pause camera display while requesting from APIs, set Emscripten frame data only when memory location changes

Makefile

@@ -111,7 +111,8 @@ EMSCRIPTEN_CFLAGS = -O0 -Wall -s USE_SDL=2 -s USE_SDL_IMAGE=2 -s SDL2_IMAGE_FORM
 	--no-heap-copy -I $(SB_LIB_DIR) -I $(SB_SRC_DIR)  -I $(HOME)/local/zbar/include -I $(HOME)/local/opencv/include/opencv4
 EMSCRIPTEN_LFLAGS = -s MIN_WEBGL_VERSION=2 -s EXPORTED_FUNCTIONS="['_main', '_malloc']" -s ALLOW_MEMORY_GROWTH=1 -s FULL_ES3=1 \
 	-sLLD_REPORT_UNDEFINED -s FETCH --bind $(wildcard $(addprefix $(HOME)/ext/software/opencv-4.6.0/build_wasm_contrib/lib/,*.a)) \
-	$(wildcard $(addprefix $(HOME)/ext/software/opencv-4.6.0/build_wasm_contrib/3rdparty/lib/,*.a))	$(HOME)/ext/software/ZBar/zbar/.libs/libzbar.a
+	$(wildcard $(addprefix $(HOME)/ext/software/opencv-4.6.0/build_wasm_contrib/3rdparty/lib/,*.a))	$(HOME)/ext/software/ZBar/zbar/.libs/libzbar.a \
+	-sNO_DISABLE_EXCEPTION_CATCHING
 EMSCRIPTEN_PRELOADS = --preload-file "BPmono.ttf"@/ --preload-file "config.json"@/ --preload-file "resource/"@/"resource/" \
 	--preload-file "src/shaders/"@/"src/shaders/"
 

config.json

@@ -59,8 +59,8 @@
         "json-save-directory": "local/scans",
         "barcode": "",
         "capture-device": "/dev/video0",
-        "brightness-addition": 30,
+        "brightness-addition": 20,
-        "contrast-multiplication": 1.6,
+        "contrast-multiplication": 1.4,
         "camera-device-id": 0,
         "sharpen": true,
         "brighten": false,

index.html

@@ -84,9 +84,12 @@
                  });
          }
 
+         /*!
+          * Pause video streaming and free pixel memory.
+          */
          function close_camera()
          {
-             Module._free
+             Module._free(image_heap_address);
              video.pause();
              video.srcObject = null;
              streaming = false;

lib/sb

@@ -1 +1 @@
-Subproject commit 346a059ee2c98fee814df51b560bbde7e6b733db
+Subproject commit 1ff7f802cbceafbd59029192421732d73ab28152

src/Pudding.cpp

@@ -335,6 +335,7 @@ void Pudding::close_camera()
     capture.release();
 #else
     emscripten_run_script("close_camera()");
+    new_frame_available = false;
 #endif
 }
 
@@ -982,14 +983,15 @@ bool Pudding::item_display_active() const
 }
 
 /*!
- * Read pixels from the camera into a `cv::Mat`.
+ * Read pixels from the camera into a `cv::Mat` and pre-process them if pre-processing methods are enabled.
  * 
  * For a Linux build: This function is meant to be launched in a separate thread, where it will run continuously. Set `new_frame_available`
  * to `false` before loading camera frame data into the `cv::Mat` object at `camera_frame`, then set it back to `true` to indicate new frame
  * data is available in `camera_frame`.
  *
- * For an Emscripten build: This will load pixel data off the Emscripten heap into a `cv::Mat`. It is intended to be called synchronously in the
+ * For an Emscripten build: This method actually does not capture frame data since that is done by associating the cv::Mat at Pudding::camera_frame
- * main thread.
+ * with the data in the Emscripten heap memory. However, pre-processing is done in this function, so it should be called synchronously once per
+ * frame.
  */
 void Pudding::capture_frame()
 {
@@ -999,30 +1001,24 @@ void Pudding::capture_frame()
     /* When the camera button is switched off, this thread will automatically finish execution. */
     while (camera_switch)
     {
+#endif
+
         /* The frame data in the `cv::Mat` at `pudding->camera_frame` is about to be modified by the rest of
          * this function, so even if there is data stored there that hasn't been read yet, it should not
          * be read by the main thread until this is set to `true`at the end of the function. */
         new_frame_available = false;
-#endif
 
         /* If requests are running, a barcode is currently being scanned and the camera image doesn't need to be scanned. */
         if (requests.empty())
         {
 
-/* Different methods of reading frames depending on the build */
+/* Emscripten loads frame data differently, so disable this part */
 #ifndef __EMSCRIPTEN__
-
             /* Load camera frame data into `cv::Mat` */
             capture >> camera_frame;
-
-#else
-
-            /* Convert the address of frame RGBA pixel data on the Emscripten heap into an unsigned 8-bit pointer and read the data
-             * into a cv::Mat. */
-            std::uint8_t* emscripten_camera_pixels = reinterpret_cast<std::uint8_t*>(emscripten_heap_offset);
-            camera_frame = cv::Mat(240, 320, CV_8UC4, emscripten_camera_pixels);
 #endif
 
+            /* Pre-process image for improved scan results */
             if (!camera_frame.empty())
             {
                 if (configuration()["scan"]["sharpen"])
@@ -1064,6 +1060,21 @@ void Pudding::capture_frame()
 /* Update parameters and draw the screen */
 void Pudding::update()
 {
+
+/* The Emscripten pixel data memory address has been set in JS, so reassociate the cv::Mat with the data. Emscripten frame data will be stored
+ * on the heap at the given offset. Once this memory is associated with the cv::Mat, the matrix will update as the memory is filled with pixel
+ * data. */
+#ifdef __EMSCRIPTEN__
+    if (emscripten_heap_offset != -1)
+    {
+        /* Convert the address of frame RGBA pixel data on the Emscripten heap into an unsigned 8-bit pointer and point a cv::Mat to
+         * the data. The memory is managed in the JS code, so don't free it. */
+        std::uint8_t* emscripten_camera_pixels = reinterpret_cast<std::uint8_t*>(emscripten_heap_offset);
+        camera_frame = cv::Mat(240, 320, CV_8UC4, emscripten_camera_pixels);
+        emscripten_heap_offset = -1;
+    }
+#endif
+
     sb::Log::gl_errors("at beginning of update");
     /* Time in seconds the game has running for */
     float time_seconds = SDL_GetTicks() / 1000.0f;
@@ -1078,12 +1089,12 @@ void Pudding::update()
         sb::Log::log(message);
     }
 
-    /* If new frame data is available, copy it from a cv::Mat into a texture, process for scanning and scan it. */
+    /* If new frame data is available and requests aren't running, pass the frame through barcode scanning and copy it into texture memory for display. */
-    if (new_frame_available)
+    if (new_frame_available && requests.empty())
     {
         
 #ifdef __EMSCRIPTEN__
-        /* Emscripten builds load pixel data into cv::Mat synchronously */
+        /* Emscripten builds call the capture frame image pre-processing synchronously */
         capture_frame();
 
         /* Pixels from Emscripten are RGBA */

src/Pudding.hpp

@@ -308,10 +308,10 @@ public:
 
 /* These variables will be bound to JS. They are placed in the global scope, so they can be read and written by both
  * C++ and JS. The associated functions are bound to JS so they can be used to write values to the variables. The
- * first flag is used by both C++ and JS builds, so it is always included. */
+ * new_frame_available flag is used by both C++ and JS builds, so it is always included. */
 bool new_frame_available = false;
 #ifdef __EMSCRIPTEN__
-unsigned int emscripten_heap_offset = 0;
+int emscripten_heap_offset = -1;
 void flag_frame();
 void set_heap_offset(int offset);
 #endif