cURL asynchronous requests

Makefile

@@ -111,7 +111,8 @@ EMSCRIPTEN_CFLAGS = -O0 -Wall -s USE_SDL=2 -s USE_SDL_IMAGE=2 -s SDL2_IMAGE_FORM
 	--no-heap-copy -I $(SB_LIB_DIR) -I $(SB_SRC_DIR)  -I $(HOME)/local/zbar/include -I $(HOME)/local/opencv/include/opencv4
 EMSCRIPTEN_LFLAGS = -s MIN_WEBGL_VERSION=2 -s EXPORTED_FUNCTIONS="['_main', '_malloc']" -s ALLOW_MEMORY_GROWTH=1 -s FULL_ES3=1 \
 	-sLLD_REPORT_UNDEFINED -s FETCH --bind $(wildcard $(addprefix $(HOME)/ext/software/opencv-4.6.0/build_wasm_contrib/lib/,*.a)) \
-	$(wildcard $(addprefix $(HOME)/ext/software/opencv-4.6.0/build_wasm_contrib/3rdparty/lib/,*.a))	$(HOME)/ext/software/ZBar/zbar/.libs/libzbar.a
+	$(wildcard $(addprefix $(HOME)/ext/software/opencv-4.6.0/build_wasm_contrib/3rdparty/lib/,*.a))	$(HOME)/ext/software/ZBar/zbar/.libs/libzbar.a \
+	-sNO_DISABLE_EXCEPTION_CATCHING
 EMSCRIPTEN_PRELOADS = --preload-file "BPmono.ttf"@/ --preload-file "config.json"@/ --preload-file "resource/"@/"resource/" \
 	--preload-file "src/shaders/"@/"src/shaders/"
 

config.json

@@ -59,8 +59,8 @@
         "json-save-directory": "local/scans",
         "barcode": "",
         "capture-device": "/dev/video0",
-        "brightness-addition": 30,
-        "contrast-multiplication": 1.6,
+        "brightness-addition": 20,
+        "contrast-multiplication": 1.4,
         "camera-device-id": 0,
         "sharpen": true,
         "brighten": false,

index.html

@@ -84,9 +84,12 @@
                  });
          }
 
+         /*!
+          * Pause video streaming and free pixel memory.
+          */
          function close_camera()
          {
-             Module._free
+             Module._free(image_heap_address);
              video.pause();
              video.srcObject = null;
              streaming = false;

lib/sb

@@ -1 +1 @@
-Subproject commit 346a059ee2c98fee814df51b560bbde7e6b733db
+Subproject commit 1ff7f802cbceafbd59029192421732d73ab28152

src/Pudding.cpp

@@ -39,6 +39,21 @@ void set_heap_offset(int offset)
 /* Initialize a Pudding instance */
 Pudding::Pudding()
 {
+#ifndef __EMSCRIPTEN__
+    /* Initialize cURL and store result. Initialize the multi request handler. */
+    curl_init_result = curl_global_init(CURL_GLOBAL_DEFAULT);
+    if (curl_init_result != CURLE_OK)
+    {
+        std::ostringstream message;
+        message << "cURL failed to initialize and will not be available " << curl_easy_strerror(curl_init_result);
+        sb::Log::log(message);
+    }
+    else
+    {
+        curl_multi_handle.reset(curl_multi_init());
+    }
+#endif
+
     /* subscribe to command events */
     get_delegate().subscribe(&Pudding::respond, this);
     get_delegate().subscribe(&Pudding::respond, this, SDL_MOUSEMOTION);
@@ -335,6 +350,7 @@ void Pudding::close_camera()
     capture.release();
 #else
     emscripten_run_script("close_camera()");
+    new_frame_available = false;
 #endif
 }
 
@@ -588,7 +604,7 @@ void Pudding::incorporate_nutritionix_api(const std::vector<std::uint8_t>& json_
     sb::Log::log(json_formatted.str(), sb::Log::DEBUG);
 
     /* test that should determine if a Nutritionix response is not empty */
-    if (!(json.contains("message") && json["message"] == NUTRITIONIX_NOT_FOUND))
+    if (json.contains("foods"))
     {
         nlohmann::json food = json["foods"][0];
         std::ostringstream message;
@@ -819,22 +835,29 @@ void Pudding::web_get_bytes(std::string url, const web_callback& callback, const
 
 #else
 
-    CURL *curl;
-    CURLcode result;
-    result = curl_global_init(CURL_GLOBAL_DEFAULT);
-    if (result != CURLE_OK)
+    if (curl_init_result != CURLE_OK)
     {
-        std::cout << "cURL initialization failed " << curl_easy_strerror(result) << std::endl;
+        std::ostringstream message;
+        message << "cURL unavailable " << curl_easy_strerror(curl_init_result);
+        sb::Log::log(message);
     }
     else
     {
-        curl = curl_easy_init();
-        if (curl)
+        CURL* curl_easy_handle = curl_easy_init();
+
+        if (curl_easy_handle)
         {
-            curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
-            curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_response);
-            curl_easy_setopt(curl, CURLOPT_WRITEDATA, request);
-            curl_easy_setopt(curl, CURLOPT_USERAGENT, configuration()["api"]["user-agent"].get<std::string>().c_str());
+            curl_easy_setopt(curl_easy_handle, CURLOPT_URL, url.c_str());
+            curl_easy_setopt(curl_easy_handle, CURLOPT_WRITEFUNCTION, curl_write_response);
+
+            /* WRITEDATA is set to the request object, so the storage in the request object can receive the transfer data */
+            curl_easy_setopt(curl_easy_handle, CURLOPT_WRITEDATA, request);
+
+            /* PRIVATE is also set to the request object, so the request can call the response function when the multi handle has
+             * determined that all the transfer packets are complete */
+            curl_easy_setopt(curl_easy_handle, CURLOPT_PRIVATE, request);
+
+            curl_easy_setopt(curl_easy_handle, CURLOPT_USERAGENT, configuration()["api"]["user-agent"].get<std::string>().c_str());
 
             /* Pass submitted headers to cURL */
             struct curl_slist* list = nullptr;
@@ -848,26 +871,18 @@ void Pudding::web_get_bytes(std::string url, const web_callback& callback, const
                     list = curl_slist_append(list, pair.c_str());
                 }
             }
-            curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
+            curl_easy_setopt(curl_easy_handle, CURLOPT_HTTPHEADER, list);
 
-            result = curl_easy_perform(curl);
-            curl_slist_free_all(list);
-
-            if (result != CURLE_OK)
-            {
-                std::cout << "cURL request failed " << curl_easy_strerror(result) << std::endl;
-            }
+            /* Add the easy handle to the multi handle, so it can be processed asynchronously in the update loop */
+            curl_multi_add_handle(curl_multi_handle.get(), curl_easy_handle);
         }
         else
         {
-            std::cout << "cURL initialization failed" << std::endl;
+            std::ostringstream message;
+            message << "cURL request initialization failed for " << url;
+            sb::Log::log(message);
         }
-        curl_easy_cleanup(curl);
     }
-    curl_global_cleanup();
-
-    /* Call the user supplied callback */
-    request->respond();
 
 #endif
 
@@ -982,14 +997,15 @@ bool Pudding::item_display_active() const
 }
 
 /*!
- * Read pixels from the camera into a `cv::Mat`.
+ * Read pixels from the camera into a `cv::Mat` and pre-process them if pre-processing methods are enabled.
  * 
  * For a Linux build: This function is meant to be launched in a separate thread, where it will run continuously. Set `new_frame_available`
  * to `false` before loading camera frame data into the `cv::Mat` object at `camera_frame`, then set it back to `true` to indicate new frame
  * data is available in `camera_frame`.
  *
- * For an Emscripten build: This will load pixel data off the Emscripten heap into a `cv::Mat`. It is intended to be called synchronously in the
- * main thread.
+ * For an Emscripten build: This method actually does not capture frame data since that is done by associating the cv::Mat at Pudding::camera_frame
+ * with the data in the Emscripten heap memory. However, pre-processing is done in this function, so it should be called synchronously once per
+ * frame.
  */
 void Pudding::capture_frame()
 {
@@ -999,30 +1015,24 @@ void Pudding::capture_frame()
     /* When the camera button is switched off, this thread will automatically finish execution. */
     while (camera_switch)
     {
+#endif
+
         /* The frame data in the `cv::Mat` at `pudding->camera_frame` is about to be modified by the rest of
          * this function, so even if there is data stored there that hasn't been read yet, it should not
          * be read by the main thread until this is set to `true`at the end of the function. */
         new_frame_available = false;
-#endif
 
         /* If requests are running, a barcode is currently being scanned and the camera image doesn't need to be scanned. */
         if (requests.empty())
         {
 
-/* Different methods of reading frames depending on the build */
+/* Emscripten loads frame data differently, so disable this part */
 #ifndef __EMSCRIPTEN__
-
             /* Load camera frame data into `cv::Mat` */
             capture >> camera_frame;
-
-#else
-
-            /* Convert the address of frame RGBA pixel data on the Emscripten heap into an unsigned 8-bit pointer and read the data
-             * into a cv::Mat. */
-            std::uint8_t* emscripten_camera_pixels = reinterpret_cast<std::uint8_t*>(emscripten_heap_offset);
-            camera_frame = cv::Mat(240, 320, CV_8UC4, emscripten_camera_pixels);
 #endif
 
+            /* Pre-process image for improved scan results */
             if (!camera_frame.empty())
             {
                 if (configuration()["scan"]["sharpen"])
@@ -1064,6 +1074,21 @@ void Pudding::capture_frame()
 /* Update parameters and draw the screen */
 void Pudding::update()
 {
+
+/* The Emscripten pixel data memory address has been set in JS, so reassociate the cv::Mat with the data. Emscripten frame data will be stored
+ * on the heap at the given offset. Once this memory is associated with the cv::Mat, the matrix will update as the memory is filled with pixel
+ * data. */
+#ifdef __EMSCRIPTEN__
+    if (emscripten_heap_offset != -1)
+    {
+        /* Convert the address of frame RGBA pixel data on the Emscripten heap into an unsigned 8-bit pointer and point a cv::Mat to
+         * the data. The memory is managed in the JS code, so don't free it. */
+        std::uint8_t* emscripten_camera_pixels = reinterpret_cast<std::uint8_t*>(emscripten_heap_offset);
+        camera_frame = cv::Mat(240, 320, CV_8UC4, emscripten_camera_pixels);
+        emscripten_heap_offset = -1;
+    }
+#endif
+
     sb::Log::gl_errors("at beginning of update");
     /* Time in seconds the game has running for */
     float time_seconds = SDL_GetTicks() / 1000.0f;
@@ -1078,12 +1103,12 @@ void Pudding::update()
         sb::Log::log(message);
     }
 
-    /* If new frame data is available, copy it from a cv::Mat into a texture, process for scanning and scan it. */
-    if (new_frame_available)
+    /* If new frame data is available and requests aren't running, pass the frame through barcode scanning and copy it into texture memory for display. */
+    if (new_frame_available && requests.empty())
     {
         
 #ifdef __EMSCRIPTEN__
-        /* Emscripten builds load pixel data into cv::Mat synchronously */
+        /* Emscripten builds call the capture frame image pre-processing synchronously */
         capture_frame();
 
         /* Pixels from Emscripten are RGBA */
@@ -1323,6 +1348,28 @@ void Pudding::update()
         previous_barcode = current_barcode;
     }
 
+#ifndef __EMSCRIPTEN__
+    /* Calling multi perform will perform transfers on handles cURL has determined are ready for transfer. Any handles that have transferred
+     * all their necessary data will cause a CURLMSG_DONE message to be in the message queue. If a handle is done, call the request callback
+     * that has been attached to it, remove the handle, and clean it up. */
+    int running_handles;
+    curl_multi_perform(curl_multi_handle.get(), &running_handles);
+    CURLMsg* message_queue;
+    do
+    {
+        int message_count;
+        message_queue = curl_multi_info_read(curl_multi_handle.get(), &message_count);
+        if (message_queue && (message_queue->msg == CURLMSG_DONE))
+        {
+            Request* request;
+            curl_easy_getinfo(message_queue->easy_handle, CURLINFO_PRIVATE, &request);
+            request->respond();
+            curl_multi_remove_handle(curl_multi_handle.get(), message_queue->easy_handle);
+            curl_easy_cleanup(message_queue->easy_handle);
+        }
+    } while(message_queue);
+#endif
+
     /* Delete and erase finished requests from the vector using iterators to erase while reading the vector */
     for (auto iter = requests.begin(); iter != requests.end();)
     {
@@ -1367,6 +1414,13 @@ void Pudding::update()
     }
 }
 
+Pudding::~Pudding()
+{
+#ifndef __EMSCRIPTEN__
+    curl_global_cleanup();
+#endif
+}
+
 Request::Request(const web_callback& callback, const std::string& url) : callback(callback), request_url(url) {}
 
 void Request::store(const std::uint8_t* buffer, const std::size_t& size)

src/Pudding.hpp

@@ -13,6 +13,18 @@
 /* Needed for functions in glm/gtx/ */
 #define GLM_ENABLE_EXPERIMENTAL
 
+/* Standard library includes */
+#include <stdlib.h>
+#include <string>
+#include <iostream>
+#include <algorithm>
+#include <thread>
+#include <mutex>
+#include <memory>
+#include <stdexcept>
+#include <functional>
+#include <chrono>
+
 /* cURL and cv::VideoCapture are not available for Emscripten, so use alternatives for Emscripten builds */
 #if defined(__EMSCRIPTEN__)
 #include <emscripten/fetch.h>
@@ -24,16 +36,6 @@ using namespace emscripten;
 #include "opencv2/highgui.hpp"
 #endif
 
-#include <stdlib.h>
-#include <string>
-#include <iostream>
-#include <algorithm>
-#include <thread>
-#include <mutex>
-#include <memory>
-#include <stdexcept>
-#include <functional>
-#include <chrono>
 #include "SDL.h"
 #include "SDL_image.h"
 #include "sdl2-gfx/SDL2_gfxPrimitives.h"
@@ -308,10 +310,10 @@ public:
 
 /* These variables will be bound to JS. They are placed in the global scope, so they can be read and written by both
  * C++ and JS. The associated functions are bound to JS so they can be used to write values to the variables. The
- * first flag is used by both C++ and JS builds, so it is always included. */
+ * new_frame_available flag is used by both C++ and JS builds, so it is always included. */
 bool new_frame_available = false;
 #ifdef __EMSCRIPTEN__
-unsigned int emscripten_heap_offset = 0;
+int emscripten_heap_offset = -1;
 void flag_frame();
 void set_heap_offset(int offset);
 #endif
@@ -429,6 +431,9 @@ private:
     int effect_id = EFFECT_NONE, pudding_triangle_vertex_count = 0, pudding_fan_vertex_count = 0;
 #ifndef __EMSCRIPTEN__
     cv::VideoCapture capture;
+    std::unique_ptr<CURLM, decltype(&curl_multi_cleanup)> curl_multi_handle = std::unique_ptr<CURLM, decltype(&curl_multi_cleanup)>(
+        nullptr, curl_multi_cleanup);
+    CURLcode curl_init_result;
 #endif
     cv::Mat camera_frame, contrasted_frame, sharpened_frame, zbar_frame, blurred, low_contrast_mask;
     zbar::ImageScanner image_scanner;
@@ -591,6 +596,7 @@ public:
     Item& current_item();
     void update();
     virtual std::string class_name() const { return "Pudding"; }
+    ~Pudding();
 
 };