pause camera display while requesting from APIs, set Emscripten frame data only when memory location changes

This commit is contained in:
ohsqueezy 2022-09-26 21:01:11 -04:00
parent 59db4015b5
commit f43203f5ae
6 changed files with 38 additions and 23 deletions

View File

@ -111,7 +111,8 @@ EMSCRIPTEN_CFLAGS = -O0 -Wall -s USE_SDL=2 -s USE_SDL_IMAGE=2 -s SDL2_IMAGE_FORM
--no-heap-copy -I $(SB_LIB_DIR) -I $(SB_SRC_DIR) -I $(HOME)/local/zbar/include -I $(HOME)/local/opencv/include/opencv4
EMSCRIPTEN_LFLAGS = -s MIN_WEBGL_VERSION=2 -s EXPORTED_FUNCTIONS="['_main', '_malloc']" -s ALLOW_MEMORY_GROWTH=1 -s FULL_ES3=1 \
-sLLD_REPORT_UNDEFINED -s FETCH --bind $(wildcard $(addprefix $(HOME)/ext/software/opencv-4.6.0/build_wasm_contrib/lib/,*.a)) \
$(wildcard $(addprefix $(HOME)/ext/software/opencv-4.6.0/build_wasm_contrib/3rdparty/lib/,*.a)) $(HOME)/ext/software/ZBar/zbar/.libs/libzbar.a
$(wildcard $(addprefix $(HOME)/ext/software/opencv-4.6.0/build_wasm_contrib/3rdparty/lib/,*.a)) $(HOME)/ext/software/ZBar/zbar/.libs/libzbar.a \
-sNO_DISABLE_EXCEPTION_CATCHING
EMSCRIPTEN_PRELOADS = --preload-file "BPmono.ttf"@/ --preload-file "config.json"@/ --preload-file "resource/"@/"resource/" \
--preload-file "src/shaders/"@/"src/shaders/"

View File

@ -59,8 +59,8 @@
"json-save-directory": "local/scans",
"barcode": "",
"capture-device": "/dev/video0",
"brightness-addition": 30,
"contrast-multiplication": 1.6,
"brightness-addition": 20,
"contrast-multiplication": 1.4,
"camera-device-id": 0,
"sharpen": true,
"brighten": false,

View File

@ -84,9 +84,12 @@
});
}
/*!
* Pause video streaming and free pixel memory.
*/
function close_camera()
{
Module._free
Module._free(image_heap_address);
video.pause();
video.srcObject = null;
streaming = false;

2
lib/sb

@ -1 +1 @@
Subproject commit 346a059ee2c98fee814df51b560bbde7e6b733db
Subproject commit 1ff7f802cbceafbd59029192421732d73ab28152

View File

@ -335,6 +335,7 @@ void Pudding::close_camera()
capture.release();
#else
emscripten_run_script("close_camera()");
new_frame_available = false;
#endif
}
@ -982,14 +983,15 @@ bool Pudding::item_display_active() const
}
/*!
* Read pixels from the camera into a `cv::Mat`.
* Read pixels from the camera into a `cv::Mat` and pre-process them if pre-processing methods are enabled.
*
* For a Linux build: This function is meant to be launched in a separate thread, where it will run continuously. Set `new_frame_available`
* to `false` before loading camera frame data into the `cv::Mat` object at `camera_frame`, then set it back to `true` to indicate new frame
* data is available in `camera_frame`.
*
* For an Emscripten build: This will load pixel data off the Emscripten heap into a `cv::Mat`. It is intended to be called synchronously in the
* main thread.
* For an Emscripten build: This method actually does not capture frame data since that is done by associating the cv::Mat at Pudding::camera_frame
* with the data in the Emscripten heap memory. However, pre-processing is done in this function, so it should be called synchronously once per
* frame.
*/
void Pudding::capture_frame()
{
@ -999,30 +1001,24 @@ void Pudding::capture_frame()
/* When the camera button is switched off, this thread will automatically finish execution. */
while (camera_switch)
{
#endif
/* The frame data in the `cv::Mat` at `pudding->camera_frame` is about to be modified by the rest of
* this function, so even if there is data stored there that hasn't been read yet, it should not
* be read by the main thread until this is set to `true`at the end of the function. */
new_frame_available = false;
#endif
/* If requests are running, a barcode is currently being scanned and the camera image doesn't need to be scanned. */
if (requests.empty())
{
/* Different methods of reading frames depending on the build */
/* Emscripten loads frame data differently, so disable this part */
#ifndef __EMSCRIPTEN__
/* Load camera frame data into `cv::Mat` */
capture >> camera_frame;
#else
/* Convert the address of frame RGBA pixel data on the Emscripten heap into an unsigned 8-bit pointer and read the data
* into a cv::Mat. */
std::uint8_t* emscripten_camera_pixels = reinterpret_cast<std::uint8_t*>(emscripten_heap_offset);
camera_frame = cv::Mat(240, 320, CV_8UC4, emscripten_camera_pixels);
#endif
/* Pre-process image for improved scan results */
if (!camera_frame.empty())
{
if (configuration()["scan"]["sharpen"])
@ -1064,6 +1060,21 @@ void Pudding::capture_frame()
/* Update parameters and draw the screen */
void Pudding::update()
{
/* The Emscripten pixel data memory address has been set in JS, so reassociate the cv::Mat with the data. Emscripten frame data will be stored
* on the heap at the given offset. Once this memory is associated with the cv::Mat, the matrix will update as the memory is filled with pixel
* data. */
#ifdef __EMSCRIPTEN__
if (emscripten_heap_offset != -1)
{
/* Convert the address of frame RGBA pixel data on the Emscripten heap into an unsigned 8-bit pointer and point a cv::Mat to
* the data. The memory is managed in the JS code, so don't free it. */
std::uint8_t* emscripten_camera_pixels = reinterpret_cast<std::uint8_t*>(emscripten_heap_offset);
camera_frame = cv::Mat(240, 320, CV_8UC4, emscripten_camera_pixels);
emscripten_heap_offset = -1;
}
#endif
sb::Log::gl_errors("at beginning of update");
/* Time in seconds the game has running for */
float time_seconds = SDL_GetTicks() / 1000.0f;
@ -1078,12 +1089,12 @@ void Pudding::update()
sb::Log::log(message);
}
/* If new frame data is available, copy it from a cv::Mat into a texture, process for scanning and scan it. */
if (new_frame_available)
/* If new frame data is available and requests aren't running, pass the frame through barcode scanning and copy it into texture memory for display. */
if (new_frame_available && requests.empty())
{
#ifdef __EMSCRIPTEN__
/* Emscripten builds load pixel data into cv::Mat synchronously */
/* Emscripten builds call the capture frame image pre-processing synchronously */
capture_frame();
/* Pixels from Emscripten are RGBA */

View File

@ -308,10 +308,10 @@ public:
/* These variables will be bound to JS. They are placed in the global scope, so they can be read and written by both
* C++ and JS. The associated functions are bound to JS so they can be used to write values to the variables. The
* first flag is used by both C++ and JS builds, so it is always included. */
* new_frame_available flag is used by both C++ and JS builds, so it is always included. */
bool new_frame_available = false;
#ifdef __EMSCRIPTEN__
unsigned int emscripten_heap_offset = 0;
int emscripten_heap_offset = -1;
void flag_frame();
void set_heap_offset(int offset);
#endif