From eb391b911a102381654bfaf3a147ce095913adb4 Mon Sep 17 00:00:00 2001 From: frank <420@shampoo.ooo> Date: Thu, 9 Sep 2021 22:57:13 -0400 Subject: [PATCH] load camera frame texture in separate thread --- config.json | 1 + lib/sb | 2 +- src/Pudding.cpp | 140 ++++++++++++++++++++++++++++++++++-------------- src/Pudding.hpp | 11 ++-- 4 files changed, 108 insertions(+), 46 deletions(-) diff --git a/config.json b/config.json index 43e5bf9..a43a453 100644 --- a/config.json +++ b/config.json @@ -43,6 +43,7 @@ }, "scan": { + "enabled": true, "json-save": true, "json-save-directory": "local/scans", "barcode": "", diff --git a/lib/sb b/lib/sb index 18f8396..87b1fa7 160000 --- a/lib/sb +++ b/lib/sb @@ -1 +1 @@ -Subproject commit 18f83968f3f2b7480d7557e18946f51706dda6b6 +Subproject commit 87b1fa735c5d663052995177c6d2f639eeac713b diff --git a/src/Pudding.cpp b/src/Pudding.cpp index 3f38c05..b9d6abb 100644 --- a/src/Pudding.cpp +++ b/src/Pudding.cpp @@ -146,6 +146,12 @@ void Pudding::set_pudding_model( void Pudding::load_gl_context() { super::load_gl_context(); + /* create another GL context for loading camera frame textures */ + SDL_GL_SetAttribute(SDL_GL_SHARE_WITH_CURRENT_CONTEXT, 1); + if ((capture_frame_thread_context = SDL_GL_CreateContext(window)) == nullptr) + { + log("could not create capture frame thread context"); + } /* load background as surface, generate texture to load pixel data into, allocate storage, bind and edit texture properties */ std::unique_ptr surface(IMG_Load("local/tptile.jpg"), SDL_FreeSurface); std::unique_ptr flipped_surface(rotozoomSurfaceXY(surface.get(), 0, 1, -1, 0), SDL_FreeSurface); @@ -226,8 +232,9 @@ void Pudding::load_gl_context() log_gl_errors(); } -/* Try to create cv::VideoCapture object using device ID #0. If successful, this will create a GL texture for displaying - * the frames, so it must be called after GL context has been created. +/* Try to create cv::VideoCapture object using device ID #0. If successful, this will create GL texture IDs and storage + * for the camera frames, so it must be called after GL context has been created. Two textures will be created, so they + * can be used as a double buffer. */ void Pudding::initialize_camera() { @@ -240,12 +247,18 @@ void Pudding::initialize_camera() message << "opened and initialized " << capture.get(cv::CAP_PROP_FRAME_WIDTH) << "x" << capture.get(cv::CAP_PROP_FRAME_HEIGHT) << ", " << capture.get(cv::CAP_PROP_FPS) << "fps video capture device ID #" << device_id << " using " << capture.getBackendName(); - /* generate the texture that will store the video frame, allocate storage for a video frame, bind and edit texture properties */ - glGenTextures(1, &video_capture_texture_id); - glBindTexture(GL_TEXTURE_2D, video_capture_texture_id); - glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGB8, capture.get(cv::CAP_PROP_FRAME_WIDTH), capture.get(cv::CAP_PROP_FRAME_HEIGHT)); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); - glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + /* generate two textures that will store the video frames with the intention of double buffering them + * for threaded texture loading */ + for (GLuint* buffer_id : {&capture_texture_front_buffer_id, &capture_texture_back_buffer_id}) + { + glGenTextures(1, buffer_id); + glBindTexture(GL_TEXTURE_2D, *buffer_id); + glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGB8, capture.get(cv::CAP_PROP_FRAME_WIDTH), capture.get(cv::CAP_PROP_FRAME_HEIGHT)); + // glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGB8, 320, 240); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST); + glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST); + } + capture_texture_id = capture_texture_front_buffer_id; } else { @@ -635,9 +648,80 @@ bool Pudding::item_display_active() const return show_item && items.size() > 0; } +/* Read pixels from the camera into a GL texture. This function is meant to be launched in a separate thread, + * so it will use its own GL context to load the pixels into either the front or back texture buffer, depending + * on which is not currently in use */ +int Pudding::capture_frame(void* game) +{ + Pudding* pudding = reinterpret_cast(game); + /* Make the thread context the current context (not sure what that means, but it doesn't seem to conflict + * with the main rendering context) */ + if (SDL_GL_MakeCurrent(pudding->window, pudding->capture_frame_thread_context) < 0) + { + pudding->log("error making thread context current"); + } + else + { + if (pudding->capture.isOpened()) + { + cv::Mat frame; + pudding->capture.read(frame); + if (!frame.empty()) + { + /* rotate the opencv matrix 180 to work with opengl coords */ + cv::flip(frame, frame, -1); + /* use whichever texture ID is not being used by the main rendering thread */ + GLuint texture_id = pudding->capture_texture_id == pudding->capture_texture_front_buffer_id ? + pudding->capture_texture_back_buffer_id : pudding->capture_texture_front_buffer_id; + /* bind texture for accepting pixel data */ + glBindTexture(GL_TEXTURE_2D, texture_id); + /* fill texture memory with last frame's pixels */ + glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, frame.cols, frame.rows, GL_BGR, GL_UNSIGNED_BYTE, frame.ptr()); + pudding->capture_texture_id = texture_id; + if (pudding->get_configuration()["scan"]["enabled"]) + { + /* convert to gray and scan with zbar */ + cv::cvtColor(frame, frame, cv::COLOR_BGR2GRAY); + zbar::Image query_image(frame.cols, frame.rows, "Y800", static_cast(frame.data), frame.cols * frame.rows); + int result = pudding->image_scanner.scan(query_image); + if (result > 0) + { + for (zbar::Image::SymbolIterator symbol = query_image.symbol_begin(); symbol != query_image.symbol_end(); ++symbol) + { + std::stringstream message; + message << "camera scanned " << symbol->get_type_name() << " symbol " << symbol->get_data(); + pudding->log(message.str()); + pudding->current_camera_barcode = symbol->get_data(); + pudding->current_barcode = pudding->current_camera_barcode; + } + } + query_image.set_data(nullptr, 0); + } + } + frame.release(); + } + SDL_GL_MakeCurrent(pudding->window, nullptr); + } + pudding->reading_capture_frame = false; + return 0; +} + /* Update parameters and draw the screen */ void Pudding::update() { + /* launch the camera capture thread if it is not currently running */ + if (capture.isOpened() && !reading_capture_frame) + { + SDL_Thread* capture_thread = SDL_CreateThread(Pudding::capture_frame, "capture frame", reinterpret_cast(this)); + if (capture_thread == nullptr) + { + log("could not create capture thread"); + } + else + { + reading_capture_frame = true; + } + } /* if the config is set to refresh automatically, there may be a new barcode available */ if (current_config_barcode != get_configuration()["scan"]["barcode"]) { @@ -706,6 +790,8 @@ void Pudding::update() /* draw pudding model */ glEnable(GL_DEPTH_TEST); glDrawArrays(GL_TRIANGLES, 0, pudding_vertices.size()); + /* regular fill mode enabled for all other drawing */ + glPolygonMode(GL_FRONT_AND_BACK, GL_FILL); /* only do more drawing if items are downloaded or camera is enabled */ if (item_display_active() || capture.isOpened()) { @@ -743,40 +829,12 @@ void Pudding::update() /* draw the camera if the camera has been opened */ if (capture.isOpened()) { - capture.read(capture_frame); viewport_box.left(window_box(true).left()); - if (!capture_frame.empty()) - { - /* rotate the opencv matrix 180 to work with opengl coords */ - cv::flip(capture_frame, capture_frame, -1); - glViewport(viewport_box.left(), viewport_box.bottom(), viewport_box.width(), viewport_box.height()); - /* bind texture, binding it to accept pixel data and to GLSL sampler */ - glBindTexture(GL_TEXTURE_2D, video_capture_texture_id); - glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, capture_frame.cols, capture_frame.rows, GL_BGR, GL_UNSIGNED_BYTE, capture_frame.ptr()); - /* draws rectangle vertices and rectangle texture using UV coords */ - glDrawArrays(GL_TRIANGLES, 0, 6); - /* convert to gray and scan with zbar */ - cv::cvtColor(capture_frame, capture_frame, cv::COLOR_BGR2GRAY); - zbar::Image query_image(capture_frame.cols, capture_frame.rows, "Y800", static_cast(capture_frame.data), - capture_frame.cols * capture_frame.rows); - int result = image_scanner.scan(query_image); - if (result > 0) - { - for (zbar::Image::SymbolIterator symbol = query_image.symbol_begin(); symbol != query_image.symbol_end(); ++symbol) - { - std::stringstream message; - message << "camera scanned " << symbol->get_type_name() << " symbol " << symbol->get_data(); - log(message.str()); - current_camera_barcode = symbol->get_data(); - current_barcode = current_camera_barcode; - } - } - query_image.set_data(nullptr, 0); - } - else - { - debug("video capture device frame empty"); - } + glViewport(viewport_box.left(), viewport_box.bottom(), viewport_box.width(), viewport_box.height()); + /* bind texture for drawing */ + glBindTexture(GL_TEXTURE_2D, capture_texture_id); + /* draws rectangle vertices and rectangle texture using UV coords */ + glDrawArrays(GL_TRIANGLES, 0, 6); } } SDL_GL_SwapWindow(get_window()); diff --git a/src/Pudding.hpp b/src/Pudding.hpp index 372ba5e..7da7dce 100644 --- a/src/Pudding.hpp +++ b/src/Pudding.hpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -21,6 +22,7 @@ #include "Color.hpp" #include "extension.hpp" #include "Item.hpp" +#include "Animation.hpp" class Pudding : public Game { @@ -54,15 +56,15 @@ private: std::vector items; int current_item_index = 0; cv::VideoCapture capture; - cv::Mat capture_frame; zbar::ImageScanner image_scanner; - GLuint flat_program, mvp_program, video_capture_texture_id, mvp_uniform_location, background_texture_id, time_uniform_location, - effect_uniform_location; + GLuint flat_program, mvp_program, capture_texture_front_buffer_id, capture_texture_back_buffer_id, capture_texture_id, + mvp_uniform_location, background_texture_id, time_uniform_location, effect_uniform_location; glm::mat4 projection, model = glm::mat4(1.0f), mvp; std::vector pudding_vertices, pudding_colors; std::vector pudding_uv; - bool show_item; + bool show_item = false, reading_capture_frame = false; int effect_id = Effect::NONE; + SDL_GLContext capture_frame_thread_context = nullptr; void set_pudding_model(float, float, int, int = 1, float = -1, float = 1, float = 0.3f); void load_gl_context(); @@ -78,6 +80,7 @@ private: std::shared_ptr texture_from_image_url(const std::string&); static void destroy_texture(GLuint*); bool item_display_active() const; + static int capture_frame(void*); public: