diff --git a/config.json b/config.json index bc58852..fb7e4a7 100644 --- a/config.json +++ b/config.json @@ -63,8 +63,8 @@ "best-buy-api-key": "vAC23XA5YWBzaYiGtOkoNlXZ", "giantbomb-api-key": "91a395231f4e1fd9f9ba8840c52a61cda343cd70", "nutronix-enabled": false, - "edamam-enabled": true, - "open-food-enabled": false, + "edamam-enabled": false, + "open-food-enabled": true, "open-products-enabled": true, "best-buy-enabled": true, "google-books-enabled": true diff --git a/src/Model.cpp b/src/Model.cpp index 129ca4e..8709a75 100644 --- a/src/Model.cpp +++ b/src/Model.cpp @@ -179,13 +179,13 @@ sb::Texture& Background::current() return carousel.current(textures())->second; } -CameraView::CameraView() : Plane() +PlaneDoubleBuffer::PlaneDoubleBuffer() : Plane() { texture(sb::Texture(), "front"); texture(sb::Texture(), "back"); } -void CameraView::generate(const glm::vec2& size) +void PlaneDoubleBuffer::generate(const glm::vec2& size) { for (sb::Texture* buffer : {&texture("front"), &texture("back")}) { @@ -193,17 +193,17 @@ void CameraView::generate(const glm::vec2& size) } } -sb::Texture& CameraView::current() +sb::Texture& PlaneDoubleBuffer::active() { return swapped ? texture("back") : texture("front"); } -sb::Texture& CameraView::free() +sb::Texture& PlaneDoubleBuffer::inactive() { return swapped ? texture("front") : texture("back"); } -void CameraView::swap() +void PlaneDoubleBuffer::swap() { swapped = !swapped; } diff --git a/src/Model.hpp b/src/Model.hpp index a3d58e7..89d2c58 100644 --- a/src/Model.hpp +++ b/src/Model.hpp @@ -80,8 +80,11 @@ public: }; -/* A Plane that contains a Carousel for cycling through active textures. Only one texture is - * active at a time, returned by Background::current. Carousel only goes forward, using Background::next. */ +/*! + * A version of `Plane` that contains a `Carousel` for cycling through active `sb::Texture` objects. Only one + * `sb::Texture` is * active at a time, returned by `Background::current`. Carousel only goes forward, using + * `Background::next`. + */ class Background : public Plane { @@ -96,7 +99,12 @@ public: }; -class CameraView : public Plane +/*! + * A version of `Plane` which contains two texture objects, one of which is active at a time. A reference + * to the active `sb::Texture` object is available from `PlaneDoubleBuffer.active`, and the inactive object is + * available from `PlaneDoubleBuffer.inactive`. The buffers can be swapped using `PlaneDoubleBuffer.swap`. + */ +class PlaneDoubleBuffer : public Plane { private: @@ -105,10 +113,10 @@ private: public: - CameraView(); + PlaneDoubleBuffer(); void generate(const glm::vec2&); - sb::Texture& current(); - sb::Texture& free(); + sb::Texture& active(); + sb::Texture& inactive(); void swap(); }; diff --git a/src/Pudding.cpp b/src/Pudding.cpp index 720abd8..07d9272 100644 --- a/src/Pudding.cpp +++ b/src/Pudding.cpp @@ -1,11 +1,11 @@ -/* _______________ ,-------------------------------------------------------------------. - //`````````````\\ \ \ - //~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \ - //=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \ - // \\ \ \ - // \\ \ code released under zlib license [git.nugget.fun/nugget/gunkiss] \ - // ☆ GUNKISS ☆ \\ \ \ - //_________________________\\ `-------------------------------------------------------------------' +/* _______________ ,--------------------------------------------------------. + //`````````````\\ \ \ + //~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \ + //=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \ + // \\ \ \ + // \\ \ zlib licensed code at [git.nugget.fun/nugget/gunkiss] \ + // ☆ GUNKISS ☆ \\ \ \ + //_________________________\\ `--------------------------------------------------------' Generate a custom pudding from food product UPC codes and help a pair of rats take over the video game industry, using their extraterrestrial ability to turn trash into performance enhancing drug puddings that enable business professionals @@ -31,16 +31,27 @@ Pudding::Pudding() get_delegate().subscribe(&Pudding::respond, this); get_delegate().subscribe(&Pudding::respond, this, SDL_MOUSEMOTION); get_delegate().subscribe(&Pudding::respond, this, SDL_MOUSEBUTTONDOWN); + /* initialize a zbar image scanner for reading barcodes of any format */ image_scanner.set_config(zbar::ZBAR_NONE, zbar::ZBAR_CFG_ENABLE, 1); + + /* Add a texture to the camera Plane for storing frame image data */ + camera_view.texture(sb::Texture()); + /* set up pudding model */ nlohmann::json pudding = get_configuration()["pudding"]; load_pudding_model(pudding["top-radius"], pudding["base-radius"], pudding["ring-vertex-count"], pudding["layer-count"], pudding["y-range"][0], pudding["y-range"][1], pudding["gradient-position"]); + /* loading GL context instead of SDL context for 3D */ load_gl_context(); + + /* Load background tiles */ load_tiles(); + + /* Load button graphics and create button objects */ load_pads(); + /* Load a pointer cursor from the system library that will be freed automatically */ poke = std::shared_ptr(SDL_CreateSystemCursor(SDL_SYSTEM_CURSOR_HAND), SDL_FreeCursor); } @@ -160,12 +171,6 @@ void Pudding::load_gl_context() { super::load_gl_context(); - /* Create another GL context for loading camera frame textures */ - if ((capture_frame_thread_context = SDL_GL_CreateContext(window)) == nullptr) - { - sb::Log::log("could not create capture frame thread context"); - } - /* Generate a vertex array object ID, bind it as current (requirement of OpenGL) */ vao.generate(); vao.bind(); @@ -265,13 +270,14 @@ void Pudding::load_pads() next_button.rotation(glm::radians(180.0f)); } -/* Try to create cv::VideoCapture object using device ID #0. If successful, this will create GL texture IDs and storage - * for the camera frames, so it must be called after GL context has been created. Two textures will be created, so they - * can be used as a double buffer. +/*! + * Try to create cv::VideoCapture object using device ID #0. If successful, this will also create a GL texture ID and + * storage for the camera frame on the GPU, so it must be called after GL context has been created. Create and detach + * a thread which will continuously read frame data. */ void Pudding::initialize_camera() { - /* initialize an opencv capture device for getting images from an attached camera */ + /* Open the OpenCV capture and assign device ID 0 for getting images from the default attached camera. */ int device_id = 0; capture.open(device_id); std::ostringstream message; @@ -280,9 +286,13 @@ void Pudding::initialize_camera() message << "opened and initialized " << capture.get(cv::CAP_PROP_FRAME_WIDTH) << "x" << capture.get(cv::CAP_PROP_FRAME_HEIGHT) << ", " << capture.get(cv::CAP_PROP_FPS) << "fps video capture device ID #" << device_id << " using " << capture.getBackendName(); - /* generate two textures that will store the video frames with the intention of double buffering them - * for threaded texture loading */ - camera_view.generate({capture.get(cv::CAP_PROP_FRAME_WIDTH), capture.get(cv::CAP_PROP_FRAME_HEIGHT)}); + + /* Use the texture object to generate a texture the size of the camera's resolution. */ + camera_view.texture().generate({capture.get(cv::CAP_PROP_FRAME_WIDTH), capture.get(cv::CAP_PROP_FRAME_HEIGHT)}); + + /* Create and detach a thread which will read frame data */ + std::thread camera_thread(&Pudding::capture_frame, this); + camera_thread.detach(); } else { @@ -818,100 +828,48 @@ bool Pudding::item_display_active() const return show_item && items.size() > 0; } -/* Read pixels from the camera into a GL texture. This function is meant to be launched in a separate thread, - * so it will use its own GL context to load the pixels into either the front or back texture buffer, depending - * on which is not currently in use */ -int Pudding::capture_frame(void* game) +/*! + * Read pixels from the camera into a `cv::Mat`. This function is meant to be launched in a separate thread, + * where it will run continuously, setting `new_frame_available` to `false` before loading camera frame data + * into the `cv::Mat` object at `camera_frame`, then setting `finished_loading_frame` to `true` to indicate + * new frame data is available in `camera_frame`. + */ +void Pudding::capture_frame() { - time_it("total thread")([&]{ - Pudding* pudding = reinterpret_cast(game); - /* Make the thread context the current context (not sure what that means, but it doesn't seem to conflict - * with the main rendering context) */ - if (time_it("make current")([&] { return SDL_GL_MakeCurrent(pudding->window, pudding->capture_frame_thread_context); }) < 0) + /* When capture is closed, this thread will automatically finish execution. */ + while (capture.isOpened()) { - sb::Log::log("error making thread context current"); - } - else - { - if (pudding->capture.isOpened()) + /* The frame data in the `cv::Mat` at `pudding->camera_frame` is about to be modified by the rest of + * this function, so even if there is data stored there that hasn't been read yet, it should not + * be read by the main thread until this is set to `true`at the end of the function. */ + new_frame_available = false; + + /* Load camera frame data into `cv::Mat` */ + time_it("read frame")([&]{ + capture >> camera_frame; + }); + + if (!camera_frame.empty()) { - cv::Mat frame; - time_it("read frame")([&]{ - pudding->capture.read(frame); + /* Rotate the frame 180 degrees to work with OpenGL coords */ + time_it("flip")([&]{ + cv::flip(camera_frame, camera_frame, -1); }); - if (!frame.empty()) - { - time_it("flip")([&]{ - /* rotate the opencv matrix 180 to work with opengl coords */ - cv::flip(frame, frame, -1); - }); - time_it("load texture")([&]{ - /* use whichever texture ID is not being used by the main rendering thread */ - sb::Texture& texture = pudding->camera_view.free(); - /* bind texture for accepting pixel data */ - texture.bind(); - /* fill texture memory with last frame's pixels */ - texture.load(frame.ptr(), {frame.cols, frame.rows}, GL_BGR, GL_UNSIGNED_BYTE); - }); - pudding->camera_view.swap(); - if (pudding->get_configuration()["scan"]["enabled"]) - { - time_it("gray")([&]{ - /* convert to gray and scan with zbar */ - cv::cvtColor(frame, frame, cv::COLOR_BGR2GRAY); - }); - zbar::Image query_image(frame.cols, frame.rows, "Y800", static_cast(frame.data), frame.cols * frame.rows); - int result = pudding->image_scanner.scan(query_image); - if (result > 0) - { - time_it("barcode lookup")([&] { - for (zbar::Image::SymbolIterator symbol = query_image.symbol_begin(); symbol != query_image.symbol_end(); ++symbol) - { - std::ostringstream message; - message << "camera scanned " << symbol->get_type_name() << " symbol " << symbol->get_data(); - sb::Log::log(message); - pudding->current_camera_barcode = symbol->get_data(); - pudding->current_barcode = pudding->current_camera_barcode; - } - }); - } - query_image.set_data(nullptr, 0); - } - } - frame.release(); + + /* Finished loading into `cv::Mat`, so it is new data that is safe to read. */ + new_frame_available = true; } - SDL_GL_MakeCurrent(pudding->window, nullptr); sb::Log::gl_errors("in capture thread, after capturing frame"); + std::this_thread::sleep_for(std::chrono::milliseconds(50)); } - pudding->reading_capture_frame = false; - // using namespace std::chrono_literals; - // std::this_thread::sleep_for(2s); - }); - return 0; } /* Update parameters and draw the screen */ void Pudding::update() { - /* number of seconds we've been running for */ + /* Time in seconds the game has running for */ float time_seconds = SDL_GetTicks() / 1000.0f; - { - /* launch the camera capture thread if it is not currently running */ - if (capture.isOpened() && !reading_capture_frame) - { - SDL_Thread* capture_thread = SDL_CreateThread(Pudding::capture_frame, "capture frame", reinterpret_cast(this)); - if (capture_thread == nullptr) - { - sb::Log::log("could not create capture thread"); - } - else - { - reading_capture_frame = true; - SDL_WaitThread(capture_thread, nullptr); - } - } - } - sb::Log::gl_errors("in main thread, after capturing frame"); + /* if the config is set to refresh automatically, there may be a new barcode available */ if (current_config_barcode != get_configuration()["scan"]["barcode"]) { @@ -921,13 +879,49 @@ void Pudding::update() message << "read new barcode from config " << current_barcode; sb::Log::log(message); } + + /* If a new frame is finished loading in the detached thread, copy the frame data into a texture, process the frame + * for scanning and scan it. */ + if (new_frame_available) + { + camera_view.texture().bind(); + /* Fill camera view texture memory with last frame's pixels */ + camera_view.texture().load(camera_frame.ptr(), {camera_frame.cols, camera_frame.rows}, GL_BGR, GL_UNSIGNED_BYTE); + /* Frame data has been loaded, so there is not a new frame available anymore. */ + new_frame_available = false; + /* Convert to grayscale for ZBar */ + cv::cvtColor(camera_frame, camera_frame, cv::COLOR_BGR2GRAY); + if (get_configuration()["scan"]["enabled"]) + { + zbar::Image query_image(camera_frame.cols, camera_frame.rows, "Y800", static_cast(camera_frame.data), + camera_frame.cols * camera_frame.rows); + int result = image_scanner.scan(query_image); + if (result > 0) + { + time_it("barcode lookup")([&] { + for (zbar::Image::SymbolIterator symbol = query_image.symbol_begin(); symbol != query_image.symbol_end(); ++symbol) + { + std::ostringstream message; + message << "camera scanned " << symbol->get_type_name() << " symbol " << symbol->get_data(); + sb::Log::log(message); + current_camera_barcode = symbol->get_data(); + current_barcode = current_camera_barcode; + } + }); + } + query_image.set_data(nullptr, 0); + } + } + /* viewport box will be used to tell GL where to draw */ viewport = window_box(true); + /* shrink viewport if item texture or camera will be displayed */ if (item_display_active() || capture.isOpened()) { viewport.drag_bottom(0.5f * get_configuration()["interface"]["pop-up-viewport-height"].get() * viewport.height()); } + /* Save the main viewport dimensions */ main_viewport = viewport; glViewport(viewport); @@ -1038,7 +1032,7 @@ void Pudding::update() glViewport(viewport); /* bind texture for drawing */ glUniformMatrix4fv(uniform["flat"]["transformation"], 1, GL_FALSE, &camera_view.transformation()[0][0]); - camera_view.current().bind(); + camera_view.texture().bind(); camera_view.enable(); /* draws rectangle vertices and rectangle texture using UV coords */ glDrawArrays(GL_TRIANGLES, 0, camera_view.attributes("position")->count()); diff --git a/src/Pudding.hpp b/src/Pudding.hpp index f2d4d84..6bf035f 100644 --- a/src/Pudding.hpp +++ b/src/Pudding.hpp @@ -345,23 +345,21 @@ private: Carousel item_carousel; int effect_id = EFFECT_NONE, pudding_triangle_vertex_count = 0, pudding_fan_vertex_count = 0; cv::VideoCapture capture; + cv::Mat camera_frame; zbar::ImageScanner image_scanner; std::map> uniform; GLuint flat_program, mvp_program; glm::mat4 projection, model {1.0f}, mvp; Model pudding_model; - Plane plane; + Plane plane, camera_view; Background background; - CameraView camera_view; - bool show_item = false, reading_capture_frame = false; - SDL_GLContext capture_frame_thread_context = nullptr; + bool show_item = false, new_frame_available = false; sb::VAO vao; sb::VBO vbo; std::map labels; Pad camera_button, previous_button, next_button, inventory_button; Box viewport, main_viewport, pop_up_viewport; std::mutex camera_mutex; - SDL_Thread* capture_thread = nullptr; void load_pudding_model(float, float, int, int = 1, float = -1.0f, float = 1.0f, float = 0.3f); void load_gl_context(); @@ -380,7 +378,7 @@ private: sb::Texture texture_from_image_url(const std::string&) const; static void destroy_texture(GLuint*); bool item_display_active() const; - static int capture_frame(void*); + void capture_frame(); /* Initialize camera on connection and release on disconnection. */ Connection<> camera_switch {