shampoo/gunkiss
shampoo
/
gunkiss
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

use a single GL context by moving texture loading out of camera frame thread and use the thread only for reading frame data into cv::Mat

This commit is contained in:
ohsqueezy 2022-08-08 19:28:25 -04:00
parent c500cd332e
commit 17dee6174f
5 changed files with 122 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
config.json
View File

@ -63,8 +63,8 @@
"best-buy-api-key": "vAC23XA5YWBzaYiGtOkoNlXZ", "best-buy-api-key": "vAC23XA5YWBzaYiGtOkoNlXZ",
"giantbomb-api-key": "91a395231f4e1fd9f9ba8840c52a61cda343cd70", "giantbomb-api-key": "91a395231f4e1fd9f9ba8840c52a61cda343cd70",
"nutronix-enabled": false, "nutronix-enabled": false,
"edamam-enabled": true, "edamam-enabled": false,
"open-food-enabled": false, "open-food-enabled": true,
"open-products-enabled": true, "open-products-enabled": true,
"best-buy-enabled": true, "best-buy-enabled": true,
"google-books-enabled": true "google-books-enabled": true

10
src/Model.cpp
View File

@ -179,13 +179,13 @@ sb::Texture& Background::current()
return carousel.current(textures())->second; return carousel.current(textures())->second;
} }
CameraView::CameraView() : Plane() PlaneDoubleBuffer::PlaneDoubleBuffer() : Plane()
{ {
texture(sb::Texture(), "front"); texture(sb::Texture(), "front");
texture(sb::Texture(), "back"); texture(sb::Texture(), "back");
} }
void CameraView::generate(const glm::vec2& size) void PlaneDoubleBuffer::generate(const glm::vec2& size)
{ {
for (sb::Texture* buffer : {&texture("front"), &texture("back")}) for (sb::Texture* buffer : {&texture("front"), &texture("back")})
{ {
@ -193,17 +193,17 @@ void CameraView::generate(const glm::vec2& size)
} }
} }
sb::Texture& CameraView::current() sb::Texture& PlaneDoubleBuffer::active()
{ {
return swapped ? texture("back") : texture("front"); return swapped ? texture("back") : texture("front");
} }
sb::Texture& CameraView::free() sb::Texture& PlaneDoubleBuffer::inactive()
{ {
return swapped ? texture("front") : texture("back"); return swapped ? texture("front") : texture("back");
} }
void CameraView::swap() void PlaneDoubleBuffer::swap()
{ {
swapped = !swapped; swapped = !swapped;
} }

20
src/Model.hpp
View File

@ -80,8 +80,11 @@ public:
}; };
/* A Plane that contains a Carousel for cycling through active textures. Only one texture is /*!
* active at a time, returned by Background::current. Carousel only goes forward, using Background::next. */ * A version of `Plane` that contains a `Carousel` for cycling through active `sb::Texture` objects. Only one
* `sb::Texture` is * active at a time, returned by `Background::current`. Carousel only goes forward, using
* `Background::next`.
*/
class Background : public Plane class Background : public Plane
{ {
@ -96,7 +99,12 @@ public:
}; };
class CameraView : public Plane /*!
* A version of `Plane` which contains two texture objects, one of which is active at a time. A reference
* to the active `sb::Texture` object is available from `PlaneDoubleBuffer.active`, and the inactive object is
* available from `PlaneDoubleBuffer.inactive`. The buffers can be swapped using `PlaneDoubleBuffer.swap`.
*/
class PlaneDoubleBuffer : public Plane
{ {
private: private:
@ -105,10 +113,10 @@ private:
public: public:
CameraView(); PlaneDoubleBuffer();
void generate(const glm::vec2&); void generate(const glm::vec2&);
sb::Texture& current(); sb::Texture& active();
sb::Texture& free(); sb::Texture& inactive();
void swap(); void swap();
}; };

200
src/Pudding.cpp
View File

@ -1,11 +1,11 @@
/* _______________ ,-------------------------------------------------------------------. /* _______________ ,--------------------------------------------------------.
//`````````````\\ \ \ //`````````````\\ \ \
//~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \ //~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \
//=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \ //=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \
// \\ \ \ // \\ \ \
// \\ \ code released under zlib license [git.nugget.fun/nugget/gunkiss] \ // \\ \ zlib licensed code at [git.nugget.fun/nugget/gunkiss] \
// ☆ GUNKISS ☆ \\ \ \ // ☆ GUNKISS ☆ \\ \ \
//_________________________\\ `-------------------------------------------------------------------' //_________________________\\ `--------------------------------------------------------'
Generate a custom pudding from food product UPC codes and help a pair of rats take over the video game industry, using Generate a custom pudding from food product UPC codes and help a pair of rats take over the video game industry, using
their extraterrestrial ability to turn trash into performance enhancing drug puddings that enable business professionals their extraterrestrial ability to turn trash into performance enhancing drug puddings that enable business professionals
@ -31,16 +31,27 @@ Pudding::Pudding()
get_delegate().subscribe(&Pudding::respond, this); get_delegate().subscribe(&Pudding::respond, this);
get_delegate().subscribe(&Pudding::respond, this, SDL_MOUSEMOTION); get_delegate().subscribe(&Pudding::respond, this, SDL_MOUSEMOTION);
get_delegate().subscribe(&Pudding::respond, this, SDL_MOUSEBUTTONDOWN); get_delegate().subscribe(&Pudding::respond, this, SDL_MOUSEBUTTONDOWN);
/* initialize a zbar image scanner for reading barcodes of any format */ /* initialize a zbar image scanner for reading barcodes of any format */
image_scanner.set_config(zbar::ZBAR_NONE, zbar::ZBAR_CFG_ENABLE, 1); image_scanner.set_config(zbar::ZBAR_NONE, zbar::ZBAR_CFG_ENABLE, 1);
/* Add a texture to the camera Plane for storing frame image data */
camera_view.texture(sb::Texture());
/* set up pudding model */ /* set up pudding model */
nlohmann::json pudding = get_configuration()["pudding"]; nlohmann::json pudding = get_configuration()["pudding"];
load_pudding_model(pudding["top-radius"], pudding["base-radius"], pudding["ring-vertex-count"], pudding["layer-count"], load_pudding_model(pudding["top-radius"], pudding["base-radius"], pudding["ring-vertex-count"], pudding["layer-count"],
pudding["y-range"][0], pudding["y-range"][1], pudding["gradient-position"]); pudding["y-range"][0], pudding["y-range"][1], pudding["gradient-position"]);
/* loading GL context instead of SDL context for 3D */ /* loading GL context instead of SDL context for 3D */
load_gl_context(); load_gl_context();
/* Load background tiles */
load_tiles(); load_tiles();
/* Load button graphics and create button objects */
load_pads(); load_pads();
/* Load a pointer cursor from the system library that will be freed automatically */ /* Load a pointer cursor from the system library that will be freed automatically */
poke = std::shared_ptr<SDL_Cursor>(SDL_CreateSystemCursor(SDL_SYSTEM_CURSOR_HAND), SDL_FreeCursor); poke = std::shared_ptr<SDL_Cursor>(SDL_CreateSystemCursor(SDL_SYSTEM_CURSOR_HAND), SDL_FreeCursor);
} }
@ -160,12 +171,6 @@ void Pudding::load_gl_context()
{ {
super::load_gl_context(); super::load_gl_context();
/* Create another GL context for loading camera frame textures */
if ((capture_frame_thread_context = SDL_GL_CreateContext(window)) == nullptr)
{
sb::Log::log("could not create capture frame thread context");
}
/* Generate a vertex array object ID, bind it as current (requirement of OpenGL) */ /* Generate a vertex array object ID, bind it as current (requirement of OpenGL) */
vao.generate(); vao.generate();
vao.bind(); vao.bind();
@ -265,13 +270,14 @@ void Pudding::load_pads()
next_button.rotation(glm::radians(180.0f)); next_button.rotation(glm::radians(180.0f));
} }
/* Try to create cv::VideoCapture object using device ID #0. If successful, this will create GL texture IDs and storage /*!
* for the camera frames, so it must be called after GL context has been created. Two textures will be created, so they * Try to create cv::VideoCapture object using device ID #0. If successful, this will also create a GL texture ID and
* can be used as a double buffer. * storage for the camera frame on the GPU, so it must be called after GL context has been created. Create and detach
* a thread which will continuously read frame data.
*/ */
void Pudding::initialize_camera() void Pudding::initialize_camera()
{ {
/* initialize an opencv capture device for getting images from an attached camera */ /* Open the OpenCV capture and assign device ID 0 for getting images from the default attached camera. */
int device_id = 0; int device_id = 0;
capture.open(device_id); capture.open(device_id);
std::ostringstream message; std::ostringstream message;
@ -280,9 +286,13 @@ void Pudding::initialize_camera()
message << "opened and initialized " << capture.get(cv::CAP_PROP_FRAME_WIDTH) << "x" << message << "opened and initialized " << capture.get(cv::CAP_PROP_FRAME_WIDTH) << "x" <<
capture.get(cv::CAP_PROP_FRAME_HEIGHT) << ", " << capture.get(cv::CAP_PROP_FPS) << capture.get(cv::CAP_PROP_FRAME_HEIGHT) << ", " << capture.get(cv::CAP_PROP_FPS) <<
"fps video capture device ID #" << device_id << " using " << capture.getBackendName(); "fps video capture device ID #" << device_id << " using " << capture.getBackendName();
/* generate two textures that will store the video frames with the intention of double buffering them
* for threaded texture loading */ /* Use the texture object to generate a texture the size of the camera's resolution. */
camera_view.generate({capture.get(cv::CAP_PROP_FRAME_WIDTH), capture.get(cv::CAP_PROP_FRAME_HEIGHT)}); camera_view.texture().generate({capture.get(cv::CAP_PROP_FRAME_WIDTH), capture.get(cv::CAP_PROP_FRAME_HEIGHT)});
/* Create and detach a thread which will read frame data */
std::thread camera_thread(&Pudding::capture_frame, this);
camera_thread.detach();
} }
else else
{ {
@ -818,100 +828,48 @@ bool Pudding::item_display_active() const
return show_item && items.size() > 0; return show_item && items.size() > 0;
} }
/* Read pixels from the camera into a GL texture. This function is meant to be launched in a separate thread, /*!
* so it will use its own GL context to load the pixels into either the front or back texture buffer, depending * Read pixels from the camera into a `cv::Mat`. This function is meant to be launched in a separate thread,
* on which is not currently in use */ * where it will run continuously, setting `new_frame_available` to `false` before loading camera frame data
int Pudding::capture_frame(void* game) * into the `cv::Mat` object at `camera_frame`, then setting `finished_loading_frame` to `true` to indicate
* new frame data is available in `camera_frame`.
*/
void Pudding::capture_frame()
{ {
time_it("total thread")([&]{ /* When capture is closed, this thread will automatically finish execution. */
Pudding* pudding = reinterpret_cast<Pudding*>(game); while (capture.isOpened())
/* Make the thread context the current context (not sure what that means, but it doesn't seem to conflict
* with the main rendering context) */
if (time_it<int>("make current")([&] { return SDL_GL_MakeCurrent(pudding->window, pudding->capture_frame_thread_context); }) < 0)
{ {
sb::Log::log("error making thread context current"); /* The frame data in the `cv::Mat` at `pudding->camera_frame` is about to be modified by the rest of
} * this function, so even if there is data stored there that hasn't been read yet, it should not
else * be read by the main thread until this is set to `true`at the end of the function. */
{ new_frame_available = false;
if (pudding->capture.isOpened())
/* Load camera frame data into `cv::Mat` */
time_it("read frame")([&]{
capture >> camera_frame;
});
if (!camera_frame.empty())
{ {
cv::Mat frame; /* Rotate the frame 180 degrees to work with OpenGL coords */
time_it("read frame")([&]{ time_it("flip")([&]{
pudding->capture.read(frame); cv::flip(camera_frame, camera_frame, -1);
}); });
if (!frame.empty())
{ /* Finished loading into `cv::Mat`, so it is new data that is safe to read. */
time_it("flip")([&]{ new_frame_available = true;
/* rotate the opencv matrix 180 to work with opengl coords */
cv::flip(frame, frame, -1);
});
time_it("load texture")([&]{
/* use whichever texture ID is not being used by the main rendering thread */
sb::Texture& texture = pudding->camera_view.free();
/* bind texture for accepting pixel data */
texture.bind();
/* fill texture memory with last frame's pixels */
texture.load(frame.ptr(), {frame.cols, frame.rows}, GL_BGR, GL_UNSIGNED_BYTE);
});
pudding->camera_view.swap();
if (pudding->get_configuration()["scan"]["enabled"])
{
time_it("gray")([&]{
/* convert to gray and scan with zbar */
cv::cvtColor(frame, frame, cv::COLOR_BGR2GRAY);
});
zbar::Image query_image(frame.cols, frame.rows, "Y800", static_cast<void*>(frame.data), frame.cols * frame.rows);
int result = pudding->image_scanner.scan(query_image);
if (result > 0)
{
time_it("barcode lookup")([&] {
for (zbar::Image::SymbolIterator symbol = query_image.symbol_begin(); symbol != query_image.symbol_end(); ++symbol)
{
std::ostringstream message;
message << "camera scanned " << symbol->get_type_name() << " symbol " << symbol->get_data();
sb::Log::log(message);
pudding->current_camera_barcode = symbol->get_data();
pudding->current_barcode = pudding->current_camera_barcode;
}
});
}
query_image.set_data(nullptr, 0);
}
}
frame.release();
} }
SDL_GL_MakeCurrent(pudding->window, nullptr);
sb::Log::gl_errors("in capture thread, after capturing frame"); sb::Log::gl_errors("in capture thread, after capturing frame");
std::this_thread::sleep_for(std::chrono::milliseconds(50));
} }
pudding->reading_capture_frame = false;
// using namespace std::chrono_literals;
// std::this_thread::sleep_for(2s);
});
return 0;
} }
/* Update parameters and draw the screen */ /* Update parameters and draw the screen */
void Pudding::update() void Pudding::update()
{ {
/* number of seconds we've been running for */ /* Time in seconds the game has running for */
float time_seconds = SDL_GetTicks() / 1000.0f; float time_seconds = SDL_GetTicks() / 1000.0f;
{
/* launch the camera capture thread if it is not currently running */
if (capture.isOpened() && !reading_capture_frame)
{
SDL_Thread* capture_thread = SDL_CreateThread(Pudding::capture_frame, "capture frame", reinterpret_cast<void*>(this));
if (capture_thread == nullptr)
{
sb::Log::log("could not create capture thread");
}
else
{
reading_capture_frame = true;
SDL_WaitThread(capture_thread, nullptr);
}
}
}
sb::Log::gl_errors("in main thread, after capturing frame");
/* if the config is set to refresh automatically, there may be a new barcode available */ /* if the config is set to refresh automatically, there may be a new barcode available */
if (current_config_barcode != get_configuration()["scan"]["barcode"]) if (current_config_barcode != get_configuration()["scan"]["barcode"])
{ {
@ -921,13 +879,49 @@ void Pudding::update()
message << "read new barcode from config " << current_barcode; message << "read new barcode from config " << current_barcode;
sb::Log::log(message); sb::Log::log(message);
} }
/* If a new frame is finished loading in the detached thread, copy the frame data into a texture, process the frame
* for scanning and scan it. */
if (new_frame_available)
{
camera_view.texture().bind();
/* Fill camera view texture memory with last frame's pixels */
camera_view.texture().load(camera_frame.ptr(), {camera_frame.cols, camera_frame.rows}, GL_BGR, GL_UNSIGNED_BYTE);
/* Frame data has been loaded, so there is not a new frame available anymore. */
new_frame_available = false;
/* Convert to grayscale for ZBar */
cv::cvtColor(camera_frame, camera_frame, cv::COLOR_BGR2GRAY);
if (get_configuration()["scan"]["enabled"])
{
zbar::Image query_image(camera_frame.cols, camera_frame.rows, "Y800", static_cast<void*>(camera_frame.data),
camera_frame.cols * camera_frame.rows);
int result = image_scanner.scan(query_image);
if (result > 0)
{
time_it("barcode lookup")([&] {
for (zbar::Image::SymbolIterator symbol = query_image.symbol_begin(); symbol != query_image.symbol_end(); ++symbol)
{
std::ostringstream message;
message << "camera scanned " << symbol->get_type_name() << " symbol " << symbol->get_data();
sb::Log::log(message);
current_camera_barcode = symbol->get_data();
current_barcode = current_camera_barcode;
}
});
}
query_image.set_data(nullptr, 0);
}
}
/* viewport box will be used to tell GL where to draw */ /* viewport box will be used to tell GL where to draw */
viewport = window_box(true); viewport = window_box(true);
/* shrink viewport if item texture or camera will be displayed */ /* shrink viewport if item texture or camera will be displayed */
if (item_display_active() || capture.isOpened()) if (item_display_active() || capture.isOpened())
{ {
viewport.drag_bottom(0.5f * get_configuration()["interface"]["pop-up-viewport-height"].get<float>() * viewport.height()); viewport.drag_bottom(0.5f * get_configuration()["interface"]["pop-up-viewport-height"].get<float>() * viewport.height());
} }
/* Save the main viewport dimensions */ /* Save the main viewport dimensions */
main_viewport = viewport; main_viewport = viewport;
glViewport(viewport); glViewport(viewport);
@ -1038,7 +1032,7 @@ void Pudding::update()
glViewport(viewport); glViewport(viewport);
/* bind texture for drawing */ /* bind texture for drawing */
glUniformMatrix4fv(uniform["flat"]["transformation"], 1, GL_FALSE, &camera_view.transformation()[0][0]); glUniformMatrix4fv(uniform["flat"]["transformation"], 1, GL_FALSE, &camera_view.transformation()[0][0]);
camera_view.current().bind(); camera_view.texture().bind();
camera_view.enable(); camera_view.enable();
/* draws rectangle vertices and rectangle texture using UV coords */ /* draws rectangle vertices and rectangle texture using UV coords */
glDrawArrays(GL_TRIANGLES, 0, camera_view.attributes("position")->count()); glDrawArrays(GL_TRIANGLES, 0, camera_view.attributes("position")->count());

10
src/Pudding.hpp
View File

@ -345,23 +345,21 @@ private:
Carousel item_carousel; Carousel item_carousel;
int effect_id = EFFECT_NONE, pudding_triangle_vertex_count = 0, pudding_fan_vertex_count = 0; int effect_id = EFFECT_NONE, pudding_triangle_vertex_count = 0, pudding_fan_vertex_count = 0;
cv::VideoCapture capture; cv::VideoCapture capture;
cv::Mat camera_frame;
zbar::ImageScanner image_scanner; zbar::ImageScanner image_scanner;
std::map<std::string, std::map<std::string, GLuint>> uniform; std::map<std::string, std::map<std::string, GLuint>> uniform;
GLuint flat_program, mvp_program; GLuint flat_program, mvp_program;
glm::mat4 projection, model {1.0f}, mvp; glm::mat4 projection, model {1.0f}, mvp;
Model pudding_model; Model pudding_model;
Plane plane; Plane plane, camera_view;
Background background; Background background;
CameraView camera_view; bool show_item = false, new_frame_available = false;
bool show_item = false, reading_capture_frame = false;
SDL_GLContext capture_frame_thread_context = nullptr;
sb::VAO vao; sb::VAO vao;
sb::VBO vbo; sb::VBO vbo;
std::map<std::string, sb::Texture> labels; std::map<std::string, sb::Texture> labels;
Pad camera_button, previous_button, next_button, inventory_button; Pad camera_button, previous_button, next_button, inventory_button;
Box viewport, main_viewport, pop_up_viewport; Box viewport, main_viewport, pop_up_viewport;
std::mutex camera_mutex; std::mutex camera_mutex;
SDL_Thread* capture_thread = nullptr;
void load_pudding_model(float, float, int, int = 1, float = -1.0f, float = 1.0f, float = 0.3f); void load_pudding_model(float, float, int, int = 1, float = -1.0f, float = 1.0f, float = 0.3f);
void load_gl_context(); void load_gl_context();
@ -380,7 +378,7 @@ private:
sb::Texture texture_from_image_url(const std::string&) const; sb::Texture texture_from_image_url(const std::string&) const;
static void destroy_texture(GLuint*); static void destroy_texture(GLuint*);
bool item_display_active() const; bool item_display_active() const;
static int capture_frame(void*); void capture_frame();
/* Initialize camera on connection and release on disconnection. */ /* Initialize camera on connection and release on disconnection. */
Connection<> camera_switch { Connection<> camera_switch {