shampoo/gunkiss
shampoo
/
gunkiss
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

use a single GL context by moving texture loading out of camera frame thread and use the thread only for reading frame data into cv::Mat

This commit is contained in:
ohsqueezy 2022-08-08 19:28:25 -04:00
parent c500cd332e
commit 17dee6174f
5 changed files with 122 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
config.json
View File

@ -63,8 +63,8 @@
"best-buy-api-key": "vAC23XA5YWBzaYiGtOkoNlXZ",
"giantbomb-api-key": "91a395231f4e1fd9f9ba8840c52a61cda343cd70",
"nutronix-enabled": false,
"edamam-enabled": true,
"open-food-enabled": false,
"edamam-enabled": false,
"open-food-enabled": true,
"open-products-enabled": true,
"best-buy-enabled": true,
"google-books-enabled": true

10
src/Model.cpp
View File

@ -179,13 +179,13 @@ sb::Texture& Background::current()
return carousel.current(textures())->second;
}
CameraView::CameraView() : Plane()
PlaneDoubleBuffer::PlaneDoubleBuffer() : Plane()
{
texture(sb::Texture(), "front");
texture(sb::Texture(), "back");
}
void CameraView::generate(const glm::vec2& size)
void PlaneDoubleBuffer::generate(const glm::vec2& size)
{
for (sb::Texture* buffer : {&texture("front"), &texture("back")})
{
@ -193,17 +193,17 @@ void CameraView::generate(const glm::vec2& size)
}
}
sb::Texture& CameraView::current()
sb::Texture& PlaneDoubleBuffer::active()
{
return swapped ? texture("back") : texture("front");
}
sb::Texture& CameraView::free()
sb::Texture& PlaneDoubleBuffer::inactive()
{
return swapped ? texture("front") : texture("back");
}
void CameraView::swap()
void PlaneDoubleBuffer::swap()
{
swapped = !swapped;
}

20
src/Model.hpp
View File

@ -80,8 +80,11 @@ public:
};
/* A Plane that contains a Carousel for cycling through active textures. Only one texture is
* active at a time, returned by Background::current. Carousel only goes forward, using Background::next. */
/*!
* A version of `Plane` that contains a `Carousel` for cycling through active `sb::Texture` objects. Only one
* `sb::Texture` is * active at a time, returned by `Background::current`. Carousel only goes forward, using
* `Background::next`.
*/
class Background : public Plane
{
@ -96,7 +99,12 @@ public:
};
class CameraView : public Plane
/*!
* A version of `Plane` which contains two texture objects, one of which is active at a time. A reference
* to the active `sb::Texture` object is available from `PlaneDoubleBuffer.active`, and the inactive object is
* available from `PlaneDoubleBuffer.inactive`. The buffers can be swapped using `PlaneDoubleBuffer.swap`.
*/
class PlaneDoubleBuffer : public Plane
{
private:
@ -105,10 +113,10 @@ private:
public:
CameraView();
PlaneDoubleBuffer();
void generate(const glm::vec2&);
sb::Texture& current();
sb::Texture& free();
sb::Texture& active();
sb::Texture& inactive();
void swap();
};

200
src/Pudding.cpp
View File

@ -1,11 +1,11 @@
/* _______________ ,-------------------------------------------------------------------.
//`````````````\\ \ \
//~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \
//=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \
// \\ \ \
// \\ \ code released under zlib license [git.nugget.fun/nugget/gunkiss] \
// ☆ GUNKISS ☆ \\ \ \
//_________________________\\ `-------------------------------------------------------------------'
/* _______________ ,--------------------------------------------------------.
//`````````````\\ \ \
//~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \
//=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \
// \\ \ \
// \\ \ zlib licensed code at [git.nugget.fun/nugget/gunkiss] \
// ☆ GUNKISS ☆ \\ \ \
//_________________________\\ `--------------------------------------------------------'
Generate a custom pudding from food product UPC codes and help a pair of rats take over the video game industry, using
their extraterrestrial ability to turn trash into performance enhancing drug puddings that enable business professionals
@ -31,16 +31,27 @@ Pudding::Pudding()
get_delegate().subscribe(&Pudding::respond, this);
get_delegate().subscribe(&Pudding::respond, this, SDL_MOUSEMOTION);
get_delegate().subscribe(&Pudding::respond, this, SDL_MOUSEBUTTONDOWN);
/* initialize a zbar image scanner for reading barcodes of any format */
image_scanner.set_config(zbar::ZBAR_NONE, zbar::ZBAR_CFG_ENABLE, 1);
/* Add a texture to the camera Plane for storing frame image data */
camera_view.texture(sb::Texture());
/* set up pudding model */
nlohmann::json pudding = get_configuration()["pudding"];
load_pudding_model(pudding["top-radius"], pudding["base-radius"], pudding["ring-vertex-count"], pudding["layer-count"],
pudding["y-range"][0], pudding["y-range"][1], pudding["gradient-position"]);
/* loading GL context instead of SDL context for 3D */
load_gl_context();
/* Load background tiles */
load_tiles();
/* Load button graphics and create button objects */
load_pads();
/* Load a pointer cursor from the system library that will be freed automatically */
poke = std::shared_ptr<SDL_Cursor>(SDL_CreateSystemCursor(SDL_SYSTEM_CURSOR_HAND), SDL_FreeCursor);
}
@ -160,12 +171,6 @@ void Pudding::load_gl_context()
{
super::load_gl_context();
/* Create another GL context for loading camera frame textures */
if ((capture_frame_thread_context = SDL_GL_CreateContext(window)) == nullptr)
{
sb::Log::log("could not create capture frame thread context");
}
/* Generate a vertex array object ID, bind it as current (requirement of OpenGL) */
vao.generate();
vao.bind();
@ -265,13 +270,14 @@ void Pudding::load_pads()
next_button.rotation(glm::radians(180.0f));
}
/* Try to create cv::VideoCapture object using device ID #0. If successful, this will create GL texture IDs and storage
* for the camera frames, so it must be called after GL context has been created. Two textures will be created, so they
* can be used as a double buffer.
/*!
* Try to create cv::VideoCapture object using device ID #0. If successful, this will also create a GL texture ID and
* storage for the camera frame on the GPU, so it must be called after GL context has been created. Create and detach
* a thread which will continuously read frame data.
*/
void Pudding::initialize_camera()
{
/* initialize an opencv capture device for getting images from an attached camera */
/* Open the OpenCV capture and assign device ID 0 for getting images from the default attached camera. */
int device_id = 0;
capture.open(device_id);
std::ostringstream message;
@ -280,9 +286,13 @@ void Pudding::initialize_camera()
message << "opened and initialized " << capture.get(cv::CAP_PROP_FRAME_WIDTH) << "x" <<
capture.get(cv::CAP_PROP_FRAME_HEIGHT) << ", " << capture.get(cv::CAP_PROP_FPS) <<
"fps video capture device ID #" << device_id << " using " << capture.getBackendName();
/* generate two textures that will store the video frames with the intention of double buffering them
* for threaded texture loading */
camera_view.generate({capture.get(cv::CAP_PROP_FRAME_WIDTH), capture.get(cv::CAP_PROP_FRAME_HEIGHT)});
/* Use the texture object to generate a texture the size of the camera's resolution. */
camera_view.texture().generate({capture.get(cv::CAP_PROP_FRAME_WIDTH), capture.get(cv::CAP_PROP_FRAME_HEIGHT)});
/* Create and detach a thread which will read frame data */
std::thread camera_thread(&Pudding::capture_frame, this);
camera_thread.detach();
}
else
{
@ -818,100 +828,48 @@ bool Pudding::item_display_active() const
return show_item && items.size() > 0;
}
/* Read pixels from the camera into a GL texture. This function is meant to be launched in a separate thread,
* so it will use its own GL context to load the pixels into either the front or back texture buffer, depending
* on which is not currently in use */
int Pudding::capture_frame(void* game)
/*!
* Read pixels from the camera into a `cv::Mat`. This function is meant to be launched in a separate thread,
* where it will run continuously, setting `new_frame_available` to `false` before loading camera frame data
* into the `cv::Mat` object at `camera_frame`, then setting `finished_loading_frame` to `true` to indicate
* new frame data is available in `camera_frame`.
*/
void Pudding::capture_frame()
{
time_it("total thread")([&]{
Pudding* pudding = reinterpret_cast<Pudding*>(game);
/* Make the thread context the current context (not sure what that means, but it doesn't seem to conflict
* with the main rendering context) */
if (time_it<int>("make current")([&] { return SDL_GL_MakeCurrent(pudding->window, pudding->capture_frame_thread_context); }) < 0)
/* When capture is closed, this thread will automatically finish execution. */
while (capture.isOpened())
{
sb::Log::log("error making thread context current");
}
else
{
if (pudding->capture.isOpened())
/* The frame data in the `cv::Mat` at `pudding->camera_frame` is about to be modified by the rest of
* this function, so even if there is data stored there that hasn't been read yet, it should not
* be read by the main thread until this is set to `true`at the end of the function. */
new_frame_available = false;
/* Load camera frame data into `cv::Mat` */
time_it("read frame")([&]{
capture >> camera_frame;
});
if (!camera_frame.empty())
{
cv::Mat frame;
time_it("read frame")([&]{
pudding->capture.read(frame);
/* Rotate the frame 180 degrees to work with OpenGL coords */
time_it("flip")([&]{
cv::flip(camera_frame, camera_frame, -1);
});
if (!frame.empty())
{
time_it("flip")([&]{
/* rotate the opencv matrix 180 to work with opengl coords */
cv::flip(frame, frame, -1);
});
time_it("load texture")([&]{
/* use whichever texture ID is not being used by the main rendering thread */
sb::Texture& texture = pudding->camera_view.free();
/* bind texture for accepting pixel data */
texture.bind();
/* fill texture memory with last frame's pixels */
texture.load(frame.ptr(), {frame.cols, frame.rows}, GL_BGR, GL_UNSIGNED_BYTE);
});
pudding->camera_view.swap();
if (pudding->get_configuration()["scan"]["enabled"])
{
time_it("gray")([&]{
/* convert to gray and scan with zbar */
cv::cvtColor(frame, frame, cv::COLOR_BGR2GRAY);
});
zbar::Image query_image(frame.cols, frame.rows, "Y800", static_cast<void*>(frame.data), frame.cols * frame.rows);
int result = pudding->image_scanner.scan(query_image);
if (result > 0)
{
time_it("barcode lookup")([&] {
for (zbar::Image::SymbolIterator symbol = query_image.symbol_begin(); symbol != query_image.symbol_end(); ++symbol)
{
std::ostringstream message;
message << "camera scanned " << symbol->get_type_name() << " symbol " << symbol->get_data();
sb::Log::log(message);
pudding->current_camera_barcode = symbol->get_data();
pudding->current_barcode = pudding->current_camera_barcode;
}
});
}
query_image.set_data(nullptr, 0);
}
}
frame.release();
/* Finished loading into `cv::Mat`, so it is new data that is safe to read. */
new_frame_available = true;
}
SDL_GL_MakeCurrent(pudding->window, nullptr);
sb::Log::gl_errors("in capture thread, after capturing frame");
std::this_thread::sleep_for(std::chrono::milliseconds(50));
}
pudding->reading_capture_frame = false;
// using namespace std::chrono_literals;
// std::this_thread::sleep_for(2s);
});
return 0;
}
/* Update parameters and draw the screen */
void Pudding::update()
{
/* number of seconds we've been running for */
/* Time in seconds the game has running for */
float time_seconds = SDL_GetTicks() / 1000.0f;
{
/* launch the camera capture thread if it is not currently running */
if (capture.isOpened() && !reading_capture_frame)
{
SDL_Thread* capture_thread = SDL_CreateThread(Pudding::capture_frame, "capture frame", reinterpret_cast<void*>(this));
if (capture_thread == nullptr)
{
sb::Log::log("could not create capture thread");
}
else
{
reading_capture_frame = true;
SDL_WaitThread(capture_thread, nullptr);
}
}
}
sb::Log::gl_errors("in main thread, after capturing frame");
/* if the config is set to refresh automatically, there may be a new barcode available */
if (current_config_barcode != get_configuration()["scan"]["barcode"])
{
@ -921,13 +879,49 @@ void Pudding::update()
message << "read new barcode from config " << current_barcode;
sb::Log::log(message);
}
/* If a new frame is finished loading in the detached thread, copy the frame data into a texture, process the frame
* for scanning and scan it. */
if (new_frame_available)
{
camera_view.texture().bind();
/* Fill camera view texture memory with last frame's pixels */
camera_view.texture().load(camera_frame.ptr(), {camera_frame.cols, camera_frame.rows}, GL_BGR, GL_UNSIGNED_BYTE);
/* Frame data has been loaded, so there is not a new frame available anymore. */
new_frame_available = false;
/* Convert to grayscale for ZBar */
cv::cvtColor(camera_frame, camera_frame, cv::COLOR_BGR2GRAY);
if (get_configuration()["scan"]["enabled"])
{
zbar::Image query_image(camera_frame.cols, camera_frame.rows, "Y800", static_cast<void*>(camera_frame.data),
camera_frame.cols * camera_frame.rows);
int result = image_scanner.scan(query_image);
if (result > 0)
{
time_it("barcode lookup")([&] {
for (zbar::Image::SymbolIterator symbol = query_image.symbol_begin(); symbol != query_image.symbol_end(); ++symbol)
{
std::ostringstream message;
message << "camera scanned " << symbol->get_type_name() << " symbol " << symbol->get_data();
sb::Log::log(message);
current_camera_barcode = symbol->get_data();
current_barcode = current_camera_barcode;
}
});
}
query_image.set_data(nullptr, 0);
}
}
/* viewport box will be used to tell GL where to draw */
viewport = window_box(true);
/* shrink viewport if item texture or camera will be displayed */
if (item_display_active() || capture.isOpened())
{
viewport.drag_bottom(0.5f * get_configuration()["interface"]["pop-up-viewport-height"].get<float>() * viewport.height());
}
/* Save the main viewport dimensions */
main_viewport = viewport;
glViewport(viewport);
@ -1038,7 +1032,7 @@ void Pudding::update()
glViewport(viewport);
/* bind texture for drawing */
glUniformMatrix4fv(uniform["flat"]["transformation"], 1, GL_FALSE, &camera_view.transformation()[0][0]);
camera_view.current().bind();
camera_view.texture().bind();
camera_view.enable();
/* draws rectangle vertices and rectangle texture using UV coords */
glDrawArrays(GL_TRIANGLES, 0, camera_view.attributes("position")->count());

10
src/Pudding.hpp
View File

@ -345,23 +345,21 @@ private:
Carousel item_carousel;
int effect_id = EFFECT_NONE, pudding_triangle_vertex_count = 0, pudding_fan_vertex_count = 0;
cv::VideoCapture capture;
cv::Mat camera_frame;
zbar::ImageScanner image_scanner;
std::map<std::string, std::map<std::string, GLuint>> uniform;
GLuint flat_program, mvp_program;
glm::mat4 projection, model {1.0f}, mvp;
Model pudding_model;
Plane plane;
Plane plane, camera_view;
Background background;
CameraView camera_view;
bool show_item = false, reading_capture_frame = false;
SDL_GLContext capture_frame_thread_context = nullptr;
bool show_item = false, new_frame_available = false;
sb::VAO vao;
sb::VBO vbo;
std::map<std::string, sb::Texture> labels;
Pad camera_button, previous_button, next_button, inventory_button;
Box viewport, main_viewport, pop_up_viewport;
std::mutex camera_mutex;
SDL_Thread* capture_thread = nullptr;
void load_pudding_model(float, float, int, int = 1, float = -1.0f, float = 1.0f, float = 0.3f);
void load_gl_context();
@ -380,7 +378,7 @@ private:
sb::Texture texture_from_image_url(const std::string&) const;
static void destroy_texture(GLuint*);
bool item_display_active() const;
static int capture_frame(void*);
void capture_frame();
/* Initialize camera on connection and release on disconnection. */
Connection<> camera_switch {