shampoo/gunkiss
shampoo
/
gunkiss
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

load camera frame texture in separate thread

This commit is contained in:
frank 2021-09-09 22:57:13 -04:00
parent 5aab43896c
commit eb391b911a
4 changed files with 108 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
config.json
View File

@ -43,6 +43,7 @@
},
"scan":
{
"enabled": true,
"json-save": true,
"json-save-directory": "local/scans",
"barcode": "",

2
lib/sb

@ -1 +1 @@
Subproject commit 18f83968f3f2b7480d7557e18946f51706dda6b6
Subproject commit 87b1fa735c5d663052995177c6d2f639eeac713b

140
src/Pudding.cpp
View File

@ -146,6 +146,12 @@ void Pudding::set_pudding_model(
void Pudding::load_gl_context()
{
super::load_gl_context();
/* create another GL context for loading camera frame textures */
SDL_GL_SetAttribute(SDL_GL_SHARE_WITH_CURRENT_CONTEXT, 1);
if ((capture_frame_thread_context = SDL_GL_CreateContext(window)) == nullptr)
{
log("could not create capture frame thread context");
}
/* load background as surface, generate texture to load pixel data into, allocate storage, bind and edit texture properties */
std::unique_ptr<SDL_Surface, decltype(&SDL_FreeSurface)> surface(IMG_Load("local/tptile.jpg"), SDL_FreeSurface);
std::unique_ptr<SDL_Surface, decltype(&SDL_FreeSurface)> flipped_surface(rotozoomSurfaceXY(surface.get(), 0, 1, -1, 0), SDL_FreeSurface);
@ -226,8 +232,9 @@ void Pudding::load_gl_context()
log_gl_errors();
}
/* Try to create cv::VideoCapture object using device ID #0. If successful, this will create a GL texture for displaying
* the frames, so it must be called after GL context has been created.
/* Try to create cv::VideoCapture object using device ID #0. If successful, this will create GL texture IDs and storage
* for the camera frames, so it must be called after GL context has been created. Two textures will be created, so they
* can be used as a double buffer.
*/
void Pudding::initialize_camera()
{
@ -240,12 +247,18 @@ void Pudding::initialize_camera()
message << "opened and initialized " << capture.get(cv::CAP_PROP_FRAME_WIDTH) << "x" <<
capture.get(cv::CAP_PROP_FRAME_HEIGHT) << ", " << capture.get(cv::CAP_PROP_FPS) <<
"fps video capture device ID #" << device_id << " using " << capture.getBackendName();
/* generate the texture that will store the video frame, allocate storage for a video frame, bind and edit texture properties */
glGenTextures(1, &video_capture_texture_id);
glBindTexture(GL_TEXTURE_2D, video_capture_texture_id);
glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGB8, capture.get(cv::CAP_PROP_FRAME_WIDTH), capture.get(cv::CAP_PROP_FRAME_HEIGHT));
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
/* generate two textures that will store the video frames with the intention of double buffering them
* for threaded texture loading */
for (GLuint* buffer_id : {&capture_texture_front_buffer_id, &capture_texture_back_buffer_id})
{
glGenTextures(1, buffer_id);
glBindTexture(GL_TEXTURE_2D, *buffer_id);
glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGB8, capture.get(cv::CAP_PROP_FRAME_WIDTH), capture.get(cv::CAP_PROP_FRAME_HEIGHT));
// glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGB8, 320, 240);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
}
capture_texture_id = capture_texture_front_buffer_id;
}
else
{
@ -635,9 +648,80 @@ bool Pudding::item_display_active() const
return show_item && items.size() > 0;
}
/* Read pixels from the camera into a GL texture. This function is meant to be launched in a separate thread,
* so it will use its own GL context to load the pixels into either the front or back texture buffer, depending
* on which is not currently in use */
int Pudding::capture_frame(void* game)
{
Pudding* pudding = reinterpret_cast<Pudding*>(game);
/* Make the thread context the current context (not sure what that means, but it doesn't seem to conflict
* with the main rendering context) */
if (SDL_GL_MakeCurrent(pudding->window, pudding->capture_frame_thread_context) < 0)
{
pudding->log("error making thread context current");
}
else
{
if (pudding->capture.isOpened())
{
cv::Mat frame;
pudding->capture.read(frame);
if (!frame.empty())
{
/* rotate the opencv matrix 180 to work with opengl coords */
cv::flip(frame, frame, -1);
/* use whichever texture ID is not being used by the main rendering thread */
GLuint texture_id = pudding->capture_texture_id == pudding->capture_texture_front_buffer_id ?
pudding->capture_texture_back_buffer_id : pudding->capture_texture_front_buffer_id;
/* bind texture for accepting pixel data */
glBindTexture(GL_TEXTURE_2D, texture_id);
/* fill texture memory with last frame's pixels */
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, frame.cols, frame.rows, GL_BGR, GL_UNSIGNED_BYTE, frame.ptr());
pudding->capture_texture_id = texture_id;
if (pudding->get_configuration()["scan"]["enabled"])
{
/* convert to gray and scan with zbar */
cv::cvtColor(frame, frame, cv::COLOR_BGR2GRAY);
zbar::Image query_image(frame.cols, frame.rows, "Y800", static_cast<void*>(frame.data), frame.cols * frame.rows);
int result = pudding->image_scanner.scan(query_image);
if (result > 0)
{
for (zbar::Image::SymbolIterator symbol = query_image.symbol_begin(); symbol != query_image.symbol_end(); ++symbol)
{
std::stringstream message;
message << "camera scanned " << symbol->get_type_name() << " symbol " << symbol->get_data();
pudding->log(message.str());
pudding->current_camera_barcode = symbol->get_data();
pudding->current_barcode = pudding->current_camera_barcode;
}
}
query_image.set_data(nullptr, 0);
}
}
frame.release();
}
SDL_GL_MakeCurrent(pudding->window, nullptr);
}
pudding->reading_capture_frame = false;
return 0;
}
/* Update parameters and draw the screen */
void Pudding::update()
{
/* launch the camera capture thread if it is not currently running */
if (capture.isOpened() && !reading_capture_frame)
{
SDL_Thread* capture_thread = SDL_CreateThread(Pudding::capture_frame, "capture frame", reinterpret_cast<void*>(this));
if (capture_thread == nullptr)
{
log("could not create capture thread");
}
else
{
reading_capture_frame = true;
}
}
/* if the config is set to refresh automatically, there may be a new barcode available */
if (current_config_barcode != get_configuration()["scan"]["barcode"])
{
@ -706,6 +790,8 @@ void Pudding::update()
/* draw pudding model */
glEnable(GL_DEPTH_TEST);
glDrawArrays(GL_TRIANGLES, 0, pudding_vertices.size());
/* regular fill mode enabled for all other drawing */
glPolygonMode(GL_FRONT_AND_BACK, GL_FILL);
/* only do more drawing if items are downloaded or camera is enabled */
if (item_display_active() || capture.isOpened())
{
@ -743,40 +829,12 @@ void Pudding::update()
/* draw the camera if the camera has been opened */
if (capture.isOpened())
{
capture.read(capture_frame);
viewport_box.left(window_box(true).left());
if (!capture_frame.empty())
{
/* rotate the opencv matrix 180 to work with opengl coords */
cv::flip(capture_frame, capture_frame, -1);
glViewport(viewport_box.left(), viewport_box.bottom(), viewport_box.width(), viewport_box.height());
/* bind texture, binding it to accept pixel data and to GLSL sampler */
glBindTexture(GL_TEXTURE_2D, video_capture_texture_id);
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, capture_frame.cols, capture_frame.rows, GL_BGR, GL_UNSIGNED_BYTE, capture_frame.ptr());
/* draws rectangle vertices and rectangle texture using UV coords */
glDrawArrays(GL_TRIANGLES, 0, 6);
/* convert to gray and scan with zbar */
cv::cvtColor(capture_frame, capture_frame, cv::COLOR_BGR2GRAY);
zbar::Image query_image(capture_frame.cols, capture_frame.rows, "Y800", static_cast<void*>(capture_frame.data),
capture_frame.cols * capture_frame.rows);
int result = image_scanner.scan(query_image);
if (result > 0)
{
for (zbar::Image::SymbolIterator symbol = query_image.symbol_begin(); symbol != query_image.symbol_end(); ++symbol)
{
std::stringstream message;
message << "camera scanned " << symbol->get_type_name() << " symbol " << symbol->get_data();
log(message.str());
current_camera_barcode = symbol->get_data();
current_barcode = current_camera_barcode;
}
}
query_image.set_data(nullptr, 0);
}
else
{
debug("video capture device frame empty");
}
glViewport(viewport_box.left(), viewport_box.bottom(), viewport_box.width(), viewport_box.height());
/* bind texture for drawing */
glBindTexture(GL_TEXTURE_2D, capture_texture_id);
/* draws rectangle vertices and rectangle texture using UV coords */
glDrawArrays(GL_TRIANGLES, 0, 6);
}
}
SDL_GL_SwapWindow(get_window());

11
src/Pudding.hpp
View File

@ -5,6 +5,7 @@
#include <string>
#include <iostream>
#include <algorithm>
#include <thread>
#include <memory>
#include <type_traits>
#include <filesystem>
@ -21,6 +22,7 @@
#include "Color.hpp"
#include "extension.hpp"
#include "Item.hpp"
#include "Animation.hpp"
class Pudding : public Game
{
@ -54,15 +56,15 @@ private:
std::vector<Item> items;
int current_item_index = 0;
cv::VideoCapture capture;
cv::Mat capture_frame;
zbar::ImageScanner image_scanner;
GLuint flat_program, mvp_program, video_capture_texture_id, mvp_uniform_location, background_texture_id, time_uniform_location,
effect_uniform_location;
GLuint flat_program, mvp_program, capture_texture_front_buffer_id, capture_texture_back_buffer_id, capture_texture_id,
mvp_uniform_location, background_texture_id, time_uniform_location, effect_uniform_location;
glm::mat4 projection, model = glm::mat4(1.0f), mvp;
std::vector<glm::vec3> pudding_vertices, pudding_colors;
std::vector<glm::vec2> pudding_uv;
bool show_item;
bool show_item = false, reading_capture_frame = false;
int effect_id = Effect::NONE;
SDL_GLContext capture_frame_thread_context = nullptr;
void set_pudding_model(float, float, int, int = 1, float = -1, float = 1, float = 0.3f);
void load_gl_context();
@ -78,6 +80,7 @@ private:
std::shared_ptr<GLuint> texture_from_image_url(const std::string&);
static void destroy_texture(GLuint*);
bool item_display_active() const;
static int capture_frame(void*);
public: