load camera frame texture in separate thread
This commit is contained in:
parent
5aab43896c
commit
eb391b911a
|
@ -43,6 +43,7 @@
|
|||
},
|
||||
"scan":
|
||||
{
|
||||
"enabled": true,
|
||||
"json-save": true,
|
||||
"json-save-directory": "local/scans",
|
||||
"barcode": "",
|
||||
|
|
2
lib/sb
2
lib/sb
|
@ -1 +1 @@
|
|||
Subproject commit 18f83968f3f2b7480d7557e18946f51706dda6b6
|
||||
Subproject commit 87b1fa735c5d663052995177c6d2f639eeac713b
|
128
src/Pudding.cpp
128
src/Pudding.cpp
|
@ -146,6 +146,12 @@ void Pudding::set_pudding_model(
|
|||
void Pudding::load_gl_context()
|
||||
{
|
||||
super::load_gl_context();
|
||||
/* create another GL context for loading camera frame textures */
|
||||
SDL_GL_SetAttribute(SDL_GL_SHARE_WITH_CURRENT_CONTEXT, 1);
|
||||
if ((capture_frame_thread_context = SDL_GL_CreateContext(window)) == nullptr)
|
||||
{
|
||||
log("could not create capture frame thread context");
|
||||
}
|
||||
/* load background as surface, generate texture to load pixel data into, allocate storage, bind and edit texture properties */
|
||||
std::unique_ptr<SDL_Surface, decltype(&SDL_FreeSurface)> surface(IMG_Load("local/tptile.jpg"), SDL_FreeSurface);
|
||||
std::unique_ptr<SDL_Surface, decltype(&SDL_FreeSurface)> flipped_surface(rotozoomSurfaceXY(surface.get(), 0, 1, -1, 0), SDL_FreeSurface);
|
||||
|
@ -226,8 +232,9 @@ void Pudding::load_gl_context()
|
|||
log_gl_errors();
|
||||
}
|
||||
|
||||
/* Try to create cv::VideoCapture object using device ID #0. If successful, this will create a GL texture for displaying
|
||||
* the frames, so it must be called after GL context has been created.
|
||||
/* Try to create cv::VideoCapture object using device ID #0. If successful, this will create GL texture IDs and storage
|
||||
* for the camera frames, so it must be called after GL context has been created. Two textures will be created, so they
|
||||
* can be used as a double buffer.
|
||||
*/
|
||||
void Pudding::initialize_camera()
|
||||
{
|
||||
|
@ -240,13 +247,19 @@ void Pudding::initialize_camera()
|
|||
message << "opened and initialized " << capture.get(cv::CAP_PROP_FRAME_WIDTH) << "x" <<
|
||||
capture.get(cv::CAP_PROP_FRAME_HEIGHT) << ", " << capture.get(cv::CAP_PROP_FPS) <<
|
||||
"fps video capture device ID #" << device_id << " using " << capture.getBackendName();
|
||||
/* generate the texture that will store the video frame, allocate storage for a video frame, bind and edit texture properties */
|
||||
glGenTextures(1, &video_capture_texture_id);
|
||||
glBindTexture(GL_TEXTURE_2D, video_capture_texture_id);
|
||||
/* generate two textures that will store the video frames with the intention of double buffering them
|
||||
* for threaded texture loading */
|
||||
for (GLuint* buffer_id : {&capture_texture_front_buffer_id, &capture_texture_back_buffer_id})
|
||||
{
|
||||
glGenTextures(1, buffer_id);
|
||||
glBindTexture(GL_TEXTURE_2D, *buffer_id);
|
||||
glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGB8, capture.get(cv::CAP_PROP_FRAME_WIDTH), capture.get(cv::CAP_PROP_FRAME_HEIGHT));
|
||||
// glTexStorage2D(GL_TEXTURE_2D, 1, GL_RGB8, 320, 240);
|
||||
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
|
||||
glTexParameteri(GL_TEXTURE_2D, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
|
||||
}
|
||||
capture_texture_id = capture_texture_front_buffer_id;
|
||||
}
|
||||
else
|
||||
{
|
||||
message << "failed to open video capture device ID #" << device_id;
|
||||
|
@ -635,9 +648,80 @@ bool Pudding::item_display_active() const
|
|||
return show_item && items.size() > 0;
|
||||
}
|
||||
|
||||
/* Read pixels from the camera into a GL texture. This function is meant to be launched in a separate thread,
|
||||
* so it will use its own GL context to load the pixels into either the front or back texture buffer, depending
|
||||
* on which is not currently in use */
|
||||
int Pudding::capture_frame(void* game)
|
||||
{
|
||||
Pudding* pudding = reinterpret_cast<Pudding*>(game);
|
||||
/* Make the thread context the current context (not sure what that means, but it doesn't seem to conflict
|
||||
* with the main rendering context) */
|
||||
if (SDL_GL_MakeCurrent(pudding->window, pudding->capture_frame_thread_context) < 0)
|
||||
{
|
||||
pudding->log("error making thread context current");
|
||||
}
|
||||
else
|
||||
{
|
||||
if (pudding->capture.isOpened())
|
||||
{
|
||||
cv::Mat frame;
|
||||
pudding->capture.read(frame);
|
||||
if (!frame.empty())
|
||||
{
|
||||
/* rotate the opencv matrix 180 to work with opengl coords */
|
||||
cv::flip(frame, frame, -1);
|
||||
/* use whichever texture ID is not being used by the main rendering thread */
|
||||
GLuint texture_id = pudding->capture_texture_id == pudding->capture_texture_front_buffer_id ?
|
||||
pudding->capture_texture_back_buffer_id : pudding->capture_texture_front_buffer_id;
|
||||
/* bind texture for accepting pixel data */
|
||||
glBindTexture(GL_TEXTURE_2D, texture_id);
|
||||
/* fill texture memory with last frame's pixels */
|
||||
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, frame.cols, frame.rows, GL_BGR, GL_UNSIGNED_BYTE, frame.ptr());
|
||||
pudding->capture_texture_id = texture_id;
|
||||
if (pudding->get_configuration()["scan"]["enabled"])
|
||||
{
|
||||
/* convert to gray and scan with zbar */
|
||||
cv::cvtColor(frame, frame, cv::COLOR_BGR2GRAY);
|
||||
zbar::Image query_image(frame.cols, frame.rows, "Y800", static_cast<void*>(frame.data), frame.cols * frame.rows);
|
||||
int result = pudding->image_scanner.scan(query_image);
|
||||
if (result > 0)
|
||||
{
|
||||
for (zbar::Image::SymbolIterator symbol = query_image.symbol_begin(); symbol != query_image.symbol_end(); ++symbol)
|
||||
{
|
||||
std::stringstream message;
|
||||
message << "camera scanned " << symbol->get_type_name() << " symbol " << symbol->get_data();
|
||||
pudding->log(message.str());
|
||||
pudding->current_camera_barcode = symbol->get_data();
|
||||
pudding->current_barcode = pudding->current_camera_barcode;
|
||||
}
|
||||
}
|
||||
query_image.set_data(nullptr, 0);
|
||||
}
|
||||
}
|
||||
frame.release();
|
||||
}
|
||||
SDL_GL_MakeCurrent(pudding->window, nullptr);
|
||||
}
|
||||
pudding->reading_capture_frame = false;
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Update parameters and draw the screen */
|
||||
void Pudding::update()
|
||||
{
|
||||
/* launch the camera capture thread if it is not currently running */
|
||||
if (capture.isOpened() && !reading_capture_frame)
|
||||
{
|
||||
SDL_Thread* capture_thread = SDL_CreateThread(Pudding::capture_frame, "capture frame", reinterpret_cast<void*>(this));
|
||||
if (capture_thread == nullptr)
|
||||
{
|
||||
log("could not create capture thread");
|
||||
}
|
||||
else
|
||||
{
|
||||
reading_capture_frame = true;
|
||||
}
|
||||
}
|
||||
/* if the config is set to refresh automatically, there may be a new barcode available */
|
||||
if (current_config_barcode != get_configuration()["scan"]["barcode"])
|
||||
{
|
||||
|
@ -706,6 +790,8 @@ void Pudding::update()
|
|||
/* draw pudding model */
|
||||
glEnable(GL_DEPTH_TEST);
|
||||
glDrawArrays(GL_TRIANGLES, 0, pudding_vertices.size());
|
||||
/* regular fill mode enabled for all other drawing */
|
||||
glPolygonMode(GL_FRONT_AND_BACK, GL_FILL);
|
||||
/* only do more drawing if items are downloaded or camera is enabled */
|
||||
if (item_display_active() || capture.isOpened())
|
||||
{
|
||||
|
@ -743,40 +829,12 @@ void Pudding::update()
|
|||
/* draw the camera if the camera has been opened */
|
||||
if (capture.isOpened())
|
||||
{
|
||||
capture.read(capture_frame);
|
||||
viewport_box.left(window_box(true).left());
|
||||
if (!capture_frame.empty())
|
||||
{
|
||||
/* rotate the opencv matrix 180 to work with opengl coords */
|
||||
cv::flip(capture_frame, capture_frame, -1);
|
||||
glViewport(viewport_box.left(), viewport_box.bottom(), viewport_box.width(), viewport_box.height());
|
||||
/* bind texture, binding it to accept pixel data and to GLSL sampler */
|
||||
glBindTexture(GL_TEXTURE_2D, video_capture_texture_id);
|
||||
glTexSubImage2D(GL_TEXTURE_2D, 0, 0, 0, capture_frame.cols, capture_frame.rows, GL_BGR, GL_UNSIGNED_BYTE, capture_frame.ptr());
|
||||
/* bind texture for drawing */
|
||||
glBindTexture(GL_TEXTURE_2D, capture_texture_id);
|
||||
/* draws rectangle vertices and rectangle texture using UV coords */
|
||||
glDrawArrays(GL_TRIANGLES, 0, 6);
|
||||
/* convert to gray and scan with zbar */
|
||||
cv::cvtColor(capture_frame, capture_frame, cv::COLOR_BGR2GRAY);
|
||||
zbar::Image query_image(capture_frame.cols, capture_frame.rows, "Y800", static_cast<void*>(capture_frame.data),
|
||||
capture_frame.cols * capture_frame.rows);
|
||||
int result = image_scanner.scan(query_image);
|
||||
if (result > 0)
|
||||
{
|
||||
for (zbar::Image::SymbolIterator symbol = query_image.symbol_begin(); symbol != query_image.symbol_end(); ++symbol)
|
||||
{
|
||||
std::stringstream message;
|
||||
message << "camera scanned " << symbol->get_type_name() << " symbol " << symbol->get_data();
|
||||
log(message.str());
|
||||
current_camera_barcode = symbol->get_data();
|
||||
current_barcode = current_camera_barcode;
|
||||
}
|
||||
}
|
||||
query_image.set_data(nullptr, 0);
|
||||
}
|
||||
else
|
||||
{
|
||||
debug("video capture device frame empty");
|
||||
}
|
||||
}
|
||||
}
|
||||
SDL_GL_SwapWindow(get_window());
|
||||
|
|
|
@ -5,6 +5,7 @@
|
|||
#include <string>
|
||||
#include <iostream>
|
||||
#include <algorithm>
|
||||
#include <thread>
|
||||
#include <memory>
|
||||
#include <type_traits>
|
||||
#include <filesystem>
|
||||
|
@ -21,6 +22,7 @@
|
|||
#include "Color.hpp"
|
||||
#include "extension.hpp"
|
||||
#include "Item.hpp"
|
||||
#include "Animation.hpp"
|
||||
|
||||
class Pudding : public Game
|
||||
{
|
||||
|
@ -54,15 +56,15 @@ private:
|
|||
std::vector<Item> items;
|
||||
int current_item_index = 0;
|
||||
cv::VideoCapture capture;
|
||||
cv::Mat capture_frame;
|
||||
zbar::ImageScanner image_scanner;
|
||||
GLuint flat_program, mvp_program, video_capture_texture_id, mvp_uniform_location, background_texture_id, time_uniform_location,
|
||||
effect_uniform_location;
|
||||
GLuint flat_program, mvp_program, capture_texture_front_buffer_id, capture_texture_back_buffer_id, capture_texture_id,
|
||||
mvp_uniform_location, background_texture_id, time_uniform_location, effect_uniform_location;
|
||||
glm::mat4 projection, model = glm::mat4(1.0f), mvp;
|
||||
std::vector<glm::vec3> pudding_vertices, pudding_colors;
|
||||
std::vector<glm::vec2> pudding_uv;
|
||||
bool show_item;
|
||||
bool show_item = false, reading_capture_frame = false;
|
||||
int effect_id = Effect::NONE;
|
||||
SDL_GLContext capture_frame_thread_context = nullptr;
|
||||
|
||||
void set_pudding_model(float, float, int, int = 1, float = -1, float = 1, float = 0.3f);
|
||||
void load_gl_context();
|
||||
|
@ -78,6 +80,7 @@ private:
|
|||
std::shared_ptr<GLuint> texture_from_image_url(const std::string&);
|
||||
static void destroy_texture(GLuint*);
|
||||
bool item_display_active() const;
|
||||
static int capture_frame(void*);
|
||||
|
||||
public:
|
||||
|
||||
|
|
Loading…
Reference in New Issue