wip of emscripten build with working webcam

This commit is contained in:
ohsqueezy 2022-09-07 20:55:06 -04:00
parent d47289007c
commit 009e374cd8
15 changed files with 526 additions and 232 deletions

3
.gitignore vendored
View File

@ -3,3 +3,6 @@ local/
BPmono.ttf
compile_commands.json
gunkiss
*.data
*.wasm
gunkiss.js

View File

@ -83,15 +83,15 @@ $(SRC_DIR)Model.o : $(addprefix $(SB_SRC_DIR),extension.hpp Attributes.hpp Textu
$(SRC_DIR)Item.o : $(addprefix $(SB_SRC_DIR),Texture.hpp Log.hpp utility.hpp) $(addprefix $(SRC_DIR),Model.hpp Carousel.hpp)
$(SRC_DIR)Pudding.o : $(SRC_H_FILES) $(SB_H_FILES)
%.o : %.cpp %.hpp
$(CXX) $(CPP_FLAGS) $< -c -o $@
$(CXX) $(CXXFLAGS) $< -c -o $@
###############
# Linux build #
###############
linux : CFLAGS = -g -Wall -Wextra -O0 -c -I$(SB_LIB_DIR) -I$(SB_SRC_DIR) $(SDL_CFLAGS) -I$(HOME)/local/zbar/include \
-I$(HOME)/local/opencv/include/opencv4
linux : CPP_FLAGS = $(CFLAGS) --std=c++17
-I $(HOME)/local/opencv/include/opencv4 -I $(HOME)/ext/software/emsdk/upstream/emscripten/system/include
linux : CXXFLAGS = $(CFLAGS) --std=c++17
linux : LFLAGS = $(SDL_LFLAGS) -Wl,--enable-new-dtags -lpthread -lGL -lGLESv2 -lSDL2_image -lSDL2_ttf -lSDL2_mixer -lstdc++fs -lcurl \
-L$(HOME)/local/opencv/lib -Wl,-rpath,$(HOME)/local/opencv/lib -lopencv_videoio -lopencv_core -lopencv_highgui -lopencv_imgproc \
-L$(HOME)/local/zbar/lib -Wl,-rpath,$(HOME)/local/zbar/lib -lzbar
@ -107,29 +107,38 @@ linux : $(GLEW_DIR)glew.o $(addprefix $(SDLGFX2_DIR),SDL2_rotozoom.o SDL2_gfxPri
# Use Emscripten to output JavaScript and an HTML index page for running in the browser
EMSCRIPTENHOME = $(HOME)/ext/software/emsdk/upstream/emscripten
EMSCRIPTEN_CFLAGS = -O3 -Wall -s USE_SDL=2 -s USE_SDL_IMAGE=2 -s SDL2_IMAGE_FORMATS="['png', 'jpg']" \
-s USE_SDL_TTF=2 -s USE_SDL_MIXER=2 -s MAX_WEBGL_VERSION=1 -s EXPORTED_FUNCTIONS="['_main']" \
-s ALLOW_MEMORY_GROWTH=1 -s GL_PREINITIALIZED_CONTEXT=1 -s ENVIRONMENT=web --shell-file shell_minimal.html \
--no-heap-copy -I$(SFW_LIB_DIR) -I$(SFW_SRC_DIR)
EMSCRIPTEN_PRELOADS = --preload-file "BPmono.ttf"@/ --preload-file "config.json" --preload-file "resource"
EMSCRIPTEN_CFLAGS = -O1 -Wall -s USE_SDL=2 -s USE_SDL_IMAGE=2 -s SDL2_IMAGE_FORMATS="['png', 'jpg']" -s USE_SDL_TTF=2 -s USE_SDL_MIXER=2 \
--no-heap-copy -I $(SB_LIB_DIR) -I $(SB_SRC_DIR) -I $(HOME)/local/zbar/include \
-I $(HOME)/ext/software/opencv-4.6.0/modules/videoio/include/ \
-I $(HOME)/ext/software/opencv-4.6.0/modules/core/include/ \
-I $(HOME)/ext/software/opencv-4.6.0/modules/highgui/include/ \
-I $(HOME)/ext/software/opencv-4.6.0/modules/imgproc/include/ \
-I $(HOME)/ext/software/opencv-4.6.0/modules/imgcodecs/include/ \
-I $(HOME)/ext/software/opencv-4.6.0/build_wasm/
EMSCRIPTEN_LFLAGS = -s MIN_WEBGL_VERSION=2 -s EXPORTED_FUNCTIONS="['_main']" -s ALLOW_MEMORY_GROWTH=1 -s FULL_ES3=1 \
-sLLD_REPORT_UNDEFINED -s FETCH --bind $(wildcard $(addprefix $(HOME)/ext/software/opencv-4.6.0/build_wasm/lib/,*.a)) \
$(HOME)/ext/software/ZBar/zbar/.libs/libzbar.a
EMSCRIPTEN_PRELOADS = --preload-file "BPmono.ttf"@/ --preload-file "config.json"@/ --preload-file "resource/"@/"resource/" \
--preload-file "src/shaders/"@/"src/shaders/"
emscripten : CC = $(EMSCRIPTENHOME)/emcc
emscripten : CXX = $(EMSCRIPTENHOME)/em++
emscripten : CFLAGS = $(EMSCRIPTEN_CFLAGS)
emscripten : CPP_FLAGS = $(CFLAGS) --std=c++17
emscripten : CXXFLAGS = $(CFLAGS) --std=c++17
emscripten : $(addprefix $(SDLGFX2_DIR),SDL2_rotozoom.o SDL2_gfxPrimitives.o) $(SB_O_FILES) $(SRC_O_FILES)
$(CREATE_FONT_SYMLINK)
$(CXX) $(CPP_FLAGS) $(EMSCRIPTEN_PRELOADS) $^ -o "index.html"
$(CXX) $^ $(CXXFLAGS) $(EMSCRIPTEN_LFLAGS) $(EMSCRIPTEN_PRELOADS) -o gunkiss.js
#########################
# Clean up object files #
#########################
clean :
-rm $(SRC_DIR)*.o
-find $(SRC_DIR) -iname "*.o" -delete
clean-all :
-find . -iname "*.o" -exec rm {} \;
clean-all : clean
-find $(SB_SRC_DIR) -iname "*.o" -delete
-find $(SB_LIB_DIR) -iname "*.o" -delete
#############
# compiledb #

View File

@ -3,9 +3,9 @@
{
"dimensions": [460, 768],
"framerate": 60,
"title": "Pudding",
"title": "Gunkiss",
"debug": false,
"render driver": "opengl",
"render driver": "opengles2",
"show-cursor": true,
"camera-resolution": [1280, 720]
},
@ -29,7 +29,7 @@
{
"screenshot-directory": "local/screenshots",
"video-directory": "local/video",
"enabled": true,
"enabled": false,
"write-mp4": true,
"video-frame-length": 33.333,
"max-video-memory": 2000,
@ -57,13 +57,13 @@
"enabled": true,
"json-save": true,
"json-save-directory": "local/scans",
"barcode": "0140231056",
"barcode": "",
"capture-device": "/dev/video0"
},
"api":
{
"user-agent": "Custom pudding creation game for http://nugget.fun",
"user-agent": "Custom pudding creation game under development for https://shampoo.ooo",
"nutronix-app-id": "ea0f2e7e",
"nutronix-app-key": "39218dde526dd3349daa028deda518ae",
"edamam-app-id": "c23b139f",

108
index.html Normal file
View File

@ -0,0 +1,108 @@
<!doctype html>
<html>
<body>
<!-- WebGL output will be drawn here through Emscripten -->
<canvas id="canvas"></canvas>
<!-- navigator.mediaDevices.getUserMedia streams the webcam video directly, displayed for testing -->
<!-- <video id="webcam"></video> -->
<script>
const FPS = 15;
const BPP = 4;
// Direct output of webcam (hidden)
var video = document.createElement("video");
video.width = 320;
video.height = 240;
// Undisplayed canvas which is used to draw the video frame and access the pixel data directly
var intermediate = document.createElement("canvas");
intermediate.width = video.width;
intermediate.height = video.height;
var context = intermediate.getContext("2d");
// Indicates whether webcam is opened or not
var streaming = false;
// Address of the webcam frame pixel data on the Emscripten heap
var image_heap_address;
var Module = {
onRuntimeInitialized: function()
{
process_video();
},
// Tell Emscripten to use this canvas for display
canvas: document.getElementById("canvas")
};
function open_camera()
{
// Open the webcam and start displaying frames if successfully opened. Allocate space for 32-bit RGBA frame pixel data
// on the Emscripten heap.
navigator.mediaDevices.getUserMedia({video: {width: video.width, height: video.height}, audio: false})
.then(function(stream) {
video.srcObject = stream;
video.play();
streaming = true;
// Get the memory address of the pixel data
image_heap_address = Module._malloc(video.width * video.height * BPP);
// Pass the address to the C++ program
Module.set_heap_offset(image_heap_address);
})
.catch(function(err) {
console.log('Camera Error: ' + err.name + ' ' + err.message);
});
}
function close_camera()
{
video.pause();
video.srcObject = null;
streaming = false;
}
// This function will run continuously, drawing the webcam frame to the intermediate canvas, reading the pixel data,
// storing the data on the heap, and setting the new frame available flag.
function process_video()
{
try
{
if (streaming)
{
// Draw the webcam frame on a hidden canvas
context.drawImage(video, 0, 0, video.width, video.height);
// Read the pixel data
image_data = context.getImageData(0, 0, video.width, video.height).data;
// Get a memory view object that provides access to the heap at the previously allocated address
image_heap_data = new Uint8Array(Module.HEAPU8.buffer, image_heap_address, video.width * video.height * BPP);
// Write the pixel data to the heap
image_heap_data.set(image_data);
// Flag the C++ that new data is available
Module.flag_frame();
}
// Loop at roughly the FPS
let begin = Date.now();
let delay = 1000/FPS - (Date.now() - begin);
setTimeout(process_video, delay);
}
catch (err)
{
console.log(err);
}
}
</script>
<!-- This file is built by Emscripten when compiling the program -->
<script src="gunkiss.js"></script>
</body>
</html>

2
lib/sb

@ -1 +1 @@
Subproject commit 0bf2e1293542da180a325455610a72df5697853d
Subproject commit b1fb77b1c8a2902fde711ede1a45b459013dc876

View File

@ -1,11 +1,11 @@
/* _______________ ,----------------------------------------------------------------.
//`````````````\\ \ \
//~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \
//=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \
// \\ \ \
// \\ \ code released under the zlib license [git.nugget.fun/pudding] \
// GUNKISS \\ \ \
//_________________________\\ `---------------------------------------------------------------*/
/* _______________ ,-------------------------------------------------.
//`````````````\\ \ \
//~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \
//=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \
// \\ \ \
// \\ \ zlib licensed code at [git.nugget.fun/pudding] \
// GUNKISS \\ \ \
//_________________________\\ `-------------------------------------------------*/
#include "Item.hpp"

View File

@ -24,6 +24,18 @@ int main()
return 0;
}
#ifdef __EMSCRIPTEN__
void flag_frame()
{
new_frame_available = true;
}
void set_heap_offset(int offset)
{
emscripten_heap_offset = offset;
}
#endif
/* Initialize a Pudding instance */
Pudding::Pudding()
{
@ -35,9 +47,6 @@ Pudding::Pudding()
/* initialize a zbar image scanner for reading barcodes of any format */
image_scanner.set_config(zbar::ZBAR_NONE, zbar::ZBAR_CFG_ENABLE, 1);
/* Add a texture to the camera Plane for storing frame image data */
camera_view.texture(sb::Texture());
/* set up pudding model */
nlohmann::json pudding = configuration()["pudding"];
load_pudding_model(pudding["top-radius"], pudding["base-radius"], pudding["ring-vertex-count"], pudding["layer-count"],
@ -46,6 +55,12 @@ Pudding::Pudding()
/* loading GL context instead of SDL context for 3D */
load_gl_context();
/* Add a texture to the camera Plane for storing frame image data */
camera_view.texture(sb::Texture());
glm::mat4 flip = glm::mat4(1);
flip[1][1] = -1;
camera_view.transformation(flip);
/* Load background tiles */
load_tiles();
@ -175,16 +190,15 @@ void Pudding::load_gl_context()
vao.generate();
vao.bind();
/* Generate ID for the vertex buffer object that will hold all vertex data. Since we're using one buffer, data
* will be copied in one after the other, offset to after the previous data location. The same buffer offset will
* be passed to the vertex attributes for each data. */
/* Generate ID for the vertex buffer object that will hold all vertex data. Using one buffer for all attributes, data
* will be copied in one after the other. */
vbo.generate();
vbo.bind();
/* Load two shader programs, one for rendering the flat objects, and one for rendering the 3D model. Load and configure
* the flat shader program first. */
GLuint vertex_shader = load_shader("src/flat.vert", GL_VERTEX_SHADER);
GLuint fragment_shader = load_shader("src/flat.frag", GL_FRAGMENT_SHADER);
GLuint vertex_shader = load_shader("src/shaders/flat.vert", GL_VERTEX_SHADER);
GLuint fragment_shader = load_shader("src/shaders/flat.frag", GL_FRAGMENT_SHADER);
flat_program = glCreateProgram();
glAttachShader(flat_program, vertex_shader);
glAttachShader(flat_program, fragment_shader);
@ -192,8 +206,8 @@ void Pudding::load_gl_context()
Plane::uv->bind(1, flat_program, "vertex_uv");
/* load, configure and link the 3D world program */
vertex_shader = load_shader("src/mvp.vert", GL_VERTEX_SHADER);
fragment_shader = load_shader("src/mvp.frag", GL_FRAGMENT_SHADER);
vertex_shader = load_shader("src/shaders/mvp.vert", GL_VERTEX_SHADER);
fragment_shader = load_shader("src/shaders/mvp.frag", GL_FRAGMENT_SHADER);
mvp_program = glCreateProgram();
glAttachShader(mvp_program, vertex_shader);
glAttachShader(mvp_program, fragment_shader);
@ -275,8 +289,9 @@ void Pudding::load_pads()
* storage for the camera frame on the GPU, so it must be called after GL context has been created. Create and detach
* a thread which will continuously read frame data.
*/
void Pudding::initialize_camera()
void Pudding::open_camera()
{
#ifndef __EMSCRIPTEN__
/* Open the OpenCV capture, using device ID #0 to get the default attached camera. */
int device_id = 0;
capture.open(device_id);
@ -314,6 +329,18 @@ void Pudding::initialize_camera()
message << "failed to open video capture device ID #" << device_id;
}
sb::Log::log(message);
#else
emscripten_run_script("open_camera()");
#endif
}
void Pudding::close_camera()
{
#ifndef __EMSCRIPTEN__
capture.release();
#else
emscripten_run_script("close_camera()");
#endif
}
/* Respond to command events */
@ -382,9 +409,9 @@ void Pudding::respond(SDL_Event& event)
glm::vec2 mouse_viewport_ndc {
mouse_ndc.x, (1.0f - (float(mouse_pixel.y) - float(viewport_pixel.top())) / viewport_pixel.height()) * 2.0f - 1.0f
};
bool over_camera_button = !capture.isOpened() && !item_display_active() && camera_button.collide(mouse_ndc),
over_inventory_button = items.size() > 0 && !item_display_active() && !capture.isOpened() && inventory_button.collide(mouse_ndc),
over_close_area = (capture.isOpened() || item_display_active()) && get_display().ndc_subsection(main_viewport).collide(mouse_ndc),
bool over_camera_button = !camera_switch && !item_display_active() && camera_button.collide(mouse_ndc),
over_inventory_button = items.size() > 0 && !item_display_active() && !camera_switch && inventory_button.collide(mouse_ndc),
over_close_area = (camera_switch || item_display_active()) && get_display().ndc_subsection(main_viewport).collide(mouse_ndc),
over_previous_button = item_display_active() && previous_button.collide(mouse_viewport_ndc),
over_next_button = item_display_active() && next_button.collide(mouse_viewport_ndc);
/* Check for collisions with anything clickable */
@ -721,7 +748,7 @@ void Pudding::save_item_json(const nlohmann::json& json, const Item& item, const
nlohmann::json Pudding::json_from_url(const std::string& url, const std::vector<std::string>& headers)
{
std::vector<std::uint8_t> storage;
curl_get_bytes(url, storage, headers);
web_get_bytes(url, storage, headers);
nlohmann::json json = nlohmann::json::parse(storage);
std::stringstream json_formatted;
json_formatted << std::setw(4) << json << std::endl;
@ -729,10 +756,35 @@ nlohmann::json Pudding::json_from_url(const std::string& url, const std::vector<
return json;
}
/* Store the byte buffer from the submitted URL downloaded by cURL into the supplied storage vector
/*!
* Store the bytes retrieved from `url` in the byte vector `storage`.
*
* The compiler will determine whether to use cURL or the Emscripten Fetch API to do the retrieval, depending on whether it is compiling for
* Emscripten.
*
* The optional `headers` parameter will be added to the request when using cURL, but not when using the Emscripten Fetch API.
*
* @param url URL containing data to be retrieved
* @param storage A reference to a vector of bytes which will be filled with the data retrieved from the URL
* @param headers A reference to a vector of strings that should be passed as headers with the request. It is only supported by the cURL version.
*/
void Pudding::curl_get_bytes(const std::string& url, std::vector<std::uint8_t>& storage, const std::vector<std::string>& headers) const
void Pudding::web_get_bytes(const std::string& url, std::vector<std::uint8_t>& storage, const std::vector<std::string>& headers) const
{
#if defined(__EMSCRIPTEN__)
/* Create a fetch attributes object. Set a callback that will be called when response data is received. Pass along the user
* storage location to be filled by the callback. */
emscripten_fetch_attr_t attr;
emscripten_fetch_attr_init(&attr);
strcpy(attr.requestMethod, "GET");
attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY;
attr.onsuccess = fetch_success;
attr.onerror = fetch_error;
attr.userData = &storage;
emscripten_fetch(&attr, url.c_str());
#else
CURL *curl;
CURLcode result;
result = curl_global_init(CURL_GLOBAL_DEFAULT);
@ -747,7 +799,6 @@ void Pudding::curl_get_bytes(const std::string& url, std::vector<std::uint8_t>&
{
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, Pudding::curl_write_response);
std::vector<std::uint8_t> food_barcode_response;
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &storage);
curl_easy_setopt(curl, CURLOPT_USERAGENT, configuration()["api"]["user-agent"].get<std::string>().c_str());
struct curl_slist* list = nullptr;
@ -773,10 +824,47 @@ void Pudding::curl_get_bytes(const std::string& url, std::vector<std::uint8_t>&
curl_easy_cleanup(curl);
}
curl_global_cleanup();
#endif
}
/* This callback will be called by cURL when it has a response char buffer. The chars will be inserted into the storage
* vector pointed to by the storage parameter.
#if defined(__EMSCRIPTEN__)
/*!
* This will be called automatically when request data is sucessfully fetched by `emscripten_fetch` from `Pudding::web_get_bytes`.
* Response bytes will be inserted into the user supplied `std::vector<std::uint8_t>&` at `fetch->userData`.
*/
void Pudding::fetch_success(emscripten_fetch_t* fetch)
{
std::vector<std::uint8_t>* storage = reinterpret_cast<std::vector<std::uint8_t>*>(fetch->userData);
storage->insert(storage->end(), fetch->data, fetch->data + fetch->numBytes);
std::stringstream message;
message << "Stored " << (fetch->numBytes / 100) << "KB of image data in memory from " << fetch->url;
sb::Log::log(message.str());
emscripten_fetch_close(fetch);
}
/*!
* This will be called automatically when request data is not successfully fetched by `emscripten_fetch` from `Pudding::web_get_bytes`.
*/
void Pudding::fetch_error(emscripten_fetch_t* fetch)
{
std::stringstream message;
message << "Downloading image from " << fetch->url << " failed with status code " << fetch->status;
sb::Log::log(message.str());
emscripten_fetch_close(fetch);
}
#else
/*!
* This will be called by cURL when it has received a buffer of data. The data will be inserted into the vector at `storage`
*
* @param buffer pointer to data
* @param size size in bytes of each value
* @param count number of values
* @param storage pointer to a vector of unsigned 8-bit values where the data will be copied to
* @return number of bytes copied
*/
size_t Pudding::curl_write_response(std::uint8_t* buffer, size_t size, size_t count, std::vector<std::uint8_t>* storage)
{
@ -785,6 +873,8 @@ size_t Pudding::curl_write_response(std::uint8_t* buffer, size_t size, size_t co
return total_size;
}
#endif
/* Allocate storage for a texture, copy the cURL response data into the storage, and return the ID that corresponds to the GL texture
*/
sb::Texture Pudding::texture_from_image_url(const std::string& url) const
@ -793,7 +883,7 @@ sb::Texture Pudding::texture_from_image_url(const std::string& url) const
sb::Texture texture;
sb::Log::log("looking up image at " + url);
std::vector<std::uint8_t> storage;
curl_get_bytes(url, storage);
web_get_bytes(url, storage);
if (!storage.empty())
{
sb::Log::log("received image data", sb::Log::DEBUG);
@ -844,15 +934,23 @@ bool Pudding::item_display_active() const
}
/*!
* Read pixels from the camera into a `cv::Mat`. This function is meant to be launched in a separate thread,
* where it will run continuously, setting `new_frame_available` to `false` before loading camera frame data
* into the `cv::Mat` object at `camera_frame`, then setting `finished_loading_frame` to `true` to indicate
* new frame data is available in `camera_frame`.
* Read pixels from the camera into a `cv::Mat`.
*
* For a Linux build: This function is meant to be launched in a separate thread, where it will run continuously. Set `new_frame_available`
* to `false` before loading camera frame data into the `cv::Mat` object at `camera_frame`, then set it back to `true` to indicate new frame
* data is available in `camera_frame`.
*
* For an Emscripten build: This will load pixel data off the Emscripten heap into a `cv::Mat`. It is intended to be called synchronously in the
* main thread.
*/
void Pudding::capture_frame()
{
/* When capture is closed, this thread will automatically finish execution. */
while (capture.isOpened())
/* Emscripten builds will call this function from the main thread, so don't run continuously */
#ifndef __EMSCRIPTEN__
/* When the camera button is switched off, this thread will automatically finish execution. */
while (camera_switch)
{
/* The frame data in the `cv::Mat` at `pudding->camera_frame` is about to be modified by the rest of
* this function, so even if there is data stored there that hasn't been read yet, it should not
@ -864,24 +962,33 @@ void Pudding::capture_frame()
capture >> camera_frame;
});
#else
/* Convert the address of frame RGBA pixel data on the Emscripten heap into an unsigned 8-bit pointer and read the data
* into a cv::Mat. */
std::uint8_t* emscripten_camera_pixels = reinterpret_cast<std::uint8_t*>(emscripten_heap_offset);
camera_frame = cv::Mat(320, 240, CV_8UC4, emscripten_camera_pixels);
#endif
if (!camera_frame.empty())
{
/* Rotate the frame 180 degrees to work with OpenGL coords */
time_it("flip")([&]{
cv::flip(camera_frame, camera_frame, -1);
});
/* Finished loading into `cv::Mat`, so it is new data that is safe to read. */
new_frame_available = true;
}
sb::Log::gl_errors("in capture thread, after capturing frame");
sb::Log::gl_errors("in capture, after capturing frame");
#ifndef __EMSCRIPTEN__
std::this_thread::sleep_for(std::chrono::milliseconds(50));
}
#endif
}
/* Update parameters and draw the screen */
void Pudding::update()
{
sb::Log::gl_errors("at beginning of update");
/* Time in seconds the game has running for */
float time_seconds = SDL_GetTicks() / 1000.0f;
@ -895,13 +1002,21 @@ void Pudding::update()
sb::Log::log(message);
}
/* If a new frame is finished loading in the detached thread, copy the frame data into a texture, process the frame
* for scanning and scan it. */
/* If new frame data is available, copy it from a cv::Mat into a texture, process for scanning and scan it. */
if (new_frame_available)
{
sb::Log::log("Hello, World!");
#ifdef __EMSCRIPTEN__
/* Emscripten builds load pixel data into cv::Mat synchronously */
capture_frame();
#endif
camera_view.texture().bind();
/* Fill camera view texture memory with last frame's pixels */
camera_view.texture().load(camera_frame.ptr(), {camera_frame.cols, camera_frame.rows}, GL_BGR, GL_UNSIGNED_BYTE);
// camera_view.texture().load(camera_frame.ptr(), {camera_frame.cols, camera_frame.rows}, GL_BGR, GL_UNSIGNED_BYTE);
// std::cout << camera_frame.size[0] << " " << camera_frame.size[1] << std::endl;
camera_view.texture().load(camera_frame.ptr(), {320, 240}, GL_RGBA, GL_UNSIGNED_BYTE);
/* Frame data has been loaded, so there is not a new frame available anymore. */
new_frame_available = false;
/* Convert to grayscale for ZBar */
@ -932,13 +1047,14 @@ void Pudding::update()
viewport = window_box(true);
/* shrink viewport if item texture or camera will be displayed */
if (item_display_active() || capture.isOpened())
if (item_display_active() || camera_switch)
{
viewport.drag_bottom(0.5f * configuration()["interface"]["pop-up-viewport-height"].get<float>() * viewport.height());
}
/* Save the main viewport dimensions */
main_viewport = viewport;
sb::Log::gl_errors("before viewport");
glViewport(viewport);
glDisable(GL_DEPTH_TEST);
glClearColor(0, 0, 0, 1);
@ -976,12 +1092,12 @@ void Pudding::update()
pudding_model.attributes("position")->enable();
if (items.size() == 0)
{
glPolygonMode(GL_FRONT_AND_BACK, GL_LINE);
// glPolygonMode(GL_FRONT_AND_BACK, GL_LINE);
// pudding_model.attributes("color")->enable();
}
else
{
glPolygonMode(GL_FRONT_AND_BACK, GL_FILL);
// glPolygonMode(GL_FRONT_AND_BACK, GL_FILL);
// pudding_model.attributes("color")->enable();
pudding_model.attributes("uv")->enable();
glUniform1i(uniform["mvp"]["pudding texture"], 0);
@ -1002,10 +1118,10 @@ void Pudding::update()
/* disable squircling for all other drawing */
glUniform1i(uniform["mvp"]["uv transformation"], UV_NONE);
/* regular fill mode enabled for all other drawing */
glPolygonMode(GL_FRONT_AND_BACK, GL_FILL);
// glPolygonMode(GL_FRONT_AND_BACK, GL_FILL);
sb::Log::gl_errors("after pudding, before item or camera view");
/* only do more drawing if items are downloaded or camera is enabled */
if (item_display_active() || capture.isOpened())
if (item_display_active() || camera_switch)
{
/* switch to flat shader for item and camera */
glUseProgram(flat_program);
@ -1023,7 +1139,7 @@ void Pudding::update()
if (item_display_active())
{
/* shrink viewport to half size if camera will also be displayed */
if (capture.isOpened())
if (camera_switch)
{
viewport.left(viewport.cx(), true);
}
@ -1040,8 +1156,8 @@ void Pudding::update()
previous_button.draw(uniform["flat"]["transformation"]);
}
}
/* draw the camera if the camera has been opened */
if (capture.isOpened())
/* draw the camera view if the camera button has been switched on */
if (camera_switch)
{
viewport.left(window_box(true).left());
glViewport(viewport);
@ -1169,3 +1285,12 @@ void glViewport(Box box)
{
glViewport(box.left(), box.bottom(), box.width(), box.height());
}
#ifdef __EMSCRIPTEN__
/* This will bind the global functions to Emscripten so the camera pixel data can be transferred */
EMSCRIPTEN_BINDINGS(my_module)
{
function("flag_frame", &flag_frame);
function("set_heap_offset", &set_heap_offset);
}
#endif

View File

@ -13,6 +13,17 @@
/* Needed for functions in glm/gtx/ */
#define GLM_ENABLE_EXPERIMENTAL
/* cURL and cv::VideoCapture are not available for Emscripten, so use alternatives for Emscripten builds */
#if defined(__EMSCRIPTEN__)
#include <emscripten/fetch.h>
#include <emscripten/bind.h>
using namespace emscripten;
#else
#include <curl/curl.h>
#include "opencv2/videoio.hpp"
#include "opencv2/highgui.hpp"
#endif
#include <stdlib.h>
#include <string>
#include <iostream>
@ -23,7 +34,6 @@
#include <stdexcept>
#include <functional>
#include <chrono>
#include <curl/curl.h>
#include "SDL.h"
#include "SDL_image.h"
#include "sdl2-gfx/SDL2_gfxPrimitives.h"
@ -31,8 +41,6 @@
#include "glm/glm.hpp"
#include "glm/gtx/matrix_decompose.hpp"
#include "opencv2/core.hpp"
#include "opencv2/videoio.hpp"
#include "opencv2/highgui.hpp"
#include "opencv2/imgproc.hpp"
#include "zbar.h"
#include "Game.hpp"
@ -297,12 +305,22 @@ public:
};
/* These variables will be bound to JS. They are placed in the global scope, so they can be read and written by both
* C++ and JS. The associated functions are bound to JS so they can be used to write values to the variables. The
* first flag is used by both C++ and JS builds, so it is always included. */
bool new_frame_available = false;
#ifdef __EMSCRIPTEN__
unsigned int emscripten_heap_offset = 0;
void flag_frame();
void set_heap_offset(int offset);
#endif
class Pudding : public Game
{
private:
/* Defines for effect IDs that will be passed to the shader program. Since COUNT is last and every value
/* Defines for effect IDs that will be passed to the shader program. Since EFFECT_COUNT is last and every value
* is the default integer, it will be set to the number of effects available. */
enum Effect
{
@ -344,7 +362,9 @@ private:
std::vector<Item> items;
Carousel item_carousel;
int effect_id = EFFECT_NONE, pudding_triangle_vertex_count = 0, pudding_fan_vertex_count = 0;
#ifndef __EMSCRIPTEN__
cv::VideoCapture capture;
#endif
cv::Mat camera_frame;
zbar::ImageScanner image_scanner;
std::map<std::string, std::map<std::string, GLuint>> uniform;
@ -353,7 +373,7 @@ private:
Model pudding_model;
Plane plane, camera_view;
Background background;
bool show_item = false, new_frame_available = false;
bool show_item = false;
sb::VAO vao;
sb::VBO vbo;
std::map<std::string, sb::Texture> labels;
@ -365,25 +385,39 @@ private:
void load_gl_context();
void load_tiles();
void load_pads();
void initialize_camera();
void open_camera();
/*!
* Release camera resources.
*/
void close_camera();
void incorporate_open_api(Item&, const std::string&);
void incorporate_nutronix_api(Item&);
void incorporate_edamam_api(Item&);
void incorporate_best_buy_api(Item&);
void incorporate_google_books_api(Item&);
void save_item_json(const nlohmann::json&, const Item&, const std::string&) const;
nlohmann::json json_from_url(const std::string&, const std::vector<std::string>& = {});
void curl_get_bytes(const std::string& url, std::vector<std::uint8_t>&, const std::vector<std::string>& = {}) const;
static size_t curl_write_response(std::uint8_t*, size_t, size_t, std::vector<std::uint8_t>*);
nlohmann::json json_from_url(const std::string& url, const std::vector<std::string>& = {});
void web_get_bytes(const std::string& url, std::vector<std::uint8_t>& storage, const std::vector<std::string>& = {}) const;
sb::Texture texture_from_image_url(const std::string&) const;
static void destroy_texture(GLuint*);
bool item_display_active() const;
void capture_frame();
/* Initialize camera on connection and release on disconnection. */
/* Define the appropriate callbacks for URL data loaders. Either cURL by default, or Fetch if compiling for Emscripten. */
#if defined(__EMSCRIPTEN__)
static void fetch_success(emscripten_fetch_t* fetch);
static void fetch_error(emscripten_fetch_t* fetch);
#else
static size_t curl_write_response(std::uint8_t*, size_t, size_t, std::vector<std::uint8_t>*);
#endif
/* Open camera on connection and close on disconnection. */
Connection<> camera_switch {
std::bind(&Pudding::initialize_camera, this),
[&] { capture.release(); }
std::bind(&Pudding::open_camera, this),
std::bind(&Pudding::close_camera, this)
// [&] { capture.release(); }
};
public:

View File

@ -1,40 +0,0 @@
/* _______________ ,----------------------------------------------------------------.
//`````````````\\ \ \
//~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \
//=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \
// \\ \ \
// \\ \ code released under the zlib license [git.nugget.fun/pudding] \
// ☆ GUNKISS ☆ \\ \ \
//_________________________\\ `---------------------------------------------------------------*/
#version 130
in vec2 uv;
uniform sampler2D base_texture;
uniform vec3 blend_min_hsv;
uniform float time;
uniform bool scroll = false;
/* from http://lolengine.net/blog/2013/07/27/rgb-to-hsv-in-glsl, licensed under WTFPL */
vec3 hsv2rgb(vec3 c)
{
vec4 K = vec4(1.0, 2.0 / 3.0, 1.0 / 3.0, 3.0);
vec3 p = abs(fract(c.xxx + K.xyz) * 6.0 - K.www);
return c.z * mix(K.xxx, clamp(p - K.xxx, 0.0, 1.0), c.y);
}
void main(void)
{
if (scroll)
{
ivec2 texture_size = textureSize(base_texture, 0);
float speed = time * 35.0;
gl_FragColor = texelFetch(base_texture, ivec2(mod(vec2(gl_FragCoord.x + speed, gl_FragCoord.y - speed), texture_size)), 0);
}
else
{
gl_FragColor = texture(base_texture, uv);
}
/* apply blending, leaving alpha unchanged */
gl_FragColor.xyz = min(gl_FragColor.xyz, hsv2rgb(blend_min_hsv));
}

View File

@ -1,21 +0,0 @@
/* _______________ ,----------------------------------------------------------------.
//`````````````\\ \ \
//~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \
//=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \
// \\ \ \
// \\ \ code released under the zlib license [git.nugget.fun/pudding] \
// ☆ GUNKISS ☆ \\ \ \
//_________________________\\ `---------------------------------------------------------------*/
#version 130
in vec2 in_position;
in vec2 vertex_uv;
out vec2 uv;
uniform mat4 transformation;
void main(void)
{
gl_Position = transformation * vec4(in_position, 0, 1);
uv = vertex_uv;
}

View File

@ -1,78 +0,0 @@
/* _______________ +---------------------------------------------------------------------------------------+
//~~~~~~~~~~~~~\\ | a game by @ohsqueezy & @sleepin |
//```````````````\\ | [ohsqueezy.itch.io] [instagram.com/sleepin] |
//_0_0_0_0_0_0_0_0_\\ | |
//_/_/_/_/___\_\_\_\_\\ | with code licensed for copy, modification and redistribution [git.nugget.fun/pudding] |
//GGGUUUNNNKKKIIISSSSSS\\ | |
//_/__/__/__/_\__\__\__\_\\ | 😀 Thank you for choosing Puddendo for your business 😀 |
+---------------------------------------------------------------------------------------+ */
#version 130
#define TRANSFORMATION_NONE 0
#define TRANSFORMATION_SQUIRCLE 1
in vec2 fragment_uv;
in vec3 ex_color;
in float x_center_proximity;
in vec3 original_coordinates;
in vec3 clip_coordinates;
uniform sampler2D pudding_texture;
uniform int uv_transformation = TRANSFORMATION_NONE;
uniform float coordinate_bound;
/* [-coordinate_bound, coordinate_bound] arbitrary box coordinates to [-1, 1] normalized coordinates */
vec2 normalize_coordinates(vec2 coordinates)
{
return coordinates / coordinate_bound;
}
/* [-1, 1] box coordinates to [0, 1] UV coordinates */
vec2 coordinates_to_uv(vec2 coordinates)
{
return (coordinates + 1) / 2;
}
/* coordinates in circle with radius <= 1 to box coordinates in [-1, 1] */
vec2 circle_to_box(vec2 circle)
{
float u = circle.x;
float v = circle.y;
float u_sq = pow(u, 2);
float v_sq = pow(v, 2);
float rt_2 = sqrt(2);
float x = .5 * sqrt(2 + 2 * u * rt_2 + u_sq - v_sq) - .5 * sqrt(2 - 2 * u * rt_2 + u_sq - v_sq);
float y = .5 * sqrt(2 + 2 * v * rt_2 - u_sq + v_sq) - .5 * sqrt(2 - 2 * v * rt_2 - u_sq + v_sq);
return vec2(x, y);
}
/* Apply color passed in from the vertex shader, compressing to one of 16 colors. Add retro effect
* by alternately darkening and lightening 2x2 pixel areas in a checker pattern. Add shadowing by
* brightening the color based on how near it is to the center in the X-dimension */
void retro()
{
vec3 shadowed = min(ex_color, 1.0);
float dx = abs(floor(gl_FragCoord[0]) - 480) / 480.0;
if (int(floor(gl_FragCoord[0] / 2) + floor(gl_FragCoord[1]) / 2) % 2 == 0)
{
gl_FragColor = vec4(shadowed * 1.2, 1);
}
else
{
gl_FragColor = vec4(shadowed * 0.7, 1);
}
gl_FragColor[0] = int(gl_FragColor[0] * 4) / 4.0;
gl_FragColor[1] = int(gl_FragColor[1] * 4) / 4.0;
gl_FragColor[2] = int(gl_FragColor[2] * 4) / 4.0;
gl_FragColor *= x_center_proximity;
}
void main()
{
vec2 uv = fragment_uv;
if (uv_transformation == TRANSFORMATION_SQUIRCLE)
{
vec2 normalized_circle_coordinates = normalize_coordinates(vec2(original_coordinates.x, original_coordinates.z));
uv = coordinates_to_uv(circle_to_box(normalized_circle_coordinates));
}
gl_FragColor = texture(pudding_texture, uv);
}

44
src/shaders/flat.frag Normal file
View File

@ -0,0 +1,44 @@
#version 300 es
/* _______________ ,--------------------------------------------------------.
//`````````````\\ \ \
//~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \
//=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \
// \\ \ \
// \\ \ zlib licensed code at [git.nugget.fun/nugget/gunkiss] \
// ☆ GUNKISS ☆ \\ \ \
//_________________________\\ `--------------------------------------------------------*/
/* The precision declaration is required by OpenGL ES */
precision mediump float;
in vec2 uv;
uniform sampler2D base_texture;
uniform vec3 blend_min_hsv;
uniform float time;
uniform bool scroll;
out vec4 outputColor;
/* from http://lolengine.net/blog/2013/07/27/rgb-to-hsv-in-glsl, licensed under WTFPL */
vec3 hsv2rgb(vec3 c)
{
vec4 K = vec4(1.0, 2.0 / 3.0, 1.0 / 3.0, 3.0);
vec3 p = abs(fract(c.xxx + K.xyz) * 6.0 - K.www);
return c.z * mix(K.xxx, clamp(p - K.xxx, 0.0, 1.0), c.y);
}
void main(void)
{
if (scroll)
{
ivec2 texture_size = textureSize(base_texture, 0);
float speed = time * 35.0;
outputColor = texelFetch(base_texture, ivec2(mod(vec2(gl_FragCoord.x + speed, gl_FragCoord.y - speed), vec2(texture_size))), 0);
}
else
{
outputColor = texture(base_texture, uv);
}
/* apply blending, leaving alpha unchanged */
outputColor.xyz = min(outputColor.xyz, hsv2rgb(blend_min_hsv));
}

24
src/shaders/flat.vert Normal file
View File

@ -0,0 +1,24 @@
#version 300 es
/* _______________ ,--------------------------------------------------------.
//`````````````\\ \ \
//~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \
//=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \
// \\ \ \
// \\ \ zlib licensed code at [git.nugget.fun/nugget/gunkiss] \
// ☆ GUNKISS ☆ \\ \ \
//_________________________\\ `--------------------------------------------------------*/
/* The precision declaration is required by OpenGL ES */
precision mediump float;
in vec2 in_position;
in vec2 vertex_uv;
uniform mat4 transformation;
out vec2 uv;
void main(void)
{
gl_Position = transformation * vec4(in_position, 0, 1);
uv = vertex_uv;
}

83
src/shaders/mvp.frag Normal file
View File

@ -0,0 +1,83 @@
#version 300 es
/* _______________ ,--------------------------------------------------------.
//`````````````\\ \ \
//~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \
//=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \
// \\ \ \
// \\ \ zlib licensed code at [git.nugget.fun/nugget/gunkiss] \
// ☆ GUNKISS ☆ \\ \ \
//_________________________\\ `--------------------------------------------------------*/
/* The precision declaration is required by OpenGL ES */
precision mediump float;
#define TRANSFORMATION_NONE 0
#define TRANSFORMATION_SQUIRCLE 1
in vec2 fragment_uv;
in vec3 ex_color;
in float x_center_proximity;
in vec3 original_coordinates;
in vec3 clip_coordinates;
uniform sampler2D pudding_texture;
uniform int uv_transformation;
uniform float coordinate_bound;
out vec4 output_color;
/* [-coordinate_bound, coordinate_bound] arbitrary box coordinates to [-1, 1] normalized coordinates */
vec2 normalize_coordinates(vec2 coordinates)
{
return coordinates / coordinate_bound;
}
/* [-1, 1] box coordinates to [0, 1] UV coordinates */
vec2 coordinates_to_uv(vec2 coordinates)
{
return (1.0 + coordinates) / 2.0;
}
/* coordinates in circle with radius <= 1 to box coordinates in [-1, 1] */
vec2 circle_to_box(vec2 circle)
{
float u = circle.x;
float v = circle.y;
float u_sq = pow(u, 2.0);
float v_sq = pow(v, 2.0);
float rt_2 = sqrt(2.0);
float x = 0.5 * sqrt(2.0 + 2.0 * u * rt_2 + u_sq - v_sq) - 0.5 * sqrt(2.0 - 2.0 * u * rt_2 + u_sq - v_sq);
float y = 0.5 * sqrt(2.0 + 2.0 * v * rt_2 - u_sq + v_sq) - 0.5 * sqrt(2.0 - 2.0 * v * rt_2 - u_sq + v_sq);
return vec2(x, y);
}
/* Apply color passed in from the vertex shader, compressing to one of 16 colors. Add retro effect
* by alternately darkening and lightening 2x2 pixel areas in a checker pattern. Add shadowing by
* brightening the color based on how near it is to the center in the X-dimension */
void retro()
{
vec3 shadowed = min(ex_color, 1.0);
float dx = abs(floor(gl_FragCoord[0]) - 480.0) / 480.0;
if (int(floor(gl_FragCoord[0] / 2.0) + floor(gl_FragCoord[1]) / 2.0) % 2 == 0)
{
output_color = vec4(shadowed * 1.2, 1);
}
else
{
output_color = vec4(shadowed * 0.7, 1);
}
output_color[0] = float(int(output_color[0] * 4.0)) / 4.0;
output_color[1] = float(int(output_color[1] * 4.0)) / 4.0;
output_color[2] = float(int(output_color[2] * 4.0)) / 4.0;
output_color *= x_center_proximity;
}
void main()
{
vec2 uv = fragment_uv;
if (uv_transformation == TRANSFORMATION_SQUIRCLE)
{
vec2 normalized_circle_coordinates = normalize_coordinates(vec2(original_coordinates.x, original_coordinates.z));
uv = coordinates_to_uv(circle_to_box(normalized_circle_coordinates));
}
output_color = texture(pudding_texture, uv);
}

View File

@ -1,13 +1,16 @@
/* _______________ ,----------------------------------------------------------------.
//`````````````\\ \ \
//~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \
//=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \
// \\ \ \
// \\ \ code released under the zlib license [git.nugget.fun/pudding] \
// ☆ GUNKISS ☆ \\ \ \
//_________________________\\ `---------------------------------------------------------------*/
#version 300 es
#version 130
/* _______________ ,--------------------------------------------------------.
//`````````````\\ \ \
//~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \
//=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \
// \\ \ \
// \\ \ zlib licensed code at [git.nugget.fun/nugget/gunkiss] \
// ☆ GUNKISS ☆ \\ \ \
//_________________________\\ `--------------------------------------------------------*/
/* The precision declaration is required by OpenGL ES */
precision mediump float;
#define PI 3.1415926535897932384626433832795
#define AMPLITUDE 0.2
@ -22,7 +25,7 @@ in vec3 vertex_color;
in vec2 vertex_uv;
uniform mat4 mvp;
uniform float time;
uniform int effect = EFFECT_NONE;
uniform int effect;
out vec3 ex_color;
out float x_center_proximity;
out vec2 fragment_uv;