shampoo/gunkiss
shampoo
/
gunkiss
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

working emscripten web export; transfer web cam pixel data from JavaScript to C++; fetch images through Emscripten API and cors-anywhere proxy

This commit is contained in:
ohsqueezy 2022-09-19 22:21:17 -04:00
parent 009e374cd8
commit 5f417a2592
9 changed files with 663 additions and 314 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Makefile
View File

@ -15,9 +15,9 @@
# `git clone --recursive git.nugget.fun/nugget/gunkiss`. The paths below are the default for the repository, but
# they can be edited as necessary.
#######################
# Location parameters #
#######################
#########
# Paths #
#########
# Location of project specific source files
SRC_DIR := src/
@ -38,7 +38,7 @@ CXX := clang++
# Location of SDL config program
SDLCONFIG := $(HOME)/local/sdl/bin/sdl2-config
# Edit to point to the location of BPmono.ttf
# Include BPmono.ttf in the project
CREATE_FONT_SYMLINK := ln -nsf $(SB_DIR)"BPmono.ttf" .
#############################
@ -89,7 +89,7 @@ $(SRC_DIR)Pudding.o : $(SRC_H_FILES) $(SB_H_FILES)
# Linux build #
###############
linux : CFLAGS = -g -Wall -Wextra -O0 -c -I$(SB_LIB_DIR) -I$(SB_SRC_DIR) $(SDL_CFLAGS) -I$(HOME)/local/zbar/include \
linux : CFLAGS = -g -Wall -Wextra -O1 -c -I$(SB_LIB_DIR) -I$(SB_SRC_DIR) $(SDL_CFLAGS) -I$(HOME)/local/zbar/include \
-I $(HOME)/local/opencv/include/opencv4 -I $(HOME)/ext/software/emsdk/upstream/emscripten/system/include
linux : CXXFLAGS = $(CFLAGS) --std=c++17
linux : LFLAGS = $(SDL_LFLAGS) -Wl,--enable-new-dtags -lpthread -lGL -lGLESv2 -lSDL2_image -lSDL2_ttf -lSDL2_mixer -lstdc++fs -lcurl \
@ -108,14 +108,8 @@ linux : $(GLEW_DIR)glew.o $(addprefix $(SDLGFX2_DIR),SDL2_rotozoom.o SDL2_gfxPri
EMSCRIPTENHOME = $(HOME)/ext/software/emsdk/upstream/emscripten
EMSCRIPTEN_CFLAGS = -O1 -Wall -s USE_SDL=2 -s USE_SDL_IMAGE=2 -s SDL2_IMAGE_FORMATS="['png', 'jpg']" -s USE_SDL_TTF=2 -s USE_SDL_MIXER=2 \
--no-heap-copy -I $(SB_LIB_DIR) -I $(SB_SRC_DIR) -I $(HOME)/local/zbar/include \
-I $(HOME)/ext/software/opencv-4.6.0/modules/videoio/include/ \
-I $(HOME)/ext/software/opencv-4.6.0/modules/core/include/ \
-I $(HOME)/ext/software/opencv-4.6.0/modules/highgui/include/ \
-I $(HOME)/ext/software/opencv-4.6.0/modules/imgproc/include/ \
-I $(HOME)/ext/software/opencv-4.6.0/modules/imgcodecs/include/ \
-I $(HOME)/ext/software/opencv-4.6.0/build_wasm/
EMSCRIPTEN_LFLAGS = -s MIN_WEBGL_VERSION=2 -s EXPORTED_FUNCTIONS="['_main']" -s ALLOW_MEMORY_GROWTH=1 -s FULL_ES3=1 \
--no-heap-copy -I $(SB_LIB_DIR) -I $(SB_SRC_DIR) -I $(HOME)/local/zbar/include -I $(HOME)/local/opencv/include/opencv4
EMSCRIPTEN_LFLAGS = -s MIN_WEBGL_VERSION=2 -s EXPORTED_FUNCTIONS="['_main', '_malloc']" -s ALLOW_MEMORY_GROWTH=1 -s FULL_ES3=1 \
-sLLD_REPORT_UNDEFINED -s FETCH --bind $(wildcard $(addprefix $(HOME)/ext/software/opencv-4.6.0/build_wasm/lib/,*.a)) \
$(HOME)/ext/software/ZBar/zbar/.libs/libzbar.a
EMSCRIPTEN_PRELOADS = --preload-file "BPmono.ttf"@/ --preload-file "config.json"@/ --preload-file "resource/"@/"resource/" \

30
config.json
View File

@ -3,11 +3,11 @@
{
"dimensions": [460, 768],
"framerate": 60,
"title": "Gunkiss",
"title": "Pudding",
"debug": false,
"render driver": "opengles2",
"render driver": "opengl",
"show-cursor": true,
"camera-resolution": [1280, 720]
"camera-resolution": [320, 240]
},
"configuration":
@ -21,8 +21,8 @@
"print-frame-length-history": ["CTRL", "SHIFT", "h"],
"toggle-camera": ["CTRL", "c"],
"toggle-item": ["CTRL", "i"],
"effect": ["CTRL", "e"],
"tile": ["CTRL", "t"]
"effect": ["e"],
"tile": ["t"]
},
"recording":
@ -55,27 +55,31 @@
"scan":
{
"enabled": true,
"json-save": true,
"json-save": false,
"json-save-directory": "local/scans",
"barcode": "",
"capture-device": "/dev/video0"
"capture-device": "/dev/video0",
"brightness-addition": 10,
"contrast-multiplication": 1.3,
"camera-device-id": 0
},
"api":
{
"user-agent": "Custom pudding creation game under development for https://shampoo.ooo",
"nutronix-app-id": "ea0f2e7e",
"nutronix-app-key": "39218dde526dd3349daa028deda518ae",
"user-agent": "Custom pudding creation game under development at https://mario.shampoo.ooo",
"nutritionix-app-id": "ea0f2e7e",
"nutritionix-app-key": "39218dde526dd3349daa028deda518ae",
"edamam-app-id": "c23b139f",
"edamam-app-key": "c54cf8c997534caf7ee92b1ccc7d95a3",
"best-buy-api-key": "vAC23XA5YWBzaYiGtOkoNlXZ",
"giantbomb-api-key": "91a395231f4e1fd9f9ba8840c52a61cda343cd70",
"nutronix-enabled": false,
"edamam-enabled": false,
"google-books-api-key": "AIzaSyBD9wXIlBJ6UrXXDIY03k6s0oR1q6ByETQ",
"nutritionix-enabled": true,
"edamam-enabled": true,
"open-food-enabled": true,
"open-products-enabled": true,
"best-buy-enabled": true,
"google-books-enabled": true
"google-books-enabled": false
},
"pudding":

BIN
favicon.ico Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.2 KiB

128
index.html
View File

@ -1,104 +1,128 @@
<!doctype html>
<html>
<head>
<style>
body
{
background: #000;
}
</style>
</head>
<body>
<!-- WebGL output will be drawn here through Emscripten -->
<!-- WebGL output will be drawn here through Emscripten. The dimensions will be set by Emscripten. -->
<canvas id="canvas"></canvas>
<!-- navigator.mediaDevices.getUserMedia streams the webcam video directly, displayed for testing -->
<!-- <video id="webcam"></video> -->
<script>
const FPS = 15;
const FPS = 10;
const BPP = 4;
// Direct output of webcam (hidden)
/* Direct output of webcam in HTML5 (not displayed) */
var video = document.createElement("video");
video.width = 320;
video.height = 240;
// Undisplayed canvas which is used to draw the video frame and access the pixel data directly
/* Undisplayed canvas which is used to draw the video frame and access the pixel data directly */
var intermediate = document.createElement("canvas");
intermediate.width = video.width;
intermediate.height = video.height;
var context = intermediate.getContext("2d");
// Indicates whether webcam is opened or not
/* Indicates whether webcam is opened or not */
var streaming = false;
// Address of the webcam frame pixel data on the Emscripten heap
/* Address of the webcam frame pixel data on the Emscripten heap */
var image_heap_address;
/* Stores the time when the last webcam frame was processed */
var previous_frame_timestamp;
var Module = {
/* When the Module is finished loading, launch the camera frame processor function. */
onRuntimeInitialized: function()
{
process_video();
window.requestAnimationFrame(process_video);
},
// Tell Emscripten to use this canvas for display
/* Set Emscripten to use a canvas for display. */
canvas: document.getElementById("canvas")
};
/*!
* Open the webcam and start displaying frames if successfully opened. Allocate space for 32-bit RGBA frame pixel data
* on the Emscripten heap.
*/
function open_camera()
{
// Open the webcam and start displaying frames if successfully opened. Allocate space for 32-bit RGBA frame pixel data
// on the Emscripten heap.
navigator.mediaDevices.getUserMedia({video: {width: video.width, height: video.height}, audio: false})
.then(function(stream) {
video.srcObject = stream;
video.play();
streaming = true;
// Get the memory address of the pixel data
image_heap_address = Module._malloc(video.width * video.height * BPP);
// Pass the address to the C++ program
Module.set_heap_offset(image_heap_address);
})
.catch(function(err) {
console.log('Camera Error: ' + err.name + ' ' + err.message);
});
navigator.mediaDevices.getUserMedia(
{
video: {
width: video.width,
height: video.height
},
audio: false
})
.then(function(stream) {
video.srcObject = stream;
video.play();
streaming = true;
/* Get the memory address of the pixel data */
image_heap_address = Module._malloc(video.width * video.height * BPP);
/* Pass the address to the game object */
Module.set_heap_offset(image_heap_address);
})
.catch(function(err) {
console.log('Camera Error: ' + err.name + ' ' + err.message);
});
}
function close_camera()
{
Module._free
video.pause();
video.srcObject = null;
streaming = false;
}
// This function will run continuously, drawing the webcam frame to the intermediate canvas, reading the pixel data,
// storing the data on the heap, and setting the new frame available flag.
function process_video()
/*!
* Run continuously, drawing the webcam frame to the intermediate canvas, reading the pixel data, storing the data on the heap,
* and setting the new frame available flag.
*
* @param timestamp time at which the method was called, provided automatically by `window.requestAnimationFrame`
*/
function process_video(timestamp)
{
try
if (timestamp - previous_frame_timestamp > 1000 / FPS || previous_frame_timestamp == undefined)
{
if (streaming)
previous_frame_timestamp = timestamp
try
{
// Draw the webcam frame on a hidden canvas
context.drawImage(video, 0, 0, video.width, video.height);
if (streaming)
{
/* Draw the webcam frame on a hidden canvas */
context.drawImage(video, 0, 0, video.width, video.height);
// Read the pixel data
image_data = context.getImageData(0, 0, video.width, video.height).data;
/* Read the pixel data */
image_data = context.getImageData(0, 0, video.width, video.height).data;
// Get a memory view object that provides access to the heap at the previously allocated address
image_heap_data = new Uint8Array(Module.HEAPU8.buffer, image_heap_address, video.width * video.height * BPP);
/* Get a memory view object that provides access to the heap at the previously allocated address */
image_heap_data = new Uint8Array(Module.HEAPU8.buffer, image_heap_address, video.width * video.height * BPP);
// Write the pixel data to the heap
image_heap_data.set(image_data);
/* Write the pixel data to the heap */
image_heap_data.set(image_data);
// Flag the C++ that new data is available
Module.flag_frame();
/* Flag the game object that new data is available */
Module.flag_frame();
}
}
catch (err)
{
console.log(err);
}
// Loop at roughly the FPS
let begin = Date.now();
let delay = 1000/FPS - (Date.now() - begin);
setTimeout(process_video, delay);
}
catch (err)
{
console.log(err);
}
window.requestAnimationFrame(process_video);
}
</script>

2
lib/sb

@ -1 +1 @@
Subproject commit b1fb77b1c8a2902fde711ede1a45b459013dc876
Subproject commit 24f6d3ed3d4962a88078c5024473834812968d1a

25
src/Item.cpp
View File

@ -1,10 +1,10 @@
/* _______________ ,-------------------------------------------------.
//`````````````\\ \ \
//~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \
//=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \
// \\ \ \
// \\ \ zlib licensed code at [git.nugget.fun/pudding] \
// ☆ GUNKISS ☆ \\ \ \
//`````````````\\ \ \
//~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \
//=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \
// \\ \ \
// \\ \ zlib licensed code at [git.nugget.fun/pudding] \
// ☆ GUNKISS ☆ \\ \ \
//_________________________\\ `-------------------------------------------------*/
#include "Item.hpp"
@ -69,6 +69,13 @@ std::string Item::full_name() const
name += " ";
}
name += product_name();
/* If no names have been set yet, try the UPC as a name */
if (name == "")
{
name = upc();
}
return name;
}
@ -125,3 +132,9 @@ void Item::to_last()
{
carousel.end(item_view.textures());
}
std::ostream& std::operator<<(std::ostream& out, const Item& item)
{
out << item.full_name();
return out;
}

12
src/Item.hpp
View File

@ -68,4 +68,16 @@ public:
};
namespace std
{
/*!
* Support passing item objects to the global stream operator.
*
* @param out The output stream
* @param item The item to be printed
* @return The submitted output stream with the text representation of item added
*/
std::ostream& operator<<(std::ostream& out, const Item& item);
}
#endif

577
src/Pudding.cpp
View File

@ -75,7 +75,7 @@ Pudding::Pudding()
void Pudding::load_pudding_model(float top_radius, float base_radius, int ring_vertex_count, int layer_count, float min_y,
float max_y, float gradient_position)
{
size_t ii;
std::size_t ii;
const glm::vec3 *layer_top_color, *layer_bottom_color;
const glm::vec2 *start_vertex, *end_vertex;
float layer_top_y, layer_top_percent, layer_base_y, layer_base_percent, u_step = 1.0f / ring_vertex_count, ring_start_vertex_u;
@ -284,16 +284,11 @@ void Pudding::load_pads()
next_button.rotation(glm::radians(180.0f));
}
/*!
* Try to create cv::VideoCapture object using device ID #0. If successful, this will also create a GL texture ID and
* storage for the camera frame on the GPU, so it must be called after GL context has been created. Create and detach
* a thread which will continuously read frame data.
*/
void Pudding::open_camera()
{
#ifndef __EMSCRIPTEN__
/* Open the OpenCV capture, using device ID #0 to get the default attached camera. */
int device_id = 0;
int device_id = configuration()["scan"]["camera-device-id"];
capture.open(device_id);
std::ostringstream message;
if (capture.isOpened())
@ -432,6 +427,15 @@ void Pudding::respond(SDL_Event& event)
if (over_camera_button)
{
camera_switch.connect();
#ifndef __EMSCRIPTEN__
/* If the camera did not open, this failed, so unflip the switch */
if (!capture.isOpened())
{
camera_switch.disconnect();
}
#endif
}
else if (over_inventory_button)
{
@ -489,151 +493,156 @@ void Pudding::respond(SDL_Event& event)
*/
void Pudding::add_item(const std::string& upc)
{
Item item;
item.upc(upc);
/* Store the UPC code in the incoming item object */
incoming_item.upc(upc);
if (configuration()["api"]["open-food-enabled"])
{
incorporate_open_api(item, OPEN_FOOD_API_URL);
web_get_bytes(OPEN_FOOD_API_URL + upc, std::bind(&Pudding::incorporate_open_api, this, std::placeholders::_1, std::placeholders::_2));
}
if (configuration()["api"]["open-products-enabled"])
{
incorporate_open_api(item, OPEN_PRODUCTS_API_URL);
web_get_bytes(OPEN_PRODUCTS_API_URL + upc, std::bind(&Pudding::incorporate_open_api, this, std::placeholders::_1, std::placeholders::_2));
}
if (configuration()["api"]["nutronix-enabled"])
if (configuration()["api"]["nutritionix-enabled"])
{
incorporate_nutronix_api(item);
/* Nutritionix requires API keys in headers for validation */
web_get_bytes(NUTRITIONIX_API_URL + upc, std::bind(&Pudding::incorporate_nutritionix_api, this, std::placeholders::_1, std::placeholders::_2), {
"x-app-id", configuration()["api"]["nutritionix-app-id"].get<std::string>(),
"x-app-key", configuration()["api"]["nutritionix-app-key"].get<std::string>()
});
}
if (configuration()["api"]["edamam-enabled"])
{
incorporate_edamam_api(item);
/* Build API request by concatenating URL and query string */
std::stringstream url;
url << "https://api.edamam.com/api/food-database/v2/parser?upc=" << upc << "&app_id=" <<
configuration()["api"]["edamam-app-id"].get<std::string>() << "&app_key=" <<
configuration()["api"]["edamam-app-key"].get<std::string>();
web_get_bytes(url.str(), std::bind(&Pudding::incorporate_edamam_api, this, std::placeholders::_1, std::placeholders::_2));
}
if (configuration()["api"]["best-buy-enabled"])
{
incorporate_best_buy_api(item);
/* Build API request by concatenating URL and query string */
std::stringstream url;
url << BEST_BUY_API_URL_1 << upc << BEST_BUY_API_URL_2 << configuration()["api"]["best-buy-api-key"].get<std::string>();
web_get_bytes(url.str(), std::bind(&Pudding::incorporate_best_buy_api, this, std::placeholders::_1, std::placeholders::_2));
}
if (configuration()["api"]["google-books-enabled"])
{
incorporate_google_books_api(item);
}
if (item.texture_count() > 0)
{
items.push_back(item);
/* Set item index to end so newest item will display. */
item_carousel.end(items);
/* Move the camera button away from center to make room for inventory button if this is the first item added. */
if (items.size() == 1)
{
const nlohmann::json& interface = configuration()["interface"];
camera_button.translation({-1.0f * interface["main-button-double-x"].get<float>(), interface["main-button-y"]});
}
}
else
{
std::ostringstream message;
message << "discarding item, no images found for " << upc;
sb::Log::log(message);
std::stringstream url;
url << GOOGLE_BOOKS_API_URL << upc << "&key=" << configuration()["api"]["google-books-api-key"].get<std::string>();
web_get_bytes(url.str(), std::bind(&Pudding::incorporate_google_books_api, this, std::placeholders::_1, std::placeholders::_2));
}
}
/* Look for item upc in the Open Food/Products API and use the result to fill out item properties if found. */
void Pudding::incorporate_open_api(Item& item, const std::string& api_url)
void Pudding::incorporate_open_api(const std::vector<std::uint8_t>& json_bytes, const std::string& url)
{
std::ostringstream checking_message;
checking_message << "checking " << api_url;
sb::Log::log(checking_message);
nlohmann::json json = json_from_url(api_url + item.upc());
/* test that should determine if an Open Food API response is not empty */
std::ostringstream message;
message << "Processing " << (json_bytes.size() / 100.0) << "KB from the Open Food/Products API";
sb::Log::log(message);
/* Use the nlohmann library to parse the raw JSON byte data retrieved from a web request. */
nlohmann::json json = nlohmann::json::parse(json_bytes);
std::stringstream json_formatted;
json_formatted << std::setw(4) << json << std::endl;
sb::Log::log(json_formatted.str(), sb::Log::DEBUG);
/* Test that should determine if an Open Food API response is not empty */
if (json.value("status", 0) && json.contains("product"))
{
std::ostringstream message;
if (json["product"].value("image_url", "") != "")
{
std::string image_url = json["product"]["image_url"];
sb::Texture texture = texture_from_image_url(image_url);
if (texture.generated())
{
item.texture(texture, image_url);
}
std::ostringstream message;
message << "Found image URL for item " << incoming_item << " from Open API at " << image_url;
sb::Log::log(message);
web_get_bytes(image_url, std::bind(&Pudding::store_web_image, this, std::placeholders::_1, std::placeholders::_2));
}
item.brand_name(json["product"].value("brands", ""));
item.product_name(json["product"].value("product_name", ""));
if (api_url == OPEN_FOOD_API_URL)
else
{
save_item_json(json, item, "Open_Food_API");
}
else if (api_url == OPEN_PRODUCTS_API_URL)
{
save_item_json(json, item, "Open_Products_API");
message << "No images found at Open API for " << incoming_item;
}
sb::Log::log(message);
incoming_item.brand_name(json["product"].value("brands", ""));
incoming_item.product_name(json["product"].value("product_name", ""));
save_item_json(json, incoming_item, "Open_Food_and_Products_API");
}
else
{
std::ostringstream results_message;
results_message << "no results from " << api_url;
sb::Log::log(results_message);
sb::Log::log("No item found in JSON from Open API");
}
}
/* Look for item upc in the Nutronix API, and use the result to fill out item properties if found
*/
void Pudding::incorporate_nutronix_api(Item& item)
void Pudding::incorporate_nutritionix_api(const std::vector<std::uint8_t>& json_bytes, const std::string& url)
{
sb::Log::log("checking Nutronix API");
/* Nutronix requires API keys in headers for validation */
nlohmann::json json = json_from_url(
NUTRONIX_API_URL + item.upc(), {
"x-app-id: " + configuration()["api"]["nutronix-app-id"].get<std::string>(),
"x-app-key: " + configuration()["api"]["nutronix-app-key"].get<std::string>()
});
/* test that should determine if a Nutronix response is not empty */
if (!(json.contains("message") && json["message"] == NUTRONIX_NOT_FOUND))
std::ostringstream message;
message << "Processing " << (json_bytes.size() / 100.0) << "KB from the Nutritionix API";
sb::Log::log(message);
/* Use the nlohmann library to parse the raw JSON byte data retrieved from a web request. */
nlohmann::json json = nlohmann::json::parse(json_bytes);
std::stringstream json_formatted;
json_formatted << std::setw(4) << json << std::endl;
sb::Log::log(json_formatted.str(), sb::Log::DEBUG);
/* test that should determine if a Nutritionix response is not empty */
if (!(json.contains("message") && json["message"] == NUTRITIONIX_NOT_FOUND))
{
nlohmann::json food = json["foods"][0];
std::ostringstream message;
if (food.contains("photo") && food["photo"].value("thumb", "") != "")
{
std::string url = food["photo"]["thumb"];
sb::Log::log("adding image listed in Nutronix API at " + url);
sb::Texture texture = texture_from_image_url(url);
if (texture.generated())
{
item.texture(texture, url);
}
std::string image_url = food["photo"]["thumb"];
message << "Found image URL for item " << incoming_item << " from Nutritionix at " << image_url;
web_get_bytes(image_url, std::bind(&Pudding::store_web_image, this, std::placeholders::_1, std::placeholders::_2));
}
item.brand_name(food.value("brand_name", ""));
item.product_name(food.value("food_name", ""));
save_item_json(json, item, "Nutronix_API");
else
{
message << "No images found at Nutritionix for " << incoming_item;
}
sb::Log::log(message);
incoming_item.brand_name(food.value("brand_name", ""));
incoming_item.product_name(food.value("food_name", ""));
save_item_json(json, incoming_item, "Nutritionix_API");
}
else
{
sb::Log::log("no results from Nutronix");
sb::Log::log("no results from Nutritionix");
}
}
/* Submit a query to Edamam API and insert relevant results into supplied Item object
*/
void Pudding::incorporate_edamam_api(Item& item)
void Pudding::incorporate_edamam_api(const std::vector<std::uint8_t>& json_bytes, const std::string& url)
{
sb::Log::log("checking Edamam API");
/* build API url by concatenating relevant values into query string */
std::stringstream url;
url << "https://api.edamam.com/api/food-database/v2/parser?upc=" << item.upc() << "&app_id=" <<
configuration()["api"]["edamam-app-id"].get<std::string>() << "&app_key=" <<
configuration()["api"]["edamam-app-key"].get<std::string>();
nlohmann::json json = json_from_url(url.str());
std::ostringstream message;
message << "Processing " << (json_bytes.size() / 100.0) << "KB from the Edamam API";
sb::Log::log(message);
/* Use the nlohmann library to parse the raw JSON byte data retrieved from a web request. */
nlohmann::json json = nlohmann::json::parse(json_bytes);
std::stringstream json_formatted;
json_formatted << std::setw(4) << json << std::endl;
sb::Log::log(json_formatted.str(), sb::Log::DEBUG);
/* test that should determine if a Edamam response has food data */
if (json.contains("hints") && json["hints"][0].contains("food"))
{
nlohmann::json food = json["hints"][0]["food"];
std::ostringstream message;
if (food.value("image", "") != "")
{
std::string url = food["image"];
sb::Texture texture = texture_from_image_url(url);
if (texture.generated())
{
item.texture(texture, url);
}
item.product_name(food.value("label", ""));
std::string image_url = food["image"];
message << "Found URL to image for item " << incoming_item << " from Edamam at " << image_url;
web_get_bytes(image_url, std::bind(&Pudding::store_web_image, this, std::placeholders::_1, std::placeholders::_2));
}
save_item_json(json, item, "Edamam_API");
else
{
message << "No images found at Edamam for " << incoming_item;
}
sb::Log::log(message);
incoming_item.product_name(food.value("label", ""));
save_item_json(json, incoming_item, "Edamam_API");
}
else
{
@ -641,16 +650,18 @@ void Pudding::incorporate_edamam_api(Item& item)
}
}
/* Submit a query to the Best Buy API and insert relevant results into supplied Item object
*/
void Pudding::incorporate_best_buy_api(Item& item)
void Pudding::incorporate_best_buy_api(const std::vector<std::uint8_t>& json_bytes, const std::string& url)
{
sb::Log::log("checking Best Buy API");
/* build API url by concatenating relevant values into query string */
std::stringstream url;
url << "https://api.bestbuy.com/v1/products(upc=" << item.upc() << ")?format=json&apiKey=" <<
configuration()["api"]["best-buy-api-key"].get<std::string>();
nlohmann::json json = json_from_url(url.str());
std::ostringstream message;
message << "Processing " << (json_bytes.size() / 100.0) << "KB from the Best Buy API";
sb::Log::log(message);
/* Use the nlohmann library to parse the raw JSON byte data retrieved from a web request. */
nlohmann::json json = nlohmann::json::parse(json_bytes);
std::stringstream json_formatted;
json_formatted << std::setw(4) << json << std::endl;
sb::Log::log(json_formatted.str(), sb::Log::DEBUG);
/* test that should determine if a Best Buy response has a result */
if (json.contains("total") && json["total"].get<int>() > 0)
{
@ -658,18 +669,21 @@ void Pudding::incorporate_best_buy_api(Item& item)
/* look up image (for games this is box art) and "alternate views image" (for games this is a screen shot) */
for (std::string key : {"alternateViewsImage", "image"})
{
std::ostringstream message;
if (product.value(key, "") != "")
{
std::string url = product[key];
sb::Texture texture = texture_from_image_url(url);
if (texture.generated())
{
item.texture(texture, url);
}
std::string image_url = product[key];
message << "Found URL to image for item " << incoming_item << " from Best Buy at " << image_url;
web_get_bytes(image_url, std::bind(&Pudding::store_web_image, this, std::placeholders::_1, std::placeholders::_2));
}
else
{
message << "No images found at Best Buy for " << incoming_item;
}
sb::Log::log(message);
}
item.product_name(product.value("name", ""));
save_item_json(json, item, "Best_Buy_API");
incoming_item.product_name(product.value("name", ""));
save_item_json(json, incoming_item, "Best_Buy_API");
}
else
{
@ -677,32 +691,45 @@ void Pudding::incorporate_best_buy_api(Item& item)
}
}
/* Look for item upc in the Google Books API and use the result to fill out item properties if found. */
void Pudding::incorporate_google_books_api(Item& item)
void Pudding::incorporate_google_books_api(const std::vector<std::uint8_t>& json_bytes, const std::string& url)
{
sb::Log::log("checking Google Books API");
nlohmann::json json = json_from_url(GOOGLE_BOOKS_API_URL + item.upc());
std::ostringstream message;
message << "Processing " << (json_bytes.size() / 100.0) << "KB from the Google Books API";
sb::Log::log(message);
/* Use the nlohmann library to parse the raw JSON byte data retrieved from a web request. */
nlohmann::json json = nlohmann::json::parse(json_bytes);
std::stringstream json_formatted;
json_formatted << std::setw(4) << json << std::endl;
sb::Log::log(json_formatted.str(), sb::Log::DEBUG);
/* test that should determine if a Google Books API response is not empty */
if (json.value<int>("totalItems", 0) > 0 && json.contains("items") && json["items"][0].contains("volumeInfo"))
{
/* book specific section of the JSON */
json = json["items"][0]["volumeInfo"];
/* get the image data */
std::ostringstream message;
if (json.contains("imageLinks") && json["imageLinks"].value("thumbnail", "") != "")
{
std::string image_url = json["imageLinks"]["thumbnail"];
sb::Texture texture = texture_from_image_url(image_url);
if (texture.generated())
{
item.texture(texture, image_url);
}
message << "Found URL to image for item " << incoming_item << " from Google Books at " << image_url;
web_get_bytes(image_url, std::bind(&Pudding::store_web_image, this, std::placeholders::_1, std::placeholders::_2));
}
else
{
message << "No images found at Google Books for " << incoming_item;
}
sb::Log::log(message);
if (json.contains("authors"))
{
item.brand_name(json["authors"][0]);
incoming_item.brand_name(json["authors"][0]);
}
item.product_name(json.value("title", ""));
save_item_json(json, item, "Google_Books_API");
incoming_item.product_name(json.value("title", ""));
save_item_json(json, incoming_item, "Google_Books_API");
}
else
{
@ -743,44 +770,51 @@ void Pudding::save_item_json(const nlohmann::json& json, const Item& item, const
}
}
/* Download the JSON data at the submitted URL, and return it as a JSON object
*/
nlohmann::json Pudding::json_from_url(const std::string& url, const std::vector<std::string>& headers)
void Pudding::web_get_bytes(std::string url, const web_callback& callback, const std::vector<std::string>& headers)
{
std::vector<std::uint8_t> storage;
web_get_bytes(url, storage, headers);
nlohmann::json json = nlohmann::json::parse(storage);
std::stringstream json_formatted;
json_formatted << std::setw(4) << json << std::endl;
sb::Log::log(json_formatted.str(), sb::Log::DEBUG);
return json;
}
std::stringstream message;
message << "Fetching data from " << url;
sb::Log::log(message.str());
/* Add a request object to the end of the vector of launched requests. */
Request* request = new Request(callback, url);
requests.push_back(request);
/* Use the CORS anywhere proxy */
url = "https://mario.shampoo.ooo:8088/" + url;
/*!
* Store the bytes retrieved from `url` in the byte vector `storage`.
*
* The compiler will determine whether to use cURL or the Emscripten Fetch API to do the retrieval, depending on whether it is compiling for
* Emscripten.
*
* The optional `headers` parameter will be added to the request when using cURL, but not when using the Emscripten Fetch API.
*
* @param url URL containing data to be retrieved
* @param storage A reference to a vector of bytes which will be filled with the data retrieved from the URL
* @param headers A reference to a vector of strings that should be passed as headers with the request. It is only supported by the cURL version.
*/
void Pudding::web_get_bytes(const std::string& url, std::vector<std::uint8_t>& storage, const std::vector<std::string>& headers) const
{
#if defined(__EMSCRIPTEN__)
/* Create a fetch attributes object. Set a callback that will be called when response data is received. Pass along the user
* storage location to be filled by the callback. */
/* Create a fetch attributes object. Set the callback that will be called when response data is received. Attach the user
* submitted callback to the userData attribute. Set the headers. */
emscripten_fetch_attr_t attr;
emscripten_fetch_attr_init(&attr);
strcpy(attr.requestMethod, "GET");
attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY;
attr.onsuccess = fetch_success;
attr.onerror = fetch_error;
attr.userData = &storage;
attr.userData = request;
/* Copy headers into a vector of C strings with null terminator for Emscripten */
if (!headers.empty())
{
std::vector<const char*>* emscripten_formatted_headers = new std::vector<const char*>();
for (const std::string& component : headers)
{
const std::string* component_c = new std::string(component.c_str());
emscripten_formatted_headers->push_back(component_c->c_str());
}
emscripten_formatted_headers->push_back(nullptr);
std::ostringstream message;
message << "Headers are";
for (const char* component : *emscripten_formatted_headers)
{
message << " " << component;
}
sb::Log::log(message);
attr.requestHeaders = emscripten_formatted_headers->data();
}
emscripten_fetch(&attr, url.c_str());
#else
@ -790,7 +824,7 @@ void Pudding::web_get_bytes(const std::string& url, std::vector<std::uint8_t>& s
result = curl_global_init(CURL_GLOBAL_DEFAULT);
if (result != CURLE_OK)
{
std::cout << "curl initialization failed " << curl_easy_strerror(result) << std::endl;
std::cout << "cURL initialization failed " << curl_easy_strerror(result) << std::endl;
}
else
{
@ -798,105 +832,119 @@ void Pudding::web_get_bytes(const std::string& url, std::vector<std::uint8_t>& s
if (curl)
{
curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, Pudding::curl_write_response);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &storage);
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_response);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, request);
curl_easy_setopt(curl, CURLOPT_USERAGENT, configuration()["api"]["user-agent"].get<std::string>().c_str());
/* Pass submitted headers to cURL */
struct curl_slist* list = nullptr;
if (headers.size() > 0)
{
for (const std::string& header : headers)
/* cURL expects headers as a list of "name: value" pair strings, so combine every two components of the headers list
* into a single string */
for (std::size_t ii = 0; ii < headers.size(); ii += 2)
{
list = curl_slist_append(list, header.c_str());
std::string pair = headers[ii] + ": " + headers[ii + 1];
list = curl_slist_append(list, pair.c_str());
}
}
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
result = curl_easy_perform(curl);
curl_slist_free_all(list);
if (result != CURLE_OK)
{
std::cout << "curl request failed " << curl_easy_strerror(result) << std::endl;
std::cout << "cURL request failed " << curl_easy_strerror(result) << std::endl;
}
}
else
{
std::cout << "curl initialization failed" << std::endl;
std::cout << "cURL initialization failed" << std::endl;
}
curl_easy_cleanup(curl);
}
curl_global_cleanup();
/* Call the user supplied callback */
request->respond();
#endif
}
#if defined(__EMSCRIPTEN__)
/*!
* This will be called automatically when request data is sucessfully fetched by `emscripten_fetch` from `Pudding::web_get_bytes`.
* Response bytes will be inserted into the user supplied `std::vector<std::uint8_t>&` at `fetch->userData`.
*/
void Pudding::fetch_success(emscripten_fetch_t* fetch)
{
std::vector<std::uint8_t>* storage = reinterpret_cast<std::vector<std::uint8_t>*>(fetch->userData);
storage->insert(storage->end(), fetch->data, fetch->data + fetch->numBytes);
std::stringstream message;
message << "Stored " << (fetch->numBytes / 100) << "KB of image data in memory from " << fetch->url;
sb::Log::log(message.str());
std::stringstream bytes_message;
bytes_message << "Found " << fetch->numBytes << " bytes using Emscripten Fetch API";
sb::Log::log(bytes_message.str());
/* Store the bytes in the request object */
Request* request = reinterpret_cast<Request*>(fetch->userData);
request->store(reinterpret_cast<const std::uint8_t*>(fetch->data), fetch->numBytes);
/* Call the user supplied callback */
request->respond();
emscripten_fetch_close(fetch);
}
/*!
* This will be called automatically when request data is not successfully fetched by `emscripten_fetch` from `Pudding::web_get_bytes`.
*/
void Pudding::fetch_error(emscripten_fetch_t* fetch)
{
std::stringstream message;
message << "Downloading image from " << fetch->url << " failed with status code " << fetch->status;
sb::Log::log(message.str());
std::ostringstream message;
message << "Failed fetching " << fetch->url << " with status code " << fetch->status;
sb::Log::log(message);
/* Since the request failed, mark it finished */
Request* request = reinterpret_cast<Request*>(fetch->userData);
request->mark_finished();
emscripten_fetch_close(fetch);
}
#else
/*!
* This will be called by cURL when it has received a buffer of data. The data will be inserted into the vector at `storage`
*
* @param buffer pointer to data
* @param size size in bytes of each value
* @param count number of values
* @param storage pointer to a vector of unsigned 8-bit values where the data will be copied to
* @return number of bytes copied
*/
size_t Pudding::curl_write_response(std::uint8_t* buffer, size_t size, size_t count, std::vector<std::uint8_t>* storage)
std::size_t Pudding::curl_write_response(std::uint8_t* buffer, std::size_t size, std::size_t count, Request* request)
{
size_t total_size = size * count;
storage->insert(storage->end(), buffer, buffer + total_size);
return total_size;
std::size_t packet_size = size * count;
std::stringstream bytes_message;
bytes_message << "Found " << packet_size << " bytes using cURL ";
sb::Log::log(bytes_message.str());
/* Store the bytes in the request object */
request->store(buffer, packet_size);
return packet_size;
}
#endif
/* Allocate storage for a texture, copy the cURL response data into the storage, and return the ID that corresponds to the GL texture
*/
sb::Texture Pudding::texture_from_image_url(const std::string& url) const
void Pudding::store_web_image(const std::vector<std::uint8_t>& image, const std::string& url)
{
/* this texture will be returned whether we load pixels into it or not */
/* Get a Texture by passing the bytes through an RW ops which will enable the Texture object to load a Surface */
sb::Texture texture;
sb::Log::log("looking up image at " + url);
std::vector<std::uint8_t> storage;
web_get_bytes(url, storage);
if (!storage.empty())
SDL_RWops* rw = SDL_RWFromConstMem(image.data(), image.size());
texture.load(rw);
SDL_RWclose(rw);
std::ostringstream message;
sb::Log::Level message_level;
if (texture.generated())
{
sb::Log::log("received image data", sb::Log::DEBUG);
/* get a Texture by passing the bytes through an RW ops which will enable the Texture object to load a Surface */
SDL_RWops* rw = SDL_RWFromConstMem(storage.data(), storage.size());
texture.load(rw);
SDL_RWclose(rw);
message << "Loaded an image from " << url << " and attached it to " << incoming_item << " at " << &incoming_item;
message_level = sb::Log::INFO;
/* Use the URL as the name for the texture */
incoming_item.texture(texture, url);
}
else
{
SDL_LogWarn(SDL_LOG_CATEGORY_CUSTOM, "image url returned no data");
message << "Could not generate texture from " << url;
message_level = sb::Log::WARN;
}
return texture;
sb::Log::log(message, message_level);
}
/* Call GL's delete texture function, and print a debug statement for testing. This is defined as a static member
@ -973,6 +1021,14 @@ void Pudding::capture_frame()
if (!camera_frame.empty())
{
/* Brightness and contrast adjustment, see https://docs.opencv.org/2.4.13.7/doc/tutorials/core/basic_linear_transform/basic_linear_transform.html */
int brightness = configuration()["scan"]["brightness-addition"];
float contrast = configuration()["scan"]["contrast-multiplication"];
if (brightness != 0 || contrast != 1.0)
{
camera_frame.convertTo(camera_frame, -1, contrast, brightness);
}
/* Finished loading into `cv::Mat`, so it is new data that is safe to read. */
new_frame_available = true;
}
@ -1006,20 +1062,31 @@ void Pudding::update()
if (new_frame_available)
{
sb::Log::log("Hello, World!");
#ifdef __EMSCRIPTEN__
/* Emscripten builds load pixel data into cv::Mat synchronously */
capture_frame();
/* Pixels from Emscripten are RGBA */
GLenum pixel_format = GL_RGBA;
/* The cv::Mat rows vs. cols (width vs. height) are correct in Emscripten? */
int camera_frame_width = camera_frame.size[0];
int camera_frame_height = camera_frame.size[1];
#else
/* Pixels from cv::VideoCapture are BGR */
GLenum pixel_format = GL_BGR;
/* The cv::Mat rows vs. cols (width vs. height) values are swapped? */
int camera_frame_width = camera_frame.size[1];
int camera_frame_height = camera_frame.size[0];
#endif
camera_view.texture().bind();
/* Fill camera view texture memory with last frame's pixels */
// camera_view.texture().load(camera_frame.ptr(), {camera_frame.cols, camera_frame.rows}, GL_BGR, GL_UNSIGNED_BYTE);
// std::cout << camera_frame.size[0] << " " << camera_frame.size[1] << std::endl;
camera_view.texture().load(camera_frame.ptr(), {320, 240}, GL_RGBA, GL_UNSIGNED_BYTE);
camera_view.texture().load(camera_frame.ptr(), {camera_frame_width, camera_frame_height}, pixel_format, GL_UNSIGNED_BYTE);
/* Frame data has been loaded, so there is not a new frame available anymore. */
new_frame_available = false;
/* Convert to grayscale for ZBar */
/* Convert to grayscale, for ZBar */
cv::cvtColor(camera_frame, camera_frame, cv::COLOR_BGR2GRAY);
if (configuration()["scan"]["enabled"])
{
@ -1090,15 +1157,20 @@ void Pudding::update()
/* disable bg attributes and enable pudding attributes */
background.disable();
pudding_model.attributes("position")->enable();
GLenum side_mode, top_mode;
if (items.size() == 0)
{
// glPolygonMode(GL_FRONT_AND_BACK, GL_LINE);
// pudding_model.attributes("color")->enable();
side_mode = GL_LINES;
top_mode = GL_LINES;
}
else
{
// glPolygonMode(GL_FRONT_AND_BACK, GL_FILL);
// pudding_model.attributes("color")->enable();
side_mode = GL_TRIANGLES;
top_mode = GL_TRIANGLE_FAN;
pudding_model.attributes("uv")->enable();
glUniform1i(uniform["mvp"]["pudding texture"], 0);
glActiveTexture(GL_TEXTURE0);
@ -1107,14 +1179,14 @@ void Pudding::update()
/* draw pudding model */
glEnable(GL_DEPTH_TEST);
/* draw the sides of the pudding */
glDrawArrays(GL_TRIANGLES, 0, pudding_triangle_vertex_count);
glDrawArrays(side_mode, 0, pudding_triangle_vertex_count);
sb::Log::gl_errors("after pudding sides, before pudding top/bottom");
/* enable squircling and draw the top and bottom of pudding */
glUniform1i(uniform["mvp"]["uv transformation"], UV_SQUIRCLE);
glUniform1f(uniform["mvp"]["coordinate bound"], configuration()["pudding"]["top-radius"]);
glDrawArrays(GL_TRIANGLE_FAN, pudding_triangle_vertex_count, pudding_fan_vertex_count);
glDrawArrays(top_mode, pudding_triangle_vertex_count, pudding_fan_vertex_count);
glUniform1f(uniform["mvp"]["coordinate bound"], configuration()["pudding"]["base-radius"]);
glDrawArrays(GL_TRIANGLE_FAN, pudding_triangle_vertex_count + pudding_fan_vertex_count, pudding_fan_vertex_count);
glDrawArrays(top_mode, pudding_triangle_vertex_count + pudding_fan_vertex_count, pudding_fan_vertex_count);
/* disable squircling for all other drawing */
glUniform1i(uniform["mvp"]["uv transformation"], UV_NONE);
/* regular fill mode enabled for all other drawing */
@ -1182,12 +1254,87 @@ void Pudding::update()
}
SDL_GL_SwapWindow(window());
sb::Log::gl_errors("at end of update");
/* add a new item if a new barcode was scanned or entered */
if (current_barcode != previous_barcode)
/* Launch requests if a new barcode was scanned or entered */
if (camera_switch && current_barcode != previous_barcode)
{
add_item(current_barcode);
previous_barcode = current_barcode;
}
/* Delete and erase finished requests from the vector using iterators to erase while reading the vector */
for (auto iter = requests.begin(); iter != requests.end();)
{
if ((*iter)->finished())
{
std::ostringstream message;
message << "Freeing and removing request object for " << (*iter)->url();
sb::Log::log(message);
/* Free the heap allocated Request */
delete *iter;
/* Get a iterator that points to the next request, which may have been moved after erase */
iter = requests.erase(iter);
}
else
{
/* Only increment the iterator when there was no erase */
iter++;
}
}
/* If requests are finished processing and the incoming item has a texture, add the item to the item list and create a new incoming item. */
if (requests.empty() && incoming_item.texture_count() > 0)
{
std::ostringstream message;
message << "Adding item " << incoming_item.full_name() << " to inventory";
sb::Log::log(message);
items.push_back(incoming_item);
/* Set item index to end so newest item will display. */
item_carousel.end(items);
/* Move the camera button away from center to make room for inventory button if this is the first item added. */
if (items.size() == 1)
{
const nlohmann::json& interface = configuration()["interface"];
camera_button.translation({-1.0f * interface["main-button-double-x"].get<float>(), interface["main-button-y"]});
}
incoming_item = Item();
}
}
Request::Request(const web_callback& callback, const std::string& url) : callback(callback), request_url(url) {}
void Request::store(const std::uint8_t* buffer, const std::size_t& size)
{
response.insert(response.end(), buffer, buffer + size);
std::stringstream store_message;
store_message << "Have " << (response.size() / 100.0) << "KB of data in memory";
sb::Log::log(store_message.str());
}
const std::string& Request::url() const
{
return request_url;
}
void Request::respond()
{
callback(response, url());
mark_finished();
}
void Request::mark_finished()
{
is_finished = true;
}
const bool& Request::finished() const
{
return is_finished;
}
/* Construct a Pad using a texture, a translation, a scale, and a callback function. A Pad is a Plane which can be clicked

183
src/Pudding.hpp
View File

@ -315,6 +315,68 @@ void flag_frame();
void set_heap_offset(int offset);
#endif
/*!
* Type declaration of a function that will accept a vector of bytes for the response data from a web request and a string for the URL.
*/
using web_callback = std::function<void(const std::vector<std::uint8_t>&, const std::string&)>;
/*!
* Store the state, response data, and response function of a request for web data sent to either cURL or Emscripten.
*/
class Request
{
private:
web_callback callback = nullptr;
std::vector<std::uint8_t> response;
bool is_finished = false;
std::string request_url;
public:
/*!
* Construct a request object, specifying a callback that will be passed the complete data in bytes. The callback must therefore accept a
* vector of bytes. The URL of the request can be stored.
*
* @param callback A function object that accepts a vector of bytes.
* @param url URL of the request
*/
Request(const web_callback& callback, const std::string& url = "");
/*!
* Get the URL of the request if it has been specified.
*
* @return The URL of the request or an empty string if the URL was not set
*/
const std::string& url() const;
/*!
* Add the bytes pointed to by buffer to the storage vector.
*/
void store(const std::uint8_t* buffer, const std::size_t& size);
/*!
* Call the user supplied callback and set state to finished.
*/
void respond();
/*!
* Set the finished state to true.
*/
void mark_finished();
/*!
* Check if the request is complete, meaning the data has been stored in memory and the callback has run.
*
* @return true if complete, false otherwise
*/
const bool& finished() const;
};
/*!
* The main game object. There is currently only support for one of these to exist at a time.
*/
class Pudding : public Game
{
@ -343,11 +405,11 @@ private:
/* Constants */
inline static const std::string OPEN_FOOD_API_URL = "https://world.openfoodfacts.org/api/v0/product/";
inline static const std::string OPEN_PRODUCTS_API_URL = "https://world.openproductsfacts.org/api/v0/product/";
inline static const std::string NUTRONIX_API_URL = "https://trackapi.nutritionix.com/v2/search/item?upc=";
inline static const std::string NUTRITIONIX_API_URL = "https://trackapi.nutritionix.com/v2/search/item?upc=";
inline static const std::string BARCODE_MONSTER_API_URL = "https://barcode.monster/api/";
inline static const std::string BEST_BUY_API_URL_1 = "https://api.bestbuy.com/v1/products(upc=";
inline static const std::string BEST_BUY_API_URL_2 = ")?format=json&apiKey=";
inline static const std::string NUTRONIX_NOT_FOUND = "resource not found";
inline static const std::string NUTRITIONIX_NOT_FOUND = "resource not found";
inline static const std::string GOOGLE_BOOKS_API_URL = "https://www.googleapis.com/books/v1/volumes?q=isbn:";
inline static const std::string GIANTBOMB_API_URL = "https://www.giantbomb.com/api/release/?api_key=";
inline static const glm::vec3 ZERO_VECTOR_3D {0, 0, 0};
@ -360,6 +422,7 @@ private:
std::shared_ptr<SDL_Cursor> poke;
std::string current_barcode, previous_barcode, current_config_barcode, current_camera_barcode;
std::vector<Item> items;
Item incoming_item;
Carousel item_carousel;
int effect_id = EFFECT_NONE, pudding_triangle_vertex_count = 0, pudding_fan_vertex_count = 0;
#ifndef __EMSCRIPTEN__
@ -379,12 +442,18 @@ private:
std::map<std::string, sb::Texture> labels;
Pad camera_button, previous_button, next_button, inventory_button;
Box viewport, main_viewport, pop_up_viewport;
std::mutex camera_mutex;
std::vector<Request*> requests;
void load_pudding_model(float, float, int, int = 1, float = -1.0f, float = 1.0f, float = 0.3f);
void load_gl_context();
void load_tiles();
void load_pads();
/*!
* Try to create cv::VideoCapture object using device ID #0. If successful, this will also create a GL texture ID and
* storage for the camera frame on the GPU, so it must be called after GL context has been created. Create and detach
* a thread which will continuously read frame data.
*/
void open_camera();
/*!
@ -392,32 +461,118 @@ private:
*/
void close_camera();
void incorporate_open_api(Item&, const std::string&);
void incorporate_nutronix_api(Item&);
void incorporate_edamam_api(Item&);
void incorporate_best_buy_api(Item&);
void incorporate_google_books_api(Item&);
/*!
* Check the response from Open Food/Products API and use the result to fill out item properties if found. Request the image
* data if an image URL is found.
*
* @param storage JSON as raw bytes fetched from the web, written to a vector
* @param url URL of the request
*/
void incorporate_open_api(const std::vector<std::uint8_t>& json_bytes, const std::string& url);
/*!
* Check the response from Nutritionix API and use the result to fill out item properties if found. Request the image data if an
* image URL is found.
*
* @param storage JSON as raw bytes fetched from the web, written to a vector
* @param url URL of the request
*/
void incorporate_nutritionix_api(const std::vector<std::uint8_t>& json_bytes, const std::string& url);
/*!
* Check the response from Edamame API and use the result to fill out item properties if found. Request the image data if an
* image URL is found.
*
* @param storage JSON as raw bytes fetched from the web, written to a vector
* @param url URL of the request
*/
void incorporate_edamam_api(const std::vector<std::uint8_t>& json_bytes, const std::string& url);
/*!
* Check the response from Best Buy API and use the result to fill out item properties if found. Request image data if an
* image URL is found.
*
* @param storage JSON as raw bytes fetched from the web, written to a vector
* @param url URL of the request
*/
void incorporate_best_buy_api(const std::vector<std::uint8_t>& json_bytes, const std::string& url);
/*!
* Check the response from Google API and use the result to fill out item properties if found. Request image data if an
* image URL is found.
*
* @param storage JSON as raw bytes fetched from the web, written to a vector
* @param url URL of the request
*/
void incorporate_google_books_api(const std::vector<std::uint8_t>& json_bytes, const std::string& url);
void save_item_json(const nlohmann::json&, const Item&, const std::string&) const;
nlohmann::json json_from_url(const std::string& url, const std::vector<std::string>& = {});
void web_get_bytes(const std::string& url, std::vector<std::uint8_t>& storage, const std::vector<std::string>& = {}) const;
sb::Texture texture_from_image_url(const std::string&) const;
/*!
* Fetch data from `url` as raw bytes. The data will be copied into a vector which will be passed to a user supplied function. A request object
* will be added to `launched_requests`. That vector can be checked to determine when all requests are complete.
*
* The compiler will determine whether to use cURL or the Emscripten Fetch API to do the retrieval, depending on whether it is compiling for
* Emscripten.
*
* @param url URL containing data to be retrieved
* @param callback A function pointer for a function that accepts a reference to a vector of bytes and a reference to an Item. The bytes are the
* response data retrieved from `url`. The function can be any arbitrary code that uses the data.
* @param headers Request headers as a vector of strings formatted as ["name1", "value1", "name2", "value2", ...]
*/
void web_get_bytes(std::string url, const web_callback& callback, const std::vector<std::string>& headers = {});
static void destroy_texture(GLuint*);
bool item_display_active() const;
void capture_frame();
/* Define the appropriate callbacks for URL data loaders. Either cURL by default, or Fetch if compiling for Emscripten. */
/*!
* Create a texture to store the image data and add it to the incoming item object.
*
* @param image image data as raw bytes fetched from the web, written to a vector
* @param url image URL which will will be the Texture's name
*/
void store_web_image(const std::vector<std::uint8_t>& image, const std::string& url);
/* Declare the appropriate callbacks for asynchronous web data loaders. Either cURL by default, or Fetch if compiling for Emscripten. */
#if defined(__EMSCRIPTEN__)
/*!
* This will be called automatically when request data is sucessfully fetched by `emscripten_fetch` from `Pudding::web_get_bytes`.
* Data will be written to a vector, and a user supplied callback will be called and passed the data. The callback requested by the
* caller is in fetch->userData.
*
* @param fetch an object created by Emscripten that stores parameters for accessing the downloaded data
*/
static void fetch_success(emscripten_fetch_t* fetch);
/*!
* This will be called automatically when request data is not successfully fetched by `emscripten_fetch` from `Pudding::web_get_bytes`.
*
* @param fetch an object created by Emscripten that stores parameters related to the request
*/
static void fetch_error(emscripten_fetch_t* fetch);
#else
static size_t curl_write_response(std::uint8_t*, size_t, size_t, std::vector<std::uint8_t>*);
/*!
* This will be called by cURL when it has received a buffer of data. The data will be stored by the object at `request`. This may
* be called multiple times before the entire data received from the originally submitted URL is received.
*
* @param buffer pointer to data cURL is transferring into memory
* @param size size in bytes of each value
* @param count number of values
* @param request pointer to a request object which will store the data which will be freed in the update loop
* @return number of bytes copied
*/
static std::size_t curl_write_response(std::uint8_t* buffer, std::size_t size, std::size_t count, Request* request);
#endif
/* Open camera on connection and close on disconnection. */
Connection<> camera_switch {
std::bind(&Pudding::open_camera, this),
std::bind(&Pudding::close_camera, this)
// [&] { capture.release(); }
};
public: