shampoo/gunkiss
shampoo
/
gunkiss
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

working emscripten web export; transfer web cam pixel data from JavaScript to C++; fetch images through Emscripten API and cors-anywhere proxy

This commit is contained in:
ohsqueezy 2022-09-19 22:21:17 -04:00
parent 009e374cd8
commit 5f417a2592
9 changed files with 663 additions and 314 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
Makefile
View File

@ -15,9 +15,9 @@
# `git clone --recursive git.nugget.fun/nugget/gunkiss`. The paths below are the default for the repository, but # `git clone --recursive git.nugget.fun/nugget/gunkiss`. The paths below are the default for the repository, but
# they can be edited as necessary. # they can be edited as necessary.
####################### #########
# Location parameters # # Paths #
####################### #########
# Location of project specific source files # Location of project specific source files
SRC_DIR := src/ SRC_DIR := src/
@ -38,7 +38,7 @@ CXX := clang++
# Location of SDL config program # Location of SDL config program
SDLCONFIG := $(HOME)/local/sdl/bin/sdl2-config SDLCONFIG := $(HOME)/local/sdl/bin/sdl2-config
# Edit to point to the location of BPmono.ttf # Include BPmono.ttf in the project
CREATE_FONT_SYMLINK := ln -nsf $(SB_DIR)"BPmono.ttf" . CREATE_FONT_SYMLINK := ln -nsf $(SB_DIR)"BPmono.ttf" .
############################# #############################
@ -89,7 +89,7 @@ $(SRC_DIR)Pudding.o : $(SRC_H_FILES) $(SB_H_FILES)
# Linux build # # Linux build #
############### ###############
linux : CFLAGS = -g -Wall -Wextra -O0 -c -I$(SB_LIB_DIR) -I$(SB_SRC_DIR) $(SDL_CFLAGS) -I$(HOME)/local/zbar/include \ linux : CFLAGS = -g -Wall -Wextra -O1 -c -I$(SB_LIB_DIR) -I$(SB_SRC_DIR) $(SDL_CFLAGS) -I$(HOME)/local/zbar/include \
-I $(HOME)/local/opencv/include/opencv4 -I $(HOME)/ext/software/emsdk/upstream/emscripten/system/include -I $(HOME)/local/opencv/include/opencv4 -I $(HOME)/ext/software/emsdk/upstream/emscripten/system/include
linux : CXXFLAGS = $(CFLAGS) --std=c++17 linux : CXXFLAGS = $(CFLAGS) --std=c++17
linux : LFLAGS = $(SDL_LFLAGS) -Wl,--enable-new-dtags -lpthread -lGL -lGLESv2 -lSDL2_image -lSDL2_ttf -lSDL2_mixer -lstdc++fs -lcurl \ linux : LFLAGS = $(SDL_LFLAGS) -Wl,--enable-new-dtags -lpthread -lGL -lGLESv2 -lSDL2_image -lSDL2_ttf -lSDL2_mixer -lstdc++fs -lcurl \
@ -108,14 +108,8 @@ linux : $(GLEW_DIR)glew.o $(addprefix $(SDLGFX2_DIR),SDL2_rotozoom.o SDL2_gfxPri
EMSCRIPTENHOME = $(HOME)/ext/software/emsdk/upstream/emscripten EMSCRIPTENHOME = $(HOME)/ext/software/emsdk/upstream/emscripten
EMSCRIPTEN_CFLAGS = -O1 -Wall -s USE_SDL=2 -s USE_SDL_IMAGE=2 -s SDL2_IMAGE_FORMATS="['png', 'jpg']" -s USE_SDL_TTF=2 -s USE_SDL_MIXER=2 \ EMSCRIPTEN_CFLAGS = -O1 -Wall -s USE_SDL=2 -s USE_SDL_IMAGE=2 -s SDL2_IMAGE_FORMATS="['png', 'jpg']" -s USE_SDL_TTF=2 -s USE_SDL_MIXER=2 \
--no-heap-copy -I $(SB_LIB_DIR) -I $(SB_SRC_DIR) -I $(HOME)/local/zbar/include \ --no-heap-copy -I $(SB_LIB_DIR) -I $(SB_SRC_DIR) -I $(HOME)/local/zbar/include -I $(HOME)/local/opencv/include/opencv4
-I $(HOME)/ext/software/opencv-4.6.0/modules/videoio/include/ \ EMSCRIPTEN_LFLAGS = -s MIN_WEBGL_VERSION=2 -s EXPORTED_FUNCTIONS="['_main', '_malloc']" -s ALLOW_MEMORY_GROWTH=1 -s FULL_ES3=1 \
-I $(HOME)/ext/software/opencv-4.6.0/modules/core/include/ \
-I $(HOME)/ext/software/opencv-4.6.0/modules/highgui/include/ \
-I $(HOME)/ext/software/opencv-4.6.0/modules/imgproc/include/ \
-I $(HOME)/ext/software/opencv-4.6.0/modules/imgcodecs/include/ \
-I $(HOME)/ext/software/opencv-4.6.0/build_wasm/
EMSCRIPTEN_LFLAGS = -s MIN_WEBGL_VERSION=2 -s EXPORTED_FUNCTIONS="['_main']" -s ALLOW_MEMORY_GROWTH=1 -s FULL_ES3=1 \
-sLLD_REPORT_UNDEFINED -s FETCH --bind $(wildcard $(addprefix $(HOME)/ext/software/opencv-4.6.0/build_wasm/lib/,*.a)) \ -sLLD_REPORT_UNDEFINED -s FETCH --bind $(wildcard $(addprefix $(HOME)/ext/software/opencv-4.6.0/build_wasm/lib/,*.a)) \
$(HOME)/ext/software/ZBar/zbar/.libs/libzbar.a $(HOME)/ext/software/ZBar/zbar/.libs/libzbar.a
EMSCRIPTEN_PRELOADS = --preload-file "BPmono.ttf"@/ --preload-file "config.json"@/ --preload-file "resource/"@/"resource/" \ EMSCRIPTEN_PRELOADS = --preload-file "BPmono.ttf"@/ --preload-file "config.json"@/ --preload-file "resource/"@/"resource/" \

30
config.json
View File

@ -3,11 +3,11 @@
{ {
"dimensions": [460, 768], "dimensions": [460, 768],
"framerate": 60, "framerate": 60,
"title": "Gunkiss", "title": "Pudding",
"debug": false, "debug": false,
"render driver": "opengles2", "render driver": "opengl",
"show-cursor": true, "show-cursor": true,
"camera-resolution": [1280, 720] "camera-resolution": [320, 240]
}, },
"configuration": "configuration":
@ -21,8 +21,8 @@
"print-frame-length-history": ["CTRL", "SHIFT", "h"], "print-frame-length-history": ["CTRL", "SHIFT", "h"],
"toggle-camera": ["CTRL", "c"], "toggle-camera": ["CTRL", "c"],
"toggle-item": ["CTRL", "i"], "toggle-item": ["CTRL", "i"],
"effect": ["CTRL", "e"], "effect": ["e"],
"tile": ["CTRL", "t"] "tile": ["t"]
}, },
"recording": "recording":
@ -55,27 +55,31 @@
"scan": "scan":
{ {
"enabled": true, "enabled": true,
"json-save": true, "json-save": false,
"json-save-directory": "local/scans", "json-save-directory": "local/scans",
"barcode": "", "barcode": "",
"capture-device": "/dev/video0" "capture-device": "/dev/video0",
"brightness-addition": 10,
"contrast-multiplication": 1.3,
"camera-device-id": 0
}, },
"api": "api":
{ {
"user-agent": "Custom pudding creation game under development for https://shampoo.ooo", "user-agent": "Custom pudding creation game under development at https://mario.shampoo.ooo",
"nutronix-app-id": "ea0f2e7e", "nutritionix-app-id": "ea0f2e7e",
"nutronix-app-key": "39218dde526dd3349daa028deda518ae", "nutritionix-app-key": "39218dde526dd3349daa028deda518ae",
"edamam-app-id": "c23b139f", "edamam-app-id": "c23b139f",
"edamam-app-key": "c54cf8c997534caf7ee92b1ccc7d95a3", "edamam-app-key": "c54cf8c997534caf7ee92b1ccc7d95a3",
"best-buy-api-key": "vAC23XA5YWBzaYiGtOkoNlXZ", "best-buy-api-key": "vAC23XA5YWBzaYiGtOkoNlXZ",
"giantbomb-api-key": "91a395231f4e1fd9f9ba8840c52a61cda343cd70", "giantbomb-api-key": "91a395231f4e1fd9f9ba8840c52a61cda343cd70",
"nutronix-enabled": false, "google-books-api-key": "AIzaSyBD9wXIlBJ6UrXXDIY03k6s0oR1q6ByETQ",
"edamam-enabled": false, "nutritionix-enabled": true,
"edamam-enabled": true,
"open-food-enabled": true, "open-food-enabled": true,
"open-products-enabled": true, "open-products-enabled": true,
"best-buy-enabled": true, "best-buy-enabled": true,
"google-books-enabled": true "google-books-enabled": false
}, },
"pudding": "pudding":

BIN
favicon.ico Normal file
View File

Binary file not shown.
Side by Side

After

Width:  |  Height:  |  Size: 4.2 KiB

128
index.html
View File

@ -1,104 +1,128 @@
<!doctype html> <!doctype html>
<html> <html>
<head>
<style>
body
{
background: #000;
}
</style>
</head>
<body> <body>
<!-- WebGL output will be drawn here through Emscripten --> <!-- WebGL output will be drawn here through Emscripten. The dimensions will be set by Emscripten. -->
<canvas id="canvas"></canvas> <canvas id="canvas"></canvas>
<!-- navigator.mediaDevices.getUserMedia streams the webcam video directly, displayed for testing -->
<!-- <video id="webcam"></video> -->
<script> <script>
const FPS = 15; const FPS = 10;
const BPP = 4; const BPP = 4;
// Direct output of webcam (hidden) /* Direct output of webcam in HTML5 (not displayed) */
var video = document.createElement("video"); var video = document.createElement("video");
video.width = 320; video.width = 320;
video.height = 240; video.height = 240;
// Undisplayed canvas which is used to draw the video frame and access the pixel data directly /* Undisplayed canvas which is used to draw the video frame and access the pixel data directly */
var intermediate = document.createElement("canvas"); var intermediate = document.createElement("canvas");
intermediate.width = video.width; intermediate.width = video.width;
intermediate.height = video.height; intermediate.height = video.height;
var context = intermediate.getContext("2d"); var context = intermediate.getContext("2d");
// Indicates whether webcam is opened or not /* Indicates whether webcam is opened or not */
var streaming = false; var streaming = false;
// Address of the webcam frame pixel data on the Emscripten heap /* Address of the webcam frame pixel data on the Emscripten heap */
var image_heap_address; var image_heap_address;
/* Stores the time when the last webcam frame was processed */
var previous_frame_timestamp;
var Module = { var Module = {
/* When the Module is finished loading, launch the camera frame processor function. */
onRuntimeInitialized: function() onRuntimeInitialized: function()
{ {
process_video(); window.requestAnimationFrame(process_video);
}, },
// Tell Emscripten to use this canvas for display /* Set Emscripten to use a canvas for display. */
canvas: document.getElementById("canvas") canvas: document.getElementById("canvas")
}; };
/*!
* Open the webcam and start displaying frames if successfully opened. Allocate space for 32-bit RGBA frame pixel data
* on the Emscripten heap.
*/
function open_camera() function open_camera()
{ {
// Open the webcam and start displaying frames if successfully opened. Allocate space for 32-bit RGBA frame pixel data navigator.mediaDevices.getUserMedia(
// on the Emscripten heap. {
navigator.mediaDevices.getUserMedia({video: {width: video.width, height: video.height}, audio: false}) video: {
.then(function(stream) { width: video.width,
video.srcObject = stream; height: video.height
video.play(); },
streaming = true; audio: false
})
// Get the memory address of the pixel data .then(function(stream) {
image_heap_address = Module._malloc(video.width * video.height * BPP); video.srcObject = stream;
video.play();
// Pass the address to the C++ program streaming = true;
Module.set_heap_offset(image_heap_address);
}) /* Get the memory address of the pixel data */
.catch(function(err) { image_heap_address = Module._malloc(video.width * video.height * BPP);
console.log('Camera Error: ' + err.name + ' ' + err.message);
}); /* Pass the address to the game object */
Module.set_heap_offset(image_heap_address);
})
.catch(function(err) {
console.log('Camera Error: ' + err.name + ' ' + err.message);
});
} }
function close_camera() function close_camera()
{ {
Module._free
video.pause(); video.pause();
video.srcObject = null; video.srcObject = null;
streaming = false; streaming = false;
} }
// This function will run continuously, drawing the webcam frame to the intermediate canvas, reading the pixel data, /*!
// storing the data on the heap, and setting the new frame available flag. * Run continuously, drawing the webcam frame to the intermediate canvas, reading the pixel data, storing the data on the heap,
function process_video() * and setting the new frame available flag.
*
* @param timestamp time at which the method was called, provided automatically by `window.requestAnimationFrame`
*/
function process_video(timestamp)
{ {
try if (timestamp - previous_frame_timestamp > 1000 / FPS || previous_frame_timestamp == undefined)
{ {
if (streaming) previous_frame_timestamp = timestamp
try
{ {
// Draw the webcam frame on a hidden canvas if (streaming)
context.drawImage(video, 0, 0, video.width, video.height); {
/* Draw the webcam frame on a hidden canvas */
context.drawImage(video, 0, 0, video.width, video.height);
// Read the pixel data /* Read the pixel data */
image_data = context.getImageData(0, 0, video.width, video.height).data; image_data = context.getImageData(0, 0, video.width, video.height).data;
// Get a memory view object that provides access to the heap at the previously allocated address /* Get a memory view object that provides access to the heap at the previously allocated address */
image_heap_data = new Uint8Array(Module.HEAPU8.buffer, image_heap_address, video.width * video.height * BPP); image_heap_data = new Uint8Array(Module.HEAPU8.buffer, image_heap_address, video.width * video.height * BPP);
// Write the pixel data to the heap /* Write the pixel data to the heap */
image_heap_data.set(image_data); image_heap_data.set(image_data);
// Flag the C++ that new data is available /* Flag the game object that new data is available */
Module.flag_frame(); Module.flag_frame();
}
}
catch (err)
{
console.log(err);
} }
// Loop at roughly the FPS
let begin = Date.now();
let delay = 1000/FPS - (Date.now() - begin);
setTimeout(process_video, delay);
}
catch (err)
{
console.log(err);
} }
window.requestAnimationFrame(process_video);
} }
</script> </script>

2
lib/sb

@ -1 +1 @@
Subproject commit b1fb77b1c8a2902fde711ede1a45b459013dc876 Subproject commit 24f6d3ed3d4962a88078c5024473834812968d1a

25
src/Item.cpp
View File

@ -1,10 +1,10 @@
/* _______________ ,-------------------------------------------------. /* _______________ ,-------------------------------------------------.
//`````````````\\ \ \ //`````````````\\ \ \
//~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \ //~~~~~~~~~~~~~~~\\ \ by @ohsqueezy & @sleepin \
//=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \ //=================\\ \ [ohsqueezy.itch.io] [sleepin.itch.io] \
// \\ \ \ // \\ \ \
// \\ \ zlib licensed code at [git.nugget.fun/pudding] \ // \\ \ zlib licensed code at [git.nugget.fun/pudding] \
// ☆ GUNKISS ☆ \\ \ \ // ☆ GUNKISS ☆ \\ \ \
//_________________________\\ `-------------------------------------------------*/ //_________________________\\ `-------------------------------------------------*/
#include "Item.hpp" #include "Item.hpp"
@ -69,6 +69,13 @@ std::string Item::full_name() const
name += " "; name += " ";
} }
name += product_name(); name += product_name();
/* If no names have been set yet, try the UPC as a name */
if (name == "")
{
name = upc();
}
return name; return name;
} }
@ -125,3 +132,9 @@ void Item::to_last()
{ {
carousel.end(item_view.textures()); carousel.end(item_view.textures());
} }
std::ostream& std::operator<<(std::ostream& out, const Item& item)
{
out << item.full_name();
return out;
}

12
src/Item.hpp
View File

@ -68,4 +68,16 @@ public:
}; };
namespace std
{
/*!
* Support passing item objects to the global stream operator.
*
* @param out The output stream
* @param item The item to be printed
* @return The submitted output stream with the text representation of item added
*/
std::ostream& operator<<(std::ostream& out, const Item& item);
}
#endif #endif

577
src/Pudding.cpp
View File

@ -75,7 +75,7 @@ Pudding::Pudding()
void Pudding::load_pudding_model(float top_radius, float base_radius, int ring_vertex_count, int layer_count, float min_y, void Pudding::load_pudding_model(float top_radius, float base_radius, int ring_vertex_count, int layer_count, float min_y,
float max_y, float gradient_position) float max_y, float gradient_position)
{ {
size_t ii; std::size_t ii;
const glm::vec3 *layer_top_color, *layer_bottom_color; const glm::vec3 *layer_top_color, *layer_bottom_color;
const glm::vec2 *start_vertex, *end_vertex; const glm::vec2 *start_vertex, *end_vertex;
float layer_top_y, layer_top_percent, layer_base_y, layer_base_percent, u_step = 1.0f / ring_vertex_count, ring_start_vertex_u; float layer_top_y, layer_top_percent, layer_base_y, layer_base_percent, u_step = 1.0f / ring_vertex_count, ring_start_vertex_u;
@ -284,16 +284,11 @@ void Pudding::load_pads()
next_button.rotation(glm::radians(180.0f)); next_button.rotation(glm::radians(180.0f));
} }
/*!
* Try to create cv::VideoCapture object using device ID #0. If successful, this will also create a GL texture ID and
* storage for the camera frame on the GPU, so it must be called after GL context has been created. Create and detach
* a thread which will continuously read frame data.
*/
void Pudding::open_camera() void Pudding::open_camera()
{ {
#ifndef __EMSCRIPTEN__ #ifndef __EMSCRIPTEN__
/* Open the OpenCV capture, using device ID #0 to get the default attached camera. */ /* Open the OpenCV capture, using device ID #0 to get the default attached camera. */
int device_id = 0; int device_id = configuration()["scan"]["camera-device-id"];
capture.open(device_id); capture.open(device_id);
std::ostringstream message; std::ostringstream message;
if (capture.isOpened()) if (capture.isOpened())
@ -432,6 +427,15 @@ void Pudding::respond(SDL_Event& event)
if (over_camera_button) if (over_camera_button)
{ {
camera_switch.connect(); camera_switch.connect();
#ifndef __EMSCRIPTEN__
/* If the camera did not open, this failed, so unflip the switch */
if (!capture.isOpened())
{
camera_switch.disconnect();
}
#endif
} }
else if (over_inventory_button) else if (over_inventory_button)
{ {
@ -489,151 +493,156 @@ void Pudding::respond(SDL_Event& event)
*/ */
void Pudding::add_item(const std::string& upc) void Pudding::add_item(const std::string& upc)
{ {
Item item; /* Store the UPC code in the incoming item object */
item.upc(upc); incoming_item.upc(upc);
if (configuration()["api"]["open-food-enabled"]) if (configuration()["api"]["open-food-enabled"])
{ {
incorporate_open_api(item, OPEN_FOOD_API_URL); web_get_bytes(OPEN_FOOD_API_URL + upc, std::bind(&Pudding::incorporate_open_api, this, std::placeholders::_1, std::placeholders::_2));
} }
if (configuration()["api"]["open-products-enabled"]) if (configuration()["api"]["open-products-enabled"])
{ {
incorporate_open_api(item, OPEN_PRODUCTS_API_URL); web_get_bytes(OPEN_PRODUCTS_API_URL + upc, std::bind(&Pudding::incorporate_open_api, this, std::placeholders::_1, std::placeholders::_2));
} }
if (configuration()["api"]["nutronix-enabled"]) if (configuration()["api"]["nutritionix-enabled"])
{ {
incorporate_nutronix_api(item); /* Nutritionix requires API keys in headers for validation */
web_get_bytes(NUTRITIONIX_API_URL + upc, std::bind(&Pudding::incorporate_nutritionix_api, this, std::placeholders::_1, std::placeholders::_2), {
"x-app-id", configuration()["api"]["nutritionix-app-id"].get<std::string>(),
"x-app-key", configuration()["api"]["nutritionix-app-key"].get<std::string>()
});
} }
if (configuration()["api"]["edamam-enabled"]) if (configuration()["api"]["edamam-enabled"])
{ {
incorporate_edamam_api(item); /* Build API request by concatenating URL and query string */
std::stringstream url;
url << "https://api.edamam.com/api/food-database/v2/parser?upc=" << upc << "&app_id=" <<
configuration()["api"]["edamam-app-id"].get<std::string>() << "&app_key=" <<
configuration()["api"]["edamam-app-key"].get<std::string>();
web_get_bytes(url.str(), std::bind(&Pudding::incorporate_edamam_api, this, std::placeholders::_1, std::placeholders::_2));
} }
if (configuration()["api"]["best-buy-enabled"]) if (configuration()["api"]["best-buy-enabled"])
{ {
incorporate_best_buy_api(item); /* Build API request by concatenating URL and query string */
std::stringstream url;
url << BEST_BUY_API_URL_1 << upc << BEST_BUY_API_URL_2 << configuration()["api"]["best-buy-api-key"].get<std::string>();
web_get_bytes(url.str(), std::bind(&Pudding::incorporate_best_buy_api, this, std::placeholders::_1, std::placeholders::_2));
} }
if (configuration()["api"]["google-books-enabled"]) if (configuration()["api"]["google-books-enabled"])
{ {
incorporate_google_books_api(item); std::stringstream url;
} url << GOOGLE_BOOKS_API_URL << upc << "&key=" << configuration()["api"]["google-books-api-key"].get<std::string>();
if (item.texture_count() > 0) web_get_bytes(url.str(), std::bind(&Pudding::incorporate_google_books_api, this, std::placeholders::_1, std::placeholders::_2));
{
items.push_back(item);
/* Set item index to end so newest item will display. */
item_carousel.end(items);
/* Move the camera button away from center to make room for inventory button if this is the first item added. */
if (items.size() == 1)
{
const nlohmann::json& interface = configuration()["interface"];
camera_button.translation({-1.0f * interface["main-button-double-x"].get<float>(), interface["main-button-y"]});
}
}
else
{
std::ostringstream message;
message << "discarding item, no images found for " << upc;
sb::Log::log(message);
} }
} }
/* Look for item upc in the Open Food/Products API and use the result to fill out item properties if found. */ void Pudding::incorporate_open_api(const std::vector<std::uint8_t>& json_bytes, const std::string& url)
void Pudding::incorporate_open_api(Item& item, const std::string& api_url)
{ {
std::ostringstream checking_message; std::ostringstream message;
checking_message << "checking " << api_url; message << "Processing " << (json_bytes.size() / 100.0) << "KB from the Open Food/Products API";
sb::Log::log(checking_message); sb::Log::log(message);
nlohmann::json json = json_from_url(api_url + item.upc());
/* test that should determine if an Open Food API response is not empty */ /* Use the nlohmann library to parse the raw JSON byte data retrieved from a web request. */
nlohmann::json json = nlohmann::json::parse(json_bytes);
std::stringstream json_formatted;
json_formatted << std::setw(4) << json << std::endl;
sb::Log::log(json_formatted.str(), sb::Log::DEBUG);
/* Test that should determine if an Open Food API response is not empty */
if (json.value("status", 0) && json.contains("product")) if (json.value("status", 0) && json.contains("product"))
{ {
std::ostringstream message;
if (json["product"].value("image_url", "") != "") if (json["product"].value("image_url", "") != "")
{ {
std::string image_url = json["product"]["image_url"]; std::string image_url = json["product"]["image_url"];
sb::Texture texture = texture_from_image_url(image_url); std::ostringstream message;
if (texture.generated()) message << "Found image URL for item " << incoming_item << " from Open API at " << image_url;
{ sb::Log::log(message);
item.texture(texture, image_url); web_get_bytes(image_url, std::bind(&Pudding::store_web_image, this, std::placeholders::_1, std::placeholders::_2));
}
} }
item.brand_name(json["product"].value("brands", "")); else
item.product_name(json["product"].value("product_name", ""));
if (api_url == OPEN_FOOD_API_URL)
{ {
save_item_json(json, item, "Open_Food_API"); message << "No images found at Open API for " << incoming_item;
}
else if (api_url == OPEN_PRODUCTS_API_URL)
{
save_item_json(json, item, "Open_Products_API");
} }
sb::Log::log(message);
incoming_item.brand_name(json["product"].value("brands", ""));
incoming_item.product_name(json["product"].value("product_name", ""));
save_item_json(json, incoming_item, "Open_Food_and_Products_API");
} }
else else
{ {
std::ostringstream results_message; sb::Log::log("No item found in JSON from Open API");
results_message << "no results from " << api_url;
sb::Log::log(results_message);
} }
} }
/* Look for item upc in the Nutronix API, and use the result to fill out item properties if found void Pudding::incorporate_nutritionix_api(const std::vector<std::uint8_t>& json_bytes, const std::string& url)
*/
void Pudding::incorporate_nutronix_api(Item& item)
{ {
sb::Log::log("checking Nutronix API"); std::ostringstream message;
/* Nutronix requires API keys in headers for validation */ message << "Processing " << (json_bytes.size() / 100.0) << "KB from the Nutritionix API";
nlohmann::json json = json_from_url( sb::Log::log(message);
NUTRONIX_API_URL + item.upc(), {
"x-app-id: " + configuration()["api"]["nutronix-app-id"].get<std::string>(), /* Use the nlohmann library to parse the raw JSON byte data retrieved from a web request. */
"x-app-key: " + configuration()["api"]["nutronix-app-key"].get<std::string>() nlohmann::json json = nlohmann::json::parse(json_bytes);
}); std::stringstream json_formatted;
/* test that should determine if a Nutronix response is not empty */ json_formatted << std::setw(4) << json << std::endl;
if (!(json.contains("message") && json["message"] == NUTRONIX_NOT_FOUND)) sb::Log::log(json_formatted.str(), sb::Log::DEBUG);
/* test that should determine if a Nutritionix response is not empty */
if (!(json.contains("message") && json["message"] == NUTRITIONIX_NOT_FOUND))
{ {
nlohmann::json food = json["foods"][0]; nlohmann::json food = json["foods"][0];
std::ostringstream message;
if (food.contains("photo") && food["photo"].value("thumb", "") != "") if (food.contains("photo") && food["photo"].value("thumb", "") != "")
{ {
std::string url = food["photo"]["thumb"]; std::string image_url = food["photo"]["thumb"];
sb::Log::log("adding image listed in Nutronix API at " + url); message << "Found image URL for item " << incoming_item << " from Nutritionix at " << image_url;
sb::Texture texture = texture_from_image_url(url); web_get_bytes(image_url, std::bind(&Pudding::store_web_image, this, std::placeholders::_1, std::placeholders::_2));
if (texture.generated())
{
item.texture(texture, url);
}
} }
item.brand_name(food.value("brand_name", "")); else
item.product_name(food.value("food_name", "")); {
save_item_json(json, item, "Nutronix_API"); message << "No images found at Nutritionix for " << incoming_item;
}
sb::Log::log(message);
incoming_item.brand_name(food.value("brand_name", ""));
incoming_item.product_name(food.value("food_name", ""));
save_item_json(json, incoming_item, "Nutritionix_API");
} }
else else
{ {
sb::Log::log("no results from Nutronix"); sb::Log::log("no results from Nutritionix");
} }
} }
/* Submit a query to Edamam API and insert relevant results into supplied Item object void Pudding::incorporate_edamam_api(const std::vector<std::uint8_t>& json_bytes, const std::string& url)
*/
void Pudding::incorporate_edamam_api(Item& item)
{ {
sb::Log::log("checking Edamam API"); std::ostringstream message;
/* build API url by concatenating relevant values into query string */ message << "Processing " << (json_bytes.size() / 100.0) << "KB from the Edamam API";
std::stringstream url; sb::Log::log(message);
url << "https://api.edamam.com/api/food-database/v2/parser?upc=" << item.upc() << "&app_id=" <<
configuration()["api"]["edamam-app-id"].get<std::string>() << "&app_key=" << /* Use the nlohmann library to parse the raw JSON byte data retrieved from a web request. */
configuration()["api"]["edamam-app-key"].get<std::string>(); nlohmann::json json = nlohmann::json::parse(json_bytes);
nlohmann::json json = json_from_url(url.str()); std::stringstream json_formatted;
json_formatted << std::setw(4) << json << std::endl;
sb::Log::log(json_formatted.str(), sb::Log::DEBUG);
/* test that should determine if a Edamam response has food data */ /* test that should determine if a Edamam response has food data */
if (json.contains("hints") && json["hints"][0].contains("food")) if (json.contains("hints") && json["hints"][0].contains("food"))
{ {
nlohmann::json food = json["hints"][0]["food"]; nlohmann::json food = json["hints"][0]["food"];
std::ostringstream message;
if (food.value("image", "") != "") if (food.value("image", "") != "")
{ {
std::string url = food["image"]; std::string image_url = food["image"];
sb::Texture texture = texture_from_image_url(url); message << "Found URL to image for item " << incoming_item << " from Edamam at " << image_url;
if (texture.generated()) web_get_bytes(image_url, std::bind(&Pudding::store_web_image, this, std::placeholders::_1, std::placeholders::_2));
{
item.texture(texture, url);
}
item.product_name(food.value("label", ""));
} }
save_item_json(json, item, "Edamam_API"); else
{
message << "No images found at Edamam for " << incoming_item;
}
sb::Log::log(message);
incoming_item.product_name(food.value("label", ""));
save_item_json(json, incoming_item, "Edamam_API");
} }
else else
{ {
@ -641,16 +650,18 @@ void Pudding::incorporate_edamam_api(Item& item)
} }
} }
/* Submit a query to the Best Buy API and insert relevant results into supplied Item object void Pudding::incorporate_best_buy_api(const std::vector<std::uint8_t>& json_bytes, const std::string& url)
*/
void Pudding::incorporate_best_buy_api(Item& item)
{ {
sb::Log::log("checking Best Buy API"); std::ostringstream message;
/* build API url by concatenating relevant values into query string */ message << "Processing " << (json_bytes.size() / 100.0) << "KB from the Best Buy API";
std::stringstream url; sb::Log::log(message);
url << "https://api.bestbuy.com/v1/products(upc=" << item.upc() << ")?format=json&apiKey=" <<
configuration()["api"]["best-buy-api-key"].get<std::string>(); /* Use the nlohmann library to parse the raw JSON byte data retrieved from a web request. */
nlohmann::json json = json_from_url(url.str()); nlohmann::json json = nlohmann::json::parse(json_bytes);
std::stringstream json_formatted;
json_formatted << std::setw(4) << json << std::endl;
sb::Log::log(json_formatted.str(), sb::Log::DEBUG);
/* test that should determine if a Best Buy response has a result */ /* test that should determine if a Best Buy response has a result */
if (json.contains("total") && json["total"].get<int>() > 0) if (json.contains("total") && json["total"].get<int>() > 0)
{ {
@ -658,18 +669,21 @@ void Pudding::incorporate_best_buy_api(Item& item)
/* look up image (for games this is box art) and "alternate views image" (for games this is a screen shot) */ /* look up image (for games this is box art) and "alternate views image" (for games this is a screen shot) */
for (std::string key : {"alternateViewsImage", "image"}) for (std::string key : {"alternateViewsImage", "image"})
{ {
std::ostringstream message;
if (product.value(key, "") != "") if (product.value(key, "") != "")
{ {
std::string url = product[key]; std::string image_url = product[key];
sb::Texture texture = texture_from_image_url(url); message << "Found URL to image for item " << incoming_item << " from Best Buy at " << image_url;
if (texture.generated()) web_get_bytes(image_url, std::bind(&Pudding::store_web_image, this, std::placeholders::_1, std::placeholders::_2));
{
item.texture(texture, url);
}
} }
else
{
message << "No images found at Best Buy for " << incoming_item;
}
sb::Log::log(message);
} }
item.product_name(product.value("name", "")); incoming_item.product_name(product.value("name", ""));
save_item_json(json, item, "Best_Buy_API"); save_item_json(json, incoming_item, "Best_Buy_API");
} }
else else
{ {
@ -677,32 +691,45 @@ void Pudding::incorporate_best_buy_api(Item& item)
} }
} }
/* Look for item upc in the Google Books API and use the result to fill out item properties if found. */ void Pudding::incorporate_google_books_api(const std::vector<std::uint8_t>& json_bytes, const std::string& url)
void Pudding::incorporate_google_books_api(Item& item)
{ {
sb::Log::log("checking Google Books API"); std::ostringstream message;
nlohmann::json json = json_from_url(GOOGLE_BOOKS_API_URL + item.upc()); message << "Processing " << (json_bytes.size() / 100.0) << "KB from the Google Books API";
sb::Log::log(message);
/* Use the nlohmann library to parse the raw JSON byte data retrieved from a web request. */
nlohmann::json json = nlohmann::json::parse(json_bytes);
std::stringstream json_formatted;
json_formatted << std::setw(4) << json << std::endl;
sb::Log::log(json_formatted.str(), sb::Log::DEBUG);
/* test that should determine if a Google Books API response is not empty */ /* test that should determine if a Google Books API response is not empty */
if (json.value<int>("totalItems", 0) > 0 && json.contains("items") && json["items"][0].contains("volumeInfo")) if (json.value<int>("totalItems", 0) > 0 && json.contains("items") && json["items"][0].contains("volumeInfo"))
{ {
/* book specific section of the JSON */ /* book specific section of the JSON */
json = json["items"][0]["volumeInfo"]; json = json["items"][0]["volumeInfo"];
/* get the image data */ /* get the image data */
std::ostringstream message;
if (json.contains("imageLinks") && json["imageLinks"].value("thumbnail", "") != "") if (json.contains("imageLinks") && json["imageLinks"].value("thumbnail", "") != "")
{ {
std::string image_url = json["imageLinks"]["thumbnail"]; std::string image_url = json["imageLinks"]["thumbnail"];
sb::Texture texture = texture_from_image_url(image_url); message << "Found URL to image for item " << incoming_item << " from Google Books at " << image_url;
if (texture.generated()) web_get_bytes(image_url, std::bind(&Pudding::store_web_image, this, std::placeholders::_1, std::placeholders::_2));
{
item.texture(texture, image_url);
}
} }
else
{
message << "No images found at Google Books for " << incoming_item;
}
sb::Log::log(message);
if (json.contains("authors")) if (json.contains("authors"))
{ {
item.brand_name(json["authors"][0]); incoming_item.brand_name(json["authors"][0]);
} }
item.product_name(json.value("title", ""));
save_item_json(json, item, "Google_Books_API"); incoming_item.product_name(json.value("title", ""));
save_item_json(json, incoming_item, "Google_Books_API");
} }
else else
{ {
@ -743,44 +770,51 @@ void Pudding::save_item_json(const nlohmann::json& json, const Item& item, const
} }
} }
/* Download the JSON data at the submitted URL, and return it as a JSON object void Pudding::web_get_bytes(std::string url, const web_callback& callback, const std::vector<std::string>& headers)
*/
nlohmann::json Pudding::json_from_url(const std::string& url, const std::vector<std::string>& headers)
{ {
std::vector<std::uint8_t> storage; std::stringstream message;
web_get_bytes(url, storage, headers); message << "Fetching data from " << url;
nlohmann::json json = nlohmann::json::parse(storage); sb::Log::log(message.str());
std::stringstream json_formatted;
json_formatted << std::setw(4) << json << std::endl; /* Add a request object to the end of the vector of launched requests. */
sb::Log::log(json_formatted.str(), sb::Log::DEBUG); Request* request = new Request(callback, url);
return json; requests.push_back(request);
}
/* Use the CORS anywhere proxy */
url = "https://mario.shampoo.ooo:8088/" + url;
/*!
* Store the bytes retrieved from `url` in the byte vector `storage`.
*
* The compiler will determine whether to use cURL or the Emscripten Fetch API to do the retrieval, depending on whether it is compiling for
* Emscripten.
*
* The optional `headers` parameter will be added to the request when using cURL, but not when using the Emscripten Fetch API.
*
* @param url URL containing data to be retrieved
* @param storage A reference to a vector of bytes which will be filled with the data retrieved from the URL
* @param headers A reference to a vector of strings that should be passed as headers with the request. It is only supported by the cURL version.
*/
void Pudding::web_get_bytes(const std::string& url, std::vector<std::uint8_t>& storage, const std::vector<std::string>& headers) const
{
#if defined(__EMSCRIPTEN__) #if defined(__EMSCRIPTEN__)
/* Create a fetch attributes object. Set a callback that will be called when response data is received. Pass along the user /* Create a fetch attributes object. Set the callback that will be called when response data is received. Attach the user
* storage location to be filled by the callback. */ * submitted callback to the userData attribute. Set the headers. */
emscripten_fetch_attr_t attr; emscripten_fetch_attr_t attr;
emscripten_fetch_attr_init(&attr); emscripten_fetch_attr_init(&attr);
strcpy(attr.requestMethod, "GET"); strcpy(attr.requestMethod, "GET");
attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY; attr.attributes = EMSCRIPTEN_FETCH_LOAD_TO_MEMORY;
attr.onsuccess = fetch_success; attr.onsuccess = fetch_success;
attr.onerror = fetch_error; attr.onerror = fetch_error;
attr.userData = &storage; attr.userData = request;
/* Copy headers into a vector of C strings with null terminator for Emscripten */
if (!headers.empty())
{
std::vector<const char*>* emscripten_formatted_headers = new std::vector<const char*>();
for (const std::string& component : headers)
{
const std::string* component_c = new std::string(component.c_str());
emscripten_formatted_headers->push_back(component_c->c_str());
}
emscripten_formatted_headers->push_back(nullptr);
std::ostringstream message;
message << "Headers are";
for (const char* component : *emscripten_formatted_headers)
{
message << " " << component;
}
sb::Log::log(message);
attr.requestHeaders = emscripten_formatted_headers->data();
}
emscripten_fetch(&attr, url.c_str()); emscripten_fetch(&attr, url.c_str());
#else #else
@ -790,7 +824,7 @@ void Pudding::web_get_bytes(const std::string& url, std::vector<std::uint8_t>& s
result = curl_global_init(CURL_GLOBAL_DEFAULT); result = curl_global_init(CURL_GLOBAL_DEFAULT);
if (result != CURLE_OK) if (result != CURLE_OK)
{ {
std::cout << "curl initialization failed " << curl_easy_strerror(result) << std::endl; std::cout << "cURL initialization failed " << curl_easy_strerror(result) << std::endl;
} }
else else
{ {
@ -798,105 +832,119 @@ void Pudding::web_get_bytes(const std::string& url, std::vector<std::uint8_t>& s
if (curl) if (curl)
{ {
curl_easy_setopt(curl, CURLOPT_URL, url.c_str()); curl_easy_setopt(curl, CURLOPT_URL, url.c_str());
curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, Pudding::curl_write_response); curl_easy_setopt(curl, CURLOPT_WRITEFUNCTION, curl_write_response);
curl_easy_setopt(curl, CURLOPT_WRITEDATA, &storage); curl_easy_setopt(curl, CURLOPT_WRITEDATA, request);
curl_easy_setopt(curl, CURLOPT_USERAGENT, configuration()["api"]["user-agent"].get<std::string>().c_str()); curl_easy_setopt(curl, CURLOPT_USERAGENT, configuration()["api"]["user-agent"].get<std::string>().c_str());
/* Pass submitted headers to cURL */
struct curl_slist* list = nullptr; struct curl_slist* list = nullptr;
if (headers.size() > 0) if (headers.size() > 0)
{ {
for (const std::string& header : headers) /* cURL expects headers as a list of "name: value" pair strings, so combine every two components of the headers list
* into a single string */
for (std::size_t ii = 0; ii < headers.size(); ii += 2)
{ {
list = curl_slist_append(list, header.c_str()); std::string pair = headers[ii] + ": " + headers[ii + 1];
list = curl_slist_append(list, pair.c_str());
} }
} }
curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list); curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
result = curl_easy_perform(curl); result = curl_easy_perform(curl);
curl_slist_free_all(list); curl_slist_free_all(list);
if (result != CURLE_OK) if (result != CURLE_OK)
{ {
std::cout << "curl request failed " << curl_easy_strerror(result) << std::endl; std::cout << "cURL request failed " << curl_easy_strerror(result) << std::endl;
} }
} }
else else
{ {
std::cout << "curl initialization failed" << std::endl; std::cout << "cURL initialization failed" << std::endl;
} }
curl_easy_cleanup(curl); curl_easy_cleanup(curl);
} }
curl_global_cleanup(); curl_global_cleanup();
/* Call the user supplied callback */
request->respond();
#endif #endif
} }
#if defined(__EMSCRIPTEN__) #if defined(__EMSCRIPTEN__)
/*!
* This will be called automatically when request data is sucessfully fetched by `emscripten_fetch` from `Pudding::web_get_bytes`.
* Response bytes will be inserted into the user supplied `std::vector<std::uint8_t>&` at `fetch->userData`.
*/
void Pudding::fetch_success(emscripten_fetch_t* fetch) void Pudding::fetch_success(emscripten_fetch_t* fetch)
{ {
std::vector<std::uint8_t>* storage = reinterpret_cast<std::vector<std::uint8_t>*>(fetch->userData); std::stringstream bytes_message;
storage->insert(storage->end(), fetch->data, fetch->data + fetch->numBytes); bytes_message << "Found " << fetch->numBytes << " bytes using Emscripten Fetch API";
std::stringstream message; sb::Log::log(bytes_message.str());
message << "Stored " << (fetch->numBytes / 100) << "KB of image data in memory from " << fetch->url;
sb::Log::log(message.str()); /* Store the bytes in the request object */
Request* request = reinterpret_cast<Request*>(fetch->userData);
request->store(reinterpret_cast<const std::uint8_t*>(fetch->data), fetch->numBytes);
/* Call the user supplied callback */
request->respond();
emscripten_fetch_close(fetch); emscripten_fetch_close(fetch);
} }
/*!
* This will be called automatically when request data is not successfully fetched by `emscripten_fetch` from `Pudding::web_get_bytes`.
*/
void Pudding::fetch_error(emscripten_fetch_t* fetch) void Pudding::fetch_error(emscripten_fetch_t* fetch)
{ {
std::stringstream message; std::ostringstream message;
message << "Downloading image from " << fetch->url << " failed with status code " << fetch->status; message << "Failed fetching " << fetch->url << " with status code " << fetch->status;
sb::Log::log(message.str()); sb::Log::log(message);
/* Since the request failed, mark it finished */
Request* request = reinterpret_cast<Request*>(fetch->userData);
request->mark_finished();
emscripten_fetch_close(fetch); emscripten_fetch_close(fetch);
} }
#else #else
/*! std::size_t Pudding::curl_write_response(std::uint8_t* buffer, std::size_t size, std::size_t count, Request* request)
* This will be called by cURL when it has received a buffer of data. The data will be inserted into the vector at `storage`
*
* @param buffer pointer to data
* @param size size in bytes of each value
* @param count number of values
* @param storage pointer to a vector of unsigned 8-bit values where the data will be copied to
* @return number of bytes copied
*/
size_t Pudding::curl_write_response(std::uint8_t* buffer, size_t size, size_t count, std::vector<std::uint8_t>* storage)
{ {
size_t total_size = size * count; std::size_t packet_size = size * count;
storage->insert(storage->end(), buffer, buffer + total_size);
return total_size; std::stringstream bytes_message;
bytes_message << "Found " << packet_size << " bytes using cURL ";
sb::Log::log(bytes_message.str());
/* Store the bytes in the request object */
request->store(buffer, packet_size);
return packet_size;
} }
#endif #endif
/* Allocate storage for a texture, copy the cURL response data into the storage, and return the ID that corresponds to the GL texture void Pudding::store_web_image(const std::vector<std::uint8_t>& image, const std::string& url)
*/
sb::Texture Pudding::texture_from_image_url(const std::string& url) const
{ {
/* this texture will be returned whether we load pixels into it or not */ /* Get a Texture by passing the bytes through an RW ops which will enable the Texture object to load a Surface */
sb::Texture texture; sb::Texture texture;
sb::Log::log("looking up image at " + url); SDL_RWops* rw = SDL_RWFromConstMem(image.data(), image.size());
std::vector<std::uint8_t> storage; texture.load(rw);
web_get_bytes(url, storage); SDL_RWclose(rw);
if (!storage.empty()) std::ostringstream message;
sb::Log::Level message_level;
if (texture.generated())
{ {
sb::Log::log("received image data", sb::Log::DEBUG); message << "Loaded an image from " << url << " and attached it to " << incoming_item << " at " << &incoming_item;
/* get a Texture by passing the bytes through an RW ops which will enable the Texture object to load a Surface */ message_level = sb::Log::INFO;
SDL_RWops* rw = SDL_RWFromConstMem(storage.data(), storage.size());
texture.load(rw); /* Use the URL as the name for the texture */
SDL_RWclose(rw); incoming_item.texture(texture, url);
} }
else else
{ {
SDL_LogWarn(SDL_LOG_CATEGORY_CUSTOM, "image url returned no data"); message << "Could not generate texture from " << url;
message_level = sb::Log::WARN;
} }
return texture; sb::Log::log(message, message_level);
} }
/* Call GL's delete texture function, and print a debug statement for testing. This is defined as a static member /* Call GL's delete texture function, and print a debug statement for testing. This is defined as a static member
@ -973,6 +1021,14 @@ void Pudding::capture_frame()
if (!camera_frame.empty()) if (!camera_frame.empty())
{ {
/* Brightness and contrast adjustment, see https://docs.opencv.org/2.4.13.7/doc/tutorials/core/basic_linear_transform/basic_linear_transform.html */
int brightness = configuration()["scan"]["brightness-addition"];
float contrast = configuration()["scan"]["contrast-multiplication"];
if (brightness != 0 || contrast != 1.0)
{
camera_frame.convertTo(camera_frame, -1, contrast, brightness);
}
/* Finished loading into `cv::Mat`, so it is new data that is safe to read. */ /* Finished loading into `cv::Mat`, so it is new data that is safe to read. */
new_frame_available = true; new_frame_available = true;
} }
@ -1006,20 +1062,31 @@ void Pudding::update()
if (new_frame_available) if (new_frame_available)
{ {
sb::Log::log("Hello, World!");
#ifdef __EMSCRIPTEN__ #ifdef __EMSCRIPTEN__
/* Emscripten builds load pixel data into cv::Mat synchronously */ /* Emscripten builds load pixel data into cv::Mat synchronously */
capture_frame(); capture_frame();
/* Pixels from Emscripten are RGBA */
GLenum pixel_format = GL_RGBA;
/* The cv::Mat rows vs. cols (width vs. height) are correct in Emscripten? */
int camera_frame_width = camera_frame.size[0];
int camera_frame_height = camera_frame.size[1];
#else
/* Pixels from cv::VideoCapture are BGR */
GLenum pixel_format = GL_BGR;
/* The cv::Mat rows vs. cols (width vs. height) values are swapped? */
int camera_frame_width = camera_frame.size[1];
int camera_frame_height = camera_frame.size[0];
#endif #endif
camera_view.texture().bind(); camera_view.texture().bind();
/* Fill camera view texture memory with last frame's pixels */ /* Fill camera view texture memory with last frame's pixels */
// camera_view.texture().load(camera_frame.ptr(), {camera_frame.cols, camera_frame.rows}, GL_BGR, GL_UNSIGNED_BYTE); camera_view.texture().load(camera_frame.ptr(), {camera_frame_width, camera_frame_height}, pixel_format, GL_UNSIGNED_BYTE);
// std::cout << camera_frame.size[0] << " " << camera_frame.size[1] << std::endl;
camera_view.texture().load(camera_frame.ptr(), {320, 240}, GL_RGBA, GL_UNSIGNED_BYTE);
/* Frame data has been loaded, so there is not a new frame available anymore. */ /* Frame data has been loaded, so there is not a new frame available anymore. */
new_frame_available = false; new_frame_available = false;
/* Convert to grayscale for ZBar */ /* Convert to grayscale, for ZBar */
cv::cvtColor(camera_frame, camera_frame, cv::COLOR_BGR2GRAY); cv::cvtColor(camera_frame, camera_frame, cv::COLOR_BGR2GRAY);
if (configuration()["scan"]["enabled"]) if (configuration()["scan"]["enabled"])
{ {
@ -1090,15 +1157,20 @@ void Pudding::update()
/* disable bg attributes and enable pudding attributes */ /* disable bg attributes and enable pudding attributes */
background.disable(); background.disable();
pudding_model.attributes("position")->enable(); pudding_model.attributes("position")->enable();
GLenum side_mode, top_mode;
if (items.size() == 0) if (items.size() == 0)
{ {
// glPolygonMode(GL_FRONT_AND_BACK, GL_LINE); // glPolygonMode(GL_FRONT_AND_BACK, GL_LINE);
// pudding_model.attributes("color")->enable(); // pudding_model.attributes("color")->enable();
side_mode = GL_LINES;
top_mode = GL_LINES;
} }
else else
{ {
// glPolygonMode(GL_FRONT_AND_BACK, GL_FILL); // glPolygonMode(GL_FRONT_AND_BACK, GL_FILL);
// pudding_model.attributes("color")->enable(); // pudding_model.attributes("color")->enable();
side_mode = GL_TRIANGLES;
top_mode = GL_TRIANGLE_FAN;
pudding_model.attributes("uv")->enable(); pudding_model.attributes("uv")->enable();
glUniform1i(uniform["mvp"]["pudding texture"], 0); glUniform1i(uniform["mvp"]["pudding texture"], 0);
glActiveTexture(GL_TEXTURE0); glActiveTexture(GL_TEXTURE0);
@ -1107,14 +1179,14 @@ void Pudding::update()
/* draw pudding model */ /* draw pudding model */
glEnable(GL_DEPTH_TEST); glEnable(GL_DEPTH_TEST);
/* draw the sides of the pudding */ /* draw the sides of the pudding */
glDrawArrays(GL_TRIANGLES, 0, pudding_triangle_vertex_count); glDrawArrays(side_mode, 0, pudding_triangle_vertex_count);
sb::Log::gl_errors("after pudding sides, before pudding top/bottom"); sb::Log::gl_errors("after pudding sides, before pudding top/bottom");
/* enable squircling and draw the top and bottom of pudding */ /* enable squircling and draw the top and bottom of pudding */
glUniform1i(uniform["mvp"]["uv transformation"], UV_SQUIRCLE); glUniform1i(uniform["mvp"]["uv transformation"], UV_SQUIRCLE);
glUniform1f(uniform["mvp"]["coordinate bound"], configuration()["pudding"]["top-radius"]); glUniform1f(uniform["mvp"]["coordinate bound"], configuration()["pudding"]["top-radius"]);
glDrawArrays(GL_TRIANGLE_FAN, pudding_triangle_vertex_count, pudding_fan_vertex_count); glDrawArrays(top_mode, pudding_triangle_vertex_count, pudding_fan_vertex_count);
glUniform1f(uniform["mvp"]["coordinate bound"], configuration()["pudding"]["base-radius"]); glUniform1f(uniform["mvp"]["coordinate bound"], configuration()["pudding"]["base-radius"]);
glDrawArrays(GL_TRIANGLE_FAN, pudding_triangle_vertex_count + pudding_fan_vertex_count, pudding_fan_vertex_count); glDrawArrays(top_mode, pudding_triangle_vertex_count + pudding_fan_vertex_count, pudding_fan_vertex_count);
/* disable squircling for all other drawing */ /* disable squircling for all other drawing */
glUniform1i(uniform["mvp"]["uv transformation"], UV_NONE); glUniform1i(uniform["mvp"]["uv transformation"], UV_NONE);
/* regular fill mode enabled for all other drawing */ /* regular fill mode enabled for all other drawing */
@ -1182,12 +1254,87 @@ void Pudding::update()
} }
SDL_GL_SwapWindow(window()); SDL_GL_SwapWindow(window());
sb::Log::gl_errors("at end of update"); sb::Log::gl_errors("at end of update");
/* add a new item if a new barcode was scanned or entered */
if (current_barcode != previous_barcode) /* Launch requests if a new barcode was scanned or entered */
if (camera_switch && current_barcode != previous_barcode)
{ {
add_item(current_barcode); add_item(current_barcode);
previous_barcode = current_barcode; previous_barcode = current_barcode;
} }
/* Delete and erase finished requests from the vector using iterators to erase while reading the vector */
for (auto iter = requests.begin(); iter != requests.end();)
{
if ((*iter)->finished())
{
std::ostringstream message;
message << "Freeing and removing request object for " << (*iter)->url();
sb::Log::log(message);
/* Free the heap allocated Request */
delete *iter;
/* Get a iterator that points to the next request, which may have been moved after erase */
iter = requests.erase(iter);
}
else
{
/* Only increment the iterator when there was no erase */
iter++;
}
}
/* If requests are finished processing and the incoming item has a texture, add the item to the item list and create a new incoming item. */
if (requests.empty() && incoming_item.texture_count() > 0)
{
std::ostringstream message;
message << "Adding item " << incoming_item.full_name() << " to inventory";
sb::Log::log(message);
items.push_back(incoming_item);
/* Set item index to end so newest item will display. */
item_carousel.end(items);
/* Move the camera button away from center to make room for inventory button if this is the first item added. */
if (items.size() == 1)
{
const nlohmann::json& interface = configuration()["interface"];
camera_button.translation({-1.0f * interface["main-button-double-x"].get<float>(), interface["main-button-y"]});
}
incoming_item = Item();
}
}
Request::Request(const web_callback& callback, const std::string& url) : callback(callback), request_url(url) {}
void Request::store(const std::uint8_t* buffer, const std::size_t& size)
{
response.insert(response.end(), buffer, buffer + size);
std::stringstream store_message;
store_message << "Have " << (response.size() / 100.0) << "KB of data in memory";
sb::Log::log(store_message.str());
}
const std::string& Request::url() const
{
return request_url;
}
void Request::respond()
{
callback(response, url());
mark_finished();
}
void Request::mark_finished()
{
is_finished = true;
}
const bool& Request::finished() const
{
return is_finished;
} }
/* Construct a Pad using a texture, a translation, a scale, and a callback function. A Pad is a Plane which can be clicked /* Construct a Pad using a texture, a translation, a scale, and a callback function. A Pad is a Plane which can be clicked

183
src/Pudding.hpp
View File

@ -315,6 +315,68 @@ void flag_frame();
void set_heap_offset(int offset); void set_heap_offset(int offset);
#endif #endif
/*!
* Type declaration of a function that will accept a vector of bytes for the response data from a web request and a string for the URL.
*/
using web_callback = std::function<void(const std::vector<std::uint8_t>&, const std::string&)>;
/*!
* Store the state, response data, and response function of a request for web data sent to either cURL or Emscripten.
*/
class Request
{
private:
web_callback callback = nullptr;
std::vector<std::uint8_t> response;
bool is_finished = false;
std::string request_url;
public:
/*!
* Construct a request object, specifying a callback that will be passed the complete data in bytes. The callback must therefore accept a
* vector of bytes. The URL of the request can be stored.
*
* @param callback A function object that accepts a vector of bytes.
* @param url URL of the request
*/
Request(const web_callback& callback, const std::string& url = "");
/*!
* Get the URL of the request if it has been specified.
*
* @return The URL of the request or an empty string if the URL was not set
*/
const std::string& url() const;
/*!
* Add the bytes pointed to by buffer to the storage vector.
*/
void store(const std::uint8_t* buffer, const std::size_t& size);
/*!
* Call the user supplied callback and set state to finished.
*/
void respond();
/*!
* Set the finished state to true.
*/
void mark_finished();
/*!
* Check if the request is complete, meaning the data has been stored in memory and the callback has run.
*
* @return true if complete, false otherwise
*/
const bool& finished() const;
};
/*!
* The main game object. There is currently only support for one of these to exist at a time.
*/
class Pudding : public Game class Pudding : public Game
{ {
@ -343,11 +405,11 @@ private:
/* Constants */ /* Constants */
inline static const std::string OPEN_FOOD_API_URL = "https://world.openfoodfacts.org/api/v0/product/"; inline static const std::string OPEN_FOOD_API_URL = "https://world.openfoodfacts.org/api/v0/product/";
inline static const std::string OPEN_PRODUCTS_API_URL = "https://world.openproductsfacts.org/api/v0/product/"; inline static const std::string OPEN_PRODUCTS_API_URL = "https://world.openproductsfacts.org/api/v0/product/";
inline static const std::string NUTRONIX_API_URL = "https://trackapi.nutritionix.com/v2/search/item?upc="; inline static const std::string NUTRITIONIX_API_URL = "https://trackapi.nutritionix.com/v2/search/item?upc=";
inline static const std::string BARCODE_MONSTER_API_URL = "https://barcode.monster/api/"; inline static const std::string BARCODE_MONSTER_API_URL = "https://barcode.monster/api/";
inline static const std::string BEST_BUY_API_URL_1 = "https://api.bestbuy.com/v1/products(upc="; inline static const std::string BEST_BUY_API_URL_1 = "https://api.bestbuy.com/v1/products(upc=";
inline static const std::string BEST_BUY_API_URL_2 = ")?format=json&apiKey="; inline static const std::string BEST_BUY_API_URL_2 = ")?format=json&apiKey=";
inline static const std::string NUTRONIX_NOT_FOUND = "resource not found"; inline static const std::string NUTRITIONIX_NOT_FOUND = "resource not found";
inline static const std::string GOOGLE_BOOKS_API_URL = "https://www.googleapis.com/books/v1/volumes?q=isbn:"; inline static const std::string GOOGLE_BOOKS_API_URL = "https://www.googleapis.com/books/v1/volumes?q=isbn:";
inline static const std::string GIANTBOMB_API_URL = "https://www.giantbomb.com/api/release/?api_key="; inline static const std::string GIANTBOMB_API_URL = "https://www.giantbomb.com/api/release/?api_key=";
inline static const glm::vec3 ZERO_VECTOR_3D {0, 0, 0}; inline static const glm::vec3 ZERO_VECTOR_3D {0, 0, 0};
@ -360,6 +422,7 @@ private:
std::shared_ptr<SDL_Cursor> poke; std::shared_ptr<SDL_Cursor> poke;
std::string current_barcode, previous_barcode, current_config_barcode, current_camera_barcode; std::string current_barcode, previous_barcode, current_config_barcode, current_camera_barcode;
std::vector<Item> items; std::vector<Item> items;
Item incoming_item;
Carousel item_carousel; Carousel item_carousel;
int effect_id = EFFECT_NONE, pudding_triangle_vertex_count = 0, pudding_fan_vertex_count = 0; int effect_id = EFFECT_NONE, pudding_triangle_vertex_count = 0, pudding_fan_vertex_count = 0;
#ifndef __EMSCRIPTEN__ #ifndef __EMSCRIPTEN__
@ -379,12 +442,18 @@ private:
std::map<std::string, sb::Texture> labels; std::map<std::string, sb::Texture> labels;
Pad camera_button, previous_button, next_button, inventory_button; Pad camera_button, previous_button, next_button, inventory_button;
Box viewport, main_viewport, pop_up_viewport; Box viewport, main_viewport, pop_up_viewport;
std::mutex camera_mutex; std::vector<Request*> requests;
void load_pudding_model(float, float, int, int = 1, float = -1.0f, float = 1.0f, float = 0.3f); void load_pudding_model(float, float, int, int = 1, float = -1.0f, float = 1.0f, float = 0.3f);
void load_gl_context(); void load_gl_context();
void load_tiles(); void load_tiles();
void load_pads(); void load_pads();
/*!
* Try to create cv::VideoCapture object using device ID #0. If successful, this will also create a GL texture ID and
* storage for the camera frame on the GPU, so it must be called after GL context has been created. Create and detach
* a thread which will continuously read frame data.
*/
void open_camera(); void open_camera();
/*! /*!
@ -392,32 +461,118 @@ private:
*/ */
void close_camera(); void close_camera();
void incorporate_open_api(Item&, const std::string&); /*!
void incorporate_nutronix_api(Item&); * Check the response from Open Food/Products API and use the result to fill out item properties if found. Request the image
void incorporate_edamam_api(Item&); * data if an image URL is found.
void incorporate_best_buy_api(Item&); *
void incorporate_google_books_api(Item&); * @param storage JSON as raw bytes fetched from the web, written to a vector
* @param url URL of the request
*/
void incorporate_open_api(const std::vector<std::uint8_t>& json_bytes, const std::string& url);
/*!
* Check the response from Nutritionix API and use the result to fill out item properties if found. Request the image data if an
* image URL is found.
*
* @param storage JSON as raw bytes fetched from the web, written to a vector
* @param url URL of the request
*/
void incorporate_nutritionix_api(const std::vector<std::uint8_t>& json_bytes, const std::string& url);
/*!
* Check the response from Edamame API and use the result to fill out item properties if found. Request the image data if an
* image URL is found.
*
* @param storage JSON as raw bytes fetched from the web, written to a vector
* @param url URL of the request
*/
void incorporate_edamam_api(const std::vector<std::uint8_t>& json_bytes, const std::string& url);
/*!
* Check the response from Best Buy API and use the result to fill out item properties if found. Request image data if an
* image URL is found.
*
* @param storage JSON as raw bytes fetched from the web, written to a vector
* @param url URL of the request
*/
void incorporate_best_buy_api(const std::vector<std::uint8_t>& json_bytes, const std::string& url);
/*!
* Check the response from Google API and use the result to fill out item properties if found. Request image data if an
* image URL is found.
*
* @param storage JSON as raw bytes fetched from the web, written to a vector
* @param url URL of the request
*/
void incorporate_google_books_api(const std::vector<std::uint8_t>& json_bytes, const std::string& url);
void save_item_json(const nlohmann::json&, const Item&, const std::string&) const; void save_item_json(const nlohmann::json&, const Item&, const std::string&) const;
nlohmann::json json_from_url(const std::string& url, const std::vector<std::string>& = {});
void web_get_bytes(const std::string& url, std::vector<std::uint8_t>& storage, const std::vector<std::string>& = {}) const; /*!
sb::Texture texture_from_image_url(const std::string&) const; * Fetch data from `url` as raw bytes. The data will be copied into a vector which will be passed to a user supplied function. A request object
* will be added to `launched_requests`. That vector can be checked to determine when all requests are complete.
*
* The compiler will determine whether to use cURL or the Emscripten Fetch API to do the retrieval, depending on whether it is compiling for
* Emscripten.
*
* @param url URL containing data to be retrieved
* @param callback A function pointer for a function that accepts a reference to a vector of bytes and a reference to an Item. The bytes are the
* response data retrieved from `url`. The function can be any arbitrary code that uses the data.
* @param headers Request headers as a vector of strings formatted as ["name1", "value1", "name2", "value2", ...]
*/
void web_get_bytes(std::string url, const web_callback& callback, const std::vector<std::string>& headers = {});
static void destroy_texture(GLuint*); static void destroy_texture(GLuint*);
bool item_display_active() const; bool item_display_active() const;
void capture_frame(); void capture_frame();
/* Define the appropriate callbacks for URL data loaders. Either cURL by default, or Fetch if compiling for Emscripten. */ /*!
* Create a texture to store the image data and add it to the incoming item object.
*
* @param image image data as raw bytes fetched from the web, written to a vector
* @param url image URL which will will be the Texture's name
*/
void store_web_image(const std::vector<std::uint8_t>& image, const std::string& url);
/* Declare the appropriate callbacks for asynchronous web data loaders. Either cURL by default, or Fetch if compiling for Emscripten. */
#if defined(__EMSCRIPTEN__) #if defined(__EMSCRIPTEN__)
/*!
* This will be called automatically when request data is sucessfully fetched by `emscripten_fetch` from `Pudding::web_get_bytes`.
* Data will be written to a vector, and a user supplied callback will be called and passed the data. The callback requested by the
* caller is in fetch->userData.
*
* @param fetch an object created by Emscripten that stores parameters for accessing the downloaded data
*/
static void fetch_success(emscripten_fetch_t* fetch); static void fetch_success(emscripten_fetch_t* fetch);
/*!
* This will be called automatically when request data is not successfully fetched by `emscripten_fetch` from `Pudding::web_get_bytes`.
*
* @param fetch an object created by Emscripten that stores parameters related to the request
*/
static void fetch_error(emscripten_fetch_t* fetch); static void fetch_error(emscripten_fetch_t* fetch);
#else #else
static size_t curl_write_response(std::uint8_t*, size_t, size_t, std::vector<std::uint8_t>*);
/*!
* This will be called by cURL when it has received a buffer of data. The data will be stored by the object at `request`. This may
* be called multiple times before the entire data received from the originally submitted URL is received.
*
* @param buffer pointer to data cURL is transferring into memory
* @param size size in bytes of each value
* @param count number of values
* @param request pointer to a request object which will store the data which will be freed in the update loop
* @return number of bytes copied
*/
static std::size_t curl_write_response(std::uint8_t* buffer, std::size_t size, std::size_t count, Request* request);
#endif #endif
/* Open camera on connection and close on disconnection. */ /* Open camera on connection and close on disconnection. */
Connection<> camera_switch { Connection<> camera_switch {
std::bind(&Pudding::open_camera, this), std::bind(&Pudding::open_camera, this),
std::bind(&Pudding::close_camera, this) std::bind(&Pudding::close_camera, this)
// [&] { capture.release(); }
}; };
public: public: