// Cloned by Daniel Peres on 22 Nov 2019 from World "Character recognition neural network" by "Coding Train" project
// Please leave this clone trail here.
// Port of Character recognition neural network from here:
// https://github.com/CodingTrain/Toy-Neural-Network-JS/tree/master/examples/mnist
// with many modifications
//#DANIEL: There are two working approaches:
// The first (useBestSlowApproach == false) will transform ALL images (training, test, demo, doodle, upload) in the following
// way: Crop the drawn area and resize it to 28by28px, without aspect ratio and border. It works very weel!
// The second (useBestSlowApproach == false) will transform the DOODLE (and UPLOAD) in the same format than the
// MNIST images. Crop the drawn area, resize keeping aspect ratio to 20by20px, put centred into a 28by28px image.
// The training, demo, test used are all original. We just change the doodle and upload to match them.
// --- defined by MNIST - do not change these ---------------------------------------
const PIXELS = 28; // images in data set are tiny
const PIXELSSQUARED = PIXELS * PIXELS;
// number of training and test exemplars in the data set:
const NOTRAIN = 60000;
const NOTEST = 10000;
const LETBORDER = 0;
//--- can modify all these --------------------------------------------------
// no of nodes in network
const noinput = PIXELSSQUARED;
const nohidden = 64;
const nooutput = 10;
const learningrate = 0.1; // default 0.1
// should we train every timestep or not
let do_training = true;
// how many to train and test per timestep
const TRAINPERSTEP = 30;
const TESTPERSTEP = 5;
// multiply it by this to magnify for display
const ZOOMFACTOR = 7;
const ZOOMPIXELS = ZOOMFACTOR * PIXELS;
// 3 rows of
// large image + 50 gap + small image
// 50 gap between rows
const canvaswidth = (PIXELS + ZOOMPIXELS) + 50;
const canvasheight = (ZOOMPIXELS * 4) + 100;
const DOODLE_THICK = 18; // thickness of doodle lines
const DOODLE_BLUR = 3; // blur factor applied to doodles
let mnist;
// all data is loaded into this
// mnist.train_images
// mnist.train_labels
// mnist.test_images
// mnist.test_labels
let nn;
let trainrun = 1;
let train_index = 0;
let testrun = 1;
let test_index = 0;
let total_tests = 0;
let total_correct = 0;
// show the accuracy by input. [input][total][correct]
let totalCorrectByInput = [];
// images in LHS:
let doodle, demo;
let doodle_exists = false;
let demo_exists = false;
let mousedrag = false; // are we in the middle of a mouse drag drawing?
// save inputs to global var to inspect
// type these names in console
var train_inputs, test_inputs, demo_inputs, doodle_inputs;
//#DANIEL: You can select which approach you want to test. Both are available.
var useBestSlowApproach = false;
//#DANIEL: If useVariableLearningRate is true, it give higher learning rate to input with low accuracy, and lower learning
//#DANIEL: rate to inputs with better accuracy. Didn’t do much difference to be honest.
var useVariableLearningRate = false;
// Matrix.randomize() is changed to point to this. Must be defined by user of Matrix.
function randomWeight() {
return (AB.randomFloatAtoB(-0.5, 0.5));
// Coding Train default is -1 to 1
}
// CSS trick
// make run header bigger
$("#runheaderbox").css({ "max-height": "95vh" });
//--- start of AB.msgs structure: ---------------------------------------------------------
// We output a serious of AB.msgs to put data at various places in the run header
var thehtml;
// 1 Doodle header
thehtml = "<hr> <h1> 1. Doodle </h1> Top row: Doodle (left) and shrunk (right). <br> " +
" Draw your doodle in top LHS. <button onclick='wipeDoodle();' class='normbutton' >Clear doodle</button> <br> ";
AB.msg(thehtml, 1);
// 2 Doodle variable data (guess)
// 3 Training header
thehtml = "<hr> <h1> 2. Training </h1> Middle row: Training image magnified (left) and original (right). <br> " +
" <button onclick='do_training = false;' class='normbutton' >Stop training</button> <br> ";
AB.msg(thehtml, 3);
// 4 variable training data
// 5 Testing header
thehtml = "<h3> Hidden tests </h3> ";
AB.msg(thehtml, 5);
// 6 variable testing data
// 7 Demo header
thehtml = "<hr> <h1> 3. Demo </h1> Bottom row: Test image magnified (left) and original (right). <br>" +
" The network is <i>not</i> trained on any of these images. <br> " +
" <button onclick='makeDemo();' class='normbutton' >Demo test image</button> <br> ";
AB.msg(thehtml, 7);
// 8 Demo variable data (random demo ID)
// 9 Demo variable data (changing guess)
// 13 Upload header
thehtml = "<hr> <h1> 4. Upload </h1> <br/> Test the uploaded image <br/>";
AB.msg(thehtml, 13);
AB.msg("<input type='file' onchange='handleFileUpload(this.files)' id='imageUpload' style='position: absolute; left: 0px; top: 0px;'>", 14);
const greenspan = "<span style='font-weight:bold; font-size:x-large; color:darkgreen'> ";
//--- end of AB.msgs structure: ---------------------------------------------------------
var uploadImageContent;
//#DANIEL Put the uploaded image on uploadImageContent variable.
function handleFileUpload(event) {
const reader = new FileReader()
reader.onload = function handleFileLoad(event) {
uploadImageContent = event.target.result;
};
reader.readAsDataURL(event[0])
//Set uploadImage as undefined, to be loaded again.
uploadImage = undefined;
}
//#DANIEL: Class that holds the test statistics by input
// Used to get the global accuracy by input
function Accuracy() {
var input;
var total;
var correct;
var accuracy;
}
function setup() {
//Init totalCorrectByInput to calculate the accuracy for every input
let i = 0;
totalCorrectByInput = [];
for (i = 0; i < 10; i++) {
//Input, Total, Corrects
totalCorrectByInput[i] = new Accuracy();
totalCorrectByInput[i].input = i;
totalCorrectByInput[i].total = 0;
totalCorrectByInput[i].correct = 0;
}
createCanvas(canvaswidth, canvasheight);
doodle = createGraphics(ZOOMPIXELS, ZOOMPIXELS); // doodle on larger canvas
doodle.pixelDensity(1);
doodle.background('black');
// JS load other JS
// maybe have a loading screen while loading the JS and the data set
AB.loadingScreen();
$.getScript("/uploads/codingtrain/matrix.js", function () {
$.getScript("/uploads/codingtrain/nn.js", function () {
$.getScript("/uploads/codingtrain/mnist.js", function () {
console.log("All JS loaded");
nn = new NeuralNetwork(noinput, nohidden, nooutput);
nn.setLearningRate(learningrate);
loadData();
});
});
});
}
// load data set from local file (on this server)
function loadData() {
loadMNIST(function (data) {
mnist = data;
console.log("All data loaded into mnist object:")
console.log(mnist);
AB.removeLoading(); // if no loading screen exists, this does nothing
});
}
function getImage(img) // make a P5 image object from a raw data array
{
let theimage = createImage(PIXELS, PIXELS); // make blank image, then populate it
theimage.loadPixels();
for (let i = 0; i < PIXELSSQUARED; i++) {
let bright = img[i];
let index = i * 4;
theimage.pixels[index + 0] = bright;
theimage.pixels[index + 1] = bright;
theimage.pixels[index + 2] = bright;
theimage.pixels[index + 3] = 255;
}
theimage.updatePixels();
return theimage;
}
function getInputs(img) // convert img array into normalised input array
{
let inputs = [];
for (let i = 0; i < PIXELSSQUARED; i++) {
let bright = img[i];
inputs[i] = bright / 255; // normalise to 0 to 1
}
return (inputs);
}
//#DANIEL: Pre-existing function.
// If useBestSlowApproach is true, it transform the train image, crop the drawn part of the image and resize it to 28by28px.
// In this approach ALL images are transformed, even the training set. (That’s the reason of the slowness)
function trainit(show)
{
let img = mnist.train_images[train_index];
let label = mnist.train_labels[train_index];
let inputs = [];
if (useBestSlowApproach){
//#DANIEL: Crop image to get just the drawn area
var usedImage = cropImage(getImage(img), LETBORDER);
//#DANIEL: Resize it to 28x28 pixels
usedImage.resize(PIXELS, PIXELS);
//#DANIEL: Get the inputs from image
inputs = getInputsFromImage(usedImage);
}else{
inputs = getInputs(img); // get inputs from data array
}
// optional - show visual of the image
if (show) {
var theimage = getImage(img); // get image from data array
image(theimage, 0, ZOOMPIXELS + 50, ZOOMPIXELS, ZOOMPIXELS); // magnified
image(theimage, ZOOMPIXELS + 50, ZOOMPIXELS + 50, PIXELS, PIXELS); // original
}
// set up the outputs
let targets = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
targets[label] = 1; // change one output location to 1, the rest stay at 0
// console.log(train_index);
// console.log(inputs);
// console.log(targets);
train_inputs = inputs; // can inspect in console
//#DANIEL: If useVariableLearningRate == true, it will decrease the learning rate for results with good accuracy
// and increase learning rate for results with bad accuracy.
// If it is false, will always use the default learning rate
if (useVariableLearningRate){
let generalAccuracy = (total_correct / total_tests) * 100;
if (!generalAccuracy || generalAccuracy <= 60){
nn.setLearningRate(learningrate);
}else{
let accuracyThisLabel = totalCorrectByInput[label].accuracy;
if (!accuracyThisLabel || accuracyThisLabel <= 60) {
nn.setLearningRate(learningrate);
} else {
let thisLearningRate = learningrate * ((1 - ( Math.pow((accuracyThisLabel/100),8)) ) + 0.05);
nn.setLearningRate(thisLearningRate);
}
}
}
nn.train(inputs, targets);
thehtml = " trainrun: " + trainrun + "<br> no: " + train_index;
AB.msg(thehtml, 4);
train_index++;
if (train_index == NOTRAIN) {
train_index = 0;
do_training = false;
console.log("finished trainrun: " + trainrun);
trainrun++;
}
}
function testit() // test the network with a single exemplar, from global var "test_index"
{
let img = mnist.test_images[test_index];
let label = mnist.test_labels[test_index];
let inputs = [];
if (useBestSlowApproach){
//#DANIEL: Crop image to get just the drawn area
var usedImage = cropImage(getImage(img), LETBORDER);
//#DANIEL: Resize it to 28x28 pixels
usedImage.resize(PIXELS, PIXELS);
//#DANIEL: Get the inputs from image
inputs = getInputsFromImage(usedImage);
}else{
inputs = getInputs(img);
}
test_inputs = inputs; // can inspect in console
let prediction = nn.predict(inputs); // array of outputs
let guess = findMax(prediction); // the top output
total_tests++;
//Hold the correct values by input
totalCorrectByInput[label].total++;
if (guess == label)
totalCorrectByInput[label].correct++;
if (guess == label) total_correct++;
let percent = (total_correct / total_tests) * 100;
thehtml = " testrun: " + testrun + "<br> no: " + total_tests + " <br> " +
" correct: " + total_correct + "<br>" +
" score: " + greenspan + percent.toFixed(2) + "</span>";
thehtml += "<br/><h3>Accuracy by input</h3>"
let i = 0;
//#DANIEL: Calcuate the accuracy for every input, not just the global accuracy.
for (i = 0; i < totalCorrectByInput.length; i++) {
totalCorrectByInput[i].accuracy = ((totalCorrectByInput[i].correct / totalCorrectByInput[i].total) * 100);
}
var totalCorrectByInputSorted = totalCorrectByInput.slice();
totalCorrectByInputSorted.sort(function (a, b) {
return b.accuracy - a.accuracy;
});
for (i = 0; i < totalCorrectByInputSorted.length; i++) {
thehtml += totalCorrectByInputSorted[i].input + " : " + totalCorrectByInputSorted[i].accuracy.toFixed(2) + "%" + " out of " + totalCorrectByInputSorted[i].total + " tests<br/>";
}
AB.msg(thehtml, 6);
test_index++;
if (test_index == NOTEST) {
console.log("finished testrun: " + testrun + " score: " + percent.toFixed(2));
testrun++;
do_training = false;
test_index = 0;
total_tests = 0;
total_correct = 0;
}
}
//--- find no.1 (and maybe no.2) output nodes ---------------------------------------
// (restriction) assumes array values start at 0 (which is true for output nodes)
//#DANIEL: Sort the prediction, and returns an array with Input and Accuracy, sorted.
// [{0,80%},{1,10%},{2,9%}…]
function findAllPredictionsSorted(prediction) // return array showing indexes of no.1 and no.2 values in array
{
let i = 0;
var predictionAndValues = [];
for (i = 0; i < prediction.length; i++) {
predictionAndValues[i] = [i, ((prediction[i] * 100).toFixed(2))];
}
//Sort by accuracy
predictionAndValues.sort(function (a, b) {
return b[1] - a[1];
});
return predictionAndValues;
}
// just get the maximum - separate function for speed - done many times
// find our guess - the max of the output nodes array
function findMax(a) {
let no1 = 0;
let no1value = 0;
for (let i = 0; i < a.length; i++) {
if (a[i] > no1value) {
no1 = i;
no1value = a[i];
}
}
return no1;
}
// --- the draw function -------------------------------------------------------------
// every step:
function draw() {
// check if libraries and data loaded yet:
if (typeof mnist == 'undefined') return;
// how can we get white doodle on black background on yellow canvas?
// background('#ffffcc'); doodle.background('black');
background('black');
//#DANIEL: Changed to stop the learning if the mouse is pressed.
// Make it easier to draw and see statistics
if (do_training && !mouseIsPressed) {
// do some training per step
for (let i = 0; i < TRAINPERSTEP; i++) {
if (i == 0) trainit(true); // show only one per step - still flashes by
else trainit(false);
}
// do some testing per step
for (let i = 0; i < TESTPERSTEP; i++)
testit();
}
// keep drawing demo and doodle images
// and keep guessing - we will update our guess as time goes on
if (demo_exists) {
drawDemo();
guessDemo();
}
if (doodle_exists) {
drawDoodle();
guessDoodle();
}
//#DANIEL: Draw and guess the uploaded image
//Can be the existing image previously loaded, or a new image data
if (uploadImageContent || uploadImage){
drawUpload();
guessUpload();
}
// detect doodle drawing
// (restriction) the following assumes doodle starts at 0,0
if (mouseIsPressed) // gets called when we click buttons, as well as if in doodle corner
{
// console.log ( mouseX + " " + mouseY + " " + pmouseX + " " + pmouseY );
var MAX = ZOOMPIXELS + 20; // can draw up to this pixels in corner
if ((mouseX < MAX) && (mouseY < MAX) && (pmouseX < MAX) && (pmouseY < MAX)) {
mousedrag = true; // start a mouse drag
doodle_exists = true;
doodle.stroke('white');
doodle.strokeWeight(DOODLE_THICK);
doodle.line(mouseX, mouseY, pmouseX, pmouseY);
}
}
else {
// are we exiting a drawing
if (mousedrag) {
mousedrag = false;
// console.log ("Exiting draw. Now blurring.");
doodle.filter(BLUR, DOODLE_BLUR); // just blur once
// console.log (doodle);
}
}
}
//--- demo -------------------------------------------------------------
// demo some test image and predict it
// get it from test set so have not used it in training
function makeDemo() {
demo_exists = true;
var i = AB.randomIntAtoB(0, NOTEST - 1);
demo = mnist.test_images[i];
var label = mnist.test_labels[i];
thehtml = "Test image no: " + i + "<br>" +
"Classification: " + label + "<br>";
AB.msg(thehtml, 8);
// type "demo" in console to see raw data
}
function drawDemo() {
var theimage = getImage(demo);
// console.log (theimage);
image(theimage, 0, canvasheight - (2 * ZOOMPIXELS), ZOOMPIXELS, ZOOMPIXELS); // magnified
image(theimage, ZOOMPIXELS + 50, canvasheight - (2 * ZOOMPIXELS), PIXELS, PIXELS); // original
}
function guessDemo() {
let inputs = [];
if (useBestSlowApproach){
var usedImage = cropImage(getImage(demo), LETBORDER);
usedImage.resize(PIXELS, PIXELS);
inputs = getInputsFromImage(usedImage);
}else{
inputs = getInputs(demo);
}
demo_inputs = inputs; // can inspect in console
let prediction = nn.predict(inputs); // array of outputs
let guess = findMax(prediction); // the top output
thehtml = " We classify it as: " + greenspan + guess + "</span>";
AB.msg(thehtml, 9);
}
//--- doodle -------------------------------------------------------------
function drawDoodle() {
// doodle is createGraphics not createImage
theimage = doodle.get();
// console.log (theimage);
image(theimage, 0, 0, ZOOMPIXELS, ZOOMPIXELS); // original
image(theimage, ZOOMPIXELS + 50, 0, PIXELS, PIXELS); // shrunk
}
//#DANIEL: Draw the uploaded image, on the last part of the canvas.
var uploadImage,uploadImageResized;
function drawUpload() {
if (!uploadImage && !uploadImageContent)
return;
//It is a new image. Load again on uploadImage
if (!uploadImage && uploadImageContent) {
var baseString = uploadImageContent.trim();
if(baseString.substring(0,4) != "data"){
baseString = "data:image/png;base64," + baseString;
}
uploadImageContent = baseString;
var raw = new Image();
raw.src = uploadImageContent; // base64 data here
raw.onload = function () {
uploadImage = createImage(raw.width, raw.height);
uploadImage.drawingContext.drawImage(raw, 0, 0);
uploadImage.loadPixels();
//uploadImageResized and uploadImage are the same.
//I just dont want to resize uploadImage to appear with the original ratio on the canvas
uploadImageResized = createImage(raw.width, raw.height);
uploadImageResized.drawingContext.drawImage(raw, 0, 0);
uploadImageResized.loadPixels();
}
uploadImageContent = undefined;
}
//Doesn't matter if it is new or the pre-existing. Draw again
if (uploadImage){
image(uploadImage, 0, canvasheight - ZOOMPIXELS, ZOOMPIXELS, ZOOMPIXELS); // magnified
image(uploadImage, ZOOMPIXELS + 50, canvasheight - ZOOMPIXELS, PIXELS, PIXELS); // original
}
}
//#DANIEL: Resize the image to the size passed as argument, keeping the aspect ratio of
// the higher dimension – height or width.
function resizeKeepingAspectRatio(image, resizeTo){
//img is taller than widther. resize using the height
if (image.height > image.width){
image.resize(0,resizeTo);
}else{
//or use width to rezise
image.resize(resizeTo,0);
}
return image;
}
//#DANIEL: This function is the core of the Approach 1. Given an image (doodle or uploaded image), it will crop it (let just the drawn part).
//Resize the drawn image to 20by20px keeping the aspect ratio, and put this image centred on a 28by28px image
//– exactly as the MNIST does. */
function getImageOnMnistFormat(img){
img.loadPixels();
//Crops the image. Let just the drawn part of the image
let usedImage = cropImage(img, LETBORDER);
//resize it to 20x20, based on bigger dimension, and keeping aspect ratio
let resizedImage2020 = resizeKeepingAspectRatio(usedImage, 20);
//create centered image 28 x 28
let borderX = (PIXELS - resizedImage2020.width) / 2;
let borderY = (PIXELS - resizedImage2020.height) / 2;
let theimage = createImage(PIXELS, PIXELS); // make blank image, then populate it
theimage.loadPixels();
for (let i = 0; i < PIXELS; i++) {
for (let j = 0; j < PIXELS; j++) {
let bright = 0;
//if pixel are in border (empty space), let it be 0;
//else, get value from original image
if (i < borderX || j < borderY)
bright = [0,0,0,255]; //left and upper border
else if (i > borderX + resizedImage2020.width || j > borderY + resizedImage2020.height)
bright = [0,0,0,255]; //bottom and right border
else {
//finally, get value from matrix
bright = resizedImage2020.get(i-borderX,j-borderY);
}
//set the bright for the image
theimage.set(i,j, bright);
};
}
theimage.updatePixels();
return theimage;
}
function guessDoodle() {
// doodle is createGraphics not createImage
let imgDoodle = doodle.get();
if (useBestSlowApproach){
//imgDoodle.resize(PIXELS, PIXELS); //TODO: IS IT WORST?
var usedImage = cropImage(imgDoodle, LETBORDER);
usedImage.resize(PIXELS, PIXELS);
inputs = getInputsFromImage(usedImage);
}else{
//get 28x28 image, with 20x20 centered used image
let img = getImageOnMnistFormat(imgDoodle);
inputs = getInputsFromImage(img);
}
doodle_inputs = inputs; // can inspect in console
// feed forward to make prediction
let prediction = nn.predict(inputs); // array of outputs
//Return the classification, and %, in order
let predictionAndValuesOrdered = findAllPredictionsSorted(prediction); // get no.1 and no.2 guesses
let i = 0;
thehtml = " Best guess is " + greenspan + predictionAndValuesOrdered[0][0] + "</span>" + " and the probabilities are: <br/>";
for (i = 0; i < predictionAndValuesOrdered.length; i++) {
thehtml += predictionAndValuesOrdered[i][0] + " : " + predictionAndValuesOrdered[i][1] + "%" + "<br/>";
}
thehtml + + "</span>";
// thehtml = " We classify it as: " + greenspan + predictionAndValuesOrdered[0] + "</span> <br>" +
// " No.2 guess is: " + greenspan + predictionAndValuesOrdered[1] + "</span>";
AB.msg(thehtml, 2);
}
function guessUpload() {
if (!uploadImage)
return;
// doodle is createGraphics not createImage
let inputs = [];
if (useBestSlowApproach){
let img = uploadImageResized;
//img.resize(PIXELS, PIXELS); //TODO: IS IT WORST?
var usedImage = cropImage(img, LETBORDER);
usedImage.resize(PIXELS, PIXELS);
inputs = getInputsFromImage(usedImage);
}else{
//get 28x28 image, with 20x20 centered used image
let img = getImageOnMnistFormat(uploadImageResized);
inputs = getInputsFromImage(img);
}
// feed forward to make prediction
let prediction = nn.predict(inputs); // array of outputs
//Return the classification, and %, in order
let predictionAndValuesOrdered = findAllPredictionsSorted(prediction); // get no.1 and no.2 guesses
let i = 0;
thehtml = " Best guess is " + greenspan + predictionAndValuesOrdered[0][0] + "</span>" + " and the probabilities are: <br/>";
for (i = 0; i < predictionAndValuesOrdered.length; i++) {
thehtml += predictionAndValuesOrdered[i][0] + " : " + predictionAndValuesOrdered[i][1] + "%" + "<br/>";
}
thehtml + + "</span>";
// thehtml = " We classify it as: " + greenspan + predictionAndValuesOrdered[0] + "</span> <br>" +
// " No.2 guess is: " + greenspan + predictionAndValuesOrdered[1] + "</span>";
AB.msg(thehtml, 15);
}
function wipeDoodle() {
doodle_exists = false;
doodle.background('black');
}
// --- debugging --------------------------------------------------
// in console
// showInputs(demo_inputs);
// showInputs(doodle_inputs);
//#DANIEL: Given a list (of image pixels), return a matrix (of pixels). Use to get the drawn part of the image.
// matrixWidth is the image width
function listToMatrix(list, matrixWidth) {
var matrix = [], i, k;
for (i = 0, k = -1; i < list.length; i++) {
if (i % matrixWidth === 0) {
k++;
matrix[k] = [];
}
matrix[k].push(list[i]);
}
return matrix;
}
//#DANIEL: Given a matrix of image pixels, it returns the boundaries min(x,y) and max(x,y) that has pixels drawn.
// This information is used to crop the image. Used on both approaches.
function getMatrixUsedBoundaries(matrix) {
let posX, posY;
//Find min posX; max posX; min posY; max posY
let minPosX = Infinity;
let minPosY = Infinity;
let maxPosX = -Infinity;
let maxPosY = -Infinity;
//For each pixel, get the first and last X,Y positions when we have values != 0.
//That's the used area, the drawn area
for (posX = 0; posX < matrix.length; posX++) {
for (posY = 0; posY < matrix[0].length; posY++) {
//I think this matrix is inverted.
let pixelValue = matrix[posX][posY];
if (pixelValue != 0) {
if (minPosX > posX)
minPosX = posX;
if (maxPosX < posX)
maxPosX = posX;
if (minPosY > posY)
minPosY = posY;
if (maxPosY < posY)
maxPosY = posY;
}
}
}
//It is confusing, something is inverted, but this returns is 100% right!
return {
minPosX: minPosY,
minPosY: minPosX,
maxPosX: maxPosY,
maxPosY: maxPosX
};
}
//#DANIEL: Similar to the pre-existent getImage(), but the input of this function is a matrix of pixels.
function getImageFromMatrix(matrix)
{
let theimage = createImage(matrix.length, matrix[0].length); // make blank image, then populate it
theimage.loadPixels();
for (let i = 0; i < matrix.length; i++) {
for (let j = 0; j < matrix[0].length; j++) {
let bright = matrix[i][j];
let index = i * 4;
theimage.pixels[index + 0] = bright;
theimage.pixels[index + 1] = bright;
theimage.pixels[index + 2] = bright;
theimage.pixels[index + 3] = 255;
}
}
theimage.updatePixels();
return theimage;
}
//#DANIEL: This function is the core of the Approach 2, but is used for both. Given an image,
// it will get the pixels array, transform it into a matrix, get the used part of the matrix, and finally
// crop the original image using this information.
function cropImage(image, letBorder) {
image.loadPixels();
let pixelArray = [];
//Get array of pixes
for (let i = 0; i < image.width * image.height; i++) {
pixelArray[i] = image.pixels[i * 4]; // divide by 255 to use as input
}
//transform into a matrix
let matrix = listToMatrix(pixelArray, image.width);
//get used area from matrix (drawn part of the image)
let usedBoundaries = getMatrixUsedBoundaries(matrix);
let getX, getY, width, height;
getX = Math.max(0, usedBoundaries.minPosX - letBorder);
getY = Math.max(0, usedBoundaries.minPosY - letBorder);
let tmpWidth = usedBoundaries.maxPosX - usedBoundaries.minPosX;
let tmpHeight = usedBoundaries.maxPosY - usedBoundaries.minPosY;
width = Math.min(image.width, tmpWidth + letBorder + letBorder);
height = Math.min(image.height, tmpHeight + letBorder + letBorder);
//finally crop the original image, based on the used area (drawn image part)
let usedImage = image.get(
getX,
getY,
width,
height
);
return usedImage;
}
//#DANIEL: Similar to the pre-existent getInputs, but the input here is an image.
function getInputsFromImage(image) {
if (image.pixels.length == 0)
image.loadPixels();
let inputs = [];
for (let i = 0; i < PIXELSSQUARED; i++) {
inputs[i] = image.pixels[i * 4] / 255;
}
return inputs;
}
//#DANIEL: Allow to download the images pressing “S”. Good to test the same doodle image among different cases
function keyPressed() {
//get the doodle image
if (keyPressed.arguments[0].key == 'S' || keyPressed.arguments[0].key == 's') {
if (useBestSlowApproach){
//doodle
if (doodle_exists){
let imgDoodle = doodle.get();
imgDoodle.save("best_doodle_original");
let usedDoodleImage = cropImage(imgDoodle, LETBORDER);
usedDoodleImage.resize(PIXELS, PIXELS);
usedDoodleImage.save("best_doodle_used");
}
if (uploadImageResized){
//img.resize(PIXELS, PIXELS); //TODO: IS IT WORST?
uploadImageResized.save("best_upload_original");
let usedUploadImage = cropImage(uploadImageResized, LETBORDER);
usedUploadImage.resize(PIXELS, PIXELS);
usedUploadImage.save("best_upload_used");
}
if (train_index){
let trainImage = getImage(mnist.train_images[train_index]);
trainImage.save("best_train_original");
let trainImageUsed = cropImage(trainImage, LETBORDER);
trainImageUsed.resize(PIXELS, PIXELS);
trainImageUsed.save("best_train_used");
}
if (demo_exists){
let demoImage = getImage(demo);
demoImage.save("best_demo_original");
let demoImageUsed = cropImage(demoImage, LETBORDER);
demoImageUsed.resize(PIXELS, PIXELS);
demoImageUsed.save("best_demo_used");
}
}else{
if (doodle_exists){
let imgDoodle = doodle.get();
imgDoodle.save("mnist_doodle_original");
let usedDoodleImage = getImageOnMnistFormat(imgDoodle);
usedDoodleImage.save("mnist_doodle_used");
}
if (uploadImageResized){
uploadImageResized.save("mnist_upload_original");
let usedUploadImage = getImageOnMnistFormat(uploadImageResized);
usedUploadImage.save("mnist_upload_used");
}
if (train_index){
let trainImage = getImage(mnist.train_images[train_index]);
trainImage.save("mnist_train_original_used");
}
if (demo_exists){
let demoImage = getImage(demo);
demoImage.save("mnist_train_original_used");
}
}
}
}
function showInputs(inputs)
// display inputs row by row, corresponding to square of pixels
{
var str = "";
for (let i = 0; i < inputs.length; i++) {
if (i % PIXELS == 0) str = str + "\n"; // new line for each row of pixels
var value = inputs[i];
str = str + " " + value.toFixed(2);
}
console.log(str);
}