// Cloned by K.Ellis on 19 Dec 2019 from World "Character recognition neural network (clone by K.Ellis)" by K.Ellis
// Please leave this clone trail here.
// Port of Character recognition neural network from here:
// https://github.com/CodingTrain/Toy-Neural-Network-JS/tree/master/examples/mnist
// with many modifications
// --- defined by MNIST - do not change these ---------------------------------------
const PIXELS = 28; // images in data set are tiny
const PIXELSSQUARED = PIXELS * PIXELS;
// number of training and test exemplars in the data set:
const NOTRAIN = 60000;
const NOTEST = 10000;
//--- can modify all these --------------------------------------------------
// no of nodes in network
const noinput = PIXELSSQUARED;
const nohidden = 24; //KE - was 64
const nooutput = 10;
const learningrate = 0.1; // default 0.1
// should we train every timestep or not
let do_training = true;
// how many to train and test per timestep
const TRAINPERSTEP = 30;
const TESTPERSTEP = 5;
// multiply it by this to magnify for display
const ZOOMFACTOR = 7;
const ZOOMPIXELS = ZOOMFACTOR * PIXELS;
// 3 rows of
// large image + 50 gap + small image
// 50 gap between rows
//const canvaswidth = ( PIXELS + ZOOMPIXELS ) + 60;
const canvaswidth = ( ZOOMPIXELS * 2 ) + 120;
const canvasheight = ( ZOOMPIXELS * 3 ) + 102;
const doodlewidth = (PIXELS);
const DOODLE_THICK = 15; // thickness of doodle lines
const DOODLE_BLUR = 3; // blur factor applied to doodles
let mnist;
// all data is loaded into this
// mnist.train_images
// mnist.train_labels
// mnist.test_images
// mnist.test_labels
let nn;
//KE added variables
let canvas, dst, src, hierarchy, contours, img, diffX, diffY, M;
let cvOutput = [];
let trainrun = 1;
let train_index = 0;
let testrun = 1;
let test_index = 0;
let total_tests = 0;
let total_correct = 0;
// images in LHS:
let doodle, demo;
let doodle_exists = false;
let demo_exists = false;
let mousedrag = false; // are we in the middle of a mouse drag drawing?
// save inputs to global var to inspect
// type these names in console
var train_inputs, test_inputs, demo_inputs, doodle_inputs;
// Matrix.randomize() is changed to point to this. Must be defined by user of Matrix.
function randomWeight()
{
return ( AB.randomFloatAtoB ( -0.5, 0.5 ) ); //-0.5, 0.5
// Coding Train default is -1 to 1
}
// CSS trick
// make run header bigger
$("#runheaderbox").css ( { "max-height": "95vh" } );
//--- start of AB.msgs structure: ---------------------------------------------------------
// We output a serious of AB.msgs to put data at various places in the run header
var thehtml;
// 1 Doodle header
thehtml = "<hr> <h1> 1. Doodle </h1> Top row: Doodle (left) and shrunk (right). <br> " +
" Draw your doodle in top LHS. <button onclick='wipeDoodle();' class='normbutton' >Clear doodle</button> <br> ";
AB.msg ( thehtml, 1 );
// 2 Doodle variable data (guess)
// 3 Training header
thehtml = "<hr> <h1> 2. Training </h1> Middle row: Training image magnified (left) and original (right). <br> " +
" <button onclick='do_training = false;' class='normbutton' >Stop training</button> <br> ";
AB.msg ( thehtml, 3 );
// 4 variable training data
// 5 Testing header
thehtml = "<h3> Hidden tests </h3> " ;
AB.msg ( thehtml, 5 );
// 6 variable testing data
// 7 Demo header
thehtml = "<hr> <h1> 3. Demo </h1> Bottom row: Test image magnified (left) and original (right). <br>" +
" The network is <i>not</i> trained on any of these images. <br> " +
" <button onclick='makeDemo();' class='normbutton' >Demo test image</button> <br> ";
AB.msg ( thehtml, 7 );
// 8 Demo variable data (random demo ID)
// 9 Demo variable data (changing guess)
const greenspan = "<span style='font-weight:bold; font-size:x-large; color:darkgreen'> " ;
//--- end of AB.msgs structure: ---------------------------------------------------------
function setup()
{
canvas = createCanvas ( canvaswidth, canvasheight);
canvas.position(10,20); // just to shift away from edge
doodle = createGraphics ( ZOOMPIXELS, ZOOMPIXELS ); // doodle on larger canvas
doodle.pixelDensity(1); // MNIST images are pixulated so keep at 1 or blank
wipeDoodle(); //KE calling wipe to have doodle prepped from load
//KE just some boxes for our boxes & some text prompts
rect(0, 0, ZOOMPIXELS, ZOOMPIXELS); //doodle
textSize(16);
textAlign(CENTER);
text('DRAW A DIGIT', ZOOMPIXELS /2, ZOOMPIXELS/2.2);
textAlign(CENTER);
text('BETWEEN 0-9', ZOOMPIXELS /2, ZOOMPIXELS/1.8);
rect(0, ZOOMPIXELS + 50, ZOOMPIXELS, ZOOMPIXELS); //train
rect(0, canvasheight - ZOOMPIXELS - 2, ZOOMPIXELS, ZOOMPIXELS); // demo
textSize(16);
textAlign(CENTER);
text(' CLICK ', 100, canvasheight - (ZOOMPIXELS / 1.8));
text('\'DEMO TEST IMAGE\'', 100, canvasheight - (ZOOMPIXELS / 2.2));
// JS load other JS
// maybe have a loading screen while loading the JS and the data set
AB.loadingScreen();
$.getScript ( "/uploads/ke737/opencv_3.3.1.js", function()
{
$.getScript ( "/uploads/ke737/matrix.js", function()
{
$.getScript ( "/uploads/ke737/nn.js", function()
{
$.getScript ( "/uploads/codingtrain/mnist.js", function()
{
console.log ("All JS loaded");
nn = new NeuralNetwork( noinput, nohidden, nooutput );
nn.setLearningRate ( learningrate );
loadData();
});
});
});
});
}
// load data set from local file (on this server)
function loadData()
{
loadMNIST ( function(data)
{
mnist = data;
console.log ("All data loaded into mnist object:");
console.log(mnist);
AB.removeLoading(); // if no loading screen exists, this does nothing
});
}
function getImage ( img ) // make a P5 image object from a raw data array
{
let theimage = createImage (PIXELS, PIXELS); // make blank image, then populate it
theimage.loadPixels();
for (let i = 0; i < PIXELSSQUARED ; i++)
{
let bright = img[i];
let index = i * 4;
//console.log("index" + index);
theimage.pixels[index + 0] = bright;
theimage.pixels[index + 1] = bright;
theimage.pixels[index + 2] = bright;
theimage.pixels[index + 3] = 255;
}
theimage.updatePixels();
return theimage;
}
function getInputs ( img ) // convert img array into normalised input array
{
let inputs = [];
for (let i = 0; i < PIXELSSQUARED ; i++)
{
let bright = img[i];
inputs[i] = bright / 255; // normalise to 0 to 1,
}
return ( inputs );
}
function trainit (show) // train the network with a single exemplar, from global var "train_index", show visual on or off
{
let img = mnist.train_images[train_index];
let label = mnist.train_labels[train_index];
// optional - show visual of the image
if (show)
{
var theimage = getImage ( img ); // get image from data array
image ( theimage, 0, ZOOMPIXELS+50, ZOOMPIXELS, ZOOMPIXELS ); // magnified
image ( theimage, ZOOMPIXELS+50, ZOOMPIXELS+50, PIXELS, PIXELS ); // original
}
// set up the inputs
let inputs = getInputs ( img ); // get inputs from data array
// set up the outputs
let targets = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
targets[label] = 1; // change one output location to 1, the rest stay at 0
train_inputs = inputs; // can inspect in console
nn.train ( inputs, targets );
thehtml = " trainrun: " + trainrun + "<br> no: " + train_index ;
AB.msg ( thehtml, 4 );
train_index++;
if ( train_index == NOTRAIN )
{
train_index = 0;
console.log( "finished trainrun: " + trainrun );
trainrun++;
}
}
function testit() // test the network with a single exemplar, from global var "test_index"
{
let img = mnist.test_images[test_index];
let label = mnist.test_labels[test_index];
// set up the inputs
let inputs = getInputs ( img );
test_inputs = inputs; // can inspect in console
let prediction = nn.predict(inputs); // array of outputs
let guess = findMax(prediction); // the top output
total_tests++;
if (guess == label) total_correct++;
let percent = (total_correct / total_tests) * 100 ;
thehtml = " testrun: " + testrun + "<br> no: " + total_tests + " <br> " +
" correct: " + total_correct + "<br>" +
" score: " + greenspan + percent.toFixed(2) + "</span>";
AB.msg ( thehtml, 6 );
test_index++;
if ( test_index == NOTEST )
{
console.log( "finished testrun: " + testrun + " score: " + percent.toFixed(2) );
testrun++;
test_index = 0;
total_tests = 0;
total_correct = 0;
}
}
//--- find no.1 (and maybe no.2) output nodes ---------------------------------------
// (restriction) assumes array values start at 0 (which is true for output nodes)
//KE changed this to list top 3 predictions
function find123 (a) // return array showing indexes of no.1 and no.2 values in array
{
let no1 = 0;
let no2 = 0;
let no3 = 0;
let no1value = 0;
let no2value = 0;
let no3value = 0;
for (let i = 0; i < a.length; i++)
{
if (a[i] > no1value)
{
no1 = i;
no1value = a[i];
}
else if (a[i] > no2value)
{
no2 = i;
no2value = a[i];
}
else if (a[i] > no3value)
{
no3 = i;
no3value = a[i];
}
}
var b = [ no1, no2, no3 ];
return b;
}
// just get the maximum - separate function for speed - done many times
// find our guess - the max of the output nodes array
function findMax (a)
{
let no1 = 0;
let no1value = 0;
for (let i = 0; i < a.length; i++)
{
if (a[i] > no1value)
{
no1 = i;
no1value = a[i];
}
}
return no1;
}
// --- the draw function -------------------------------------------------------------
// every step:
function draw()
{
// check if libraries and data loaded yet:
if ( typeof mnist == 'undefined' ) return;
// how can we get white doodle on black background on yellow canvas?
// background('#ffffcc'); doodle.background('black');
//background ('black'); //KE edited out so as to apply the following instead
rect(0, 0, ZOOMPIXELS, ZOOMPIXELS); //doodle
textSize(16);
textAlign(CENTER);
text('DRAW A DIGIT', ZOOMPIXELS /2, ZOOMPIXELS/2.2);
textAlign(CENTER);
text('BETWEEN 0-9', ZOOMPIXELS /2, ZOOMPIXELS/1.8);
if ( do_training )
{
// do some training per step
for (let i = 0; i < TRAINPERSTEP; i++)
{
if (i === 0) trainit(true); // show only one per step - still flashes by
else trainit(false);
}
// do some testing per step
for (let i = 0; i < TESTPERSTEP; i++)
testit();
}
// keep drawing demo and doodle images
// and keep guessing - we will update our guess as time goes on
if ( demo_exists )
{
drawDemo();
guessDemo();
}
if ( doodle_exists )
{
drawDoodle();
guessDoodle();
}
// detect doodle drawing
// (restriction) the following assumes doodle starts at 0,0
if ( mouseIsPressed ) // gets called when we click buttons, as well as if in doodle corner
{
// console.log ( mouseX + " " + mouseY + " " + pmouseX + " " + pmouseY );
var MAX = ZOOMPIXELS - 2; // can draw up to this pixels in corner
if ( (mouseX < MAX) && (mouseY < MAX) && (pmouseX < MAX) && (pmouseY < MAX) )
{
mousedrag = true; // start a mouse drag
doodle_exists = true;
doodle.stroke('cream');
strokeJoin(BEVEL);
doodle.strokeWeight( DOODLE_THICK );
doodle.line(mouseX, mouseY, pmouseX, pmouseY);
}
}
else
{
// are we exiting a drawing
if ( mousedrag )
{
mousedrag = false;
console.log ("Doodle detected");
//KE main addition to code follows
// using opencv.js to find contours, centre of mass, bounding box
// & using warp affine to shift our image
/*Applying opencv.js_3.3.1 lib
Contours are key to recognition:
https://docs.opencv.org/master/d5/daa/tutorial_js_contours_begin.html
https://docs.opencv.org/master/d0/d43/tutorial_js_table_of_contents_contours.html
https://docs.opencv.org/3.4/dc/dcf/tutorial_js_contour_features.html
*/
// Common lines for Contours, Moments, Bounding Box etc
// added next 5 lines as cannot use imread
img = doodle.get(); //get();
img.resize(PIXELS, PIXELS);
img.loadPixels();
//console.log(img);
imagedata = img.imageData;
src = cv.matFromImageData(imagedata);
//src = cv.imread('canvasInput'); //issue using cv.imread, think because doodle graphic not canvas
dst = cv.Mat.zeros(src.cols, src.rows, cv.CV_8UC3);
cv.cvtColor(src, src, cv.COLOR_RGBA2GRAY, 0);
cv.threshold(src, src, 120, 255, cv.THRESH_BINARY); // 120 , 200
contours = new cv.MatVector();
hierarchy = new cv.Mat();
cv.findContours(src, contours, hierarchy, cv.RETR_CCOMP, cv.CHAIN_APPROX_SIMPLE);
/// ** Moments**
let cnt = contours.get(0);
let Moments = cv.moments(cnt, false);
//console.log("Moments.m00 = " + Moments.m00);
M = Moments.m00; //KE this is our centre of mass of the object, understanding its position relative to 14,14 is key
let cx = Math.round((Moments.m10/Moments.m00) * 100) / 100;
let cy = Math.round((Moments.m01/Moments.m00) * 100) / 100;
//console.log("cx = " + cx + ", cy = " + cy); // print the Centroid for number X & Y axis
let x = PIXELS/2; //
let y = PIXELS/2;
//console.log("x = " + x + ", y = " + y);
diffX = Math.round(x - cx); // this is my X axis offset for centering, the centre of mass of doodle
diffY = Math.round(y - cy); // this is my Y axis offset for centering, the centre of mass of doodle
//console.log("diff x = " + diffX + ", diff y = " + diffY);
// ** Rect bounding box **
let rect = cv.boundingRect(cnt);
let contoursColor = new cv.Scalar(255, 255, 255);
let rectangleColor = new cv.Scalar(255, 0, 0);
cv.drawContours(dst, contours, 0, contoursColor, 1, 8, hierarchy, 100);
let point1 = new cv.Point(rect.x, rect.y);
let point2 = new cv.Point(rect.x + rect.width, rect.y + rect.height);
cv.rectangle(dst, point1, point2, rectangleColor, 2, cv.LINE_AA, 0);
//console.log("bounding box before offset");
//console.log(rect);
//console.log("bounding box after offset");
rect.x = Math.round(rect.x + diffX);
rect.y = Math.round(rect.y + diffY);
//console.log(rect);
//diffH = rect.height + (20 - rect.height);
// ** warp Affine transform **
// KE - key is to apply matrix with X-axis & Y-axis shifts from moments
let Matrix = cv.matFromArray(2, 3, cv.CV_64FC1, [1, 0, diffX, 0, 1, diffY]);
let dsize = new cv.Size(src.rows, src.cols);
//let dsize = new cv.Size(src.rows, src.cols);
cv.warpAffine(src, dst, Matrix, dsize, cv.INTER_LINEAR, cv.BORDER_CONSTANT, new cv.Scalar());
//cv.warpAffine(src, dst, Matrix, dsize, cv.INTER_AREA, cv.BORDER_CONSTANT, new cv.Scalar());
cvOutput = getInputs ( dst.data8S);
// KE - Print shifted image to screen
image ( img, (ZOOMPIXELS + 120) + (diffX * ZOOMFACTOR), 0 + (diffY * ZOOMFACTOR), ZOOMPIXELS, ZOOMPIXELS);
//KE - These are the different filters available vi p5.js https://p5js.org/reference/#/p5/filter
// found that none where needed to match demo tests level of accuracy
// These are operations on doodle graphic whereas we sending back offset image data for prediction
/*
///
doodle.filter(THRESHOLD, 0.5); //THRESHOLD Converts the image to black and white pixels depending if they are above or below the threshold defined by the level parameter. The parameter must be between 0.0 (black) and 1.0 (white). If no level is specified, 0.5 is used.
doodle.filter(GRAY); // Converts any colors in the image to grayscale equivalents. No parameter is used.
doodle.filter(OPAQUE); //OPAQUE Sets the alpha channel to entirely opaque. No parameter is used.
doodle.filter(INVERT); //INVERT Sets each pixel to its inverse value. No parameter is used.
doodle.filter(POSTERIZE, 255); //2 -255 Limits each channel of the image to the number of colors specified as the parameter. The parameter can be set to values between 2 and 255, but results are most noticeable in the lower ranges.
doodle.filter(DILATE); //Increases the light areas. No parameter is used.
doodle.filter(BLUR, DOODLE_BLUR); // just blur once // BLUR implements Gaussian blur
doodle.filter(ERODE); //Reduces the light areas. No parameter is used.
*/
console.log('%c ***** DOODLE CENTERED ***** ', 'background: #222; color: #bada55');
return cvOutput;
}
}
}
//--- demo -------------------------------------------------------------
// demo some test image and predict it
// get it from test set so have not used it in training
function makeDemo()
{
demo_exists = true;
var i = AB.randomIntAtoB ( 0, NOTEST - 1 );
demo = mnist.test_images[i];
var label = mnist.test_labels[i];
thehtml = "Test image no: " + i + "<br>" +
"Classification: " + label + "<br>" ;
AB.msg ( thehtml, 8 );
// type "demo" in console to see raw data
}
function drawDemo()
{
var theimage = getImage ( demo );
// console.log (theimage); //normalised 0 -255
image ( theimage, 0, canvasheight - (ZOOMPIXELS + 2), ZOOMPIXELS, ZOOMPIXELS ); // magnified
image ( theimage, ZOOMPIXELS+50, canvasheight - ZOOMPIXELS, PIXELS, PIXELS ); // original
}
function guessDemo()
{
let inputs = getInputs ( demo );
demo_inputs = inputs; // can inspect in console
let prediction = nn.predict(inputs); // array of outputs
let guess = findMax(prediction); // the top output
thehtml = " We classify it as: " + greenspan + guess + "</span>" ;
AB.msg ( thehtml, 9 );
}
//--- doodle -------------------------------------------------------------
function drawDoodle()
{
// doodle is createGraphics not createImage
let theimage = doodle.get();
image ( theimage, 0, 0, ZOOMPIXELS, ZOOMPIXELS ); // original
image ( theimage, ZOOMPIXELS+50, 0, PIXELS, PIXELS ); // shrunk
}
function guessDoodle()
{
// doodle is createGraphics not createImage
img = doodle.get();
img.resize ( PIXELS, PIXELS );
img.loadPixels();
// set up inputs
let inputs = [];
for (let i = 0; i < PIXELSSQUARED ; i++)
{
inputs[i] = img.pixels[i * 4] / 255; // doodle_input values sent for classifcation/prediction
}
//KE - copy & then overwrite array for doodle inputs
let ke = Array.from(inputs);
//KE - overwrite inputs with our cvOutput
for (let i = 0; i < ke.length ; i++)
{
ke[i] = cvOutput[i];
}
//KE - copy array back to inputs & normalise
inputs = Array.from(ke);
for (let i = 0; i < inputs.length ; i++)
{
inputs[i] = inputs[i] * -255;
}
// can inspect in console
doodle_inputs = inputs;
// feed forward to make prediction
let prediction = nn.predict(inputs); // KE - now inputs are not doodle.get() but rather cvOutput
let b = find123(prediction); // KE - get no.1, no.2 & no.3 guesses
//console.log("guess # 1,2,3 are: " + b[0] + " , " + b[1] + " , " + b[2]);
thehtml = " We classify it as: " + greenspan + b[0] + "</span> <br>" +
" No.2 guess is: " + greenspan + b[1] + "</span> <br>" +
"No.3 guess is: " + greenspan + b[2] + "</span>";
AB.msg ( thehtml, 2 );
AB.msg ( thehtml, 2 );
}
function wipeDoodle()
{
doodle_exists = false;
doodle.background('black');
}
// --- debugging --------------------------------------------------
// in console
// showInputs(demo_inputs);
// showInputs(doodle_inputs);
function showInputs ( inputs )
// display inputs row by row, corresponding to square of pixels
{
var str = "";
for (let i = 0; i < inputs.length; i++)
{
if ( i % PIXELS === 0 ) str = str + "\n"; // new line for each row of pixels
var value = inputs[i];
str = str + " " + value.toFixed(2) ;
}
console.log (str);
}