// Cloned by Dillan de Langen on 11 Dec 2019 from World "Character recognition neural network" by "Coding Train" project
// Please leave this clone trail here.
// Port of Character recognition neural network from here:
// This world is a port of the MNIST ConvNetJS demo created by Andrej Karpathy ported into
// the Character reconition neural network world by Coding Train with many modifications done by
// Dillan de Langen
// The original ConvNetJS MNIST demo can be viewed here: https://cs.stanford.edu/people/karpathy/convnetjs/demo/mnist.html
// The code for the ConvNetJS MNIST demo can be used here: https://github.com/karpathy/convnetjs/blob/master/demo/mnist.html
// --- defined by MNIST - do not change these ---------------------------------------
const PIXELS = 28; // images in data set are tiny
const PIXELSSQUARED = PIXELS * PIXELS;
//--- can modify all these --------------------------------------------------
// no of nodes in network
const noinput = PIXELSSQUARED;
const nohidden = 64;
const nooutput = 10;
const learningrate = 0.01; // default 0.01
//Settings to be used to "tune" then training of the neural network
const momentum = 0.9;
const batchSize = 20;
const weightDecay = 0.001;
//Global variables used to hold the MNIST data set:
var num_batches = 21; // 20 training batches, 1 test, each batch contains 3000 images
var data_img_elts = new Array(num_batches);
var img_data = new Array(num_batches);
var loaded = new Array(num_batches);
var loaded_train_batches = [];
var step_num = 0;
var paused = false;
var use_validation_data = true;
var imagesSeen = 0;
var imagesCorrect = 0;
classes_txt = ['0', '1', '2', '3', '4', '5', '6', '7', '8', '9'];
// should we train every timestep or not
let do_training = true;
// how many to train and test per timestep
const TRAINPERSTEP = 30;
const TESTPERSTEP = 5;
// multiply it by this to magnify for display
const ZOOMFACTOR = 7;
const ZOOMPIXELS = ZOOMFACTOR * PIXELS;
// 3 rows of
// large image + 50 gap + small image
// 50 gap between rows
const canvaswidth = ( PIXELS + ZOOMPIXELS ) + 50;
const canvasheight = ( ZOOMPIXELS * 3 ) + 100;
const DOODLE_THICK = 18; // thickness of doodle lines
const DOODLE_BLUR = 3; // blur factor applied to doodles
let convNeuralNetwork;
let nnTrainer;
let trainrun = 1;
let train_index = 0;
let testrun = 1;
let test_index = 0;
let total_tests = 0;
let total_correct = 0;
// images in LHS:
let doodle, demo, trainingImage;
let doodle_exists = false;
let demo_exists = false;
let trainingImage_exists = false;
let mousedrag = false; // are we in the middle of a mouse drag drawing?
// save inputs to global var to inspect
// type these names in console
var train_inputs, test_inputs, demo_inputs, doodle_inputs;
// CSS trick
// make run header bigger
$("#runheaderbox").css ( { "max-height": "95vh" } );
//Set the background image of the world.
//Done with jQuery as p5 backfround did not want to work.
//Background image credit to Engadget orginally from
//https://www.engadget.com/2019/03/22/mit-ai-automated-neural-network-design/?guccounter=1&guce_referrer=aHR0cHM6Ly93d3cuZ29vZ2xlLmNvbS8&guce_referrer_sig=AQAAAIfGoDWXLdfKpXCuvP_S5m7ZoRe5alAA6nF3auVTjFSXR8-V_QFLNneZezrUwmmXJv3UMVZkYIZAQkzsrde0Nzng3zRja8wfVBub23eEAmaRQ6uMpHsUbtk7fWlyk2TbsTer8SEN9iL6dXetIjZLBF4wyMzfN1Qm7eXtBgTyB3Rm
$("body").css("background", "url('/uploads/dillan/NeuralNetworkbackground.jpg')");
//--- start of AB.msgs structure: ---------------------------------------------------------
// We output a serious of AB.msgs to put data at various places in the run header
var thehtml;
// 1 Doodle header
thehtml = "<hr> <h1> 1. Doodle </h1> Top row: Doodle (left) and shrunk (right). <br> " +
" Draw your doodle in top LHS. <button onclick='wipeDoodle();' class='normbutton' >Clear doodle</button> <br> ";
AB.msg ( thehtml, 1 );
// 2 Doodle variable data (guess)
// 3 Training header
thehtml = "<hr> <h1> 2. Training </h1> Middle row: Training image magnified (left) and original (right). <br> " +
" <button onclick='stopTraining()' class='normbutton' >Stop training</button> <br> ";
AB.msg ( thehtml, 3 );
// 4 variable training data
// 5 Testing header
thehtml = "<h3> Hidden tests </h3> " ;
AB.msg ( thehtml, 5 );
// 6 variable testing data
// 7 Demo header
thehtml = "<hr> <h1> 3. Demo </h1> Bottom row: Test image magnified (left) and original (right). <br>" +
" The network is <i>not</i> trained on any of these images. <br> " +
" <button onclick='demoTestImage();' class='normbutton' >Demo test image</button> <br> ";
AB.msg ( thehtml, 7 );
// 8 Demo variable data (random demo ID)
// 9 Demo variable data (changing guess)
const greenSpan = "<span style='font-weight:bold; font-size:x-large; color:darkgreen'>{{content}}</span>" ;
const redSpan = "<span style='font-weight:bold; font-size:x-large; color:red'>{{content}}</span>" ;
//--- end of AB.msgs structure: ---------------------------------------------------------
function setup()
{
createCanvas ( canvaswidth, canvasheight );
doodle = createGraphics ( ZOOMPIXELS, ZOOMPIXELS ); // doodle on larger canvas
doodle.pixelDensity(1);
// JS load other JS
// maybe have a loading screen while loading the JS and the data set
AB.loadingScreen();
$.getScript("/uploads/dillan/convnet-min.js", function(){
$.getScript("/uploads/dillan/mnist_labels.js", function(){
createNeuralNetwork();
//Trainer for the Neural Network
nnTrainer = new convnetjs.SGDTrainer(convNeuralNetwork, {method:'adadelta', batch_size:20, l2_decay:0.001});
for (var k = 0; k < loaded.length; k++) {
loaded[k] = false;
}
loadData();
start_fun();
});
});
}
//Instantiate a new Neural Network with 4 hidden layers
function createNeuralNetwork(){
convNeuralNetwork = new convnetjs.Net();
let layers = [];
//Input layer of the neural network:
//Crop the orignal 28 x 28 pixel image to a 24 x 24 pixel window to be passed to the next layer.
layers.push({type:'input', out_sx:24, out_sy:24, out_depth:1});
layers.push({type:'conv', sx:5, filters:8, stride:1, pad:2, activation:'relu'});
layers.push({type:'pool', sx:2, stride:2});
layers.push({type:'conv', sx:5, filters:16, stride:1, pad:2, activation:'relu'});
layers.push({type:'pool', sx:3, stride:3});
//This will be the output layer of the neural network:
//We use 10 output classes to classify images as a number from 0 to 9.
layers.push({type:'softmax', num_classes:10});
convNeuralNetwork.makeLayers(layers);
}
//Train the NN on a sample image
//sample = a random image from the training set.
function step(sample) {
let x = sample.x;
let y = sample.label;
// train on it with network
let stats = nnTrainer.train(x, y);
imagesSeen++;
let lossx = stats.cost_loss;
let lossw = stats.l2_decay_loss;
// keep track of stats such as the average training error and loss
let yhat = convNeuralNetwork.getPrediction();
if(yhat == y){
imagesCorrect++;
}
if(step_num % 25 === 0){
displayTrainingStats();
trainingImage = sample.pixels;
trainingImage_exists = true;
}
step_num++;
}
function displayTrainingStats(){
let percentCorrect = Math.round(imagesCorrect / imagesSeen * 10000) / 100;
let span;
if(percentCorrect > 80.0){
span = greenSpan.replace("{{content}}", percentCorrect + "%");
}
else{
span = redSpan.replace("{{content}}", percentCorrect + "%")
}
let html = "Images seen: " + imagesSeen + "<br>";
html += "Images correct: " + imagesCorrect + "<br>";
html += "Score: " + span;
AB.msg(html, 6);
}
//Get a random image from the TRAINING set
function sample_training_instance () {
// find an unloaded batch
let batchIndex = Math.floor(Math.random() * loaded_train_batches.length);
let batch = loaded_train_batches[batchIndex];
let k = Math.floor(Math.random() * 3000); // random sample within the batch
let n = batch * 3000 + k;
// load more batches over time
if (step_num % 5000 === 0 && step_num > 0) {
for (let i = 0; i < num_batches; i++) {
if (!loaded[i]) {
// load it
load_data_batch(i);
break; // okay for now
}
}
}
// fetch the appropriate row of the training image and reshape into a Vol
let p = img_data[batch].data;
let x = new convnetjs.Vol(28, 28, 1, 0.0);
let W = 28 * 28;
for (let i = 0; i < W; i++) {
let ix = ((W * k) + i) * 4;
x.w[i] = p[ix] / 255.0;
}
let pixels = [].slice.call(x.w).map(function(pixel){
return Math.floor(pixel * 255);
});
x = convnetjs.augment(x, 24);
let isval = use_validation_data && n % 10 === 0 ? true : false;
return {
x: x,
label: labels[n],
isval: isval,
pixels: pixels
};
}
//Get a random image from the TEST set.
function sample_test_instance() {
let b = 20; //Batch 20 is the test batch.
let k = Math.floor(Math.random() * 3000); //Get a random image from the test set
let n = b * 3000 + k;
let p = img_data[b].data;
let x = new convnetjs.Vol(28, 28, 1, 0.0);
let W = 28 * 28;
for (let i = 0; i < W; i++) {
let ix = ((W * k) + i) * 4;
x.w[i] = p[ix] / 255.0;
}
let xs = [];
for (let i = 0; i < 4; i++) {
xs.push(convnetjs.augment(x, 24));
}
let pixels = [].slice.call(x.w).map(function(pixel){
return Math.floor(pixel * 255);
});
// return multiple augmentations, and we will average the network over them
// to increase performance
return {
x: xs,
label: labels[n],
pixels: pixels
};
}
function load_and_step() {
if (paused)
return;
var sample = sample_training_instance();
step(sample); // Train the NN with the sample image.
}
function start_fun() {
//loaded[0] indicates the first training batch has loaded.
//loaded[20] indicates the TEST images have loaded.
if (loaded[0] && loaded[20]) {
console.log('starting!');
setInterval(load_and_step, 0);
}
else {
console.log('Either training 0 or the test set have not loaded yet. Waiting 200ms.');
//If the first training set and the test set have not loaded
//yet then we wait for a 200ms and try again.
setTimeout(start_fun, 200);
}
}
function load_data_batch(batch_num) {
// Load the dataset with JS in background
data_img_elts[batch_num] = new Image();
var data_img_elt = data_img_elts[batch_num];
data_img_elt.onload = function () {
var data_canvas = document.createElement('canvas');
data_canvas.width = data_img_elt.width;
data_canvas.height = data_img_elt.height;
var data_ctx = data_canvas.getContext("2d");
data_ctx.drawImage(data_img_elt, 0, 0); // copy it over... bit wasteful :(
img_data[batch_num] = data_ctx.getImageData(0, 0, data_canvas.width, data_canvas.height);
loaded[batch_num] = true;
if (batch_num < 20) {
loaded_train_batches.push(batch_num);
}
console.log('finished loading data batch ' + batch_num);
};
data_img_elt.src = "/uploads/dillan/mnist_batch_" + batch_num + ".png";
}
function stopTraining(){
paused = true;
}
//See if the NN can classify a test image correctly.
function demoTestImage(){
let num_classes = convNeuralNetwork.layers[convNeuralNetwork.layers.length - 1].out_depth;
let testImage = sample_test_instance();
demo = testImage.pixels;
demo_exists = true;
let aavg = new convnetjs.Vol(1, 1, num_classes, 0.0);
let xs = [].concat(testImage.x);
let n = xs.length;
for (let i = 0; i < n; i++) {
let a = convNeuralNetwork.forward(xs[i]);
aavg.addFrom(a);
}
let predictions = [];
for (let k = 0; k < aavg.w.length; k++) {
predictions.push({ k: k, p: aavg.w[k] });
}
//Sort the prediction so that the most confident prediction is first
predictions.sort(function (a, b) { return a.p < b.p ? 1 : -1; });
console.log('Predictions: ', predictions);
let bestGuess = predictions[0];
let span = testImage.label == bestGuess.k ? greenSpan.replace("{{content}}", bestGuess.k) : redSpan.replace("{{content}}", bestGuess.k);
let html = "Classification: " + testImage.label + "<br>";
html += "Our guess: " + span;
AB.msg(html, 9);
drawDemo();
}
function drawTrainingImage(){
var theimage = getImage ( trainingImage ); // get image from data array
image ( theimage, 0, ZOOMPIXELS+50, ZOOMPIXELS, ZOOMPIXELS ); // magnified
image ( theimage, ZOOMPIXELS+50, ZOOMPIXELS+50, PIXELS, PIXELS ); // original
}
// load the MNIST data from the batch images.
function loadData()
{
console.log('Loading MNIST data set...');
for(let i = 0; i < num_batches; i++){
load_data_batch(i);
}
console.log('Training and test data loaded.');
AB.removeLoading();
}
function getImage ( img ) // make a P5 image object from a raw data array
{
let theimage = createImage (PIXELS, PIXELS); // make blank image, then populate it
theimage.loadPixels();
for (let i = 0; i < PIXELSSQUARED ; i++)
{
let bright = img[i];
let index = i * 4;
theimage.pixels[index + 0] = bright;
theimage.pixels[index + 1] = bright;
theimage.pixels[index + 2] = bright;
theimage.pixels[index + 3] = 255;
}
theimage.updatePixels();
return theimage;
}
// --- the draw function -------------------------------------------------------------
// every step:
function draw()
{
background ('black');
// keep drawing demo and doodle images
// and keep guessing - we will update our guess as time goes on
if ( doodle_exists )
{
drawDoodle();
guessDoodle();
}
if ( demo_exists )
{
drawDemo();
}
if(trainingImage_exists){
drawTrainingImage();
}
// detect doodle drawing
// (restriction) the following assumes doodle starts at 0,0
if ( mouseIsPressed ) // gets called when we click buttons, as well as if in doodle corner
{
// console.log ( mouseX + " " + mouseY + " " + pmouseX + " " + pmouseY );
var MAX = ZOOMPIXELS + 20; // can draw up to this pixels in corner
if ( (mouseX < MAX) && (mouseY < MAX) && (pmouseX < MAX) && (pmouseY < MAX) )
{
mousedrag = true; // start a mouse drag
doodle_exists = true;
doodle.stroke('white');
doodle.strokeWeight( DOODLE_THICK );
doodle.line(mouseX, mouseY, pmouseX, pmouseY);
}
}
else
{
// are we exiting a drawing
if ( mousedrag )
{
mousedrag = false;
// console.log ("Exiting draw. Now blurring.");
doodle.filter (BLUR, DOODLE_BLUR); // just blur once
// console.log (doodle);
}
}
}
//--- demo -------------------------------------------------------------
// demo some test image and predict it
// get it from test set so have not used it in training
function drawDemo()
{
var theimage = getImage ( demo );
image ( theimage, 0, canvasheight - ZOOMPIXELS, ZOOMPIXELS, ZOOMPIXELS ); // magnified
image ( theimage, ZOOMPIXELS+50, canvasheight - ZOOMPIXELS, PIXELS, PIXELS ); // original
}
//--- doodle -------------------------------------------------------------
function drawDoodle()
{
// doodle is createGraphics not createImage
let theimage = doodle.get();
image ( theimage, 0, 0, ZOOMPIXELS, ZOOMPIXELS ); // original
image ( theimage, ZOOMPIXELS+50, 0, PIXELS, PIXELS ); // shrunk
}
function guessDoodle()
{
let num_classes = convNeuralNetwork.layers[convNeuralNetwork.layers.length - 1].out_depth;
let aavg = new convnetjs.Vol(1, 1, num_classes, 0.0);
// doodle is createGraphics not createImage
let img = doodle.get();
img.resize ( PIXELS, PIXELS );
img.loadPixels();
// set up inputs
let inputs = [];
var x = new convnetjs.Vol(28, 28, 1, 0.0);
for (let i = 0; i < PIXELSSQUARED ; i++)
{
inputs[i] = img.pixels[i * 4] / 255.0;
x.w[i] = inputs[i];
}
doodle_inputs = inputs;
x = convnetjs.augment(x, 24);
let xs = [].concat(x);
let n = xs.length;
for (let i = 0; i < n; i++) {
let a = convNeuralNetwork.forward(xs[i]);
aavg.addFrom(a);
}
let predictions = [];
for (let k = 0; k < aavg.w.length; k++) {
predictions.push({ k: k, p: aavg.w[k] });
}
predictions.sort(function (a, b) { return a.p < b.p ? 1 : -1; });
console.log(predictions);
let bestGuess = predictions[0];
let secondGuess = predictions[1];
for(let i = 0; i < predictions.length; i++){
if(predictions[i].p > bestGuess.p){
bestGuess = predictions[i];
}
}
thehtml = "We classify it as: " + bestGuess.k + "</span> <br>" +
" No.2 guess is: " + secondGuess.k + "</span>";
AB.msg ( thehtml, 2 );
}
function wipeDoodle()
{
doodle_exists = false;
doodle.background('black');
}