Code viewer [Ancient Brain]

Code viewer for World: NN Project - Submission Entry
// Cloned by Brendan on 9 Dec 2019 from World "Character recognition neural network (clone by Brendan)" by Brendan 
// Please leave this clone trail here.



// Cloned by Brendan on 1 Dec 2019 from World "Character recognition neural network" by "Coding Train" project 
// Please leave this clone trail here.


// Port of Character recognition neural network from here:
// https://github.com/CodingTrain/Toy-Neural-Network-JS/tree/master/examples/mnist
// with many modifications 


// --- defined by MNIST - do not change these ---------------------------------------

const PIXELS = 28; // images in data set are tiny 
const PIXELSSQUARED = PIXELS * PIXELS;

// number of training and test exemplars in the data set:
const NOTRAIN = 60000;
const NOTEST = 10000;

//--- can modify all these --------------------------------------------------

// no of nodes in network 
const noinput = PIXELSSQUARED;
const nohidden = 128; // no of hidden layers
const nooutput = 10;

var learningRate = 0.1;
const LEARNING_MULTIPLIER = 0.01; // Addition of variable learning
const DEFAULT_LEARNING_RATE = 0.1;
var dynamicLearning = true; // Set to true for adaptable learning

// should we train every timestep or not 
let do_training = true;
let show_training = true;
var showBrain = true;
var brainDead = false;

// how many to train and test per timestep 
const TRAINPERSTEP = 60; // 6:1 train v test
const TESTPERSTEP = 10;
var PerStepFactor = 0.2;   // best mixture of drawing and learning

// multiply it by this to magnify for display 
const ZOOMFACTOR = 7;
const ZOOMPIXELS = ZOOMFACTOR * PIXELS;

// 3 rows of
// large image + 50 gap + small image    
// 50 gap between rows 

const canvaswidth = (PIXELS + ZOOMPIXELS) + 50;
const canvasheight = (ZOOMPIXELS * 3) + 50;


const DOODLE_THICK = 15; // thickness of doodle lines 
const DOODLE_BLUR = 6; // blur factor applied to doodles 
const DOODLE_POSTERIZE = 2;
const DOODLE_COLOUR = '#aaaaaa';

var mnist;
// all data is loaded into this 
// mnist.train_images
// mnist.train_labels
// mnist.test_images
// mnist.test_labels


let nn;

let trainrun = 1;
let train_index = 0;

let testrun = 1;
let test_index = 0;
let total_tests = 0;
let total_correct = 0;

// images in LHS:
let doodle, demo;
let doodle_exists = false;
let demo_exists = false;

let mousedrag = false; // are we in the middle of a mouse drag drawing?  


// save inputs to global var to inspect
// type these names in console 
var train_inputs, test_inputs, demo_inputs, doodle_inputs;


// Matrix.randomize() is changed to point to this. Must be defined by user of Matrix. 

// test randomise range at different values

function randomWeight() {
  return (AB.randomFloatAtoB(-0.1, 0.1));
  // Coding Train default is -1 to 1
}



// CSS trick 
// make run header bigger 
$("#runheaderbox").css({
  "max-height": "95vh"
});



//--- start of AB.msgs structure: ---------------------------------------------------------
// We output a series of AB.msgs to put data at various places in the run header 
var thehtml;

var currPredictLayer = 3;
const PREDICT_LAYERS = 6;

// 1 Doodle header 
thehtml = "<hr> <b>doodle:</b><br> " +
  "<button onclick='wipeDoodle();' class='normbutton' >clear doodle</button>" +
  "<button onclick='currPredictLayer=(currPredictLayer==0)?PREDICT_LAYERS:currPredictLayer-1; nn.setPredictionLayer(currPredictLayer);' class='normbutton' >\<</button>" +
  "<button onclick='showBrain = !showBrain;' class='normbutton' >show brain</button>" + 
  "<button onclick='currPredictLayer=(currPredictLayer>=PREDICT_LAYERS)?0:currPredictLayer+1; nn.setPredictionLayer(currPredictLayer);' class='normbutton' >\></button> <br>";

AB.msg(thehtml, 1);

// 2 Doodle variable data (guess)

// 3 Training header
thehtml = "<hr><b>training:</b><br>  " +
  " <button onclick='show_training = !show_training;' class='normbutton' >show</button>" +
  " <button onclick='PerStepFactor = PerStepFactor * 2;' class='normbutton' >-</button>" +
  " <button onclick='PerStepFactor = PerStepFactor /2;' class='normbutton' >+</button>" +
  " <button onclick='dynamicLearning = !dynamicLearning;' class='normbutton' >dyn</button>" +
  " <button onclick='do_training = !do_training;' class='normbutton' >training</button> <br>";



AB.msg(thehtml, 3);

// 4 variable training data 

// 5 Testing header
thehtml = "<b><br>tests:</b> ";
AB.msg(thehtml, 5);

// 6 variable testing data 

// 7 Demo header 
thehtml = "<hr><b>demo:</b><br>" +
  "<button onclick='makeDemo();' class='normbutton' >demo</button> <br> ";
AB.msg(thehtml, 7);

// 8 Demo variable data (random demo ID)
// 9 Demo variable data (changing guess)

const greenspan = "<span style='font-weight:bold; color:darkgreen'> ";
const braindead = "<h1><span style='font-weight:bold; color:red'>BRAIN DEAD</span><h1>";

//--- end of AB.msgs structure: ---------------------------------------------------------




function setup() {
  createCanvas(canvaswidth, canvasheight);

  doodle = createGraphics(ZOOMPIXELS, ZOOMPIXELS); // doodle on larger canvas 
  doodle.pixelDensity(4);

  // JS load other JS 
  // maybe have a loading screen while loading the JS and the data set 

  AB.loadingScreen();

  $.getScript("/uploads/brendanb/matrix.js", function () {
    $.getScript("/uploads/brendanb/nn.js", function () {
      $.getScript("/uploads/brendanb/mnist.js", function () {
        console.log("All JS loaded");
        // nn = new NeuralNetwork( [noinput, nohidden, nohidden/2, nooutput] );
        nn = new NeuralNetwork(noinput, nohidden, nooutput);
        nn.setLearningRate(learningRate);
        loadData();
      });
    });
  });
}

// load data set from local file (on this server)

function loadData() {
  loadMNIST(function (data) {
    mnist = data;
    console.log("All data loaded into mnist object:")
    console.log(mnist);
    AB.removeLoading(); // if no loading screen exists, this does nothing 
  });
}

function getImage(img) // make a P5 image object from a raw data array   
{
  let theimage = createImage(PIXELS, PIXELS); // make blank image, then populate it 
  theimage.loadPixels();

  for (let i = 0; i < PIXELSSQUARED; i++) {
    let bright = img[i]; // greyscale, so RGB the same
    let index = i * 4;
    theimage.pixels[index + 0] = bright;
    theimage.pixels[index + 1] = bright;
    theimage.pixels[index + 2] = bright;
    theimage.pixels[index + 3] = 255;
  }

  theimage.updatePixels();
  return theimage;
}


function getInputs(img) // convert img array into normalised input array 
{

  // todo: rotate image randomly
  let startTracking = false;
  var ignoreRows = 0;

  let inputs = [];

  for (let i = 0; i < PIXELSSQUARED; i++) {

    let bright = img[i];
    inputs[i] = bright / 255; // normalise to 0 to 1
  }
  return (inputs);
}

function getImageFromInputs(inputs) // convert img array into normalised input array 
{

  var imageSize = Math.sqrt(inputs.length).toFixed(); 
  let img = createImage(imageSize, imageSize);
  img.loadPixels();

  // helper for writing color to array
  function writeGreyscale(image, x, y, value) {
    let index = (x + y * image.width) * 4;
    image.pixels[index] = value;
    image.pixels[index + 1] = value;
    image.pixels[index + 2] = value;
    image.pixels[index + 3] = 255;
  }


  let x, y, point = 0;
  // fill with random colors
  for (y = 0; y < img.height; y++) {
    for (x = 0; x < img.width; x++) {
      writeGreyscale(img, x, y, inputs[point++]);
    }
  }
  
  img.updatePixels();
  return(img);
}

var targets_count = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0];

function trainit(show) // train the network with a single exemplar, from global var "train_index", show visual on or off 
{
  let img = mnist.train_images[train_index];
  let label = mnist.train_labels[train_index];

  // optional - show visual of the image 
  if (show && show_training) // && (trainrun > 1))                
  {
    var theimage = getImage(img); // get image from data array 

    image(theimage, 0, ZOOMPIXELS + 25, ZOOMPIXELS, ZOOMPIXELS); // magnified 
    image(theimage, ZOOMPIXELS + 25, ZOOMPIXELS + 50, PIXELS, PIXELS); // original
  }

  // set up the inputs
  let inputs = getInputs(img); // get inputs from data array 

  // set up the outputs
  let targets = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
  targets[label] = 1; // change one output location to 1, the rest stay at 0 
  targets_count[label] += 1;
  // console.log(train_index);
  // console.log(inputs);
  // console.log(targets);

  var plearningRate = learningRate;

  if (dynamicLearning) {
    //  Reduce LearningRate as we become more accurate
    if (accuracy >= 0.90) {
      learningRate = 1;
    } else if (accuracy >= 0.75) {
      learningRate = 5;
    } else if (accuracy >= 0.50) {
      learningRate = 8;
    } else {
      // default rate is 12.5%
      learningRate = 10;
    }


    learningRate = learningRate * LEARNING_MULTIPLIER;
    //forget tiered learning, implement continuous rates
    //        learningRate =  Math.round(1/(accuracy * 100)*100)/100;
    nn.setLearningRate(learningRate);
  }
  else {
      // reset back to default learning rate
      learningRate = DEFAULT_LEARNING_RATE;
      nn.setLearningRate(learningRate);
  }


  train_inputs = inputs; // can inspect in console 
  nn.train(inputs, targets);
        //debug when NaN enters the array

  thehtml = "train: " + trainrun + " / " + train_index +
    "<br>learning rate: " + learningRate;

  let t1 = targets_count.reduce((a, b) => a + b, 0);

  // console.log("the targets are : " + targets_count);
  AB.msg(thehtml, 4);

  train_index++;
  if (train_index == NOTRAIN) {
    train_index = 0;
    console.log("finished trainrun: " + trainrun);
    trainrun++;
  }
}

var accuracy;

function testit() // test the network with a single exemplar, from global var "test_index"
{
  let img = mnist.test_images[test_index];
  let label = mnist.test_labels[test_index];

  // set up the inputs
  let inputs = getInputs(img);

  test_inputs = inputs; // can inspect in console 
  let prediction = nn.predict(inputs); // array of outputs 
  let guess = findMax(prediction); // the top output 

  total_tests++;
  if (guess == label) total_correct++;

  accuracy = (total_correct / total_tests);
  let percent = accuracy * 100;
  /*
      thehtml = "run: <table><tr><td>" + testrun + " ( " + total_correct +
          " / " + total_tests +
          " ) " + greenspan + percent.toFixed(2) + "%</span>";
  */

  thehtml = "<table><tr><th>run</th><th>correct</th><th>tests</th><th>accuracy</th><tr>" +
    "<td>" + testrun + "</td><td>" + total_correct + "</td><td>" + total_tests +
    "</td><td>" + greenspan + percent.toFixed(2) + "%</span></td></tr>";
  AB.msg(thehtml, 6);

  test_index++;
  if (test_index == NOTEST) {
    console.log("finished testrun: " + testrun + " score: " + percent.toFixed(2));
    testrun++;
    test_index = 0;
    total_tests = 0;
    total_correct = 0;
  }
}


//--- find no.1 (and maybe no.2) output nodes ---------------------------------------
// (restriction) assumes array values start at 0 (which is true for output nodes) 


function find12(a) // return array showing indexes of no.1 and no.2 values in array 
{
  let no1 = 0;
  let no2 = 0;
  let no1value = 0;
  let no2value = 0;

  for (let i = 0; i < a.length; i++) {
    if (a[i] > no1value) {
      no1 = i;
      no1value = a[i];
    } else if (a[i] > no2value) {
      no2 = i;
      no2value = a[i];
    }
  }

  var b = [no1, no2];
  return b;
}


// just get the maximum - separate function for speed - done many times 
// find our guess - the max of the output nodes array

function findMax(a) {
  let no1 = 0;
  let no1value = 0;

  for (let i = 0; i < a.length; i++) {
    if (a[i] > no1value) { //temporarily switch from max to min
      no1 = i;
      no1value = a[i];
    }
  }

  return no1;
}


// --- the draw function -------------------------------------------------------------
// every step:

function draw() {
  // check if libraries and data loaded yet:
  if (typeof mnist == 'undefined') return;


  // how can we get white doodle on black background on yellow canvas?
  //        background('#ffffcc');    doodle.background('black');

    background('black');
    stroke(127);
    fill(0, 0, 0);

    rect(0, 0, ZOOMPIXELS, ZOOMPIXELS, 5);
    rect(0, (ZOOMPIXELS * 1) + 24, ZOOMPIXELS + 2, ZOOMPIXELS + 2, 5);
    rect(0, (ZOOMPIXELS * 2) + 48, ZOOMPIXELS + 2, ZOOMPIXELS + 2, 5);

    if (do_training) {
      // do some training per step 
      for (let i = 0; i < (TRAINPERSTEP * PerStepFactor); i++) {
        if (i === 0) trainit(true); // show only one per step - still flashes by  
        else trainit(false);

      }

      // do some testing per step 
      for (let i = 0; i < (TESTPERSTEP * PerStepFactor); i++)
        testit();
    }


    // keep drawing demo and doodle images 
    // and keep guessing - we will update our guess as time goes on 

    if (demo_exists) {
      drawDemo();
      guessDemo();
      

    }

    if (doodle_exists) {
      drawDoodle();
      guessDoodle();
      

    }

  // detect doodle drawing 
  // (restriction) the following assumes doodle starts at 0,0 

  if (mouseIsPressed) // gets called when we click buttons, as well as if in doodle corner  
  {
    // console.log ( mouseX + " " + mouseY + " " + pmouseX + " " + pmouseY );
    var MAX = ZOOMPIXELS + 20; // can draw up to this pixels in corner 
    if ((mouseX < MAX) && (mouseY < MAX) && (pmouseX < MAX) && (pmouseY < MAX)) {
      mousedrag = true; // start a mouse drag 
      doodle_exists = true;
      doodle.stroke(DOODLE_COLOUR); // change colour to match MNIST
      doodle.strokeWeight(DOODLE_THICK);
      doodle.line(mouseX, mouseY, pmouseX, pmouseY);
    }
  } else {
    // are we exiting a drawing
    if (mousedrag) {
      mousedrag = false;
      // console.log ("Exiting draw. Now blurring.");
      // pixelate(doodle, 4);

      doodle.filter(POSTERIZE, DOODLE_POSTERIZE); // run posterize filter to reduce edges
      doodle.filter(BLUR, DOODLE_BLUR); // just blur once 
      
        let theimage = doodle.get();
        
        // best method for matching MNIST is to reduce and enlarge
        theimage.loadPixels();
        theimage.resize(PIXELS, PIXELS);
        theimage.resize(ZOOMPIXELS, ZOOMPIXELS);
        theimage.updatePixels();

      //   console.log (doodle);
    }
  }
  
    if (showBrain){
        var arr = nn.getPredictionArray();
        if (Array.isArray(arr)){
           // var min = Math.min.apply(null, arr), max = Math.max.apply(null, arr);
            var predictArray = arr.scaleBetween(0, 255);
            drawArray(predictArray);
        }
    }

    
    if (isNaN(nn.weights_ho.data[0][0])){
            AB.msg(braindead, 6);
      noLoop();
    }
    
}

// Array to see the current range of the neural network layer
var predictMax = 0;
var predictMin = 0;

// Scale and array between (0 and 255) to turn into image
Array.prototype.scaleBetween = function(scaledMin, scaledMax) {
  predictMax = Math.max.apply(Math, this);
  predictMin = Math.min.apply(Math, this);
  return this.map(num => (scaledMax-scaledMin)*(num-predictMin)/(predictMax-predictMin)+scaledMin);
}

var resizeDoodle = false;


//--- demo -------------------------------------------------------------
// demo some test image and predict it
// get it from test set so have not used it in training


function makeDemo() {
  demo_exists = true;
  var i = AB.randomIntAtoB(0, NOTEST - 1);

  demo = mnist.test_images[i];
  var label = mnist.test_labels[i];

  thehtml = "test[" + i + "]" +
    " = " + label + "<br>";
  AB.msg(thehtml, 8);

  // type "demo" in console to see raw data 
}


function drawDemo() {
  var theimage = getImage(demo);
  //  console.log (theimage);

  image(theimage, 0, canvasheight - ZOOMPIXELS, ZOOMPIXELS, ZOOMPIXELS); // magnified 
  image(theimage, ZOOMPIXELS + 50, canvasheight - ZOOMPIXELS, PIXELS, PIXELS); // original
}


function drawArray(inputs) {
  var arrayImage = getImageFromInputs(inputs);
  image(arrayImage, 40, canvasheight - ZOOMPIXELS+5, ZOOMPIXELS*0.60, ZOOMPIXELS*0.60); // magnified 

}

function guessDemo() {
  let inputs = getInputs(demo);

  demo_inputs = inputs; // can inspect in console 

  let prediction = nn.predict(inputs); // array of outputs 
  let guess = findMax(prediction); // the top output 

  thehtml = "predict: " + greenspan + guess + "</span>";
  AB.msg(thehtml, 9);
}

//--- doodle -------------------------------------------------------------

function drawDoodle() {
  // doodle is createGraphics not createImage
  let theimage = doodle.get();

  image(theimage, 0, 0, ZOOMPIXELS, ZOOMPIXELS); // original 
  image(theimage, ZOOMPIXELS + 50, 0, PIXELS, PIXELS); // shrunk

}


function guessDoodle() {
  // doodle is createGraphics not createImage
  let img = doodle.get();

  // best simulation of MNIST is resize down and up
  img.resize(PIXELS, PIXELS);
  img.loadPixels();

  // set up inputs   
  let inputs = [];
  for (let i = 0; i < PIXELSSQUARED; i++) {
    inputs[i] = img.pixels[i * 4] / 260;    // take the first pixel and "over divide" by 255
  }

  doodle_inputs = inputs; // can inspect in console 

  // feed forward to make prediction 
  let prediction = nn.predict(inputs); // array of outputs 
  let b = find12(prediction); // get no.1 and no.2 guesses  
  showMatrix = false;

    // return predict creates a predictionArray in an array
    // It takes from the weights_ho component of the matrix


  thehtml = "predict 1: " + greenspan + b[0] + "</span><br>" +
    "predict 2: " + greenspan + b[1] + "</span><br>";

    
    switch(currPredictLayer){
        case 0:  thehtml = thehtml + " nn: inputs"; break;
        case 1:  thehtml = thehtml + " nn: weights_ih"; break;
        case 2:  thehtml = thehtml + " nn: bias_h"; break;
        case 3:  thehtml = thehtml + " nn: hidden"; break;
        case 4:  thehtml = thehtml + " nn: weights_ho"; break;
        case 5:  thehtml = thehtml + " nn: bias_o"; break;
        case 6:  thehtml = thehtml + " nn: output"; break;
    }
    var d = (predictMax < 1)?2:0;
    thehtml = thehtml + "[" + predictMin.toFixed(d) + " : " + predictMax.toFixed(d) + "]";

  AB.msg(thehtml, 2);

  // We tried two methods of drawing predictions - this is the best - draw on demo.
  fill(128,128,128);
  for (var pLoop = 0; pLoop < prediction.length; pLoop++) {
    var value = prediction[pLoop].toFixed(1) * 10;
    let w = 10;
    let h = 5 * (value + 0.2);
    let x = (pLoop * 18) + 10;
    let y = (ZOOMPIXELS * 3) + 48 - h;

    rect(x, y, w, h);
  }
}


var showMatrix = false;

function wipeDoodle() {
  doodle_exists = false;
  doodle.background('black');
}


// --- debugging --------------------------------------------------
// in console
// showInputs(demo_inputs);
// showInputs(doodle_inputs);

function showInputs(inputs)
// display inputs row by row, corresponding to square of pixels 
{
  var str = "";
  for (let i = 0; i < inputs.length; i++) {
    if (i % PIXELS === 0) str = str + "\n"; // new line for each row of pixels 
    var value = inputs[i];
    str = str + " " + value.toFixed(2);
  }
  console.log(str);
}

// Experiment for improvement of drawing accuracy
// MNIST uses anti-aliasing on the reduces images, but this
// Did not work as well as double resize!
function pixelate(doodle, sample_size) {

  var image = doodle.pixels;

  var w = ZOOMPIXELS;
  var h = ZOOMPIXELS;

  for (var y = 0; y < h; y += sample_size) {
    for (var x = 0; x < w; x += sample_size) {

      var pos = (x + y * w) * 4;
      var red = doodle[pos];
      var green = doodle[pos + 1];
      var blue = doodle[pos + 2];

      for (var n = 1; n < sample_size; n++) {
        doodle[pos + (4 * n) + 0] = red;
        doodle[pos + (4 * n) + 1] = green;
        doodle[pos + (4 * n) + 2] = blue;
      }
    }
  }
}