Code viewer for World: Character recognition neur...
// Cloned by K.Ellis on 9 Dec 2019 from World "Character recognition neural network" by "Coding Train" project 
// Please leave this clone trail here.
 

// Port of Character recognition neural network from here:
// https://github.com/CodingTrain/Toy-Neural-Network-JS/tree/master/examples/mnist
// with many modifications 


// --- defined by MNIST - do not change these ---------------------------------------

const PIXELS        = 28;                       // images in data set are tiny 
const PIXELSSQUARED = PIXELS * PIXELS;

// number of training and test exemplars in the data set:
const NOTRAIN = 60000;
const NOTEST  = 10000;



//--- can modify all these --------------------------------------------------

// no of nodes in network 
const noinput  = PIXELSSQUARED;
const nohidden = 24; //KE - was 64
const nooutput = 10;

const learningrate = 0.1;   // default 0.1  

// should we train every timestep or not 
let do_training = true;

// how many to train and test per timestep 
const TRAINPERSTEP = 30; 
const TESTPERSTEP  = 5;  

// multiply it by this to magnify for display 
const ZOOMFACTOR    = 7;                        
const ZOOMPIXELS    = ZOOMFACTOR * PIXELS; 

// 3 rows of
// large image + 50 gap + small image    
// 50 gap between rows 

//const canvaswidth = ( PIXELS + ZOOMPIXELS ) + 60;
const canvaswidth = ( ZOOMPIXELS * 2 ) + 120;
const canvasheight = ( ZOOMPIXELS * 3 ) + 102;

const doodlewidth = (PIXELS);

const DOODLE_THICK = 15;    // thickness of doodle lines 
const DOODLE_BLUR = 3;      // blur factor applied to doodles 


let mnist;      
// all data is loaded into this 
// mnist.train_images
// mnist.train_labels
// mnist.test_images
// mnist.test_labels


let nn;

//KE added variables
let canvas, dst, src, hierarchy, contours, img, diffX, diffY, M; 
let cvOutput = [];

let trainrun = 1;
let train_index = 0;

let testrun = 1;
let test_index = 0;
let total_tests = 0;
let total_correct = 0;

// images in LHS:
let doodle, demo;
let doodle_exists = false;
let demo_exists = false;
let mousedrag = false;      // are we in the middle of a mouse drag drawing?  


// save inputs to global var to inspect
// type these names in console 
var train_inputs, test_inputs, demo_inputs, doodle_inputs;


// Matrix.randomize() is changed to point to this. Must be defined by user of Matrix. 

function randomWeight()
{
    return ( AB.randomFloatAtoB ( -0.5, 0.5 ) ); //-0.5, 0.5
            // Coding Train default is -1 to 1
}    



// CSS trick 
// make run header bigger 
 $("#runheaderbox").css ( { "max-height": "95vh" } );



//--- start of AB.msgs structure: ---------------------------------------------------------
// We output a serious of AB.msgs to put data at various places in the run header 
var thehtml;

  // 1 Doodle header 
  thehtml = "<hr> <h1> 1. Doodle </h1> Top row: Doodle (left) and shrunk (right). <br> " +
        " Draw your doodle in top LHS. <button onclick='wipeDoodle();' class='normbutton' >Clear doodle</button> <br> ";
   AB.msg ( thehtml, 1 );

  // 2 Doodle variable data (guess)
  
  // 3 Training header
  thehtml = "<hr> <h1> 2. Training </h1> Middle row: Training image magnified (left) and original (right). <br>  " +
        " <button onclick='do_training = false;' class='normbutton' >Stop training</button> <br> ";
  AB.msg ( thehtml, 3 );
     
  // 4 variable training data 
  
  // 5 Testing header
  thehtml = "<h3> Hidden tests </h3> " ;
  AB.msg ( thehtml, 5 );
           
  // 6 variable testing data 
  
  // 7 Demo header 
  thehtml = "<hr> <h1> 3. Demo </h1> Bottom row: Test image magnified (left) and  original (right). <br>" +
        " The network is <i>not</i> trained on any of these images. <br> " +
        " <button onclick='makeDemo();' class='normbutton' >Demo test image</button> <br> ";
   AB.msg ( thehtml, 7 );
   
  // 8 Demo variable data (random demo ID)
  // 9 Demo variable data (changing guess)
  
const greenspan = "<span style='font-weight:bold; font-size:x-large; color:darkgreen'> "  ;

//--- end of AB.msgs structure: ---------------------------------------------------------




function setup() 
{
    
    canvas = createCanvas ( canvaswidth, canvasheight);
    canvas.position(10,20); // just to shift away from edge
    doodle = createGraphics ( ZOOMPIXELS, ZOOMPIXELS );       // doodle on larger canvas 
    doodle.pixelDensity(1); // MNIST images are pixulated so keep at 1 or blank
    wipeDoodle(); //KE calling wipe to have doodle prepped from load 
    
    //KE just some boxes for our boxes & some text prompts
    rect(0, 0, ZOOMPIXELS, ZOOMPIXELS); //doodle
        textSize(16);
        textAlign(CENTER);
        text('DRAW A DIGIT', ZOOMPIXELS /2, ZOOMPIXELS/2.2);
        textAlign(CENTER);
        text('BETWEEN 0-9', ZOOMPIXELS /2, ZOOMPIXELS/1.8);
        
    rect(0,                 ZOOMPIXELS + 50, ZOOMPIXELS, ZOOMPIXELS); //train
    
    rect(0, canvasheight - ZOOMPIXELS - 2, ZOOMPIXELS, ZOOMPIXELS); // demo
        textSize(16);
        textAlign(CENTER);
        text('     CLICK     ', 100, canvasheight - (ZOOMPIXELS / 1.8));
        text('\'DEMO TEST IMAGE\'', 100, canvasheight - (ZOOMPIXELS / 2.2));

    
// JS load other JS 
// maybe have a loading screen while loading the JS and the data set 
    AB.loadingScreen();
  
        $.getScript ( "/uploads/ke737/opencv_3.3.1.js", function()
        {
             $.getScript ( "/uploads/ke737/matrix.js", function()
             {
               $.getScript ( "/uploads/ke737/nn.js", function()
               {
                    $.getScript ( "/uploads/codingtrain/mnist.js", function()
                    {
                        console.log ("All JS loaded");
                        nn = new NeuralNetwork(  noinput, nohidden, nooutput );
                        nn.setLearningRate ( learningrate );
                        loadData();
                    });
               });
             });
        });
  }


// load data set from local file (on this server)

function loadData()    
{
  loadMNIST ( function(data)    
  {
    mnist = data;
    console.log ("All data loaded into mnist object:");
    console.log(mnist);
    AB.removeLoading();     // if no loading screen exists, this does nothing 
  });
}



function getImage ( img )      // make a P5 image object from a raw data array   
{
    let theimage  = createImage (PIXELS, PIXELS);    // make blank image, then populate it 
    theimage.loadPixels();  
    
    for (let i = 0; i < PIXELSSQUARED ; i++) 
    {
        let bright = img[i];
        let index = i * 4;
        //console.log("index" + index);
        theimage.pixels[index + 0] = bright;
        theimage.pixels[index + 1] = bright;
        theimage.pixels[index + 2] = bright;
        theimage.pixels[index + 3] = 255;
    }
    
    theimage.updatePixels();
    return theimage;
}



function getInputs ( img )      // convert img array into normalised input array 
{
    let inputs = [];
    for (let i = 0; i < PIXELSSQUARED ; i++)          
    {
        let bright = img[i];
        inputs[i] = bright / 255;       // normalise to 0 to 1,
        
    } 
    return ( inputs );
    
}

 

function trainit (show)        // train the network with a single exemplar, from global var "train_index", show visual on or off 
{
  let img   = mnist.train_images[train_index];
  let label = mnist.train_labels[train_index];
  
  // optional - show visual of the image 
  if (show)                
  {
    var theimage = getImage ( img );    // get image from data array 
    image ( theimage,   0,                ZOOMPIXELS+50,    ZOOMPIXELS,     ZOOMPIXELS  );      // magnified 
    image ( theimage,   ZOOMPIXELS+50,    ZOOMPIXELS+50,    PIXELS,         PIXELS      );      // original
  }

  // set up the inputs
  let inputs = getInputs ( img );       // get inputs from data array

  // set up the outputs
  let targets = [0, 0, 0, 0, 0, 0, 0, 0, 0, 0];
  targets[label] = 1;       // change one output location to 1, the rest stay at 0 

  train_inputs = inputs;        // can inspect in console 
  nn.train ( inputs, targets );

  thehtml = " trainrun: " + trainrun + "<br> no: " + train_index ;
  AB.msg ( thehtml, 4 );

  train_index++;
  if ( train_index == NOTRAIN ) 
  {
    train_index = 0;
    console.log( "finished trainrun: " + trainrun );
    trainrun++;
  }
}



function testit()    // test the network with a single exemplar, from global var "test_index"
{ 
  let img   = mnist.test_images[test_index];
  let label = mnist.test_labels[test_index];

  // set up the inputs
  let inputs = getInputs ( img ); 
  
  test_inputs = inputs;        // can inspect in console 
  let prediction    = nn.predict(inputs);       // array of outputs 
  let guess         = findMax(prediction);      // the top output 

  total_tests++;
  if (guess == label)  total_correct++;

  let percent = (total_correct / total_tests) * 100 ;
  
  thehtml =  " testrun: " + testrun + "<br> no: " + total_tests + " <br> " +
        " correct: " + total_correct + "<br>" +
        "  score: " + greenspan + percent.toFixed(2) + "</span>";
  AB.msg ( thehtml, 6 );

  test_index++;
  if ( test_index == NOTEST ) 
  {
    console.log( "finished testrun: " + testrun + " score: " + percent.toFixed(2) );
    testrun++;
    test_index = 0;
    total_tests = 0;
    total_correct = 0;
  }
}




//--- find no.1 (and maybe no.2) output nodes ---------------------------------------
// (restriction) assumes array values start at 0 (which is true for output nodes) 

//KE changed this to list top 3 predictions
function find123 (a)         // return array showing indexes of no.1 and no.2 values in array 
{
  let no1 = 0;
  let no2 = 0;
  let no3 = 0;
  let no1value = 0;     
  let no2value = 0;
  let no3value = 0;
  
  for (let i = 0; i < a.length; i++) 
  {
    if (a[i] > no1value) 
    {
      no1 = i;
      no1value = a[i];
    }
    else if (a[i] > no2value) 
    {
      no2 = i;
      no2value = a[i];
    }
     else if (a[i] > no3value) 
    {
      no3 = i;
      no3value = a[i];
    }
  }
  
  var b = [ no1, no2, no3 ];
  return b;
}


// just get the maximum - separate function for speed - done many times 
// find our guess - the max of the output nodes array

function findMax (a)        
{
  let no1 = 0;
  let no1value = 0;     
  
  for (let i = 0; i < a.length; i++) 
  {
    if (a[i] > no1value) 
    {
      no1 = i;
      no1value = a[i];
    }
  }
  return no1;
}




// --- the draw function -------------------------------------------------------------
// every step:
 
function draw() 
{
    
    // check if libraries and data loaded yet:
    if ( typeof mnist == 'undefined' ) return;


// how can we get white doodle on black background on yellow canvas?
//        background('#ffffcc');    doodle.background('black');

    //background ('black'); //KE edited out so as to apply the following instead
     rect(0, 0, ZOOMPIXELS, ZOOMPIXELS); //doodle
        textSize(16);
        textAlign(CENTER);
        text('DRAW A DIGIT', ZOOMPIXELS /2, ZOOMPIXELS/2.2);
        textAlign(CENTER);
        text('BETWEEN 0-9', ZOOMPIXELS /2, ZOOMPIXELS/1.8);
        
   
if ( do_training )    
{
  // do some training per step 
    for (let i = 0; i < TRAINPERSTEP; i++) 
    {
      if (i === 0)    trainit(true);    // show only one per step - still flashes by  
      else           trainit(false);
    }
    
  // do some testing per step 
    for (let i = 0; i < TESTPERSTEP; i++) 
      testit();
}



  // keep drawing demo and doodle images 
  // and keep guessing - we will update our guess as time goes on 
  
  if ( demo_exists ) 
  {
    drawDemo();
    guessDemo();
  }
  if ( doodle_exists ) 
  {
    drawDoodle();
    guessDoodle();
  }


// detect doodle drawing 
// (restriction) the following assumes doodle starts at 0,0 

  if ( mouseIsPressed )         // gets called when we click buttons, as well as if in doodle corner  
  {
          
     // console.log ( mouseX + " " + mouseY + " " + pmouseX + " " + pmouseY );
     var MAX = ZOOMPIXELS  - 2;     // can draw up to this pixels in corner 
     if ( (mouseX < MAX) && (mouseY < MAX) && (pmouseX < MAX) && (pmouseY < MAX) )
     {
        mousedrag = true;       // start a mouse drag 
        doodle_exists = true;
        doodle.stroke('cream');
        strokeJoin(BEVEL);
        doodle.strokeWeight( DOODLE_THICK );
        doodle.line(mouseX, mouseY, pmouseX, pmouseY);      
     }
  }
  else 
  {
      // are we exiting a drawing
      if ( mousedrag )
      {
            mousedrag = false;
            
            console.log ("Doodle detected");
            
            //KE main addition to code follows
            // using opencv.js to find contours, centre of mass, bounding box
            // & using warp affine to shift our image
        
            /*Applying opencv.js_3.3.1 lib
            Contours are key to recognition:
            https://docs.opencv.org/master/d5/daa/tutorial_js_contours_begin.html
            https://docs.opencv.org/master/d0/d43/tutorial_js_table_of_contents_contours.html
            https://docs.opencv.org/3.4/dc/dcf/tutorial_js_contour_features.html
            */

            // Common lines for Contours, Moments, Bounding Box etc
            // added next 5 lines as cannot use imread
	        img = doodle.get(); //get();
	        img.resize(PIXELS, PIXELS);
	        img.loadPixels();
                //console.log(img);
            imagedata = img.imageData;
            src = cv.matFromImageData(imagedata);
            //src = cv.imread('canvasInput'); //issue using cv.imread, think because doodle graphic not canvas
            dst = cv.Mat.zeros(src.cols, src.rows, cv.CV_8UC3);
            cv.cvtColor(src, src, cv.COLOR_RGBA2GRAY, 0);
            cv.threshold(src, src, 120, 255, cv.THRESH_BINARY); // 120 , 200
            contours = new cv.MatVector();
            hierarchy = new cv.Mat();
            cv.findContours(src, contours, hierarchy, cv.RETR_CCOMP, cv.CHAIN_APPROX_SIMPLE);
            
            
            ///  ** Moments** 
            let cnt = contours.get(0);
            let Moments = cv.moments(cnt, false);  
                //console.log("Moments.m00 = " + Moments.m00);
            M = Moments.m00; //KE this is our centre of mass of the object, understanding its position relative to 14,14 is key
                
            let cx = Math.round((Moments.m10/Moments.m00) * 100) / 100;
            let cy = Math.round((Moments.m01/Moments.m00) * 100) / 100;
                //console.log("cx = " + cx + ", cy = " + cy); // print the Centroid for number X & Y axis
            let x = PIXELS/2; //
            let y = PIXELS/2;
                //console.log("x = " + x + ", y = " + y);
            diffX = Math.round(x - cx); // this is my X axis offset for centering, the centre of mass of doodle
            diffY = Math.round(y - cy); // this is my Y axis offset for centering, the centre of mass of doodle
                //console.log("diff x = " + diffX + ", diff y = " + diffY); 
            
           
            /*
            // ** Rect bounding box **
            let rect = cv.boundingRect(cnt);
            let contoursColor = new cv.Scalar(255, 255, 255);
            let rectangleColor = new cv.Scalar(255, 0, 0);
            cv.drawContours(dst, contours, 0, contoursColor, 1, 8, hierarchy, 100);
            let point1 = new cv.Point(rect.x, rect.y);
            let point2 = new cv.Point(rect.x + rect.width, rect.y + rect.height);
            cv.rectangle(dst, point1, point2, rectangleColor, 2, cv.LINE_AA, 0);
                //console.log("bounding box before offset");
                //console.log(rect);
                //console.log("bounding box after offset");
            rect.x = Math.round(rect.x + diffX);
            rect.y = Math.round(rect.y + diffY);
                //console.log(rect);
            diffH = rect.height + (20 - rect.height);*/
  
            // ** warp Affine transform **
            // KE - key is to apply matrix with X-axis & Y-axis shift from Moments
            let Matrix = cv.matFromArray(2, 3, cv.CV_64FC1, [1, 0, diffX, 0, 1, diffY]); 
            dsize = new cv.Size(src.rows, src.cols);
            cv.warpAffine(src, dst, Matrix, dsize, cv.INTER_LINEAR, cv.BORDER_CONSTANT, new cv.Scalar());
            //cv.warpAffine(src, dst, Matrix, dsize, cv.INTER_AREA, cv.BORDER_CONSTANT, new cv.Scalar());
            
            cvOutput = getInputs ( dst.data8S); 
           
            // KE - Print shifted image to screen
            
            image ( img,   (ZOOMPIXELS + 120) + (diffX * ZOOMFACTOR), 0 + (diffY * ZOOMFACTOR), ZOOMPIXELS, ZOOMPIXELS);
            
            
          
            //KE - These are the different filters available vi p5.js https://p5js.org/reference/#/p5/filter
            //     found that none where needed to match demo tests level of accuracy
            //     These are operations on doodle graphic whereas we sending back offset image data for prediction
            
            /*
            /// 
            doodle.filter(THRESHOLD, 0.5); //THRESHOLD Converts the image to black and white pixels depending if they are above or below the threshold defined by the level parameter. The parameter must be between 0.0 (black) and 1.0 (white). If no level is specified, 0.5 is used.
            doodle.filter(GRAY); // Converts any colors in the image to grayscale equivalents. No parameter is used.
            doodle.filter(OPAQUE); //OPAQUE Sets the alpha channel to entirely opaque. No parameter is used.
            doodle.filter(INVERT); //INVERT Sets each pixel to its inverse value. No parameter is used.
            doodle.filter(POSTERIZE, 255); //2 -255 Limits each channel of the image to the number of colors specified as the parameter. The parameter can be set to values between 2 and 255, but results are most noticeable in the lower ranges.
            doodle.filter(DILATE); //Increases the light areas. No parameter is used.
            doodle.filter(BLUR, DOODLE_BLUR); // just blur once // BLUR implements Gaussian blur
            doodle.filter(ERODE); //Reduces the light areas. No parameter is used.
           */
           
            console.log('%c ***** DOODLE CENTERED  ***** ', 'background: #222; color: #bada55');
           
            return cvOutput; 
            
      }
  
  }
}




//--- demo -------------------------------------------------------------
// demo some test image and predict it
// get it from test set so have not used it in training




function makeDemo()
{
    demo_exists = true;
    var  i = AB.randomIntAtoB ( 0, NOTEST - 1 );  
    
    demo        = mnist.test_images[i]; 
    
    var label   = mnist.test_labels[i];
    
   thehtml =  "Test image no: " + i + "<br>" + 
            "Classification: " + label + "<br>" ;
   AB.msg ( thehtml, 8 );
   
   // type "demo" in console to see raw data 
}


function drawDemo()
{
    var theimage = getImage ( demo );
     //  console.log (theimage); //normalised 0 -255
     
    image ( theimage,   0,                canvasheight - (ZOOMPIXELS + 2),    ZOOMPIXELS,     ZOOMPIXELS  );      // magnified 
    image ( theimage,   ZOOMPIXELS+50,    canvasheight - ZOOMPIXELS,    PIXELS,         PIXELS      );      // original
}


function guessDemo()
{
   let inputs = getInputs ( demo ); 
   
  demo_inputs = inputs;  // can inspect in console 
  
  let prediction    = nn.predict(inputs);       // array of outputs 
  let guess         = findMax(prediction);      // the top output 

   thehtml =   " We classify it as: " + greenspan + guess + "</span>" ;
   AB.msg ( thehtml, 9 );
}




//--- doodle -------------------------------------------------------------

function drawDoodle()
{
    // doodle is createGraphics not createImage
    let theimage = doodle.get();
    
    image ( theimage,   0,                0,    ZOOMPIXELS,     ZOOMPIXELS  );      // original 
    image ( theimage,   ZOOMPIXELS+50,    0,    PIXELS,         PIXELS      );      // shrunk

}


function guessDoodle() 
{
   // doodle is createGraphics not createImage
  img = doodle.get();
  img.resize ( PIXELS, PIXELS ); 
  img.loadPixels();

  // set up inputs  
  let inputs = [];
  for (let i = 0; i < PIXELSSQUARED ; i++) 
  {
     inputs[i] = img.pixels[i * 4] / 255; // doodle_input values sent for classifcation/prediction
  }
  

  //KE - copy & then overwrite array for doodle inputs
  let ke = Array.from(inputs);
  
  //KE - overwrite inputs with our cvOutput
  for (let i = 0; i < ke.length ; i++) 
  {
     ke[i] = cvOutput[i];
  }
  
  //KE - copy array back to inputs & normalise
  inputs = Array.from(ke);
  for (let i = 0; i < inputs.length ; i++) 
  {
     inputs[i] = inputs[i] * -255;
  }
    // can inspect in console 
    doodle_inputs = inputs;

  // feed forward to make prediction 
  let prediction    = nn.predict(inputs);  // KE - now inputs are not doodle.get() but rather cvOutput
  let b             = find123(prediction); // KE - get no.1, no.2 & no.3 guesses  
  
  //console.log("guess # 1,2,3 are: " + b[0] + " , " + b[1] + " , " + b[2]);

  thehtml =   " We classify it as: " + greenspan + b[0] + "</span> <br>" +
            " No.2 guess is: " + greenspan + b[1] + "</span> <br>" +
            "No.3 guess is: " + greenspan + b[2] + "</span>";
  AB.msg ( thehtml, 2 );
  AB.msg ( thehtml, 2 );
}


function wipeDoodle()    
{
    doodle_exists = false; 
    doodle.background('black');
}




// --- debugging --------------------------------------------------
// in console
// showInputs(demo_inputs);
// showInputs(doodle_inputs);


function showInputs ( inputs )
// display inputs row by row, corresponding to square of pixels 
{
    var str = "";
    for (let i = 0; i < inputs.length; i++) 
    {
      if ( i % PIXELS === 0 )    str = str + "\n";                                   // new line for each row of pixels 
      var value = inputs[i];
      str = str + " " + value.toFixed(2) ; 
    }
    console.log (str);
}