//Sanchit Shreeprakash Akhauri
// 20210205 , CA686
// Cloned by Sanchit Akhauri on 29 Nov 2020 from World "Character recognition neural network" by "Coding Train" project
// Please leave this clone trail here.
// Port of Character recognition neural network from here:
// https://github.com/CodingTrain/Toy-Neural-Network-JS/tree/master/examples/mnist
// with many modifications
const PIXELS = 28; // images in data set are tiny
const PIXELSSQUARED = PIXELS * PIXELS;
let RESIZEDPIXELS = 24; // Sanchit : - Cropped Image Pexel
//--- can modify all these --------------------------------------------------
// multiply it by this to magnify for display
const ZOOMFACTOR = 7;
const ZOOMPIXELS = ZOOMFACTOR * PIXELS;
// 3 rows of
// large image + 50 gap + small image
// 50 gap between rows
const canvaswidth = (PIXELS + ZOOMPIXELS) + 50;
const canvasheight = ZOOMPIXELS + 10;
const DOODLE_THICK = 18; // thickness of doodle lines
const DOODLE_BLUR = 0; // Sanchit : - blur factor disabled
// images in LHS:
let doodle, demo;
let doodle_exists = false;
let mousedrag = false; // are we in the middle of a mouse drag drawing?
// save inputs to global var to inspect
// type these names in console
var doodle_inputs;
//--- start of AB.msgs structure: ---------------------------------------------------------
// We output a serious of AB.msgs to put data at various places in the run header
var thehtml;
// 1 Doodle header
thehtml = "<hr> <h1> 1. Doodle </h1> Top row: Doodle (left) and shrunk (right). <br> " +
" Draw your doodle in top LHS. <button onclick='wipeDoodle();' class='normbutton' >Clear doodle</button> <br> ";
AB.msg(thehtml, 1);
const greenspan = "<span style='font-weight:bold; font-size:x-large; color:darkgreen'> ";
//--- end of AB.msgs structure: ---------------------------------------------------------
function setup() {
createCanvas(canvaswidth, canvasheight);
doodle = createGraphics(ZOOMPIXELS, ZOOMPIXELS); // doodle on larger canvas
doodle.pixelDensity(1);
// JS load other JS
// maybe have a loading screen while loading the JS and the data set
AB.loadingScreen();
uploadFiles(); // Sanchit :- Call all the scripts
}
// Sanchit :- Upload Scripts
function uploadFiles(){
$.getScript("/uploads/codingtrain/matrix.js", function () {
$.getScript("/uploads/codingtrain/nn.js", function () {
$.getScript("/uploads/codingtrain/mnist.js", function () {
$.getScript("/uploads/sanchit08/webcnn.js", function () { // Sanchit :- Load webCNN JS file to implement CNN
$.getScript("/uploads/sanchit08/mathutils.js", function () { // Sanchit :- Instantiates a pseudo-random number
$.ajax({
url: "/uploads/sanchit08/cnn_mnist_10_20_98accuracy.json", // Sanchit :- Loads pre-recorded weights to give high accuracy
dataType: "json",
success: loadWeights
});
});
});
});
});
});
}
function loadWeights(response) {
loadNetworkFromWeights(response); // Sanchit :- Initialize CNN
AB.removeLoading();
}
// Sanchit :- Initialize CNN
function loadNetworkFromWeights(networkJSON) {
cnn = new WebCNN();
// Initialize All the layers from JSON File
for (var i = 0; i < networkJSON.layers.length; ++i) {
let layerDesc = networkJSON.layers[i];
cnn.newLayer(layerDesc);
}
for (var j = 0; j < networkJSON.layers.length; ++j) { // initialize weights and biases
let layerDesc = networkJSON.layers[j];
switch (networkJSON.layers[j].type) {
case LAYER_TYPE_CONV:
case LAYER_TYPE_FULLY_CONNECTED:
{
if (layerDesc.weights != undefined && layerDesc.biases != undefined) {
cnn.layers[j].setWeightsAndBiases(layerDesc.weights, layerDesc.biases);
}
break;
}
}
}
cnn.initialize();
}
function getImage(img, size = PIXELS) // make a P5 image object from a raw data array
{
let theimage = createImage(PIXELS, PIXELS); // make blank image, then populate it
theimage.loadPixels();
for (let i = 0; i < size * size; i++) {
let bright = img[i];
let index = i * 4;
theimage.pixels[index + 0] = bright;
theimage.pixels[index + 1] = bright;
theimage.pixels[index + 2] = bright;
theimage.pixels[index + 3] = 255;
}
theimage.updatePixels();
return theimage;
}
function getInputs(img) // convert img array into normalised input array
{
let inputs = [];
for (let i = 0; i < PIXELSSQUARED; i++) {
let bright = img[i];
inputs[i] = bright / 255; // normalise to 0 to 1
}
return (inputs);
}
//--- find no.1 (and maybe no.2) output nodes ---------------------------------------
// (restriction) assumes array values start at 0 (which is true for output nodes)
function find12(predictionValue) // return array showing indexes of no.1 and no.2 values in array
{
let Guess1 = 0;
let Guess2 = 0;
let Value1 = 0;
let Value2 = 0;
for (let i = 0; i < 10; i++) {
let predictedVal = predictionValue[0].getValue(0, 0, i);
if (predictedVal > Value1) {
Guess1 = i;
Value1 = predictedVal;
}
}
for (let i = 0; i < 10; i++) {
let predictedVal = predictionValue[0].getValue(0, 0, i);
if ((Guess1 != i) && (predictedVal > Value2)) {
Guess2 = i;
Value2 = predictedVal;
}
}
return [Guess1, Guess2];
}
// --- the draw function -------------------------------------------------------------
// every step:
function draw() {
background('black');
if (doodle_exists) {
drawDoodle();
guessDoodle();
}
// detect doodle drawing
// (restriction) the following assumes doodle starts at 0,0
if (mouseIsPressed) // gets called when we click buttons, as well as if in doodle corner
{
// console.log ( mouseX + " " + mouseY + " " + pmouseX + " " + pmouseY );
var MAX = ZOOMPIXELS + 20; // can draw up to this pixels in corner
if ((mouseX < MAX) && (mouseY < MAX) && (pmouseX < MAX) && (pmouseY < MAX)) {
mousedrag = true; // start a mouse drag
doodle_exists = true;
doodle.stroke('white');
doodle.strokeWeight(DOODLE_THICK);
doodle.line(mouseX, mouseY, pmouseX, pmouseY);
}
}
else {
// are we exiting a drawing
if (mousedrag) {
mousedrag = false;
// console.log ("Exiting draw. Now blurring.");
doodle.filter(BLUR, DOODLE_BLUR); // just blur once
// console.log (doodle);
}
}
}
//--- doodle -------------------------------------------------------------
function drawDoodle() {
// doodle is createGraphics not createImage
let theimage = doodle.get();
//console.log (theimage);
image(theimage, 0, 0, ZOOMPIXELS, ZOOMPIXELS); // original
image(theimage, ZOOMPIXELS + 50, 0, PIXELS, PIXELS); // shrunk
}
// Sanchit :- Returns the Object containg image, height and width
function changeFormat(image, size) {
return {"width": size,"height": size,"data": getImage(cropImage(image, size), size).pixels
}
}
// Sanchit :- Crop image from 28*28 to 24*24
function cropImage(img, size) {
const startIndex = PIXELS - size;
let randomX = Math.floor(Math.random() * startIndex);
let randomY = Math.floor(Math.random() * startIndex);
const Newpixels = size;
let xEndIndex = randomX + Newpixels;
let yEndIndex = randomY + Newpixels;
let inputs = [];
for (let i = randomX; i < xEndIndex; i++) {
for (let j = randomY; j < yEndIndex; j++) {
inputs.push(img[i * PIXELS + j])
}
}
return inputs;
}
function guessDoodle() {
// doodle is createGraphics not createImage
let img = doodle.get();
img.resize(PIXELS, PIXELS);
img.loadPixels();
// set up inputs
let inputs = [];
for (let i = 0; i < PIXELSSQUARED; i++) {
inputs[i] = img.pixels[i * 4] / 255;
}
doodle_inputs = inputs; // can inspect in console
let centeredImage = positionImageCenter(img.pixels, PIXELS); // Sanchit :- positions the image in center.
let format = changeFormat(centeredImage, RESIZEDPIXELS)
// feed forward to make prediction
let prediction = cnn.classifyImages([format]); // Sanchit :- Predice the image
let b = find12(prediction); // get no.1 and no.2 guesses
thehtml = " We classify it as: " + greenspan + b[0] + "</span> <br>" +
" No.2 guess is: " + greenspan + b[1] + "</span>";
AB.msg(thehtml, 2);
}
//Sanchit : - Utility function to convert 1D pixel array to 2D array
function convertTo2D(pixels,size){
let array_1D = []
for (let i = 0; i < size; i++) {
array_1D[i] = [];
for (let j = 0; j < size; j++) {
array_1D[i][j] = pixels[(i * size + j) * 4];
}
}
return array_1D;
}
// Sanchit :- Calculate the indexes of corners of doodle
function calculateCornersIndex(matrix,top,left,bottom,right){
for (var y = 0; y < matrix.length; y++) {
var l = matrix[y].indexOf(255);
var r = matrix[y].lastIndexOf(255);
if (l >= 0 && l < left) left = l;
if (r >= 0 && r > right) right = r;
if (l >= 0 && y < top) top = y;
if (l >= 0 && y > bottom) bottom = y;
}
return [left,right,top,bottom];
}
//Sanchit :- Position the image to center
function positionImageCenter(pixels, size) {
//convert to matrix
let matrix = [];
matrix = convertTo2D(pixels,size);
// Calculate the indexes of the four corners
let positions = calculateCornersIndex(matrix,Number.MAX_VALUE,Number.MAX_VALUE,-1,-1);
//Center Points
let Y = Math.floor((size - positions[3] - positions[2]) / 2);
let X = Math.floor((size - positions[1] - positions[0]) / 2);
//Create the new array
let new2DArray = Array(size).fill().map(() => Array(size).fill(0));
for (i = positions[2]; i <= positions[3]; i++) {
for (j = positions[0]; j <= positions[1]; j++) {
new2DArray[i + Y][j + X] = matrix[i][j];
}
}
//Convert to 1D array
let result = [];
for (let i = 0; i < size; i++) {
for (let j = 0; j < size; j++) {
result[i * size + j] = new2DArray[i][j];
}
}
return result;
}
function wipeDoodle() {
doodle_exists = false;
doodle.background('black');
}
// --- debugging --------------------------------------------------
// in console
// showInputs(demo_inputs);
// showInputs(doodle_inputs);
function showInputs(inputs)
// display inputs row by row, corresponding to square of pixels
{
var str = "";
for (let i = 0; i < inputs.length; i++) {
if (i % PIXELS == 0) str = str + "\n"; // new line for each row of pixels
var value = inputs[i];
str = str + " " + value.toFixed(2);
}
console.log(str);
}