// Cloned by Colin McCabe on 20 Nov 2022 from World "Character recognition neural network" by "Coding Train" project
// Please leave this clone trail here.
// Port of Character recognition neural network from here:
// https://github.com/CodingTrain/Toy-Neural-Network-JS/tree/master/examples/mnist
// with many modifications
// --------------------------------------------------CNN
// ML5 image recognition using MobileNet
// uses AB framework
// enter URL of image at runtime
// no run, pause, step
AB.drawRunControls = false;
const PIXELS = 28; // images in data set are tiny
const PIXELSSQUARED = PIXELS * PIXELS;
let doodle, demo;
let doodle_exists = false;
let mousedrag = false;
const ZOOMFACTOR = 10;
const ZOOMPIXELS = ZOOMFACTOR * PIXELS;
let resultsDiv;
let inputImage;
let ml5nn;
let doodle_inputs;
let img = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20,114,125,82,37,37,39,115,157,94,21,10,32,37,37,37,37,37,37,37,82,125,114,20,0,0,0,3,109,245,249,233,217,217,217,245,251,234,172,142,203,217,217,217,217,217,217,217,233,249,245,109,3,0,0,4,114,253,254,254,254,254,254,254,254,254,252,251,254,254,254,254,254,254,254,254,254,254,254,127,4,0,0,0,45,214,233,251,254,254,254,254,255,255,255,255,255,255,255,255,255,255,255,255,254,254,251,113,4,0,0,0,0,5,22,82,128,160,184,217,217,218,249,254,255,255,255,255,255,254,254,247,216,170,83,7,0,0,0,0,0,0,0,2,5,16,25,37,40,50,227,254,255,255,254,254,252,245,220,159,38,21,3,0,0,0,0,0,0,0,0,0,0,0,0,9,77,140,247,254,254,254,254,244,177,114,50,20,0,0,0,0,0,0,0,0,0,0,0,0,0,3,34,175,246,254,254,254,253,221,115,46,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,34,84,175,251,254,254,253,234,202,91,4,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,10,79,204,233,251,254,255,255,239,116,35,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,20,123,220,254,254,254,255,255,255,248,165,37,21,32,9,4,4,4,4,0,0,0,0,0,0,0,0,0,100,238,254,255,255,255,255,255,255,254,253,235,234,245,222,217,217,217,203,115,32,0,0,0,0,0,0,0,33,163,244,254,254,254,255,255,255,255,254,254,254,254,254,254,254,254,254,240,111,3,0,0,0,0,0,0,1,33,126,216,233,250,254,255,255,255,255,255,255,255,255,255,255,254,254,233,82,2,0,0,0,0,0,0,0,0,0,5,22,82,251,255,255,255,255,255,255,254,254,250,233,217,203,110,18,0,0,0,0,0,0,0,0,0,0,0,10,128,254,255,255,254,254,254,254,246,208,140,82,39,32,4,0,0,0,0,0,0,0,0,0,0,0,22,95,221,254,255,254,253,234,217,215,127,46,9,2,0,0,0,0,0,0,0,0,0,0,0,0,0,3,154,232,253,254,254,252,207,95,39,37,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,2,77,247,254,255,254,242,131,9,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,125,254,254,252,232,131,32,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,4,113,253,246,177,109,22,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,8,111,77,8,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,3,2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0];
let classifier;
// asynchronous loads of resources, with callback functions when ready
let classifierLoaded = false;
new p5();
function modelLoaded() {
console.log('model ready!');
}
function classifyImage() {
// console.log('In classify image!');
ml5nn.classify(
{
image: doodle_inputs
},
gotResults
);
}
function gotResults(err, results) {
if (err) {
console.error(err);
return;
}
let label = results[0].label;
let confidence = nf(100 * results[0].confidence, 2, 0);
resultsDiv.html(`here ${label} ${confidence}%`);
// resultsDiv = " Label: " + label"<br>" +" Confidence: " + confidence + " % "+"</span>";
// AB.msg ( resultsDiv, 6 );
//console.log(results);
}
AB.world.newRun = function()
{
ABWorld.init ('red');
// AB.runReady = false; // prevent screenshot while waiting on splash screen
AB.headerRHS();
// background(255,0,0);
doodle = createGraphics(ZOOMPIXELS, ZOOMPIXELS);
doodle.pixelDensity(1);
doodle.background(0);
let options = {
inputs: [28, 28, 1],
task: 'imageClassification',
};
ml5nn = ml5.neuralNetwork(options);
let modelDetails = {
model: '/uploads/tesla/model.json',
metadata: '/uploads/tesla/model_meta.json',
weights: '/uploads/tesla/model.weights.bin'
}
// resultsDiv = "<hr> <h1> loading model.. </h1> Top row: Doodle (left) and shrunk (right). <br> " +" Draw your doodle in top LHS.";
// AB.msg ( resultsDiv, 1 );
resultsDiv = createDiv('loading model.').position(0,300);
// resultsDiv.style('font-size', '16px');
// resultsDiv.position(200, 0);
// resultsDiv.html('loading model.');
ml5nn.load(modelDetails, modelLoaded);
};
AB.world.nextStep = function()
{
background(255,0,0);
if ( doodle_exists )
{
drawDoodle();
guessDoodle();
}
if (mouseIsPressed) {
doodle_exists = true;
doodle.stroke(255);
doodle.strokeWeight(16);
doodle.line(mouseX, mouseY, pmouseX, pmouseY);
}
else
{
// are we exiting a drawing
if ( mousedrag )
{
mousedrag = false;
// console.log ("Exiting draw. Now blurring.");
// doodle.filter (BLUR, DOODLE_BLUR); // just blur once
// console.log (doodle);
}
}
};
function keyPressed()
{
console.log('Key Pressed');
doodle.background(0);
}
function mousePressed()
{
console.log('Mouse Pressed');
classifyImage();
}
function getImage ( img ) // make a P5 image object from a raw data array
{
let theimage = createImage (PIXELS, PIXELS); // make blank image, then populate it
theimage.loadPixels();
for (let i = 0; i < PIXELSSQUARED ; i++)
{
let bright = img[i];
let index = i * 4;
theimage.pixels[index + 0] = bright;
theimage.pixels[index + 1] = bright;
theimage.pixels[index + 2] = bright;
theimage.pixels[index + 3] = 255;
}
theimage.updatePixels();
return theimage;
}
function getInputs ( img ) // convert img array into normalised input array
{
let inputs = [];
for (let i = 0; i < PIXELSSQUARED ; i++)
{
let bright = img[i];
inputs[i] = bright; // normalise to 0 to 1
}
return ( inputs );
}
function drawDoodle()
{
// doodle is createGraphics not createImage
let theimage = doodle.get();
// console.log ('Teh Image is = '+theimage);
image ( theimage, 0, 0, ZOOMPIXELS, ZOOMPIXELS ); // original
image ( theimage, ZOOMPIXELS+50, 0, PIXELS, PIXELS ); // shrunk
}
function guessDoodle()
{
// doodle is createGraphics not createImage
let img = doodle.get();
img.resize ( PIXELS, PIXELS );
img.loadPixels();
// set up inputs
let inputs = [];
let test=0;
for (let i = 0; i < PIXELS ; i++)
{
for (let j = 0; j < PIXELS ; j++)
{
// inputs[test] = img.pixels[( i * (j*PIXELS))*4];
inputs[test] = img.pixels[( i + (j*PIXELS))*4];
// console.log('Count = '+test+' ij = '+( i + (j*PIXELS)));
test++;
}
}
doodle_inputs = inputs; // can inspect in console
classifyImage();
console.log('Count = '+test+' Doodle inputs = '+doodle_inputs);
}
function convlab(num){
var alphabet = ["Dummy","a","b","c","d","e","f","g","h","i","j","k","l","m","n","o","p","q","r","s","t","u","v","w","x","y","z"];
return alphabet[num];
}