// Use JS to write whatever HTML and data you want to the page
// One way of using JS to write HTML and data is with a multi-line string
// https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Template_literals
//Project utils
function loadCss(url) {
var head = document.getElementsByTagName('head')[0];
var link = document.createElement('link');
link.rel = 'stylesheet';
link.type = 'text/css';
link.href = url;
link.media = 'all';
head.appendChild(link);
}
//CANVAS
const PIXELS = 28;
let canvas, ctx, flag = false,
prevX = 0,
currX = 0,
prevY = 0,
currY = 0,
dot_flag = false,
scaleC = 6;
let x = "black",
y = 2;
function init() {
canvas = document.getElementById('can');
ctx = canvas.getContext("2d");
//ctx.scale(scaleC, scaleC);
w = canvas.width;
h = canvas.height;
canvas.addEventListener("mousemove", function (e) {
findxy('move', e)
}, false);
canvas.addEventListener("mousedown", function (e) {
findxy('down', e)
}, false);
canvas.addEventListener("mouseup", function (e) {
findxy('up', e)
}, false);
canvas.addEventListener("mouseout", function (e) {
findxy('out', e)
}, false);
}
function draw() {
ctx.beginPath();
ctx.moveTo(prevX/scaleC, prevY/scaleC);
ctx.lineTo(currX/scaleC, currY/scaleC);
ctx.strokeStyle = x;
ctx.lineWidth = y;
ctx.stroke();
ctx.closePath();
}
function erase() {
ctx.clearRect(0, 0, w, h);
$('#predictionDiv').html('');
}
function findxy(res, e) {
if (res == 'down') {
prevX = currX;
prevY = currY;
currX = e.clientX - canvas.offsetLeft;
currY = e.clientY - canvas.offsetTop;
flag = true;
dot_flag = true;
if (dot_flag) {
ctx.beginPath();
ctx.fillStyle = x;
ctx.fillRect(currX, currY, 2, 2);
ctx.closePath();
dot_flag = false;
}
}
if (res == 'up' || res == "out") {
flag = false;
}
if (res == 'move') {
if (flag) {
prevX = currX;
prevY = currY;
currX = e.clientX - canvas.offsetLeft;
currY = e.clientY - canvas.offsetTop;
draw();
}
}
}
// ANN
/**
* Class: ANN
* e: an array containing the structure of the nn in terms of layers
* e.g. e = [3,5,4] -> 3 input neurons, 5 hidden neurons, 4 output neurons
* a: array of activation functions called at every layer e.g. sigmoid
* aDer: derivative of activation functions e.g. sigmoidDerivative
* lr: learning rate
* bias: boolean that determines if there is a bias neuron
*/
class ANN{
constructor(structure = [2,2,2], activationFunction = [sigmoid, sigmoid], lr = .2, bias = false) {
this.structure = structure;
this.activation = activationFunction;
this.lr = lr;
this.bias = bias;
this.x = this.generateLayers();
this.w = this.generateWeights();
this.activation.splice(0, 0, null);
this.trainingExampleFed = 0;
this.testingExampleFed = 0;
this.rightPrediction = 0; //prediction are valid only while testing, not while training
//this.checkInitialization();
}
generateLayers() {
let layers = [];
for (let j in this.structure) {
layers.push(new Matrix(this.structure[j], 1));
}
return layers;
}
generateWeights(){
let ep = this.structure.length - 1; //do not consider the last hidden layer
let w = new Array(ep-1);
w[0] = null;
for (let n = 0; n < ep; n++) {
let b = this.x[n].rows;
let bp = this.x[n+1].rows;
w[n+1] = new Matrix(bp, b);
w[n+1].randomize();
}
return w;
}
net(i_layer) {
return Matrix.multiply(this.w[i_layer], this.x[i_layer-1]);
}
act(i_layer) {
return this.net(i_layer).map(this.activation[i_layer].func);
}
onlyPropagate(input) {
input = this.transformInput(input);
this.x[0] = input;
for(let i = 1; i < this.x.length; i++) {
this.x[i] = this.act(i);
}
}
propagate(input) {
this.x[0] = input;
for(let i = 1; i < this.x.length; i++) {
this.x[i] = this.act(i);
}
}
calculateError(target) {
let etot = 0;
for(let i in target.data) {
for(let j in target.data[i])
etot += Math.pow((target.data[i][j] - this.x[this.x.length-1].data[i][j]),2)/2;
}
return etot;
}
backpropagate(target) {
let errors = [];
let output_errors = Matrix.subtract(target, this.x[this.x.length-1]);
let gradient = Matrix.map(this.x[this.x.length-1], this.activation[this.x.length-1].dfunc);
gradient.multiply(output_errors);
gradient.multiply(this.lr);
let deltaOut = Matrix.multiply(gradient, Matrix.transpose(this.x[this.x.length-2]));
this.w[this.x.length-1].add(deltaOut);
errors[this.x.length-1] = output_errors;
for(let i = this.x.length-2; i > 0; i--) {
let h_errors = Matrix.multiply(Matrix.transpose(this.w[i+1]), errors[i+1]);
let h_gradients = Matrix.map(this.x[i], this.activation[i].dfunc);
h_gradients.multiply(h_errors);
h_gradients.multiply(this.lr);
let delta_h = Matrix.multiply(h_gradients, Matrix.transpose(this.x[i-1]));
this.w[i].add(delta_h);
errors[i] = h_errors;
}
}
backpropagateCodingTrain(target) {
let errors = [];
let output_errors = Matrix.subtract(target, this.x[this.x.length-1]);
let gradient = Matrix.map(this.x[this.x.length-1], this.activation[this.x.length-1].dfunc);
gradient.multiply(output_errors);
gradient.multiply(this.lr);
let deltaOut = Matrix.multiply(gradient, Matrix.transpose(this.x[this.x.length-2]));
this.w[this.x.length-1].add(deltaOut);
errors[this.x.length-1] = output_errors;
for(let i = this.x.length-2; i > 0; i--) {
let h_errors = Matrix.multiply(Matrix.transpose(this.w[i+1]), errors[i+1]);
let h_gradients = Matrix.map(this.x[i], this.activation[i].dfunc);
h_gradients.multiply(h_errors);
h_gradients.multiply(this.lr);
let delta_h = Matrix.multiply(h_gradients, Matrix.transpose(this.x[i-1]));
this.w[i].add(delta_h);
errors[i] = h_errors;
}
}
train(input, target) {
input = this.transformInput(input);
target = this.transformtarget(target);
this.propagate(input);
this.backpropagate(target);
let t = outNumeric(target);
let p = this.getPrediction();
this.trainingExampleFed += 1;
if(t == p) this.rightPrediction += 1;
console.log('error: ', this.calculateError(target).toFixed(4),
' target: ', t,
' prediction: ', p,
' examples fed: ', this.trainingExampleFed,
(t == p) ? '<- CORRECT PREDICTION': ' ');
}
test(input, target) {
input = this.transformInput(input);
target = this.transformtarget(target);
this.propagate(input);
//this.backpropagate(target);
let t = outNumeric(target);
let p = this.getPrediction();
this.testingExampleFed += 1;
if(t == p) this.rightPrediction += 1;
console.log('error: ', this.calculateError(target).toFixed(4),
' target: ', t,
' prediction: ', p,
' precision: ', this.rightPrediction, '/', this.testingExampleFed,
(t == p) ? 'GOT IT': ' ');
}
getPrediction() {
// console.log ( this.x ); // MH debug
return outNumeric(this.x[this.x.length-1]);
}
transformInput(input) {
input = Array.from(input);
input = input.map((x) => x/255);
input = Matrix.fromArray(input);
return input;
}
transformtarget(target) {
target = outOneHot(this.structure[this.structure.length-1], target);
target = Matrix.fromArray(target);
return target;
}
// HTML GRAPHICS
// USAGE: Create an Html Div with ID #ANNDiv, instanciate one ANN in a variable called 'nn'
generateHtml() {
return `
<div id="ANNGenerator"><h4>Generalized ANN</h4>
<p class="ANNSubtitle"><i>customize it</i></p>
<div id="LRChanger">Learning Rate =
<span onclick="nn.showLRChanger()" class="jsLink">`+this.lr+`</span>
</div>
<br>
`+ this.generateInputSection() +`
`+ this.generateHiddenSections() +`
`+ this.generateOutputSection() +`
</div>
`;
}
showLRChanger() {
$('#LRChanger').html(`
Learning rate =
<input type="number" id="ANNLR" value="`+this.lr+`" />
<button onclick="nn.changeLR()">Save</button>
`);
}
changeLR() {
this.lr = Number($('#ANNLR').val());
$('#ANNDiv').html(this.generateHtml());
}
generateInputSection() {
return `
<div class="ANNSection">
<div class="ANNTitleContainer">
<span class="ANNTitle"><b>Input layer</b></span>
<span class="ANNFunction"></span>
</div>
<div class="ANNCenter"># of Neurons: <b>`+ this.structure[0] +`</b></div>
<div class="ANNNeurons"><b><i>I</i></b><sub>0</sub> ... <b><i>I</i></b><sub>` + (this.structure[0]-1) +`</sub></div>
</div>
`;
}
generateHiddenSection(num) {
return `
<div class="ANNSectionAdd" onclick="nn.createNewLayer(` + (num) + `)">
<b>+</b>
</div>
<div class="ANNSection" id="hiddenLayer` + num + `">
<div class="ANNTitleContainer">
<span class="ANNTitle"><b>Hidden ` + num + ` layer</b></span>
<span class="ANNFunction">f = `+ this.activation[num].name +`</span>
</div>
<div class="ANNCenter"># of Neurons: <b>`+ this.structure[num] +`</b></div>
<div class="ANNNeurons">
<span class="ANNNeuronNames">
<b><i>H` + num + `</i></b><sub>0</sub> ...
<b><i>H` + num + `</i></b><sub>`+ (this.structure[num]-1) +`</sub>
</span>
<span class="ANNSettings">
<span class="jsLink" onclick="$('#hiddenLayer` + num + `').html(nn.generateHiddenSectionSettings(` + num + `))">change settings</span>
<span>
</div>
</div>
`;
}
generateHiddenSectionSettings(num) {
return `
<div class="ANNSection" id="hiddenLayer` + num + `">
<div class="ANNTitleContainer">
<span class="ANNTitle"><b>Hidden ` + num + ` layer</b></span>
<span class="ANNFunction">`+ this.activationFunctionSelection('hidFunction' + num) +`</span>
</div>
<div class="ANNCenter"># of Neurons:
<input type="number" id="hidNumber` + num + `" value="`+ this.structure[num] +`">
</div>
<div class="ANNNeurons">
<span class="ANNNeuronNames">
<b><i>H` + num + `</i></b><sub>0</sub> ...
<b><i>H` + num + `</i></b><sub>`+ (this.structure[num]-1) +`</sub>
</span>
<span class="ANNSettings">
<span class="jsLink" onclick="nn.updateHiddenSettings(` + num + `, $('#hidNumber` + num + `').val(), $('#hidFunction` + num + `').val())">save settings</span>
<span>
</div>
</div>
`;
}
updateHiddenSettings(layer, num, func) {
this.structure[layer] = Number(num);
this.activation[layer] = allFunctions[Number(func)];
this.x = this.generateLayers();
this.w = this.generateWeights();
this.writeHtmlOnPage();
}
generateOutputSectionSettings() {
return `
<div class="ANNTitleContainer">
<span class="ANNTitle"><b>Output layer</b></span>
<span class="ANNFunction">`+ this.activationFunctionSelection('outFunction') +` </span>
</div>
<div class="ANNCenter"># of Neurons:
<input type="number" id="outNumber" value="`+ this.structure[this.structure.length-1] +`">
</div>
<div class="ANNNeurons">
<span class="ANNNeuronNames">
<b><i>O</i></b><sub>0</sub> ...
<b><i>O</i></b><sub>`+ (this.structure[this.structure.length-1]-1) +`</sub>
</span>
<span class="ANNSettings">
<span class="jsLink" onclick="nn.updateOutputSettings($('#outNumber').val(), $('#outFunction').val())">save settings</span>
<span>
</div>
`;
}
generateHiddenSections() {
let s = '';
for(let i = 1; i < this.structure.length-1; i++) {
s += this.generateHiddenSection(i);
}
return s;
}
generateOutputSection() {
return `
<div class="ANNSectionAdd" onclick="nn.createNewLayer(` + (this.structure.length-1) + `)">
<b>+</b>
</div>
<div class="ANNSection" id="outputLayer">
<div class="ANNTitleContainer">
<span class="ANNTitle"><b>Output layer</b></span>
<span class="ANNFunction">f = `+ this.activation[this.structure.length-1].name +` </span>
</div>
<div class="ANNCenter"># of Neurons: <b>`+ this.structure[this.structure.length-1] +`</b></div>
<div class="ANNNeurons">
<span class="ANNNeuronNames">
<b><i>O</i></b><sub>0</sub> ...
<b><i>O</i></b><sub>`+ (this.structure[this.structure.length-1]-1) +`</sub>
</span>
<span class="ANNSettings">
<span class="jsLink" onclick="$('#outputLayer').html(nn.generateOutputSectionSettings())">change settings</span>
<span>
</div>
</div>
`;
}
generateOutputSectionSettings() {
return `
<div class="ANNTitleContainer">
<span class="ANNTitle"><b>Output layer</b></span>
<span class="ANNFunction">`+ this.activationFunctionSelection('outFunction') +` </span>
</div>
<div class="ANNCenter"># of Neurons:
<input type="number" id="outNumber" value="`+ this.structure[this.structure.length-1] +`">
</div>
<div class="ANNNeurons">
<span class="ANNNeuronNames">
<b><i>O</i></b><sub>0</sub> ...
<b><i>O</i></b><sub>`+ (this.structure[this.structure.length-1]-1) +`</sub>
</span>
<span class="ANNSettings">
<span class="jsLink" onclick="nn.updateOutputSettings($('#outNumber').val(), $('#outFunction').val())">save settings</span>
<span>
</div>
`;
}
updateOutputSettings(num, func) {
this.structure[this.structure.length-1] = Number(num);
this.activation[this.activation.length-1] = allFunctions[Number(func)];
this.x = this.generateLayers();
this.w = this.generateWeights();
this.writeHtmlOnPage();
//alert(num + func);
}
activationFunctionSelection(idName) {
let s = '';
for(let i in allFunctions) {
s += '<option value="' + i + '">'+allFunctions[i].name+'</option>';
}
return `
<label for="` + idName + `">f = </label>
<select id="` + idName + `">
`+ s +`
</select>
`;
}
createNewLayer(num) {
this.structure.splice(num, 0, 2); //add a layer with 2 neurons
this.activation.splice(num, 0, allFunctions[0]);
this.x = this.generateLayers();
this.w = this.generateWeights();
this.writeHtmlOnPage();
}
writeHtmlOnPage() {
$('#ANNDiv').html(this.generateHtml());
}
}
//load resources
loadCss('https://ancientbrain.com/uploads/stefano/general_ann_style.css');
loadCss('https://fonts.googleapis.com/css2?family=Noto+Serif:ital,wght@1,700&family=Nunito:wght@300&display=swap');
let nn;
let mnist;
let n_unseen = 300;
function getUnseenScore() {
let l = [];
for (let i = 0; i < mnist.test_images.length; i++) {
l.push(i);
}
let ar = [];
// get #n_unseen random examples
for(let i = 0; i < n_unseen; i++) {
let rand = Math.floor(Math.random() * l.length);
ar.push(l.splice(rand, 1)[0]);
}
let correct = 0;
for(let i = 0; i < n_unseen; i++) {
nn.onlyPropagate(mnist.test_images[ar[i]]);
if(nn.getPrediction() == mnist.test_labels[ar[i]]) {
correct += 1;
}
}
//console.log((correct/n_unseen*100).toFixed(2) + '%');
$('#scoreDiv').html($('#scoreDiv').html() +
'<br>Score on unseen data: ' + (correct/n_unseen*100).toFixed(2) + '%');
}
function loadData() {
loadMNIST ( function(data) {
mnist = data;
console.log ("All data loaded into mnist object");
//console.log(mnist);
AB.removeLoading(); // if no loading screen exists, this does nothing
console.log('%c Remember: this project does not need the console to be executed. \n' +
' For a better experience do not use the console while drawing, \n TIP: scroll to the top of the page before drawing in the canvas. ',
'background: #222; color: #bada55; font-size:15px');
});
}
$.getScript ( "/uploads/codingtrain/matrix.js", function() {
$.getScript( "/uploads/stefano/utils.js", function() {
$.getScript ( "/uploads/stefano/generalANN.js", function() {
$.getScript ( "/uploads/codingtrain/mnist.js", function() {
console.log ("All JS loaded");
//activationStructure = [sigmoid, sigmoid, sigmoid];
//nn = new ANN(layersStructure, activationStructure, 0.1, false);
loadData();
nn = new ANN([PIXELS*PIXELS, 64, 10]);
nn.writeHtmlOnPage();
});
});
});
});
//training epochs
let epoch_size = 20;
let tests_number = 100;
let trains_number = tests_number * 6;
let epoch = 0;
const max_epochs = 60000 / epoch_size;
let super_epoch = 15;
let scoreArray = [];
function executeEpochTraining() {
let start = epoch*epoch_size;
if(epoch < max_epochs){
for(let i = start; i < start + epoch_size; i++) {
nn.train(mnist.train_images[i], mnist.train_labels[i]);
//$('#scoreDiv').html(i + '/' + start);
}
epoch += 1;
scoreArray.push({epoch: epoch, value: (nn.rightPrediction/nn.trainingExampleFed*100).toFixed(2)});
} else {
console.log('Training completed');
}
}
function executeSuperEpoch() {
AB.removeLoading();
AB.loadingScreen();
if(epoch < max_epochs) {
for(let i = 0; i < super_epoch; i++) {
executeEpochTraining();
}
}
createGraph();
getUnseenScore();
AB.removeLoading();
}
//GRAPH
function createGraph() {
$('#my_dataviz').html('');
data = scoreArray;
let margin = {
top: 10,
right: 30,
bottom: 40,
left: 40
},
width = 500 - margin.left - margin.right,
height = 300 - margin.top - margin.bottom;
let svg = d3.select("#my_dataviz")
.append("svg")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append("g")
.attr("transform",
"translate(" + margin.left + "," + margin.top + ")");
// Set the ranges
let y = d3.scaleLinear()
.domain([0, 100])
.range([ height, 0 ]);
svg.append("g")
.call(d3.axisLeft(y));
let x = d3.scaleLinear()
.domain([1, max_epochs])
.range([ 0, width ]);
svg.append("g")
.attr("transform", "translate(0," + height + ")")
.call(d3.axisBottom(x));
// append the svg object to the body of the page
// Add the valueline path.
svg.append("path")
.datum(data)
.attr("fill", "none")
.attr("stroke", "steelblue")
.attr("stroke-width", 2)
.attr("d", d3.line()
.x(function(d) { return x(d.epoch * epoch_size) })
.y(function(d) { return y(d.value) })
);
// text label for the y axis
svg.append("text")
.attr("transform", "rotate(-90)")
.attr("y", 0 - margin.left)
.attr("x",0 - (height / 2))
.attr("dy", ".8em")
.style("text-anchor", "middle")
.text("Score");
// text label for the x axis
svg.append("text")
.attr("transform",
"translate(" + (width/2) + " ," +
(height + margin.top + 25) + ")")
.attr("dx", ".8em")
.style("text-anchor", "middle")
.text("Epoch");
if(nn != undefined)
$('#scoreDiv').html('Current score: ' + (nn.rightPrediction/nn.trainingExampleFed*100).toFixed(2) + '%<br>Example fed: ' + nn.trainingExampleFed);
}
$.getScript("https://d3js.org/d3.v4.js", function() {
console.log("D3JS loaded");
createGraph();
});
//
function resetANN() {
location.reload();
}
function executeTraining() {
for (let j = 0; j < trains_number; j++) {
nn.train(mnist.train_images[j], mnist.train_labels[j]);
}
}
function executeTesting() {
for (let j = 0; j < tests_number; j++) {
nn.test(mnist.test_images[j], mnist.test_labels[j]);
}
}
//user input prediction
function predictInput() {
let myImageData = ctx.getImageData(0, 0, w, h);
let alphaVal = [];
for(let i = 0; i < myImageData.data.length; i++) {
if((i+1) % 4 == 0) {
alphaVal.push(myImageData.data[i]);
}
}
let imgUint8 = new Uint8Array(alphaVal);
nn.onlyPropagate(imgUint8);
let pr = nn.getPrediction();
$('#predictionDiv').html('Prediction: ' + pr);
console.log('The drawn number seems to be: ' + pr);
}
//HTML Template
document.write ( `
<html>
<body onload="init()">
<div style="position:relative; display:inline-block">
<div id="ANNDiv" style="position:relative; display:inline-block"></div>
<div style="display:block">
<button onclick="executeSuperEpoch()" style="display:inline-block; width:150px; margin-top:15px">Feed `+epoch_size*super_epoch+` examples</button>
<button onclick="resetANN()" style="display:inline-block; width:90px; margin-top:15px">Reset ANN</button>
</div>
</div>
<div style="display:inline-block">
<div id="my_dataviz" style="display:inline-block"></div>
<div id="scoreDiv" style="display:block; margin-top:20px">Current score: 0%</div>
</div>
Draw
<canvas id="can" width="28" height="28" style="margin:20px;position:relative;border:2px solid;width:`+PIXELS*scaleC+`px; height:`+PIXELS*scaleC+`px;display:inline-block"></canvas>
<div style="display:inline-block">
<button id="clr" onclick="erase()" style="position:relative; display:block">Clear</button>
<button id="clr" onclick="predictInput()" style="position:relative; display:block">Predict</button>
<div id="predictionDiv"></div>
</div>
</body>
</html>
` );