// Cloned by Jorge Blanco on 8 Dec 2019 from World "Ported Code" by Adrian Sweeney
// Please leave this clone trail here.
// Cloned by Adrian Sweeney on 3 Dec 2019 from World "Complex World" by Starter user
// Please leave this clone trail here.
<!DOCTYPE html>
<html lang="en" xmlns="http://www.w3.org/1999/html">
<head>
<meta charset="UTF-8">
<title>Handwritten Digit Recognition by Convolutional Neural Network</title>
<script src="http://ajax.googleapis.com/ajax/libs/jquery/1.9.1/jquery.min.js"></script>
<script type="text/javascript" src="mathutils.js"></script>
<script type="text/javascript" src="mnist_digit_labels.js"></script>
<script type="text/javascript" src="webcnn.js"></script>
<script type="text/javascript" src="digitdemo.js"></script>
<link href="digitdemo.css" rel="stylesheet"/>
</head>
<body>
<a target="_blank" href="https://github.com/diaphone/webcnn"><img style="position: absolute; top: 0; right: 0; border: 0;" src="https://camo.githubusercontent.com/a6677b08c955af8400f44c6298f40e7d19cc5b2d/68747470733a2f2f73332e616d617a6f6e6177732e636f6d2f6769746875622f726962626f6e732f666f726b6d655f72696768745f677261795f3664366436642e706e67" alt="Fork me on GitHub" data-canonical-src="https://s3.amazonaws.com/github/ribbons/forkme_right_gray_6d6d6d.png"></a>
<div id="headerDiv"><a href="http://denseinl2.com">DenseInL2.com</a></div>
<div id="contentDiv">
<h1>Handwritten Digit Recognition by Convolutional Neural Network</h1>
<p>
This is a demonstration of my <a target="_blank" href="https://github.com/diaphone/webcnn">JavaScript-based Convolutional Neural Network</a>.
Draw a digit from 0 to 9
in the left box, and the network will attempt to recognize it. The system evaluates your drawing after
each stroke (mouse button up), so expect incorrect intermediate results if you draw your digit using
more than one stroke. Because the pre-processing of your input records and scales strokes, rather than
downsampling the canvas bitmap directly, building up a drawing from
short dashes or dots may not give accurate classification.
<p>
The associated training page for this network is here: <a target="_blank" href="digittraining.html">digittraining.html</a>
</p>
</p>
<div id="recognizer">
<div class="inlineDiv">
<div id="drawingCanvasDiv"><canvas id="drawCanvas" width="300" height="300"></canvas></div>
<button class="silverButton" onclick="buttonClick(2)" id="clearButton">Reset</button>
</div>
<div class="inlineDiv">
<div id="guessNumberDiv"></div>
<div>Confidence: <span id="confidence">0%</span></div>
</div>
</div>
<div style="text-align: center; font-size: 12px; padding: 8px 0px 0px 0px;">Note: This demo uses ES6 features and will not run in Safari 9.x. Try Chrome 52+, Firefox 50+.</div>
<h2>But this has been done before...</h2>
<p>
Yes, well, sort of. It's a classic dataset now, and there are a couple of other JavaScript demos similar to this one on the web already,
but what makes this one different is that I've coded the full training of the network, not merely the forward evaluation.
Other online demos I've found that let you draw digits for recognition
have only the forward implementation in JavaScript, and their network was trained externally in a commercial
product like MATLAB or TensorFlow. <a target="_blank" href="http://cs.stanford.edu/people/karpathy/convnetjs/demo/mnist.html">Andrej Karpathy's ConvNetJS</a> is the only other complete JavaScript CNN implementation
that I'm aware of. Because my motivation for doing this project is to learn about how these things
work--at the most fundamental level--and because deriving and implementing
the gradient descent training of the network is where the real math is, I decided not to cut any corners and to
intentionally re-invent the wheel for my own understanding. You can see
my somewhat crude training page here: <a target="_blank" href="digittraining.html">digittraining.html</a>
This network instance has been trained on the MNIST set of 60,000 handwritten digit images, and scores
approximately 98% accuracy on the associated set of 10,000 validation images.
</p>
<h2>So, what is this doing?</h2>
<p>
The images you draw in the box above are being fed into a <a target="_blank" href="https://en.wikipedia.org/wiki/Convolutional_neural_network">Convolutional Neural Network</a> that I wrote
in JavaScript/ES6 and trained on the <a target="_blank" href="http://yann.lecun.com/exdb/mnist/">MNIST dataset of handwritten digits</a>.
The network consists of digital filters that start out initialized with random values in their
kernels. The network "learns" to distinguish the features of digits by a negative feedback process known as
<a target="_blank" href="https://en.wikipedia.org/wiki/Gradient_descent">gradient descent</a>. Labeled example
digits are fed into the network, and any error in their classification is used to tune the network--making small adjustments to values in the
filter kernels and to the weights of connections to the output layer--to produce a more accurate score. This
is repeated for tens of thousands of example digit images, until the network has converged on a set of filters that can accurately
differentiate between all 10 digits.
</p>
<h2>This network's architecture</h2>
The network analyzing the digits consists of:
<ul>
<li>The input image pre-processor, which crops your drawing down to a 24x24 pixel input image, redrawing it
with a thinner stroke if it does not fill the box</li>
<li>A convolution layer with (10) 5x5x1 filter kernels, ReLU activation</li>
<li>A modified<sup>1</sup> max pooling layer, 2x2 with strides of 2</li>
<li>A convolution layer with (20) 5x5x10 filter kernels, ReLU activation</li>
<li>A modified max pooling layer, 2x2 with strides of 2</li>
<li>A fully-connected layer with 10 units with soft-max activation</li>
</ul>
<p>
During training of the network, I used a small amount of input image augmentation suggested by
<a target="_blank" href="http://cs.stanford.edu/people/karpathy/">Andrej Karpathy</a> in the notes accompanying his MNIST example
here: <a target="_blank" href="http://cs.stanford.edu/people/karpathy/convnetjs/demo/mnist.html">http://cs.stanford.edu/people/karpathy/convnetjs/demo/mnist.html</a>
A random 24x24 crop is taken from each of the 28x28 digit images. The network was trained using 2 passes of the dataset, a total
of 120,000 impressions.
</p>
<div class="footnote"><sup>1</sup>I've implemented the max pool layers to backpropagate error evenly to all input pixels that attained the maximum during training, when there
is more than one, rather than the more traditional approach of randomly selecting one when this occurs.</div>
</div>
<div id="footerDiv">
Dense in L2 · Adam Smith ©2017
</div>
<script>
function updateSliderDisplay( id, value )
{
document.querySelector('#'+id).value = value;
}
</script>
</body>
</html>
let cnn;
window.onload = Init;
let drawCanvas;
let ctx_draw;
let drawing = false;
let lastPos;
let lineWidth = 28;
let drawingPathIndex = -1;
let drawingDotsIndex = 0;
let drawingPaths = [];
let drawingDots = [];
const TAU = Math.PI * 2;
let freshCanvas = true;
function Init()
{
drawCanvas = document.getElementById("drawCanvas");
ctx_draw = drawCanvas.getContext("2d");
ctx_draw.mozImageSmoothingEnabled = false;
ctx_draw.webkitImageSmoothingEnabled = false;
ctx_draw.msImageSmoothingEnabled = false;
ctx_draw.imageSmoothingEnabled = false;
resetDrawingCanvas();
drawCanvas.addEventListener("mousedown",onMouseDown,false);
window.addEventListener("mouseup",onMouseUp,false);
drawCanvas.addEventListener("mousemove",onMouseMove,false);
drawCanvas.addEventListener("contextmenu",onContextMenu,false);
drawCanvas.addEventListener("mouseout",onMouseOut,false);
drawCanvas.addEventListener("mouseover",onMouseOver,false);
// Load the network
$.ajax({
url: "cnn_mnist_10_20_98accuracy.json",
dataType: "json",
success: onJSONLoaded
});
}
function resetDrawingCanvas()
{
if ( ctx_draw == undefined )
return;
drawingDotsIndex = 0;
drawingDots = [];
freshCanvas = true;
ctx_draw.fillStyle = "white";
ctx_draw.fillRect( 0, 0, drawCanvas.width, drawCanvas.height );
ctx_draw.fillStyle = "rgb(200,200,200)";
ctx_draw.font = "22px Verdana";
ctx_draw.fillText("Draw a digit (0-9) here", 24, 150);
}
function onJSONLoaded( response )
{
loadNetworkFromJSON( response );
console.log("JSON Loaded!");
}
function onContextMenu(e)
{
e.preventDefault();
}
function onMouseDown(e)
{
if ( freshCanvas )
{
freshCanvas = false;
ctx_draw.fillStyle = "white";
ctx_draw.fillRect( 0, 0, drawCanvas.width, drawCanvas.height );
}
drawing = true;
drawingPathIndex++;
drawingPaths[ drawingPathIndex ] = [];
lastPos = [ e.offsetX, e.offsetY ];
drawingDots[ drawingDotsIndex ] = lastPos;
drawingDotsIndex++;
ctx_draw.strokeStyle = "black";
ctx_draw.fillStyle = "black";
ctx_draw.lineCap = "round";
ctx_draw.lineJoin = "round";
ctx_draw.beginPath();
ctx_draw.arc( e.offsetX, e.offsetY, lineWidth / 2, 0, TAU );
ctx_draw.fill();
}
function onMouseUp(e)
{
if (drawing)
{
guessNumber();
drawing = false;
lastPos = undefined;
}
}
function onMouseOut(e)
{
drawing = false;
lastPos = undefined;
}
function onMouseOver(e)
{
}
function onMouseMove(e)
{
if ( !drawing ) return;
if ( e.target != drawCanvas )
{
drawing = false;
lastPos = undefined;
return;
}
var x = Math.max( 0, Math.min( e.target.width, e.offsetX ) );
var y = Math.max( 0, Math.min( e.target.height, e.offsetY ) );
if ( e.offsetX > 0 && e.offsetX < e.target.width && e.offsetY > 0 && e.offsetY < e.target.height )
{
ctx_draw.lineWidth = lineWidth;
if ( lastPos != undefined )
{
//ctx_draw.beginPath();
//ctx_draw.arc( x, y, 14, 0, TAU);
//ctx_draw.fill();
ctx_draw.beginPath();
ctx_draw.moveTo( lastPos[ 0 ], lastPos[ 1 ] );
ctx_draw.lineTo( x, y );
ctx_draw.stroke();
}
else
{
drawingPathIndex++;
ctx_draw.beginPath();
ctx_draw.arc( x, y, lineWidth / 2, 0, TAU );
ctx_draw.fill();
}
if ( drawingPaths[ drawingPathIndex ] == undefined )
{
drawingPaths[ drawingPathIndex ] = [];
}
drawingPaths[ drawingPathIndex ].push( [ x, y ] );
lastPos = [ x, y ];
}
else
{
lastPos = undefined;
}
}
function buttonClick( n )
{
switch (n)
{
case 1:
{
guessNumber();
break;
}
case 2:
{
drawingPaths = [];
drawingPathIndex = -1;
lastPos = undefined;
document.getElementById("guessNumberDiv").innerHTML = "";
document.getElementById("confidence").innerHTML = "";
resetDrawingCanvas();
break;
}
}
}
function preProcessDrawing()
{
let drawnImageData = ctx_draw.getImageData( 0, 0, ctx_draw.canvas.width, ctx_draw.canvas.height );
var xmin = ctx_draw.canvas.width - 1;
var xmax = 0;
var ymin = ctx_draw.canvas.height - 1;
var ymax = 0;
var w = ctx_draw.canvas.width;
var h = ctx_draw.canvas.height;
// Find bounding rect of drawing
for ( let i = 0; i < drawnImageData.data.length; i+=4 )
{
var x = Math.floor( i / 4 ) % w;
var y = Math.floor( i / ( 4 * w ) );
if ( drawnImageData.data[ i ] < 255 || drawnImageData.data[ i + 1 ] < 255 || drawnImageData.data[ i + 2 ] < 255 )
{
xmin = Math.min( xmin, x );
xmax = Math.max( xmax, x );
ymin = Math.min( ymin, y );
ymax = Math.max( ymax, y );
}
}
const cropWidth = xmax - xmin;
const cropHeight = ymax - ymin;
if ( cropWidth > 0 && cropHeight > 0 && ( cropWidth < w || cropHeight < h ) )
{
// Crop and scale drawing
const scaleX = cropWidth / w;
const scaleY = cropHeight / h;
const scale = Math.max( scaleX, scaleY );
const scaledLineWidth = Math.max( 1, Math.floor( lineWidth * scale ) );
const scaledDotWidth = Math.max( 1, Math.floor( scaledLineWidth / 2 ) );
//console.log(scale);
// Scaling down, redraw image with scale lineWidth
const tempCanvas = document.createElement("canvas");
//document.body.appendChild(tempCanvas);
tempCanvas.width = w;
tempCanvas.height = h;
const ctx_temp = tempCanvas.getContext("2d");
ctx_temp.strokeStyle = "black";
ctx_temp.fillStyle = "black";
ctx_temp.lineCap = "round";
ctx_temp.lineJoin = "round";
ctx_temp.lineWidth = scaledLineWidth;
//console.log(drawingPaths);
//console.log(drawingPaths);
for ( var pathIndex = 0; pathIndex < drawingPaths.length; ++pathIndex )
{
var path = drawingPaths[ pathIndex ];
if ( path == undefined || path.length == 0)
{
continue;
}
var p = path[0];
ctx_temp.beginPath();
ctx_temp.moveTo( p[0], p[1] );
for ( var i = 1; i < path.length; ++i )
{
p = path[ i ];
ctx_temp.lineTo( p[0], p[1] );
}
ctx_temp.stroke();
}
for ( var dotIndex = 0; dotIndex < drawingDots.length; ++dotIndex )
{
var dotPos = drawingDots[ dotIndex ];
ctx_temp.beginPath();
ctx_temp.arc( dotPos[ 0 ], dotPos[ 1 ], scaledDotWidth, 0, TAU );
ctx_temp.fill();
}
drawnImageData = ctx_temp.getImageData( xmin, ymin, cropWidth, cropHeight );
}
// Invert black and white to match training data
for ( var i = 0; i < drawnImageData.data.length; i+=4 )
{
drawnImageData.data[i] = 255 - drawnImageData.data[i];
drawnImageData.data[i+1] = 255 - drawnImageData.data[i+1];
drawnImageData.data[i+2] = 255 - drawnImageData.data[i+2];
}
let canvas2 = document.createElement( "canvas" );
canvas2.width = drawnImageData.width;
canvas2.height = drawnImageData.height;
//document.body.appendChild(canvas2);
let ctx2 = canvas2.getContext( "2d" );
ctx2.mozImageSmoothingEnabled = false;
ctx2.webkitImageSmoothingEnabled = false;
ctx2.msImageSmoothingEnabled = false;
ctx2.imageSmoothingEnabled = false;
ctx2.putImageData( drawnImageData, 0, 0 );
let canvas = document.createElement( "canvas" );
canvas.width = 24;
canvas.height = 24;
//document.body.appendChild(canvas);
let ctx = canvas.getContext( "2d" );
ctx.mozImageSmoothingEnabled = false;
ctx.webkitImageSmoothingEnabled = false;
ctx.msImageSmoothingEnabled = false;
ctx.imageSmoothingEnabled = false;
// Preserve aspect ratio of cropped section, center it
var xOffset = 0;
var yOffset = 0;
var xScale = 1;
var yScale = 1;
const padding = 1;
if ( canvas2.width > canvas2.height )
{
yOffset = ( canvas.width / ( canvas2.width + 2 * padding) ) * ( canvas2.width - canvas2.height ) / 2 + padding;
yScale = canvas2.height / canvas2.width;
xOffset = padding;
}
else if ( canvas2.height > canvas2.width )
{
xOffset = ( canvas.height / canvas2.height ) * ( canvas2.height - canvas2.width ) / 2 + padding;
xScale = canvas2.width / canvas2.height;
yOffset = padding;
}
ctx.fillStyle = "black";
ctx.fillRect( 0, 0, canvas.width, canvas.height );
ctx.drawImage( canvas2, xOffset, yOffset, canvas.width * xScale - 2 * padding, canvas.height * yScale - 2 * padding );
return ctx.getImageData( 0, 0, 24, 24 );
}
function guessNumber()
{
const inputImageData = preProcessDrawing();
if ( cnn == undefined ) return;
const result = cnn.classifyImages( [ inputImageData ] );
let guess = 0;
let max = 0;
for ( var i = 0; i < 10; ++i )
{
if ( result[ 0 ].getValue( 0, 0, i ) > max )
{
max = result[ 0 ].getValue( 0, 0, i );
guess = i;
}
}
//console.log("Is it a "+guess+" ? ("+Math.floor( 1000 * max ) / 10.0 + "%");
document.getElementById("guessNumberDiv").innerHTML = ( max > 0.666667 ) ? String( guess ) : "?";
document.getElementById("confidence").innerHTML = String( Math.min( 100, Math.floor( 1000 * ( max + 0.1 ) ) / 10.0 ) ) + "% it's a "+String( guess );
}
function loadNetworkFromJSON( networkJSON )
{
cnn = new WebCNN();
if (networkJSON.momentum != undefined) cnn.setMomentum( networkJSON.momentum );
if (networkJSON.lambda != undefined) cnn.setLambda( networkJSON.lambda );
if (networkJSON.learningRate != undefined) cnn.setLearningRate( networkJSON.learningRate );
for ( var layerIndex = 0; layerIndex < networkJSON.layers.length; ++layerIndex )
{
let layerDesc = networkJSON.layers[ layerIndex ];
console.log( layerDesc );
cnn.newLayer( layerDesc );
}
for ( var layerIndex = 0; layerIndex < networkJSON.layers.length; ++layerIndex )
{
let layerDesc = networkJSON.layers[ layerIndex ];
switch ( networkJSON.layers[ layerIndex ].type )
{
case LAYER_TYPE_CONV:
case LAYER_TYPE_FULLY_CONNECTED:
{
if (layerDesc.weights != undefined && layerDesc.biases != undefined )
{
cnn.layers[ layerIndex ].setWeightsAndBiases( layerDesc.weights, layerDesc.biases );
}
break;
}
}
}
cnn.initialize();
}
function OnNetworkJSONLoaded(evt)
{
console.log("parsing");
let networkFileText = evt.target.result;
let networkJSON = JSON.parse( networkFileText );
loadNetworkFromJSON( networkJSON );
}