/* PART:
* 1. Project Setup
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*/
/***********************************************************************************/
/***********************************************************************************
/* Setup procedure for creating a new Phaser Game object on window load event
/***********************************************************************************/
window.onload = function () {
// create a new game object which is an instance of Phaser.Game
var game = new Phaser.Game(1280, 720, Phaser.CANVAS);
// add all States to the game object (this program has only the Main State)
game.state.add('MainState', App.MainState);
// start the Main State
game.state.start('MainState');
};
/***********************************************************************************
/* The Application Namespace
/***********************************************************************************/
var App = App || {};
// ---------------------------------------------------------------------------------
// Global constants and variables
// ---------------------------------------------------------------------------------
// ---------------------------------------------------------------------------------
// The Main State constructor
// ---------------------------------------------------------------------------------
App.MainState = function(){
// constants describing all modes of the main state
this.MODE_INIT = 1;
this.MODE_OPEN_FILE = 2;
this.MODE_LOAD_FILE = 3;
this.MODE_START_TRAIN = 4;
this.MODE_DO_TRAIN = 5;
this.MODE_START_PREDICT = 6;
this.MODE_DO_PREDICT = 7;
this.MODE_DRAW = 8;
// set initial mode
this.mode = this.MODE_INIT;
};
// ---------------------------------------------------------------------------------
// The Main State prototype
// ---------------------------------------------------------------------------------
App.MainState.prototype = {
/**
* Automatically called only once to load all assets.
*/
preload : function(){
this.game.load.image('imgBack', '../assets/img_back_1.png');
this.game.load.image('btnMoreGames', '../assets/btn_moregames.png');
this.game.load.image('btnAuthor', '../assets/btn_author.png');
this.load.bitmapFont('fntBlackChars', '../assets/fnt_black_chars.png', '../assets/fnt_black_chars.fnt');
},
/**
* Automatically called immediately after all assets are loaded to create all objects.
*/
create : function(){
// scale game to cover the entire screen
this.scale.scaleMode = Phaser.ScaleManager.SHOW_ALL;
this.scale.pageAlignVertically = true;
this.scale.pageAlignHorizontally = true;
// keep game running if it loses the focus
this.game.stage.disableVisibilityChange = true;
// create background
this.game.add.sprite(0, 0, 'imgBack');
// create user interface with buttons, bitmaps and texts
this.ui = new UI(this);
},
/**
* Automatically called on every tick representing the main loop.
*/
update : function(){
switch(this.mode){
// initialize the game
case this.MODE_INIT:
this.ui.disableButtons();
this.ui.showStatusBar("Game initialized.");
this.mode = this.MODE_OPEN_FILE;
break;
case this.MODE_OPEN_FILE:
this.mode = this.MODE_LOAD_FILE;
break;
case this.MODE_LOAD_FILE:
this.mode = this.MODE_START_TRAIN;
break;
case this.MODE_START_TRAIN:
this.mode = this.MODE_DO_TRAIN;
break;
case this.MODE_DO_TRAIN:
this.mode = this.MODE_START_PREDICT;
break;
case this.MODE_START_PREDICT:
this.mode = this.MODE_DO_PREDICT;
break;
case this.MODE_DO_PREDICT:
this.mode = this.MODE_DRAW;
break;
case this.MODE_DRAW:
break;
}
}
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 1)
*
* MODULE:
* ui.js - UI Class (User Interface)
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 1. Project Setup
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
// ---------------------------------------------------------------------------------
// UI Constructor
// ---------------------------------------------------------------------------------
var UI = function(main){
// reference to the Main State
this.main = main;
// reference to the Phaser Game object
var game = this.main.game;
// create buttons
this.btnMoreGames = game.add.button(1048, 625, 'btnMoreGames', this.onMoreGamesClick, this);
this.btnAuthor = game.add.button(1130, 703, 'btnAuthor', this.onMoreGamesClick, this);
this.btnAuthor.anchor.setTo(0.5);
// create a text which shows messages in the status bar
this.txtStatBar = game.add.bitmapText(10, 695, "fntBlackChars", "", 18);
};
// ---------------------------------------------------------------------------------
// UI Prototype
// ---------------------------------------------------------------------------------
/**
* Disables buttons.
*/
UI.prototype.disableButtons = function(){
};
/**
* Enables buttons.
*/
UI.prototype.enableButtons = function(){
};
/**
* Opens Official Web Site when click on "Play More Games" button.
*/
UI.prototype.onMoreGamesClick = function(){
window.open("http://www.askforgametask.com", "_blank");
};
/**
* Shows a message in the status bar.
*
* @param {String} strText - the text to be shown in the status bar
*/
UI.prototype.showStatusBar = function(strText){
this.txtStatBar.text = strText;
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 2)
*
* MODULE:
* cnn.js - CNN Class (Convolution Neural Network)
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 2. Getting Data
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
// ---------------------------------------------------------------------------------
// CNN Constructor
// ---------------------------------------------------------------------------------
var CNN = function(main){
// reference to the Main State
this.main = main;
this.NUM_CLASSES = App.DATASETS.length; // number of classes which can be recognized by CNN model
this.IMAGE_SIZE = 784; // size of an image in a dataset
this.NUM_TRAIN_IMAGES = 400; // number of training images in a dataset
this.NUM_TEST_IMAGES = 100; // number of test images in a dataset
// total number of training images in all classes
const TOTAL_TRAIN_IMAGES = this.NUM_CLASSES * this.NUM_TRAIN_IMAGES;
// total number of test images in all classes
const TOTAL_TEST_IMAGES = this.NUM_CLASSES * this.NUM_TEST_IMAGES;
// create Training Data arrays for storing training images and their corresponding classes
this.aTrainImages = new Float32Array(TOTAL_TRAIN_IMAGES * this.IMAGE_SIZE);
this.aTrainClasses = new Uint8Array(TOTAL_TRAIN_IMAGES);
// shuffle Training Data by creating an array of shuffled Train indices
this.aTrainIndices = tf.util.createShuffledIndices(TOTAL_TRAIN_IMAGES);
// the reference to the current element in the aTrainIndices[] array
this.trainElement = -1;
// create arrays of Test Data for storing test images and their corresponding classes
this.aTestImages = new Float32Array(TOTAL_TEST_IMAGES * this.IMAGE_SIZE);
this.aTestClasses = new Uint8Array(TOTAL_TEST_IMAGES);
// shuffle Test Data by creating an array of shuffled Test indices
this.aTestIndices = tf.util.createShuffledIndices(TOTAL_TEST_IMAGES);
// the reference to the current element in the aTestIndices[] array
this.testElement = -1;
};
// ---------------------------------------------------------------------------------
// CNN Prototype
// ---------------------------------------------------------------------------------
/**
* Splits the entire dataset into training data and test data.
*
* @param {Uint8Array} imagesBuffer - the array with binary data of all images in the dataset
* @param {integer} dataset - the ordinal number of the dataset
*/
CNN.prototype.splitDataset = function(imagesBuffer, dataset){
// slice dataset to get training images and normalize them
var trainBuffer = new Float32Array(imagesBuffer.slice(0, this.IMAGE_SIZE * this.NUM_TRAIN_IMAGES));
trainBuffer = trainBuffer.map(function (cv){return cv/255.0});
// add training images and their corresponding classes into Training Data arrays
var start = dataset * this.NUM_TRAIN_IMAGES;
this.aTrainImages.set(trainBuffer, start * this.IMAGE_SIZE);
this.aTrainClasses.fill(dataset, start, start + this.NUM_TRAIN_IMAGES);
// slice dataset to get test images and normalize them
var testBuffer = new Float32Array(imagesBuffer.slice(this.IMAGE_SIZE * this.NUM_TRAIN_IMAGES));
testBuffer = testBuffer.map(function (cv){return cv/255.0});
// add test images and their corresponding classes into Test Data arrays
start = dataset * this.NUM_TEST_IMAGES;
this.aTestImages.set(testBuffer, start * this.IMAGE_SIZE);
this.aTestClasses.fill(dataset, start, start + this.NUM_TEST_IMAGES);
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 2)
*
* MODULE:
* main.js - Main Program
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 2. Getting Data
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
/***********************************************************************************
/* Setup procedure for creating a new Phaser Game object on window load event
/***********************************************************************************/
window.onload = function () {
// create a new game object which is an instance of Phaser.Game
var game = new Phaser.Game(1280, 720, Phaser.CANVAS);
// add all States to the game object (this program has only the Main State)
game.state.add('MainState', App.MainState);
// start the Main State
game.state.start('MainState');
};
/***********************************************************************************
/* The Application Namespace
/***********************************************************************************/
var App = App || {};
// ---------------------------------------------------------------------------------
// Global constants and variables
// ---------------------------------------------------------------------------------
// the names of all datasets
App.DATASETS = ['car', 'fish', 'snowman'];
// ---------------------------------------------------------------------------------
// The Main State constructor
// ---------------------------------------------------------------------------------
App.MainState = function(){
// constants describing all modes of the main state
this.MODE_INIT = 1;
this.MODE_OPEN_FILE = 2;
this.MODE_LOAD_FILE = 3;
this.MODE_START_TRAIN = 4;
this.MODE_DO_TRAIN = 5;
this.MODE_START_PREDICT = 6;
this.MODE_DO_PREDICT = 7;
this.MODE_DRAW = 8;
// set initial mode
this.mode = this.MODE_INIT;
// the counter of currently loaded datasets
this.dataset = 0;
};
// ---------------------------------------------------------------------------------
// The Main State prototype
// ---------------------------------------------------------------------------------
App.MainState.prototype = {
/**
* Automatically called only once to load all assets.
*/
preload : function(){
this.game.load.image('imgBack', '../assets/img_back_2.png');
this.game.load.image('btnMoreGames', '../assets/btn_moregames.png');
this.game.load.image('btnAuthor', '../assets/btn_author.png');
this.load.bitmapFont('fntBlackChars', '../assets/fnt_black_chars.png', '../assets/fnt_black_chars.fnt');
},
/**
* Automatically called immediately after all assets are loaded to create all objects.
*/
create : function(){
// scale game to cover the entire screen
this.scale.scaleMode = Phaser.ScaleManager.SHOW_ALL;
this.scale.pageAlignVertically = true;
this.scale.pageAlignHorizontally = true;
// keep game running if it loses the focus
this.game.stage.disableVisibilityChange = true;
// create background
this.game.add.sprite(0, 0, 'imgBack');
// create a loader for loading datasets
this.loader = new Phaser.Loader(this.game);
// create user interface with buttons, bitmaps and texts
this.ui = new UI(this);
// create a convolution neural network
this.cnn = new CNN(this);
},
/**
* Automatically called on every tick representing the main loop.
*/
update : function(){
switch(this.mode){
// initialize the game
case this.MODE_INIT:
this.ui.disableButtons();
this.mode = this.MODE_OPEN_FILE;
break;
// open dataset file and start loading it
case this.MODE_OPEN_FILE:
var fileName = App.DATASETS[this.dataset] + '.bin';
this.loader.reset();
this.loader.binary('input_file', '../data/'+fileName);
this.loader.start();
this.ui.showStatusBar("Loading " + fileName + " file.");
this.mode = this.MODE_LOAD_FILE;
break;
// wait on dataset file to be loaded
case this.MODE_LOAD_FILE:
if (this.loader.hasLoaded){
// split the loaded dataset into training data and test data
this.cnn.splitDataset(
new Uint8Array(this.game.cache.getBinary('input_file')),
this.dataset
);
// increase the number of loaded datasets
this.dataset++;
// if we have not loaded all datasets yet then go to load the next one
if (this.dataset < App.DATASETS.length){
this.mode = this.MODE_OPEN_FILE;
} else {
this.ui.showStatusBar("All datasets loaded.");
this.mode = this.MODE_START_TRAIN;
}
}
break;
case this.MODE_START_TRAIN:
this.mode = this.MODE_DO_TRAIN;
break;
case this.MODE_DO_TRAIN:
this.mode = this.MODE_START_PREDICT;
break;
case this.MODE_START_PREDICT:
this.mode = this.MODE_DO_PREDICT;
break;
case this.MODE_DO_PREDICT:
this.mode = this.MODE_DRAW;
break;
case this.MODE_DRAW:
break;
}
}
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 2)
*
* MODULE:
* ui.js - UI Class (User Interface)
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 2. Getting Data
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
// ---------------------------------------------------------------------------------
// UI Constructor
// ---------------------------------------------------------------------------------
var UI = function(main){
// reference to the Main State
this.main = main;
// reference to the Phaser Game object
var game = this.main.game;
// create buttons
this.btnMoreGames = game.add.button(1048, 625, 'btnMoreGames', this.onMoreGamesClick, this);
this.btnAuthor = game.add.button(1130, 703, 'btnAuthor', this.onMoreGamesClick, this);
this.btnAuthor.anchor.setTo(0.5);
// create a text which shows messages in the status bar
this.txtStatBar = game.add.bitmapText(10, 695, "fntBlackChars", "", 18);
};
// ---------------------------------------------------------------------------------
// UI Prototype
// ---------------------------------------------------------------------------------
/**
* Disables buttons.
*/
UI.prototype.disableButtons = function(){
};
/**
* Enables buttons.
*/
UI.prototype.enableButtons = function(){
};
/**
* Opens Official Web Site when click on "Play More Games" button.
*/
UI.prototype.onMoreGamesClick = function(){
window.open("http://www.askforgametask.com", "_blank");
};
/**
* Shows a message in the status bar.
*
* @param {String} strText - the text to be shown in the status bar
*/
UI.prototype.showStatusBar = function(strText){
this.txtStatBar.text = strText;
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 3)
*
* MODULE:
* cnn.js - CNN Class (Convolution Neural Network)
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 3. Building Model
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
// ---------------------------------------------------------------------------------
// CNN Constructor
// ---------------------------------------------------------------------------------
var CNN = function(main){
// reference to the Main State
this.main = main;
this.NUM_CLASSES = App.DATASETS.length; // number of classes which can be recognized by CNN model
this.IMAGE_SIZE = 784; // size of an image in a dataset
this.NUM_TRAIN_IMAGES = 400; // number of training images in a dataset
this.NUM_TEST_IMAGES = 100; // number of test images in a dataset
// total number of training images in all classes
const TOTAL_TRAIN_IMAGES = this.NUM_CLASSES * this.NUM_TRAIN_IMAGES;
// total number of test images in all classes
const TOTAL_TEST_IMAGES = this.NUM_CLASSES * this.NUM_TEST_IMAGES;
// create Training Data arrays for storing training images and their corresponding classes
this.aTrainImages = new Float32Array(TOTAL_TRAIN_IMAGES * this.IMAGE_SIZE);
this.aTrainClasses = new Uint8Array(TOTAL_TRAIN_IMAGES);
// shuffle Training Data by creating an array of shuffled Train indices
this.aTrainIndices = tf.util.createShuffledIndices(TOTAL_TRAIN_IMAGES);
// the reference to the current element in the aTrainIndices[] array
this.trainElement = -1;
// create arrays of Test Data for storing test images and their corresponding classes
this.aTestImages = new Float32Array(TOTAL_TEST_IMAGES * this.IMAGE_SIZE);
this.aTestClasses = new Uint8Array(TOTAL_TEST_IMAGES);
// shuffle Test Data by creating an array of shuffled Test indices
this.aTestIndices = tf.util.createShuffledIndices(TOTAL_TEST_IMAGES);
// the reference to the current element in the aTestIndices[] array
this.testElement = -1;
// create a CNN model using a Sequential model type in which
// tensors are consecutively passed from one layer to the next
this.model = tf.sequential();
// add a convolutional layer
this.model.add(tf.layers.conv2d({
inputShape: [28, 28, 1],
kernelSize: 5,
filters: 8,
strides: 1,
activation: 'relu',
kernelInitializer: 'varianceScaling'
}));
// add a max pooling layer
this.model.add(tf.layers.maxPooling2d({
poolSize: [2, 2],
strides: [2, 2]
}));
// add a second convolutional layer
this.model.add(tf.layers.conv2d({
kernelSize: 5,
filters: 16,
strides: 1,
activation: 'relu',
kernelInitializer: 'varianceScaling'
}));
// add a second max pooling layer
this.model.add(tf.layers.maxPooling2d({
poolSize: [2, 2],
strides: [2, 2]
}));
// add a flatten layer to flatten the output of the previous layer to a vector
this.model.add(tf.layers.flatten());
// add a dense (fully connected) layer to perform the final classification
this.model.add(tf.layers.dense({
units: this.NUM_CLASSES,
kernelInitializer: 'varianceScaling',
activation: 'softmax'
}));
// compile the model
this.model.compile({
optimizer: tf.train.sgd(0.15), // optimizer with learning rate
loss: 'categoricalCrossentropy', // loss function
metrics: ['accuracy'], // evaluation metric
});
};
// ---------------------------------------------------------------------------------
// CNN Prototype
// ---------------------------------------------------------------------------------
/**
* Splits the entire dataset into training data and test data.
*
* @param {Uint8Array} imagesBuffer - the array with binary data of all images in the dataset
* @param {integer} dataset - the ordinal number of the dataset
*/
CNN.prototype.splitDataset = function(imagesBuffer, dataset){
// slice dataset to get training images and normalize them
var trainBuffer = new Float32Array(imagesBuffer.slice(0, this.IMAGE_SIZE * this.NUM_TRAIN_IMAGES));
trainBuffer = trainBuffer.map(function (cv){return cv/255.0});
// add training images and their corresponding classes into Training Data arrays
var start = dataset * this.NUM_TRAIN_IMAGES;
this.aTrainImages.set(trainBuffer, start * this.IMAGE_SIZE);
this.aTrainClasses.fill(dataset, start, start + this.NUM_TRAIN_IMAGES);
// slice dataset to get test images and normalize them
var testBuffer = new Float32Array(imagesBuffer.slice(this.IMAGE_SIZE * this.NUM_TRAIN_IMAGES));
testBuffer = testBuffer.map(function (cv){return cv/255.0});
// add test images and their corresponding classes into Test Data arrays
start = dataset * this.NUM_TEST_IMAGES;
this.aTestImages.set(testBuffer, start * this.IMAGE_SIZE);
this.aTestClasses.fill(dataset, start, start + this.NUM_TEST_IMAGES);
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 3)
*
* MODULE:
* main.js - Main Program
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 3. Building Model
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
/***********************************************************************************
/* Setup procedure for creating a new Phaser Game object on window load event
/***********************************************************************************/
window.onload = function () {
// create a new game object which is an instance of Phaser.Game
var game = new Phaser.Game(1280, 720, Phaser.CANVAS);
// add all States to the game object (this program has only the Main State)
game.state.add('MainState', App.MainState);
// start the Main State
game.state.start('MainState');
};
/***********************************************************************************
/* The Application Namespace
/***********************************************************************************/
var App = App || {};
// ---------------------------------------------------------------------------------
// Global constants and variables
// ---------------------------------------------------------------------------------
// the names of all datasets
App.DATASETS = ['car', 'fish', 'snowman'];
// ---------------------------------------------------------------------------------
// The Main State constructor
// ---------------------------------------------------------------------------------
App.MainState = function(){
// constants describing all modes of the main state
this.MODE_INIT = 1;
this.MODE_OPEN_FILE = 2;
this.MODE_LOAD_FILE = 3;
this.MODE_START_TRAIN = 4;
this.MODE_DO_TRAIN = 5;
this.MODE_START_PREDICT = 6;
this.MODE_DO_PREDICT = 7;
this.MODE_DRAW = 8;
// set initial mode
this.mode = this.MODE_INIT;
// the counter of currently loaded datasets
this.dataset = 0;
};
// ---------------------------------------------------------------------------------
// The Main State prototype
// ---------------------------------------------------------------------------------
App.MainState.prototype = {
/**
* Automatically called only once to load all assets.
*/
preload : function(){
this.game.load.image('imgBack', '../assets/img_back_3.png');
this.game.load.image('btnMoreGames', '../assets/btn_moregames.png');
this.game.load.image('btnAuthor', '../assets/btn_author.png');
this.load.bitmapFont('fntBlackChars', '../assets/fnt_black_chars.png', '../assets/fnt_black_chars.fnt');
},
/**
* Automatically called immediately after all assets are loaded to create all objects.
*/
create : function(){
// scale game to cover the entire screen
this.scale.scaleMode = Phaser.ScaleManager.SHOW_ALL;
this.scale.pageAlignVertically = true;
this.scale.pageAlignHorizontally = true;
// keep game running if it loses the focus
this.game.stage.disableVisibilityChange = true;
// create background
this.game.add.sprite(0, 0, 'imgBack');
// create a loader for loading datasets
this.loader = new Phaser.Loader(this.game);
// create user interface with buttons, bitmaps and texts
this.ui = new UI(this);
// create a convolution neural network
this.cnn = new CNN(this);
},
/**
* Automatically called on every tick representing the main loop.
*/
update : function(){
switch(this.mode){
// initialize the game
case this.MODE_INIT:
this.ui.disableButtons();
this.mode = this.MODE_OPEN_FILE;
break;
// open dataset file and start loading it
case this.MODE_OPEN_FILE:
var fileName = App.DATASETS[this.dataset] + '.bin';
this.loader.reset();
this.loader.binary('input_file', '../data/'+fileName);
this.loader.start();
this.ui.showStatusBar("Loading " + fileName + " file.");
this.mode = this.MODE_LOAD_FILE;
break;
// wait on dataset file to be loaded
case this.MODE_LOAD_FILE:
if (this.loader.hasLoaded){
// split the loaded dataset into training data and test data
this.cnn.splitDataset(
new Uint8Array(this.game.cache.getBinary('input_file')),
this.dataset
);
// increase the number of loaded datasets
this.dataset++;
// if we have not loaded all datasets yet then go to load the next one
// else go to the CNN training
if (this.dataset < App.DATASETS.length){
this.mode = this.MODE_OPEN_FILE;
} else {
this.ui.showStatusBar("All datasets loaded and CNN model created.");
this.mode = this.MODE_START_TRAIN;
}
}
break;
case this.MODE_START_TRAIN:
this.mode = this.MODE_DO_TRAIN;
break;
case this.MODE_DO_TRAIN:
this.mode = this.MODE_START_PREDICT;
break;
case this.MODE_START_PREDICT:
this.mode = this.MODE_DO_PREDICT;
break;
case this.MODE_DO_PREDICT:
this.mode = this.MODE_DRAW;
break;
case this.MODE_DRAW:
break;
}
}
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 3)
*
* MODULE:
* ui.js - UI Class (User Interface)
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 3. Building Model
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
// ---------------------------------------------------------------------------------
// UI Constructor
// ---------------------------------------------------------------------------------
var UI = function(main){
// reference to the Main State
this.main = main;
// reference to the Phaser Game object
var game = this.main.game;
// create buttons
this.btnMoreGames = game.add.button(1048, 625, 'btnMoreGames', this.onMoreGamesClick, this);
this.btnAuthor = game.add.button(1130, 703, 'btnAuthor', this.onMoreGamesClick, this);
this.btnAuthor.anchor.setTo(0.5);
// create a text which shows messages in the status bar
this.txtStatBar = game.add.bitmapText(10, 695, "fntBlackChars", "", 18);
};
// ---------------------------------------------------------------------------------
// UI Prototype
// ---------------------------------------------------------------------------------
/**
* Disables buttons.
*/
UI.prototype.disableButtons = function(){
};
/**
* Enables buttons.
*/
UI.prototype.enableButtons = function(){
};
/**
* Opens Official Web Site when click on "Play More Games" button.
*/
UI.prototype.onMoreGamesClick = function(){
window.open("http://www.askforgametask.com", "_blank");
};
/**
* Shows a message in the status bar.
*
* @param {String} strText - the text to be shown in the status bar
*/
UI.prototype.showStatusBar = function(strText){
this.txtStatBar.text = strText;
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 4)
*
* MODULE:
* cnn.js - CNN Class (Convolution Neural Network)
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 4. Training Model
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
// ---------------------------------------------------------------------------------
// CNN Constructor
// ---------------------------------------------------------------------------------
var CNN = function(main){
// reference to the Main State
this.main = main;
this.NUM_CLASSES = App.DATASETS.length; // number of classes which can be recognized by CNN model
this.IMAGE_SIZE = 784; // size of an image in a dataset
this.NUM_TRAIN_IMAGES = 400; // number of training images in a dataset
this.NUM_TEST_IMAGES = 100; // number of test images in a dataset
this.TRAIN_ITERATIONS = 50; // total number of training iterations
this.TRAIN_BATCH_SIZE = 100; // number of training images used to train model during one iteration
this.TEST_FREQUENCY = 5; // frequency of testing model accuracy (one test on every 5 training iterations)
this.TEST_BATCH_SIZE = 50; // number of test images used to test model accuracy
this.trainIteration = 0; // current number of executed training iterations
this.aLoss = []; // array to store model's loss values during training
this.aAccu = []; // array to store model's accuracy values during training
// total number of training images in all classes
const TOTAL_TRAIN_IMAGES = this.NUM_CLASSES * this.NUM_TRAIN_IMAGES;
// total number of test images in all classes
const TOTAL_TEST_IMAGES = this.NUM_CLASSES * this.NUM_TEST_IMAGES;
// create Training Data arrays for storing training images and their corresponding classes
this.aTrainImages = new Float32Array(TOTAL_TRAIN_IMAGES * this.IMAGE_SIZE);
this.aTrainClasses = new Uint8Array(TOTAL_TRAIN_IMAGES);
// shuffle Training Data by creating an array of shuffled Train indices
this.aTrainIndices = tf.util.createShuffledIndices(TOTAL_TRAIN_IMAGES);
// the reference to the current element in the aTrainIndices[] array
this.trainElement = -1;
// create arrays of Test Data for storing test images and their corresponding classes
this.aTestImages = new Float32Array(TOTAL_TEST_IMAGES * this.IMAGE_SIZE);
this.aTestClasses = new Uint8Array(TOTAL_TEST_IMAGES);
// shuffle Test Data by creating an array of shuffled Test indices
this.aTestIndices = tf.util.createShuffledIndices(TOTAL_TEST_IMAGES);
// the reference to the current element in the aTestIndices[] array
this.testElement = -1;
// create a CNN model using a Sequential model type in which
// tensors are consecutively passed from one layer to the next
this.model = tf.sequential();
// add a convolutional layer
this.model.add(tf.layers.conv2d({
inputShape: [28, 28, 1],
kernelSize: 5,
filters: 8,
strides: 1,
activation: 'relu',
kernelInitializer: 'varianceScaling'
}));
// add a max pooling layer
this.model.add(tf.layers.maxPooling2d({
poolSize: [2, 2],
strides: [2, 2]
}));
// add a second convolutional layer
this.model.add(tf.layers.conv2d({
kernelSize: 5,
filters: 16,
strides: 1,
activation: 'relu',
kernelInitializer: 'varianceScaling'
}));
// add a second max pooling layer
this.model.add(tf.layers.maxPooling2d({
poolSize: [2, 2],
strides: [2, 2]
}));
// add a flatten layer to flatten the output of the previous layer to a vector
this.model.add(tf.layers.flatten());
// add a dense (fully connected) layer to perform the final classification
this.model.add(tf.layers.dense({
units: this.NUM_CLASSES,
kernelInitializer: 'varianceScaling',
activation: 'softmax'
}));
// compile the model
this.model.compile({
optimizer: tf.train.sgd(0.15), // optimizer with learning rate
loss: 'categoricalCrossentropy', // loss function
metrics: ['accuracy'], // evaluation metric
});
};
// ---------------------------------------------------------------------------------
// CNN Prototype
// ---------------------------------------------------------------------------------
/**
* Splits the entire dataset into training data and test data.
*
* @param {Uint8Array} imagesBuffer - the array with binary data of all images in the dataset
* @param {integer} dataset - the ordinal number of the dataset
*/
CNN.prototype.splitDataset = function(imagesBuffer, dataset){
// slice dataset to get training images and normalize them
var trainBuffer = new Float32Array(imagesBuffer.slice(0, this.IMAGE_SIZE * this.NUM_TRAIN_IMAGES));
trainBuffer = trainBuffer.map(function (cv){return cv/255.0});
// add training images and their corresponding classes into Training Data arrays
var start = dataset * this.NUM_TRAIN_IMAGES;
this.aTrainImages.set(trainBuffer, start * this.IMAGE_SIZE);
this.aTrainClasses.fill(dataset, start, start + this.NUM_TRAIN_IMAGES);
// slice dataset to get test images and normalize them
var testBuffer = new Float32Array(imagesBuffer.slice(this.IMAGE_SIZE * this.NUM_TRAIN_IMAGES));
testBuffer = testBuffer.map(function (cv){return cv/255.0});
// add test images and their corresponding classes into Test Data arrays
start = dataset * this.NUM_TEST_IMAGES;
this.aTestImages.set(testBuffer, start * this.IMAGE_SIZE);
this.aTestClasses.fill(dataset, start, start + this.NUM_TEST_IMAGES);
};
/**
* Trains the model
*/
CNN.prototype.train = async function(){
// reset the training flag to know the training is currently in progress
this.isTrainCompleted = false;
for (let i = 0; i < this.TRAIN_ITERATIONS; i++){
// increase the number of training iterations
this.trainIteration++;
this.main.ui.showStatusBar("Training the CNN - iteration " + this.trainIteration + ".");
// fetch the next Training Batch
let trainBatch = this.nextTrainBatch(this.TRAIN_BATCH_SIZE);
// create new Test Batch and Validation Set
let testBatch;
let validationSet;
if (i % this.TEST_FREQUENCY === 0){ // every few training passes...
// fetch the next Test Batch
testBatch = this.nextTestBatch(this.TEST_BATCH_SIZE);
// build Validation Set by using images and corresponding labels from Test Batch
validationSet = [testBatch.images, testBatch.labels];
}
// train the model
const training = await this.model.fit(
trainBatch.images,
trainBatch.labels,
{batchSize: this.TRAIN_BATCH_SIZE, validationData: validationSet, epochs: 1}
);
// get the model loss from the last training iteration and plot the Loss Chart
var maxLossLength = this.main.ui.bmpLossChart.width;
if (this.aLoss.length > maxLossLength) this.aLoss.shift();
this.aLoss.push(1 - Math.min(1, training.history.loss[0]));
this.main.ui.plotChart(this.main.ui.bmpLossChart, this.aLoss, 1);
if (testBatch != null) {
// get the model accuracy from the last training iteration and plot the Accuracy Chart
var maxAccuLength = this.main.ui.bmpAccuChart.width;
if (this.aAccu.length * this.TEST_FREQUENCY > maxAccuLength) this.aAccu.shift();
this.aAccu.push(1 - Math.min(1, training.history.acc[0]));
this.main.ui.plotChart(this.main.ui.bmpAccuChart, this.aAccu, this.TEST_FREQUENCY);
// dispose Test Batch from memory
testBatch.images.dispose();
testBatch.labels.dispose();
}
// dispose Training Batch from memory
trainBatch.images.dispose();
trainBatch.labels.dispose();
// mitigate blocking the UI thread and freezing the tab during training
await tf.nextFrame();
}
// set the training flag to know the training is completed
this.isTrainCompleted = true;
};
/**
* Returns a batch of images and their corresponding classes from the Training Data
*
* @param {integer} batchSize - how many images are included in training batch
*/
CNN.prototype.nextTrainBatch = function(batchSize){
return this.fetchBatch(
batchSize, this.aTrainImages, this.aTrainClasses,
() => {
this.trainElement = (this.trainElement + 1) % this.aTrainIndices.length;
return this.aTrainIndices[this.trainElement];
}
);
};
/**
* Returns a batch of images and their corresponding classes from the Test Data
*
* @param {integer} batchSize - how many images are included in test batch
*/
CNN.prototype.nextTestBatch = function(batchSize){
return this.fetchBatch(
batchSize, this.aTestImages, this.aTestClasses,
() => {
this.testElement = (this.testElement + 1) % this.aTestIndices.length;
return this.aTestIndices[this.testElement];
}
);
};
/**
* Fetches a batch of images and their corresponding classes
*
* @param {integer} batchSize - how many images are included in the batch
* @param {Float32Array} aImages - array of images
* @param {Uint8Array} aClasses - array of corresponding classes
* @param {integer} getIndex - a function which returns the index of an image that must be fetched
*/
CNN.prototype.fetchBatch = function(batchSize, aImages, aClasses, getIndex){
// create batch arrays
const batchImages = new Float32Array(batchSize * this.IMAGE_SIZE);
const batchLabels = new Uint8Array(batchSize * this.NUM_CLASSES);
for (let i = 0; i < batchSize; i++){
// get the index of the image we want to fetch
const idx = getIndex();
// fetch the image
const image = aImages.slice(idx * this.IMAGE_SIZE, (idx + 1) * this.IMAGE_SIZE);
// add the image to the batch of images
batchImages.set(image, i * this.IMAGE_SIZE);
// get the class number of this image
const class_num = aClasses[idx];
// generate the label for this image by using "one hot encoding method":
// define a vector where all elements are 0, beside one element
// which points to the correct class of this image
const label = new Uint8Array(this.NUM_CLASSES);
label[class_num] = 1;
// add the label to the batch of labels
batchLabels.set(label, i * this.NUM_CLASSES);
}
// convert batch of images to a temporary tensor
const images_temp = tf.tensor2d(batchImages, [batchSize, this.IMAGE_SIZE]);
// reshape the temporary tensor to the size of the model input shape
const images = images_temp.reshape([batchSize, 28, 28, 1]);
// dispose the temporary tensor to free memory
images_temp.dispose();
// convert batch of labels to tensor
const labels = tf.tensor2d(batchLabels, [batchSize, this.NUM_CLASSES]);
return {images, labels};
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 4)
*
* MODULE:
* main.js - Main Program
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 4. Training Model
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
/***********************************************************************************
/* Setup procedure for creating a new Phaser Game object on window load event
/***********************************************************************************/
window.onload = function () {
// create a new game object which is an instance of Phaser.Game
var game = new Phaser.Game(1280, 720, Phaser.CANVAS);
// add all States to the game object (this program has only the Main State)
game.state.add('MainState', App.MainState);
// start the Main State
game.state.start('MainState');
};
/***********************************************************************************
/* The Application Namespace
/***********************************************************************************/
var App = App || {};
// ---------------------------------------------------------------------------------
// Global constants and variables
// ---------------------------------------------------------------------------------
// the names of all datasets
App.DATASETS = ['car', 'fish', 'snowman'];
// ---------------------------------------------------------------------------------
// The Main State constructor
// ---------------------------------------------------------------------------------
App.MainState = function(){
// constants describing all modes of the main state
this.MODE_INIT = 1;
this.MODE_OPEN_FILE = 2;
this.MODE_LOAD_FILE = 3;
this.MODE_START_TRAIN = 4;
this.MODE_DO_TRAIN = 5;
this.MODE_START_PREDICT = 6;
this.MODE_DO_PREDICT = 7;
this.MODE_DRAW = 8;
this.MODE_CLICK_ON_TRAIN = 9;
// set initial mode
this.mode = this.MODE_INIT;
// the counter of currently loaded datasets
this.dataset = 0;
};
// ---------------------------------------------------------------------------------
// The Main State prototype
// ---------------------------------------------------------------------------------
App.MainState.prototype = {
/**
* Automatically called only once to load all assets.
*/
preload : function(){
this.game.load.image('imgBack', '../assets/img_back_4.png');
this.game.load.image('btnTrain', '../assets/btn_train.png');
this.game.load.image('btnMoreGames', '../assets/btn_moregames.png');
this.game.load.image('btnAuthor', '../assets/btn_author.png');
this.load.bitmapFont('fntBlackChars', '../assets/fnt_black_chars.png', '../assets/fnt_black_chars.fnt');
},
/**
* Automatically called immediately after all assets are loaded to create all objects.
*/
create : function(){
// scale game to cover the entire screen
this.scale.scaleMode = Phaser.ScaleManager.SHOW_ALL;
this.scale.pageAlignVertically = true;
this.scale.pageAlignHorizontally = true;
// keep game running if it loses the focus
this.game.stage.disableVisibilityChange = true;
// create background
this.game.add.sprite(0, 0, 'imgBack');
// create a loader for loading datasets
this.loader = new Phaser.Loader(this.game);
// create user interface with buttons, bitmaps and texts
this.ui = new UI(this);
// create a convolution neural network
this.cnn = new CNN(this);
},
/**
* Automatically called on every tick representing the main loop.
*/
update : function(){
switch(this.mode){
// initialize the game
case this.MODE_INIT:
this.ui.disableButtons();
this.mode = this.MODE_OPEN_FILE;
break;
// open dataset file and start loading it
case this.MODE_OPEN_FILE:
var fileName = App.DATASETS[this.dataset] + '.bin';
this.loader.reset();
this.loader.binary('input_file', '../data/'+fileName);
this.loader.start();
this.ui.showStatusBar("Loading " + fileName + " file.");
this.mode = this.MODE_LOAD_FILE;
break;
// wait on dataset file to be loaded
case this.MODE_LOAD_FILE:
if (this.loader.hasLoaded){
// split the loaded dataset into training data and test data
this.cnn.splitDataset(
new Uint8Array(this.game.cache.getBinary('input_file')),
this.dataset
);
// increase the number of loaded datasets
this.dataset++;
// if we have not loaded all datasets yet then go to load the next one
// else go to the CNN training
if (this.dataset < App.DATASETS.length){
this.mode = this.MODE_OPEN_FILE;
} else {
this.ui.showStatusBar("Initializing training.");
this.mode = this.MODE_START_TRAIN;
}
}
break;
// start with CNN training
case this.MODE_START_TRAIN:
this.ui.disableButtons();
this.cnn.train();
this.mode = this.MODE_DO_TRAIN;
break;
// wait for completion of the CNN training
case this.MODE_DO_TRAIN:
if (this.cnn.isTrainCompleted){
this.ui.showStatusBar("Training completed.");
this.ui.enableButtons();
this.mode = this.MODE_START_PREDICT;
}
break;
case this.MODE_START_PREDICT:
this.mode = this.MODE_DO_PREDICT;
break;
case this.MODE_DO_PREDICT:
this.mode = this.MODE_DRAW;
break;
case this.MODE_DRAW:
break;
// actions on clicking "Train More" button
case this.MODE_CLICK_ON_TRAIN:
this.mode = this.MODE_START_TRAIN;
break;
}
}
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 4)
*
* MODULE:
* ui.js - UI Class (User Interface)
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 4. Training Model
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
// ---------------------------------------------------------------------------------
// UI Constructor
// ---------------------------------------------------------------------------------
var UI = function(main){
// reference to the Main State
this.main = main;
// reference to the Phaser Game object
var game = this.main.game;
// create buttons
this.btnTrain = game.add.button(460, 625, 'btnTrain', this.onTrainClick, this);
this.btnMoreGames = game.add.button(1048, 625, 'btnMoreGames', this.onMoreGamesClick, this);
this.btnAuthor = game.add.button(1130, 703, 'btnAuthor', this.onMoreGamesClick, this);
this.btnAuthor.anchor.setTo(0.5);
// create a bitmap for plotting CNN loss trend during training
this.bmpAccuChart = game.add.bitmapData(350, 250);
this.bmpAccuChart.addToWorld(45, 95);
// create a bitmap for plotting CNN accuracy trend during training
this.bmpLossChart = game.add.bitmapData(350, 250);
this.bmpLossChart.addToWorld(45, 410);
// create a text which shows messages in the status bar
this.txtStatBar = game.add.bitmapText(10, 695, "fntBlackChars", "", 18);
};
// ---------------------------------------------------------------------------------
// UI Prototype
// ---------------------------------------------------------------------------------
/**
* Disables buttons.
*/
UI.prototype.disableButtons = function(){
this.btnTrain.kill();
};
/**
* Enables buttons.
*/
UI.prototype.enableButtons = function(){
this.btnTrain.revive();
};
/**
* Triggers on pressing "Train More" button.
*/
UI.prototype.onTrainClick = function(){
if (this.main.mode == this.main.MODE_DRAW){
this.main.mode = this.main.MODE_CLICK_ON_TRAIN;
}
};
/**
* Opens Official Web Site when click on "Play More Games" button.
*/
UI.prototype.onMoreGamesClick = function(){
window.open("http://www.askforgametask.com", "_blank");
};
/**
* Plots a line chart to show a trend over time.
*
* @param {Phaser.BitmapData} bmpChart - the bitmap used to draw lines between points on it
* @param {float Array} aValues - the array of trend values on the Y-axis
* @param {integer} dx - the space between two points on the X-axis
*/
UI.prototype.plotChart = function(bmpChart, aValues, dx){
bmpChart.clear();
for (var i = 1; i < aValues.length; i++){
var x1 = (i-1) * dx;
var y1 = bmpChart.height * aValues[i-1];
var x2 = i * dx;
var y2 = bmpChart.height * aValues[i];
bmpChart.line(x1, y1, x2, y2, '#61bc6d', 2);
}
};
/**
* Shows a message in the status bar.
*
* @param {String} strText - the text to be shown in the status bar
*/
UI.prototype.showStatusBar = function(strText){
this.txtStatBar.text = strText;
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 5)
*
* MODULE:
* cnn.js - CNN Class (Convolution Neural Network)
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 5. Predicting Samples
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
// ---------------------------------------------------------------------------------
// CNN Constructor
// ---------------------------------------------------------------------------------
var CNN = function(main){
// reference to the Main State
this.main = main;
this.NUM_CLASSES = App.DATASETS.length; // number of classes which can be recognized by CNN model
this.IMAGE_SIZE = 784; // size of an image in a dataset
this.NUM_TRAIN_IMAGES = 400; // number of training images in a dataset
this.NUM_TEST_IMAGES = 100; // number of test images in a dataset
this.TRAIN_ITERATIONS = 50; // total number of training iterations
this.TRAIN_BATCH_SIZE = 100; // number of training images used to train model during one iteration
this.TEST_FREQUENCY = 5; // frequency of testing model accuracy (one test on every 5 training iterations)
this.TEST_BATCH_SIZE = 50; // number of test images used to test model accuracy
this.trainIteration = 0; // current number of executed training iterations
this.aLoss = []; // array to store model's loss values during training
this.aAccu = []; // array to store model's accuracy values during training
// total number of training images in all classes
const TOTAL_TRAIN_IMAGES = this.NUM_CLASSES * this.NUM_TRAIN_IMAGES;
// total number of test images in all classes
const TOTAL_TEST_IMAGES = this.NUM_CLASSES * this.NUM_TEST_IMAGES;
// create Training Data arrays for storing training images and their corresponding classes
this.aTrainImages = new Float32Array(TOTAL_TRAIN_IMAGES * this.IMAGE_SIZE);
this.aTrainClasses = new Uint8Array(TOTAL_TRAIN_IMAGES);
// shuffle Training Data by creating an array of shuffled Train indices
this.aTrainIndices = tf.util.createShuffledIndices(TOTAL_TRAIN_IMAGES);
// the reference to the current element in the aTrainIndices[] array
this.trainElement = -1;
// create arrays of Test Data for storing test images and their corresponding classes
this.aTestImages = new Float32Array(TOTAL_TEST_IMAGES * this.IMAGE_SIZE);
this.aTestClasses = new Uint8Array(TOTAL_TEST_IMAGES);
// shuffle Test Data by creating an array of shuffled Test indices
this.aTestIndices = tf.util.createShuffledIndices(TOTAL_TEST_IMAGES);
// the reference to the current element in the aTestIndices[] array
this.testElement = -1;
// create a CNN model using a Sequential model type in which
// tensors are consecutively passed from one layer to the next
this.model = tf.sequential();
// add a convolutional layer
this.model.add(tf.layers.conv2d({
inputShape: [28, 28, 1],
kernelSize: 5,
filters: 8,
strides: 1,
activation: 'relu',
kernelInitializer: 'varianceScaling'
}));
// add a max pooling layer
this.model.add(tf.layers.maxPooling2d({
poolSize: [2, 2],
strides: [2, 2]
}));
// add a second convolutional layer
this.model.add(tf.layers.conv2d({
kernelSize: 5,
filters: 16,
strides: 1,
activation: 'relu',
kernelInitializer: 'varianceScaling'
}));
// add a second max pooling layer
this.model.add(tf.layers.maxPooling2d({
poolSize: [2, 2],
strides: [2, 2]
}));
// add a flatten layer to flatten the output of the previous layer to a vector
this.model.add(tf.layers.flatten());
// add a dense (fully connected) layer to perform the final classification
this.model.add(tf.layers.dense({
units: this.NUM_CLASSES,
kernelInitializer: 'varianceScaling',
activation: 'softmax'
}));
// compile the model
this.model.compile({
optimizer: tf.train.sgd(0.15), // optimizer with learning rate
loss: 'categoricalCrossentropy', // loss function
metrics: ['accuracy'], // evaluation metric
});
};
// ---------------------------------------------------------------------------------
// CNN Prototype
// ---------------------------------------------------------------------------------
/**
* Splits the entire dataset into training data and test data.
*
* @param {Uint8Array} imagesBuffer - the array with binary data of all images in the dataset
* @param {integer} dataset - the ordinal number of the dataset
*/
CNN.prototype.splitDataset = function(imagesBuffer, dataset){
// slice dataset to get training images and normalize them
var trainBuffer = new Float32Array(imagesBuffer.slice(0, this.IMAGE_SIZE * this.NUM_TRAIN_IMAGES));
trainBuffer = trainBuffer.map(function (cv){return cv/255.0});
// add training images and their corresponding classes into Training Data arrays
var start = dataset * this.NUM_TRAIN_IMAGES;
this.aTrainImages.set(trainBuffer, start * this.IMAGE_SIZE);
this.aTrainClasses.fill(dataset, start, start + this.NUM_TRAIN_IMAGES);
// slice dataset to get test images and normalize them
var testBuffer = new Float32Array(imagesBuffer.slice(this.IMAGE_SIZE * this.NUM_TRAIN_IMAGES));
testBuffer = testBuffer.map(function (cv){return cv/255.0});
// add test images and their corresponding classes into Test Data arrays
start = dataset * this.NUM_TEST_IMAGES;
this.aTestImages.set(testBuffer, start * this.IMAGE_SIZE);
this.aTestClasses.fill(dataset, start, start + this.NUM_TEST_IMAGES);
};
/**
* Trains the model
*/
CNN.prototype.train = async function(){
// reset the training flag to know the training is currently in progress
this.isTrainCompleted = false;
for (let i = 0; i < this.TRAIN_ITERATIONS; i++){
// increase the number of training iterations
this.trainIteration++;
this.main.ui.showStatusBar("Training the CNN - iteration " + this.trainIteration + ".");
// fetch the next Training Batch
let trainBatch = this.nextTrainBatch(this.TRAIN_BATCH_SIZE);
// create new Test Batch and Validation Set
let testBatch;
let validationSet;
if (i % this.TEST_FREQUENCY === 0){ // every few training passes...
// fetch the next Test Batch
testBatch = this.nextTestBatch(this.TEST_BATCH_SIZE);
// build Validation Set by using images and corresponding labels from Test Batch
validationSet = [testBatch.images, testBatch.labels];
}
// train the model
const training = await this.model.fit(
trainBatch.images,
trainBatch.labels,
{batchSize: this.TRAIN_BATCH_SIZE, validationData: validationSet, epochs: 1}
);
// get the model loss from the last training iteration and plot the Loss Chart
var maxLossLength = this.main.ui.bmpLossChart.width;
if (this.aLoss.length > maxLossLength) this.aLoss.shift();
this.aLoss.push(1 - Math.min(1, training.history.loss[0]));
this.main.ui.plotChart(this.main.ui.bmpLossChart, this.aLoss, 1);
if (testBatch != null) {
// get the model accuracy from the last training iteration and plot the Accuracy Chart
var maxAccuLength = this.main.ui.bmpAccuChart.width;
if (this.aAccu.length * this.TEST_FREQUENCY > maxAccuLength) this.aAccu.shift();
this.aAccu.push(1 - Math.min(1, training.history.acc[0]));
this.main.ui.plotChart(this.main.ui.bmpAccuChart, this.aAccu, this.TEST_FREQUENCY);
// dispose Test Batch from memory
testBatch.images.dispose();
testBatch.labels.dispose();
}
// dispose Training Batch from memory
trainBatch.images.dispose();
trainBatch.labels.dispose();
// mitigate blocking the UI thread and freezing the tab during training
await tf.nextFrame();
}
// set the training flag to know the training is completed
this.isTrainCompleted = true;
};
/**
* Predicts a batch of sample images fetched from Test Data
*/
CNN.prototype.predictSamples = async function(){
this.isSamplesPredicted = false;
const samplesBatch = this.nextTestBatch(App.NUM_SAMPLES);
tf.tidy(() => {
const output = this.model.predict(samplesBatch.images);
const aClassifications = Array.from(samplesBatch.labels.argMax(1).dataSync());
const aPredictions = Array.from(output.argMax(1).dataSync());
this.main.ui.showSamplePredictions(aClassifications, aPredictions);
});
tf.dispose(samplesBatch);
this.isSamplesPredicted = true;
};
/**
* Returns a batch of images and their corresponding classes from the Training Data
*
* @param {integer} batchSize - how many images are included in training batch
*/
CNN.prototype.nextTrainBatch = function(batchSize){
return this.fetchBatch(
batchSize, this.aTrainImages, this.aTrainClasses,
() => {
this.trainElement = (this.trainElement + 1) % this.aTrainIndices.length;
return this.aTrainIndices[this.trainElement];
}
);
};
/**
* Returns a batch of images and their corresponding classes from the Test Data
*
* @param {integer} batchSize - how many images are included in test batch
*/
CNN.prototype.nextTestBatch = function(batchSize){
return this.fetchBatch(
batchSize, this.aTestImages, this.aTestClasses,
() => {
this.testElement = (this.testElement + 1) % this.aTestIndices.length;
return this.aTestIndices[this.testElement];
}
);
};
/**
* Fetches a batch of images and their corresponding classes
*
* @param {integer} batchSize - how many images are included in the batch
* @param {Float32Array} aImages - array of images
* @param {Uint8Array} aClasses - array of corresponding classes
* @param {integer} getIndex - a function which returns the index of an image that must be fetched
*/
CNN.prototype.fetchBatch = function(batchSize, aImages, aClasses, getIndex){
// create batch arrays
const batchImages = new Float32Array(batchSize * this.IMAGE_SIZE);
const batchLabels = new Uint8Array(batchSize * this.NUM_CLASSES);
for (let i = 0; i < batchSize; i++){
// get the index of the image we want to fetch
const idx = getIndex();
// fetch the image
const image = aImages.slice(idx * this.IMAGE_SIZE, (idx + 1) * this.IMAGE_SIZE);
// add the image to the batch of images
batchImages.set(image, i * this.IMAGE_SIZE);
// get the class number of this image
const class_num = aClasses[idx];
// generate the label for this image by using "one hot encoding method":
// define a vector where all elements are 0, beside one element
// which points to the correct class of this image
const label = new Uint8Array(this.NUM_CLASSES);
label[class_num] = 1;
// add the label to the batch of labels
batchLabels.set(label, i * this.NUM_CLASSES);
}
// convert batch of images to a temporary tensor
const images_temp = tf.tensor2d(batchImages, [batchSize, this.IMAGE_SIZE]);
// reshape the temporary tensor to the size of the model input shape
const images = images_temp.reshape([batchSize, 28, 28, 1]);
// dispose the temporary tensor to free memory
images_temp.dispose();
// convert batch of labels to tensor
const labels = tf.tensor2d(batchLabels, [batchSize, this.NUM_CLASSES]);
return {images, labels};
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 5)
*
* MODULE:
* main.js - Main Program
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 5. Predicting Samples
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
/***********************************************************************************
/* Setup procedure for creating a new Phaser Game object on window load event
/***********************************************************************************/
window.onload = function () {
// create a new game object which is an instance of Phaser.Game
var game = new Phaser.Game(1280, 720, Phaser.CANVAS);
// add all States to the game object (this program has only the Main State)
game.state.add('MainState', App.MainState);
// start the Main State
game.state.start('MainState');
};
/***********************************************************************************
/* The Application Namespace
/***********************************************************************************/
var App = App || {};
// ---------------------------------------------------------------------------------
// Global constants and variables
// ---------------------------------------------------------------------------------
// the names of all datasets
App.DATASETS = ['car', 'fish', 'snowman'];
// the number of test data samples intended for predicting by CNN model
App.NUM_SAMPLES = 16;
// ---------------------------------------------------------------------------------
// The Main State constructor
// ---------------------------------------------------------------------------------
App.MainState = function(){
// constants describing all modes of the main state
this.MODE_INIT = 1;
this.MODE_OPEN_FILE = 2;
this.MODE_LOAD_FILE = 3;
this.MODE_START_TRAIN = 4;
this.MODE_DO_TRAIN = 5;
this.MODE_START_PREDICT = 6;
this.MODE_DO_PREDICT = 7;
this.MODE_DRAW = 8;
this.MODE_CLICK_ON_TRAIN = 9;
this.MODE_CLICK_ON_TEST = 10;
// set initial mode
this.mode = this.MODE_INIT;
// the counter of currently loaded datasets
this.dataset = 0;
};
// ---------------------------------------------------------------------------------
// The Main State prototype
// ---------------------------------------------------------------------------------
App.MainState.prototype = {
/**
* Automatically called only once to load all assets.
*/
preload : function(){
this.game.load.image('imgBack', '../assets/img_back_5.png');
this.game.load.image('btnTrain', '../assets/btn_train.png');
this.game.load.image('btnTest', '../assets/btn_test.png');
this.game.load.image('btnMoreGames', '../assets/btn_moregames.png');
this.game.load.image('btnAuthor', '../assets/btn_author.png');
this.load.bitmapFont('fntBlackChars', '../assets/fnt_black_chars.png', '../assets/fnt_black_chars.fnt');
},
/**
* Automatically called immediately after all assets are loaded to create all objects.
*/
create : function(){
// scale game to cover the entire screen
this.scale.scaleMode = Phaser.ScaleManager.SHOW_ALL;
this.scale.pageAlignVertically = true;
this.scale.pageAlignHorizontally = true;
// keep game running if it loses the focus
this.game.stage.disableVisibilityChange = true;
// create background
this.game.add.sprite(0, 0, 'imgBack');
// create a loader for loading datasets
this.loader = new Phaser.Loader(this.game);
// create user interface with buttons, bitmaps and texts
this.ui = new UI(this);
// create a convolution neural network
this.cnn = new CNN(this);
},
/**
* Automatically called on every tick representing the main loop.
*/
update : function(){
switch(this.mode){
// initialize the game
case this.MODE_INIT:
this.ui.disableButtons();
this.mode = this.MODE_OPEN_FILE;
break;
// open dataset file and start loading it
case this.MODE_OPEN_FILE:
var fileName = App.DATASETS[this.dataset] + '.bin';
this.loader.reset();
this.loader.binary('input_file', '../data/'+fileName);
this.loader.start();
this.ui.showStatusBar("Loading " + fileName + " file.");
this.mode = this.MODE_LOAD_FILE;
break;
// wait on dataset file to be loaded
case this.MODE_LOAD_FILE:
if (this.loader.hasLoaded){
// split the loaded dataset into training data and test data
this.cnn.splitDataset(
new Uint8Array(this.game.cache.getBinary('input_file')),
this.dataset
);
// increase the number of loaded datasets
this.dataset++;
// if we have not loaded all datasets yet then go to load the next one
// else go to the CNN training
if (this.dataset < App.DATASETS.length){
this.mode = this.MODE_OPEN_FILE;
} else {
this.ui.showStatusBar("Initializing training.");
this.mode = this.MODE_START_TRAIN;
}
}
break;
// start with CNN training
case this.MODE_START_TRAIN:
this.ui.disableButtons();
this.cnn.train();
this.mode = this.MODE_DO_TRAIN;
break;
// wait for completion of the CNN training
case this.MODE_DO_TRAIN:
if (this.cnn.isTrainCompleted){
this.ui.showStatusBar("Training completed. Predicting samples...");
this.mode = this.MODE_START_PREDICT;
}
break;
// draw sample images and start with predicting them by using CNN
case this.MODE_START_PREDICT:
this.ui.drawSampleImages();
this.cnn.predictSamples();
this.mode = this.MODE_DO_PREDICT;
break;
// wait for CNN to make predictions for all samples
case this.MODE_DO_PREDICT:
if (this.cnn.isSamplesPredicted){
this.ui.enableButtons();
this.ui.showStatusBar("Predicting samples completed.");
this.mode = this.MODE_DRAW;
}
break;
case this.MODE_DRAW:
break;
// actions on clicking "Train More" button
case this.MODE_CLICK_ON_TRAIN:
this.mode = this.MODE_START_TRAIN;
break;
// actions on clicking "Next Test" button
case this.MODE_CLICK_ON_TEST:
this.mode = this.MODE_START_PREDICT;
break;
}
}
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 5)
*
* MODULE:
* ui.js - UI Class (User Interface)
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 5. Predicting Samples
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
// ---------------------------------------------------------------------------------
// UI Constructor
// ---------------------------------------------------------------------------------
var UI = function(main){
// reference to the Main State
this.main = main;
// reference to the Phaser Game object
var game = this.main.game;
// create buttons
this.btnTrain = game.add.button(460, 625, 'btnTrain', this.onTrainClick, this);
this.btnTest = game.add.button(652, 625, 'btnTest', this.onTestClick, this);
this.btnMoreGames = game.add.button(1048, 625, 'btnMoreGames', this.onMoreGamesClick, this);
this.btnAuthor = game.add.button(1130, 703, 'btnAuthor', this.onMoreGamesClick, this);
this.btnAuthor.anchor.setTo(0.5);
// create a bitmap for plotting CNN loss trend during training
this.bmpAccuChart = game.add.bitmapData(350, 250);
this.bmpAccuChart.addToWorld(45, 95);
// create a bitmap for plotting CNN accuracy trend during training
this.bmpLossChart = game.add.bitmapData(350, 250);
this.bmpLossChart.addToWorld(45, 410);
// create a bitmap for showing all sample images intended for predicting by CNN
this.bmpSampleImages = game.add.bitmapData(28, (28+4) * App.NUM_SAMPLES);
this.bmpSampleImages.addToWorld(470, 95);
// create a bitmap for drawing green/red rectangles as results of CNN predictions on sample images
this.bmpSampleResults = game.add.bitmapData(125, (28+4) * App.NUM_SAMPLES);
this.bmpSampleResults.addToWorld(665, 95);
// create text objects
this.txtSampleClasses = []; // array of texts describing correct classes of samples
this.txtSamplePredictions = []; // array of texts describing predictions for samples
for (var i=0; i<App.NUM_SAMPLES; i++){
var y = 100 + i*32;
this.txtSampleClasses.push(
game.add.bitmapText(550, y, "fntBlackChars", "", 18)
);
this.txtSamplePredictions.push(
game.add.bitmapText(670, y, "fntBlackChars", "", 18)
);
}
// create a text which shows messages in the status bar
this.txtStatBar = game.add.bitmapText(10, 695, "fntBlackChars", "", 18);
};
// ---------------------------------------------------------------------------------
// UI Prototype
// ---------------------------------------------------------------------------------
/**
* Disables buttons.
*/
UI.prototype.disableButtons = function(){
this.btnTrain.kill();
this.btnTest.kill();
};
/**
* Enables buttons.
*/
UI.prototype.enableButtons = function(){
this.btnTrain.revive();
this.btnTest.revive();
};
/**
* Triggers on pressing "Train More" button.
*/
UI.prototype.onTrainClick = function(){
if (this.main.mode == this.main.MODE_DRAW){
this.main.mode = this.main.MODE_CLICK_ON_TRAIN;
}
};
/**
* Triggers on pressing "Next Test" button.
*/
UI.prototype.onTestClick = function(){
if (this.main.mode == this.main.MODE_DRAW){
this.main.mode = this.main.MODE_CLICK_ON_TEST;
}
};
/**
* Opens Official Web Site when click on "Play More Games" button.
*/
UI.prototype.onMoreGamesClick = function(){
window.open("http://www.askforgametask.com", "_blank");
};
/**
* Plots a line chart to show a trend over time.
*
* @param {Phaser.BitmapData} bmpChart - the bitmap used to draw lines between points on it
* @param {float Array} aValues - the array of trend values on the Y-axis
* @param {integer} dx - the space between two points on the X-axis
*/
UI.prototype.plotChart = function(bmpChart, aValues, dx){
bmpChart.clear();
for (var i = 1; i < aValues.length; i++){
var x1 = (i-1) * dx;
var y1 = bmpChart.height * aValues[i-1];
var x2 = i * dx;
var y2 = bmpChart.height * aValues[i];
bmpChart.line(x1, y1, x2, y2, '#61bc6d', 2);
}
};
/**
* Draws all sample images intended for CNN prediction.
*/
UI.prototype.drawSampleImages = function(){
// clear the bitmap used for drawing samples on it
this.bmpSampleImages.clear();
// get the reference to the first sample
var sample = this.main.cnn.testElement;
// for all samples...
for (var n = 0; n < App.NUM_SAMPLES; n++){
// get the reference to the next sample
sample = (sample + 1) % this.main.cnn.aTestIndices.length;
// get the starting position of the first pixel of this sample
var index = this.main.cnn.aTestIndices[sample];
var start = index * this.main.cnn.IMAGE_SIZE;
// for all pixels of this sample...
for (var i = 0; i < this.main.cnn.IMAGE_SIZE; i++){
// get the color of the current pixel
var pixel = this.main.cnn.aTestImages[i + start];
var color = 255 - ((pixel * 255)>>0) & 0xFF;
// calculate XY position of this pixel on the bitmap
var x = i%28;
var y = (i/28)>>0;
// set this pixel on the bitmap
this.bmpSampleImages.setPixel32(x, y + n*32, color, color, color, 255, false);
}
}
// put the image data of the bitmap on the context to show all samples
this.bmpSampleImages.context.putImageData(this.bmpSampleImages.imageData, 0, 0);
};
/**
* Shows correct classifications for all sample images
* with their predictions displayed inside green/red rectangle.
*
* @param {integer Array} aClassifications
* - the array of index numbers pointing to the correct classes of samples
*
* @param {integer Array} aPredictions
* - the array of index numbers pointing to the predicted classes of samples
*/
UI.prototype.showSamplePredictions = function(aClassifications, aPredictions){
this.bmpSampleResults.clear();
for (var i=0; i<aClassifications.length; i++){
// set the text of correct class of this sample
this.txtSampleClasses[i].text = App.DATASETS[aClassifications[i]];
// set the text of predicted class of this sample
this.txtSamplePredictions[i].text = App.DATASETS[aPredictions[i]];
// if (classification = prediction) then draw green rectangle else draw red rectangle
var color = (this.txtSampleClasses[i].text === this.txtSamplePredictions[i].text) ? '#61bc6d' : '#e24939';
this.bmpSampleResults.rect(0, 2 + i*32, this.bmpSampleResults.width, 24, color);
}
};
/**
* Shows a message in the status bar.
*
* @param {String} strText - the text to be shown in the status bar
*/
UI.prototype.showStatusBar = function(strText){
this.txtStatBar.text = strText;
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 6)
*
* MODULE:
* cnn.js - CNN Class (Convolution Neural Network)
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 6. Drawing Doodle
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
// ---------------------------------------------------------------------------------
// CNN Constructor
// ---------------------------------------------------------------------------------
var CNN = function(main){
// reference to the Main State
this.main = main;
this.NUM_CLASSES = App.DATASETS.length; // number of classes which can be recognized by CNN model
this.IMAGE_SIZE = 784; // size of an image in a dataset
this.NUM_TRAIN_IMAGES = 400; // number of training images in a dataset
this.NUM_TEST_IMAGES = 100; // number of test images in a dataset
this.TRAIN_ITERATIONS = 50; // total number of training iterations
this.TRAIN_BATCH_SIZE = 100; // number of training images used to train model during one iteration
this.TEST_FREQUENCY = 5; // frequency of testing model accuracy (one test on every 5 training iterations)
this.TEST_BATCH_SIZE = 50; // number of test images used to test model accuracy
this.trainIteration = 0; // current number of executed training iterations
this.aLoss = []; // array to store model's loss values during training
this.aAccu = []; // array to store model's accuracy values during training
// total number of training images in all classes
const TOTAL_TRAIN_IMAGES = this.NUM_CLASSES * this.NUM_TRAIN_IMAGES;
// total number of test images in all classes
const TOTAL_TEST_IMAGES = this.NUM_CLASSES * this.NUM_TEST_IMAGES;
// create Training Data arrays for storing training images and their corresponding classes
this.aTrainImages = new Float32Array(TOTAL_TRAIN_IMAGES * this.IMAGE_SIZE);
this.aTrainClasses = new Uint8Array(TOTAL_TRAIN_IMAGES);
// shuffle Training Data by creating an array of shuffled Train indices
this.aTrainIndices = tf.util.createShuffledIndices(TOTAL_TRAIN_IMAGES);
// the reference to the current element in the aTrainIndices[] array
this.trainElement = -1;
// create arrays of Test Data for storing test images and their corresponding classes
this.aTestImages = new Float32Array(TOTAL_TEST_IMAGES * this.IMAGE_SIZE);
this.aTestClasses = new Uint8Array(TOTAL_TEST_IMAGES);
// shuffle Test Data by creating an array of shuffled Test indices
this.aTestIndices = tf.util.createShuffledIndices(TOTAL_TEST_IMAGES);
// the reference to the current element in the aTestIndices[] array
this.testElement = -1;
// create a CNN model using a Sequential model type in which
// tensors are consecutively passed from one layer to the next
this.model = tf.sequential();
// add a convolutional layer
this.model.add(tf.layers.conv2d({
inputShape: [28, 28, 1],
kernelSize: 5,
filters: 8,
strides: 1,
activation: 'relu',
kernelInitializer: 'varianceScaling'
}));
// add a max pooling layer
this.model.add(tf.layers.maxPooling2d({
poolSize: [2, 2],
strides: [2, 2]
}));
// add a second convolutional layer
this.model.add(tf.layers.conv2d({
kernelSize: 5,
filters: 16,
strides: 1,
activation: 'relu',
kernelInitializer: 'varianceScaling'
}));
// add a second max pooling layer
this.model.add(tf.layers.maxPooling2d({
poolSize: [2, 2],
strides: [2, 2]
}));
// add a flatten layer to flatten the output of the previous layer to a vector
this.model.add(tf.layers.flatten());
// add a dense (fully connected) layer to perform the final classification
this.model.add(tf.layers.dense({
units: this.NUM_CLASSES,
kernelInitializer: 'varianceScaling',
activation: 'softmax'
}));
// compile the model
this.model.compile({
optimizer: tf.train.sgd(0.15), // optimizer with learning rate
loss: 'categoricalCrossentropy', // loss function
metrics: ['accuracy'], // evaluation metric
});
};
// ---------------------------------------------------------------------------------
// CNN Prototype
// ---------------------------------------------------------------------------------
/**
* Splits the entire dataset into training data and test data.
*
* @param {Uint8Array} imagesBuffer - the array with binary data of all images in the dataset
* @param {integer} dataset - the ordinal number of the dataset
*/
CNN.prototype.splitDataset = function(imagesBuffer, dataset){
// slice dataset to get training images and normalize them
var trainBuffer = new Float32Array(imagesBuffer.slice(0, this.IMAGE_SIZE * this.NUM_TRAIN_IMAGES));
trainBuffer = trainBuffer.map(function (cv){return cv/255.0});
// add training images and their corresponding classes into Training Data arrays
var start = dataset * this.NUM_TRAIN_IMAGES;
this.aTrainImages.set(trainBuffer, start * this.IMAGE_SIZE);
this.aTrainClasses.fill(dataset, start, start + this.NUM_TRAIN_IMAGES);
// slice dataset to get test images and normalize them
var testBuffer = new Float32Array(imagesBuffer.slice(this.IMAGE_SIZE * this.NUM_TRAIN_IMAGES));
testBuffer = testBuffer.map(function (cv){return cv/255.0});
// add test images and their corresponding classes into Test Data arrays
start = dataset * this.NUM_TEST_IMAGES;
this.aTestImages.set(testBuffer, start * this.IMAGE_SIZE);
this.aTestClasses.fill(dataset, start, start + this.NUM_TEST_IMAGES);
};
/**
* Trains the model
*/
CNN.prototype.train = async function(){
// reset the training flag to know the training is currently in progress
this.isTrainCompleted = false;
for (let i = 0; i < this.TRAIN_ITERATIONS; i++){
// increase the number of training iterations
this.trainIteration++;
this.main.ui.showStatusBar("Training the CNN - iteration " + this.trainIteration + ".");
// fetch the next Training Batch
let trainBatch = this.nextTrainBatch(this.TRAIN_BATCH_SIZE);
// create new Test Batch and Validation Set
let testBatch;
let validationSet;
if (i % this.TEST_FREQUENCY === 0){ // every few training passes...
// fetch the next Test Batch
testBatch = this.nextTestBatch(this.TEST_BATCH_SIZE);
// build Validation Set by using images and corresponding labels from Test Batch
validationSet = [testBatch.images, testBatch.labels];
}
// train the model
const training = await this.model.fit(
trainBatch.images,
trainBatch.labels,
{batchSize: this.TRAIN_BATCH_SIZE, validationData: validationSet, epochs: 1}
);
// get the model loss from the last training iteration and plot the Loss Chart
var maxLossLength = this.main.ui.bmpLossChart.width;
if (this.aLoss.length > maxLossLength) this.aLoss.shift();
this.aLoss.push(1 - Math.min(1, training.history.loss[0]));
this.main.ui.plotChart(this.main.ui.bmpLossChart, this.aLoss, 1);
if (testBatch != null) {
// get the model accuracy from the last training iteration and plot the Accuracy Chart
var maxAccuLength = this.main.ui.bmpAccuChart.width;
if (this.aAccu.length * this.TEST_FREQUENCY > maxAccuLength) this.aAccu.shift();
this.aAccu.push(1 - Math.min(1, training.history.acc[0]));
this.main.ui.plotChart(this.main.ui.bmpAccuChart, this.aAccu, this.TEST_FREQUENCY);
// dispose Test Batch from memory
testBatch.images.dispose();
testBatch.labels.dispose();
}
// dispose Training Batch from memory
trainBatch.images.dispose();
trainBatch.labels.dispose();
// mitigate blocking the UI thread and freezing the tab during training
await tf.nextFrame();
}
// set the training flag to know the training is completed
this.isTrainCompleted = true;
};
/**
* Predicts a batch of sample images fetched from Test Data
*/
CNN.prototype.predictSamples = async function(){
this.isSamplesPredicted = false;
const samplesBatch = this.nextTestBatch(App.NUM_SAMPLES);
tf.tidy(() => {
const output = this.model.predict(samplesBatch.images);
const aClassifications = Array.from(samplesBatch.labels.argMax(1).dataSync());
const aPredictions = Array.from(output.argMax(1).dataSync());
this.main.ui.showSamplePredictions(aClassifications, aPredictions);
});
tf.dispose(samplesBatch);
this.isSamplesPredicted = true;
};
/**
* Returns a batch of images and their corresponding classes from the Training Data
*
* @param {integer} batchSize - how many images are included in training batch
*/
CNN.prototype.nextTrainBatch = function(batchSize){
return this.fetchBatch(
batchSize, this.aTrainImages, this.aTrainClasses,
() => {
this.trainElement = (this.trainElement + 1) % this.aTrainIndices.length;
return this.aTrainIndices[this.trainElement];
}
);
};
/**
* Returns a batch of images and their corresponding classes from the Test Data
*
* @param {integer} batchSize - how many images are included in test batch
*/
CNN.prototype.nextTestBatch = function(batchSize){
return this.fetchBatch(
batchSize, this.aTestImages, this.aTestClasses,
() => {
this.testElement = (this.testElement + 1) % this.aTestIndices.length;
return this.aTestIndices[this.testElement];
}
);
};
/**
* Fetches a batch of images and their corresponding classes
*
* @param {integer} batchSize - how many images are included in the batch
* @param {Float32Array} aImages - array of images
* @param {Uint8Array} aClasses - array of corresponding classes
* @param {integer} getIndex - a function which returns the index of an image that must be fetched
*/
CNN.prototype.fetchBatch = function(batchSize, aImages, aClasses, getIndex){
// create batch arrays
const batchImages = new Float32Array(batchSize * this.IMAGE_SIZE);
const batchLabels = new Uint8Array(batchSize * this.NUM_CLASSES);
for (let i = 0; i < batchSize; i++){
// get the index of the image we want to fetch
const idx = getIndex();
// fetch the image
const image = aImages.slice(idx * this.IMAGE_SIZE, (idx + 1) * this.IMAGE_SIZE);
// add the image to the batch of images
batchImages.set(image, i * this.IMAGE_SIZE);
// get the class number of this image
const class_num = aClasses[idx];
// generate the label for this image by using "one hot encoding method":
// define a vector where all elements are 0, beside one element
// which points to the correct class of this image
const label = new Uint8Array(this.NUM_CLASSES);
label[class_num] = 1;
// add the label to the batch of labels
batchLabels.set(label, i * this.NUM_CLASSES);
}
// convert batch of images to a temporary tensor
const images_temp = tf.tensor2d(batchImages, [batchSize, this.IMAGE_SIZE]);
// reshape the temporary tensor to the size of the model input shape
const images = images_temp.reshape([batchSize, 28, 28, 1]);
// dispose the temporary tensor to free memory
images_temp.dispose();
// convert batch of labels to tensor
const labels = tf.tensor2d(batchLabels, [batchSize, this.NUM_CLASSES]);
return {images, labels};
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 6)
*
* MODULE:
* main.js - Main Program
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 6. Drawing Doodle
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
/***********************************************************************************
/* Setup procedure for creating a new Phaser Game object on window load event
/***********************************************************************************/
window.onload = function () {
// create a new game object which is an instance of Phaser.Game
var game = new Phaser.Game(1280, 720, Phaser.CANVAS);
// add all States to the game object (this program has only the Main State)
game.state.add('MainState', App.MainState);
// start the Main State
game.state.start('MainState');
};
/***********************************************************************************
/* The Application Namespace
/***********************************************************************************/
var App = App || {};
// ---------------------------------------------------------------------------------
// Global constants and variables
// ---------------------------------------------------------------------------------
// the names of all datasets
App.DATASETS = ['car', 'fish', 'snowman'];
// the number of test data samples intended for predicting by CNN model
App.NUM_SAMPLES = 16;
// ---------------------------------------------------------------------------------
// The Main State constructor
// ---------------------------------------------------------------------------------
App.MainState = function(){
// constants describing all modes of the main state
this.MODE_INIT = 1;
this.MODE_OPEN_FILE = 2;
this.MODE_LOAD_FILE = 3;
this.MODE_START_TRAIN = 4;
this.MODE_DO_TRAIN = 5;
this.MODE_START_PREDICT = 6;
this.MODE_DO_PREDICT = 7;
this.MODE_DRAW = 8;
this.MODE_CLICK_ON_TRAIN = 9;
this.MODE_CLICK_ON_TEST = 10;
this.MODE_CLICK_ON_CLEAR = 11;
// set initial mode
this.mode = this.MODE_INIT;
// the counter of currently loaded datasets
this.dataset = 0;
};
// ---------------------------------------------------------------------------------
// The Main State prototype
// ---------------------------------------------------------------------------------
App.MainState.prototype = {
/**
* Automatically called only once to load all assets.
*/
preload : function(){
this.game.load.image('imgBack', '../assets/img_back_6.png');
this.game.load.image('imgDisable', '../assets/img_disable.png');
this.game.load.image('btnTrain', '../assets/btn_train.png');
this.game.load.image('btnTest', '../assets/btn_test.png');
this.game.load.image('btnClear', '../assets/btn_clear.png');
this.game.load.image('btnMoreGames', '../assets/btn_moregames.png');
this.game.load.image('btnAuthor', '../assets/btn_author.png');
this.load.bitmapFont('fntBlackChars', '../assets/fnt_black_chars.png', '../assets/fnt_black_chars.fnt');
},
/**
* Automatically called immediately after all assets are loaded to create all objects.
*/
create : function(){
// scale game to cover the entire screen
this.scale.scaleMode = Phaser.ScaleManager.SHOW_ALL;
this.scale.pageAlignVertically = true;
this.scale.pageAlignHorizontally = true;
// keep game running if it loses the focus
this.game.stage.disableVisibilityChange = true;
// create background
this.game.add.sprite(0, 0, 'imgBack');
// create a loader for loading datasets
this.loader = new Phaser.Loader(this.game);
// create user interface with buttons, bitmaps and texts
this.ui = new UI(this);
// create a convolution neural network
this.cnn = new CNN(this);
// create a painter for drawing doodles
this.painter = new Painter(this);
},
/**
* Automatically called on every tick representing the main loop.
*/
update : function(){
switch(this.mode){
// initialize the game
case this.MODE_INIT:
this.painter.reset();
this.painter.disable();
this.ui.disableButtons();
this.mode = this.MODE_OPEN_FILE;
break;
// open dataset file and start loading it
case this.MODE_OPEN_FILE:
var fileName = App.DATASETS[this.dataset] + '.bin';
this.loader.reset();
this.loader.binary('input_file', '../data/'+fileName);
this.loader.start();
this.ui.showStatusBar("Loading " + fileName + " file.");
this.mode = this.MODE_LOAD_FILE;
break;
// wait on dataset file to be loaded
case this.MODE_LOAD_FILE:
if (this.loader.hasLoaded){
// split the loaded dataset into training data and test data
this.cnn.splitDataset(
new Uint8Array(this.game.cache.getBinary('input_file')),
this.dataset
);
// increase the number of loaded datasets
this.dataset++;
// if we have not loaded all datasets yet then go to load the next one
// else go to the CNN training
if (this.dataset < App.DATASETS.length){
this.mode = this.MODE_OPEN_FILE;
} else {
this.ui.showStatusBar("Initializing training.");
this.mode = this.MODE_START_TRAIN;
}
}
break;
// start with CNN training
case this.MODE_START_TRAIN:
this.painter.disable();
this.ui.disableButtons();
this.cnn.train();
this.mode = this.MODE_DO_TRAIN;
break;
// wait for completion of the CNN training
case this.MODE_DO_TRAIN:
if (this.cnn.isTrainCompleted){
this.ui.showStatusBar("Training completed. Predicting samples...");
this.mode = this.MODE_START_PREDICT;
}
break;
// draw sample images and start with predicting them by using CNN
case this.MODE_START_PREDICT:
this.ui.drawSampleImages();
this.cnn.predictSamples();
this.mode = this.MODE_DO_PREDICT;
break;
// wait for CNN to make predictions for all samples
case this.MODE_DO_PREDICT:
if (this.cnn.isSamplesPredicted){
this.painter.enable();
this.ui.enableButtons();
this.ui.showStatusBar(
"Draw " + App.DATASETS.join(", ") +
" to recognize your drawing!"
);
this.mode = this.MODE_DRAW;
}
break;
// draw a doodle
case this.MODE_DRAW:
if (this.game.input.mousePointer.isDown){
this.painter.draw(this.game.input.x, this.game.input.y);
} else {
this.painter.isDrawing = false;
}
break;
// actions on clicking "Train More" button
case this.MODE_CLICK_ON_TRAIN:
this.mode = this.MODE_START_TRAIN;
break;
// actions on clicking "Next Test" button
case this.MODE_CLICK_ON_TEST:
this.mode = this.MODE_START_PREDICT;
break;
// actions on clicking "Clear" button
case this.MODE_CLICK_ON_CLEAR:
this.painter.reset();
this.mode = this.MODE_DRAW;
break;
}
}
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 6)
*
* MODULE:
* painter.js - Painter Class
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 6. Drawing Doodle
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
// ---------------------------------------------------------------------------------
// Painter Constructor
// ---------------------------------------------------------------------------------
var Painter = function(main){
// reference to the Main State
this.main = main;
// reference to the Phaser Game object
var game = this.main.game;
// define a drawing area
this.DRAWING_AREA = new Phaser.Rectangle(842, 95, 416, 416);
// create a bitmap of the same size as the drawing area to draw a doodle on it
this.bmpOrigDrawing = game.make.bitmapData(this.DRAWING_AREA.width+2, this.DRAWING_AREA.height+2);
this.bmpOrigDrawing.addToWorld(this.DRAWING_AREA.x-1, this.DRAWING_AREA.y-1);
// create a sprite that visualizes that drawing a doodle is disabled
this.sprDisableEffect = game.add.sprite(this.DRAWING_AREA.x-1, this.DRAWING_AREA.y-1, 'imgDisable');
this.sprDisableEffect.width = this.bmpOrigDrawing.width;
this.sprDisableEffect.height = this.bmpOrigDrawing.height;
};
// ---------------------------------------------------------------------------------
// Painter Prototype
// ---------------------------------------------------------------------------------
/**
* Resets Painter parameters.
*/
Painter.prototype.reset = function(){
// reset the flag to know if something new is drawn
this.isDrawing = false;
// reset pencil position and size
this.pencil = {x:0, y:0, prevX:0, prevY:0, left:0, top:0, right:0, bottom:0};
// clear bitmaps by filling them with white color
this.bmpOrigDrawing.fill(255, 255, 255);
};
/**
* Hides the sprite with disabling effect over the drawing area.
*/
Painter.prototype.enable = function(){
this.sprDisableEffect.kill();
};
/**
* Shows the sprite with disabling effect over the drawing area.
*/
Painter.prototype.disable = function(){
this.sprDisableEffect.revive();
};
/**
* Draws a quadratic curve from the previous point to the current point
*
* @param {number} x - the X coordinate of the mouse
* @param {number} y - the Y coordinate of the mouse
*/
Painter.prototype.draw = function(x, y){
if (this.DRAWING_AREA.contains(x, y)){ // if mouse is inside the drawing area...
// calculate pencil position and size
this.pencil.prevX = this.pencil.x;
this.pencil.prevY = this.pencil.y;
this.pencil.x = x - this.DRAWING_AREA.x;
this.pencil.y = y - this.DRAWING_AREA.y;
this.pencil.left = this.pencil.x - 5;
this.pencil.top = this.pencil.y - 5;
this.pencil.right = this.pencil.x + 5;
this.pencil.bottom = this.pencil.y + 5;
// draw a circle at the current pencil position
this.bmpOrigDrawing.circle(this.pencil.x, this.pencil.y, 4, '#000');
if (!this.isDrawing){
// if this is the first drawing point, don't draw anything more,
// just set the flag to know that drawing has been started
this.isDrawing = true;
} else {
// else draw a quadratic curve from the previous to the current point
var xc = (this.pencil.prevX + this.pencil.x) / 2;
var yc = (this.pencil.prevY + this.pencil.y) / 2;
var ctx = this.bmpOrigDrawing.context;
ctx.beginPath();
ctx.quadraticCurveTo(this.pencil.prevX, this.pencil.prevY, xc, yc);
ctx.quadraticCurveTo(xc, yc, this.pencil.x, this.pencil.y);
ctx.lineWidth = 9;
ctx.strokeStyle = '#000';
ctx.stroke();
ctx.closePath();
}
} else { // else if mouse is outside of the drawing area...
this.isDrawing = false;
}
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 6)
*
* MODULE:
* ui.js - UI Class (User Interface)
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 6. Drawing Doodle
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
// ---------------------------------------------------------------------------------
// UI Constructor
// ---------------------------------------------------------------------------------
var UI = function(main){
// reference to the Main State
this.main = main;
// reference to the Phaser Game object
var game = this.main.game;
// create buttons
this.btnTrain = game.add.button(460, 625, 'btnTrain', this.onTrainClick, this);
this.btnTest = game.add.button(652, 625, 'btnTest', this.onTestClick, this);
this.btnClear = game.add.button(842, 625, 'btnClear', this.onClearClick, this);
this.btnMoreGames = game.add.button(1048, 625, 'btnMoreGames', this.onMoreGamesClick, this);
this.btnAuthor = game.add.button(1130, 703, 'btnAuthor', this.onMoreGamesClick, this);
this.btnAuthor.anchor.setTo(0.5);
// create a bitmap for plotting CNN loss trend during training
this.bmpAccuChart = game.add.bitmapData(350, 250);
this.bmpAccuChart.addToWorld(45, 95);
// create a bitmap for plotting CNN accuracy trend during training
this.bmpLossChart = game.add.bitmapData(350, 250);
this.bmpLossChart.addToWorld(45, 410);
// create a bitmap for showing all sample images intended for predicting by CNN
this.bmpSampleImages = game.add.bitmapData(28, (28+4) * App.NUM_SAMPLES);
this.bmpSampleImages.addToWorld(470, 95);
// create a bitmap for drawing green/red rectangles as results of CNN predictions on sample images
this.bmpSampleResults = game.add.bitmapData(125, (28+4) * App.NUM_SAMPLES);
this.bmpSampleResults.addToWorld(665, 95);
// create text objects
this.txtSampleClasses = []; // array of texts describing correct classes of samples
this.txtSamplePredictions = []; // array of texts describing predictions for samples
for (var i=0; i<App.NUM_SAMPLES; i++){
var y = 100 + i*32;
this.txtSampleClasses.push(
game.add.bitmapText(550, y, "fntBlackChars", "", 18)
);
this.txtSamplePredictions.push(
game.add.bitmapText(670, y, "fntBlackChars", "", 18)
);
}
// create a text which shows messages in the status bar
this.txtStatBar = game.add.bitmapText(10, 695, "fntBlackChars", "", 18);
};
// ---------------------------------------------------------------------------------
// UI Prototype
// ---------------------------------------------------------------------------------
/**
* Disables buttons.
*/
UI.prototype.disableButtons = function(){
this.btnTrain.kill();
this.btnTest.kill();
this.btnClear.kill();
};
/**
* Enables buttons.
*/
UI.prototype.enableButtons = function(){
this.btnTrain.revive();
this.btnTest.revive();
this.btnClear.revive();
};
/**
* Triggers on pressing "Train More" button.
*/
UI.prototype.onTrainClick = function(){
if (this.main.mode == this.main.MODE_DRAW){
this.main.mode = this.main.MODE_CLICK_ON_TRAIN;
}
};
/**
* Triggers on pressing "Next Test" button.
*/
UI.prototype.onTestClick = function(){
if (this.main.mode == this.main.MODE_DRAW){
this.main.mode = this.main.MODE_CLICK_ON_TEST;
}
};
/**
* Triggers on pressing "Clear" button.
*/
UI.prototype.onClearClick = function(){
if (this.main.mode == this.main.MODE_DRAW){
this.main.mode = this.main.MODE_CLICK_ON_CLEAR;
}
};
/**
* Opens Official Web Site when click on "Play More Games" button.
*/
UI.prototype.onMoreGamesClick = function(){
window.open("http://www.askforgametask.com", "_blank");
};
/**
* Plots a line chart to show a trend over time.
*
* @param {Phaser.BitmapData} bmpChart - the bitmap used to draw lines between points on it
* @param {float Array} aValues - the array of trend values on the Y-axis
* @param {integer} dx - the space between two points on the X-axis
*/
UI.prototype.plotChart = function(bmpChart, aValues, dx){
bmpChart.clear();
for (var i = 1; i < aValues.length; i++){
var x1 = (i-1) * dx;
var y1 = bmpChart.height * aValues[i-1];
var x2 = i * dx;
var y2 = bmpChart.height * aValues[i];
bmpChart.line(x1, y1, x2, y2, '#61bc6d', 2);
}
};
/**
* Draws all sample images intended for CNN prediction.
*/
UI.prototype.drawSampleImages = function(){
// clear the bitmap used for drawing samples on it
this.bmpSampleImages.clear();
// get the reference to the first sample
var sample = this.main.cnn.testElement;
// for all samples...
for (var n = 0; n < App.NUM_SAMPLES; n++){
// get the reference to the next sample
sample = (sample + 1) % this.main.cnn.aTestIndices.length;
// get the starting position of the first pixel of this sample
var index = this.main.cnn.aTestIndices[sample];
var start = index * this.main.cnn.IMAGE_SIZE;
// for all pixels of this sample...
for (var i = 0; i < this.main.cnn.IMAGE_SIZE; i++){
// get the color of the current pixel
var pixel = this.main.cnn.aTestImages[i + start];
var color = 255 - ((pixel * 255)>>0) & 0xFF;
// calculate XY position of this pixel on the bitmap
var x = i%28;
var y = (i/28)>>0;
// set this pixel on the bitmap
this.bmpSampleImages.setPixel32(x, y + n*32, color, color, color, 255, false);
}
}
// put the image data of the bitmap on the context to show all samples
this.bmpSampleImages.context.putImageData(this.bmpSampleImages.imageData, 0, 0);
};
/**
* Shows correct classifications for all sample images
* with their predictions displayed inside green/red rectangle.
*
* @param {integer Array} aClassifications
* - the array of index numbers pointing to the correct classes of samples
*
* @param {integer Array} aPredictions
* - the array of index numbers pointing to the predicted classes of samples
*/
UI.prototype.showSamplePredictions = function(aClassifications, aPredictions){
this.bmpSampleResults.clear();
for (var i=0; i<aClassifications.length; i++){
// set the text of correct class of this sample
this.txtSampleClasses[i].text = App.DATASETS[aClassifications[i]];
// set the text of predicted class of this sample
this.txtSamplePredictions[i].text = App.DATASETS[aPredictions[i]];
// if (classification = prediction) then draw green rectangle else draw red rectangle
var color = (this.txtSampleClasses[i].text === this.txtSamplePredictions[i].text) ? '#61bc6d' : '#e24939';
this.bmpSampleResults.rect(0, 2 + i*32, this.bmpSampleResults.width, 24, color);
}
};
/**
* Shows a message in the status bar.
*
* @param {String} strText - the text to be shown in the status bar
*/
UI.prototype.showStatusBar = function(strText){
this.txtStatBar.text = strText;
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 7)
*
* MODULE:
* cnn.js - CNN Class (Convolution Neural Network)
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 7. Predicting Doodle
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
// ---------------------------------------------------------------------------------
// CNN Constructor
// ---------------------------------------------------------------------------------
var CNN = function(main){
// reference to the Main State
this.main = main;
this.NUM_CLASSES = App.DATASETS.length; // number of classes which can be recognized by CNN model
this.IMAGE_SIZE = 784; // size of an image in a dataset
this.NUM_TRAIN_IMAGES = 400; // number of training images in a dataset
this.NUM_TEST_IMAGES = 100; // number of test images in a dataset
this.TRAIN_ITERATIONS = 50; // total number of training iterations
this.TRAIN_BATCH_SIZE = 100; // number of training images used to train model during one iteration
this.TEST_FREQUENCY = 5; // frequency of testing model accuracy (one test on every 5 training iterations)
this.TEST_BATCH_SIZE = 50; // number of test images used to test model accuracy
this.trainIteration = 0; // current number of executed training iterations
this.aLoss = []; // array to store model's loss values during training
this.aAccu = []; // array to store model's accuracy values during training
// total number of training images in all classes
const TOTAL_TRAIN_IMAGES = this.NUM_CLASSES * this.NUM_TRAIN_IMAGES;
// total number of test images in all classes
const TOTAL_TEST_IMAGES = this.NUM_CLASSES * this.NUM_TEST_IMAGES;
// create Training Data arrays for storing training images and their corresponding classes
this.aTrainImages = new Float32Array(TOTAL_TRAIN_IMAGES * this.IMAGE_SIZE);
this.aTrainClasses = new Uint8Array(TOTAL_TRAIN_IMAGES);
// shuffle Training Data by creating an array of shuffled Train indices
this.aTrainIndices = tf.util.createShuffledIndices(TOTAL_TRAIN_IMAGES);
// the reference to the current element in the aTrainIndices[] array
this.trainElement = -1;
// create arrays of Test Data for storing test images and their corresponding classes
this.aTestImages = new Float32Array(TOTAL_TEST_IMAGES * this.IMAGE_SIZE);
this.aTestClasses = new Uint8Array(TOTAL_TEST_IMAGES);
// shuffle Test Data by creating an array of shuffled Test indices
this.aTestIndices = tf.util.createShuffledIndices(TOTAL_TEST_IMAGES);
// the reference to the current element in the aTestIndices[] array
this.testElement = -1;
// create a CNN model using a Sequential model type in which
// tensors are consecutively passed from one layer to the next
this.model = tf.sequential();
// add a convolutional layer
this.model.add(tf.layers.conv2d({
inputShape: [28, 28, 1],
kernelSize: 5,
filters: 8,
strides: 1,
activation: 'relu',
kernelInitializer: 'varianceScaling'
}));
// add a max pooling layer
this.model.add(tf.layers.maxPooling2d({
poolSize: [2, 2],
strides: [2, 2]
}));
// add a second convolutional layer
this.model.add(tf.layers.conv2d({
kernelSize: 5,
filters: 16,
strides: 1,
activation: 'relu',
kernelInitializer: 'varianceScaling'
}));
// add a second max pooling layer
this.model.add(tf.layers.maxPooling2d({
poolSize: [2, 2],
strides: [2, 2]
}));
// add a flatten layer to flatten the output of the previous layer to a vector
this.model.add(tf.layers.flatten());
// add a dense (fully connected) layer to perform the final classification
this.model.add(tf.layers.dense({
units: this.NUM_CLASSES,
kernelInitializer: 'varianceScaling',
activation: 'softmax'
}));
// compile the model
this.model.compile({
optimizer: tf.train.sgd(0.15), // optimizer with learning rate
loss: 'categoricalCrossentropy', // loss function
metrics: ['accuracy'], // evaluation metric
});
};
// ---------------------------------------------------------------------------------
// CNN Prototype
// ---------------------------------------------------------------------------------
/**
* Splits the entire dataset into training data and test data.
*
* @param {Uint8Array} imagesBuffer - the array with binary data of all images in the dataset
* @param {integer} dataset - the ordinal number of the dataset
*/
CNN.prototype.splitDataset = function(imagesBuffer, dataset){
// slice dataset to get training images and normalize them
var trainBuffer = new Float32Array(imagesBuffer.slice(0, this.IMAGE_SIZE * this.NUM_TRAIN_IMAGES));
trainBuffer = trainBuffer.map(function (cv){return cv/255.0});
// add training images and their corresponding classes into Training Data arrays
var start = dataset * this.NUM_TRAIN_IMAGES;
this.aTrainImages.set(trainBuffer, start * this.IMAGE_SIZE);
this.aTrainClasses.fill(dataset, start, start + this.NUM_TRAIN_IMAGES);
// slice dataset to get test images and normalize them
var testBuffer = new Float32Array(imagesBuffer.slice(this.IMAGE_SIZE * this.NUM_TRAIN_IMAGES));
testBuffer = testBuffer.map(function (cv){return cv/255.0});
// add test images and their corresponding classes into Test Data arrays
start = dataset * this.NUM_TEST_IMAGES;
this.aTestImages.set(testBuffer, start * this.IMAGE_SIZE);
this.aTestClasses.fill(dataset, start, start + this.NUM_TEST_IMAGES);
};
/**
* Trains the model
*/
CNN.prototype.train = async function(){
// reset the training flag to know the training is currently in progress
this.isTrainCompleted = false;
for (let i = 0; i < this.TRAIN_ITERATIONS; i++){
// increase the number of training iterations
this.trainIteration++;
this.main.ui.showStatusBar("Training the CNN - iteration " + this.trainIteration + ".");
// fetch the next Training Batch
let trainBatch = this.nextTrainBatch(this.TRAIN_BATCH_SIZE);
// create new Test Batch and Validation Set
let testBatch;
let validationSet;
if (i % this.TEST_FREQUENCY === 0){ // every few training passes...
// fetch the next Test Batch
testBatch = this.nextTestBatch(this.TEST_BATCH_SIZE);
// build Validation Set by using images and corresponding labels from Test Batch
validationSet = [testBatch.images, testBatch.labels];
}
// train the model
const training = await this.model.fit(
trainBatch.images,
trainBatch.labels,
{batchSize: this.TRAIN_BATCH_SIZE, validationData: validationSet, epochs: 1}
);
// get the model loss from the last training iteration and plot the Loss Chart
var maxLossLength = this.main.ui.bmpLossChart.width;
if (this.aLoss.length > maxLossLength) this.aLoss.shift();
this.aLoss.push(1 - Math.min(1, training.history.loss[0]));
this.main.ui.plotChart(this.main.ui.bmpLossChart, this.aLoss, 1);
if (testBatch != null) {
// get the model accuracy from the last training iteration and plot the Accuracy Chart
var maxAccuLength = this.main.ui.bmpAccuChart.width;
if (this.aAccu.length * this.TEST_FREQUENCY > maxAccuLength) this.aAccu.shift();
this.aAccu.push(1 - Math.min(1, training.history.acc[0]));
this.main.ui.plotChart(this.main.ui.bmpAccuChart, this.aAccu, this.TEST_FREQUENCY);
// dispose Test Batch from memory
testBatch.images.dispose();
testBatch.labels.dispose();
}
// dispose Training Batch from memory
trainBatch.images.dispose();
trainBatch.labels.dispose();
// mitigate blocking the UI thread and freezing the tab during training
await tf.nextFrame();
}
// set the training flag to know the training is completed
this.isTrainCompleted = true;
};
/**
* Predicts a batch of sample images fetched from Test Data
*/
CNN.prototype.predictSamples = async function(){
this.isSamplesPredicted = false;
const samplesBatch = this.nextTestBatch(App.NUM_SAMPLES);
tf.tidy(() => {
const output = this.model.predict(samplesBatch.images);
const aClassifications = Array.from(samplesBatch.labels.argMax(1).dataSync());
const aPredictions = Array.from(output.argMax(1).dataSync());
this.main.ui.showSamplePredictions(aClassifications, aPredictions);
});
tf.dispose(samplesBatch);
this.isSamplesPredicted = true;
};
/**
* Predicts a doodle drawing
*
* @param {Float32Array} aNormalizedPixels - array of normalized pixels representing a doodle drawing
*/
CNN.prototype.predictDoodle = async function(aNormalizedPixels){
const input = tf.tensor2d(aNormalizedPixels, [1, this.IMAGE_SIZE]);
tf.tidy(() => {
const output = this.model.predict(
input.reshape([1, 28, 28, 1])
);
const aPrediction = Array.from(output.argMax(1).dataSync());
this.main.ui.showDoodlePrediction(aPrediction);
});
tf.dispose(input);
};
/**
* Returns a batch of images and their corresponding classes from the Training Data
*
* @param {integer} batchSize - how many images are included in training batch
*/
CNN.prototype.nextTrainBatch = function(batchSize){
return this.fetchBatch(
batchSize, this.aTrainImages, this.aTrainClasses,
() => {
this.trainElement = (this.trainElement + 1) % this.aTrainIndices.length;
return this.aTrainIndices[this.trainElement];
}
);
};
/**
* Returns a batch of images and their corresponding classes from the Test Data
*
* @param {integer} batchSize - how many images are included in test batch
*/
CNN.prototype.nextTestBatch = function(batchSize){
return this.fetchBatch(
batchSize, this.aTestImages, this.aTestClasses,
() => {
this.testElement = (this.testElement + 1) % this.aTestIndices.length;
return this.aTestIndices[this.testElement];
}
);
};
/**
* Fetches a batch of images and their corresponding classes
*
* @param {integer} batchSize - how many images are included in the batch
* @param {Float32Array} aImages - array of images
* @param {Uint8Array} aClasses - array of corresponding classes
* @param {integer} getIndex - a function which returns the index of an image that must be fetched
*/
CNN.prototype.fetchBatch = function(batchSize, aImages, aClasses, getIndex){
// create batch arrays
const batchImages = new Float32Array(batchSize * this.IMAGE_SIZE);
const batchLabels = new Uint8Array(batchSize * this.NUM_CLASSES);
for (let i = 0; i < batchSize; i++){
// get the index of the image we want to fetch
const idx = getIndex();
// fetch the image
const image = aImages.slice(idx * this.IMAGE_SIZE, (idx + 1) * this.IMAGE_SIZE);
// add the image to the batch of images
batchImages.set(image, i * this.IMAGE_SIZE);
// get the class number of this image
const class_num = aClasses[idx];
// generate the label for this image by using "one hot encoding method":
// define a vector where all elements are 0, beside one element
// which points to the correct class of this image
const label = new Uint8Array(this.NUM_CLASSES);
label[class_num] = 1;
// add the label to the batch of labels
batchLabels.set(label, i * this.NUM_CLASSES);
}
// convert batch of images to a temporary tensor
const images_temp = tf.tensor2d(batchImages, [batchSize, this.IMAGE_SIZE]);
// reshape the temporary tensor to the size of the model input shape
const images = images_temp.reshape([batchSize, 28, 28, 1]);
// dispose the temporary tensor to free memory
images_temp.dispose();
// convert batch of labels to tensor
const labels = tf.tensor2d(batchLabels, [batchSize, this.NUM_CLASSES]);
return {images, labels};
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 7)
*
* MODULE:
* main.js - Main Program
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 7. Predicting Doodle
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
/***********************************************************************************
/* Setup procedure for creating a new Phaser Game object on window load event
/***********************************************************************************/
window.onload = function () {
// create a new game object which is an instance of Phaser.Game
var game = new Phaser.Game(1280, 720, Phaser.CANVAS);
// add all States to the game object (this program has only the Main State)
game.state.add('MainState', App.MainState);
// start the Main State
game.state.start('MainState');
};
/***********************************************************************************
/* The Application Namespace
/***********************************************************************************/
var App = App || {};
// ---------------------------------------------------------------------------------
// Global constants and variables
// ---------------------------------------------------------------------------------
// the names of all datasets
App.DATASETS = ['car', 'fish', 'snowman'];
// the number of test data samples intended for predicting by CNN model
App.NUM_SAMPLES = 16;
// ---------------------------------------------------------------------------------
// The Main State constructor
// ---------------------------------------------------------------------------------
App.MainState = function(){
// constants describing all modes of the main state
this.MODE_INIT = 1;
this.MODE_OPEN_FILE = 2;
this.MODE_LOAD_FILE = 3;
this.MODE_START_TRAIN = 4;
this.MODE_DO_TRAIN = 5;
this.MODE_START_PREDICT = 6;
this.MODE_DO_PREDICT = 7;
this.MODE_DRAW = 8;
this.MODE_CLICK_ON_TRAIN = 9;
this.MODE_CLICK_ON_TEST = 10;
this.MODE_CLICK_ON_CLEAR = 11;
// set initial mode
this.mode = this.MODE_INIT;
// the counter of currently loaded datasets
this.dataset = 0;
};
// ---------------------------------------------------------------------------------
// The Main State prototype
// ---------------------------------------------------------------------------------
App.MainState.prototype = {
/**
* Automatically called only once to load all assets.
*/
preload : function(){
this.game.load.image('imgBack', '../assets/img_back_7.png');
this.game.load.image('imgDisable', '../assets/img_disable.png');
this.game.load.image('btnTrain', '../assets/btn_train.png');
this.game.load.image('btnTest', '../assets/btn_test.png');
this.game.load.image('btnClear', '../assets/btn_clear.png');
this.game.load.image('btnMoreGames', '../assets/btn_moregames.png');
this.game.load.image('btnAuthor', '../assets/btn_author.png');
this.load.bitmapFont('fntBlackChars', '../assets/fnt_black_chars.png', '../assets/fnt_black_chars.fnt');
},
/**
* Automatically called immediately after all assets are loaded to create all objects.
*/
create : function(){
// scale game to cover the entire screen
this.scale.scaleMode = Phaser.ScaleManager.SHOW_ALL;
this.scale.pageAlignVertically = true;
this.scale.pageAlignHorizontally = true;
// keep game running if it loses the focus
this.game.stage.disableVisibilityChange = true;
// create background
this.game.add.sprite(0, 0, 'imgBack');
// create a loader for loading datasets
this.loader = new Phaser.Loader(this.game);
// create user interface with buttons, bitmaps and texts
this.ui = new UI(this);
// create a convolution neural network
this.cnn = new CNN(this);
// create a painter for drawing doodles
this.painter = new Painter(this);
},
/**
* Automatically called on every tick representing the main loop.
*/
update : function(){
switch(this.mode){
// initialize the game
case this.MODE_INIT:
this.painter.reset();
this.painter.disable();
this.ui.disableButtons();
this.mode = this.MODE_OPEN_FILE;
break;
// open dataset file and start loading it
case this.MODE_OPEN_FILE:
var fileName = App.DATASETS[this.dataset] + '.bin';
this.loader.reset();
this.loader.binary('input_file', '../data/'+fileName);
this.loader.start();
this.ui.showStatusBar("Loading " + fileName + " file.");
this.mode = this.MODE_LOAD_FILE;
break;
// wait on dataset file to be loaded
case this.MODE_LOAD_FILE:
if (this.loader.hasLoaded){
// split the loaded dataset into training data and test data
this.cnn.splitDataset(
new Uint8Array(this.game.cache.getBinary('input_file')),
this.dataset
);
// increase the number of loaded datasets
this.dataset++;
// if we have not loaded all datasets yet then go to load the next one
// else go to the CNN training
if (this.dataset < App.DATASETS.length){
this.mode = this.MODE_OPEN_FILE;
} else {
this.ui.showStatusBar("Initializing training.");
this.mode = this.MODE_START_TRAIN;
}
}
break;
// start with CNN training
case this.MODE_START_TRAIN:
this.painter.disable();
this.ui.disableButtons();
this.cnn.train();
this.mode = this.MODE_DO_TRAIN;
break;
// wait for completion of the CNN training
case this.MODE_DO_TRAIN:
if (this.cnn.isTrainCompleted){
this.ui.showStatusBar("Training completed. Predicting samples...");
this.mode = this.MODE_START_PREDICT;
}
break;
// draw sample images and start with predicting them by using CNN
case this.MODE_START_PREDICT:
this.ui.drawSampleImages();
this.cnn.predictSamples();
this.mode = this.MODE_DO_PREDICT;
break;
// wait for CNN to make predictions for all samples
case this.MODE_DO_PREDICT:
if (this.cnn.isSamplesPredicted){
this.painter.enable();
this.ui.enableButtons();
this.ui.showStatusBar(
"Draw " + App.DATASETS.join(", ") +
" to recognize your drawing!"
);
this.mode = this.MODE_DRAW;
}
break;
// draw a doodle and recognize it by using CNN
case this.MODE_DRAW:
if (this.game.input.mousePointer.isDown){
this.painter.draw(this.game.input.x, this.game.input.y);
} else {
this.painter.recognize();
}
break;
// actions on clicking "Train More" button
case this.MODE_CLICK_ON_TRAIN:
this.mode = this.MODE_START_TRAIN;
break;
// actions on clicking "Next Test" button
case this.MODE_CLICK_ON_TEST:
this.mode = this.MODE_START_PREDICT;
break;
// actions on clicking "Clear" button
case this.MODE_CLICK_ON_CLEAR:
this.painter.reset();
this.ui.txtDoodlePrediction.setText("");
this.mode = this.MODE_DRAW;
break;
}
}
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 7)
*
* MODULE:
* painter.js - Painter Class
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 7. Predicting Doodle
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
// ---------------------------------------------------------------------------------
// Painter Constructor
// ---------------------------------------------------------------------------------
var Painter = function(main){
// reference to the Main State
this.main = main;
// reference to the Phaser Game object
var game = this.main.game;
// define a drawing area
this.DRAWING_AREA = new Phaser.Rectangle(842, 95, 416, 416);
// create a bitmap of the same size as the drawing area to draw a doodle on it
this.bmpOrigDrawing = game.make.bitmapData(this.DRAWING_AREA.width+2, this.DRAWING_AREA.height+2);
this.bmpOrigDrawing.addToWorld(this.DRAWING_AREA.x-1, this.DRAWING_AREA.y-1);
// create a bitmap of the cropped drawing where empty borders are cut away
this.bmpCropDrawing = game.make.bitmapData(this.DRAWING_AREA.width, this.DRAWING_AREA.height);
// create a bitmap of the cropped drawing downsampled to 104x104 pixels
this.bmpDownSample1 = game.make.bitmapData(104, 104);
// create a bitmap of the cropped drawing downsampled to 52x52 pixels
this.bmpDownSample2 = game.make.bitmapData(52, 52);
// create a bitmap of the final drawing downsampled to 28x28 pixels which is applicable to fed the CNN
this.bmpFinalDrawing = game.make.bitmapData(28, 28);
// create a sprite that visualizes that drawing a doodle is disabled
this.sprDisableEffect = game.add.sprite(this.DRAWING_AREA.x-1, this.DRAWING_AREA.y-1, 'imgDisable');
this.sprDisableEffect.width = this.bmpOrigDrawing.width;
this.sprDisableEffect.height = this.bmpOrigDrawing.height;
};
// ---------------------------------------------------------------------------------
// Painter Prototype
// ---------------------------------------------------------------------------------
/**
* Resets Painter parameters.
*/
Painter.prototype.reset = function(){
// reset the flag to know if something new is drawn
this.isDrawing = false;
// reset pencil position and size
this.pencil = {x:0, y:0, prevX:0, prevY:0, left:0, top:0, right:0, bottom:0};
// reset the cropping area used to cut empty borders away from the original drawing
this.cropArea = {left:2000, top:2000, right:-2000, bottom:-2000, width:0, height:0, tx:0, ty:0};
// clear bitmaps by filling them with white color
this.bmpOrigDrawing.fill(255, 255, 255);
this.bmpCropDrawing.fill(255, 255, 255);
this.bmpFinalDrawing.fill(255, 255, 255);
};
/**
* Hides the sprite with disabling effect over the drawing area.
*/
Painter.prototype.enable = function(){
this.sprDisableEffect.kill();
};
/**
* Shows the sprite with disabling effect over the drawing area.
*/
Painter.prototype.disable = function(){
this.sprDisableEffect.revive();
};
/**
* Draws a quadratic curve from the previous point to the current point
*
* @param {number} x - the X coordinate of the mouse
* @param {number} y - the Y coordinate of the mouse
*/
Painter.prototype.draw = function(x, y){
if (this.DRAWING_AREA.contains(x, y)){ // if mouse is inside the drawing area...
// calculate pencil position and size
this.pencil.prevX = this.pencil.x;
this.pencil.prevY = this.pencil.y;
this.pencil.x = x - this.DRAWING_AREA.x;
this.pencil.y = y - this.DRAWING_AREA.y;
this.pencil.left = this.pencil.x - 5;
this.pencil.top = this.pencil.y - 5;
this.pencil.right = this.pencil.x + 5;
this.pencil.bottom = this.pencil.y + 5;
// draw a circle at the current pencil position
this.bmpOrigDrawing.circle(this.pencil.x, this.pencil.y, 4, '#000');
if (!this.isDrawing){
// if this is the first drawing point, don't draw anything more,
// just set the flag to know that drawing has been started
this.isDrawing = true;
} else {
// else draw a quadratic curve from the previous to the current point
var xc = (this.pencil.prevX + this.pencil.x) / 2;
var yc = (this.pencil.prevY + this.pencil.y) / 2;
var ctx = this.bmpOrigDrawing.context;
ctx.beginPath();
ctx.quadraticCurveTo(this.pencil.prevX, this.pencil.prevY, xc, yc);
ctx.quadraticCurveTo(xc, yc, this.pencil.x, this.pencil.y);
ctx.lineWidth = 9;
ctx.strokeStyle = '#000';
ctx.stroke();
ctx.closePath();
}
// recalculate the cropping area by finding the edge points of the original drawing
if (this.pencil.left < this.cropArea.left){
this.cropArea.left = this.pencil.left;
if (this.cropArea.left < 0) this.cropArea.left = 0;
}
if (this.pencil.right > this.cropArea.right){
this.cropArea.right = this.pencil.right;
if (this.cropArea.right > this.DRAWING_AREA.width) this.cropArea.right = this.DRAWING_AREA.width;
}
if (this.pencil.top < this.cropArea.top){
this.cropArea.top = this.pencil.top;
if (this.cropArea.top < 0) this.cropArea.top = 0;
}
if (this.pencil.bottom > this.cropArea.bottom){
this.cropArea.bottom = this.pencil.bottom;
if (this.cropArea.bottom > this.DRAWING_AREA.height) this.cropArea.bottom = this.DRAWING_AREA.height;
}
this.cropArea.width = this.cropArea.right - this.cropArea.left;
this.cropArea.height = this.cropArea.bottom - this.cropArea.top;
this.cropArea.tx = 0;
this.cropArea.ty = 0;
if (this.cropArea.width > this.cropArea.height){
this.cropArea.ty = (this.cropArea.width - this.cropArea.height)/2;
}
if (this.cropArea.width < this.cropArea.height){
this.cropArea.tx = (this.cropArea.height - this.cropArea.width)/2;
}
// resize the original drawing
this.resizeDrawing();
} else { // else if mouse is outside of the drawing area...
// start recognizing the doodle drawing
this.recognize();
}
};
/**
* Resizes the original drawing to the size of 28x28 pixels.
*/
Painter.prototype.resizeDrawing = function(){
// resize the bitmap of the cropped drawing to the cropping area size
this.bmpCropDrawing.resize(
this.cropArea.width + this.cropArea.tx * 2,
this.cropArea.height + this.cropArea.ty * 2
);
// fill the bitmap of the cropped drawing with the white color
this.bmpCropDrawing.fill(255, 255, 255);
// copy the original drawing without empty borders to the cropped drawing using the cropping area
this.bmpCropDrawing.copy(
this.bmpOrigDrawing,
this.cropArea.left, this.cropArea.top,
this.cropArea.width, this.cropArea.height,
this.cropArea.tx, this.cropArea.ty
);
// gradually downsample the cropped drawing to the size of 28x28 pixels as follows:
// step 1: downsample the cropped drawing to 104x104
this.bmpDownSample1.copy(this.bmpCropDrawing, null, null, null, null, 0, 0, 104, 104);
// step 2: downsample the drawing from 104x104 to 52x52
this.bmpDownSample2.copy(this.bmpDownSample1, null, null, null, null, 0, 0, 52, 52);
// step 3: downsample the drawing from 52x52 to 26x26 and translate it
// for 1px right and 1px down to put it in the center of the final 28x28 bitmap
this.bmpFinalDrawing.copy(this.bmpDownSample2, null, null, null, null, 1, 1, 26, 26);
};
/**
* Initiates the drawing recognition by normalizing input array of pixels.
*/
Painter.prototype.recognize = function(){
if (this.isDrawing){ // only if something new has been drawn...
// update the final 28x28 bitmap
this.bmpFinalDrawing.update();
// set all pixels from the final 28x28 bitmap to a local Float32 Array
// because this is a grayscale drawing, we are mapping only one color component of each pixel
var aPixels = Float32Array.from(
this.bmpFinalDrawing.pixels.map(function (cv){return cv & 255;})
);
// normalize pixels to be in the range between 0.0 and 1.0
var aNormalizedPixels = aPixels.map(function (cv){return (255-cv)/255.0;});
// use CNN to predict the doodle drawing
this.main.cnn.predictDoodle(aNormalizedPixels);
// reset the drawing flag so we know there is no need
// for the next recognition until something new is drawn
this.isDrawing = false;
}
};
/***********************************************************************************
*
* PROGRAM:
* Doodle Predictor (Part 7)
*
* MODULE:
* ui.js - UI Class (User Interface)
*
* EXTERNAL LIBRARIES:
* phaser.min.js - Phaser 2 Framework
* tensorflow.js - TensorFlow Library
*
* DESCRIPTION:
* Recognizing Doodles using Deep Machine Learning with Convolutional Neural Network
*
* PART:
* 7. Predicting Doodle
*
* LINKS
* @website https://www.askforgametask.com
* @videos https://www.youtube.com/ssusnic
* @repos https://www.github.com/ssusnic
*
* ABOUT:
* @author Srdjan Susnic
* @copyright 2019 Ask For Game Task
*
* This program comes with ABSOLUTELY NO WARRANTY.
*
/***********************************************************************************/
// ---------------------------------------------------------------------------------
// UI Constructor
// ---------------------------------------------------------------------------------
var UI = function(main){
// reference to the Main State
this.main = main;
// reference to the Phaser Game object
var game = this.main.game;
// create buttons
this.btnTrain = game.add.button(460, 625, 'btnTrain', this.onTrainClick, this);
this.btnTest = game.add.button(652, 625, 'btnTest', this.onTestClick, this);
this.btnClear = game.add.button(842, 625, 'btnClear', this.onClearClick, this);
this.btnMoreGames = game.add.button(1048, 625, 'btnMoreGames', this.onMoreGamesClick, this);
this.btnAuthor = game.add.button(1130, 703, 'btnAuthor', this.onMoreGamesClick, this);
this.btnAuthor.anchor.setTo(0.5);
// create a bitmap for plotting CNN loss trend during training
this.bmpAccuChart = game.add.bitmapData(350, 250);
this.bmpAccuChart.addToWorld(45, 95);
// create a bitmap for plotting CNN accuracy trend during training
this.bmpLossChart = game.add.bitmapData(350, 250);
this.bmpLossChart.addToWorld(45, 410);
// create a bitmap for showing all sample images intended for predicting by CNN
this.bmpSampleImages = game.add.bitmapData(28, (28+4) * App.NUM_SAMPLES);
this.bmpSampleImages.addToWorld(470, 95);
// create a bitmap for drawing green/red rectangles as results of CNN predictions on sample images
this.bmpSampleResults = game.add.bitmapData(125, (28+4) * App.NUM_SAMPLES);
this.bmpSampleResults.addToWorld(665, 95);
// create text objects
this.txtSampleClasses = []; // array of texts describing correct classes of samples
this.txtSamplePredictions = []; // array of texts describing predictions for samples
for (var i=0; i<App.NUM_SAMPLES; i++){
var y = 100 + i*32;
this.txtSampleClasses.push(
game.add.bitmapText(550, y, "fntBlackChars", "", 18)
);
this.txtSamplePredictions.push(
game.add.bitmapText(670, y, "fntBlackChars", "", 18)
);
}
// create a text which shows messages in the status bar
this.txtStatBar = game.add.bitmapText(10, 695, "fntBlackChars", "", 18);
// create a text which shows CNN prediction for the doodle drawing
this.txtDoodlePrediction = game.add.bitmapText(1050, 572, "fntBlackChars", "", 36);
this.txtDoodlePrediction.anchor.setTo(0.5);
};
// ---------------------------------------------------------------------------------
// UI Prototype
// ---------------------------------------------------------------------------------
/**
* Disables buttons.
*/
UI.prototype.disableButtons = function(){
this.btnTrain.kill();
this.btnTest.kill();
this.btnClear.kill();
};
/**
* Enables buttons.
*/
UI.prototype.enableButtons = function(){
this.btnTrain.revive();
this.btnTest.revive();
this.btnClear.revive();
};
/**
* Triggers on pressing "Train More" button.
*/
UI.prototype.onTrainClick = function(){
if (this.main.mode == this.main.MODE_DRAW){
this.main.mode = this.main.MODE_CLICK_ON_TRAIN;
}
};
/**
* Triggers on pressing "Next Test" button.
*/
UI.prototype.onTestClick = function(){
if (this.main.mode == this.main.MODE_DRAW){
this.main.mode = this.main.MODE_CLICK_ON_TEST;
}
};
/**
* Triggers on pressing "Clear" button.
*/
UI.prototype.onClearClick = function(){
if (this.main.mode == this.main.MODE_DRAW){
this.main.mode = this.main.MODE_CLICK_ON_CLEAR;
}
};
/**
* Opens Official Web Site when click on "Play More Games" button.
*/
UI.prototype.onMoreGamesClick = function(){
window.open("http://www.askforgametask.com", "_blank");
};
/**
* Plots a line chart to show a trend over time.
*
* @param {Phaser.BitmapData} bmpChart - the bitmap used to draw lines between points on it
* @param {float Array} aValues - the array of trend values on the Y-axis
* @param {integer} dx - the space between two points on the X-axis
*/
UI.prototype.plotChart = function(bmpChart, aValues, dx){
bmpChart.clear();
for (var i = 1; i < aValues.length; i++){
var x1 = (i-1) * dx;
var y1 = bmpChart.height * aValues[i-1];
var x2 = i * dx;
var y2 = bmpChart.height * aValues[i];
bmpChart.line(x1, y1, x2, y2, '#61bc6d', 2);
}
};
/**
* Draws all sample images intended for CNN prediction.
*/
UI.prototype.drawSampleImages = function(){
// clear the bitmap used for drawing samples on it
this.bmpSampleImages.clear();
// get the reference to the first sample
var sample = this.main.cnn.testElement;
// for all samples...
for (var n = 0; n < App.NUM_SAMPLES; n++){
// get the reference to the next sample
sample = (sample + 1) % this.main.cnn.aTestIndices.length;
// get the starting position of the first pixel of this sample
var index = this.main.cnn.aTestIndices[sample];
var start = index * this.main.cnn.IMAGE_SIZE;
// for all pixels of this sample...
for (var i = 0; i < this.main.cnn.IMAGE_SIZE; i++){
// get the color of the current pixel
var pixel = this.main.cnn.aTestImages[i + start];
var color = 255 - ((pixel * 255)>>0) & 0xFF;
// calculate XY position of this pixel on the bitmap
var x = i%28;
var y = (i/28)>>0;
// set this pixel on the bitmap
this.bmpSampleImages.setPixel32(x, y + n*32, color, color, color, 255, false);
}
}
// put the image data of the bitmap on the context to show all samples
this.bmpSampleImages.context.putImageData(this.bmpSampleImages.imageData, 0, 0);
};
/**
* Shows correct classifications for all sample images
* with their predictions displayed inside green/red rectangle.
*
* @param {integer Array} aClassifications
* - the array of index numbers pointing to the correct classes of samples
*
* @param {integer Array} aPredictions
* - the array of index numbers pointing to the predicted classes of samples
*/
UI.prototype.showSamplePredictions = function(aClassifications, aPredictions){
this.bmpSampleResults.clear();
for (var i=0; i<aClassifications.length; i++){
// set the text of correct class of this sample
this.txtSampleClasses[i].text = App.DATASETS[aClassifications[i]];
// set the text of predicted class of this sample
this.txtSamplePredictions[i].text = App.DATASETS[aPredictions[i]];
// if (classification = prediction) then draw green rectangle else draw red rectangle
var color = (this.txtSampleClasses[i].text === this.txtSamplePredictions[i].text) ? '#61bc6d' : '#e24939';
this.bmpSampleResults.rect(0, 2 + i*32, this.bmpSampleResults.width, 24, color);
}
};
/**
* Shows a prediction for the doodle drawing.
*
* @param {integer Array} aPredictions
* - the array with only one element pointing to the predicted class of the doodle drawing
*/
UI.prototype.showDoodlePrediction = function(aPredictions){
this.txtDoodlePrediction.text = "It's " + App.DATASETS[aPredictions[0]] + ".";
};
/**
* Shows a message in the status bar.
*
* @param {String} strText - the text to be shown in the status bar
*/
UI.prototype.showStatusBar = function(strText){
this.txtStatBar.text = strText;
};