Merge pull request rasmusbergpalm#29 from rasmusbergpalm/pr/28

Cleaned up pull request from @skaae. Thanks!

Author: rasmusbergpalm

NN/nneval.m

@@ -0,0 +1,27 @@
+function [loss] = nneval(nn, loss, train_x, train_y, val_x, val_y)
+%NNEVAL evaluates performance of neural network
+% Returns a updated loss struct
+assert(nargin == 4 || nargin == 6, 'Wrong number of arguments');
+
+% training performance
+nn                    = nnff(nn, train_x, train_y);
+loss.train.e(end + 1) = nn.L;
+
+% validation performance
+if nargin == 6
+    nn                    = nnff(nn, val_x, val_y);
+    loss.val.e(end + 1)   = nn.L;
+end
+
+%calc misclassification rate if softmax
+if strcmp(nn.output,'softmax')
+    [er_train, ~]               = nntest(nn, train_x, train_y);
+    loss.train.e_frac(end+1)    = er_train;
+    
+    if nargin == 6
+        [er_val, ~]             = nntest(nn, val_x, val_y);
+        loss.val.e_frac(end+1)  = er_val;
+    end
+end
+
+end

NN/nnpredict.m

@@ -1,8 +1,4 @@
 function labels = nnpredict(nn, x)
-    if nn.normalize_input==1;
-       x = (x-repmat(nn.normalizeMean,size(x,1),1))./repmat(nn.normalizeStd,size(x,1),1);
-    end
-    
     nn.testing = 1;
     nn = nnff(nn, x, zeros(size(x,1), nn.size(end)));
     nn.testing = 0;

NN/nnsetup.m

@@ -6,7 +6,6 @@
     nn.size   = architecture;
     nn.n      = numel(nn.size);
 
-    nn.normalize_input                  = 1;            %  normalize input elements to be between [-1 1]. Note: use a linear output function if training auto-encoders with normalized inputs
     nn.activation_function              = 'tanh_opt';   %  Activation functions of hidden layers: 'sigm' (sigmoid) or 'tanh_opt' (optimal tanh).
     nn.learningRate                     = 2;            %  learning rate Note: typically needs to be lower when using 'sigm' activation function and non-normalized inputs.
     nn.momentum                         = 0.5;          %  Momentum

NN/nntrain.m

@@ -1,57 +1,75 @@
-function [nn, L] = nntrain(nn, x, y, opts)
+function [nn, L]  = nntrain(nn, train_x, train_y, opts, val_x, val_y)
 %NNTRAIN trains a neural net
-% [nn, L] = nnff(nn, x, y, opts) trains the neural network nn with input x and 
+% [nn, L] = nnff(nn, x, y, opts) trains the neural network nn with input x and
 % output y for opts.numepochs epochs, with minibatches of size
 % opts.batchsize. Returns a neural network nn with updated activations,
-% errors, weights and biases, (nn.a, nn.e, nn.W, nn.b) and L, the sum 
+% errors, weights and biases, (nn.a, nn.e, nn.W, nn.b) and L, the sum
 % squared error for each training minibatch.
 
-    assert(isfloat(x), 'x must be a float');
-    m = size(x, 1);
-    
-    if nn.normalize_input==1
-        [x, mu, sigma] = zscore(x);
-        nn.normalizeMean = mu;
-        sigma(sigma==0) = 0.0001;%this should be very small value. 
-        nn.normalizeStd  = sigma;
-    end
+assert(isfloat(train_x), 'train_x must be a float');
+assert(nargin == 4 || nargin == 6,'number ofinput arguments must be 4 or 6')
+
+loss.train.e               = [];
+loss.train.e_frac          = [];
+loss.val.e                 = [];
+loss.val.e_frac            = [];
+opts.validation = 0;
+if nargin == 6
+    opts.validation = 1;
+end
+
+fhandle = [];
+if isfield(opts,'plot') && opts.plot == 1
+    fhandle = figure();
+end
+
+m = size(train_x, 1);
+
+batchsize = opts.batchsize;
+numepochs = opts.numepochs;
+
+numbatches = m / batchsize;
+
+assert(rem(numbatches, 1) == 0, 'numbatches must be a integer');
+
+L = zeros(numepochs*numbatches,1);
+n = 1;
+for i = 1 : numepochs
+    tic;
 
-    batchsize = opts.batchsize;
-    numepochs = opts.numepochs;
-
-    numbatches = m / batchsize;
-
-    assert(rem(numbatches, 1) == 0, 'numbatches must be a integer');
-
-    L = zeros(numepochs*numbatches,1);
-    n = 1;
-    for i = 1 : numepochs
-        tic;
-
-        kk = randperm(m);
-        for l = 1 : numbatches
-            batch_x = x(kk((l - 1) * batchsize + 1 : l * batchsize), :);
-            
-            %Add noise to input (for use in denoising autoencoder)
-            if(nn.inputZeroMaskedFraction ~= 0)
-                batch_x = batch_x.*(rand(size(batch_x))>nn.inputZeroMaskedFraction);
-            end
-            
-            batch_y = y(kk((l - 1) * batchsize + 1 : l * batchsize), :);
-            
-            nn = nnff(nn, batch_x, batch_y);
-            nn = nnbp(nn);            
-            nn = nnapplygrads(nn);
-
-            L(n) = nn.L;
-            
-            n = n + 1;
+    kk = randperm(m);
+    for l = 1 : numbatches
+        batch_x = train_x(kk((l - 1) * batchsize + 1 : l * batchsize), :);
+        
+        %Add noise to input (for use in denoising autoencoder)
+        if(nn.inputZeroMaskedFraction ~= 0)
+            batch_x = batch_x.*(rand(size(batch_x))>nn.inputZeroMaskedFraction);
         end
-
-        t = toc;
 
-        disp(['epoch ' num2str(i) '/' num2str(opts.numepochs) '. Took ' num2str(t) ' seconds' '. Mean squared error on training set is ' num2str(mean(L((n-numbatches):(n-1))))]);
+        batch_y = train_y(kk((l - 1) * batchsize + 1 : l * batchsize), :);
+        
+        nn = nnff(nn, batch_x, batch_y);
+        nn = nnbp(nn);
+        nn = nnapplygrads(nn);
 
+        L(n) = nn.L;
+        
+        n = n + 1;
     end
+    
+    t = toc;
+    
+    if ishandle(fhandle)
+        if opts.validation == 1
+            loss = nneval(nn, loss, train_x, train_y, val_x, val_y);
+        else
+            loss = nneval(nn, loss, train_x, train_y);
+        end
+        nnupdatefigures(nn, fhandle, loss, opts, i);
+    end
+        
+    disp(['epoch ' num2str(i) '/' num2str(opts.numepochs) '. Took ' num2str(t) ' seconds' '. Mean squared error on training set is ' num2str(mean(L((n-numbatches):(n-1))))]);
+    
+end
 end
 

NN/nnupdatefigures.m

@@ -0,0 +1,82 @@
+function nnupdatefigures(nn,fhandle,L,opts,i)
+%NNUPDATEFIGURES updates figures during training
+if i > 1 %dont plot first point, its only a point   
+    x_ax = 1:i;
+    % create legend
+    if opts.validation == 1
+        M            = {'Training','Validation'};
+    else
+        M            = {'Training'};
+    end
+    
+    %create data for plots
+    if strcmp(nn.output,'softmax')
+        plot_x       = x_ax';
+        plot_ye      = L.train.e';
+        plot_yfrac   = L.train.e_frac';
+        
+    else
+        plot_x       = x_ax';
+        plot_ye      = L.train.e';
+    end
+    
+    %add error on validation data if present
+    if opts.validation == 1
+        plot_x       = [plot_x, x_ax'];
+        plot_ye      = [plot_ye,L.val.e'];
+    end
+    
+    
+    %add classification error on validation data if present
+    if opts.validation == 1 && strcmp(nn.output,'softmax')
+        plot_yfrac   = [plot_yfrac, L.val.e_frac'];        
+    end
+    
+%    plotting
+    figure(fhandle);   
+    if strcmp(nn.output,'softmax')  %also plot classification error
+                
+        p1 = subplot(1,2,1);
+        plot(plot_x,plot_ye);
+        xlabel('Number of epochs'); ylabel('Error');title('Error');
+        title('Error')
+        legend(p1, M,'Location','NorthEast');
+        set(p1, 'Xlim',[0,opts.numepochs + 1])
+        
+        if i ==2 % speeds up plotting by factor of ~2
+            set(gca,'LegendColorbarListeners',[]);
+            setappdata(gca,'LegendColorbarManualSpace',1);
+            setappdata(gca,'LegendColorbarReclaimSpace',1);
+        end
+               
+        p2 = subplot(1,2,2);
+        plot(plot_x,plot_yfrac);
+        xlabel('Number of epochs'); ylabel('Misclassification rate');
+        title('Misclassification rate')
+        legend(p2, M,'Location','NorthEast');
+        set(p2, 'Xlim',[0,opts.numepochs + 1])
+        
+        if i ==2 % speeds up plotting by factor of ~2
+            set(gca,'LegendColorbarListeners',[]);
+            setappdata(gca,'LegendColorbarManualSpace',1);
+            setappdata(gca,'LegendColorbarReclaimSpace',1);
+        end
+
+    else
+        
+        p = plot(plot_x,plot_ye);
+        xlabel('Number of epochs'); ylabel('Error');title('Error');
+        legend(p, M,'Location','NorthEast');
+        set(gca, 'Xlim',[0,opts.numepochs + 1])
+        
+        if i ==2 % speeds up plotting by factor of ~2
+            set(gca,'LegendColorbarListeners',[]);
+            setappdata(gca,'LegendColorbarManualSpace',1);
+            setappdata(gca,'LegendColorbarReclaimSpace',1);
+            
+        end
+        
+    end
+    drawnow;
+end
+end

README.md

@@ -92,7 +92,6 @@ dbn = dbntrain(dbn, train_x, opts);
 
 %unfold dbn to nn
 nn = dbnunfoldtonn(dbn, 10);
-nn.normalize_input = 0;
 nn.activation_function = 'sigm';
 
 %train nn
@@ -122,7 +121,6 @@ test_y  = double(test_y);
 %  Setup and train a stacked denoising autoencoder (SDAE)
 rng(0);
 sae = saesetup([784 100]);
-sae.ae{1}.normalize_input           = 0;
 sae.ae{1}.activation_function       = 'sigm';
 sae.ae{1}.learningRate              = 1;
 sae.ae{1}.inputZeroMaskedFraction   = 0.5;
@@ -133,7 +131,6 @@ visualize(sae.ae{1}.W{1}(:,2:end)')
 
 % Use the SDAE to initialize a FFNN
 nn = nnsetup([784 100 10]);
-nn.normalize_input                  = 0;
 nn.activation_function              = 'sigm';
 nn.learningRate                     = 1;
 nn.W{1} = sae.ae{1}.W{1};
@@ -149,12 +146,10 @@ assert(er < 0.16, 'Too big error');
 %  Setup and train a stacked denoising autoencoder (SDAE)
 rng(0);
 sae = saesetup([784 100 100]);
-sae.ae{1}.normalize_input           = 0;
 sae.ae{1}.activation_function       = 'sigm';
 sae.ae{1}.learningRate              = 1;
 sae.ae{1}.inputZeroMaskedFraction   = 0.5;
 
-sae.ae{2}.normalize_input           = 0;
 sae.ae{2}.activation_function       = 'sigm';
 sae.ae{2}.learningRate              = 1;
 sae.ae{2}.inputZeroMaskedFraction   = 0.5;
@@ -166,7 +161,6 @@ visualize(sae.ae{1}.W{1}(:,2:end)')
 
 % Use the SDAE to initialize a FFNN
 nn = nnsetup([784 100 100 10]);
-nn.normalize_input                  = 0;
 nn.activation_function              = 'sigm';
 nn.learningRate                     = 1;
 
@@ -237,6 +231,10 @@ test_x  = double(test_x)  / 255;
 train_y = double(train_y);
 test_y  = double(test_y);
 
+% normalize
+[train_x, mu, sigma] = zscore(train_x);
+test_x = normalize(test_x, mu, sigma);
+
 %% ex1 vanilla neural net
 rng(0);
 nn = nnsetup([784 100 10]);
@@ -283,18 +281,48 @@ nn = nntrain(nn, train_x, train_y, opts);
 [er, bad] = nntest(nn, test_x, test_y);
 assert(er < 0.1, 'Too big error');
 
-%% ex4 neural net with sigmoid activation function, and without normalizing inputs
+%% ex4 neural net with sigmoid activation function
 rng(0);
 nn = nnsetup([784 100 10]);
 
 nn.activation_function = 'sigm';    %  Sigmoid activation function
-nn.normalize_input = 0;             %  Don't normalize inputs
-nn.learningRate = 1;                %  Sigm and non-normalized inputs require a lower learning rate
+nn.learningRate = 1;                %  Sigm require a lower learning rate
 opts.numepochs =  1;                %  Number of full sweeps through data
 opts.batchsize = 100;               %  Take a mean gradient step over this many samples
 
 nn = nntrain(nn, train_x, train_y, opts);
 
+[er, bad] = nntest(nn, test_x, test_y);
+assert(er < 0.1, 'Too big error');
+
+%% ex5 plotting functionality
+rng(0);
+nn = nnsetup([784 20 10]);
+opts.numepochs         = 5;            %  Number of full sweeps through data
+nn.output              = 'softmax';    %  use softmax output
+opts.batchsize         = 1000;         %  Take a mean gradient step over this many samples
+opts.plot              = 1;            %  enable plotting
+
+nn = nntrain(nn, train_x, train_y, opts);
+
+[er, bad] = nntest(nn, test_x, test_y);
+assert(er < 0.1, 'Too big error');
+
+%% ex6 neural net with sigmoid activation and plotting of validation and training error
+% split training data into training and validation data
+vx   = train_x(1:10000,:);
+tx = train_x(10001:end,:);
+vy   = train_y(1:10000,:);
+ty = train_y(10001:end,:);
+
+rng(0);
+nn                      = nnsetup([784 20 10]);     
+nn.output               = 'softmax';                   %  use softmax output
+opts.numepochs          = 5;                           %  Number of full sweeps through data
+opts.batchsize          = 1000;                        %  Take a mean gradient step over this many samples
+opts.plot               = 1;                           %  enable plotting
+nn = nntrain(nn, tx, ty, opts, vx, vy);                %  nntrain takes validation set as last two arguments (optionally)
+
 [er, bad] = nntest(nn, test_x, test_y);
 assert(er < 0.1, 'Too big error');
 ```

tests/test_example_DBN.m

@@ -30,7 +30,6 @@
 
 %unfold dbn to nn
 nn = dbnunfoldtonn(dbn, 10);
-nn.normalize_input = 0;
 nn.activation_function = 'sigm';
 
 %train nn