merged upstream/master

Author: albertoandreottiATgmail

.travis.yml

@@ -0,0 +1,8 @@
+before_script:
+  - sudo apt-get install octave
+
+script:
+  - sh -c "octave tests/runalltests.m"
+
+notifications:
+  email: false

CAE/caeexamples.m

@@ -1,6 +1,6 @@
 %%  mnist data
 clear all; close all; clc;
-load ../data/mnist_uint8;
+load mnist_uint8;
 x = cell(100, 1);
 N = 600;
 for i = 1 : 100
@@ -29,4 +29,4 @@
     mm = cae.ok{1}{i}(1,:,:); 
     ff(i,:) = mm(:); 
 end; 
-figure;visualize(ff',1)
+figure;visualize(ff')

CNN/cnnsetup.m

@@ -1,4 +1,5 @@
 function net = cnnsetup(net, x, y)
+    assert(exist('OCTAVE_VERSION')==0, 'CNNs does not work with Octave as there is a bug in the implementation of convolution in octave. See: http://savannah.gnu.org/bugs/?39314');
     inputmaps = 1;
     mapsize = size(squeeze(x(:, :, 1)));
 

CONTRIBUTING.md

@@ -0,0 +1,20 @@
+Thank you so much for wanting to give back to the toolbox. Here's some info on how to contribute:
+
+#General:
+ 
+Don't bunch up changes, e.g. if you have bug-fixes, new features and style changes, rather make 3 seperate pull requests.
+
+Ensure that you introduce tests/examples for any new functionality
+
+# Guide
+1. Fork repository
+2. Create a new branch, e.g. `checkout -b my-stuff`
+3. Commit and push your changes to that branch
+4. Make sure that the test works (!) (see known errors)
+5. Create a pull request
+6. I accept your pull request
+
+
+
+
+

DBN/rbmtrain.m

@@ -1,5 +1,6 @@
 function rbm = rbmtrain(rbm, x, opts)
     assert(isfloat(x), 'x must be a float');
+    assert(all(x>=0) && all(x<=1), "all data in x must be in [0:1]");
     m = size(x, 1);
     numbatches = m / opts.batchsize;
 
@@ -14,7 +15,7 @@
             v1 = batch;
             h1 = sigmrnd(repmat(rbm.c', opts.batchsize, 1) + v1 * rbm.W');
             v2 = sigmrnd(repmat(rbm.b', opts.batchsize, 1) + h1 * rbm.W);
-            h2 = sigmrnd(repmat(rbm.c', opts.batchsize, 1) + v2 * rbm.W');
+            h2 = sigm(repmat(rbm.c', opts.batchsize, 1) + v2 * rbm.W');
 
             c1 = h1' * v1;
             c2 = h2' * v2;

NN/nnapplygrads.m

@@ -5,7 +5,7 @@
 
     for i = 1 : (nn.n - 1)
         if(nn.weightPenaltyL2>0)
-            dW = nn.dW{i} + nn.weightPenaltyL2 * nn.W{i};
+            dW = nn.dW{i} + nn.weightPenaltyL2 * [zeros(size(nn.W{i},1),1) nn.W{i}(:,2:end)];
         else
             dW = nn.dW{i};
         end

NN/nnchecknumgrad.m

@@ -8,9 +8,9 @@ function nnchecknumgrad(nn, x, y)
                 nn_m = nn; nn_p = nn;
                 nn_m.W{l}(i, j) = nn.W{l}(i, j) - epsilon;
                 nn_p.W{l}(i, j) = nn.W{l}(i, j) + epsilon;
-                rng(0);
+				rand('state',0)
                 nn_m = nnff(nn_m, x, y);
-                rng(0);
+                rand('state',0)
                 nn_p = nnff(nn_p, x, y);
                 dW = (nn_p.L - nn_m.L) / (2 * epsilon);
                 e = abs(dW - nn.dW{l}(i, j));

NN/nntrain.m

@@ -58,17 +58,19 @@
     end
 
     t = toc;
-    
+
+    if opts.validation == 1
+        loss = nneval(nn, loss, train_x, train_y, val_x, val_y);
+        str_perf = sprintf('; Full-batch train mse = %f, val mse = %f', loss.train.e(end), loss.val.e(end));
+    else
+        loss = nneval(nn, loss, train_x, train_y);
+        str_perf = sprintf('; Full-batch train err = %f', loss.train.e(end));
+    end
     if ishandle(fhandle)
-        if opts.validation == 1
-            loss = nneval(nn, loss, train_x, train_y, val_x, val_y);
-        else
-            loss = nneval(nn, loss, train_x, train_y);
-        end
         nnupdatefigures(nn, fhandle, loss, opts, i);
     end
 
-    disp(['epoch ' num2str(i) '/' num2str(opts.numepochs) '. Took ' num2str(t) ' seconds' '. Mean squared error on training set is ' num2str(mean(L((n-numbatches):(n-1))))]);
+    disp(['epoch ' num2str(i) '/' num2str(opts.numepochs) '. Took ' num2str(t) ' seconds' '. Mini-batch mean squared error on training set is ' num2str(mean(L((n-numbatches):(n-1)))) str_perf]);
     nn.learningRate = nn.learningRate * nn.scaling_learningRate;
 end
 end

NN/nnupdatefigures.m

@@ -43,40 +43,21 @@ function nnupdatefigures(nn,fhandle,L,opts,i)
         legend(p1, M,'Location','NorthEast');
         set(p1, 'Xlim',[0,opts.numepochs + 1])
 
-        if i ==2 % speeds up plotting by factor of ~2
-            set(gca,'LegendColorbarListeners',[]);
-            setappdata(gca,'LegendColorbarManualSpace',1);
-            setappdata(gca,'LegendColorbarReclaimSpace',1);
-        end
-               
         p2 = subplot(1,2,2);
         plot(plot_x,plot_yfrac);
         xlabel('Number of epochs'); ylabel('Misclassification rate');
         title('Misclassification rate')
         legend(p2, M,'Location','NorthEast');
         set(p2, 'Xlim',[0,opts.numepochs + 1])
 
-        if i ==2 % speeds up plotting by factor of ~2
-            set(gca,'LegendColorbarListeners',[]);
-            setappdata(gca,'LegendColorbarManualSpace',1);
-            setappdata(gca,'LegendColorbarReclaimSpace',1);
-        end
-
     else
 
         p = plot(plot_x,plot_ye);
         xlabel('Number of epochs'); ylabel('Error');title('Error');
         legend(p, M,'Location','NorthEast');
         set(gca, 'Xlim',[0,opts.numepochs + 1])
-        
-        if i ==2 % speeds up plotting by factor of ~2
-            set(gca,'LegendColorbarListeners',[]);
-            setappdata(gca,'LegendColorbarManualSpace',1);
-            setappdata(gca,'LegendColorbarReclaimSpace',1);
-            
-        end
-        
+                
     end
     drawnow;
 end
-end
+end

README.md

@@ -1,3 +1,4 @@
+
 DeepLearnToolbox
 ================
 
@@ -53,9 +54,12 @@ Setup
 1. Download.
 2. addpath(genpath('DeepLearnToolbox'));
 
-Everything is work in progress
+Known errors
 ------------------------------
 
+`test_cnn_gradients_are_numerically_correct` fails on Octave because of a bug in Octave's convn implementation. See http://savannah.gnu.org/bugs/?39314
+
+`test_example_CNN` fails in Octave for the same reason.
 Example: Deep Belief Network
 ---------------------
 ```matlab
@@ -69,7 +73,7 @@ train_y = double(train_y);
 test_y  = double(test_y);
 
 %%  ex1 train a 100 hidden unit RBM and visualize its weights
-rng(0);
+rand('state',0)
 dbn.sizes = [100];
 opts.numepochs =   1;
 opts.batchsize = 100;
@@ -80,7 +84,7 @@ dbn = dbntrain(dbn, train_x, opts);
 figure; visualize(dbn.rbm{1}.W');   %  Visualize the RBM weights
 
 %%  ex2 train a 100-100 hidden unit DBN and use its weights to initialize a NN
-rng(0);
+rand('state',0)
 %train dbn
 dbn.sizes = [100 100];
 opts.numepochs =   1;
@@ -119,7 +123,7 @@ test_y  = double(test_y);
 
 %%  ex1 train a 100 hidden unit SDAE and use it to initialize a FFNN
 %  Setup and train a stacked denoising autoencoder (SDAE)
-rng(0);
+rand('state',0)
 sae = saesetup([784 100]);
 sae.ae{1}.activation_function       = 'sigm';
 sae.ae{1}.learningRate              = 1;
@@ -142,39 +146,6 @@ nn = nntrain(nn, train_x, train_y, opts);
 [er, bad] = nntest(nn, test_x, test_y);
 assert(er < 0.16, 'Too big error');
 
-%% ex2 train a 100-100 hidden unit SDAE and use it to initialize a FFNN
-%  Setup and train a stacked denoising autoencoder (SDAE)
-rng(0);
-sae = saesetup([784 100 100]);
-sae.ae{1}.activation_function       = 'sigm';
-sae.ae{1}.learningRate              = 1;
-sae.ae{1}.inputZeroMaskedFraction   = 0.5;
-
-sae.ae{2}.activation_function       = 'sigm';
-sae.ae{2}.learningRate              = 1;
-sae.ae{2}.inputZeroMaskedFraction   = 0.5;
-
-opts.numepochs =   1;
-opts.batchsize = 100;
-sae = saetrain(sae, train_x, opts);
-visualize(sae.ae{1}.W{1}(:,2:end)')
-
-% Use the SDAE to initialize a FFNN
-nn = nnsetup([784 100 100 10]);
-nn.activation_function              = 'sigm';
-nn.learningRate                     = 1;
-
-%add pretrained weights
-nn.W{1} = sae.ae{1}.W{1};
-nn.W{2} = sae.ae{2}.W{1};
-
-% Train the FFNN
-opts.numepochs =   1;
-opts.batchsize = 100;
-nn = nntrain(nn, train_x, train_y, opts);
-[er, bad] = nntest(nn, test_x, test_y);
-assert(er < 0.1, 'Too big error');
-
 ```
 
 
@@ -193,7 +164,7 @@ test_y = double(test_y');
 %% ex1 Train a 6c-2s-12c-2s Convolutional neural network 
 %will run 1 epoch in about 200 second and get around 11% error. 
 %With 100 epochs you'll get around 1.2% error
-rng(0)
+rand('state',0)
 cnn.layers = {
     struct('type', 'i') %input layer
     struct('type', 'c', 'outputmaps', 6, 'kernelsize', 5) %convolution layer
@@ -236,7 +207,7 @@ test_y  = double(test_y);
 test_x = normalize(test_x, mu, sigma);
 
 %% ex1 vanilla neural net
-rng(0);
+rand('state',0)
 nn = nnsetup([784 100 10]);
 opts.numepochs =  1;   %  Number of full sweeps through data
 opts.batchsize = 100;  %  Take a mean gradient step over this many samples
@@ -246,16 +217,8 @@ opts.batchsize = 100;  %  Take a mean gradient step over this many samples
 
 assert(er < 0.08, 'Too big error');
 
-% Make an artificial one and verify that we can predict it
-x = zeros(1,28,28);
-x(:, 14:15, 6:22) = 1;
-x = reshape(x,1,28^2);
-figure; visualize(x');
-predicted = nnpredict(nn,x)-1;
-
-assert(predicted == 1);
 %% ex2 neural net with L2 weight decay
-rng(0);
+rand('state',0)
 nn = nnsetup([784 100 10]);
 
 nn.weightPenaltyL2 = 1e-4;  %  L2 weight decay
@@ -269,7 +232,7 @@ assert(er < 0.1, 'Too big error');
 
 
 %% ex3 neural net with dropout
-rng(0);
+rand('state',0)
 nn = nnsetup([784 100 10]);
 
 nn.dropoutFraction = 0.5;   %  Dropout fraction 
@@ -282,7 +245,7 @@ nn = nntrain(nn, train_x, train_y, opts);
 assert(er < 0.1, 'Too big error');
 
 %% ex4 neural net with sigmoid activation function
-rng(0);
+rand('state',0)
 nn = nnsetup([784 100 10]);
 
 nn.activation_function = 'sigm';    %  Sigmoid activation function
@@ -296,7 +259,7 @@ nn = nntrain(nn, train_x, train_y, opts);
 assert(er < 0.1, 'Too big error');
 
 %% ex5 plotting functionality
-rng(0);
+rand('state',0)
 nn = nnsetup([784 20 10]);
 opts.numepochs         = 5;            %  Number of full sweeps through data
 nn.output              = 'softmax';    %  use softmax output
@@ -315,7 +278,7 @@ tx = train_x(10001:end,:);
 vy   = train_y(1:10000,:);
 ty = train_y(10001:end,:);
 
-rng(0);
+rand('state',0)
 nn                      = nnsetup([784 20 10]);     
 nn.output               = 'softmax';                   %  use softmax output
 opts.numepochs          = 5;                           %  Number of full sweeps through data
@@ -325,6 +288,7 @@ nn = nntrain(nn, tx, ty, opts, vx, vy);                %  nntrain takes validati
 
 [er, bad] = nntest(nn, test_x, test_y);
 assert(er < 0.1, 'Too big error');
+
 ```