Merged code from Ole Winther. Thanks! Adds linear and softmax ouput neurons to NN

Author: rasmusbergpalm

NN/nnbp.m

@@ -5,8 +5,12 @@
 
     n = nn.n;
     sparsityError = 0;
-    
-    d{n} = - nn.e .* (nn.a{n} .* (1 - nn.a{n}));
+    switch nn.output
+        case 'sigm'
+            d{n} = - nn.e .* (nn.a{n} .* (1 - nn.a{n}));
+        case {'softmax','linear'}
+             d{n} = - nn.e;
+    end
     for i = (n - 1) : -1 : 2
         if(nn.nonSparsityPenalty>0)
             pi = repmat(nn.p{i}, size(nn.a{i}, 1), 1);

NN/nnchecknumgrad.m

@@ -1,6 +1,6 @@
 function nnchecknumgrad(nn, x, y)
     epsilon = 1e-6;
-    er = 1e-10;
+    er = 1e-8;
     n = nn.n;
     for l = 1 : (n - 1)
         for i = 1 : size(nn.W{l}, 1)

NN/nnff.m

@@ -9,23 +9,37 @@
     nn.a{1} = x;
 
     %feedforward pass
-    for i = 2 : n
+    for i = 2 : n-1
         nn.a{i} = sigm(repmat(nn.b{i - 1}', m, 1) + nn.a{i - 1} * nn.W{i - 1}');
-        if(nn.dropoutFraction > 0 && i<n)
+        if(nn.dropoutFraction > 0)
             if(nn.testing)
                 nn.a{i} = nn.a{i}.*(1 - nn.dropoutFraction);
             else
                 nn.a{i} = nn.a{i}.*(rand(size(nn.a{i}))>nn.dropoutFraction);
             end
         end
-        
         %calculate running exponential activations for use with sparsity
         if(nn.nonSparsityPenalty>0)
             nn.p{i} = 0.99 * nn.p{i} + 0.01 * mean(nn.a{i}, 1);
         end
     end
+    switch nn.output 
+        case 'sigm'
+            nn.a{n} = sigm(repmat(nn.b{n - 1}', m, 1) + nn.a{n - 1} * nn.W{n - 1}');
+        case 'linear'
+            nn.a{n} = repmat(nn.b{n - 1}', m, 1) + nn.a{n - 1} * nn.W{n - 1}';
+        case 'softmax'
+            nn.a{n} = repmat(nn.b{n - 1}', m, 1) + nn.a{n - 1} * nn.W{n - 1}';
+            nn.a{n} = exp(bsxfun(@minus, nn.a{n}, max(nn.a{n},[],2)));
+            nn.a{n} = bsxfun(@rdivide, nn.a{n}, sum(nn.a{n}, 2)); 
+    end
 
     %error and loss
     nn.e = y - nn.a{n};
-    nn.L = 1/2 * sum(sum(nn.e .^ 2)) / m; 
+    switch nn.output
+        case {'sigm','linear'}
+            nn.L = 1/2 * sum(sum(nn.e .^ 2)) / m; 
+        case 'softmax'
+            nn.L = -sum(sum(y .* log(nn.a{n}))) / m;
+    end
 end

NN/nnsetup.m

@@ -14,6 +14,7 @@
     nn.inputZeroMaskedFraction          = 0;      %  Used for Denoising AutoEncoders
     nn.dropoutFraction                  = 0;      %  Dropout level (http://www.cs.toronto.edu/~hinton/absps/dropout.pdf)
     nn.testing                          = 0;      %  Internal variable. nntest sets this to one.
+    nn.output                           = 'sigm'; %  output unit 'sigm' (=logistic), 'softmax' and 'linear'
 
     for i = 2 : nn.n
         % biases and bias momentum

tests/test_nn_gradients_are_numerically_correct.m

@@ -1,13 +1,29 @@
 function test_nn_gradients_are_numerically_correct
-nn = nnsetup([5 3 2]);
 batch_x = rand(20, 5);
 batch_y = rand(20, 2);
+
+nn = nnsetup([5 3 2]);
+nn.output='sigm';
+nn = nnff(nn, batch_x, batch_y);
+nn = nnbp(nn);
+nnchecknumgrad(nn, batch_x, batch_y);
+
+nn = nnsetup([5 3 2]);
+nn.output='linear';
 nn = nnff(nn, batch_x, batch_y);
 nn = nnbp(nn);
 nnchecknumgrad(nn, batch_x, batch_y);
 
+nn = nnsetup([5 3 2]);
+batch_y=batch_y==repmat(max(batch_y,[],2),1,size(batch_y,2));
+nn.output='softmax';
+nn = nnff(nn, batch_x, batch_y);
+nn = nnbp(nn);
+nnchecknumgrad(nn, batch_x, batch_y);
+
+nn = nnsetup([5 3 2]);
 nn.dropoutFraction=0.5;
 rng(0);
 nn = nnff(nn, batch_x, batch_y);
 nn = nnbp(nn);
-nnchecknumgrad(nn, batch_x, batch_y);
+nnchecknumgrad(nn, batch_x, batch_y);