better numerical gradient checking

Author: rasmusbergpalm

NN/nnchecknumgrad.m

@@ -1,34 +1,35 @@
 function nnchecknumgrad(nn, x, y)
-    epsilon = 1e-4;
-    er = 1e-9;
+    epsilon = 1e-6;
+    er = 1e-10;
     n = nn.n;
     for l = 1 : (n - 1)
         for i = 1 : size(nn.W{l}, 1)
             for j = 1 : size(nn.W{l}, 2)
                 nn_m = nn; nn_p = nn;
                 nn_m.W{l}(i, j) = nn.W{l}(i, j) - epsilon;
                 nn_p.W{l}(i, j) = nn.W{l}(i, j) + epsilon;
+                rng(0);
                 nn_m = nnff(nn_m, x, y);
+                rng(0);
                 nn_p = nnff(nn_p, x, y);
                 dW = (nn_p.L - nn_m.L) / (2 * epsilon);
                 e = abs(dW - nn.dW{l}(i, j));
-                if e > er
-                    error('numerical gradient checking failed');
-                end
+                
+                assert(e < er, 'numerical gradient checking failed');
             end
         end
 
         for i = 1 : size(nn.b{l}, 1)
             nn_m = nn; nn_p = nn;
             nn_m.b{l}(i) = nn.b{l}(i) - epsilon;
             nn_p.b{l}(i) = nn.b{l}(i) + epsilon;
+            rng(0);
             nn_m = nnff(nn_m, x, y);
+            rng(0);
             nn_p = nnff(nn_p, x, y);
             db = (nn_p.L - nn_m.L) / (2 * epsilon);
             e = abs(db - nn.db{l}(i));
-            if e > er
-                error('numerical gradient checking failed');
-            end
+            assert(e < er, 'numerical gradient checking failed');
         end
     end
 end

tests/test_nn_gradients_are_numerically_correct.m

@@ -1,7 +1,13 @@
 function test_nn_gradients_are_numerically_correct
-nn = nnsetup([5 2]);
+nn = nnsetup([5 3 2]);
 batch_x = rand(20, 5);
 batch_y = rand(20, 2);
 nn = nnff(nn, batch_x, batch_y);
 nn = nnbp(nn);
-nnchecknumgrad(nn, batch_x, batch_y);
+nnchecknumgrad(nn, batch_x, batch_y);
+
+nn.dropoutFraction=0.5;
+rng(0);
+nn = nnff(nn, batch_x, batch_y);
+nn = nnbp(nn);
+nnchecknumgrad(nn, batch_x, batch_y);