Adding generate_gradient.py

Author: mnielsen

.gitignore

@@ -1,4 +1,4 @@
-*~
-*.org
-*.pkl
+*~
+*.org
+*.pkl
 *.pyc

README.md

@@ -1,33 +1,33 @@
-# Code samples for "Neural Networks and Deep Learning"
-
-This repository contains code samples for my (forthcoming) book on
-"Neural Networks and Deep Learning".
-
-As the code is written to accompany the book, I don't intend to add
-new features.  However, bug reports are welcome, and you should feel
-free to fork and modify the code.
-
-## License
-
-MIT License
-
-Copyright (c) 2012-2013 Michael Nielsen
-
-Permission is hereby granted, free of charge, to any person obtaining
-a copy of this software and associated documentation files (the
-"Software"), to deal in the Software without restriction, including
-without limitation the rights to use, copy, modify, merge, publish,
-distribute, sublicense, and/or sell copies of the Software, and to
-permit persons to whom the Software is furnished to do so, subject to
-the following conditions:
-
-The above copyright notice and this permission notice shall be
-included in all copies or substantial portions of the Software.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
-NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
-LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
-OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
-WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
+# Code samples for "Neural Networks and Deep Learning"
+
+This repository contains code samples for my (forthcoming) book on
+"Neural Networks and Deep Learning".
+
+As the code is written to accompany the book, I don't intend to add
+new features.  However, bug reports are welcome, and you should feel
+free to fork and modify the code.
+
+## License
+
+MIT License
+
+Copyright (c) 2012-2013 Michael Nielsen
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

code/mnist_average_darkness.py

@@ -1,64 +1,64 @@
-"""
-mnist_average_darkness
-~~~~~~~~~~~~~~~~~~~~~~
-
-A naive classifier for recognizing handwritten digits from the MNIST
-data set.  The program classifies digits based on how dark they are
---- the idea is that digits like "1" tend to be less dark than digits
-like "8", simply because the latter has a more complex shape.  When
-shown an image the classifier returns whichever digit in the training
-data had the closest average darkness.
-
-The program works in two steps: first it trains the classifier, and
-then it applies the classifier to the MNIST test data to see how many
-digits are correctly classified.
-
-Needless to say, this isn't a very good way of recognizing handwritten
-digits!  Still, it's useful to show what sort of performance we get
-from naive ideas."""
-
-#### Libraries
-# Standard library
-from collections import defaultdict
-
-# My libraries
-import mnist_loader
-
-def main():
-    training_data, validation_data, test_data = mnist_loader.load_data()
-    # training phase: compute the average darknesses for each digit,
-    # based on the training data
-    avgs = avg_darknesses(training_data)
-    # testing phase: see how many of the test images are classified
-    # correctly
-    num_correct = sum(int(guess_digit(image, avgs) == digit)
-                      for image, digit in zip(test_data[0], test_data[1]))
-    print "Baseline classifier using average darkness of image."
-    print "%s of %s values correct." % (num_correct, len(test_data[1]))
-
-def avg_darknesses(training_data):
-    """ Return a defaultdict whose keys are the digits 0 through 9.
-    For each digit we compute a value which is the average darkness of
-    training images containing that digit.  The darkness for any
-    particular image is just the sum of the darknesses for each pixel."""
-    digit_counts = defaultdict(int)
-    darknesses = defaultdict(float)
-    for image, digit in zip(training_data[0], training_data[1]):
-        digit_counts[digit] += 1
-        darknesses[digit] += sum(image)
-    avgs = defaultdict(float)
-    for digit, n in digit_counts.iteritems():
-        avgs[digit] = darknesses[digit] / n
-    return avgs
-
-def guess_digit(image, avgs):
-    """Return the digit whose average darkness in the training data is
-    closest to the darkness of ``image``.  Note that ``avgs`` is
-    assumed to be a defaultdict whose keys are 0...9, and whose values
-    are the corresponding average darknesses across the training data."""
-    darkness = sum(image)
-    distances = {k: abs(v-darkness) for k, v in avgs.iteritems()}
-    return min(distances, key=distances.get)
-
-if __name__ == "__main__":
-    main()
+"""
+mnist_average_darkness
+~~~~~~~~~~~~~~~~~~~~~~
+
+A naive classifier for recognizing handwritten digits from the MNIST
+data set.  The program classifies digits based on how dark they are
+--- the idea is that digits like "1" tend to be less dark than digits
+like "8", simply because the latter has a more complex shape.  When
+shown an image the classifier returns whichever digit in the training
+data had the closest average darkness.
+
+The program works in two steps: first it trains the classifier, and
+then it applies the classifier to the MNIST test data to see how many
+digits are correctly classified.
+
+Needless to say, this isn't a very good way of recognizing handwritten
+digits!  Still, it's useful to show what sort of performance we get
+from naive ideas."""
+
+#### Libraries
+# Standard library
+from collections import defaultdict
+
+# My libraries
+import mnist_loader
+
+def main():
+    training_data, validation_data, test_data = mnist_loader.load_data()
+    # training phase: compute the average darknesses for each digit,
+    # based on the training data
+    avgs = avg_darknesses(training_data)
+    # testing phase: see how many of the test images are classified
+    # correctly
+    num_correct = sum(int(guess_digit(image, avgs) == digit)
+                      for image, digit in zip(test_data[0], test_data[1]))
+    print "Baseline classifier using average darkness of image."
+    print "%s of %s values correct." % (num_correct, len(test_data[1]))
+
+def avg_darknesses(training_data):
+    """ Return a defaultdict whose keys are the digits 0 through 9.
+    For each digit we compute a value which is the average darkness of
+    training images containing that digit.  The darkness for any
+    particular image is just the sum of the darknesses for each pixel."""
+    digit_counts = defaultdict(int)
+    darknesses = defaultdict(float)
+    for image, digit in zip(training_data[0], training_data[1]):
+        digit_counts[digit] += 1
+        darknesses[digit] += sum(image)
+    avgs = defaultdict(float)
+    for digit, n in digit_counts.iteritems():
+        avgs[digit] = darknesses[digit] / n
+    return avgs
+
+def guess_digit(image, avgs):
+    """Return the digit whose average darkness in the training data is
+    closest to the darkness of ``image``.  Note that ``avgs`` is
+    assumed to be a defaultdict whose keys are 0...9, and whose values
+    are the corresponding average darknesses across the training data."""
+    darkness = sum(image)
+    distances = {k: abs(v-darkness) for k, v in avgs.iteritems()}
+    return min(distances, key=distances.get)
+
+if __name__ == "__main__":
+    main()

code/mnist_loader.py

@@ -1,85 +1,85 @@
-"""
-mnist_loader
-~~~~~~~~~~~~
-
-A library to load the MNIST image data.  For details of the data
-structures that are returned, see the doc strings for ``load_data``
-and ``load_data_wrapper``.  In practice, ``load_data_wrapper`` is the
-function usually called by our neural network code.
-"""
-
-#### Libraries
-# Standard library
-import cPickle
-import gzip
-
-# Third-party libraries
-import numpy as np
-
-def load_data():
-    """Return the MNIST data as a tuple containing the training data,
-    the validation data, and the test data.
-
-    The ``training_data`` is returned as a tuple with two entries.
-    The first entry contains the actual training images.  This is a
-    numpy ndarray with 50,000 entries.  Each entry is, in turn, a
-    numpy ndarray with 784 values, representing the 28 * 28 = 784
-    pixels in a single MNIST image.
-
-    The second entry in the ``training_data`` tuple is a numpy ndarray
-    containing 50,000 entries.  Those entries are just the digit
-    values (0...9) for the corresponding images contained in the first
-    entry of the tuple.
-
-    The ``validation_data`` and ``test_data`` are similar, except
-    each contains only 10,000 images.
-
-    This is a nice data format, but for use in neural networks it's
-    helpful to modify the format of the ``training_data`` a little.
-    That's done in the wrapper function ``load_data_wrapper()``, see
-    below.
-    """
-    f = gzip.open('../data/mnist.pkl.gz', 'rb')
-    training_data, validation_data, test_data = cPickle.load(f)
-    f.close()
-    return (training_data, validation_data, test_data)
-
-def load_data_wrapper():
-    """Return a tuple containing ``(training_data, validation_data,
-    test_data)``. Based on ``load_data``, but the format is more
-    convenient for use in our implementation of neural networks.
-
-    In particular, ``training_data`` is a list containing 50,000
-    2-tuples ``(x, y)``.  ``x`` is a 784-dimensional numpy.ndarray
-    containing the input image.  ``y`` is a 10-dimensional
-    numpy.ndarray representing the unit vector corresponding to the
-    correct digit for ``x``.
-
-    ``validation_data`` and ``test_data`` are lists containing 10,000
-    2-tuples ``(x, y)``.  In each case, ``x`` is a 784-dimensional
-    numpy.ndarry containing the input image, and ``y`` is the
-    corresponding classification, i.e., the digit values (integers)
-    corresponding to ``x``.
-
-    Obviously, this means we're using slightly different formats for
-    the training data and the validation / test data.  These formats
-    turn out to be the most convenient for use in our neural network
-    code."""
-    tr_d, va_d, te_d = load_data()
-    training_inputs = [np.reshape(x, (784, 1)) for x in tr_d[0]]
-    training_results = [vectorized_result(y) for y in tr_d[1]]
-    training_data = zip(training_inputs, training_results)
-    validation_inputs = [np.reshape(x, (784, 1)) for x in va_d[0]]
-    validation_data = zip(validation_inputs, va_d[1])
-    test_inputs = [np.reshape(x, (784, 1)) for x in te_d[0]]
-    test_data = zip(test_inputs, te_d[1])
-    return (training_data, validation_data, test_data)
-
-def vectorized_result(j):
-    """Return a 10-dimensional unit vector with a 1.0 in the jth
-    position and zeroes elsewhere.  This is used to convert a digit
-    (0...9) into a corresponding desired output from the neural
-    network."""
-    e = np.zeros((10, 1))
-    e[j] = 1.0
-    return e
+"""
+mnist_loader
+~~~~~~~~~~~~
+
+A library to load the MNIST image data.  For details of the data
+structures that are returned, see the doc strings for ``load_data``
+and ``load_data_wrapper``.  In practice, ``load_data_wrapper`` is the
+function usually called by our neural network code.
+"""
+
+#### Libraries
+# Standard library
+import cPickle
+import gzip
+
+# Third-party libraries
+import numpy as np
+
+def load_data():
+    """Return the MNIST data as a tuple containing the training data,
+    the validation data, and the test data.
+
+    The ``training_data`` is returned as a tuple with two entries.
+    The first entry contains the actual training images.  This is a
+    numpy ndarray with 50,000 entries.  Each entry is, in turn, a
+    numpy ndarray with 784 values, representing the 28 * 28 = 784
+    pixels in a single MNIST image.
+
+    The second entry in the ``training_data`` tuple is a numpy ndarray
+    containing 50,000 entries.  Those entries are just the digit
+    values (0...9) for the corresponding images contained in the first
+    entry of the tuple.
+
+    The ``validation_data`` and ``test_data`` are similar, except
+    each contains only 10,000 images.
+
+    This is a nice data format, but for use in neural networks it's
+    helpful to modify the format of the ``training_data`` a little.
+    That's done in the wrapper function ``load_data_wrapper()``, see
+    below.
+    """
+    f = gzip.open('../data/mnist.pkl.gz', 'rb')
+    training_data, validation_data, test_data = cPickle.load(f)
+    f.close()
+    return (training_data, validation_data, test_data)
+
+def load_data_wrapper():
+    """Return a tuple containing ``(training_data, validation_data,
+    test_data)``. Based on ``load_data``, but the format is more
+    convenient for use in our implementation of neural networks.
+
+    In particular, ``training_data`` is a list containing 50,000
+    2-tuples ``(x, y)``.  ``x`` is a 784-dimensional numpy.ndarray
+    containing the input image.  ``y`` is a 10-dimensional
+    numpy.ndarray representing the unit vector corresponding to the
+    correct digit for ``x``.
+
+    ``validation_data`` and ``test_data`` are lists containing 10,000
+    2-tuples ``(x, y)``.  In each case, ``x`` is a 784-dimensional
+    numpy.ndarry containing the input image, and ``y`` is the
+    corresponding classification, i.e., the digit values (integers)
+    corresponding to ``x``.
+
+    Obviously, this means we're using slightly different formats for
+    the training data and the validation / test data.  These formats
+    turn out to be the most convenient for use in our neural network
+    code."""
+    tr_d, va_d, te_d = load_data()
+    training_inputs = [np.reshape(x, (784, 1)) for x in tr_d[0]]
+    training_results = [vectorized_result(y) for y in tr_d[1]]
+    training_data = zip(training_inputs, training_results)
+    validation_inputs = [np.reshape(x, (784, 1)) for x in va_d[0]]
+    validation_data = zip(validation_inputs, va_d[1])
+    test_inputs = [np.reshape(x, (784, 1)) for x in te_d[0]]
+    test_data = zip(test_inputs, te_d[1])
+    return (training_data, validation_data, test_data)
+
+def vectorized_result(j):
+    """Return a 10-dimensional unit vector with a 1.0 in the jth
+    position and zeroes elsewhere.  This is used to convert a digit
+    (0...9) into a corresponding desired output from the neural
+    network."""
+    e = np.zeros((10, 1))
+    e[j] = 1.0
+    return e