Merge pull request deeplearning4j#889 from deeplearning4j/canovatravis

clean up optimization and deprecations

Author: AlexDBlack

.travis.yml

@@ -10,7 +10,9 @@ jdk:
 # for running tests on Travis CI container infrastructure for faster builds
 sudo: true
 
-
+env:
+  global:
+    JAVA_OPTS=-Xmx2g
 
 before_install:
   - sudo apt-get install build-essential git

deeplearning4j-cli/deeplearning4j-cli-api/src/main/java/org/deeplearning4j/cli/subcommands/Train.java

@@ -170,7 +170,7 @@ public void execLocal() {
                 MultiLayerConfiguration conf = MultiLayerConfiguration.fromJson(FileUtils.readFileToString(new File(modelPath)));
                 FeedForwardLayer outputLayer = (FeedForwardLayer) conf.getConf(conf.getConfs().size() - 1).getLayer();
 
-                DataSetIterator iter = new RecordReaderDataSetIterator( reader , conf.getConf(0).getBatchSize(),-1, outputLayer.getNOut());
+                DataSetIterator iter = new RecordReaderDataSetIterator( reader ,1,-1, outputLayer.getNOut());
 
                 MultiLayerNetwork network = new MultiLayerNetwork(conf);
                 if(verbose) {
@@ -195,7 +195,7 @@ public void execLocal() {
                 NeuralNetConfiguration conf = NeuralNetConfiguration.fromJson(FileUtils.readFileToString(new File(modelPath)));
                 LayerFactory factory = LayerFactories.getFactory(conf);
                 Layer l = factory.create(conf);
-                DataSetIterator iter = new RecordReaderDataSetIterator( reader , conf.getBatchSize());
+                DataSetIterator iter = new RecordReaderDataSetIterator( reader , 1);
                 while(iter.hasNext()) {
                     l.fit(iter.next().getFeatureMatrix());
                 }

deeplearning4j-cli/deeplearning4j-cli-api/src/test/java/org/deeplearning4j/cli/TrainMultiLayerConfigTest.java

@@ -46,7 +46,6 @@ public void testMultiLayerConfig() throws Exception {
         Model testModelFlag = new Model();
         MultiLayerConfiguration conf = new NeuralNetConfiguration.Builder()
                 .iterations(100)
-                .batchSize(10)
                 .learningRate(1e-1f).momentum(0.9).regularization(true).l2(2e-4)
                 .optimizationAlgo(OptimizationAlgorithm.LBFGS).constrainGradientToUnitNorm(true)
                 .list(2)

deeplearning4j-core/src/main/java/org/deeplearning4j/gradientcheck/GradientCheckUtil.java

@@ -60,13 +60,13 @@ public static boolean checkGradients( MultiLayerNetwork mln, double epsilon, dou
 
         if(useUpdater) {
             Updater updater = UpdaterCreator.getUpdater(mln);
-            updater.update(mln, gradAndScore.getFirst(), 0);
+            updater.update(mln, gradAndScore.getFirst(), 0, mln.batchSize());
         }
 
         INDArray gradientToCheck = gradAndScore.getFirst().gradient();
         INDArray originalParams = mln.params();
 
-        int nParams = mln.numParams();
+        int nParams = originalParams.length();
 
         int totalNFailures = 0;
         double maxError = 0.0;

deeplearning4j-core/src/main/java/org/deeplearning4j/nn/api/Updater.java

@@ -12,10 +12,11 @@
 public interface Updater extends Serializable {
     /**
      * Updater: updates the model
+     *
      * @param layer
      * @param gradient
-     * @param  iteration
+     * @param iteration
      */
-    void update(Layer layer,Gradient gradient,int iteration);
+    void update(Layer layer, Gradient gradient, int iteration, int miniBatchSize);
 
 }

deeplearning4j-core/src/main/java/org/deeplearning4j/nn/conf/InputPreProcessor.java

@@ -54,16 +54,18 @@ public interface InputPreProcessor extends Serializable, Cloneable {
     /**
      * Pre preProcess input/activations for a multi layer network
      * @param input the input to pre preProcess
+     * @param miniBatchSize
      * @return the processed input
      */
-    INDArray preProcess(INDArray input, Layer layer);
+    INDArray preProcess(INDArray input, int miniBatchSize);
 
     /**Reverse the preProcess during backprop. Process Gradient/epsilons before
      * passing them to the layer below.
-     * @param output which is a pair of the gradient and epsilon 
+     * @param output which is a pair of the gradient and epsilon
+     * @param miniBatchSize
      * @return the reverse of the pre preProcess step (if any)
      */
-    INDArray backprop(INDArray output, Layer layer);
+    INDArray backprop(INDArray output, int miniBatchSize);
 
     InputPreProcessor clone();
 }

deeplearning4j-core/src/main/java/org/deeplearning4j/nn/conf/NeuralNetConfiguration.java

@@ -55,7 +55,6 @@ public class NeuralNetConfiguration implements Serializable,Cloneable {
 
     protected Layer layer;
     //batch size: primarily used for conv nets. Will be reinforced if set.
-    protected int batchSize = 1;
     protected boolean miniBatch = true;
     protected int numIterations = 5;
     //number of line search iterations
@@ -65,8 +64,6 @@ public class NeuralNetConfiguration implements Serializable,Cloneable {
     //gradient keys used for ensuring order when getting and setting the gradient
     protected List<String> variables = new ArrayList<>();
     //whether to constrain the gradient to unit norm or not
-    @Deprecated
-    protected boolean constrainGradientToUnitNorm = false;
     //adadelta - weight for how much to consider previous history
     protected StepFunction stepFunction;
     protected boolean useRegularization = false;
@@ -285,7 +282,6 @@ public static class Builder implements Cloneable {
         private double adamMeanDecay = 0.9;
         private double adamVarDecay = 0.999;
         private Layer layer;
-        private int batchSize = 1;
         private boolean miniBatch = true;
         private int numIterations = 5;
         private int maxNumLineSearchIterations = 5;
@@ -352,11 +348,7 @@ public Builder maxNumLineSearchIterations(int maxNumLineSearchIterations) {
             return this;
         }
 
-        /** Minibatch size. Number of examples in a batch.*/
-        public Builder batchSize(int batchSize) {
-            this.batchSize = batchSize;
-            return this;
-        }
+
 
         /** Layer class. */
         public Builder layer(Layer layer) {
@@ -590,13 +582,11 @@ public NeuralNetConfiguration build() {
 
             conf.minimize = minimize;
             conf.maxNumLineSearchIterations = maxNumLineSearchIterations;
-            conf.batchSize = batchSize;
             conf.layer = layer;
             conf.numIterations = numIterations;
             conf.useRegularization = useRegularization;
             conf.useSchedules = useSchedules;
             conf.optimizationAlgo = optimizationAlgo;
-            conf.constrainGradientToUnitNorm = constrainGradientToUnitNorm;
             conf.seed = seed;
             conf.timeSeriesLength = timeSeriesLength;
             conf.stepFunction = stepFunction;

deeplearning4j-core/src/main/java/org/deeplearning4j/nn/conf/preprocessor/BinomialSamplingPreProcessor.java

@@ -21,7 +21,6 @@
 
 import lombok.Data;
 
-import org.deeplearning4j.nn.api.Layer;
 import org.nd4j.linalg.api.ndarray.INDArray;
 import org.nd4j.linalg.factory.Nd4j;
 
@@ -33,13 +32,13 @@
 public class BinomialSamplingPreProcessor extends BaseInputPreProcessor {
 
 	@Override
-    public INDArray preProcess(INDArray input, Layer layer) {
+    public INDArray preProcess(INDArray input, int miniBatchSize) {
         return Nd4j.getDistributions().createBinomial(1, input).sample(input.shape());
     }
 
 
     @Override
-    public INDArray backprop(INDArray output, Layer layer) {
+    public INDArray backprop(INDArray output, int miniBatchSize) {
         return output;	//No op?
     }
 }

deeplearning4j-core/src/main/java/org/deeplearning4j/nn/conf/preprocessor/CnnToFeedForwardPreProcessor.java

@@ -23,7 +23,6 @@
 
 import lombok.Data;
 
-import org.deeplearning4j.nn.api.Layer;
 import org.deeplearning4j.nn.conf.InputPreProcessor;
 import org.nd4j.linalg.api.ndarray.INDArray;
 import org.nd4j.linalg.api.shape.Shape;
@@ -78,7 +77,7 @@ public CnnToFeedForwardPreProcessor(){}
 
     @Override
     // return 2 dimensions
-    public INDArray preProcess(INDArray input, Layer layer) {
+    public INDArray preProcess(INDArray input, int miniBatchSize) {
         int[] otherOutputs = null;
 
         //this.inputHeight = input.size(-2);
@@ -104,7 +103,7 @@ else if(input.shape().length == 3) {
     }
 
     @Override
-    public INDArray backprop(INDArray output, Layer layer){
+    public INDArray backprop(INDArray output, int miniBatchSize){
         if (output.shape().length == 4)
             return output;
         if (output.columns() != inputWidth * inputHeight * numChannels)

deeplearning4j-core/src/main/java/org/deeplearning4j/nn/conf/preprocessor/CnnToRnnPreProcessor.java

@@ -2,9 +2,7 @@
 
 import com.fasterxml.jackson.annotation.JsonCreator;
 import com.fasterxml.jackson.annotation.JsonProperty;
-import lombok.AllArgsConstructor;
 import lombok.Data;
-import org.deeplearning4j.nn.api.Layer;
 import org.deeplearning4j.nn.conf.InputPreProcessor;
 import org.nd4j.linalg.api.ndarray.INDArray;
 
@@ -41,20 +39,19 @@ public CnnToRnnPreProcessor(@JsonProperty("inputHeight") int inputHeight,
     }
 
     @Override
-    public INDArray preProcess(INDArray input, Layer layer) {
+    public INDArray preProcess(INDArray input, int miniBatchSize) {
         if(input.rank() != 4) throw new IllegalArgumentException("Invalid input: expect CNN activations with rank 4 (received input with shape "
             + Arrays.toString(input.shape())+")");
         //Input: 4d activations (CNN)
         //Output: 3d activations (RNN)
 
         int[] shape = input.shape();    //[timeSeriesLength*miniBatchSize, numChannels, inputHeight, inputWidth]
-        int miniBatchSize = layer.getInputMiniBatchSize();
         INDArray reshaped = input.reshape(miniBatchSize,shape[0]/miniBatchSize,product);
         return reshaped.permute(0,2,1);
     }
 
     @Override
-    public INDArray backprop(INDArray output, Layer layer) {
+    public INDArray backprop(INDArray output, int miniBatchSize) {
         int[] shape = output.shape();
         INDArray output2d;
         if(shape[0]==1){