1. # from neural_net import AltTwoLayerNet
  2.  
  3. tol = 1e-5
  4. # You should expect loss to go down and train and val accuracy go up for every epoch
  5. input_size = train_x.shape[1]
  6. hidden_size = 100
  7. output_size = 10
  8. reg = 0.1
  9. std = 0.001 # multiplier to init
  10.  
  11. model = TwoLayerNet(input_size=nis, hidden_size=hidden_size, output_size=output_size, reg=reg)
  12. # model2 = AltTwoLayerNet(input_size=nis, hidden_size=nhs, output_size=nos, reg=reg)
  13. dataset = Dataset(train_x, train_y, val_x, val_y)
  14.  
  15. # optimizers = {}
  16. # for name, param in model.params.items():
  17. #     optimizers[name] = SGD()
  18.  
  19. batch_size = 30
  20.  
  21. np.random.seed(69)
  22. num_train = dataset.train_x.shape[0]
  23. shuffled_indices = np.arange(num_train)
  24. np.random.shuffle(shuffled_indices)
  25. sections = np.arange(batch_size, num_train, batch_size)
  26. batches_indices = np.array_split(shuffled_indices, sections)
  27.  
  28. batch_indices = batches_indices[0]
  29. batch_x = dataset.train_x[batch_indices]
  30. batch_y = dataset.train_y[batch_indices]
  31.  
  32. learning_rate = 0.05
  33.  
  34. for i in range(10000):
  35.     loss, grads = model.compute_loss_and_gradients(batch_x, batch_y)
  36.     params = model.params
  37.     if i % 1000 == 0:
  38.         print("loss = ", loss)
  39.         print("values = ", {k: np.sum(v) for k, v in params.items()})
  40.         print("grads = ", {k: np.sum(v) for k, v in grads.items()})
  41.         print()
  42. #         print(batch_y, " > ", model.predict(batch_x))
  43.  
  44.     for name, param in model.params.items():
  45.         grad = grads[name]
  46.         optimizer = optimizers[name]
  47.         params[name] = params[name] - learning_rate * grad
  48.  
  49.  
  50. # trainer = Trainer(model2, dataset, SGD(), num_epochs=1, batch_size=batch_size,
  51. #                   learning_rate=0.01, learning_rate_decay=0.9)
  52. # # loss_history, train_history, val_history = trainer.fit()
  53. # # model1.predict(train_x[0])
  54. # print("batches_indices[0]:", train_x[batches_indices[0]].shape)
  55. # model1.predict(train_x[batches_indices[0]])
  56.  
  57. ################
  58. #### output ####
  59. ################
  60.  
  61. loss =  2.302738695380354
  62. values =  {'w1': -0.004068116397824662, 'b1': 0.01, 'w2': -0.0020747624855106516, 'b2': 0.001}
  63. grads =  {'w1': 0.019029607815834754, 'b1': -8.17012587906322e-05, 'w2': -0.0002074762485510651, 'b2': -1.3877787807814457e-17}
  64.  
  65. loss =  2.305641633884892
  66. values =  {'w1': -0.9555485071895646, 'b1': 0.014085062939531566, 'w2': 0.008299049942042487, 'b2': 0.000999999999726775}
  67. grads =  {'w1': 0.019029607815834754, 'b1': -8.17012587906322e-05, 'w2': -0.0002074762485510651, 'b2': -1.3877787807814457e-17}
  68.  
  69. loss =  2.3174225788290426
  70. values =  {'w1': -1.9070288979812922, 'b1': 0.018170125879062932, 'w2': 0.018672862369594933, 'b2': 0.0009999999995944364}
  71. grads =  {'w1': 0.019029607815834754, 'b1': -8.17012587906322e-05, 'w2': -0.0002074762485510651, 'b2': -1.3877787807814457e-17}
  72.  
  73. loss =  2.3380815302128046
  74. values =  {'w1': -2.8585092887729777, 'b1': 0.022255188818594434, 'w2': 0.029046674797153428, 'b2': 0.0009999999999514841}
  75. grads =  {'w1': 0.019029607815834754, 'b1': -8.17012587906322e-05, 'w2': -0.0002074762485510651, 'b2': -1.3877787807814457e-17}
  76.  
  77. loss =  2.367618488036179
  78. values =  {'w1': -3.809989679564654, 'b1': 0.026340251758126456, 'w2': 0.03942048722472072, 'b2': 0.001000000001369017}
  79. grads =  {'w1': 0.019029607815834754, 'b1': -8.17012587906322e-05, 'w2': -0.0002074762485510651, 'b2': -1.3877787807814457e-17}
  80.  
  81. loss =  2.4060334522991647
  82. values =  {'w1': -4.761470070356343, 'b1': 0.030425314697659713, 'w2': 0.04979429965227849, 'b2': 0.0010000000031453737}
  83. grads =  {'w1': 0.019029607815834754, 'b1': -8.17012587906322e-05, 'w2': -0.0002074762485510651, 'b2': -1.3877787807814457e-17}
  84.  
  85. loss =  2.4533264230017626
  86. values =  {'w1': -5.712950461148048, 'b1': 0.034510377637193, 'w2': 0.06016811207983852, 'b2': 0.0010000000049217306}
  87. grads =  {'w1': 0.019029607815834754, 'b1': -8.17012587906322e-05, 'w2': -0.0002074762485510651, 'b2': -1.3877787807814457e-17}
  88.  
  89. loss =  2.5094974001439723
  90. values =  {'w1': -6.664430851939748, 'b1': 0.03859544057672655, 'w2': 0.07054192450739312, 'b2': 0.0009999999981715746}
  91. grads =  {'w1': 0.019029607815834754, 'b1': -8.17012587906322e-05, 'w2': -0.0002074762485510651, 'b2': -1.3877787807814457e-17}
  92.  
  93. loss =  2.574546383725794
  94. values =  {'w1': -7.61591124273147, 'b1': 0.042680503516257406, 'w2': 0.08091573693496038, 'b2': 0.0009999999857370767}
  95. grads =  {'w1': 0.019029607815834754, 'b1': -8.17012587906322e-05, 'w2': -0.0002074762485510651, 'b2': -1.3877787807814457e-17}
  96.  
  97. loss =  2.648473373747228
  98. values =  {'w1': -8.567391633523291, 'b1': 0.04676556645578911, 'w2': 0.09128954936252516, 'b2': 0.0009999999733025788}
  99. grads =  {'w1': 0.019029607815834754, 'b1': -8.17012587906322e-05, 'w2': -0.0002074762485510651, 'b2': -1.3877787807814457e-17}