Implement Softplus double backwards.

Author: gchanan

test/common_nn.py

@@ -139,15 +139,21 @@
         module_name='Softplus',
         input_size=(10, 20),
         reference_fn=lambda i, _: torch.log(1 + torch.exp(i)),
-        check_gradgrad=False,
     ),
     dict(
         module_name='Softplus',
         constructor_args=(2,),
         input_size=(10, 20),
         reference_fn=lambda i, _: 1. / 2. * torch.log(1 + torch.exp(2 * i)),
         desc='beta',
-        check_gradgrad=False,
+    ),
+    dict(
+        module_name='Softplus',
+        constructor_args=(2, -100),
+        input_size=(10, 20),
+        reference_fn=(lambda i, _: ((i * 2) > -100).type_as(i) * i +
+                                   ((i * 2) <= -100).type_as(i) * 1. / 2. * torch.log(1 + torch.exp(2 * i))),
+        desc='beta_threshold',
     ),
     dict(
         module_name='Softshrink',

torch/nn/_functions/thnn/auto_double_backwards.py

@@ -91,6 +91,25 @@ def softmax_double_backwards(ctx, ggI):
     return gI, ggO, None, None, None, None
 
 
+def softplus_double_backwards(ctx, ggI):
+    t = ctx.saved_variables
+    input, gO, output = t[0], t[1], t[2]
+    beta, threshold = ctx.additional_args[0], ctx.additional_args[1]
+
+    input_beta = input * beta
+    above_threshold = ((input_beta) > threshold).type_as(ggI)
+    below_threshold = ((input_beta) <= threshold).type_as(ggI)
+
+    exp_output_beta = (output * beta).exp()
+    first_deriv = (exp_output_beta - 1) / exp_output_beta
+    first_deriv_below_threshold = first_deriv * below_threshold
+
+    gI = ggI * gO * first_deriv_below_threshold * beta / exp_output_beta
+    ggO = ggI * (above_threshold + first_deriv_below_threshold)
+
+    return gI, ggO, None, None, None, None
+
+
 def threshold_double_backwards(ctx, ggI):
     t = ctx.saved_variables
     input = t[0]
@@ -158,6 +177,7 @@ def nllloss_double_backwards(ctx, ggI):
     'LeakyReLU': leakyrelu_double_backwards,
     'LogSoftmax': logsoftmax_double_backwards,
     'Softmax': softmax_double_backwards,
+    'Softplus': softplus_double_backwards,
     'Threshold': threshold_double_backwards,
     'L1Loss': l1loss_double_backwards,
     'NLLLoss': nllloss_double_backwards,