We read every piece of feedback, and take your input very seriously.
To see all available qualifiers, see our documentation.
There was an error while loading. Please reload this page.
1 parent c7392f2 commit b21d6ceCopy full SHA for b21d6ce
lion/lion_pytorch.py
@@ -78,10 +78,7 @@ def step(self, closure=None):
78
# Weight update
79
update = exp_avg * beta1 + grad * (1 - beta1)
80
81
- p.add_(torch.sign(update), alpha=-group['lr'], inplace=True)
82
- #This has been made more efficient by using the torch.sign function's inplace mode.
83
- #This will prevent the need to create a new tensor for the updated parameter,
84
- #which can save a significant amount of time for large models.
+ p.add_(update.sign_(), alpha=-group['lr'])
85
86
# Decay the momentum running average coefficient
87
exp_avg.mul_(beta2).add_(grad, alpha=1 - beta2)
0 commit comments