Typo: SGD->AdamW (ml-explore#471)

kgourgou · web-flow · commit cfeef6d9d2c4 · 2024-02-20T15:47:17.000-08:00
diff --git a/transformer_lm/main.py b/transformer_lm/main.py
@@ -189,7 +189,7 @@ def step(inputs, targets):
         "--num_iters", type=int, default=100000, help="Iterations to train for."
     )
     parser.add_argument(
-        "--learning_rate", type=float, default=3e-4, help="SGD learning rate."
+        "--learning_rate", type=float, default=3e-4, help="AdamW learning rate."
     )
     parser.add_argument(
         "--weight_decay", type=float, default=1e-5, help="Set the weight decay"

Original file line number	Diff line number	Diff line change
`@@ -189,7 +189,7 @@ def step(inputs, targets):`
`189`	`189`	`"--num_iters", type=int, default=100000, help="Iterations to train for."`
`190`	`190`	`)`
`191`	`191`	`parser.add_argument(`
`192`		`- "--learning_rate", type=float, default=3e-4, help="SGD learning rate."`
	`192`	`+ "--learning_rate", type=float, default=3e-4, help="AdamW learning rate."`
`193`	`193`	`)`
`194`	`194`	`parser.add_argument(`
`195`	`195`	`"--weight_decay", type=float, default=1e-5, help="Set the weight decay"`