16 lines · 364 BytesRaw
train.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/train.py b/train.py
index 9bec56e..203c66b 100644
--- a/train.py
+++ b/train.py
@@ -1,6 +1,6 @@
MODEL = "llama-7b"
LORA_RANK = 64
-LR = 3e-4
-WARMUP_STEPS = 100
+LR = 1e-4 # 3e-4 and 2e-4 diverged (run-17, run-18)
+WARMUP_STEPS = 500
GRAD_CLIP = None
EARLY_STOP_PATIENCE = None