fix preqconditioner

PrimeIntellect-ai · Dec 10, 2024 · fb3a721 · fb3a721
1 parent 63f7f88
commit fb3a721
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/src/zeroband/optimizers/__init__.py b/src/zeroband/optimizers/__init__.py
@@ -7,7 +7,7 @@
 
 class SoapConfig(BaseConfig):
     lr: float = 4e-4
-    weight_decay: float = 0.1
+    weight_decay: float = 1e-05
     betas1: float = 0.9
     betas2: float = 0.95