Merged.

PrimeIntellect-ai · Jan 18, 2025 · 22e14e9 · 22e14e9
2 parents 0e87e1d + f6e238a
commit 22e14e9
Show file tree

Hide file tree

Showing 3 changed files with 35 additions and 5 deletions.
diff --git a/configs/150M/H100-fast.toml b/configs/150M/H100-fast.toml
@@ -0,0 +1,22 @@
+name_model = "150M"
+project = "debug_150m_zero_band"
+type_model = "llama2"
+
+[train]
+micro_bs = 64 # change this base on the gpu
+reshard_after_forward = true
+
+[optim]
+batch_size = 512
+warmup_steps = 278
+total_steps = 8192
+
+[optim.optim]
+lr = 0.003551730141097694
+betas1 = 0.9454835470717078
+betas2 = 0.9190488086654895
+weight_decay = 0.24530252977858977
+
+[data]
+seq_length = 1024
+dataset_name_or_paths = "datasets/fineweb-edu"
diff --git a/pyproject.toml b/pyproject.toml
@@ -17,7 +17,7 @@ dependencies = [
     "ninja",
     "zstandard",
     "pyarrow",
-    "toposolve",
+    "toposolve>=0.1.17",
     "psutil",
     "torch-shampoo @ git+https://github.com/facebookresearch/optimizers.git@main",
     "liger-kernel",

diff --git a/uv.lock b/uv.lock