crystallm_mpts_52_large.yaml

out_dir: 'out/crystallm_mpts_52_large'
eval_interval: 250  # how often to evaluate against the validation set
eval_iters_train: 200
log_interval: 1  # how often to print to the console (1 = every iteration)

# whether to always save a checkpoint
always_save_checkpoint: True

# whether to validate with a validation set
validate: True

dataset: 'out/tokens_mpts_52'
gradient_accumulation_steps: 4
batch_size: 16
block_size: 2048

n_layer: 16
n_head: 16
n_embd: 1024
dropout: 0.1

learning_rate: 1e-3
decay_lr: True
max_iters: 3500
lr_decay_iters: 3500
min_lr: 1e-4
beta2: 0.99

warmup_iters: 100