Merge branch 'demo/emnlp' of https://github.com/zhudotexe/redel into …

…demo/emnlp
zhudotexe · Dec 11, 2024 · 3b50a13 · 3b50a13
2 parents 32b8475 + 521651e
commit 3b50a13
Show file tree

Hide file tree

Showing 140 changed files with 559 additions and 606 deletions.
diff --git a/README.md b/README.md
@@ -221,6 +221,9 @@ Apache-2.0 license, contained in `experiments/webarena/vendor/LICENSE`.
 First, you'll need to set up your own WebArena environment.
 See https://github.com/web-arena-x/webarena/blob/main/environment_docker/README.md for instructions.
 
+> [!NOTE]
+> https://github.com/gasse/webarena-setup/tree/main/webarena is a helpful unofficial setup script.
+
 Next, run the following to setup the webarena configuration:
 
 ```shell

diff --git a/slurm/.gitignore b/slurm/.gitignore
@@ -1,3 +1,2 @@
 env.sh
 fanout-eval-env.sh
-webarena-env.sh
diff --git a/slurm/claude/fanoutqa-1-full.sh b/slurm/claude/fanoutqa-1-full.sh
@@ -15,5 +15,4 @@
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-python bench_fanoutqa.py --config full --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/full 
+python bench_fanoutqa.py --config full --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/full
diff --git a/slurm/claude/fanoutqa-2-root-fc.sh b/slurm/claude/fanoutqa-2-root-fc.sh
@@ -15,5 +15,4 @@
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-python bench_fanoutqa.py --config root-fc --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/root-fc 
+python bench_fanoutqa.py --config root-fc --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/root-fc
diff --git a/slurm/claude/fanoutqa-3-baseline.sh b/slurm/claude/fanoutqa-3-baseline.sh
@@ -15,5 +15,4 @@
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-python bench_fanoutqa.py --config baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/baseline 
+python bench_fanoutqa.py --config baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/baseline
diff --git a/slurm/claude/fanoutqa-4-small-leaf.sh b/slurm/claude/fanoutqa-4-small-leaf.sh
@@ -15,5 +15,4 @@
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-python bench_fanoutqa.py --config small-leaf --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/small-leaf 
+python bench_fanoutqa.py --config small-leaf --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/small-leaf
diff --git a/slurm/claude/fanoutqa-5-small-all.sh b/slurm/claude/fanoutqa-5-small-all.sh
@@ -15,5 +15,4 @@
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-python bench_fanoutqa.py --config small-all --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/small-all 
+python bench_fanoutqa.py --config small-all --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/small-all
diff --git a/slurm/claude/fanoutqa-6-small-baseline.sh b/slurm/claude/fanoutqa-6-small-baseline.sh
@@ -15,5 +15,4 @@
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-python bench_fanoutqa.py --config small-baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/small-baseline 
+python bench_fanoutqa.py --config small-baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/small-baseline
diff --git a/slurm/claude/fanoutqa-7-short-context.sh b/slurm/claude/fanoutqa-7-short-context.sh
@@ -15,5 +15,4 @@
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-python bench_fanoutqa.py --config short-context --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/short-context 
+python bench_fanoutqa.py --config short-context --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/short-context
diff --git a/slurm/claude/fanoutqa-8-short-baseline.sh b/slurm/claude/fanoutqa-8-short-baseline.sh
@@ -15,5 +15,4 @@
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-python bench_fanoutqa.py --config short-baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/short-baseline 
+python bench_fanoutqa.py --config short-baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/short-baseline
diff --git a/slurm/claude/fanoutqa-all.sh b/slurm/claude/fanoutqa-all.sh
@@ -15,33 +15,11 @@
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-python bench_fanoutqa.py --config full --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/full 
-
-
-
-python bench_fanoutqa.py --config root-fc --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/root-fc 
-
-
-
-python bench_fanoutqa.py --config baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/baseline 
-
-
-
-python bench_fanoutqa.py --config small-leaf --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/small-leaf 
-
-
-
-python bench_fanoutqa.py --config small-all --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/small-all 
-
-
-
-python bench_fanoutqa.py --config small-baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/small-baseline 
-
-
-
-python bench_fanoutqa.py --config short-context --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/short-context 
-
-
-
-python bench_fanoutqa.py --config short-baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/short-baseline 
+python bench_fanoutqa.py --config full --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/full
+python bench_fanoutqa.py --config root-fc --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/root-fc
+python bench_fanoutqa.py --config baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/baseline
+python bench_fanoutqa.py --config small-leaf --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/small-leaf
+python bench_fanoutqa.py --config small-all --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/small-all
+python bench_fanoutqa.py --config small-baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/small-baseline
+python bench_fanoutqa.py --config short-context --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/short-context
+python bench_fanoutqa.py --config short-baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/fanoutqa/claude/short-baseline
diff --git a/slurm/claude/travelplanner-1-full.sh b/slurm/claude/travelplanner-1-full.sh
@@ -15,5 +15,4 @@
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-python bench_travelplanner.py --config full --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/full 
+python bench_travelplanner.py --config full --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/full
diff --git a/slurm/claude/travelplanner-2-root-fc.sh b/slurm/claude/travelplanner-2-root-fc.sh
@@ -15,5 +15,4 @@
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-python bench_travelplanner.py --config root-fc --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/root-fc 
+python bench_travelplanner.py --config root-fc --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/root-fc
diff --git a/slurm/claude/travelplanner-3-baseline.sh b/slurm/claude/travelplanner-3-baseline.sh
@@ -15,5 +15,4 @@
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-python bench_travelplanner.py --config baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/baseline 
+python bench_travelplanner.py --config baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/baseline
diff --git a/slurm/claude/travelplanner-4-small-leaf.sh b/slurm/claude/travelplanner-4-small-leaf.sh
@@ -15,5 +15,4 @@
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-python bench_travelplanner.py --config small-leaf --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/small-leaf 
+python bench_travelplanner.py --config small-leaf --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/small-leaf
diff --git a/slurm/claude/travelplanner-5-small-all.sh b/slurm/claude/travelplanner-5-small-all.sh
@@ -15,5 +15,4 @@
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-python bench_travelplanner.py --config small-all --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/small-all 
+python bench_travelplanner.py --config small-all --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/small-all
diff --git a/slurm/claude/travelplanner-6-small-baseline.sh b/slurm/claude/travelplanner-6-small-baseline.sh
@@ -15,5 +15,4 @@
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-python bench_travelplanner.py --config small-baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/small-baseline 
+python bench_travelplanner.py --config small-baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/small-baseline
diff --git a/slurm/claude/travelplanner-7-short-context.sh b/slurm/claude/travelplanner-7-short-context.sh
@@ -15,5 +15,4 @@
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-python bench_travelplanner.py --config short-context --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/short-context 
+python bench_travelplanner.py --config short-context --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/short-context
diff --git a/slurm/claude/travelplanner-8-short-baseline.sh b/slurm/claude/travelplanner-8-short-baseline.sh
@@ -15,5 +15,4 @@
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-python bench_travelplanner.py --config short-baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/short-baseline 
+python bench_travelplanner.py --config short-baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/short-baseline
diff --git a/slurm/claude/travelplanner-all.sh b/slurm/claude/travelplanner-all.sh
@@ -15,33 +15,11 @@
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
-
-python bench_travelplanner.py --config full --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/full 
-
-
-
-python bench_travelplanner.py --config root-fc --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/root-fc 
-
-
-
-python bench_travelplanner.py --config baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/baseline 
-
-
-
-python bench_travelplanner.py --config small-leaf --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/small-leaf 
-
-
-
-python bench_travelplanner.py --config small-all --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/small-all 
-
-
-
-python bench_travelplanner.py --config small-baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/small-baseline 
-
-
-
-python bench_travelplanner.py --config short-context --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/short-context 
-
-
-
-python bench_travelplanner.py --config short-baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/short-baseline 
+python bench_travelplanner.py --config full --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/full
+python bench_travelplanner.py --config root-fc --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/root-fc
+python bench_travelplanner.py --config baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/baseline
+python bench_travelplanner.py --config small-leaf --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/small-leaf
+python bench_travelplanner.py --config small-all --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/small-all
+python bench_travelplanner.py --config small-baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/small-baseline
+python bench_travelplanner.py --config short-context --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/short-context
+python bench_travelplanner.py --config short-baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/travelplanner/claude/short-baseline
diff --git a/slurm/claude/webarena-1-full.sh b/slurm/claude/webarena-1-full.sh
@@ -7,15 +7,22 @@
 #SBATCH --time=7-0
 #SBATCH --nodes=1
 #SBATCH -c 1
-#SBATCH --mem=32G
+#SBATCH --mem=256G
 #SBATCH --gpus=0
 #SBATCH [email protected]
 #SBATCH --mail-type=END,FAIL
+#SBATCH --nodelist=nlpgpu05
 
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
+dockerd-rootless.sh &
+DOCKER_PID=$!
+sleep 15
 source slurm/webarena-env.sh
+bash slurm/webarena-startup.sh
+sleep 600
 curl -X GET ${RESTART_URL}
-sleep 300
-python bench_webarena.py --config full --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/webarena/claude/full 
+sleep 600
+python bench_webarena.py --config full --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/webarena/claude/full
+kill $DOCKER_PID
diff --git a/slurm/claude/webarena-2-root-fc.sh b/slurm/claude/webarena-2-root-fc.sh
@@ -7,15 +7,22 @@
 #SBATCH --time=7-0
 #SBATCH --nodes=1
 #SBATCH -c 1
-#SBATCH --mem=32G
+#SBATCH --mem=256G
 #SBATCH --gpus=0
 #SBATCH [email protected]
 #SBATCH --mail-type=END,FAIL
+#SBATCH --nodelist=nlpgpu05
 
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
+dockerd-rootless.sh &
+DOCKER_PID=$!
+sleep 15
 source slurm/webarena-env.sh
+bash slurm/webarena-startup.sh
+sleep 600
 curl -X GET ${RESTART_URL}
-sleep 300
-python bench_webarena.py --config root-fc --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/webarena/claude/root-fc 
+sleep 600
+python bench_webarena.py --config root-fc --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/webarena/claude/root-fc
+kill $DOCKER_PID
diff --git a/slurm/claude/webarena-3-baseline.sh b/slurm/claude/webarena-3-baseline.sh
@@ -7,15 +7,22 @@
 #SBATCH --time=7-0
 #SBATCH --nodes=1
 #SBATCH -c 1
-#SBATCH --mem=32G
+#SBATCH --mem=256G
 #SBATCH --gpus=0
 #SBATCH [email protected]
 #SBATCH --mail-type=END,FAIL
+#SBATCH --nodelist=nlpgpu05
 
 
 source slurm/env.sh
 export VLLM_WORKER_MULTIPROC_METHOD=spawn
+dockerd-rootless.sh &
+DOCKER_PID=$!
+sleep 15
 source slurm/webarena-env.sh
+bash slurm/webarena-startup.sh
+sleep 600
 curl -X GET ${RESTART_URL}
-sleep 300
-python bench_webarena.py --config baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/webarena/claude/baseline 
+sleep 600
+python bench_webarena.py --config baseline --model-class claude --large-model claude-3-5-sonnet-20241022 --small-model claude-3-5-haiku-20241022 --save-dir /nlpgpu/data/andrz/redel/experiments/webarena/claude/baseline
+kill $DOCKER_PID