Add first-party support for & setup tests against zsh and dash

.github/workflows/test.yml

@@ -19,7 +19,7 @@ jobs:
           uname -a
           compgen -ac | sort | grep '^ex'
 
-      - run: make test
+      - run: make tests_excluding_ash
 
   lint:
     runs-on: ubuntu-latest
@@ -37,7 +37,7 @@ jobs:
           version: "2024.1.1"
           install-go: false
 
-  test-alpine:
+  test-all-shells:
     runs-on: ubuntu-latest
     steps:
       - name: Setup Alpine
@@ -48,20 +48,60 @@ jobs:
       - name: Setup Alpine dependencies
         run: |
           cat /etc/alpine-release
-          apk add git go make bash binutils-gold python3
+          apk add git go make bash dash zsh binutils-gold python3
         shell: alpine.sh --root {0}
 
       - name: Checkout
         uses: actions/checkout@v2
 
-      - name: Run tests against ash
+      - name: Setup git config
         run: |
           git config --global --add safe.directory /home/runner/work/shell-tester/shell-tester 
           # repo is owned by user and we intend to execute as root
+        shell: alpine.sh --root {0}
+
+      - name: Run tests against all shells on alpine
+        run: |
           make test_ash
+          make test_bash
+          make test_dash
+        shell: alpine.sh --root {0}
+
+      - name: setup-ubuntu-env
+        run: |
+          sudo apt install zsh
+          sudo apt remove bpfcc-tools
+          zsh --version
+
+      - name: Run tests against all shells on alpine
+        run: |
+          sudo make test_zsh
+        shell: bash
+
+  test-alpine:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Setup Alpine
+        uses: jirutka/setup-alpine@v1
+        with:
+          branch: latest-stable
+
+      - name: Setup Alpine dependencies
+        run: |
+          cat /etc/alpine-release
+          apk add git go make bash dash zsh binutils-gold python3
+        shell: alpine.sh --root {0}
+
+      - name: Checkout
+        uses: actions/checkout@v2
+
+      - name: Setup git config
+        run: |
+          git config --global --add safe.directory /home/runner/work/shell-tester/shell-tester 
+          # repo is owned by user and we intend to execute as root
         shell: alpine.sh --root {0}
 
-      - name: Run tests against bash on alpine
+      - name: Run tests against bash & ash on alpine
         run: |
           make test
         shell: alpine.sh --root {0}

Makefile

@@ -17,6 +17,10 @@ build:
 test:
 	TESTER_DIR=$(shell pwd) go test -count=1 -p 1 -v ./internal/...
 
+# Include bash but not ash
+tests_excluding_ash:
+	TESTER_DIR=$(shell pwd) go test -count=1 -p 1 -v -skip "TestStages/.*[^b]ash" ./internal/...
+
 test_ls_against_bsd_ls:
 	TESTER_DIR=$(shell pwd) go test -count=1 -p 1 -v ./internal/custom_executable/ls/... -system
 
@@ -58,55 +62,80 @@ build_executables:
 		done; \
 	done
 
-BASE_STAGES = [ \
-	{\"slug\":\"oo8\",\"tester_log_prefix\":\"tester::\#oo8\",\"title\":\"Stage\#1: Init\"}, \
-	{\"slug\":\"cz2\",\"tester_log_prefix\":\"tester::\#cz2\",\"title\":\"Stage\#2: Invalid Command\"}, \
-	{\"slug\":\"ff0\",\"tester_log_prefix\":\"tester::\#ff0\",\"title\":\"Stage\#3: REPL\"}, \
-	{\"slug\":\"pn5\",\"tester_log_prefix\":\"tester::\#pn5\",\"title\":\"Stage\#4: Exit\"}, \
-	{\"slug\":\"iz3\",\"tester_log_prefix\":\"tester::\#iz3\",\"title\":\"Stage\#5: Echo\"}, \
-	{\"slug\":\"ez5\",\"tester_log_prefix\":\"tester::\#ez5\",\"title\":\"Stage\#6: Type built-in\"}, \
-	{\"slug\":\"mg5\",\"tester_log_prefix\":\"tester::\#mg5\",\"title\":\"Stage\#7: Type for executables\"}, \
-	{\"slug\":\"ip1\",\"tester_log_prefix\":\"tester::\#ip1\",\"title\":\"Stage\#8: Run a program\"} \
+define _BASE_STAGES
+[ \
+	{"slug":"oo8","tester_log_prefix":"tester::#oo8","title":"Stage#1: Init"}, \
+	{"slug":"cz2","tester_log_prefix":"tester::#cz2","title":"Stage#2: Invalid Command"}, \
+	{"slug":"ff0","tester_log_prefix":"tester::#ff0","title":"Stage#3: REPL"}, \
+	{"slug":"pn5","tester_log_prefix":"tester::#pn5","title":"Stage#4: Exit"}, \
+	{"slug":"iz3","tester_log_prefix":"tester::#iz3","title":"Stage#5: Echo"}, \
+	{"slug":"ez5","tester_log_prefix":"tester::#ez5","title":"Stage#6: Type built-in"}, \
+	{"slug":"mg5","tester_log_prefix":"tester::#mg5","title":"Stage#7: Type for executables"}, \
+	{"slug":"ip1","tester_log_prefix":"tester::#ip1","title":"Stage#8: Run a program"} \
 ]
+endef
 
-NAVIGATION_STAGES = [ \
-	{\"slug\":\"ei0\",\"tester_log_prefix\":\"tester::\#ei0\",\"title\":\"Stage\#9: PWD\"}, \
-	{\"slug\":\"ra6\",\"tester_log_prefix\":\"tester::\#ra6\",\"title\":\"Stage\#10: CD-1\"}, \
-	{\"slug\":\"gq9\",\"tester_log_prefix\":\"tester::\#gq9\",\"title\":\"Stage\#11: CD-2\"}, \
-	{\"slug\":\"gp4\",\"tester_log_prefix\":\"tester::\#gp4\",\"title\":\"Stage\#12: CD-3\"} \
+define _NAVIGATION_STAGES
+[ \
+	{"slug":"ei0","tester_log_prefix":"tester::#ei0","title":"Stage#9: PWD"}, \
+	{"slug":"ra6","tester_log_prefix":"tester::#ra6","title":"Stage#10: CD-1"}, \
+	{"slug":"gq9","tester_log_prefix":"tester::#gq9","title":"Stage#11: CD-2"}, \
+	{"slug":"gp4","tester_log_prefix":"tester::#gp4","title":"Stage#12: CD-3"} \
 ]
+endef
 
-QUOTING_STAGES = [ \
-	{\"slug\":\"ni6\",\"tester_log_prefix\":\"tester::\#ni6\",\"title\":\"Stage\#13: Quoting with single quotes\"}, \
-	{\"slug\":\"tg6\",\"tester_log_prefix\":\"tester::\#tg6\",\"title\":\"Stage\#14: Quoting with double quotes\"}, \
-	{\"slug\":\"yt5\",\"tester_log_prefix\":\"tester::\#yt5\",\"title\":\"Stage\#15: Quoting with backslashes\"}, \
-	{\"slug\":\"le5\",\"tester_log_prefix\":\"tester::\#le5\",\"title\":\"Stage\#16: Quoting with single and double quotes\"}, \
-	{\"slug\":\"gu3\",\"tester_log_prefix\":\"tester::\#gu3\",\"title\":\"Stage\#17: Quoting with mixed quotes\"}, \
-	{\"slug\":\"qj0\",\"tester_log_prefix\":\"tester::\#qj0\",\"title\":\"Stage\#18: Quoting program names\"} \
+define _QUOTING_STAGES
+[ \
+	{"slug":"ni6","tester_log_prefix":"tester::#ni6","title":"Stage#13: Quoting with single quotes"}, \
+	{"slug":"tg6","tester_log_prefix":"tester::#tg6","title":"Stage#14: Quoting with double quotes"}, \
+	{"slug":"yt5","tester_log_prefix":"tester::#yt5","title":"Stage#15: Quoting with backslashes"}, \
+	{"slug":"le5","tester_log_prefix":"tester::#le5","title":"Stage#16: Quoting with single and double quotes"}, \
+	{"slug":"gu3","tester_log_prefix":"tester::#gu3","title":"Stage#17: Quoting with mixed quotes"}, \
+	{"slug":"qj0","tester_log_prefix":"tester::#qj0","title":"Stage#18: Quoting program names"} \
 ]
+endef
 
-REDIRECTIONS_STAGES = [ \
-	{\"slug\":\"jv1\",\"tester_log_prefix\":\"tester::\#jv1\",\"title\":\"Stage\#19: Redirect stdout\"}, \
-	{\"slug\":\"vz4\",\"tester_log_prefix\":\"tester::\#vz4\",\"title\":\"Stage\#20: Redirect stderr\"}, \
-	{\"slug\":\"el9\",\"tester_log_prefix\":\"tester::\#el9\",\"title\":\"Stage\#21: Append stdout\"}, \
-	{\"slug\":\"un3\",\"tester_log_prefix\":\"tester::\#un3\",\"title\":\"Stage\#22: Append stderr\"} \
+define _REDIRECTIONS_STAGES
+[ \
+	{"slug":"jv1","tester_log_prefix":"tester::#jv1","title":"Stage#19: Redirect stdout"}, \
+	{"slug":"vz4","tester_log_prefix":"tester::#vz4","title":"Stage#20: Redirect stderr"}, \
+	{"slug":"el9","tester_log_prefix":"tester::#el9","title":"Stage#21: Append stdout"}, \
+	{"slug":"un3","tester_log_prefix":"tester::#un3","title":"Stage#22: Append stderr"} \
 ]
+endef
 
-COMPLETIONS_STAGES = [ \
-	{\"slug\":\"qp2\",\"tester_log_prefix\":\"tester::\#qp2\",\"title\":\"Stage\#1: builtins completion\"}, \
-	{\"slug\":\"gm9\",\"tester_log_prefix\":\"tester::\#gm9\",\"title\":\"Stage\#2: completion with args\"}, \
-	{\"slug\":\"qm8\",\"tester_log_prefix\":\"tester::\#qm8\",\"title\":\"Stage\#3: completion with invalid command\"}, \
-	{\"slug\":\"gy5\",\"tester_log_prefix\":\"tester::\#gy5\",\"title\":\"Stage\#4: completion with valid command\"}, \
-	{\"slug\":\"wh6\",\"tester_log_prefix\":\"tester::\#wh6\",\"title\":\"Stage\#5: completion with multiple executables\"}, \
-	{\"slug\":\"wt6\",\"tester_log_prefix\":\"tester::\#wt6\",\"title\":\"Stage\#6: partial completions\"} \
+define _COMPLETION_STAGES_BASE
+[ \
+  {"slug":"qp2","tester_log_prefix":"tester::#qp2","title":"Stage#1: builtins completion"}, \
+  {"slug":"qm8","tester_log_prefix":"tester::#qm8","title":"Stage#3: completion with invalid command"}, \
+  {"slug":"gy5","tester_log_prefix":"tester::#gy5","title":"Stage#4: valid command"}, \
+  {"slug":"wt6","tester_log_prefix":"tester::#wt6","title":"Stage#6: partial completions"} \
 ]
+endef
+
+define _COMPLETIONS_STAGES_COMPLEX
+  {"slug":"gm9","tester_log_prefix":"tester::#gm9","title":"Stage#2: completion with args"}, \
+  {"slug":"wh6","tester_log_prefix":"tester::#wh6","title":"Stage#5: completion with multiple executables"}
+endef
+
+# Use eval to properly escape the stage arrays
+define quote_strings
+	$(shell echo '$(1)' | sed 's/"/\\"/g')
+endef
 
 define run_test
 	CODECRAFTERS_REPOSITORY_DIR=./internal/test_helpers/$(2) \
 	CODECRAFTERS_TEST_CASES_JSON="$(1)" \
 	dist/main.out
 endef
 
+BASE_STAGES = $(call quote_strings,$(_BASE_STAGES))
+NAVIGATION_STAGES = $(call quote_strings,$(_NAVIGATION_STAGES))
+QUOTING_STAGES = $(call quote_strings,$(_QUOTING_STAGES))
+REDIRECTIONS_STAGES = $(call quote_strings,$(_REDIRECTIONS_STAGES))
+COMPLETIONS_STAGES_ZSH = $(call quote_strings,$(_COMPLETION_STAGES_BASE))
+COMPLETIONS_STAGES = $(shell echo '$(_COMPLETION_STAGES_BASE)' | sed 's/]$$/, $(_COMPLETIONS_STAGES_COMPLEX)]/' | sed 's/"/\\"/g')
+
 test_base_w_ash: build
 	$(call run_test,$(BASE_STAGES),ash)
 
@@ -162,7 +191,7 @@ test_redirections_w_zsh: build
 	$(call run_test,$(REDIRECTIONS_STAGES),zsh)
 
 test_completions_w_zsh: build
-	$(call run_test,$(COMPLETIONS_STAGES),zsh)
+	$(call run_test,$(COMPLETIONS_STAGES_ZSH),zsh)
 
 test_ash:
 	make test_base_w_ash

internal/stage10.go

@@ -39,6 +39,8 @@ func testCd1(stageHarness *test_case_harness.TestCaseHarness) error {
 		FallbackPatterns: []*regexp.Regexp{
 			regexp.MustCompile(fmt.Sprintf(`^(can't cd to %s|((bash: )?cd: )?%s: No such file or directory)$`, directory, directory)),
 			regexp.MustCompile(`^ash: cd: can't cd to ` + directory + `: No such file or directory$`),
+			regexp.MustCompile(`^cd: no such file or directory: ` + directory + `$`),
+			regexp.MustCompile(`^dash: [0-9]+: cd: can't cd to ` + directory + `$`),
 		},
 		SuccessMessage: "✓ Received error message",
 	}

internal/stage_q4.go

@@ -3,6 +3,7 @@ package internal
 import (
 	"fmt"
 	"path"
+	"regexp"
 	"strings"
 
 	"github.com/codecrafters-io/shell-tester/internal/logged_shell_asserter"
@@ -48,25 +49,33 @@ func testQ4(stageHarness *test_case_harness.TestCaseHarness) error {
 		return err
 	}
 
+	// TODO: Simplify test case creation
 	inputs := []string{
-		fmt.Sprintf(`echo '%s\\\n%s'`, L[0], L[1]),
+		fmt.Sprintf(`echo '%s\\n%s'`, L[0], L[1]),
 		fmt.Sprintf(`echo '%s\"%s%s\"%s'`, L[2], L[3], L[4], L[0]),
 		fmt.Sprintf(`echo '%s\\n%s'`, L[4], L[1]),
 		fmt.Sprintf(`%s "%s" "%s" "%s"`, CUSTOM_CAT_COMMAND, filePaths[0], filePaths[1], filePaths[2]),
 	}
 	expectedOutputs := []string{
-		fmt.Sprintf(`%s\\\n%s`, L[0], L[1]),
+		fmt.Sprintf(`%s\\n%s`, L[0], L[1]),
 		fmt.Sprintf(`%s\"%s%s\"%s`, L[2], L[3], L[4], L[0]),
 		fmt.Sprintf(`%s\\n%s`, L[4], L[1]),
 		fileContents[0] + fileContents[1] + strings.TrimRight(fileContents[2], "\n"),
 	}
-	testCaseContents := newTestCaseContents(inputs, expectedOutputs)
+	fallbackPatterns := [][]*regexp.Regexp{
+		{regexp.MustCompile(fmt.Sprintf(`%s\\n%s`, L[0], L[1]))},
+		nil,
+		{regexp.MustCompile(fmt.Sprintf(`%s\\n%s`, L[4], L[1]))},
+		nil,
+	}
+
+	testCaseContents := newTestCaseContentsWithFallbackPatterns(inputs, expectedOutputs, fallbackPatterns)
 
 	for _, testCaseContent := range testCaseContents[:3] {
 		testCase := test_cases.CommandResponseTestCase{
 			Command:          testCaseContent.Input,
 			ExpectedOutput:   testCaseContent.ExpectedOutput,
-			FallbackPatterns: nil,
+			FallbackPatterns: testCaseContent.FallbackPatterns,
 			SuccessMessage:   "✓ Received expected response",
 		}
 		if err := testCase.Run(asserter, shell, logger); err != nil {

internal/stage_q5.go

@@ -49,13 +49,13 @@ func testQ5(stageHarness *test_case_harness.TestCaseHarness) error {
 
 	L := random.RandomElementsFromArray(LARGE_WORDS, 5)
 	inputs := []string{
-		fmt.Sprintf(`echo "%s'%s'\\n'%s"`, L[0], L[1], L[2]),
+		fmt.Sprintf(`echo "%s'%s'\\'%s"`, L[0], L[1], L[2]),
 		fmt.Sprintf(`echo "%s\"insidequotes"%s\"`, L[0], L[1]),
 		fmt.Sprintf(`echo "mixed\"quote'%s'\\"`, L[4]),
 		fmt.Sprintf(`%s '%s' '%s' '%s'`, CUSTOM_CAT_COMMAND, filePaths[0], filePaths[1], filePaths[2]),
 	}
 	expectedOutputs := []string{
-		fmt.Sprintf(`%s'%s'\n'%s`, L[0], L[1], L[2]),
+		fmt.Sprintf(`%s'%s'\'%s`, L[0], L[1], L[2]),
 		fmt.Sprintf(`%s"insidequotes%s"`, L[0], L[1]),
 		fmt.Sprintf(`mixed"quote'%s'\`, L[4]),
 		fileContents[0] + fileContents[1] + strings.TrimRight(fileContents[2], "\n"),

internal/stages_test.go

@@ -12,20 +12,6 @@ func TestStages(t *testing.T) {
 	os.Setenv("CODECRAFTERS_RANDOM_SEED", "1234567890")
 
 	testCases := map[string]testerUtilsTesting.TesterOutputTestCase{
-		"base_stages_pass": {
-			UntilStageSlug:      "ip1",
-			CodePath:            "./test_helpers/bash",
-			ExpectedExitCode:    0,
-			StdoutFixturePath:   "./test_helpers/fixtures/base/pass",
-			NormalizeOutputFunc: normalizeTesterOutput,
-		},
-		"navigation_pass": {
-			UntilStageSlug:      "gp4",
-			CodePath:            "./test_helpers/bash",
-			ExpectedExitCode:    0,
-			StdoutFixturePath:   "./test_helpers/fixtures/navigation/pass",
-			NormalizeOutputFunc: normalizeTesterOutput,
-		},
 		"missing_command_fail": {
 			UntilStageSlug:      "cz2",
 			CodePath:            "./test_helpers/scenarios/wrong_output",
@@ -47,25 +33,76 @@ func TestStages(t *testing.T) {
 			StdoutFixturePath:   "./test_helpers/fixtures/escape_codes",
 			NormalizeOutputFunc: normalizeTesterOutput,
 		},
-		"quoting_pass": {
+		"base_pass_bash": {
+			UntilStageSlug:      "ip1",
+			CodePath:            "./test_helpers/bash",
+			ExpectedExitCode:    0,
+			StdoutFixturePath:   "./test_helpers/fixtures/bash/base/pass",
+			NormalizeOutputFunc: normalizeTesterOutput,
+		},
+		"navigation_pass_bash": {
+			UntilStageSlug:      "gp4",
+			CodePath:            "./test_helpers/bash",
+			ExpectedExitCode:    0,
+			StdoutFixturePath:   "./test_helpers/fixtures/bash/navigation/pass",
+			NormalizeOutputFunc: normalizeTesterOutput,
+		},
+		"quoting_pass_bash": {
 			StageSlugs:          []string{"ni6", "tg6", "yt5", "le5", "gu3", "qj0"},
 			CodePath:            "./test_helpers/bash",
 			ExpectedExitCode:    0,
-			StdoutFixturePath:   "./test_helpers/fixtures/quoting/pass",
+			StdoutFixturePath:   "./test_helpers/fixtures/bash/quoting/pass",
 			NormalizeOutputFunc: normalizeTesterOutput,
 		},
-		"redirection_pass": {
+		"redirection_pass_bash": {
 			StageSlugs:          []string{"jv1", "vz4", "el9", "un3"},
 			CodePath:            "./test_helpers/bash",
 			ExpectedExitCode:    0,
-			StdoutFixturePath:   "./test_helpers/fixtures/redirection/pass",
+			StdoutFixturePath:   "./test_helpers/fixtures/bash/redirection/pass",
 			NormalizeOutputFunc: normalizeTesterOutput,
 		},
-		"autocomplete_pass": {
+		"completions_pass_bash": {
 			StageSlugs:          []string{"qp2", "gm9", "qm8", "gy5", "wh6", "wt6"},
 			CodePath:            "./test_helpers/bash",
 			ExpectedExitCode:    0,
-			StdoutFixturePath:   "./test_helpers/fixtures/autocomplete/pass",
+			StdoutFixturePath:   "./test_helpers/fixtures/bash/completions/pass",
+			NormalizeOutputFunc: normalizeTesterOutput,
+		},
+		"base_pass_ash": {
+			UntilStageSlug:      "ip1",
+			CodePath:            "./test_helpers/ash",
+			ExpectedExitCode:    0,
+			StdoutFixturePath:   "./test_helpers/fixtures/ash/base/pass",
+			NormalizeOutputFunc: normalizeTesterOutput,
+		},
+		"navigation_pass_ash": {
+			UntilStageSlug:      "gp4",
+			CodePath:            "./test_helpers/ash",
+			ExpectedExitCode:    0,
+			StdoutFixturePath:   "./test_helpers/fixtures/ash/navigation/pass",
+			NormalizeOutputFunc: normalizeTesterOutput,
+		},
+		"quoting_pass_ash": {
+			StageSlugs:          []string{"ni6", "tg6", "yt5", "le5", "gu3", "qj0"},
+			CodePath:            "./test_helpers/ash",
+			ExpectedExitCode:    0,
+			StdoutFixturePath:   "./test_helpers/fixtures/ash/quoting/pass",
+			NormalizeOutputFunc: normalizeTesterOutput,
+		},
+		"redirection_pass_ash": {
+			StageSlugs:          []string{"jv1", "vz4", "el9", "un3"},
+			CodePath:            "./test_helpers/ash",
+			ExpectedExitCode:    0,
+			StdoutFixturePath:   "./test_helpers/fixtures/ash/redirection/pass",
+			NormalizeOutputFunc: normalizeTesterOutput,
+		},
+		"completions_pass_ash": {
+			// TODO debug why this stage fails in make test ?
+			// "gy5"
+			StageSlugs:          []string{"qp2", "gm9", "qm8", "wh6", "wt6"},
+			CodePath:            "./test_helpers/ash",
+			ExpectedExitCode:    0,
+			StdoutFixturePath:   "./test_helpers/fixtures/ash/completions/pass",
 			NormalizeOutputFunc: normalizeTesterOutput,
 		},
 	}

internal/test_cases/invalid_command_test_case.go

@@ -58,5 +58,7 @@ func (t *InvalidCommandTestCase) getFallbackPatterns() []*regexp.Regexp {
 	return []*regexp.Regexp{
 		regexp.MustCompile(`^(bash: )?` + t.Command + `: (command )?not found$`),
 		regexp.MustCompile(`^ash: ` + t.Command + `: not found$`),
+		regexp.MustCompile(`^zsh: command not found: ` + t.Command + `$`),
+		regexp.MustCompile(`^dash: [0-9]+: ` + t.Command + `: not found$`),
 	}
 }