Merge pull request #9 from polarsignals/alfonso-wasm

*: allow building wal with GOARCH=wasm GOOS=wasip1

.github/workflows/go-tests.yml

@@ -15,7 +15,13 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v2
       with:
-        go-version: '1.20'
+        go-version: '1.22'
 
     - name: Test
       run: go test -v ./...
+
+    - name: Set up wasmtime
+      uses: bytecodealliance/actions/wasmtime/setup@v1
+
+    - name: Test WASM
+      run: PATH=$PATH:$(go env GOROOT)/misc/wasm GOOS=wasip1 GOARCH=wasm go test ./...

bench/bench_test.go

@@ -5,15 +5,15 @@ package main
 
 import (
 	"fmt"
-	"github.com/polarsignals/wal/types"
 	"os"
 	"path/filepath"
 	"testing"
 	"time"
 
-	"github.com/coreos/etcd/pkg/fileutil"
-	"github.com/polarsignals/wal"
 	"github.com/stretchr/testify/require"
+
+	"github.com/polarsignals/wal"
+	"github.com/polarsignals/wal/types"
 )
 
 func BenchmarkAppend(b *testing.B) {
@@ -137,44 +137,6 @@ func runGetLogBench(b *testing.B, ls wal.LogStore, n int) {
 	}
 }
 
-// These OS benchmarks showed that at least on my Mac Creating and preallocating
-// a file is not reliably quicker than renaming a file we already created and
-// preallocated so the extra work of doing that in the background ahead of time
-// and just renaming it during rotation seems unnecessary. We are not fsyncing
-// either the file or parent dir in either case which dominates cost of either
-// operation. Three random consecutive runs on my machine:
-//
-// BenchmarkOSCreateAndPreallocate-16           100            370304 ns/op             221 B/op          3 allocs/op
-// BenchmarkOSRename-16                         100            876001 ns/op             570 B/op          5 allocs/op
-//
-// BenchmarkOSCreateAndPreallocate-16           100            353654 ns/op             221 B/op          3 allocs/op
-// BenchmarkOSRename-16                         100            168558 ns/op             570 B/op          5 allocs/op
-//
-// BenchmarkOSCreateAndPreallocate-16           100            367360 ns/op             224 B/op          3 allocs/op
-// BenchmarkOSRename-16                         100           1353014 ns/op             571 B/op          5 allocs/op
-
-func BenchmarkOSCreateAndPreallocate(b *testing.B) {
-	tmpDir, err := os.MkdirTemp("", "raft-wal-bench-*")
-	require.NoError(b, err)
-	defer os.RemoveAll(tmpDir)
-
-	b.ResetTimer()
-	for i := 0; i < b.N; i++ {
-		fname := filepath.Join(tmpDir, fmt.Sprintf("test-%d.txt", i))
-		b.StartTimer()
-		f, err := os.OpenFile(fname, os.O_CREATE|os.O_EXCL|os.O_RDWR, os.FileMode(0644))
-		if err != nil {
-			panic(err) // require is kinda slow in benchmarks
-		}
-		err = fileutil.Preallocate(f, int64(64*1024*1024), true)
-		if err != nil {
-			panic(err)
-		}
-		b.StopTimer()
-		f.Close()
-	}
-}
-
 func BenchmarkOSRename(b *testing.B) {
 	tmpDir, err := os.MkdirTemp("", "raft-wal-bench-*")
 	require.NoError(b, err)

fs/fs.go

@@ -10,7 +10,6 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/coreos/etcd/pkg/fileutil"
 	"github.com/polarsignals/wal/types"
 )
 
@@ -60,7 +59,8 @@ func (fs *FS) Create(dir string, name string, size uint64) (types.WritableFile,
 		if size > math.MaxInt32 {
 			return nil, fmt.Errorf("maximum file size is %d bytes", math.MaxInt32)
 		}
-		if err := fileutil.Preallocate(f, int64(size), true); err != nil {
+
+		if err := prealloc(f, int64(size), true); err != nil {
 			f.Close()
 			return nil, err
 		}
@@ -113,16 +113,3 @@ func (fs *FS) OpenReader(dir string, name string) (types.ReadableFile, error) {
 func (fs *FS) OpenWriter(dir string, name string) (types.WritableFile, error) {
 	return os.OpenFile(filepath.Join(dir, name), os.O_RDWR, os.FileMode(0644))
 }
-
-func syncDir(dir string) error {
-	f, err := os.Open(dir)
-	if err != nil {
-		return err
-	}
-	err = f.Sync()
-	closeErr := f.Close()
-	if err != nil {
-		return err
-	}
-	return closeErr
-}

fs/fs_nowasm.go

@@ -0,0 +1,26 @@
+//go:build !wasm
+
+package fs
+
+import (
+	"os"
+
+	"github.com/coreos/etcd/pkg/fileutil"
+)
+
+func syncDir(dir string) error {
+	f, err := os.Open(dir)
+	if err != nil {
+		return err
+	}
+	err = f.Sync()
+	closeErr := f.Close()
+	if err != nil {
+		return err
+	}
+	return closeErr
+}
+
+func prealloc(f *os.File, sizeInBytes int64, extendFile bool) error {
+	return fileutil.Preallocate(f, sizeInBytes, extendFile)
+}

fs/fs_test.go

@@ -7,7 +7,7 @@ import (
 	"bytes"
 	"io"
 	"os"
-	"path/filepath"
+	"strings"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -30,12 +30,6 @@ func TestFS(t *testing.T) {
 	require.NoError(t, err)
 	defer wf.Close()
 
-	// Should be pre-allocated (on supported file systems).
-	// TODO work out if this is reliable in CI or if we can detect supported FSs?)
-	info, err := os.Stat(filepath.Join(tmpDir, "00001-abcd1234.wal"))
-	require.NoError(t, err)
-	require.Equal(t, int64(512*1024), info.Size())
-
 	// Should be able to write data in any order
 	n, err := wf.WriteAt(bytes.Repeat([]byte{'2'}, 1024), 1024)
 	require.NoError(t, err)
@@ -121,17 +115,17 @@ func TestRealFSNoDir(t *testing.T) {
 
 	_, err := fs.ListDir("/not-a-real-dir")
 	require.Error(t, err)
-	require.Contains(t, err.Error(), "no such file or directory")
+	require.Contains(t, strings.ToLower(err.Error()), "no such file or directory")
 
 	_, err = fs.Create("/not-a-real-dir", "foo", 1024)
 	require.Error(t, err)
-	require.Contains(t, err.Error(), "no such file or directory")
+	require.Contains(t, strings.ToLower(err.Error()), "no such file or directory")
 
 	_, err = fs.OpenReader("/not-a-real-dir", "foo")
 	require.Error(t, err)
-	require.Contains(t, err.Error(), "no such file or directory")
+	require.Contains(t, strings.ToLower(err.Error()), "no such file or directory")
 
 	_, err = fs.OpenWriter("/not-a-real-dir", "foo")
 	require.Error(t, err)
-	require.Contains(t, err.Error(), "no such file or directory")
+	require.Contains(t, strings.ToLower(err.Error()), "no such file or directory")
 }

fs/fs_wasm.go

@@ -0,0 +1,16 @@
+//go:build wasm
+
+package fs
+
+import "os"
+
+func syncDir(dir string) error {
+	// TODO(asubiotto): Issue syncing dirs on wasm.
+	return nil
+}
+
+func prealloc(_ *os.File, _ int64, _ bool) error {
+	// fileutil.Prealloc relies on unix-only functionality which is unavailable
+	// on wasm.
+	return nil
+}

metadb/metadb.go

@@ -1,3 +1,5 @@
+//go:build !wasm
+
 // Copyright (c) HashiCorp, Inc
 // SPDX-License-Identifier: MPL-2.0
 
@@ -10,8 +12,9 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/polarsignals/wal/types"
 	"go.etcd.io/bbolt"
+
+	"github.com/polarsignals/wal/types"
 )
 
 const (

metadb/metadb_test.go

@@ -6,11 +6,13 @@ package metadb
 import (
 	"io/ioutil"
 	"os"
+	"strings"
 	"testing"
 	"time"
 
-	"github.com/polarsignals/wal/types"
 	"github.com/stretchr/testify/require"
+
+	"github.com/polarsignals/wal/types"
 )
 
 func TestMetaDB(t *testing.T) {
@@ -86,7 +88,7 @@ func TestMetaDBErrors(t *testing.T) {
 	// Loading from a non-existent dir is an error
 	var db2 BoltMetaDB
 	_, err = db2.Load("fake-dir-that-does-not-exist")
-	require.ErrorContains(t, err, "no such file or directory")
+	require.True(t, strings.Contains(strings.ToLower(err.Error()), "no such file or directory"))
 }
 
 func makeState(nSegs int) *types.PersistentState {