feat(kurtosis-devnet): user devnets

kurtosis-devnet/.gitignore

@@ -1 +1 @@
-*.json
+*-user.json

kurtosis-devnet/cmd/main.go

@@ -199,6 +199,7 @@ func (m *Main) renderTemplate(dir string) (*bytes.Buffer, error) {
 		m.localDockerImageOption(),
 		m.localContractArtifactsOption(dir),
 		m.localPrestateOption(dir),
+		tmpl.WithBaseDir(m.cfg.baseDir),
 	}
 
 	// Read and parse the data file if provided

kurtosis-devnet/foo-user.example.json

@@ -0,0 +1,16 @@
+{
+    "interop": true,
+    "l2s": {
+        "2151908": {
+            "nodes": ["op-geth", "op-geth"]
+        },
+        "2151909": {
+            "nodes": ["op-reth"]
+        }
+    },
+    "overrides": {
+        "flags": {
+            "log_level": "--log.level=debug"
+        }
+    }
+}

kurtosis-devnet/interop.yaml

@@ -7,7 +7,7 @@
   "op_supervisor" (localDockerImage "op-supervisor")
 -}}
 {{- $urls := dict
-  "prestate" "http://fileserver/proofs/op-program/cannon"
+  "prestate" (localPrestate.URL)
   "l1_artifacts" (localContractArtifacts "l1")
   "l2_artifacts" (localContractArtifacts "l2")
 -}}

kurtosis-devnet/justfile

@@ -41,13 +41,23 @@ op-wheel-image TAG='op-wheel:devnet': (_docker_build_stack TAG "op-wheel-target"
 KURTOSIS_PACKAGE := "github.com/ethpandaops/optimism-package"
 
 # Devnet template recipe
-devnet TEMPLATE_FILE DATA_FILE="":
+devnet TEMPLATE_FILE DATA_FILE="" NAME="":
+    #!/usr/bin/env bash
+    export DEVNET_NAME={{NAME}}
+    if [ -z "{{NAME}}" ]; then
+        export DEVNET_NAME=`basename {{TEMPLATE_FILE}} .yaml`
+        if [ -n "{{DATA_FILE}}" ]; then
+            export DATA_FILE_NAME=`basename {{DATA_FILE}} .json`
+            export DEVNET_NAME="$DEVNET_NAME-$DATA_FILE_NAME"
+        fi
+    fi
+    export ENCL_NAME="$DEVNET_NAME"-devnet
     go run cmd/main.go -kurtosis-package {{KURTOSIS_PACKAGE}} \
-        -environment tests/`basename {{TEMPLATE_FILE}} .yaml`-devnet.json \
+        -environment "tests/$ENCL_NAME.json" \
         -template "{{TEMPLATE_FILE}}" \
         -data "{{DATA_FILE}}" \
-        -enclave `basename {{TEMPLATE_FILE}} .yaml`-devnet
-    cat tests/`basename {{TEMPLATE_FILE}} .yaml`-devnet.json
+        -enclave "$ENCL_NAME" \
+    && cat "tests/$ENCL_NAME.json"
 
 devnet-test DEVNET *TEST:
     #!/usr/bin/env bash
@@ -69,3 +79,7 @@ simple-devnet: (devnet "simple.yaml")
 # Interop devnet
 interop-devnet: (devnet "interop.yaml")
 interop-devnet-test: (devnet-test "interop-devnet" "interop-smoke-test.sh")
+
+# User devnet
+user-devnet DATA_FILE:
+    {{just_executable()}} devnet "user.yaml" {{DATA_FILE}} {{file_stem(DATA_FILE)}}

kurtosis-devnet/pkg/tmpl/cmd/main.go

@@ -1,18 +1,20 @@
 package main
 
 import (
+	"encoding/json"
 	"flag"
 	"fmt"
 	"os"
 	"strings"
 
+	"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/tmpl"
 	"github.com/ethereum-optimism/optimism/kurtosis-devnet/pkg/tmpl/fake"
 )
 
 func main() {
-
 	// Parse command line flags
 	templateFile := flag.String("template", "", "Path to template file")
+	dataFile := flag.String("data", "", "Optional JSON data file")
 	flag.Parse()
 
 	if *templateFile == "" {
@@ -42,6 +44,23 @@ func main() {
 	// Create template context
 	ctx := fake.NewFakeTemplateContext(enclave)
 
+	// Load data file if provided
+	if *dataFile != "" {
+		dataBytes, err := os.ReadFile(*dataFile)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "Error reading data file: %v\n", err)
+			os.Exit(1)
+		}
+
+		var data interface{}
+		if err := json.Unmarshal(dataBytes, &data); err != nil {
+			fmt.Fprintf(os.Stderr, "Error parsing data file as JSON: %v\n", err)
+			os.Exit(1)
+		}
+
+		tmpl.WithData(data)(ctx)
+	}
+
 	// Process template and write to stdout
 	if err := ctx.InstantiateTemplate(f, os.Stdout); err != nil {
 		fmt.Fprintf(os.Stderr, "Error processing template: %v\n", err)

kurtosis-devnet/pkg/tmpl/tmpl.go

@@ -1,24 +1,37 @@
 package tmpl
 
 import (
+	"bytes"
+	"encoding/json"
 	"fmt"
 	"io"
+	"os"
+	"path/filepath"
 	"text/template"
 
 	sprig "github.com/go-task/slim-sprig/v3"
+	"gopkg.in/yaml.v3"
 )
 
 // TemplateFunc represents a function that can be used in templates
 type TemplateFunc any
 
 // TemplateContext contains data and functions to be passed to templates
 type TemplateContext struct {
-	Data      interface{}
-	Functions map[string]TemplateFunc
+	baseDir      string
+	Data         interface{}
+	Functions    map[string]TemplateFunc
+	includeStack []string // Track stack of included files to detect circular includes
 }
 
 type TemplateContextOptions func(*TemplateContext)
 
+func WithBaseDir(basedir string) TemplateContextOptions {
+	return func(ctx *TemplateContext) {
+		ctx.baseDir = basedir
+	}
+}
+
 func WithFunction(name string, fn TemplateFunc) TemplateContextOptions {
 	return func(ctx *TemplateContext) {
 		ctx.Functions[name] = fn
@@ -34,7 +47,9 @@ func WithData(data interface{}) TemplateContextOptions {
 // NewTemplateContext creates a new TemplateContext with default functions
 func NewTemplateContext(opts ...TemplateContextOptions) *TemplateContext {
 	ctx := &TemplateContext{
-		Functions: make(map[string]TemplateFunc),
+		baseDir:      ".",
+		Functions:    make(map[string]TemplateFunc),
+		includeStack: make([]string, 0),
 	}
 
 	for _, opt := range opts {
@@ -44,6 +59,73 @@ func NewTemplateContext(opts ...TemplateContextOptions) *TemplateContext {
 	return ctx
 }
 
+// includeFile reads and processes a template file relative to the given context's baseDir,
+// parses the content as YAML, and returns its JSON representation.
+// We use JSON because it can be inlined without worrying about indentation, while remaining valid YAML.
+// Note: to protect against infinite recursion, we check for circular includes.
+func (ctx *TemplateContext) includeFile(fname string, data ...interface{}) (string, error) {
+	// Resolve the file path relative to baseDir
+	path := filepath.Join(ctx.baseDir, fname)
+
+	// Check for circular includes
+	absPath, err := filepath.Abs(path)
+	if err != nil {
+		return "", fmt.Errorf("error resolving absolute path: %w", err)
+	}
+	for _, includedFile := range ctx.includeStack {
+		if includedFile == absPath {
+			return "", fmt.Errorf("circular include detected for file %s", fname)
+		}
+	}
+
+	// Read the included file
+	file, err := os.Open(path)
+	if err != nil {
+		return "", fmt.Errorf("error opening include file: %w", err)
+	}
+	defer file.Close()
+
+	// Create buffer for output
+	var buf bytes.Buffer
+
+	var tplData interface{}
+	switch len(data) {
+	case 0:
+		tplData = nil
+	case 1:
+		tplData = data[0]
+	default:
+		return "", fmt.Errorf("invalid number of arguments for includeFile: %d", len(data))
+	}
+
+	// Create new context with updated baseDir and include stack
+	includeCtx := &TemplateContext{
+		baseDir:      filepath.Dir(path),
+		Data:         tplData,
+		Functions:    ctx.Functions,
+		includeStack: append(append([]string{}, ctx.includeStack...), absPath),
+	}
+
+	// Process the included template
+	if err := includeCtx.InstantiateTemplate(file, &buf); err != nil {
+		return "", fmt.Errorf("error processing include file: %w", err)
+	}
+
+	// Parse the buffer content as YAML
+	var yamlData interface{}
+	if err := yaml.Unmarshal(buf.Bytes(), &yamlData); err != nil {
+		return "", fmt.Errorf("error parsing YAML: %w", err)
+	}
+
+	// Convert to JSON
+	jsonBytes, err := json.Marshal(yamlData)
+	if err != nil {
+		return "", fmt.Errorf("error converting to JSON: %w", err)
+	}
+
+	return string(jsonBytes), nil
+}
+
 // InstantiateTemplate reads a template from the reader, executes it with the context,
 // and writes the result to the writer
 func (ctx *TemplateContext) InstantiateTemplate(reader io.Reader, writer io.Writer) error {
@@ -54,7 +136,9 @@ func (ctx *TemplateContext) InstantiateTemplate(reader io.Reader, writer io.Writ
 	}
 
 	// Convert TemplateFunc map to FuncMap
-	funcMap := template.FuncMap{}
+	funcMap := template.FuncMap{
+		"include": ctx.includeFile,
+	}
 	for name, fn := range ctx.Functions {
 		funcMap[name] = fn
 	}
@@ -71,10 +155,35 @@ func (ctx *TemplateContext) InstantiateTemplate(reader io.Reader, writer io.Writ
 		return fmt.Errorf("failed to parse template: %w", err)
 	}
 
-	// Execute template with context
-	if err := tmpl.Execute(writer, ctx.Data); err != nil {
+	// Execute template into a buffer
+	var buf bytes.Buffer
+	if err := tmpl.Execute(&buf, ctx.Data); err != nil {
 		return fmt.Errorf("failed to execute template: %w", err)
 	}
 
+	// If this is the top-level rendering, we want to write the output as "pretty" YAML
+	if len(ctx.includeStack) == 0 {
+		var yamlData interface{}
+		// Parse the buffer content as YAML
+		if err := yaml.Unmarshal(buf.Bytes(), &yamlData); err != nil {
+			return fmt.Errorf("error parsing template output as YAML: %w. Template output: %s", err, buf.String())
+		}
+
+		// Create YAML encoder with default indentation
+		encoder := yaml.NewEncoder(writer)
+		encoder.SetIndent(2)
+
+		// Write the YAML document
+		if err := encoder.Encode(yamlData); err != nil {
+			return fmt.Errorf("error writing YAML output: %w", err)
+		}
+
+	} else {
+		// Otherwise, just write the buffer content to the writer
+		if _, err := buf.WriteTo(writer); err != nil {
+			return fmt.Errorf("failed to write template output: %w", err)
+		}
+	}
+
 	return nil
 }

kurtosis-devnet/pkg/tmpl/tmpl_test.go

@@ -44,7 +44,7 @@ func TestInstantiateTemplate(t *testing.T) {
 		err := ctx.InstantiateTemplate(input, &output)
 		require.NoError(t, err)
 
-		expected := "Hello world!"
+		expected := "Hello world!\n"
 		require.Equal(t, expected, output.String())
 	})
 
@@ -61,7 +61,7 @@ func TestInstantiateTemplate(t *testing.T) {
 		err := ctx.InstantiateTemplate(input, &output)
 		require.NoError(t, err)
 
-		expected := "Hello WORLD!"
+		expected := "Hello WORLD!\n"
 		require.Equal(t, expected, output.String())
 	})
 
@@ -104,7 +104,7 @@ func TestInstantiateTemplate(t *testing.T) {
 		err := ctx.InstantiateTemplate(input, &output)
 		require.NoError(t, err)
 
-		expected := "HELLO world!"
+		expected := "HELLO world!\n"
 		require.Equal(t, expected, output.String())
 	})
 }

kurtosis-devnet/templates/devnet.yaml

@@ -0,0 +1,47 @@
+{{- $context := or . (dict)}}
+{{- $default_l2s := dict 
+    "2151908" (dict "nodes" (list "op-geth")) 
+    "2151909" (dict "nodes" (list "op-geth"))
+}}
+{{- $l2s := dig "l2s" $default_l2s $context }}
+{{- $overrides := dig "overrides" (dict) $context }}
+{{- $interop := dig "interop" false $context }}
+---
+optimism_package:
+{{ if $interop }}
+  interop:
+    enabled: true
+    supervisor_params:
+      image: {{ dig "overrides" "images" "op_supervisor" (localDockerImage "op-supervisor") $context }}
+      extra_params:
+      - {{ dig "overrides" "flags" "log_level" "!!str" $context }}
+{{ end }}
+  chains:
+  {{ range $l2_id, $l2 := $l2s }}
+    - {{ include "l2.yaml" (dict "chain_id" $l2_id "overrides" $overrides "nodes" $l2.nodes) }}
+  {{ end }}
+  op_contract_deployer_params:
+    image: {{ dig "overrides" "images" "op_deployer" (localDockerImage "op-deployer") $context }}
+    l1_artifacts_locator: {{ dig "overrides" "urls" "l1_artifacts" (localContractArtifacts "l1") $context }}
+    l2_artifacts_locator: {{ dig "overrides" "urls" "l2_artifacts" (localContractArtifacts "l2") $context }}
+{{ if $interop }}
+    global_deploy_overrides:
+      faultGameAbsolutePrestate: {{ dig "overrides" "deployer" "prestate" (localPrestate.Hashes.prestate) $context }}
+{{ end }}
+  global_log_level: "info"
+  global_node_selectors: {}
+  global_tolerations: []
+  persistent: false
+ethereum_package:
+  network_params:
+    preset: minimal
+    genesis_delay: 5
+    additional_preloaded_contracts: |
+      {
+        "0x4e59b44847b379578588920cA78FbF26c0B4956C": {
+          "balance": "0ETH",
+          "code": "0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe03601600081602082378035828234f58015156039578182fd5b8082525050506014600cf3",
+          "storage": {},
+          "nonce": "1"
+        }
+      }