Merge branch 'main' into feature/required-label

.github/workflows/benchmark.yml

@@ -3,7 +3,13 @@ name: "benchmark"
 on:
   # Run using manual triggers from GitHub UI:
   # https://docs.github.com/en/actions/managing-workflow-runs/manually-running-a-workflow
-  workflow_dispatch: {}
+  workflow_dispatch:
+    inputs:
+      dryRun:
+        description: "Attempt a local run and report results here, without updating the bencher dashboard."
+        type: "boolean"
+        required: true
+        default: true
 
   # Run on pushes to 'main' branch:
   push:
@@ -19,8 +25,8 @@ jobs:
   benchmark:
     runs-on: "ubuntu-22.04" # _SLANG_DEV_CONTAINER_BASE_IMAGE_ (keep in sync)
 
-    # Only run on the main repo (not forks):
-    if: "${{ github.repository == 'NomicFoundation/slang' }}"
+    # Only run on the main repo (not forks), unless it is a dry run:
+    if: "${{ github.repository == 'NomicFoundation/slang' || inputs.dryRun == true }}"
 
     steps:
       - name: "Checkout Repository"
@@ -48,7 +54,7 @@ jobs:
       - name: "infra perf benchmark"
         uses: "./.github/actions/devcontainer/run"
         with:
-          runCmd: "./scripts/bin/infra perf benchmark"
+          runCmd: "./scripts/bin/infra perf benchmark ${{ inputs.dryRun == true && '--dry-run' || '' }}"
           env: |
             BENCHER_API_TOKEN=${{ secrets.BENCHER_API_TOKEN }}
 

crates/codegen/ebnf/src/builder.rs

@@ -466,7 +466,7 @@ impl Builder {
                 let expression = if chars.len() == 1 {
                     Expression::new_atom(chars[0].to_string())
                 } else {
-                    Expression::new_sequence(
+                    Expression::new_choice(
                         chars
                             .iter()
                             .map(|ch| Expression::new_atom(ch.to_string()))
@@ -485,8 +485,11 @@ impl Builder {
             Scanner::Atom { atom } => Expression::new_atom(atom.clone()),
             Scanner::TrailingContext {
                 scanner,
-                not_followed_by: _,
-            } => Self::build_scanner(scanner),
+                not_followed_by,
+            } => Expression::new_sequence(vec![
+                Self::build_scanner(scanner),
+                Expression::new_negative_look_ahead(Self::build_scanner(not_followed_by).into()),
+            ]),
             Scanner::Fragment { reference } => Self::build_ref(None, reference),
         }
     }

crates/codegen/ebnf/src/model.rs

@@ -89,6 +89,9 @@ pub enum Expression {
     Atom {
         atom: String,
     },
+    NegativeLookAhead {
+        expression: Box<Self>,
+    },
     Reference {
         leading_comment: Option<String>,
         reference: Identifier,
@@ -101,7 +104,7 @@ impl Expression {
         // This separates members of the same precedence, like both "a b (c | d)" and "a | b | (c d)".
         match self {
             // Binary
-            Self::Choice { .. } | Self::Range { .. } | Self::Sequence { .. } => 1,
+            Self::Choice { .. } | Self::Sequence { .. } => 1,
 
             // Prefix
             Self::Not { .. } => 2,
@@ -110,7 +113,10 @@ impl Expression {
             Self::OneOrMore { .. } | Self::Optional { .. } | Self::ZeroOrMore { .. } => 3,
 
             // Primary
-            Self::Atom { .. } | Self::Reference { .. } => 4,
+            Self::Range { .. }
+            | Self::Atom { .. }
+            | Self::NegativeLookAhead { .. }
+            | Self::Reference { .. } => 4,
         }
     }
 }

crates/codegen/ebnf/src/serializer.rs

@@ -140,13 +140,18 @@ impl<'s, W: EbnfWriter> Serializer<'s, W> {
                 inclusive_start,
                 inclusive_end,
             } => {
-                self.serialize_child_expr(parent, inclusive_start)?;
+                self.serialize_expr(inclusive_start)?;
                 self.serialize_punctuation("…")?;
-                self.serialize_child_expr(parent, inclusive_end)?;
+                self.serialize_expr(inclusive_end)?;
             }
             Expression::Atom { atom } => {
                 self.serialize_string_literal(atom)?;
             }
+            Expression::NegativeLookAhead { expression } => {
+                self.serialize_punctuation("(?!")?;
+                self.serialize_expr(expression)?;
+                self.serialize_punctuation(")")?;
+            }
             Expression::Reference {
                 leading_comment,
                 reference,

crates/codegen/runtime/cargo/crate/src/runtime/cst/edge_label.rs.jinja2

@@ -15,20 +15,25 @@
 )]
 #[strum(serialize_all = "snake_case")]
 #[derive(Clone, Copy)]
+/// Represents the different types of relationships between nodes in the syntax tree.
 pub enum EdgeLabel {
-    // Built-in:
+    {# Built-in: #}
+
     {% for label in model.kinds.predefined_labels -%}
+        /// Represents a child node with the label `{{ label | snake_case }}`.
         {{ label | pascal_case }},
     {%- endfor %}
 
-    // Generated:
+    {# Generated: #}
+
     {% if rendering_in_stubs -%}
         Stub1,
         Stub2,
         Stub3,
     {%- else -%}
-        {% for variant in model.kinds.labels -%}
-            {{ variant | pascal_case }},
+        {% for label in model.kinds.labels -%}
+            /// Represents a child node with the label `{{ label | snake_case }}`.
+            {{ label | pascal_case }},
         {%- endfor -%}
     {%- endif -%}
 }

crates/codegen/runtime/cargo/crate/src/runtime/cst/nonterminal_kind.rs.jinja2

@@ -24,10 +24,14 @@ pub enum NonterminalKind {
         Stub3,
     {%- else -%}
         {%- for variant in model.kinds.nonterminal_kinds -%}
+            /// Represents a node with kind `{{ variant.id | pascal_case }}`, having the following structure:
+            ///
             /// ```ebnf
-            {{ variant.documentation | indent(prefix = "/// ", first = true, blank = true) }}
+            {%- for line in variant.documentation | split(pat="\n") %}
+            /// {{ line }}
+            {%- endfor %}
             /// ```
-            {{ variant.id }},
+            {{ variant.id | pascal_case }},
         {%- endfor -%}
     {%- endif -%}
 }

crates/codegen/runtime/cargo/crate/src/runtime/cst/terminal_kind.rs.jinja2

@@ -18,21 +18,27 @@
 #[allow(clippy::doc_markdown)]
 #[allow(clippy::doc_link_with_quotes)]
 pub enum TerminalKind {
-    // Built-in:
+    {# Built-in: #}
+
     UNRECOGNIZED,
     MISSING,
 
-    // Generated:
+    {# Generated: #}
+
     {% if rendering_in_stubs -%}
         Stub1,
         Stub2,
         Stub3,
     {%- else -%}
         {%- for variant in model.kinds.terminal_kinds -%}
+            /// Represents a node with kind `{{ variant.id | pascal_case }}`, having the following structure:
+            ///
             /// ```ebnf
-            {{ variant.documentation | indent(prefix = "/// ", first = true, blank = true) }}
+            {%- for line in variant.documentation | split(pat="\n") %}
+            /// {{ line }}
+            {%- endfor %}
             /// ```
-            {{ variant.id }},
+            {{ variant.id | pascal_case }},
         {%- endfor -%}
     {%- endif -%}
 }

crates/codegen/runtime/cargo/wasm/src/runtime/interface/ast.wit.jinja2

@@ -1,6 +1,7 @@
 interface ast {
     use cst.{node, nonterminal-node};
 
+    /// @internal
     resource selectors {
         sequence: static func(node: borrow<nonterminal-node>) -> result<list<option<node>>, string>;
         choice: static func(node: borrow<nonterminal-node>) -> result<node, string>;

crates/codegen/runtime/cargo/wasm/src/runtime/interface/compilation.wit.jinja2

@@ -7,6 +7,8 @@ interface compilation {
     ///
     /// This is an internal API, and exposed via a public `CompilationBuilder` wrapper class written in TypeScript.
     /// This allows storing/invoking user supplied callbacks in TypeScript, rather than Rust, which has its limitations.
+    ///
+    /// @internal
     resource internal-compilation-builder {
         /// Creates a new compilation builder for the specified language version.
         create: static func(language-version: string) -> result<internal-compilation-builder, string>;

crates/codegen/runtime/cargo/wasm/src/runtime/interface/cst.wit.jinja2

@@ -11,7 +11,7 @@ interface cst {
             stub3,
         {%- else %}
             {%- for variant in model.kinds.nonterminal_kinds %}
-                /// This kind represents a `{{ variant.id }}` node, with the following structure: 
+                /// Represents a node with kind `{{ variant.id | pascal_case }}`, having the following structure:
                 ///
                 /// ```ebnf
                 {%- for line in variant.documentation | split(pat="\n") %}
@@ -42,7 +42,7 @@ interface cst {
             stub3,
         {%- else %}
             {%- for variant in model.kinds.terminal_kinds %}
-                /// This kind represents a `{{ variant.id }}` node, with the following structure: 
+                /// Represents a node with kind `{{ variant.id | pascal_case }}`, having the following structure:
                 ///
                 /// ```ebnf
                 {%- for line in variant.documentation | split(pat="\n") %}
@@ -65,6 +65,7 @@ interface cst {
     /// Represents the different types of relationships between nodes in the syntax tree.
     enum edge-label {
         {%- for label in model.kinds.predefined_labels %}
+            /// Represents a child node with the label `{{ label | snake_case }}`.
             {{ label | wit_case }},
         {%- endfor %}
 
@@ -76,8 +77,9 @@ interface cst {
             /// Generated stub.
             stub3,
         {%- else %}
-            {%- for variant in model.kinds.labels %}
-                {{ variant | wit_case }},
+            {%- for label in model.kinds.labels %}
+                /// Represents a child node with the label `{{ label | snake_case }}`.
+                {{ label | wit_case }},
             {%- endfor %}
         {%- endif %}
     }