Final import syntax for import with explicit format (tweag#2070)

* Change import syntax and change raw -> text Change the syntax of import with explicit format specification from the temporary `import '<Format> <file>` to `import <file> as 'Format`, as decided with the Nickel team. Doing so, we also make the various input and export formats name consistent: we get rid of `raw`, and use `text` instead, which is more precise. For backward compatibility reason, we still support `raw` as an alias on the CLI. The pretty-printer is also fixed: some spaces were previously missing around import with explicit format specification. * Apply suggestions from code review Co-authored-by: jneem <[email protected]> --------- Co-authored-by: jneem <[email protected]>
yonasBSD · Oct 16, 2024 · 00dfa7b · 00dfa7b
1 parent 02d66b2
commit 00dfa7b
Show file tree

Hide file tree

Showing 13 changed files with 70 additions and 47 deletions.
diff --git a/cli/src/cli.rs b/cli/src/cli.rs
@@ -65,10 +65,10 @@ pub struct GlobalOptions {
 pub enum Command {
     /// Evaluates a Nickel program and pretty-prints the result.
     Eval(EvalCommand),
-    /// Converts the parsed representation (AST) back to Nickel source code and
-    /// prints it. Used for debugging purpose
+    /// Converts the parsed representation (AST) back to Nickel source code and prints it. Used for
+    /// debugging purpose
     PprintAst(PprintAstCommand),
-    /// Exports the result to a different format
+    /// Evaluates a Nickel program and serializes the result to a given format
     Export(ExportCommand),
     /// Prints the metadata attached to an attribute, given as a path
     Query(QueryCommand),
@@ -77,13 +77,13 @@ pub enum Command {
     /// Starts a REPL session
     #[cfg(feature = "repl")]
     Repl(ReplCommand),
-    /// Generates the documentation files for the specified nickel file
+    /// Generates the documentation files for the a specified nickel file
     #[cfg(feature = "doc")]
     Doc(DocCommand),
     /// Tests the documentation examples in the specified nickel file
     #[cfg(feature = "doc")]
     Test(TestCommand),
-    /// Format Nickel files
+    /// Formats Nickel files
     #[cfg(feature = "format")]
     Format(FormatCommand),
 

diff --git a/cli/src/input.rs b/cli/src/input.rs
@@ -6,7 +6,8 @@ use crate::{cli::GlobalOptions, customize::Customize, error::CliResult};
 
 #[derive(clap::Parser, Debug)]
 pub struct InputOptions<Customize: clap::Args> {
-    /// Input files, omit to read from stdin
+    /// Input files. Omit to read from stdin. If multiple files are provided, the corresponding
+    /// Nickel expressions are merged (combined with `&`) to produce the result.
     pub files: Vec<PathBuf>,
 
     #[cfg(debug_assertions)]

diff --git a/core/src/cache.rs b/core/src/cache.rs
@@ -40,7 +40,7 @@ pub enum InputFormat {
     Toml,
     #[cfg(feature = "nix-experimental")]
     Nix,
-    Raw,
+    Text,
 }
 
 impl InputFormat {
@@ -53,7 +53,7 @@ impl InputFormat {
             Some("toml") => Some(InputFormat::Toml),
             #[cfg(feature = "nix-experimental")]
             Some("nix") => Some(InputFormat::Nix),
-            Some("txt") => Some(InputFormat::Raw),
+            Some("txt") => Some(InputFormat::Text),
             _ => None,
         }
     }
@@ -62,7 +62,7 @@ impl InputFormat {
         Some(match tag {
             "Json" => InputFormat::Json,
             "Nickel" => InputFormat::Nickel,
-            "Raw" => InputFormat::Raw,
+            "Text" => InputFormat::Text,
             "Yaml" => InputFormat::Yaml,
             "Toml" => InputFormat::Toml,
             #[cfg(feature = "nix-experimental")]
@@ -77,7 +77,7 @@ impl InputFormat {
             InputFormat::Json => "Json",
             InputFormat::Yaml => "Yaml",
             InputFormat::Toml => "Toml",
-            InputFormat::Raw => "Raw",
+            InputFormat::Text => "Text",
             #[cfg(feature = "nix-experimental")]
             InputFormat::Nix => "Nix",
         }
@@ -624,7 +624,7 @@ impl Cache {
                     .map(|t| (attach_pos(t), ParseErrors::default()))
                     .map_err(|err| ParseError::from_serde_json(err, file_id, &self.files))
             }
-            InputFormat::Raw => Ok((
+            InputFormat::Text => Ok((
                 attach_pos(Term::Str(self.files.source(file_id).into()).into()),
                 ParseErrors::default(),
             )),

diff --git a/core/src/error/mod.rs b/core/src/error/mod.rs
@@ -2122,7 +2122,7 @@ impl IntoDiagnostics<FileId> for ParseError {
                 .with_message("unknown import format tag")
                 .with_labels(vec![primary(&span)])
                 .with_notes(vec![
-                    "Examples of valid format tags: 'Nickel 'Json 'Yaml 'Toml 'Raw"
+                    "Examples of valid format tags: 'Nickel, 'Json, 'Yaml, 'Toml, 'Text"
                         .to_owned()
                 ]),
         };

diff --git a/core/src/eval/tests.rs b/core/src/eval/tests.rs
@@ -131,15 +131,15 @@ fn imports() {
         .add_source(String::from("bad"), String::from("^$*/.23ab 0°@"));
     vm.import_resolver_mut().add_source(
         String::from("nested"),
-        String::from("let x = import 'Nickel \"two\" in x + 1"),
+        String::from("let x = import \"two\" as 'Nickel in x + 1"),
     );
     vm.import_resolver_mut().add_source(
         String::from("cycle"),
-        String::from("let x = import 'Nickel \"cycle_b\" in {a = 1, b = x.a}"),
+        String::from("let x = import \"cycle_b\" as 'Nickel in {a = 1, b = x.a}"),
     );
     vm.import_resolver_mut().add_source(
         String::from("cycle_b"),
-        String::from("let x = import 'Nickel \"cycle\" in {a = x.a}"),
+        String::from("let x = import \"cycle\" as 'Nickel in {a = x.a}"),
     );
 
     fn mk_import<R>(

diff --git a/core/src/parser/grammar.lalrpop b/core/src/parser/grammar.lalrpop
@@ -255,7 +255,7 @@ UniTerm: UniTerm = {
     "import" <l: @L> <s: StandardStaticString> <r: @R> =>? {
         Ok(UniTerm::from(mk_import_based_on_filename(s, mk_span(src_id, l, r))?))
     },
-    "import" <l: @L> <t: EnumTag> <r: @R> <s: StandardStaticString> =>? {
+    "import" <s: StandardStaticString> "as" <l: @L> <t: EnumTag> <r: @R> =>? {
         Ok(UniTerm::from(mk_import_explicit(s, t, mk_span(src_id, l, r))?))
     },
 };
@@ -944,18 +944,20 @@ ExtendedIdent: LocIdent = {
     Ident,
 };
 
-// The "or" keyword, parsed as an indent.
+// The "or" contextual keyword, parsed as an indent.
 IdentOr: LocIdent = "or" => LocIdent::new("or");
+// The "as" contextual keyword, parsed as an indent.
+IdentAs: LocIdent = "as" => LocIdent::new("as");
 
 // The set of pure identifiers, which are never keywords in any context.
 RestrictedIdent: LocIdent = "identifier" => LocIdent::new(<>);
 
-// Identifiers allowed everywhere, which include pure identifiers and the "or"
-// contextual keyword. With a bit of effort around pattern, we can make it a
-// valid identifier unambiguously.
+// Identifiers allowed everywhere, which includes pure identifiers and contextual
+// keywords.
 #[inline]
 Ident: LocIdent = {
     WithPos<IdentOr>,
+    WithPos<IdentAs>,
     WithPos<RestrictedIdent>,
 };
 
@@ -1486,6 +1488,7 @@ extern {
         "true" => Token::Normal(NormalToken::True),
         "false" => Token::Normal(NormalToken::False),
         "or" => Token::Normal(NormalToken::Or),
+        "as" => Token::Normal(NormalToken::As),
 
         "?" => Token::Normal(NormalToken::QuestionMark),
         "," => Token::Normal(NormalToken::Comma),

diff --git a/core/src/parser/lexer.rs b/core/src/parser/lexer.rs
@@ -125,6 +125,10 @@ pub enum NormalToken<'input> {
     /// identifier because it's not ambiguous) within patterns.
     #[token("or")]
     Or,
+    /// As isn't a reserved keyword. It is a contextual keyword (a keyword that can be used as an
+    /// identifier because it's not ambiguous) within the `import xxx as yyy` construct.
+    #[token("as")]
+    As,
 
     #[token("?")]
     QuestionMark,

diff --git a/core/src/pretty.rs b/core/src/pretty.rs
@@ -1061,14 +1061,22 @@ where
                 docs![
                     allocator,
                     "import",
+                    allocator.space(),
+                    allocator.as_string(path.to_string_lossy()).double_quotes(),
                     if Some(*format)
                         != InputFormat::from_path(std::path::Path::new(path.as_os_str()))
                     {
-                        docs![allocator, "'", format.to_tag(), allocator.space()]
+                        docs![
+                            allocator,
+                            allocator.space(),
+                            "as",
+                            allocator.space(),
+                            "'",
+                            format.to_tag()
+                        ]
                     } else {
-                        allocator.space()
+                        allocator.nil()
                     },
-                    allocator.as_string(path.to_string_lossy()).double_quotes()
                 ]
             }
             ResolvedImport(id) => allocator.text(format!("import <file_id: {id:?}>")),

diff --git a/core/src/repl/wasm_frontend.rs b/core/src/repl/wasm_frontend.rs
@@ -249,7 +249,7 @@ pub struct ReplState(ReplImpl<CacheImpl>);
 /// WASM-compatible wrapper around `serialize::ExportFormat`.
 #[wasm_bindgen]
 pub enum WasmExportFormat {
-    Raw = "raw",
+    Text = "text",
     Json = "json",
     Yaml = "yaml",
     Toml = "toml",
@@ -262,7 +262,7 @@ impl TryInto<ExportFormat> for WasmExportFormat {
 
     fn try_into(self) -> Result<ExportFormat, ExportFormaParseError> {
         match self {
-            WasmExportFormat::Raw => Ok(ExportFormat::Raw),
+            WasmExportFormat::Text => Ok(ExportFormat::Text),
             WasmExportFormat::Json => Ok(ExportFormat::Json),
             WasmExportFormat::Yaml => Ok(ExportFormat::Yaml),
             WasmExportFormat::Toml => Ok(ExportFormat::Toml),

diff --git a/core/src/serialize.rs b/core/src/serialize.rs
@@ -28,7 +28,10 @@ use std::{fmt, io};
 /// Available export formats.
 #[derive(Copy, Clone, Eq, PartialEq, Debug, Default, clap::ValueEnum)]
 pub enum ExportFormat {
-    Raw,
+    /// Evalute a Nickel expression to a string and write that text to the output
+    /// Note: `raw` is a deprecated alias for `text`; prefer `text` instead.
+    #[value(alias("raw"))]
+    Text,
     #[default]
     Json,
     Yaml,
@@ -38,7 +41,7 @@ pub enum ExportFormat {
 impl fmt::Display for ExportFormat {
     fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result {
         match self {
-            Self::Raw => write!(f, "raw"),
+            Self::Text => write!(f, "text"),
             Self::Json => write!(f, "json"),
             Self::Yaml => write!(f, "yaml"),
             Self::Toml => write!(f, "toml"),
@@ -371,7 +374,7 @@ pub fn validate(format: ExportFormat, t: &RichTerm) -> Result<(), ExportError> {
         }
     }
 
-    if format == ExportFormat::Raw {
+    if format == ExportFormat::Text {
         if let Term::Str(_) = t.term.as_ref() {
             Ok(())
         } else {
@@ -432,7 +435,7 @@ where
                     .write_all(s.as_bytes())
                     .map_err(|err| ExportErrorData::Other(err.to_string()))
             }),
-        ExportFormat::Raw => match rt.as_ref() {
+        ExportFormat::Text => match rt.as_ref() {
             Term::Str(s) => writer
                 .write_all(s.as_bytes())
                 .map_err(|err| ExportErrorData::Other(err.to_string())),

diff --git a/core/tests/integration/inputs/imports/explicit.ncl b/core/tests/integration/inputs/imports/explicit.ncl
@@ -1,11 +1,11 @@
 # test.type = 'pass'
 
 [
-  (import 'Nickel "imported/file_without_extension") == 1234,
-  (import 'Raw    "imported/file_without_extension") |> std.string.is_match "^1200\\+34\\s*$",
-  (import 'Nickel "imported/file_with_unknown_extension.tst") == 1234,
-  (import 'Raw    "imported/file_with_unknown_extension.tst") |> std.string.is_match "^34\\+1200\\s*$",
-  (import 'Raw    "imported/empty.yaml") == "",
-  (import 'Raw    "imported/two.ncl") |> std.string.is_match "^\\s*\\#",
+  (import "imported/file_without_extension" as 'Nickel) == 1234,
+  (import "imported/file_without_extension" as 'Text) |> std.string.is_match "^1200\\+34\\s*$",
+  (import "imported/file_with_unknown_extension.tst" as 'Nickel) == 1234,
+  (import "imported/file_with_unknown_extension.tst" as 'Text) |> std.string.is_match "^34\\+1200\\s*$",
+  (import "imported/empty.yaml" as 'Text) == "",
+  (import "imported/two.ncl" as 'Text) |> std.string.is_match "^\\s*\\#",
 ]
 |> std.test.assert_all
diff --git a/core/tests/integration/inputs/imports/explicit_unknowntag.ncl b/core/tests/integration/inputs/imports/explicit_unknowntag.ncl
@@ -3,4 +3,4 @@
 # [test.metadata]
 # error = 'ParseError'
 
-import 'Qqq "imported/empty.yaml"
+import "imported/empty.yaml" as 'Qqq
diff --git a/doc/manual/syntax.md b/doc/manual/syntax.md
@@ -1307,24 +1307,28 @@ record is serialized. This includes the output of the `nickel export` command:
 
 ## Imports
 
-There is special keyword `import`, which can be followed by either a
-string literal or an enum tag and a string literal.
+A Nickel program can import other Nickel files using the `import` keyword: `let
+lib = import "lib.ncl" in lib.base64_encode [01, 02, 03]`. Nickel can import
+other Nickel files, but also JSON, TOML, YAML, or raw text.
+
+There is special keyword `import`, which can be followed by either a string
+literal or an enum tag and a string literal.
 
 This causes Nickel to read, evaluate and return the specified file.
 
 The file is searched in directories specified by `NICKEL_IMPORT_PATH`
-environment variable or similar command line option, with default being
-the current directory.
+environment variable or similar command line option, with default being the
+current directory.
 
-One-argument import, like `import "myfile.ncl"`, uses filename extension
-to determine the file format. Nickel embeds a short list of known filename
-extensions: `ncl`, `json`, `yml`, `yaml`, `toml`, `txt` and
-`nix` with a fallback to a Nickel file if there is no extetnsion
-or the extension is unknown.
+One-argument import, like `import "myfile.ncl"`, uses filename extension to
+determine the file format. Nickel automatically recognizes the extensions
+`ncl`, `json`, `yml`, `yaml`, `toml` and `txt`. When compiled with experimental Nix
+support, it also recognizes `nix`. If the file's extension is not recognized, it
+will default to Nickel format.
 
-Two-argument import, like `import 'Raw "test.html"` uses a special enum
+Two-argument import, like `import "test.html" as 'Text` uses a special enum
 tag to determine the format. Currently the tags are `'Nickel`, `'Json`,
-`'Yaml`, `'Toml`, `'Raw` and `'Nix`. Some of the formats may be unavailable
+`'Yaml`, `'Toml`, `'Text` and `'Nix`. Some of the formats may be unavailable
 depending on compilation options of the Nickel interpreter.
 
 [nix-string-context]: https://shealevy.com/blog/2018/08/05/understanding-nixs-string-context/