From 00dfa7b97878f04f7901eb7637ab396ad85509f5 Mon Sep 17 00:00:00 2001 From: Yann Hamdaoui Date: Wed, 16 Oct 2024 17:34:04 +0200 Subject: [PATCH] Final import syntax for import with explicit format (#2070) * Change import syntax and change raw -> text Change the syntax of import with explicit format specification from the temporary `import ' ` to `import as 'Format`, as decided with the Nickel team. Doing so, we also make the various input and export formats name consistent: we get rid of `raw`, and use `text` instead, which is more precise. For backward compatibility reason, we still support `raw` as an alias on the CLI. The pretty-printer is also fixed: some spaces were previously missing around import with explicit format specification. * Apply suggestions from code review Co-authored-by: jneem --------- Co-authored-by: jneem --- cli/src/cli.rs | 10 +++---- cli/src/input.rs | 3 ++- core/src/cache.rs | 10 +++---- core/src/error/mod.rs | 2 +- core/src/eval/tests.rs | 6 ++--- core/src/parser/grammar.lalrpop | 13 ++++++---- core/src/parser/lexer.rs | 4 +++ core/src/pretty.rs | 14 +++++++--- core/src/repl/wasm_frontend.rs | 4 +-- core/src/serialize.rs | 11 +++++--- .../integration/inputs/imports/explicit.ncl | 12 ++++----- .../inputs/imports/explicit_unknowntag.ncl | 2 +- doc/manual/syntax.md | 26 +++++++++++-------- 13 files changed, 70 insertions(+), 47 deletions(-) diff --git a/cli/src/cli.rs b/cli/src/cli.rs index 37339f2a40..80a49ea18a 100644 --- a/cli/src/cli.rs +++ b/cli/src/cli.rs @@ -65,10 +65,10 @@ pub struct GlobalOptions { pub enum Command { /// Evaluates a Nickel program and pretty-prints the result. Eval(EvalCommand), - /// Converts the parsed representation (AST) back to Nickel source code and - /// prints it. Used for debugging purpose + /// Converts the parsed representation (AST) back to Nickel source code and prints it. Used for + /// debugging purpose PprintAst(PprintAstCommand), - /// Exports the result to a different format + /// Evaluates a Nickel program and serializes the result to a given format Export(ExportCommand), /// Prints the metadata attached to an attribute, given as a path Query(QueryCommand), @@ -77,13 +77,13 @@ pub enum Command { /// Starts a REPL session #[cfg(feature = "repl")] Repl(ReplCommand), - /// Generates the documentation files for the specified nickel file + /// Generates the documentation files for the a specified nickel file #[cfg(feature = "doc")] Doc(DocCommand), /// Tests the documentation examples in the specified nickel file #[cfg(feature = "doc")] Test(TestCommand), - /// Format Nickel files + /// Formats Nickel files #[cfg(feature = "format")] Format(FormatCommand), diff --git a/cli/src/input.rs b/cli/src/input.rs index 502d275a1f..58605071de 100644 --- a/cli/src/input.rs +++ b/cli/src/input.rs @@ -6,7 +6,8 @@ use crate::{cli::GlobalOptions, customize::Customize, error::CliResult}; #[derive(clap::Parser, Debug)] pub struct InputOptions { - /// Input files, omit to read from stdin + /// Input files. Omit to read from stdin. If multiple files are provided, the corresponding + /// Nickel expressions are merged (combined with `&`) to produce the result. pub files: Vec, #[cfg(debug_assertions)] diff --git a/core/src/cache.rs b/core/src/cache.rs index e52e4e3e7d..e07dcb3d64 100644 --- a/core/src/cache.rs +++ b/core/src/cache.rs @@ -40,7 +40,7 @@ pub enum InputFormat { Toml, #[cfg(feature = "nix-experimental")] Nix, - Raw, + Text, } impl InputFormat { @@ -53,7 +53,7 @@ impl InputFormat { Some("toml") => Some(InputFormat::Toml), #[cfg(feature = "nix-experimental")] Some("nix") => Some(InputFormat::Nix), - Some("txt") => Some(InputFormat::Raw), + Some("txt") => Some(InputFormat::Text), _ => None, } } @@ -62,7 +62,7 @@ impl InputFormat { Some(match tag { "Json" => InputFormat::Json, "Nickel" => InputFormat::Nickel, - "Raw" => InputFormat::Raw, + "Text" => InputFormat::Text, "Yaml" => InputFormat::Yaml, "Toml" => InputFormat::Toml, #[cfg(feature = "nix-experimental")] @@ -77,7 +77,7 @@ impl InputFormat { InputFormat::Json => "Json", InputFormat::Yaml => "Yaml", InputFormat::Toml => "Toml", - InputFormat::Raw => "Raw", + InputFormat::Text => "Text", #[cfg(feature = "nix-experimental")] InputFormat::Nix => "Nix", } @@ -624,7 +624,7 @@ impl Cache { .map(|t| (attach_pos(t), ParseErrors::default())) .map_err(|err| ParseError::from_serde_json(err, file_id, &self.files)) } - InputFormat::Raw => Ok(( + InputFormat::Text => Ok(( attach_pos(Term::Str(self.files.source(file_id).into()).into()), ParseErrors::default(), )), diff --git a/core/src/error/mod.rs b/core/src/error/mod.rs index d33f98b8a6..fca69a1df6 100644 --- a/core/src/error/mod.rs +++ b/core/src/error/mod.rs @@ -2122,7 +2122,7 @@ impl IntoDiagnostics for ParseError { .with_message("unknown import format tag") .with_labels(vec![primary(&span)]) .with_notes(vec![ - "Examples of valid format tags: 'Nickel 'Json 'Yaml 'Toml 'Raw" + "Examples of valid format tags: 'Nickel, 'Json, 'Yaml, 'Toml, 'Text" .to_owned() ]), }; diff --git a/core/src/eval/tests.rs b/core/src/eval/tests.rs index ea3c9f0de5..38945d9e7f 100644 --- a/core/src/eval/tests.rs +++ b/core/src/eval/tests.rs @@ -131,15 +131,15 @@ fn imports() { .add_source(String::from("bad"), String::from("^$*/.23ab 0°@")); vm.import_resolver_mut().add_source( String::from("nested"), - String::from("let x = import 'Nickel \"two\" in x + 1"), + String::from("let x = import \"two\" as 'Nickel in x + 1"), ); vm.import_resolver_mut().add_source( String::from("cycle"), - String::from("let x = import 'Nickel \"cycle_b\" in {a = 1, b = x.a}"), + String::from("let x = import \"cycle_b\" as 'Nickel in {a = 1, b = x.a}"), ); vm.import_resolver_mut().add_source( String::from("cycle_b"), - String::from("let x = import 'Nickel \"cycle\" in {a = x.a}"), + String::from("let x = import \"cycle\" as 'Nickel in {a = x.a}"), ); fn mk_import( diff --git a/core/src/parser/grammar.lalrpop b/core/src/parser/grammar.lalrpop index 1f504d9006..5b1eebc9f1 100644 --- a/core/src/parser/grammar.lalrpop +++ b/core/src/parser/grammar.lalrpop @@ -255,7 +255,7 @@ UniTerm: UniTerm = { "import" =>? { Ok(UniTerm::from(mk_import_based_on_filename(s, mk_span(src_id, l, r))?)) }, - "import" =>? { + "import" "as" =>? { Ok(UniTerm::from(mk_import_explicit(s, t, mk_span(src_id, l, r))?)) }, }; @@ -944,18 +944,20 @@ ExtendedIdent: LocIdent = { Ident, }; -// The "or" keyword, parsed as an indent. +// The "or" contextual keyword, parsed as an indent. IdentOr: LocIdent = "or" => LocIdent::new("or"); +// The "as" contextual keyword, parsed as an indent. +IdentAs: LocIdent = "as" => LocIdent::new("as"); // The set of pure identifiers, which are never keywords in any context. RestrictedIdent: LocIdent = "identifier" => LocIdent::new(<>); -// Identifiers allowed everywhere, which include pure identifiers and the "or" -// contextual keyword. With a bit of effort around pattern, we can make it a -// valid identifier unambiguously. +// Identifiers allowed everywhere, which includes pure identifiers and contextual +// keywords. #[inline] Ident: LocIdent = { WithPos, + WithPos, WithPos, }; @@ -1486,6 +1488,7 @@ extern { "true" => Token::Normal(NormalToken::True), "false" => Token::Normal(NormalToken::False), "or" => Token::Normal(NormalToken::Or), + "as" => Token::Normal(NormalToken::As), "?" => Token::Normal(NormalToken::QuestionMark), "," => Token::Normal(NormalToken::Comma), diff --git a/core/src/parser/lexer.rs b/core/src/parser/lexer.rs index 31e7809bc4..305252f02c 100644 --- a/core/src/parser/lexer.rs +++ b/core/src/parser/lexer.rs @@ -125,6 +125,10 @@ pub enum NormalToken<'input> { /// identifier because it's not ambiguous) within patterns. #[token("or")] Or, + /// As isn't a reserved keyword. It is a contextual keyword (a keyword that can be used as an + /// identifier because it's not ambiguous) within the `import xxx as yyy` construct. + #[token("as")] + As, #[token("?")] QuestionMark, diff --git a/core/src/pretty.rs b/core/src/pretty.rs index c1795740c2..b09400f85b 100644 --- a/core/src/pretty.rs +++ b/core/src/pretty.rs @@ -1061,14 +1061,22 @@ where docs![ allocator, "import", + allocator.space(), + allocator.as_string(path.to_string_lossy()).double_quotes(), if Some(*format) != InputFormat::from_path(std::path::Path::new(path.as_os_str())) { - docs![allocator, "'", format.to_tag(), allocator.space()] + docs![ + allocator, + allocator.space(), + "as", + allocator.space(), + "'", + format.to_tag() + ] } else { - allocator.space() + allocator.nil() }, - allocator.as_string(path.to_string_lossy()).double_quotes() ] } ResolvedImport(id) => allocator.text(format!("import ")), diff --git a/core/src/repl/wasm_frontend.rs b/core/src/repl/wasm_frontend.rs index bfd1e5bb85..b7160eb411 100644 --- a/core/src/repl/wasm_frontend.rs +++ b/core/src/repl/wasm_frontend.rs @@ -249,7 +249,7 @@ pub struct ReplState(ReplImpl); /// WASM-compatible wrapper around `serialize::ExportFormat`. #[wasm_bindgen] pub enum WasmExportFormat { - Raw = "raw", + Text = "text", Json = "json", Yaml = "yaml", Toml = "toml", @@ -262,7 +262,7 @@ impl TryInto for WasmExportFormat { fn try_into(self) -> Result { match self { - WasmExportFormat::Raw => Ok(ExportFormat::Raw), + WasmExportFormat::Text => Ok(ExportFormat::Text), WasmExportFormat::Json => Ok(ExportFormat::Json), WasmExportFormat::Yaml => Ok(ExportFormat::Yaml), WasmExportFormat::Toml => Ok(ExportFormat::Toml), diff --git a/core/src/serialize.rs b/core/src/serialize.rs index f00dcfcb04..7b513a2c78 100644 --- a/core/src/serialize.rs +++ b/core/src/serialize.rs @@ -28,7 +28,10 @@ use std::{fmt, io}; /// Available export formats. #[derive(Copy, Clone, Eq, PartialEq, Debug, Default, clap::ValueEnum)] pub enum ExportFormat { - Raw, + /// Evalute a Nickel expression to a string and write that text to the output + /// Note: `raw` is a deprecated alias for `text`; prefer `text` instead. + #[value(alias("raw"))] + Text, #[default] Json, Yaml, @@ -38,7 +41,7 @@ pub enum ExportFormat { impl fmt::Display for ExportFormat { fn fmt(&self, f: &mut fmt::Formatter) -> fmt::Result { match self { - Self::Raw => write!(f, "raw"), + Self::Text => write!(f, "text"), Self::Json => write!(f, "json"), Self::Yaml => write!(f, "yaml"), Self::Toml => write!(f, "toml"), @@ -371,7 +374,7 @@ pub fn validate(format: ExportFormat, t: &RichTerm) -> Result<(), ExportError> { } } - if format == ExportFormat::Raw { + if format == ExportFormat::Text { if let Term::Str(_) = t.term.as_ref() { Ok(()) } else { @@ -432,7 +435,7 @@ where .write_all(s.as_bytes()) .map_err(|err| ExportErrorData::Other(err.to_string())) }), - ExportFormat::Raw => match rt.as_ref() { + ExportFormat::Text => match rt.as_ref() { Term::Str(s) => writer .write_all(s.as_bytes()) .map_err(|err| ExportErrorData::Other(err.to_string())), diff --git a/core/tests/integration/inputs/imports/explicit.ncl b/core/tests/integration/inputs/imports/explicit.ncl index dfa8be93f0..6f5da06ce4 100644 --- a/core/tests/integration/inputs/imports/explicit.ncl +++ b/core/tests/integration/inputs/imports/explicit.ncl @@ -1,11 +1,11 @@ # test.type = 'pass' [ - (import 'Nickel "imported/file_without_extension") == 1234, - (import 'Raw "imported/file_without_extension") |> std.string.is_match "^1200\\+34\\s*$", - (import 'Nickel "imported/file_with_unknown_extension.tst") == 1234, - (import 'Raw "imported/file_with_unknown_extension.tst") |> std.string.is_match "^34\\+1200\\s*$", - (import 'Raw "imported/empty.yaml") == "", - (import 'Raw "imported/two.ncl") |> std.string.is_match "^\\s*\\#", + (import "imported/file_without_extension" as 'Nickel) == 1234, + (import "imported/file_without_extension" as 'Text) |> std.string.is_match "^1200\\+34\\s*$", + (import "imported/file_with_unknown_extension.tst" as 'Nickel) == 1234, + (import "imported/file_with_unknown_extension.tst" as 'Text) |> std.string.is_match "^34\\+1200\\s*$", + (import "imported/empty.yaml" as 'Text) == "", + (import "imported/two.ncl" as 'Text) |> std.string.is_match "^\\s*\\#", ] |> std.test.assert_all diff --git a/core/tests/integration/inputs/imports/explicit_unknowntag.ncl b/core/tests/integration/inputs/imports/explicit_unknowntag.ncl index 23102f5ca1..c0c3793854 100644 --- a/core/tests/integration/inputs/imports/explicit_unknowntag.ncl +++ b/core/tests/integration/inputs/imports/explicit_unknowntag.ncl @@ -3,4 +3,4 @@ # [test.metadata] # error = 'ParseError' -import 'Qqq "imported/empty.yaml" +import "imported/empty.yaml" as 'Qqq diff --git a/doc/manual/syntax.md b/doc/manual/syntax.md index d8d7d6d52f..eefc7510e6 100644 --- a/doc/manual/syntax.md +++ b/doc/manual/syntax.md @@ -1307,24 +1307,28 @@ record is serialized. This includes the output of the `nickel export` command: ## Imports -There is special keyword `import`, which can be followed by either a -string literal or an enum tag and a string literal. +A Nickel program can import other Nickel files using the `import` keyword: `let +lib = import "lib.ncl" in lib.base64_encode [01, 02, 03]`. Nickel can import +other Nickel files, but also JSON, TOML, YAML, or raw text. + +There is special keyword `import`, which can be followed by either a string +literal or an enum tag and a string literal. This causes Nickel to read, evaluate and return the specified file. The file is searched in directories specified by `NICKEL_IMPORT_PATH` -environment variable or similar command line option, with default being -the current directory. +environment variable or similar command line option, with default being the +current directory. -One-argument import, like `import "myfile.ncl"`, uses filename extension -to determine the file format. Nickel embeds a short list of known filename -extensions: `ncl`, `json`, `yml`, `yaml`, `toml`, `txt` and -`nix` with a fallback to a Nickel file if there is no extetnsion -or the extension is unknown. +One-argument import, like `import "myfile.ncl"`, uses filename extension to +determine the file format. Nickel automatically recognizes the extensions +`ncl`, `json`, `yml`, `yaml`, `toml` and `txt`. When compiled with experimental Nix +support, it also recognizes `nix`. If the file's extension is not recognized, it +will default to Nickel format. -Two-argument import, like `import 'Raw "test.html"` uses a special enum +Two-argument import, like `import "test.html" as 'Text` uses a special enum tag to determine the format. Currently the tags are `'Nickel`, `'Json`, -`'Yaml`, `'Toml`, `'Raw` and `'Nix`. Some of the formats may be unavailable +`'Yaml`, `'Toml`, `'Text` and `'Nix`. Some of the formats may be unavailable depending on compilation options of the Nickel interpreter. [nix-string-context]: https://shealevy.com/blog/2018/08/05/understanding-nixs-string-context/