From 4bf73c7a8ac32c3c5221160e249a074a37da6634 Mon Sep 17 00:00:00 2001 From: Lucien Greathouse Date: Sat, 28 Mar 2020 00:36:01 -0700 Subject: [PATCH] Implement support for turning .json files into Lua modules (#308) * Stub implementation * Flesh out feature and add tests. Other snapshots currently failing. * Blacklist .meta.json in JSON handler * Write to correct property (Source) instead of Value * Update changelog --- CHANGELOG.md | 1 + .../rojo_test__build_test__json_as_lua.snap | 23 ++ .../json_as_lua/default.project.json | 6 + .../json_as_lua/make-me-a-script.json | 12 + rojo-test/src/build_test.rs | 1 + src/lib.rs | 1 + src/lua_ast.rs | 279 ++++++++++++++++++ src/snapshot_middleware/error.rs | 21 +- src/snapshot_middleware/json.rs | 142 +++++++++ src/snapshot_middleware/mod.rs | 3 + ...leware__json__test__instance_from_vfs.snap | 20 ++ 11 files changed, 505 insertions(+), 4 deletions(-) create mode 100644 rojo-test/build-test-snapshots/rojo_test__build_test__json_as_lua.snap create mode 100644 rojo-test/build-tests/json_as_lua/default.project.json create mode 100644 rojo-test/build-tests/json_as_lua/make-me-a-script.json create mode 100644 src/lua_ast.rs create mode 100644 src/snapshot_middleware/json.rs create mode 100644 src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json__test__instance_from_vfs.snap diff --git a/CHANGELOG.md b/CHANGELOG.md index c5bfe708..259f2d75 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ * "Open Scripts Externally": When enabled, opening a script in Studio will instead open it in your default text editor. * "Two-Way Sync": When enabled, Rojo will attempt to save changes to your place back to the filesystem. **Very early feature, very broken, beware!** * Added `--color` option to force-enable or force-disable color in Rojo's output. +* Added support for turning `.json` files into `ModuleScript` instances ([#308](https://github.com/rojo-rbx/rojo/pull/308)) * The server half of **experimental** two-way sync is now enabled by default. * Increased default logging verbosity in commands like `rojo build`. * Rojo now requires a project file again, just like 0.5.4. diff --git a/rojo-test/build-test-snapshots/rojo_test__build_test__json_as_lua.snap b/rojo-test/build-test-snapshots/rojo_test__build_test__json_as_lua.snap new file mode 100644 index 00000000..8fcead0a --- /dev/null +++ b/rojo-test/build-test-snapshots/rojo_test__build_test__json_as_lua.snap @@ -0,0 +1,23 @@ +--- +source: rojo-test/src/build_test.rs +expression: contents +--- + + + + json_as_lua + return { + ["1invalidident"] = "nice", + array = {1, 2, 3}, + ["false"] = false, + float = 1234.5452, + int = 1234, + null = nil, + object = { + hello = "world", + }, + ["true"] = true, +} + + + diff --git a/rojo-test/build-tests/json_as_lua/default.project.json b/rojo-test/build-tests/json_as_lua/default.project.json new file mode 100644 index 00000000..1e984c68 --- /dev/null +++ b/rojo-test/build-tests/json_as_lua/default.project.json @@ -0,0 +1,6 @@ +{ + "name": "json_as_lua", + "tree": { + "$path": "make-me-a-script.json" + } +} \ No newline at end of file diff --git a/rojo-test/build-tests/json_as_lua/make-me-a-script.json b/rojo-test/build-tests/json_as_lua/make-me-a-script.json new file mode 100644 index 00000000..c252006c --- /dev/null +++ b/rojo-test/build-tests/json_as_lua/make-me-a-script.json @@ -0,0 +1,12 @@ +{ + "array": [1, 2, 3], + "object": { + "hello": "world" + }, + "true": true, + "false": false, + "null": null, + "int": 1234, + "float": 1234.5452, + "1invalidident": "nice" +} \ No newline at end of file diff --git a/rojo-test/src/build_test.rs b/rojo-test/src/build_test.rs index c51cacf9..7939f206 100644 --- a/rojo-test/src/build_test.rs +++ b/rojo-test/src/build_test.rs @@ -31,6 +31,7 @@ gen_build_tests! { init_meta_class_name, init_meta_properties, init_with_children, + json_as_lua, json_model_in_folder, json_model_legacy_name, module_in_folder, diff --git a/src/lib.rs b/src/lib.rs index 454449b2..7e594414 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -11,6 +11,7 @@ mod auth_cookie; mod change_processor; mod error; mod glob; +mod lua_ast; mod message_queue; mod multimap; mod path_serializer; diff --git a/src/lua_ast.rs b/src/lua_ast.rs new file mode 100644 index 00000000..ace9b4d7 --- /dev/null +++ b/src/lua_ast.rs @@ -0,0 +1,279 @@ +//! Defines module for defining a small Lua AST for simple codegen. + +use std::{ + fmt::{self, Write}, + num::FpCategory, +}; + +/// Trait that helps turn a type into an equivalent Lua snippet. +/// +/// Designed to be similar to the `Display` trait from Rust's std. +trait FmtLua { + fn fmt_lua(&self, output: &mut LuaStream<'_>) -> fmt::Result; + + /// Used to override how this type will appear when used as a table key. + /// Some types, like strings, can have a shorter representation as a table + /// key than the default, safe approach. + fn fmt_table_key(&self, output: &mut LuaStream<'_>) -> fmt::Result { + write!(output, "[")?; + self.fmt_lua(output)?; + write!(output, "]") + } +} + +pub(crate) enum Statement { + Return(Expression), +} + +impl FmtLua for Statement { + fn fmt_lua(&self, output: &mut LuaStream<'_>) -> fmt::Result { + match self { + Self::Return(literal) => { + write!(output, "return ")?; + literal.fmt_lua(output) + } + } + } +} + +impl fmt::Display for Statement { + fn fmt(&self, output: &mut fmt::Formatter) -> fmt::Result { + let mut stream = LuaStream::new(output); + FmtLua::fmt_lua(self, &mut stream) + } +} + +pub(crate) enum Expression { + Nil, + Bool(bool), + Number(f64), + String(String), + Table(Table), + + /// Arrays are not technically distinct from other tables in Lua, but this + /// representation is more convenient. + Array(Vec), +} + +impl Expression { + pub fn table(entries: Vec<(Expression, Expression)>) -> Self { + Self::Table(Table { entries }) + } +} + +impl FmtLua for Expression { + fn fmt_lua(&self, output: &mut LuaStream<'_>) -> fmt::Result { + match self { + Self::Nil => write!(output, "nil"), + Self::Bool(inner) => inner.fmt_lua(output), + Self::Number(inner) => inner.fmt_lua(output), + Self::String(inner) => inner.fmt_lua(output), + Self::Table(inner) => inner.fmt_lua(output), + Self::Array(inner) => inner.fmt_lua(output), + } + } + + fn fmt_table_key(&self, output: &mut LuaStream<'_>) -> fmt::Result { + match self { + Self::Nil => panic!("nil cannot be a table key"), + Self::Bool(inner) => inner.fmt_table_key(output), + Self::Number(inner) => inner.fmt_table_key(output), + Self::String(inner) => inner.fmt_table_key(output), + Self::Table(inner) => inner.fmt_table_key(output), + Self::Array(inner) => inner.fmt_table_key(output), + } + } +} + +impl From for Expression { + fn from(value: String) -> Self { + Self::String(value) + } +} + +impl From<&'_ str> for Expression { + fn from(value: &str) -> Self { + Self::String(value.to_owned()) + } +} + +impl From for Expression { + fn from(value: Table) -> Self { + Self::Table(value) + } +} + +impl FmtLua for bool { + fn fmt_lua(&self, output: &mut LuaStream<'_>) -> fmt::Result { + write!(output, "{}", self) + } +} + +impl FmtLua for f64 { + fn fmt_lua(&self, output: &mut LuaStream<'_>) -> fmt::Result { + match self.classify() { + FpCategory::Nan => write!(output, "0/0"), + FpCategory::Infinite => { + if self.is_sign_positive() { + write!(output, "math.huge") + } else { + write!(output, "-math.huge") + } + } + _ => write!(output, "{}", self), + } + } +} + +impl FmtLua for String { + fn fmt_lua(&self, output: &mut LuaStream<'_>) -> fmt::Result { + write!(output, "\"{}\"", self) + } + + fn fmt_table_key(&self, output: &mut LuaStream<'_>) -> fmt::Result { + if is_valid_ident(self) { + write!(output, "{}", self) + } else { + write!(output, "[\"{}\"]", self) + } + } +} + +impl FmtLua for Vec { + fn fmt_lua(&self, output: &mut LuaStream<'_>) -> fmt::Result { + write!(output, "{{")?; + + for (index, value) in self.iter().enumerate() { + value.fmt_lua(output)?; + + if index < self.len() - 1 { + write!(output, ", ")?; + } + } + + write!(output, "}}") + } +} + +pub(crate) struct Table { + pub entries: Vec<(Expression, Expression)>, +} + +impl FmtLua for Table { + fn fmt_lua(&self, output: &mut LuaStream<'_>) -> fmt::Result { + writeln!(output, "{{")?; + output.indent(); + + for (key, value) in &self.entries { + key.fmt_table_key(output)?; + write!(output, " = ")?; + value.fmt_lua(output)?; + writeln!(output, ",")?; + } + + output.unindent(); + write!(output, "}}") + } +} + +fn is_valid_ident_char_start(value: char) -> bool { + value.is_ascii_alphabetic() || value == '_' +} + +fn is_valid_ident_char(value: char) -> bool { + value.is_ascii_alphanumeric() || value == '_' +} + +fn is_keyword(value: &str) -> bool { + match value { + "and" | "break" | "do" | "else" | "elseif" | "end" | "false" | "for" | "function" + | "if" | "in" | "local" | "nil" | "not" | "or" | "repeat" | "return" | "then" | "true" + | "until" | "while" => true, + _ => false, + } +} + +/// Tells whether the given string is a valid Lua identifier. +fn is_valid_ident(value: &str) -> bool { + if is_keyword(value) { + return false; + } + + let mut chars = value.chars(); + + match chars.next() { + Some(first) => { + if !is_valid_ident_char_start(first) { + return false; + } + } + None => return false, + } + + chars.all(is_valid_ident_char) +} + +/// Wraps a `fmt::Write` with additional tracking to do pretty-printing of Lua. +/// +/// Behaves similarly to `fmt::Formatter`. This trait's relationship to `LuaFmt` +/// is very similar to `Formatter`'s relationship to `Display`. +struct LuaStream<'a> { + indent_level: usize, + is_start_of_line: bool, + inner: &'a mut (dyn fmt::Write + 'a), +} + +impl fmt::Write for LuaStream<'_> { + /// Method to support the `write!` and `writeln!` macros. Instead of using a + /// trait directly, these macros just call `write_str` on their first + /// argument. + /// + /// This method is also available on `io::Write` and `fmt::Write`. + fn write_str(&mut self, value: &str) -> fmt::Result { + let mut is_first_line = true; + + for line in value.split('\n') { + if is_first_line { + is_first_line = false; + } else { + self.line()?; + } + + if !line.is_empty() { + if self.is_start_of_line { + self.is_start_of_line = false; + let indentation = "\t".repeat(self.indent_level); + self.inner.write_str(&indentation)?; + } + + self.inner.write_str(line)?; + } + } + + Ok(()) + } +} + +impl<'a> LuaStream<'a> { + fn new(inner: &'a mut (dyn fmt::Write + 'a)) -> Self { + LuaStream { + indent_level: 0, + is_start_of_line: true, + inner, + } + } + + fn indent(&mut self) { + self.indent_level += 1; + } + + fn unindent(&mut self) { + assert!(self.indent_level > 0); + self.indent_level -= 1; + } + + fn line(&mut self) -> fmt::Result { + self.is_start_of_line = true; + self.inner.write_str("\n") + } +} diff --git a/src/snapshot_middleware/error.rs b/src/snapshot_middleware/error.rs index 78aaaf95..a686fcbc 100644 --- a/src/snapshot_middleware/error.rs +++ b/src/snapshot_middleware/error.rs @@ -7,30 +7,36 @@ pub enum SnapshotError { #[error("file name had malformed Unicode")] FileNameBadUnicode { path: PathBuf }, - #[error("file had malformed Unicode contents")] + #[error("file had malformed Unicode contents at path {}", .path.display())] FileContentsBadUnicode { source: std::str::Utf8Error, path: PathBuf, }, - #[error("malformed project file")] + #[error("malformed project file at path {}", .path.display())] MalformedProject { source: serde_json::Error, path: PathBuf, }, - #[error("malformed .model.json file")] + #[error("malformed .model.json file at path {}", .path.display())] MalformedModelJson { source: serde_json::Error, path: PathBuf, }, - #[error("malformed .meta.json file")] + #[error("malformed .meta.json file at path {}", .path.display())] MalformedMetaJson { source: serde_json::Error, path: PathBuf, }, + #[error("malformed JSON at path {}", .path.display())] + MalformedJson { + source: serde_json::Error, + path: PathBuf, + }, + #[error(transparent)] Io { #[from] @@ -76,4 +82,11 @@ impl SnapshotError { path: path.into(), } } + + pub(crate) fn malformed_json(source: serde_json::Error, path: impl Into) -> Self { + Self::MalformedJson { + source, + path: path.into(), + } + } } diff --git a/src/snapshot_middleware/json.rs b/src/snapshot_middleware/json.rs new file mode 100644 index 00000000..4b3508e3 --- /dev/null +++ b/src/snapshot_middleware/json.rs @@ -0,0 +1,142 @@ +use std::path::Path; + +use maplit::hashmap; +use memofs::{IoResultExt, Vfs}; +use rbx_dom_weak::RbxValue; + +use crate::{ + lua_ast::{Expression, Statement}, + snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot}, +}; + +use super::{ + error::SnapshotError, + meta_file::AdjacentMetadata, + middleware::{SnapshotInstanceResult, SnapshotMiddleware}, + util::match_file_name, +}; + +/// Catch-all middleware for snapshots on JSON files that aren't used for other +/// features, like Rojo projects, JSON models, or meta files. +pub struct SnapshotJson; + +impl SnapshotMiddleware for SnapshotJson { + fn from_vfs(context: &InstanceContext, vfs: &Vfs, path: &Path) -> SnapshotInstanceResult { + let meta = vfs.metadata(path)?; + + if meta.is_dir() { + return Ok(None); + } + + // FIXME: This middleware should not need to know about the .meta.json + // middleware. Should there be a way to signal "I'm not returning an + // instance and no one should"? + if match_file_name(path, ".meta.json").is_some() { + return Ok(None); + } + + let instance_name = match match_file_name(path, ".json") { + Some(name) => name, + None => return Ok(None), + }; + + let contents = vfs.read(path)?; + + let value: serde_json::Value = serde_json::from_slice(&contents) + .map_err(|err| SnapshotError::malformed_json(err, path))?; + + let as_lua = json_to_lua(value).to_string(); + + let properties = hashmap! { + "Source".to_owned() => RbxValue::String { + value: as_lua, + }, + }; + + let meta_path = path.with_file_name(format!("{}.meta.json", instance_name)); + + let mut snapshot = InstanceSnapshot::new() + .name(instance_name) + .class_name("ModuleScript") + .properties(properties) + .metadata( + InstanceMetadata::new() + .instigating_source(path) + .relevant_paths(vec![path.to_path_buf(), meta_path.clone()]) + .context(context), + ); + + if let Some(meta_contents) = vfs.read(&meta_path).with_not_found()? { + let mut metadata = AdjacentMetadata::from_slice(&meta_contents, &meta_path)?; + metadata.apply_all(&mut snapshot); + } + + Ok(Some(snapshot)) + } +} + +fn json_to_lua(value: serde_json::Value) -> Statement { + Statement::Return(json_to_lua_value(value)) +} + +fn json_to_lua_value(value: serde_json::Value) -> Expression { + use serde_json::Value; + + match value { + Value::Null => Expression::Nil, + Value::Bool(value) => Expression::Bool(value), + Value::Number(value) => Expression::Number(value.as_f64().unwrap()), + Value::String(value) => Expression::String(value), + Value::Array(values) => { + Expression::Array(values.into_iter().map(json_to_lua_value).collect()) + } + Value::Object(values) => Expression::table( + values + .into_iter() + .map(|(key, value)| (key.into(), json_to_lua_value(value))) + .collect(), + ), + } +} + +#[cfg(test)] +mod test { + use super::*; + + use memofs::{InMemoryFs, VfsSnapshot}; + + #[test] + fn instance_from_vfs() { + let mut imfs = InMemoryFs::new(); + imfs.load_snapshot( + "/foo.json", + VfsSnapshot::file( + r#"{ + "array": [1, 2, 3], + "object": { + "hello": "world" + }, + "true": true, + "false": false, + "null": null, + "int": 1234, + "float": 1234.5452, + "1invalidident": "nice" + }"#, + ), + ) + .unwrap(); + + let mut vfs = Vfs::new(imfs.clone()); + + let instance_snapshot = SnapshotJson::from_vfs( + &InstanceContext::default(), + &mut vfs, + Path::new("/foo.json"), + ) + .unwrap() + .unwrap(); + + insta::assert_yaml_snapshot!(instance_snapshot); + } +} diff --git a/src/snapshot_middleware/mod.rs b/src/snapshot_middleware/mod.rs index dff62bc0..561949c2 100644 --- a/src/snapshot_middleware/mod.rs +++ b/src/snapshot_middleware/mod.rs @@ -6,6 +6,7 @@ mod csv; mod dir; mod error; +mod json; mod json_model; mod lua; mod meta_file; @@ -26,6 +27,7 @@ use crate::snapshot::InstanceContext; use self::{ csv::SnapshotCsv, dir::SnapshotDir, + json::SnapshotJson, json_model::SnapshotJsonModel, lua::SnapshotLua, middleware::{SnapshotInstanceResult, SnapshotMiddleware}, @@ -71,5 +73,6 @@ middlewares! { SnapshotLua, SnapshotCsv, SnapshotTxt, + SnapshotJson, SnapshotDir, } diff --git a/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json__test__instance_from_vfs.snap b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json__test__instance_from_vfs.snap new file mode 100644 index 00000000..59378750 --- /dev/null +++ b/src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json__test__instance_from_vfs.snap @@ -0,0 +1,20 @@ +--- +source: src/snapshot_middleware/json.rs +expression: instance_snapshot +--- +snapshot_id: ~ +metadata: + ignore_unknown_instances: false + instigating_source: + Path: /foo.json + relevant_paths: + - /foo.json + - /foo.meta.json + context: {} +name: foo +class_name: ModuleScript +properties: + Source: + Type: String + Value: "return {\n\t[\"1invalidident\"] = \"nice\",\n\tarray = {1, 2, 3},\n\t[\"false\"] = false,\n\tfloat = 1234.5452,\n\tint = 1234,\n\tnull = nil,\n\tobject = {\n\t\thello = \"world\",\n\t},\n\t[\"true\"] = true,\n}" +children: []