Implement support for turning .json files into Lua modules (#308)

* Stub implementation

* Flesh out feature and add tests. Other snapshots currently failing.

* Blacklist .meta.json in JSON handler

* Write to correct property (Source) instead of Value

* Update changelog

CHANGELOG.md

@@ -5,6 +5,7 @@
     * "Open Scripts Externally": When enabled, opening a script in Studio will instead open it in your default text editor.
     * "Two-Way Sync": When enabled, Rojo will attempt to save changes to your place back to the filesystem. **Very early feature, very broken, beware!**
 * Added `--color` option to force-enable or force-disable color in Rojo's output.
+* Added support for turning `.json` files into `ModuleScript` instances ([#308](https://github.com/rojo-rbx/rojo/pull/308))
 * The server half of **experimental** two-way sync is now enabled by default.
 * Increased default logging verbosity in commands like `rojo build`.
 * Rojo now requires a project file again, just like 0.5.4.

rojo-test/build-test-snapshots/rojo_test__build_test__json_as_lua.snap

@@ -0,0 +1,23 @@
+---
+source: rojo-test/src/build_test.rs
+expression: contents
+---
+<roblox version="4">
+  <Item class="ModuleScript" referent="0">
+    <Properties>
+      <string name="Name">json_as_lua</string>
+      <string name="Source">return {
+	["1invalidident"] = "nice",
+	array = {1, 2, 3},
+	["false"] = false,
+	float = 1234.5452,
+	int = 1234,
+	null = nil,
+	object = {
+		hello = "world",
+	},
+	["true"] = true,
+}</string>
+    </Properties>
+  </Item>
+</roblox>

rojo-test/build-tests/json_as_lua/default.project.json

@@ -0,0 +1,6 @@
+{
+  "name": "json_as_lua",
+  "tree": {
+    "$path": "make-me-a-script.json"
+  }
+}

rojo-test/build-tests/json_as_lua/make-me-a-script.json

@@ -0,0 +1,12 @@
+{
+  "array": [1, 2, 3],
+  "object": {
+    "hello": "world"
+  },
+  "true": true,
+  "false": false,
+  "null": null,
+  "int": 1234,
+  "float": 1234.5452,
+  "1invalidident": "nice"
+}

rojo-test/src/build_test.rs

@@ -31,6 +31,7 @@ gen_build_tests! {
     init_meta_class_name,
     init_meta_properties,
     init_with_children,
+    json_as_lua,
     json_model_in_folder,
     json_model_legacy_name,
     module_in_folder,

src/lib.rs

@@ -11,6 +11,7 @@ mod auth_cookie;
 mod change_processor;
 mod error;
 mod glob;
+mod lua_ast;
 mod message_queue;
 mod multimap;
 mod path_serializer;

src/lua_ast.rs

@@ -0,0 +1,279 @@
+//! Defines module for defining a small Lua AST for simple codegen.
+
+use std::{
+    fmt::{self, Write},
+    num::FpCategory,
+};
+
+/// Trait that helps turn a type into an equivalent Lua snippet.
+///
+/// Designed to be similar to the `Display` trait from Rust's std.
+trait FmtLua {
+    fn fmt_lua(&self, output: &mut LuaStream<'_>) -> fmt::Result;
+
+    /// Used to override how this type will appear when used as a table key.
+    /// Some types, like strings, can have a shorter representation as a table
+    /// key than the default, safe approach.
+    fn fmt_table_key(&self, output: &mut LuaStream<'_>) -> fmt::Result {
+        write!(output, "[")?;
+        self.fmt_lua(output)?;
+        write!(output, "]")
+    }
+}
+
+pub(crate) enum Statement {
+    Return(Expression),
+}
+
+impl FmtLua for Statement {
+    fn fmt_lua(&self, output: &mut LuaStream<'_>) -> fmt::Result {
+        match self {
+            Self::Return(literal) => {
+                write!(output, "return ")?;
+                literal.fmt_lua(output)
+            }
+        }
+    }
+}
+
+impl fmt::Display for Statement {
+    fn fmt(&self, output: &mut fmt::Formatter) -> fmt::Result {
+        let mut stream = LuaStream::new(output);
+        FmtLua::fmt_lua(self, &mut stream)
+    }
+}
+
+pub(crate) enum Expression {
+    Nil,
+    Bool(bool),
+    Number(f64),
+    String(String),
+    Table(Table),
+
+    /// Arrays are not technically distinct from other tables in Lua, but this
+    /// representation is more convenient.
+    Array(Vec<Expression>),
+}
+
+impl Expression {
+    pub fn table(entries: Vec<(Expression, Expression)>) -> Self {
+        Self::Table(Table { entries })
+    }
+}
+
+impl FmtLua for Expression {
+    fn fmt_lua(&self, output: &mut LuaStream<'_>) -> fmt::Result {
+        match self {
+            Self::Nil => write!(output, "nil"),
+            Self::Bool(inner) => inner.fmt_lua(output),
+            Self::Number(inner) => inner.fmt_lua(output),
+            Self::String(inner) => inner.fmt_lua(output),
+            Self::Table(inner) => inner.fmt_lua(output),
+            Self::Array(inner) => inner.fmt_lua(output),
+        }
+    }
+
+    fn fmt_table_key(&self, output: &mut LuaStream<'_>) -> fmt::Result {
+        match self {
+            Self::Nil => panic!("nil cannot be a table key"),
+            Self::Bool(inner) => inner.fmt_table_key(output),
+            Self::Number(inner) => inner.fmt_table_key(output),
+            Self::String(inner) => inner.fmt_table_key(output),
+            Self::Table(inner) => inner.fmt_table_key(output),
+            Self::Array(inner) => inner.fmt_table_key(output),
+        }
+    }
+}
+
+impl From<String> for Expression {
+    fn from(value: String) -> Self {
+        Self::String(value)
+    }
+}
+
+impl From<&'_ str> for Expression {
+    fn from(value: &str) -> Self {
+        Self::String(value.to_owned())
+    }
+}
+
+impl From<Table> for Expression {
+    fn from(value: Table) -> Self {
+        Self::Table(value)
+    }
+}
+
+impl FmtLua for bool {
+    fn fmt_lua(&self, output: &mut LuaStream<'_>) -> fmt::Result {
+        write!(output, "{}", self)
+    }
+}
+
+impl FmtLua for f64 {
+    fn fmt_lua(&self, output: &mut LuaStream<'_>) -> fmt::Result {
+        match self.classify() {
+            FpCategory::Nan => write!(output, "0/0"),
+            FpCategory::Infinite => {
+                if self.is_sign_positive() {
+                    write!(output, "math.huge")
+                } else {
+                    write!(output, "-math.huge")
+                }
+            }
+            _ => write!(output, "{}", self),
+        }
+    }
+}
+
+impl FmtLua for String {
+    fn fmt_lua(&self, output: &mut LuaStream<'_>) -> fmt::Result {
+        write!(output, "\"{}\"", self)
+    }
+
+    fn fmt_table_key(&self, output: &mut LuaStream<'_>) -> fmt::Result {
+        if is_valid_ident(self) {
+            write!(output, "{}", self)
+        } else {
+            write!(output, "[\"{}\"]", self)
+        }
+    }
+}
+
+impl FmtLua for Vec<Expression> {
+    fn fmt_lua(&self, output: &mut LuaStream<'_>) -> fmt::Result {
+        write!(output, "{{")?;
+
+        for (index, value) in self.iter().enumerate() {
+            value.fmt_lua(output)?;
+
+            if index < self.len() - 1 {
+                write!(output, ", ")?;
+            }
+        }
+
+        write!(output, "}}")
+    }
+}
+
+pub(crate) struct Table {
+    pub entries: Vec<(Expression, Expression)>,
+}
+
+impl FmtLua for Table {
+    fn fmt_lua(&self, output: &mut LuaStream<'_>) -> fmt::Result {
+        writeln!(output, "{{")?;
+        output.indent();
+
+        for (key, value) in &self.entries {
+            key.fmt_table_key(output)?;
+            write!(output, " = ")?;
+            value.fmt_lua(output)?;
+            writeln!(output, ",")?;
+        }
+
+        output.unindent();
+        write!(output, "}}")
+    }
+}
+
+fn is_valid_ident_char_start(value: char) -> bool {
+    value.is_ascii_alphabetic() || value == '_'
+}
+
+fn is_valid_ident_char(value: char) -> bool {
+    value.is_ascii_alphanumeric() || value == '_'
+}
+
+fn is_keyword(value: &str) -> bool {
+    match value {
+        "and" | "break" | "do" | "else" | "elseif" | "end" | "false" | "for" | "function"
+        | "if" | "in" | "local" | "nil" | "not" | "or" | "repeat" | "return" | "then" | "true"
+        | "until" | "while" => true,
+        _ => false,
+    }
+}
+
+/// Tells whether the given string is a valid Lua identifier.
+fn is_valid_ident(value: &str) -> bool {
+    if is_keyword(value) {
+        return false;
+    }
+
+    let mut chars = value.chars();
+
+    match chars.next() {
+        Some(first) => {
+            if !is_valid_ident_char_start(first) {
+                return false;
+            }
+        }
+        None => return false,
+    }
+
+    chars.all(is_valid_ident_char)
+}
+
+/// Wraps a `fmt::Write` with additional tracking to do pretty-printing of Lua.
+///
+/// Behaves similarly to `fmt::Formatter`. This trait's relationship to `LuaFmt`
+/// is very similar to `Formatter`'s relationship to `Display`.
+struct LuaStream<'a> {
+    indent_level: usize,
+    is_start_of_line: bool,
+    inner: &'a mut (dyn fmt::Write + 'a),
+}
+
+impl fmt::Write for LuaStream<'_> {
+    /// Method to support the `write!` and `writeln!` macros. Instead of using a
+    /// trait directly, these macros just call `write_str` on their first
+    /// argument.
+    ///
+    /// This method is also available on `io::Write` and `fmt::Write`.
+    fn write_str(&mut self, value: &str) -> fmt::Result {
+        let mut is_first_line = true;
+
+        for line in value.split('\n') {
+            if is_first_line {
+                is_first_line = false;
+            } else {
+                self.line()?;
+            }
+
+            if !line.is_empty() {
+                if self.is_start_of_line {
+                    self.is_start_of_line = false;
+                    let indentation = "\t".repeat(self.indent_level);
+                    self.inner.write_str(&indentation)?;
+                }
+
+                self.inner.write_str(line)?;
+            }
+        }
+
+        Ok(())
+    }
+}
+
+impl<'a> LuaStream<'a> {
+    fn new(inner: &'a mut (dyn fmt::Write + 'a)) -> Self {
+        LuaStream {
+            indent_level: 0,
+            is_start_of_line: true,
+            inner,
+        }
+    }
+
+    fn indent(&mut self) {
+        self.indent_level += 1;
+    }
+
+    fn unindent(&mut self) {
+        assert!(self.indent_level > 0);
+        self.indent_level -= 1;
+    }
+
+    fn line(&mut self) -> fmt::Result {
+        self.is_start_of_line = true;
+        self.inner.write_str("\n")
+    }
+}

src/snapshot_middleware/error.rs

@@ -7,30 +7,36 @@ pub enum SnapshotError {
     #[error("file name had malformed Unicode")]
     FileNameBadUnicode { path: PathBuf },
 
-    #[error("file had malformed Unicode contents")]
+    #[error("file had malformed Unicode contents at path {}", .path.display())]
     FileContentsBadUnicode {
         source: std::str::Utf8Error,
         path: PathBuf,
     },
 
-    #[error("malformed project file")]
+    #[error("malformed project file at path {}", .path.display())]
     MalformedProject {
         source: serde_json::Error,
         path: PathBuf,
     },
 
-    #[error("malformed .model.json file")]
+    #[error("malformed .model.json file at path {}", .path.display())]
     MalformedModelJson {
         source: serde_json::Error,
         path: PathBuf,
     },
 
-    #[error("malformed .meta.json file")]
+    #[error("malformed .meta.json file at path {}", .path.display())]
     MalformedMetaJson {
         source: serde_json::Error,
         path: PathBuf,
     },
 
+    #[error("malformed JSON at path {}", .path.display())]
+    MalformedJson {
+        source: serde_json::Error,
+        path: PathBuf,
+    },
+
     #[error(transparent)]
     Io {
         #[from]
@@ -76,4 +82,11 @@ impl SnapshotError {
             path: path.into(),
         }
     }
+
+    pub(crate) fn malformed_json(source: serde_json::Error, path: impl Into<PathBuf>) -> Self {
+        Self::MalformedJson {
+            source,
+            path: path.into(),
+        }
+    }
 }

src/snapshot_middleware/json.rs

@@ -0,0 +1,142 @@
+use std::path::Path;
+
+use maplit::hashmap;
+use memofs::{IoResultExt, Vfs};
+use rbx_dom_weak::RbxValue;
+
+use crate::{
+    lua_ast::{Expression, Statement},
+    snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot},
+};
+
+use super::{
+    error::SnapshotError,
+    meta_file::AdjacentMetadata,
+    middleware::{SnapshotInstanceResult, SnapshotMiddleware},
+    util::match_file_name,
+};
+
+/// Catch-all middleware for snapshots on JSON files that aren't used for other
+/// features, like Rojo projects, JSON models, or meta files.
+pub struct SnapshotJson;
+
+impl SnapshotMiddleware for SnapshotJson {
+    fn from_vfs(context: &InstanceContext, vfs: &Vfs, path: &Path) -> SnapshotInstanceResult {
+        let meta = vfs.metadata(path)?;
+
+        if meta.is_dir() {
+            return Ok(None);
+        }
+
+        // FIXME: This middleware should not need to know about the .meta.json
+        // middleware. Should there be a way to signal "I'm not returning an
+        // instance and no one should"?
+        if match_file_name(path, ".meta.json").is_some() {
+            return Ok(None);
+        }
+
+        let instance_name = match match_file_name(path, ".json") {
+            Some(name) => name,
+            None => return Ok(None),
+        };
+
+        let contents = vfs.read(path)?;
+
+        let value: serde_json::Value = serde_json::from_slice(&contents)
+            .map_err(|err| SnapshotError::malformed_json(err, path))?;
+
+        let as_lua = json_to_lua(value).to_string();
+
+        let properties = hashmap! {
+            "Source".to_owned() => RbxValue::String {
+                value: as_lua,
+            },
+        };
+
+        let meta_path = path.with_file_name(format!("{}.meta.json", instance_name));
+
+        let mut snapshot = InstanceSnapshot::new()
+            .name(instance_name)
+            .class_name("ModuleScript")
+            .properties(properties)
+            .metadata(
+                InstanceMetadata::new()
+                    .instigating_source(path)
+                    .relevant_paths(vec![path.to_path_buf(), meta_path.clone()])
+                    .context(context),
+            );
+
+        if let Some(meta_contents) = vfs.read(&meta_path).with_not_found()? {
+            let mut metadata = AdjacentMetadata::from_slice(&meta_contents, &meta_path)?;
+            metadata.apply_all(&mut snapshot);
+        }
+
+        Ok(Some(snapshot))
+    }
+}
+
+fn json_to_lua(value: serde_json::Value) -> Statement {
+    Statement::Return(json_to_lua_value(value))
+}
+
+fn json_to_lua_value(value: serde_json::Value) -> Expression {
+    use serde_json::Value;
+
+    match value {
+        Value::Null => Expression::Nil,
+        Value::Bool(value) => Expression::Bool(value),
+        Value::Number(value) => Expression::Number(value.as_f64().unwrap()),
+        Value::String(value) => Expression::String(value),
+        Value::Array(values) => {
+            Expression::Array(values.into_iter().map(json_to_lua_value).collect())
+        }
+        Value::Object(values) => Expression::table(
+            values
+                .into_iter()
+                .map(|(key, value)| (key.into(), json_to_lua_value(value)))
+                .collect(),
+        ),
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    use memofs::{InMemoryFs, VfsSnapshot};
+
+    #[test]
+    fn instance_from_vfs() {
+        let mut imfs = InMemoryFs::new();
+        imfs.load_snapshot(
+            "/foo.json",
+            VfsSnapshot::file(
+                r#"{
+                  "array": [1, 2, 3],
+                  "object": {
+                    "hello": "world"
+                  },
+                  "true": true,
+                  "false": false,
+                  "null": null,
+                  "int": 1234,
+                  "float": 1234.5452,
+                  "1invalidident": "nice"
+                }"#,
+            ),
+        )
+        .unwrap();
+
+        let mut vfs = Vfs::new(imfs.clone());
+
+        let instance_snapshot = SnapshotJson::from_vfs(
+            &InstanceContext::default(),
+            &mut vfs,
+            Path::new("/foo.json"),
+        )
+        .unwrap()
+        .unwrap();
+
+        insta::assert_yaml_snapshot!(instance_snapshot);
+    }
+}

src/snapshot_middleware/mod.rs

@@ -6,6 +6,7 @@
 mod csv;
 mod dir;
 mod error;
+mod json;
 mod json_model;
 mod lua;
 mod meta_file;
@@ -26,6 +27,7 @@ use crate::snapshot::InstanceContext;
 use self::{
     csv::SnapshotCsv,
     dir::SnapshotDir,
+    json::SnapshotJson,
     json_model::SnapshotJsonModel,
     lua::SnapshotLua,
     middleware::{SnapshotInstanceResult, SnapshotMiddleware},
@@ -71,5 +73,6 @@ middlewares! {
     SnapshotLua,
     SnapshotCsv,
     SnapshotTxt,
+    SnapshotJson,
     SnapshotDir,
 }

src/snapshot_middleware/snapshots/librojo__snapshot_middleware__json__test__instance_from_vfs.snap

@@ -0,0 +1,20 @@
+---
+source: src/snapshot_middleware/json.rs
+expression: instance_snapshot
+---
+snapshot_id: ~
+metadata:
+  ignore_unknown_instances: false
+  instigating_source:
+    Path: /foo.json
+  relevant_paths:
+    - /foo.json
+    - /foo.meta.json
+  context: {}
+name: foo
+class_name: ModuleScript
+properties:
+  Source:
+    Type: String
+    Value: "return {\n\t[\"1invalidident\"] = \"nice\",\n\tarray = {1, 2, 3},\n\t[\"false\"] = false,\n\tfloat = 1234.5452,\n\tint = 1234,\n\tnull = nil,\n\tobject = {\n\t\thello = \"world\",\n\t},\n\t[\"true\"] = true,\n}"
+children: []