Add YAML middleware that behaves like TOML and JSON (#1093)

CHANGELOG.md

@@ -1,7 +1,7 @@
 # Rojo Changelog
 
 ## Unreleased
-
+* Add support for syncing `yml` and `yaml` files (behaves similar to JSON and TOML) ([#1093])
 * Fixed colors of Table diff ([#1084])
 * Fixed `sourcemap` command outputting paths with OS-specific path separators ([#1085])
 * Fixed nil -> nil properties showing up as failing to sync in plugin's patch visualizer ([#1081])
@@ -9,6 +9,7 @@
 * Fixed `Auto Connect Playtest Server` no longer functioning due to Roblox change ([#1066])
 * Added an update indicator to the version header when a new version of the plugin is available. ([#1069])
 
+[#1093]: https://github.com/rojo-rbx/rojo/pull/1093
 [#1084]: https://github.com/rojo-rbx/rojo/pull/1084
 [#1085]: https://github.com/rojo-rbx/rojo/pull/1085
 [#1081]: https://github.com/rojo-rbx/rojo/pull/1081

Cargo.lock

@@ -45,6 +45,12 @@ version = "1.0.80"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "5ad32ce52e4161730f7098c077cd2ed6229b5804ccf99e5366be1ab72a98b4e1"
 
+[[package]]
+name = "arraydeque"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7d902e3d592a523def97af8f317b08ce16b7ab854c1985a0c671e6f15cebc236"
+
 [[package]]
 name = "arrayref"
 version = "0.3.7"
@@ -519,6 +525,12 @@ version = "1.0.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
+[[package]]
+name = "foldhash"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
+
 [[package]]
 name = "form_urlencoded"
 version = "1.2.1"
@@ -751,6 +763,24 @@ version = "0.14.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "290f1a1d9242c78d09ce40a5e87e7554ee637af1351968159f4952f028f75604"
 
+[[package]]
+name = "hashbrown"
+version = "0.15.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5"
+dependencies = [
+ "foldhash",
+]
+
+[[package]]
+name = "hashlink"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7382cf6263419f2d8df38c55d7da83da5c18aef87fc7a7fc1fb1e344edfe14c1"
+dependencies = [
+ "hashbrown 0.15.4",
+]
+
 [[package]]
 name = "heck"
 version = "0.4.1"
@@ -1879,6 +1909,7 @@ dependencies = [
  "uuid",
  "walkdir",
  "winreg 0.10.1",
+ "yaml-rust2",
 ]
 
 [[package]]
@@ -2877,6 +2908,17 @@ dependencies = [
  "linked-hash-map",
 ]
 
+[[package]]
+name = "yaml-rust2"
+version = "0.10.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4ce2a4ff45552406d02501cea6c18d8a7e50228e7736a872951fe2fe75c91be7"
+dependencies = [
+ "arraydeque",
+ "encoding_rs",
+ "hashlink",
+]
+
 [[package]]
 name = "yansi"
 version = "0.5.1"

Cargo.toml

@@ -93,6 +93,7 @@ tokio = { version = "1.36.0", features = ["rt", "rt-multi-thread"] }
 uuid = { version = "1.7.0", features = ["v4", "serde"] }
 clap = { version = "3.2.25", features = ["derive"] }
 profiling = "1.0.15"
+yaml-rust2 = "0.10.3"
 
 [target.'cfg(windows)'.dependencies]
 winreg = "0.10.1"

src/snapshot_middleware/mod.rs

@@ -17,6 +17,7 @@ mod rbxmx;
 mod toml;
 mod txt;
 mod util;
+mod yaml;
 
 use std::{
     path::{Path, PathBuf},
@@ -41,6 +42,7 @@ use self::{
     rbxmx::snapshot_rbxmx,
     toml::snapshot_toml,
     txt::snapshot_txt,
+    yaml::snapshot_yaml,
 };
 
 pub use self::{project::snapshot_project_node, util::emit_legacy_scripts_default};
@@ -212,6 +214,7 @@ pub enum Middleware {
     Rbxmx,
     Toml,
     Text,
+    Yaml,
     Ignore,
 }
 
@@ -250,6 +253,7 @@ impl Middleware {
             Self::Rbxmx => snapshot_rbxmx(context, vfs, path, name),
             Self::Toml => snapshot_toml(context, vfs, path, name),
             Self::Text => snapshot_txt(context, vfs, path, name),
+            Self::Yaml => snapshot_yaml(context, vfs, path, name),
             Self::Ignore => Ok(None),
         }
     }
@@ -315,6 +319,7 @@ pub fn default_sync_rules() -> &'static [SyncRule] {
             sync_rule!("*.txt", Text),
             sync_rule!("*.rbxmx", Rbxmx),
             sync_rule!("*.rbxm", Rbxm),
+            sync_rule!("*.{yml,yaml}", Yaml),
         ]
     })
 }

src/snapshot_middleware/snapshots/librojo__snapshot_middleware__yaml__test__instance_from_vfs-2.snap

@@ -0,0 +1,27 @@
+---
+source: src/snapshot_middleware/yaml.rs
+expression: source
+---
+return {
+	string = "this is a string",
+	boolean = true,
+	integer = 1337,
+	float = 123456789.5,
+	["value-with-hypen"] = "it sure is",
+	sequence = {"wow", 8675309},
+	map = {
+		key = "value",
+		key2 = "value 2",
+		key3 = "value 3",
+	},
+	["nested-map"] = {{
+		key = "value",
+	}, {
+		key2 = "value 2",
+	}, {
+		key3 = "value 3",
+	}},
+	whatever_this_is = {"i imagine", "it's", "a", "sequence?"},
+	null1 = nil,
+	null2 = nil,
+}

src/snapshot_middleware/snapshots/librojo__snapshot_middleware__yaml__test__instance_from_vfs.snap

@@ -0,0 +1,21 @@
+---
+source: src/snapshot_middleware/yaml.rs
+expression: instance_snapshot
+---
+snapshot_id: "00000000000000000000000000000000"
+metadata:
+  ignore_unknown_instances: false
+  instigating_source:
+    Path: /foo.yaml
+  relevant_paths:
+    - /foo.yaml
+    - /foo.meta.json
+  context:
+    emit_legacy_scripts: true
+  specified_id: ~
+name: foo
+class_name: ModuleScript
+properties:
+  Source:
+    String: "return {\n\tstring = \"this is a string\",\n\tboolean = true,\n\tinteger = 1337,\n\tfloat = 123456789.5,\n\t[\"value-with-hypen\"] = \"it sure is\",\n\tsequence = {\"wow\", 8675309},\n\tmap = {\n\t\tkey = \"value\",\n\t\tkey2 = \"value 2\",\n\t\tkey3 = \"value 3\",\n\t},\n\t[\"nested-map\"] = {{\n\t\tkey = \"value\",\n\t}, {\n\t\tkey2 = \"value 2\",\n\t}, {\n\t\tkey3 = \"value 3\",\n\t}},\n\twhatever_this_is = {\"i imagine\", \"it's\", \"a\", \"sequence?\"},\n\tnull1 = nil,\n\tnull2 = nil,\n}"
+children: []

src/snapshot_middleware/yaml.rs

@@ -0,0 +1,234 @@
+use std::path::Path;
+
+use anyhow::Context as _;
+use memofs::{IoResultExt, Vfs};
+use rbx_dom_weak::ustr;
+use yaml_rust2::{Yaml, YamlLoader};
+
+use crate::{
+    lua_ast::{Expression, Statement},
+    snapshot::{InstanceContext, InstanceMetadata, InstanceSnapshot},
+};
+
+use super::meta_file::AdjacentMetadata;
+
+pub fn snapshot_yaml(
+    context: &InstanceContext,
+    vfs: &Vfs,
+    path: &Path,
+    name: &str,
+) -> anyhow::Result<Option<InstanceSnapshot>> {
+    let contents = vfs.read_to_string(path)?;
+
+    let mut values = YamlLoader::load_from_str(&contents)?;
+    let value = values
+        .pop()
+        .context("all YAML documents must contain a document")?;
+    if !values.is_empty() {
+        anyhow::bail!("Rojo does not currently support multiple documents in a YAML file")
+    }
+
+    let as_lua = Statement::Return(yaml_to_luau(value)?);
+
+    let meta_path = path.with_file_name(format!("{}.meta.json", name));
+
+    let mut snapshot = InstanceSnapshot::new()
+        .name(name)
+        .class_name("ModuleScript")
+        .property(ustr("Source"), as_lua.to_string())
+        .metadata(
+            InstanceMetadata::new()
+                .instigating_source(path)
+                .relevant_paths(vec![path.to_path_buf(), meta_path.clone()])
+                .context(context),
+        );
+
+    if let Some(meta_contents) = vfs.read(&meta_path).with_not_found()? {
+        let mut metadata = AdjacentMetadata::from_slice(&meta_contents, meta_path)?;
+        metadata.apply_all(&mut snapshot)?;
+    }
+
+    Ok(Some(snapshot))
+}
+
+fn yaml_to_luau(value: Yaml) -> anyhow::Result<Expression> {
+    const MAX_FLOAT_INT: i64 = 1 << 53;
+
+    Ok(match value {
+        Yaml::String(str) => Expression::String(str),
+        Yaml::Boolean(bool) => Expression::Bool(bool),
+        Yaml::Integer(int) => {
+            if int <= MAX_FLOAT_INT {
+                Expression::Number(int as f64)
+            } else {
+                anyhow::bail!(
+                    "the integer '{int}' cannot be losslessly converted into a Luau number"
+                )
+            }
+        }
+        Yaml::Real(_) => {
+            let value = value.as_f64().expect("value should be a valid f64");
+            Expression::Number(value)
+        }
+        Yaml::Null => Expression::Nil,
+        Yaml::Array(values) => {
+            let new_values: anyhow::Result<Vec<Expression>> =
+                values.into_iter().map(yaml_to_luau).collect();
+            Expression::Array(new_values?)
+        }
+        Yaml::Hash(map) => {
+            let new_values: anyhow::Result<Vec<(Expression, Expression)>> = map
+                .into_iter()
+                .map(|(k, v)| {
+                    let k = yaml_to_luau(k)?;
+                    let v = yaml_to_luau(v)?;
+                    Ok((k, v))
+                })
+                .collect();
+            Expression::table(new_values?)
+        }
+        Yaml::Alias(_) => {
+            anyhow::bail!("Rojo cannot convert YAML aliases to Luau")
+        }
+        Yaml::BadValue => {
+            anyhow::bail!("Rojo cannot convert YAML to Luau because of a parsing error")
+        }
+    })
+}
+
+#[cfg(test)]
+mod test {
+    use super::*;
+
+    use memofs::{InMemoryFs, VfsSnapshot};
+    use rbx_dom_weak::types::Variant;
+
+    #[test]
+    fn instance_from_vfs() {
+        let mut imfs = InMemoryFs::new();
+        imfs.load_snapshot(
+            "/foo.yaml",
+            VfsSnapshot::file(
+                r#"
+---
+string: this is a string
+boolean: true
+integer: 1337
+float: 123456789.5
+value-with-hypen: it sure is
+sequence:
+  - wow
+  - 8675309
+map:
+  key: value
+  key2: "value 2"
+  key3: 'value 3'
+nested-map:
+  - key: value
+  - key2: "value 2"
+  - key3: 'value 3'
+whatever_this_is: [i imagine, it's, a, sequence?]
+null1: ~
+null2: null"#,
+            ),
+        )
+        .unwrap();
+
+        let vfs = Vfs::new(imfs.clone());
+
+        let instance_snapshot = snapshot_yaml(
+            &InstanceContext::default(),
+            &vfs,
+            Path::new("/foo.yaml"),
+            "foo",
+        )
+        .unwrap()
+        .unwrap();
+
+        insta::assert_yaml_snapshot!(instance_snapshot);
+
+        let source = instance_snapshot
+            .properties
+            .get(&ustr("Source"))
+            .expect("the result from snapshot_yaml should have a Source property");
+        if let Variant::String(source) = source {
+            insta::assert_snapshot!(source)
+        } else {
+            panic!("the Source property from snapshot_yaml was not a String")
+        }
+    }
+
+    #[test]
+    #[should_panic(expected = "multiple documents")]
+    fn multiple_documents() {
+        let mut imfs = InMemoryFs::new();
+        imfs.load_snapshot(
+            "/foo.yaml",
+            VfsSnapshot::file(
+                r#"
+---
+document-1: this is a document
+---
+document-2: this is also a document"#,
+            ),
+        )
+        .unwrap();
+
+        let vfs = Vfs::new(imfs.clone());
+
+        snapshot_yaml(
+            &InstanceContext::default(),
+            &vfs,
+            Path::new("/foo.yaml"),
+            "foo",
+        )
+        .unwrap()
+        .unwrap();
+    }
+
+    #[test]
+    #[should_panic = "cannot be losslessly converted into a Luau number"]
+    fn integer_border() {
+        let mut imfs = InMemoryFs::new();
+        imfs.load_snapshot(
+            "/allowed.yaml",
+            VfsSnapshot::file(
+                r#"
+value: 9007199254740992
+"#,
+            ),
+        )
+        .unwrap();
+        imfs.load_snapshot(
+            "/not-allowed.yaml",
+            VfsSnapshot::file(
+                r#"
+value: 9007199254740993
+"#,
+            ),
+        )
+        .unwrap();
+
+        let vfs = Vfs::new(imfs.clone());
+
+        assert!(
+            snapshot_yaml(
+                &InstanceContext::default(),
+                &vfs,
+                Path::new("/allowed.yaml"),
+                "allowed",
+            )
+            .is_ok(),
+            "snapshot_yaml failed to snapshot document with integer '9007199254740992' in it"
+        );
+
+        snapshot_yaml(
+            &InstanceContext::default(),
+            &vfs,
+            Path::new("/not-allowed.yaml"),
+            "not-allowed",
+        )
+        .unwrap()
+        .unwrap();
+    }
+}