Implement Syncback to support converting Roblox files to a Rojo project (#937)

This is a very large commit.
Consider checking the linked PR for more information.

tests/rojo_test/io_util.rs

@@ -10,6 +10,8 @@ use walkdir::WalkDir;
 pub static ROJO_PATH: &str = env!("CARGO_BIN_EXE_rojo");
 pub static BUILD_TESTS_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/rojo-test/build-tests");
 pub static SERVE_TESTS_PATH: &str = concat!(env!("CARGO_MANIFEST_DIR"), "/rojo-test/serve-tests");
+pub static SYNCBACK_TESTS_PATH: &str =
+    concat!(env!("CARGO_MANIFEST_DIR"), "/rojo-test/syncback-tests");
 
 pub fn get_working_dir_path() -> PathBuf {
     let mut manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));

tests/rojo_test/mod.rs

@@ -1,3 +1,4 @@
 pub mod internable;
 pub mod io_util;
 pub mod serve_util;
+pub mod syncback_util;

tests/rojo_test/syncback_util.rs

@@ -0,0 +1,116 @@
+use std::{io::Write as _, path::Path, process::Command};
+
+use insta::{assert_snapshot, assert_yaml_snapshot};
+use tempfile::tempdir;
+
+use crate::rojo_test::io_util::SYNCBACK_TESTS_PATH;
+
+use super::io_util::{copy_recursive, ROJO_PATH};
+
+const INPUT_FILE_PROJECT: &str = "input-project";
+const INPUT_FILE_PLACE: &str = "input.rbxl";
+const INPUT_FILE_MODEL: &str = "input.rbxm";
+
+/// Convenience method to run a `rojo syncback` test.
+///
+/// Test projects should be defined in the `syncback-tests` folder; their filename
+/// should be given as the first parameter.
+///
+/// The passed in callback is where the actual test body should go. Setup and
+/// cleanup happens automatically.
+pub fn run_syncback_test(name: &str, callback: impl FnOnce(&Path)) {
+    let _ = env_logger::try_init();
+
+    // let working_dir = get_working_dir_path();
+
+    let source_path = Path::new(SYNCBACK_TESTS_PATH)
+        .join(name)
+        .join(INPUT_FILE_PROJECT);
+    // We want to support both rbxls and rbxms as input
+    let input_file = {
+        let mut path = Path::new(SYNCBACK_TESTS_PATH)
+            .join(name)
+            .join(INPUT_FILE_PLACE);
+        if !path.exists() {
+            path.set_file_name(INPUT_FILE_MODEL);
+        }
+        path
+    };
+
+    let test_dir = tempdir().expect("Couldn't create temporary directory");
+    let project_path = test_dir
+        .path()
+        .canonicalize()
+        .expect("Couldn't canonicalize temporary directory path")
+        .join(name);
+
+    let source_is_file = fs_err::metadata(&source_path).unwrap().is_file();
+
+    if source_is_file {
+        fs_err::copy(&source_path, &project_path).expect("couldn't copy project file");
+    } else {
+        fs_err::create_dir(&project_path).expect("Couldn't create temporary project subdirectory");
+
+        copy_recursive(&source_path, &project_path)
+            .expect("Couldn't copy project to temporary directory");
+    };
+
+    let output = Command::new(ROJO_PATH)
+        // I don't really understand why setting the working directory breaks this, but it does.
+        // It's a bit concerning but I'm more interested in writing tests than debugging it right now.
+        // TODO: Figure out why and fix it.
+        // .current_dir(working_dir)
+        .args([
+            "--color",
+            "never",
+            "syncback",
+            project_path.to_str().unwrap(),
+            "--input",
+            input_file.to_str().unwrap(),
+            "--non-interactive",
+            "--list",
+        ])
+        .output()
+        .expect("Couldn't spawn syncback process");
+
+    if !output.status.success() {
+        let mut lock = std::io::stderr().lock();
+        writeln!(
+            lock,
+            "Rojo exited with status code {:?}",
+            output.status.code()
+        )
+        .unwrap();
+        writeln!(lock, "Stdout from process:").unwrap();
+        lock.write_all(&output.stdout).unwrap();
+        writeln!(lock, "Stderr from process:").unwrap();
+        lock.write_all(&output.stderr).unwrap();
+
+        std::process::exit(1)
+    }
+
+    let mut settings = insta::Settings::new();
+    let snapshot_path = Path::new(SYNCBACK_TESTS_PATH)
+        .parent()
+        .unwrap()
+        .join("syncback-test-snapshots");
+    settings.set_snapshot_path(snapshot_path);
+    settings.set_sort_maps(true);
+
+    settings.bind(|| {
+        assert_snapshot!(
+            format!("{name}-stdout"),
+            String::from_utf8_lossy(&output.stdout)
+        )
+    });
+
+    settings.bind(|| callback(project_path.as_path()))
+}
+
+pub fn snapshot_rbxm(name: &str, input: Vec<u8>, file_name: &str) {
+    assert_yaml_snapshot!(
+        name,
+        rbx_binary::text_format::DecodedModel::from_reader(input.as_slice()),
+        file_name
+    )
+}

tests/tests/mod.rs

@@ -1,2 +1,3 @@
 mod build;
 mod serve;
+mod syncback;

tests/tests/syncback.rs

@@ -0,0 +1,81 @@
+use std::ffi::OsStr;
+
+use insta::assert_snapshot;
+
+use crate::rojo_test::syncback_util::{run_syncback_test, snapshot_rbxm};
+
+macro_rules! syncback_tests {
+    ($($test_name:ident => $list:expr$(,)?),*) => {$(
+        #[test]
+        fn $test_name() {
+            run_syncback_test(stringify!($test_name), |path| {
+                for name in $list {
+                    let snapshot_name = format!(concat!(stringify!($test_name), "-{}"), name);
+                    let new = path.join::<&str>(name);
+                    if let Some("rbxm") = new.extension().and_then(OsStr::to_str) {
+                        let content = fs_err::read(new).unwrap();
+                        snapshot_rbxm(&snapshot_name, content, name);
+                    } else {
+                        let content = fs_err::read_to_string(new).unwrap();
+                        assert_snapshot!(snapshot_name, content, name);
+                    }
+                }
+            });
+        }
+    )*};
+}
+
+syncback_tests! {
+    // Ensures that there's only one copy written to disk if navigating a
+    // project file might yield two copies
+    child_but_not => ["OnlyOneCopy/child_of_one.luau", "ReplicatedStorage/child_replicated_storage.luau"],
+    // Ensures that syncback works with CSVs
+    csv => ["src/csv_init/init.csv", "src/csv.csv"],
+    // Ensures that if a RojoId is duplicated somewhere in the project, it's
+    // rewritten rather than synced back as a conflict
+    duplicate_rojo_id => ["container.model.json"],
+    // Ensures that the `ignorePaths` setting works for additions
+    ignore_paths_adding => ["src/int_value.model.json", "src/subfolder/string_value.txt"],
+    // Ensures that the `ignorePaths` setting works for `init` files
+    ignore_paths_init => ["src/non-init.luau", "src/init-file/init.luau"],
+    // Ensures that the `ignorePaths` setting works for removals
+    ignore_paths_removing => ["src/Message.rbxm"],
+    // Ensures that `ignoreTrees` works for additions
+    ignore_trees_adding => [],
+    // Ensures that `ignoreTrees` works for removals
+    ignore_trees_removing => [],
+    // Ensures that all of the JSON middlewares are handled as expected
+    json_middlewares => ["src/dir_with_meta/init.meta.json", "src/model_json.model.json", "src/project_json.project.json"],
+    // Ensures projects that refer to other projects work as expected
+    nested_projects => ["nested.project.json", "string_value.txt"],
+    // Ensures files that are ignored by nested projects are picked up if
+    // they're included in second project. Unusual but perfectly workable
+    // pattern that syncback has to support.
+    nested_projects_weird => ["src/modules/ClientModule.luau", "src/modules/ServerModule.luau"],
+    // Ensures that projects respect `init` files when they're directly referenced from a node
+    project_init => ["src/init.luau"],
+    // Ensures that projects can be reserialized by syncback and that
+    // default.project.json doesn't change unexpectedly.
+    project_reserialize => ["attribute_mismatch.luau", "property_mismatch.project.json"],
+    // Confirms that Instances that cannot serialize as directories serialize as rbxms
+    rbxm_fallback => ["src/ChildWithDuplicates.rbxm"],
+    // Ensures that ref properties are linked properly on the file system
+    ref_properties => ["src/pointer.model.json", "src/target.model.json"],
+    // Ensures that ref properties are linked when no attributes are manually
+    // set in the DataModel
+    ref_properties_blank => ["src/pointer.model.json", "src/target.meta.json", "src/target.txt"],
+    // Ensures that if there is a conflict in RojoRefs, one of them is rewritten.
+    ref_properties_conflict => ["src/Pointer_2.model.json", "src/Target_2.model.json"],
+    // Ensures that having multiple pointers that are aimed at the same target doesn't trigger ref rewrites.
+    ref_properties_duplicate => [],
+    // Ensures that the old middleware is respected during syncback
+    respect_old_middleware => ["default.project.json", "src/model_json.model.json", "src/rbxm.rbxm", "src/rbxmx.rbxmx"],
+    // Ensures that StringValues inside project files are written to the
+    // project file, but only if they don't have `$path` set
+    string_value_project => ["default.project.json"],
+    // Ensures that sync rules are respected. This is really just a test to
+    // ensure it uses the old path when possible, but we want the coverage.
+    sync_rules => ["src/module.modulescript", "src/text.text"],
+    // Ensures that the `syncUnscriptable` setting works
+    unscriptable_properties => ["default.project.json"],
+}