Implement Syncback to support converting Roblox files to a Rojo project (#937)

This is a very large commit.
Consider checking the linked PR for more information.

src/snapshot_middleware/meta_file.rs

@@ -1,14 +1,18 @@
-use std::{
-    collections::HashMap,
-    path::{Path, PathBuf},
-};
+use std::path::{Path, PathBuf};
 
 use anyhow::{format_err, Context};
+use indexmap::IndexMap;
 use memofs::{IoResultExt as _, Vfs};
-use rbx_dom_weak::{types::Attributes, Ustr, UstrMap};
+use rbx_dom_weak::{
+    types::{Attributes, Variant},
+    Ustr,
+};
 use serde::{Deserialize, Serialize};
 
-use crate::{json, resolution::UnresolvedValue, snapshot::InstanceSnapshot, RojoRef};
+use crate::{
+    json, resolution::UnresolvedValue, snapshot::InstanceSnapshot, syncback::SyncbackSnapshot,
+    RojoRef,
+};
 
 /// Represents metadata in a sibling file with the same basename.
 ///
@@ -26,11 +30,11 @@ pub struct AdjacentMetadata {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub ignore_unknown_instances: Option<bool>,
 
-    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
-    pub properties: UstrMap<UnresolvedValue>,
+    #[serde(default, skip_serializing_if = "IndexMap::is_empty")]
+    pub properties: IndexMap<Ustr, UnresolvedValue>,
 
-    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
-    pub attributes: HashMap<String, UnresolvedValue>,
+    #[serde(default, skip_serializing_if = "IndexMap::is_empty")]
+    pub attributes: IndexMap<String, UnresolvedValue>,
 
     #[serde(skip)]
     pub path: PathBuf,
@@ -80,6 +84,76 @@ impl AdjacentMetadata {
         Ok(meta)
     }
 
+    /// Constructs an `AdjacentMetadata` from the provided snapshot, assuming it
+    /// will be at the provided path.
+    pub fn from_syncback_snapshot(
+        snapshot: &SyncbackSnapshot,
+        path: PathBuf,
+    ) -> anyhow::Result<Option<Self>> {
+        let mut properties = IndexMap::new();
+        let mut attributes = IndexMap::new();
+        // TODO make this more granular.
+        // I am breaking the cycle of bad TODOs. This is in reference to the fact
+        // that right now, this will just not write any metadata at all for
+        // project nodes, which is not always desirable. We should try to be
+        // smarter about it.
+        if let Some(old_inst) = snapshot.old_inst() {
+            if let Some(source) = &old_inst.metadata().instigating_source {
+                let source = source.path();
+                if source != path {
+                    log::debug!(
+                        "Instigating source for Instance is mismatched so its metadata is being skipped.\nPath: {}",
+                        path.display()
+                    );
+                    return Ok(None);
+                }
+            }
+        }
+
+        let ignore_unknown_instances = snapshot
+            .old_inst()
+            .map(|inst| inst.metadata().ignore_unknown_instances)
+            .unwrap_or_default();
+
+        let class = &snapshot.new_inst().class;
+        for (name, value) in snapshot.get_path_filtered_properties(snapshot.new).unwrap() {
+            match value {
+                Variant::Attributes(attrs) => {
+                    for (attr_name, attr_value) in attrs.iter() {
+                        // We (probably) don't want to preserve internal
+                        // attributes, only user defined ones.
+                        if attr_name.starts_with("RBX") {
+                            continue;
+                        }
+                        attributes.insert(
+                            attr_name.clone(),
+                            UnresolvedValue::from_variant_unambiguous(attr_value.clone()),
+                        );
+                    }
+                }
+                _ => {
+                    properties.insert(
+                        name,
+                        UnresolvedValue::from_variant(value.clone(), class, &name),
+                    );
+                }
+            }
+        }
+
+        Ok(Some(Self {
+            ignore_unknown_instances: if ignore_unknown_instances {
+                Some(true)
+            } else {
+                None
+            },
+            properties,
+            attributes,
+            path,
+            id: None,
+            schema: None,
+        }))
+    }
+
     pub fn apply_ignore_unknown_instances(&mut self, snapshot: &mut InstanceSnapshot) {
         if let Some(ignore) = self.ignore_unknown_instances.take() {
             snapshot.metadata.ignore_unknown_instances = ignore;
@@ -89,7 +163,10 @@ impl AdjacentMetadata {
     pub fn apply_properties(&mut self, snapshot: &mut InstanceSnapshot) -> anyhow::Result<()> {
         let path = &self.path;
 
-        for (key, unresolved) in self.properties.drain() {
+        // BTreeMaps don't have an equivalent to HashMap::drain, so the next
+        // best option is to take ownership of the entire map. Not free, but
+        // very cheap.
+        for (key, unresolved) in std::mem::take(&mut self.properties) {
             let value = unresolved
                 .resolve(&snapshot.class_name, &key)
                 .with_context(|| format!("error applying meta file {}", path.display()))?;
@@ -100,7 +177,7 @@ impl AdjacentMetadata {
         if !self.attributes.is_empty() {
             let mut attributes = Attributes::new();
 
-            for (key, unresolved) in self.attributes.drain() {
+            for (key, unresolved) in std::mem::take(&mut self.attributes) {
                 let value = unresolved.resolve_unambiguous()?;
                 attributes.insert(key, value);
             }
@@ -131,6 +208,18 @@ impl AdjacentMetadata {
         Ok(())
     }
 
+    /// Returns whether the metadata is 'empty', meaning it doesn't have anything
+    /// worth persisting in it. Specifically:
+    ///
+    /// - The number of properties and attributes is 0
+    /// - `ignore_unknown_instances` is None
+    #[inline]
+    pub fn is_empty(&self) -> bool {
+        self.attributes.is_empty()
+            && self.properties.is_empty()
+            && self.ignore_unknown_instances.is_none()
+    }
+
     // TODO: Add method to allow selectively applying parts of metadata and
     // throwing errors if invalid parts are specified.
 }
@@ -151,11 +240,11 @@ pub struct DirectoryMetadata {
     #[serde(skip_serializing_if = "Option::is_none")]
     pub ignore_unknown_instances: Option<bool>,
 
-    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
-    pub properties: UstrMap<UnresolvedValue>,
+    #[serde(default, skip_serializing_if = "IndexMap::is_empty")]
+    pub properties: IndexMap<Ustr, UnresolvedValue>,
 
-    #[serde(default, skip_serializing_if = "HashMap::is_empty")]
-    pub attributes: HashMap<String, UnresolvedValue>,
+    #[serde(default, skip_serializing_if = "IndexMap::is_empty")]
+    pub attributes: IndexMap<String, UnresolvedValue>,
 
     #[serde(skip_serializing_if = "Option::is_none")]
     pub class_name: Option<Ustr>,
@@ -207,6 +296,80 @@ impl DirectoryMetadata {
         Ok(meta)
     }
 
+    /// Constructs a `DirectoryMetadata` from the provided snapshot, assuming it
+    /// will be at the provided path.
+    ///
+    /// This function does not set `ClassName` manually as most uses won't
+    /// want it set.
+    pub fn from_syncback_snapshot(
+        snapshot: &SyncbackSnapshot,
+        path: PathBuf,
+    ) -> anyhow::Result<Option<Self>> {
+        let mut properties = IndexMap::new();
+        let mut attributes = IndexMap::new();
+        // TODO make this more granular.
+        // I am breaking the cycle of bad TODOs. This is in reference to the fact
+        // that right now, this will just not write any metadata at all for
+        // project nodes, which is not always desirable. We should try to be
+        // smarter about it.
+        if let Some(old_inst) = snapshot.old_inst() {
+            if let Some(source) = &old_inst.metadata().instigating_source {
+                let source = source.path();
+                if source != path {
+                    log::debug!(
+                        "Instigating source for Instance is mismatched so its metadata is being skipped.\nPath: {}",
+                        path.display()
+                    );
+                    return Ok(None);
+                }
+            }
+        }
+
+        let ignore_unknown_instances = snapshot
+            .old_inst()
+            .map(|inst| inst.metadata().ignore_unknown_instances)
+            .unwrap_or_default();
+
+        let class = &snapshot.new_inst().class;
+        for (name, value) in snapshot.get_path_filtered_properties(snapshot.new).unwrap() {
+            match value {
+                Variant::Attributes(attrs) => {
+                    for (name, value) in attrs.iter() {
+                        // We (probably) don't want to preserve internal
+                        // attributes, only user defined ones.
+                        if name.starts_with("RBX") {
+                            continue;
+                        }
+                        attributes.insert(
+                            name.to_owned(),
+                            UnresolvedValue::from_variant_unambiguous(value.clone()),
+                        );
+                    }
+                }
+                _ => {
+                    properties.insert(
+                        name,
+                        UnresolvedValue::from_variant(value.clone(), class, &name),
+                    );
+                }
+            }
+        }
+
+        Ok(Some(Self {
+            ignore_unknown_instances: if ignore_unknown_instances {
+                Some(true)
+            } else {
+                None
+            },
+            properties,
+            attributes,
+            class_name: None,
+            path,
+            id: None,
+            schema: None,
+        }))
+    }
+
     pub fn apply_all(&mut self, snapshot: &mut InstanceSnapshot) -> anyhow::Result<()> {
         self.apply_ignore_unknown_instances(snapshot);
         self.apply_class_name(snapshot)?;
@@ -241,7 +404,7 @@ impl DirectoryMetadata {
     fn apply_properties(&mut self, snapshot: &mut InstanceSnapshot) -> anyhow::Result<()> {
         let path = &self.path;
 
-        for (key, unresolved) in self.properties.drain() {
+        for (key, unresolved) in std::mem::take(&mut self.properties) {
             let value = unresolved
                 .resolve(&snapshot.class_name, &key)
                 .with_context(|| format!("error applying meta file {}", path.display()))?;
@@ -252,7 +415,7 @@ impl DirectoryMetadata {
         if !self.attributes.is_empty() {
             let mut attributes = Attributes::new();
 
-            for (key, unresolved) in self.attributes.drain() {
+            for (key, unresolved) in std::mem::take(&mut self.attributes) {
                 let value = unresolved.resolve_unambiguous()?;
                 attributes.insert(key, value);
             }
@@ -275,6 +438,53 @@ impl DirectoryMetadata {
         snapshot.metadata.specified_id = self.id.take().map(RojoRef::new);
         Ok(())
     }
+
+    /// Returns whether the metadata is 'empty', meaning it doesn't have anything
+    /// worth persisting in it. Specifically:
+    ///
+    /// - The number of properties and attributes is 0
+    /// - `ignore_unknown_instances` is None
+    /// - `class_name` is either None or not Some("Folder")
+    #[inline]
+    pub fn is_empty(&self) -> bool {
+        self.attributes.is_empty()
+            && self.properties.is_empty()
+            && self.ignore_unknown_instances.is_none()
+            && if let Some(class) = &self.class_name {
+                class == "Folder"
+            } else {
+                true
+            }
+    }
+}
+
+/// Retrieves the meta file that should be applied for the provided directory,
+/// if it exists.
+pub fn dir_meta(vfs: &Vfs, path: &Path) -> anyhow::Result<Option<DirectoryMetadata>> {
+    let meta_path = path.join("init.meta.json");
+
+    if let Some(meta_contents) = vfs.read(&meta_path).with_not_found()? {
+        let metadata = DirectoryMetadata::from_slice(&meta_contents, meta_path)?;
+        Ok(Some(metadata))
+    } else {
+        Ok(None)
+    }
+}
+
+/// Retrieves the meta file that should be applied for the provided file,
+/// if it exists.
+///
+/// The `name` field should be the name the metadata should have.
+pub fn file_meta(vfs: &Vfs, path: &Path, name: &str) -> anyhow::Result<Option<AdjacentMetadata>> {
+    let mut meta_path = path.with_file_name(name);
+    meta_path.set_extension("meta.json");
+
+    if let Some(meta_contents) = vfs.read(&meta_path).with_not_found()? {
+        let metadata = AdjacentMetadata::from_slice(&meta_contents, meta_path)?;
+        Ok(Some(metadata))
+    } else {
+        Ok(None)
+    }
 }
 
 #[cfg(test)]