Port over to new snapshot system

server/src/rbx_session.rs

@@ -1,23 +1,20 @@
 use std::{
-    borrow::Cow,
     collections::HashMap,
-    fmt,
     path::{Path, PathBuf},
     str,
     sync::{Arc, Mutex},
 };
 
-use serde_derive::{Serialize, Deserialize};
 use log::{info, trace};
-use failure::Fail;
+use rbx_tree::{RbxTree, RbxId};
-use rbx_tree::{RbxTree, RbxInstanceProperties, RbxValue, RbxId};
 
 use crate::{
-    project::{Project, ProjectNode, InstanceProjectNodeMetadata},
+    project::{Project, InstanceProjectNodeMetadata},
     message_queue::MessageQueue,
-    imfs::{Imfs, ImfsItem, ImfsFile},
+    imfs::{Imfs, ImfsItem},
     path_map::PathMap,
-    snapshot_reconciler::{RbxSnapshotInstance, InstanceChanges, snapshot_from_tree, reify_root, reconcile_subtree},
+    rbx_snapshot::{SnapshotMetadata, snapshot_project_tree, snapshot_imfs_path},
+    snapshot_reconciler::{InstanceChanges, reify_root, reconcile_subtree},
 };
 
 const INIT_SCRIPT: &str = "init.lua";
@@ -45,7 +42,7 @@ impl RbxSession {
 
         let tree = {
             let temp_imfs = imfs.lock().unwrap();
-            construct_initial_tree(&project, &temp_imfs, &mut path_map, &mut instance_metadata_map, &mut sync_point_names)
+            reify_initial_tree(&project, &temp_imfs, &mut path_map, &mut instance_metadata_map, &mut sync_point_names)
         };
 
         RbxSession {
@@ -83,7 +80,10 @@ impl RbxSession {
 
             trace!("Snapshotting path {}", path_to_snapshot.display());
 
-            let maybe_snapshot = snapshot_instances_from_imfs(&imfs, &path_to_snapshot, &mut self.sync_point_names)
+            let mut snapshot_meta = SnapshotMetadata {
+                sync_point_names: &mut self.sync_point_names,
+            };
+            let maybe_snapshot = snapshot_imfs_path(&imfs, &mut snapshot_meta, &path_to_snapshot)
                 .unwrap_or_else(|_| panic!("Could not generate instance snapshot for path {}", path_to_snapshot.display()));
 
             let snapshot = match maybe_snapshot {
@@ -169,372 +169,25 @@ pub fn construct_oneoff_tree(project: &Project, imfs: &Imfs) -> RbxTree {
     let mut path_map = PathMap::new();
     let mut instance_metadata_map = HashMap::new();
     let mut sync_point_names = HashMap::new();
-    construct_initial_tree(project, imfs, &mut path_map, &mut instance_metadata_map, &mut sync_point_names)
+    reify_initial_tree(project, imfs, &mut path_map, &mut instance_metadata_map, &mut sync_point_names)
 }
 
-fn construct_initial_tree(
+fn reify_initial_tree(
     project: &Project,
     imfs: &Imfs,
     path_map: &mut PathMap<RbxId>,
     instance_metadata_map: &mut HashMap<RbxId, InstanceProjectNodeMetadata>,
     sync_point_names: &mut HashMap<PathBuf, String>,
 ) -> RbxTree {
-    let snapshot = construct_project_node(
+    let mut meta = SnapshotMetadata {
-        imfs,
-        &project.name,
-        &project.tree,
         sync_point_names,
-    );
+    };
+    let snapshot = snapshot_project_tree(imfs, &mut meta, project)
+        .expect("Could not snapshot project tree")
+        .expect("Project did not produce any instances");
 
     let mut changes = InstanceChanges::default();
     let tree = reify_root(&snapshot, path_map, instance_metadata_map, &mut changes);
 
     tree
-}
-
-fn construct_project_node<'a>(
-    imfs: &'a Imfs,
-    instance_name: &'a str,
-    project_node: &'a ProjectNode,
-    sync_point_names: &mut HashMap<PathBuf, String>,
-) -> RbxSnapshotInstance<'a> {
-    match project_node {
-        ProjectNode::Instance(node) => {
-            let mut children = Vec::new();
-
-            for (child_name, child_project_node) in &node.children {
-                children.push(construct_project_node(imfs, child_name, child_project_node, sync_point_names));
-            }
-
-            RbxSnapshotInstance {
-                class_name: Cow::Borrowed(&node.class_name),
-                name: Cow::Borrowed(instance_name),
-                properties: node.properties.clone(),
-                children,
-                source_path: None,
-                metadata: Some(node.metadata.clone()),
-            }
-        },
-        ProjectNode::SyncPoint(node) => {
-            // TODO: Propagate errors upward instead of dying
-            let mut snapshot = snapshot_instances_from_imfs(imfs, &node.path, sync_point_names)
-                .expect("Could not reify nodes from Imfs")
-                .expect("Sync point node did not result in an instance");
-
-            snapshot.name = Cow::Borrowed(instance_name);
-            sync_point_names.insert(node.path.clone(), instance_name.to_string());
-
-            snapshot
-        },
-    }
-}
-
-#[derive(Debug, Clone, Copy)]
-enum FileType {
-    ModuleScript,
-    ServerScript,
-    ClientScript,
-    StringValue,
-    LocalizationTable,
-    XmlModel,
-    BinaryModel,
-}
-
-fn get_trailing<'a>(input: &'a str, trailer: &str) -> Option<&'a str> {
-    if input.ends_with(trailer) {
-        let end = input.len().saturating_sub(trailer.len());
-        Some(&input[..end])
-    } else {
-        None
-    }
-}
-
-fn classify_file(file: &ImfsFile) -> Option<(&str, FileType)> {
-    static EXTENSIONS_TO_TYPES: &[(&str, FileType)] = &[
-        (".server.lua", FileType::ServerScript),
-        (".client.lua", FileType::ClientScript),
-        (".lua", FileType::ModuleScript),
-        (".csv", FileType::LocalizationTable),
-        (".txt", FileType::StringValue),
-        (".rbxmx", FileType::XmlModel),
-        (".rbxm", FileType::BinaryModel),
-    ];
-
-    let file_name = file.path.file_name()?.to_str()?;
-
-    for (extension, file_type) in EXTENSIONS_TO_TYPES {
-        if let Some(instance_name) = get_trailing(file_name, extension) {
-            return Some((instance_name, *file_type))
-        }
-    }
-
-    None
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(rename_all = "PascalCase")]
-struct LocalizationEntryCsv {
-    key: String,
-    context: String,
-    example: String,
-    source: String,
-    #[serde(flatten)]
-    values: HashMap<String, String>,
-}
-
-impl LocalizationEntryCsv {
-    fn to_json(self) -> LocalizationEntryJson {
-        LocalizationEntryJson {
-            key: self.key,
-            context: self.context,
-            example: self.example,
-            source: self.source,
-            values: self.values,
-        }
-    }
-}
-
-#[derive(Debug, Serialize, Deserialize)]
-#[serde(rename_all = "camelCase")]
-struct LocalizationEntryJson {
-    key: String,
-    context: String,
-    example: String,
-    source: String,
-    values: HashMap<String, String>,
-}
-
-#[derive(Debug, Fail)]
-enum SnapshotError {
-    DidNotExist(PathBuf),
-
-    // TODO: Add file path to the error message?
-    Utf8Error {
-        #[fail(cause)]
-        inner: str::Utf8Error,
-        path: PathBuf,
-    },
-
-    XmlModelDecodeError {
-        inner: rbx_xml::DecodeError,
-        path: PathBuf,
-    },
-
-    BinaryModelDecodeError {
-        inner: rbx_binary::DecodeError,
-        path: PathBuf,
-    },
-}
-
-impl fmt::Display for SnapshotError {
-    fn fmt(&self, output: &mut fmt::Formatter) -> fmt::Result {
-        match self {
-            SnapshotError::DidNotExist(path) => write!(output, "Path did not exist: {}", path.display()),
-            SnapshotError::Utf8Error { inner, path } => {
-                write!(output, "Invalid UTF-8: {} in path {}", inner, path.display())
-            },
-            SnapshotError::XmlModelDecodeError { inner, path } => {
-                write!(output, "Malformed rbxmx model: {:?} in path {}", inner, path.display())
-            },
-            SnapshotError::BinaryModelDecodeError { inner, path } => {
-                write!(output, "Malformed rbxm model: {:?} in path {}", inner, path.display())
-            },
-        }
-    }
-}
-
-fn snapshot_xml_model<'a>(
-    instance_name: Cow<'a, str>,
-    file: &ImfsFile,
-) -> Result<Option<RbxSnapshotInstance<'a>>, SnapshotError> {
-    let mut temp_tree = RbxTree::new(RbxInstanceProperties {
-        name: "Temp".to_owned(),
-        class_name: "Folder".to_owned(),
-        properties: HashMap::new(),
-    });
-
-    let root_id = temp_tree.get_root_id();
-    rbx_xml::decode(&mut temp_tree, root_id, file.contents.as_slice())
-        .map_err(|inner| SnapshotError::XmlModelDecodeError {
-            inner,
-            path: file.path.clone(),
-        })?;
-
-    let root_instance = temp_tree.get_instance(root_id).unwrap();
-    let children = root_instance.get_children_ids();
-
-    match children.len() {
-        0 => Ok(None),
-        1 => {
-            let mut snapshot = snapshot_from_tree(&temp_tree, children[0]).unwrap();
-            snapshot.name = instance_name;
-            Ok(Some(snapshot))
-        },
-        _ => panic!("Rojo doesn't have support for model files with multiple roots yet"),
-    }
-}
-
-fn snapshot_binary_model<'a>(
-    instance_name: Cow<'a, str>,
-    file: &ImfsFile,
-) -> Result<Option<RbxSnapshotInstance<'a>>, SnapshotError> {
-    let mut temp_tree = RbxTree::new(RbxInstanceProperties {
-        name: "Temp".to_owned(),
-        class_name: "Folder".to_owned(),
-        properties: HashMap::new(),
-    });
-
-    let root_id = temp_tree.get_root_id();
-    rbx_binary::decode(&mut temp_tree, root_id, file.contents.as_slice())
-        .map_err(|inner| SnapshotError::BinaryModelDecodeError {
-            inner,
-            path: file.path.clone(),
-        })?;
-
-    let root_instance = temp_tree.get_instance(root_id).unwrap();
-    let children = root_instance.get_children_ids();
-
-    match children.len() {
-        0 => Ok(None),
-        1 => {
-            let mut snapshot = snapshot_from_tree(&temp_tree, children[0]).unwrap();
-            snapshot.name = instance_name;
-            Ok(Some(snapshot))
-        },
-        _ => panic!("Rojo doesn't have support for model files with multiple roots yet"),
-    }
-}
-
-fn snapshot_instances_from_imfs<'a>(
-    imfs: &'a Imfs,
-    imfs_path: &Path,
-    sync_point_names: &HashMap<PathBuf, String>,
-) -> Result<Option<RbxSnapshotInstance<'a>>, SnapshotError> {
-    match imfs.get(imfs_path) {
-        Some(ImfsItem::File(file)) => {
-            let (instance_name, file_type) = match classify_file(file) {
-                Some(info) => info,
-                None => return Ok(None),
-            };
-
-            let instance_name = if let Some(actual_name) = sync_point_names.get(imfs_path) {
-                Cow::Owned(actual_name.clone())
-            } else {
-                Cow::Borrowed(instance_name)
-            };
-
-            let class_name = match file_type {
-                FileType::ModuleScript => "ModuleScript",
-                FileType::ServerScript => "Script",
-                FileType::ClientScript => "LocalScript",
-                FileType::StringValue => "StringValue",
-                FileType::LocalizationTable => "LocalizationTable",
-                FileType::XmlModel => return snapshot_xml_model(instance_name, file),
-                FileType::BinaryModel => return snapshot_binary_model(instance_name, file),
-            };
-
-            let contents = str::from_utf8(&file.contents)
-                .map_err(|inner| SnapshotError::Utf8Error {
-                    inner,
-                    path: imfs_path.to_path_buf(),
-                })?;
-
-            let mut properties = HashMap::new();
-
-            match file_type {
-                FileType::ModuleScript | FileType::ServerScript | FileType::ClientScript => {
-                    properties.insert(String::from("Source"), RbxValue::String {
-                        value: contents.to_string(),
-                    });
-                },
-                FileType::StringValue => {
-                    properties.insert(String::from("Value"), RbxValue::String {
-                        value: contents.to_string(),
-                    });
-                },
-                FileType::LocalizationTable => {
-                    let entries: Vec<LocalizationEntryJson> = csv::Reader::from_reader(contents.as_bytes())
-                        .deserialize()
-                        .map(|result| result.expect("Malformed localization table found!"))
-                        .map(LocalizationEntryCsv::to_json)
-                        .collect();
-
-                    let table_contents = serde_json::to_string(&entries)
-                        .expect("Could not encode JSON for localization table");
-
-                    properties.insert(String::from("Contents"), RbxValue::String {
-                        value: table_contents,
-                    });
-                },
-                FileType::XmlModel | FileType::BinaryModel => unreachable!(),
-            }
-
-            Ok(Some(RbxSnapshotInstance {
-                name: instance_name,
-                class_name: Cow::Borrowed(class_name),
-                properties,
-                children: Vec::new(),
-                source_path: Some(file.path.clone()),
-                metadata: None,
-            }))
-        },
-        Some(ImfsItem::Directory(directory)) => {
-            // TODO: Expand init support to handle server and client scripts
-            let init_path = directory.path.join(INIT_SCRIPT);
-            let init_server_path = directory.path.join(INIT_SERVER_SCRIPT);
-            let init_client_path = directory.path.join(INIT_CLIENT_SCRIPT);
-
-            let mut instance = if directory.children.contains(&init_path) {
-                snapshot_instances_from_imfs(imfs, &init_path, sync_point_names)?
-                    .expect("Could not snapshot instance from file that existed!")
-            } else if directory.children.contains(&init_server_path) {
-                snapshot_instances_from_imfs(imfs, &init_server_path, sync_point_names)?
-                    .expect("Could not snapshot instance from file that existed!")
-            } else if directory.children.contains(&init_client_path) {
-                snapshot_instances_from_imfs(imfs, &init_client_path, sync_point_names)?
-                    .expect("Could not snapshot instance from file that existed!")
-            } else {
-                RbxSnapshotInstance {
-                    class_name: Cow::Borrowed("Folder"),
-                    name: Cow::Borrowed(""),
-                    properties: HashMap::new(),
-                    children: Vec::new(),
-                    source_path: Some(directory.path.clone()),
-                    metadata: None,
-                }
-            };
-
-            // We have to be careful not to lose instance names that are
-            // specified in the project manifest. We store them in
-            // sync_point_names when the original tree is constructed.
-            instance.name = if let Some(actual_name) = sync_point_names.get(&directory.path) {
-                Cow::Owned(actual_name.clone())
-            } else {
-                Cow::Borrowed(directory.path
-                    .file_name().expect("Could not extract file name")
-                    .to_str().expect("Could not convert path to UTF-8"))
-            };
-
-            for child_path in &directory.children {
-                match child_path.file_name().unwrap().to_str().unwrap() {
-                    INIT_SCRIPT | INIT_SERVER_SCRIPT | INIT_CLIENT_SCRIPT => {
-                        // The existence of files with these names modifies the
-                        // parent instance and is handled above, so we can skip
-                        // them here.
-                    },
-                    _ => {
-                        match snapshot_instances_from_imfs(imfs, child_path, sync_point_names)? {
-                            Some(child) => {
-                                instance.children.push(child);
-                            },
-                            None => {},
-                        }
-                    },
-                }
-            }
-
-            Ok(Some(instance))
-        },
-        None => Err(SnapshotError::DidNotExist(imfs_path.to_path_buf())),
-    }
 }

server/src/rbx_snapshot.rs

@@ -8,7 +8,7 @@ use std::{
 
 use serde_derive::{Serialize, Deserialize};
 use maplit::hashmap;
-use rbx_tree::RbxValue;
+use rbx_tree::{RbxTree, RbxValue, RbxInstanceProperties};
 use failure::Fail;
 
 use crate::{
@@ -24,7 +24,10 @@ use crate::{
         InstanceProjectNode,
         SyncPointProjectNode,
     },
-    snapshot_reconciler::RbxSnapshotInstance,
+    snapshot_reconciler::{
+        RbxSnapshotInstance,
+        snapshot_from_tree,
+    },
 };
 
 const INIT_MODULE_NAME: &str = "init.lua";
@@ -34,7 +37,7 @@ const INIT_CLIENT_NAME: &str = "init.client.lua";
 pub type SnapshotResult<'a> = Result<Option<RbxSnapshotInstance<'a>>, SnapshotError>;
 
 pub struct SnapshotMetadata<'meta> {
-    sync_point_names: &'meta mut HashMap<PathBuf, String>,
+    pub sync_point_names: &'meta mut HashMap<PathBuf, String>,
 }
 
 #[derive(Debug, Fail)]
@@ -140,7 +143,7 @@ fn snapshot_sync_point_node<'source>(
     Ok(Some(snapshot))
 }
 
-fn snapshot_imfs_path<'source>(
+pub fn snapshot_imfs_path<'source>(
     imfs: &'source Imfs,
     metadata: &mut SnapshotMetadata,
     path: &Path
@@ -159,7 +162,7 @@ fn snapshot_imfs_item<'source>(
     item: &'source ImfsItem,
 ) -> SnapshotResult<'source> {
     match item {
-        ImfsItem::File(file) => snapshot_imfs_file(imfs, metadata, file),
+        ImfsItem::File(file) => snapshot_imfs_file(metadata, file),
         ImfsItem::Directory(directory) => snapshot_imfs_directory(imfs, metadata, directory),
     }
 }
@@ -224,7 +227,6 @@ fn snapshot_imfs_directory<'source>(
 }
 
 fn snapshot_imfs_file<'source>(
-    imfs: &'source Imfs,
     metadata: &mut SnapshotMetadata,
     file: &'source ImfsFile,
 ) -> SnapshotResult<'source> {
@@ -232,11 +234,11 @@ fn snapshot_imfs_file<'source>(
         .map(|v| v.to_str().expect("Could not convert extension to UTF-8"));
 
     let mut maybe_snapshot = match extension {
-        Some("lua") => snapshot_lua_file(metadata, file)?,
+        Some("lua") => snapshot_lua_file(file)?,
-        Some("csv") => snapshot_csv_file(metadata, file)?,
+        Some("csv") => snapshot_csv_file(file)?,
-        Some("txt") => snapshot_txt_file(metadata, file)?,
+        Some("txt") => snapshot_txt_file(file)?,
-        Some("rbxmx") => snapshot_xml_model_file(metadata, file)?,
+        Some("rbxmx") => snapshot_xml_model_file(file)?,
-        Some("rbxm") => snapshot_binary_model_file(metadata, file)?,
+        Some("rbxm") => snapshot_binary_model_file(file)?,
         Some(_) | None => return Ok(None),
     };
 
@@ -251,7 +253,6 @@ fn snapshot_imfs_file<'source>(
 }
 
 fn snapshot_lua_file<'source>(
-    metadata: &mut SnapshotMetadata,
     file: &'source ImfsFile,
 ) -> SnapshotResult<'source> {
     let file_name = file.path
@@ -296,7 +297,6 @@ fn match_trailing<'a>(input: &'a str, trailer: &str) -> Option<&'a str> {
 }
 
 fn snapshot_txt_file<'source>(
-    metadata: &mut SnapshotMetadata,
     file: &'source ImfsFile,
 ) -> SnapshotResult<'source> {
     let instance_name = file.path
@@ -324,7 +324,6 @@ fn snapshot_txt_file<'source>(
 }
 
 fn snapshot_csv_file<'source>(
-    metadata: &mut SnapshotMetadata,
     file: &'source ImfsFile,
 ) -> SnapshotResult<'source> {
     let instance_name = file.path
@@ -389,15 +388,69 @@ struct LocalizationEntryJson {
 }
 
 fn snapshot_xml_model_file<'source>(
-    metadata: &mut SnapshotMetadata,
     file: &'source ImfsFile,
 ) -> SnapshotResult<'source> {
-    unimplemented!()
+    let instance_name = file.path
+        .file_stem().expect("Could not extract file stem")
+        .to_str().expect("Could not convert path to UTF-8");
+
+    let mut temp_tree = RbxTree::new(RbxInstanceProperties {
+        name: "Temp".to_owned(),
+        class_name: "Folder".to_owned(),
+        properties: HashMap::new(),
+    });
+
+    let root_id = temp_tree.get_root_id();
+    rbx_xml::decode(&mut temp_tree, root_id, file.contents.as_slice())
+        .map_err(|inner| SnapshotError::XmlModelDecodeError {
+            inner,
+            path: file.path.clone(),
+        })?;
+
+    let root_instance = temp_tree.get_instance(root_id).unwrap();
+    let children = root_instance.get_children_ids();
+
+    match children.len() {
+        0 => Ok(None),
+        1 => {
+            let mut snapshot = snapshot_from_tree(&temp_tree, children[0]).unwrap();
+            snapshot.name = Cow::Borrowed(instance_name);
+            Ok(Some(snapshot))
+        },
+        _ => panic!("Rojo doesn't have support for model files with multiple roots yet"),
+    }
 }
 
 fn snapshot_binary_model_file<'source>(
-    metadata: &mut SnapshotMetadata,
     file: &'source ImfsFile,
 ) -> SnapshotResult<'source> {
-    unimplemented!()
+    let instance_name = file.path
+        .file_stem().expect("Could not extract file stem")
+        .to_str().expect("Could not convert path to UTF-8");
+
+    let mut temp_tree = RbxTree::new(RbxInstanceProperties {
+        name: "Temp".to_owned(),
+        class_name: "Folder".to_owned(),
+        properties: HashMap::new(),
+    });
+
+    let root_id = temp_tree.get_root_id();
+    rbx_binary::decode(&mut temp_tree, root_id, file.contents.as_slice())
+        .map_err(|inner| SnapshotError::BinaryModelDecodeError {
+            inner,
+            path: file.path.clone(),
+        })?;
+
+    let root_instance = temp_tree.get_instance(root_id).unwrap();
+    let children = root_instance.get_children_ids();
+
+    match children.len() {
+        0 => Ok(None),
+        1 => {
+            let mut snapshot = snapshot_from_tree(&temp_tree, children[0]).unwrap();
+            snapshot.name = Cow::Borrowed(instance_name);
+            Ok(Some(snapshot))
+        },
+        _ => panic!("Rojo doesn't have support for model files with multiple roots yet"),
+    }
 }