From 585806837e07320bddb990333c90646631c81929 Mon Sep 17 00:00:00 2001 From: Lucien Greathouse Date: Tue, 15 Jan 2019 18:04:06 -0800 Subject: [PATCH] Port over to new snapshot system --- server/src/rbx_session.rs | 381 ++----------------------------------- server/src/rbx_snapshot.rs | 89 +++++++-- 2 files changed, 88 insertions(+), 382 deletions(-) diff --git a/server/src/rbx_session.rs b/server/src/rbx_session.rs index 7b740f09..1820424e 100644 --- a/server/src/rbx_session.rs +++ b/server/src/rbx_session.rs @@ -1,23 +1,20 @@ use std::{ - borrow::Cow, collections::HashMap, - fmt, path::{Path, PathBuf}, str, sync::{Arc, Mutex}, }; -use serde_derive::{Serialize, Deserialize}; use log::{info, trace}; -use failure::Fail; -use rbx_tree::{RbxTree, RbxInstanceProperties, RbxValue, RbxId}; +use rbx_tree::{RbxTree, RbxId}; use crate::{ - project::{Project, ProjectNode, InstanceProjectNodeMetadata}, + project::{Project, InstanceProjectNodeMetadata}, message_queue::MessageQueue, - imfs::{Imfs, ImfsItem, ImfsFile}, + imfs::{Imfs, ImfsItem}, path_map::PathMap, - snapshot_reconciler::{RbxSnapshotInstance, InstanceChanges, snapshot_from_tree, reify_root, reconcile_subtree}, + rbx_snapshot::{SnapshotMetadata, snapshot_project_tree, snapshot_imfs_path}, + snapshot_reconciler::{InstanceChanges, reify_root, reconcile_subtree}, }; const INIT_SCRIPT: &str = "init.lua"; @@ -45,7 +42,7 @@ impl RbxSession { let tree = { let temp_imfs = imfs.lock().unwrap(); - construct_initial_tree(&project, &temp_imfs, &mut path_map, &mut instance_metadata_map, &mut sync_point_names) + reify_initial_tree(&project, &temp_imfs, &mut path_map, &mut instance_metadata_map, &mut sync_point_names) }; RbxSession { @@ -83,7 +80,10 @@ impl RbxSession { trace!("Snapshotting path {}", path_to_snapshot.display()); - let maybe_snapshot = snapshot_instances_from_imfs(&imfs, &path_to_snapshot, &mut self.sync_point_names) + let mut snapshot_meta = SnapshotMetadata { + sync_point_names: &mut self.sync_point_names, + }; + let maybe_snapshot = snapshot_imfs_path(&imfs, &mut snapshot_meta, &path_to_snapshot) .unwrap_or_else(|_| panic!("Could not generate instance snapshot for path {}", path_to_snapshot.display())); let snapshot = match maybe_snapshot { @@ -169,372 +169,25 @@ pub fn construct_oneoff_tree(project: &Project, imfs: &Imfs) -> RbxTree { let mut path_map = PathMap::new(); let mut instance_metadata_map = HashMap::new(); let mut sync_point_names = HashMap::new(); - construct_initial_tree(project, imfs, &mut path_map, &mut instance_metadata_map, &mut sync_point_names) + reify_initial_tree(project, imfs, &mut path_map, &mut instance_metadata_map, &mut sync_point_names) } -fn construct_initial_tree( +fn reify_initial_tree( project: &Project, imfs: &Imfs, path_map: &mut PathMap, instance_metadata_map: &mut HashMap, sync_point_names: &mut HashMap, ) -> RbxTree { - let snapshot = construct_project_node( - imfs, - &project.name, - &project.tree, + let mut meta = SnapshotMetadata { sync_point_names, - ); + }; + let snapshot = snapshot_project_tree(imfs, &mut meta, project) + .expect("Could not snapshot project tree") + .expect("Project did not produce any instances"); let mut changes = InstanceChanges::default(); let tree = reify_root(&snapshot, path_map, instance_metadata_map, &mut changes); tree -} - -fn construct_project_node<'a>( - imfs: &'a Imfs, - instance_name: &'a str, - project_node: &'a ProjectNode, - sync_point_names: &mut HashMap, -) -> RbxSnapshotInstance<'a> { - match project_node { - ProjectNode::Instance(node) => { - let mut children = Vec::new(); - - for (child_name, child_project_node) in &node.children { - children.push(construct_project_node(imfs, child_name, child_project_node, sync_point_names)); - } - - RbxSnapshotInstance { - class_name: Cow::Borrowed(&node.class_name), - name: Cow::Borrowed(instance_name), - properties: node.properties.clone(), - children, - source_path: None, - metadata: Some(node.metadata.clone()), - } - }, - ProjectNode::SyncPoint(node) => { - // TODO: Propagate errors upward instead of dying - let mut snapshot = snapshot_instances_from_imfs(imfs, &node.path, sync_point_names) - .expect("Could not reify nodes from Imfs") - .expect("Sync point node did not result in an instance"); - - snapshot.name = Cow::Borrowed(instance_name); - sync_point_names.insert(node.path.clone(), instance_name.to_string()); - - snapshot - }, - } -} - -#[derive(Debug, Clone, Copy)] -enum FileType { - ModuleScript, - ServerScript, - ClientScript, - StringValue, - LocalizationTable, - XmlModel, - BinaryModel, -} - -fn get_trailing<'a>(input: &'a str, trailer: &str) -> Option<&'a str> { - if input.ends_with(trailer) { - let end = input.len().saturating_sub(trailer.len()); - Some(&input[..end]) - } else { - None - } -} - -fn classify_file(file: &ImfsFile) -> Option<(&str, FileType)> { - static EXTENSIONS_TO_TYPES: &[(&str, FileType)] = &[ - (".server.lua", FileType::ServerScript), - (".client.lua", FileType::ClientScript), - (".lua", FileType::ModuleScript), - (".csv", FileType::LocalizationTable), - (".txt", FileType::StringValue), - (".rbxmx", FileType::XmlModel), - (".rbxm", FileType::BinaryModel), - ]; - - let file_name = file.path.file_name()?.to_str()?; - - for (extension, file_type) in EXTENSIONS_TO_TYPES { - if let Some(instance_name) = get_trailing(file_name, extension) { - return Some((instance_name, *file_type)) - } - } - - None -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "PascalCase")] -struct LocalizationEntryCsv { - key: String, - context: String, - example: String, - source: String, - #[serde(flatten)] - values: HashMap, -} - -impl LocalizationEntryCsv { - fn to_json(self) -> LocalizationEntryJson { - LocalizationEntryJson { - key: self.key, - context: self.context, - example: self.example, - source: self.source, - values: self.values, - } - } -} - -#[derive(Debug, Serialize, Deserialize)] -#[serde(rename_all = "camelCase")] -struct LocalizationEntryJson { - key: String, - context: String, - example: String, - source: String, - values: HashMap, -} - -#[derive(Debug, Fail)] -enum SnapshotError { - DidNotExist(PathBuf), - - // TODO: Add file path to the error message? - Utf8Error { - #[fail(cause)] - inner: str::Utf8Error, - path: PathBuf, - }, - - XmlModelDecodeError { - inner: rbx_xml::DecodeError, - path: PathBuf, - }, - - BinaryModelDecodeError { - inner: rbx_binary::DecodeError, - path: PathBuf, - }, -} - -impl fmt::Display for SnapshotError { - fn fmt(&self, output: &mut fmt::Formatter) -> fmt::Result { - match self { - SnapshotError::DidNotExist(path) => write!(output, "Path did not exist: {}", path.display()), - SnapshotError::Utf8Error { inner, path } => { - write!(output, "Invalid UTF-8: {} in path {}", inner, path.display()) - }, - SnapshotError::XmlModelDecodeError { inner, path } => { - write!(output, "Malformed rbxmx model: {:?} in path {}", inner, path.display()) - }, - SnapshotError::BinaryModelDecodeError { inner, path } => { - write!(output, "Malformed rbxm model: {:?} in path {}", inner, path.display()) - }, - } - } -} - -fn snapshot_xml_model<'a>( - instance_name: Cow<'a, str>, - file: &ImfsFile, -) -> Result>, SnapshotError> { - let mut temp_tree = RbxTree::new(RbxInstanceProperties { - name: "Temp".to_owned(), - class_name: "Folder".to_owned(), - properties: HashMap::new(), - }); - - let root_id = temp_tree.get_root_id(); - rbx_xml::decode(&mut temp_tree, root_id, file.contents.as_slice()) - .map_err(|inner| SnapshotError::XmlModelDecodeError { - inner, - path: file.path.clone(), - })?; - - let root_instance = temp_tree.get_instance(root_id).unwrap(); - let children = root_instance.get_children_ids(); - - match children.len() { - 0 => Ok(None), - 1 => { - let mut snapshot = snapshot_from_tree(&temp_tree, children[0]).unwrap(); - snapshot.name = instance_name; - Ok(Some(snapshot)) - }, - _ => panic!("Rojo doesn't have support for model files with multiple roots yet"), - } -} - -fn snapshot_binary_model<'a>( - instance_name: Cow<'a, str>, - file: &ImfsFile, -) -> Result>, SnapshotError> { - let mut temp_tree = RbxTree::new(RbxInstanceProperties { - name: "Temp".to_owned(), - class_name: "Folder".to_owned(), - properties: HashMap::new(), - }); - - let root_id = temp_tree.get_root_id(); - rbx_binary::decode(&mut temp_tree, root_id, file.contents.as_slice()) - .map_err(|inner| SnapshotError::BinaryModelDecodeError { - inner, - path: file.path.clone(), - })?; - - let root_instance = temp_tree.get_instance(root_id).unwrap(); - let children = root_instance.get_children_ids(); - - match children.len() { - 0 => Ok(None), - 1 => { - let mut snapshot = snapshot_from_tree(&temp_tree, children[0]).unwrap(); - snapshot.name = instance_name; - Ok(Some(snapshot)) - }, - _ => panic!("Rojo doesn't have support for model files with multiple roots yet"), - } -} - -fn snapshot_instances_from_imfs<'a>( - imfs: &'a Imfs, - imfs_path: &Path, - sync_point_names: &HashMap, -) -> Result>, SnapshotError> { - match imfs.get(imfs_path) { - Some(ImfsItem::File(file)) => { - let (instance_name, file_type) = match classify_file(file) { - Some(info) => info, - None => return Ok(None), - }; - - let instance_name = if let Some(actual_name) = sync_point_names.get(imfs_path) { - Cow::Owned(actual_name.clone()) - } else { - Cow::Borrowed(instance_name) - }; - - let class_name = match file_type { - FileType::ModuleScript => "ModuleScript", - FileType::ServerScript => "Script", - FileType::ClientScript => "LocalScript", - FileType::StringValue => "StringValue", - FileType::LocalizationTable => "LocalizationTable", - FileType::XmlModel => return snapshot_xml_model(instance_name, file), - FileType::BinaryModel => return snapshot_binary_model(instance_name, file), - }; - - let contents = str::from_utf8(&file.contents) - .map_err(|inner| SnapshotError::Utf8Error { - inner, - path: imfs_path.to_path_buf(), - })?; - - let mut properties = HashMap::new(); - - match file_type { - FileType::ModuleScript | FileType::ServerScript | FileType::ClientScript => { - properties.insert(String::from("Source"), RbxValue::String { - value: contents.to_string(), - }); - }, - FileType::StringValue => { - properties.insert(String::from("Value"), RbxValue::String { - value: contents.to_string(), - }); - }, - FileType::LocalizationTable => { - let entries: Vec = csv::Reader::from_reader(contents.as_bytes()) - .deserialize() - .map(|result| result.expect("Malformed localization table found!")) - .map(LocalizationEntryCsv::to_json) - .collect(); - - let table_contents = serde_json::to_string(&entries) - .expect("Could not encode JSON for localization table"); - - properties.insert(String::from("Contents"), RbxValue::String { - value: table_contents, - }); - }, - FileType::XmlModel | FileType::BinaryModel => unreachable!(), - } - - Ok(Some(RbxSnapshotInstance { - name: instance_name, - class_name: Cow::Borrowed(class_name), - properties, - children: Vec::new(), - source_path: Some(file.path.clone()), - metadata: None, - })) - }, - Some(ImfsItem::Directory(directory)) => { - // TODO: Expand init support to handle server and client scripts - let init_path = directory.path.join(INIT_SCRIPT); - let init_server_path = directory.path.join(INIT_SERVER_SCRIPT); - let init_client_path = directory.path.join(INIT_CLIENT_SCRIPT); - - let mut instance = if directory.children.contains(&init_path) { - snapshot_instances_from_imfs(imfs, &init_path, sync_point_names)? - .expect("Could not snapshot instance from file that existed!") - } else if directory.children.contains(&init_server_path) { - snapshot_instances_from_imfs(imfs, &init_server_path, sync_point_names)? - .expect("Could not snapshot instance from file that existed!") - } else if directory.children.contains(&init_client_path) { - snapshot_instances_from_imfs(imfs, &init_client_path, sync_point_names)? - .expect("Could not snapshot instance from file that existed!") - } else { - RbxSnapshotInstance { - class_name: Cow::Borrowed("Folder"), - name: Cow::Borrowed(""), - properties: HashMap::new(), - children: Vec::new(), - source_path: Some(directory.path.clone()), - metadata: None, - } - }; - - // We have to be careful not to lose instance names that are - // specified in the project manifest. We store them in - // sync_point_names when the original tree is constructed. - instance.name = if let Some(actual_name) = sync_point_names.get(&directory.path) { - Cow::Owned(actual_name.clone()) - } else { - Cow::Borrowed(directory.path - .file_name().expect("Could not extract file name") - .to_str().expect("Could not convert path to UTF-8")) - }; - - for child_path in &directory.children { - match child_path.file_name().unwrap().to_str().unwrap() { - INIT_SCRIPT | INIT_SERVER_SCRIPT | INIT_CLIENT_SCRIPT => { - // The existence of files with these names modifies the - // parent instance and is handled above, so we can skip - // them here. - }, - _ => { - match snapshot_instances_from_imfs(imfs, child_path, sync_point_names)? { - Some(child) => { - instance.children.push(child); - }, - None => {}, - } - }, - } - } - - Ok(Some(instance)) - }, - None => Err(SnapshotError::DidNotExist(imfs_path.to_path_buf())), - } } \ No newline at end of file diff --git a/server/src/rbx_snapshot.rs b/server/src/rbx_snapshot.rs index 3ccc4926..35446f14 100644 --- a/server/src/rbx_snapshot.rs +++ b/server/src/rbx_snapshot.rs @@ -8,7 +8,7 @@ use std::{ use serde_derive::{Serialize, Deserialize}; use maplit::hashmap; -use rbx_tree::RbxValue; +use rbx_tree::{RbxTree, RbxValue, RbxInstanceProperties}; use failure::Fail; use crate::{ @@ -24,7 +24,10 @@ use crate::{ InstanceProjectNode, SyncPointProjectNode, }, - snapshot_reconciler::RbxSnapshotInstance, + snapshot_reconciler::{ + RbxSnapshotInstance, + snapshot_from_tree, + }, }; const INIT_MODULE_NAME: &str = "init.lua"; @@ -34,7 +37,7 @@ const INIT_CLIENT_NAME: &str = "init.client.lua"; pub type SnapshotResult<'a> = Result>, SnapshotError>; pub struct SnapshotMetadata<'meta> { - sync_point_names: &'meta mut HashMap, + pub sync_point_names: &'meta mut HashMap, } #[derive(Debug, Fail)] @@ -140,7 +143,7 @@ fn snapshot_sync_point_node<'source>( Ok(Some(snapshot)) } -fn snapshot_imfs_path<'source>( +pub fn snapshot_imfs_path<'source>( imfs: &'source Imfs, metadata: &mut SnapshotMetadata, path: &Path @@ -159,7 +162,7 @@ fn snapshot_imfs_item<'source>( item: &'source ImfsItem, ) -> SnapshotResult<'source> { match item { - ImfsItem::File(file) => snapshot_imfs_file(imfs, metadata, file), + ImfsItem::File(file) => snapshot_imfs_file(metadata, file), ImfsItem::Directory(directory) => snapshot_imfs_directory(imfs, metadata, directory), } } @@ -224,7 +227,6 @@ fn snapshot_imfs_directory<'source>( } fn snapshot_imfs_file<'source>( - imfs: &'source Imfs, metadata: &mut SnapshotMetadata, file: &'source ImfsFile, ) -> SnapshotResult<'source> { @@ -232,11 +234,11 @@ fn snapshot_imfs_file<'source>( .map(|v| v.to_str().expect("Could not convert extension to UTF-8")); let mut maybe_snapshot = match extension { - Some("lua") => snapshot_lua_file(metadata, file)?, - Some("csv") => snapshot_csv_file(metadata, file)?, - Some("txt") => snapshot_txt_file(metadata, file)?, - Some("rbxmx") => snapshot_xml_model_file(metadata, file)?, - Some("rbxm") => snapshot_binary_model_file(metadata, file)?, + Some("lua") => snapshot_lua_file(file)?, + Some("csv") => snapshot_csv_file(file)?, + Some("txt") => snapshot_txt_file(file)?, + Some("rbxmx") => snapshot_xml_model_file(file)?, + Some("rbxm") => snapshot_binary_model_file(file)?, Some(_) | None => return Ok(None), }; @@ -251,7 +253,6 @@ fn snapshot_imfs_file<'source>( } fn snapshot_lua_file<'source>( - metadata: &mut SnapshotMetadata, file: &'source ImfsFile, ) -> SnapshotResult<'source> { let file_name = file.path @@ -296,7 +297,6 @@ fn match_trailing<'a>(input: &'a str, trailer: &str) -> Option<&'a str> { } fn snapshot_txt_file<'source>( - metadata: &mut SnapshotMetadata, file: &'source ImfsFile, ) -> SnapshotResult<'source> { let instance_name = file.path @@ -324,7 +324,6 @@ fn snapshot_txt_file<'source>( } fn snapshot_csv_file<'source>( - metadata: &mut SnapshotMetadata, file: &'source ImfsFile, ) -> SnapshotResult<'source> { let instance_name = file.path @@ -389,15 +388,69 @@ struct LocalizationEntryJson { } fn snapshot_xml_model_file<'source>( - metadata: &mut SnapshotMetadata, file: &'source ImfsFile, ) -> SnapshotResult<'source> { - unimplemented!() + let instance_name = file.path + .file_stem().expect("Could not extract file stem") + .to_str().expect("Could not convert path to UTF-8"); + + let mut temp_tree = RbxTree::new(RbxInstanceProperties { + name: "Temp".to_owned(), + class_name: "Folder".to_owned(), + properties: HashMap::new(), + }); + + let root_id = temp_tree.get_root_id(); + rbx_xml::decode(&mut temp_tree, root_id, file.contents.as_slice()) + .map_err(|inner| SnapshotError::XmlModelDecodeError { + inner, + path: file.path.clone(), + })?; + + let root_instance = temp_tree.get_instance(root_id).unwrap(); + let children = root_instance.get_children_ids(); + + match children.len() { + 0 => Ok(None), + 1 => { + let mut snapshot = snapshot_from_tree(&temp_tree, children[0]).unwrap(); + snapshot.name = Cow::Borrowed(instance_name); + Ok(Some(snapshot)) + }, + _ => panic!("Rojo doesn't have support for model files with multiple roots yet"), + } } fn snapshot_binary_model_file<'source>( - metadata: &mut SnapshotMetadata, file: &'source ImfsFile, ) -> SnapshotResult<'source> { - unimplemented!() + let instance_name = file.path + .file_stem().expect("Could not extract file stem") + .to_str().expect("Could not convert path to UTF-8"); + + let mut temp_tree = RbxTree::new(RbxInstanceProperties { + name: "Temp".to_owned(), + class_name: "Folder".to_owned(), + properties: HashMap::new(), + }); + + let root_id = temp_tree.get_root_id(); + rbx_binary::decode(&mut temp_tree, root_id, file.contents.as_slice()) + .map_err(|inner| SnapshotError::BinaryModelDecodeError { + inner, + path: file.path.clone(), + })?; + + let root_instance = temp_tree.get_instance(root_id).unwrap(); + let children = root_instance.get_children_ids(); + + match children.len() { + 0 => Ok(None), + 1 => { + let mut snapshot = snapshot_from_tree(&temp_tree, children[0]).unwrap(); + snapshot.name = Cow::Borrowed(instance_name); + Ok(Some(snapshot)) + }, + _ => panic!("Rojo doesn't have support for model files with multiple roots yet"), + } } \ No newline at end of file