WIP: Server plugins via rlua (Lua 5.3) (#125)

* Add 'plugins' field to project and add rlua

* Scaffold out new SnapshotContext type (again) with plugin state

* Almost functional snapshot system with rlua proof-of-concept

* Gate plugin config on 'plugins-enabled' feature, tell Travis to test all features

* Guard remaining plugin setup code behind feature

* Bump minimum version to 1.33, should've caught this before

* Whoops, latest Rust is 1.32, not 1.33

.travis.yml

@@ -26,11 +26,12 @@ matrix:
         - luacov-coveralls -e $TRAVIS_BUILD_DIR/lua_install
 
     - language: rust
-      rust: 1.31.1
+      rust: 1.32.0
       cache: cargo
 
       script:
         - cargo test --verbose
+        - cargo test --verbose --all-features
 
     - language: rust
       rust: stable
@@ -38,10 +39,12 @@ matrix:
 
       script:
         - cargo test --verbose
+        - cargo test --verbose --all-features
 
     - language: rust
       rust: beta
       cache: cargo
 
       script:
-        - cargo test --verbose
+        - cargo test --verbose
+        - cargo test --verbose --all-features

README.md

@@ -62,7 +62,7 @@ If you use a plugin that _isn't_ Rojo for syncing code, open an issue and let me
 ## Contributing
 Pull requests are welcome!
 
-Rojo supports Rust 1.31.1 and newer. Any changes to the minimum required compiler version require a _minor_ version bump.
+Rojo supports Rust 1.32 and newer. Any changes to the minimum required compiler version require a _minor_ version bump.
 
 ## License
 Rojo is available under the terms of the Mozilla Public License, Version 2.0. See [LICENSE.txt](LICENSE.txt) for details.

server/Cargo.toml

@@ -7,6 +7,10 @@ license = "MIT"
 repository = "https://github.com/LPGhatguy/rojo"
 edition = "2018"
 
+[features]
+default = []
+server-plugins = []
+
 [lib]
 name = "librojo"
 path = "src/lib.rs"
@@ -31,6 +35,7 @@ rbx_dom_weak = "0.3.0"
 rbx_xml = "0.3.0"
 regex = "1.0"
 reqwest = "0.9.5"
+rlua = "0.16"
 ritz = "0.1.0"
 serde = "1.0"
 serde_derive = "1.0"

server/src/project.rs

@@ -24,6 +24,10 @@ struct SourceProject {
     name: String,
     tree: SourceProjectNode,
 
+    #[cfg_attr(not(feature = "plugins-enabled"), serde(skip_deserializing))]
+    #[serde(default = "Vec::new", skip_serializing_if = "Vec::is_empty")]
+    plugins: Vec<SourcePlugin>,
+
     #[serde(skip_serializing_if = "Option::is_none")]
     serve_port: Option<u16>,
 
@@ -33,12 +37,17 @@ struct SourceProject {
 
 impl SourceProject {
     /// Consumes the SourceProject and yields a Project, ready for prime-time.
-    pub fn into_project(self, project_file_location: &Path) -> Project {
+    pub fn into_project(mut self, project_file_location: &Path) -> Project {
         let tree = self.tree.into_project_node(project_file_location);
+        let plugins = self.plugins
+            .drain(..)
+            .map(|source_plugin| source_plugin.into_plugin(project_file_location))
+            .collect();
 
         Project {
             name: self.name,
             tree,
+            plugins,
             serve_port: self.serve_port,
             serve_place_ids: self.serve_place_ids,
             file_location: PathBuf::from(project_file_location),
@@ -95,6 +104,26 @@ impl SourceProjectNode {
     }
 }
 
+#[derive(Debug, Serialize, Deserialize)]
+struct SourcePlugin {
+    path: String,
+}
+
+impl SourcePlugin {
+    pub fn into_plugin(self, project_file_location: &Path) -> Plugin {
+        let path = if Path::new(&self.path).is_absolute() {
+            PathBuf::from(self.path)
+        } else {
+            let project_folder_location = project_file_location.parent().unwrap();
+            project_folder_location.join(self.path)
+        };
+
+        Plugin {
+            path,
+        }
+    }
+}
+
 /// Error returned by Project::load_exact
 #[derive(Debug, Fail)]
 pub enum ProjectLoadExactError {
@@ -198,10 +227,30 @@ impl ProjectNode {
     }
 }
 
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
+pub struct Plugin {
+    pub path: PathBuf,
+}
+
+impl Plugin {
+    fn to_source_plugin(&self, project_file_location: &Path) -> SourcePlugin {
+        let project_folder_location = project_file_location.parent().unwrap();
+        let path = match self.path.strip_prefix(project_folder_location) {
+            Ok(stripped) => stripped.to_str().unwrap().replace("\\", "/"),
+            Err(_) => format!("{}", self.path.display()),
+        };
+
+        SourcePlugin {
+            path,
+        }
+    }
+}
+
 #[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct Project {
     pub name: String,
     pub tree: ProjectNode,
+    pub plugins: Vec<Plugin>,
     pub serve_port: Option<u16>,
     pub serve_place_ids: Option<HashSet<u64>>,
     pub file_location: PathBuf,
@@ -246,6 +295,7 @@ impl Project {
         let project = Project {
             name: project_name.to_string(),
             tree,
+            plugins: Vec::new(),
             serve_port: None,
             serve_place_ids: None,
             file_location: project_path.clone(),
@@ -274,6 +324,7 @@ impl Project {
         let project = Project {
             name: project_name.to_string(),
             tree,
+            plugins: Vec::new(),
             serve_port: None,
             serve_place_ids: None,
             file_location: project_path.clone(),
@@ -384,9 +435,15 @@ impl Project {
     }
 
     fn to_source_project(&self) -> SourceProject {
+        let plugins = self.plugins
+            .iter()
+            .map(|plugin| plugin.to_source_plugin(&self.file_location))
+            .collect();
+
         SourceProject {
             name: self.name.clone(),
             tree: self.tree.to_source_node(&self.file_location),
+            plugins,
             serve_port: self.serve_port,
             serve_place_ids: self.serve_place_ids.clone(),
         }

server/src/rbx_session.rs

@@ -6,6 +6,7 @@ use std::{
     sync::{Arc, Mutex},
 };
 
+use rlua::Lua;
 use serde_derive::{Serialize, Deserialize};
 use log::{info, trace};
 use rbx_dom_weak::{RbxTree, RbxId};
@@ -15,7 +16,14 @@ use crate::{
     message_queue::MessageQueue,
     imfs::{Imfs, ImfsItem},
     path_map::PathMap,
-    rbx_snapshot::{snapshot_project_tree, snapshot_project_node, snapshot_imfs_path},
+    rbx_snapshot::{
+        SnapshotContext,
+        SnapshotPluginContext,
+        SnapshotPluginEntry,
+        snapshot_project_tree,
+        snapshot_project_node,
+        snapshot_imfs_path,
+    },
     snapshot_reconciler::{InstanceChanges, reify_root, reconcile_subtree},
 };
 
@@ -62,9 +70,41 @@ impl RbxSession {
         let mut instances_per_path = PathMap::new();
         let mut metadata_per_instance = HashMap::new();
 
+        let plugin_context = if cfg!(feature = "server-plugins") {
+            let lua = Lua::new();
+            let mut callback_key = None;
+
+            lua.context(|context| {
+                let callback = context.load(r#"
+                    return function(snapshot)
+                        print("got my snapshot:", snapshot)
+                        print("name:", snapshot.name, "class name:", snapshot.className)
+                    end"#)
+                    .set_name("a cool plugin").unwrap()
+                    .call::<(), rlua::Function>(()).unwrap();
+
+                callback_key = Some(context.create_registry_value(callback).unwrap());
+            });
+
+            let plugins = vec![
+                SnapshotPluginEntry {
+                    file_name_filter: String::new(),
+                    callback: callback_key.unwrap(),
+                }
+            ];
+
+            Some(SnapshotPluginContext { lua, plugins })
+        } else {
+            None
+        };
+
+        let context = SnapshotContext {
+            plugin_context,
+        };
+
         let tree = {
             let temp_imfs = imfs.lock().unwrap();
-            reify_initial_tree(&project, &temp_imfs, &mut instances_per_path, &mut metadata_per_instance)
+            reify_initial_tree(&project, &context, &temp_imfs, &mut instances_per_path, &mut metadata_per_instance)
         };
 
         RbxSession {
@@ -104,17 +144,21 @@ impl RbxSession {
                 .expect("Metadata did not exist for path")
                 .clone();
 
+            let context = SnapshotContext {
+                plugin_context: None,
+            };
+
             for instance_id in &instances_at_path {
                 let instance_metadata = self.metadata_per_instance.get(&instance_id)
                     .expect("Metadata for instance ID did not exist");
 
                 let maybe_snapshot = match &instance_metadata.project_definition {
                     Some((instance_name, project_node)) => {
-                        snapshot_project_node(&imfs, &project_node, Cow::Owned(instance_name.clone()))
+                        snapshot_project_node(&context, &imfs, &project_node, Cow::Owned(instance_name.clone()))
                             .unwrap_or_else(|_| panic!("Could not generate instance snapshot for path {}", path_to_snapshot.display()))
                     },
                     None => {
-                        snapshot_imfs_path(&imfs, &path_to_snapshot, None)
+                        snapshot_imfs_path(&context, &imfs, &path_to_snapshot, None)
                             .unwrap_or_else(|_| panic!("Could not generate instance snapshot for path {}", path_to_snapshot.display()))
                     },
                 };
@@ -202,16 +246,20 @@ impl RbxSession {
 pub fn construct_oneoff_tree(project: &Project, imfs: &Imfs) -> RbxTree {
     let mut instances_per_path = PathMap::new();
     let mut metadata_per_instance = HashMap::new();
-    reify_initial_tree(project, imfs, &mut instances_per_path, &mut metadata_per_instance)
+    let context = SnapshotContext {
+        plugin_context: None,
+    };
+    reify_initial_tree(project, &context, imfs, &mut instances_per_path, &mut metadata_per_instance)
 }
 
 fn reify_initial_tree(
     project: &Project,
+    context: &SnapshotContext,
     imfs: &Imfs,
     instances_per_path: &mut PathMap<HashSet<RbxId>>,
     metadata_per_instance: &mut HashMap<RbxId, MetadataPerInstance>,
 ) -> RbxTree {
-    let snapshot = snapshot_project_tree(imfs, project)
+    let snapshot = snapshot_project_tree(&context, imfs, project)
         .expect("Could not snapshot project tree")
         .expect("Project did not produce any instances");
 

server/src/rbx_snapshot.rs

@@ -9,6 +9,7 @@ use std::{
     str,
 };
 
+use rlua::Lua;
 use failure::Fail;
 use log::info;
 use maplit::hashmap;
@@ -38,6 +39,53 @@ const INIT_MODULE_NAME: &str = "init.lua";
 const INIT_SERVER_NAME: &str = "init.server.lua";
 const INIT_CLIENT_NAME: &str = "init.client.lua";
 
+pub struct SnapshotContext {
+    pub plugin_context: Option<SnapshotPluginContext>,
+}
+
+/// Context that's only relevant to generating snapshots if there are plugins
+/// associated with the project.
+///
+/// It's possible that this needs some sort of extra nesting/filtering to
+/// support nested projects, since their plugins should only apply to
+/// themselves.
+pub struct SnapshotPluginContext {
+    pub lua: Lua,
+    pub plugins: Vec<SnapshotPluginEntry>,
+}
+
+pub struct SnapshotPluginEntry {
+    /// Simple file name suffix filter to avoid running plugins on every file
+    /// change.
+    pub file_name_filter: String,
+
+    /// A key into the Lua registry created by [`create_registry_value`] that
+    /// refers to a function that can be called to transform a file/instance
+    /// pair according to how the plugin needs to operate.
+    ///
+    /// [`create_registry_value`]: https://docs.rs/rlua/0.16.2/rlua/struct.Context.html#method.create_registry_value
+    pub callback: rlua::RegistryKey,
+}
+
+#[derive(Debug, Clone)]
+struct LuaRbxSnapshot(RbxSnapshotInstance<'static>);
+
+impl rlua::UserData for LuaRbxSnapshot {
+    fn add_methods<'lua, M: rlua::UserDataMethods<'lua, Self>>(methods: &mut M) {
+        methods.add_meta_method(rlua::MetaMethod::Index, |_context, this, key: String| {
+            match key.as_str() {
+                "name" => Ok(this.0.name.clone().into_owned()),
+                "className" => Ok(this.0.class_name.clone().into_owned()),
+                _ => Err(rlua::Error::RuntimeError(format!("{} is not a valid member of RbxSnapshotInstance", &key))),
+            }
+        });
+
+        methods.add_meta_method(rlua::MetaMethod::ToString, |_context, this, _args: ()| {
+            Ok("RbxSnapshotInstance")
+        });
+    }
+}
+
 pub type SnapshotResult<'a> = Result<Option<RbxSnapshotInstance<'a>>, SnapshotError>;
 
 #[derive(Debug, Fail)]
@@ -104,19 +152,21 @@ impl fmt::Display for SnapshotError {
 }
 
 pub fn snapshot_project_tree<'source>(
+    context: &SnapshotContext,
     imfs: &'source Imfs,
     project: &'source Project,
 ) -> SnapshotResult<'source> {
-    snapshot_project_node(imfs, &project.tree, Cow::Borrowed(&project.name))
+    snapshot_project_node(context, imfs, &project.tree, Cow::Borrowed(&project.name))
 }
 
 pub fn snapshot_project_node<'source>(
+    context: &SnapshotContext,
     imfs: &'source Imfs,
     node: &ProjectNode,
     instance_name: Cow<'source, str>,
 ) -> SnapshotResult<'source> {
     let maybe_snapshot = match &node.path {
-        Some(path) => snapshot_imfs_path(imfs, &path, Some(instance_name.clone()))?,
+        Some(path) => snapshot_imfs_path(context, imfs, &path, Some(instance_name.clone()))?,
         None => match &node.class_name {
             Some(_class_name) => Some(RbxSnapshotInstance {
                 name: instance_name.clone(),
@@ -170,7 +220,7 @@ pub fn snapshot_project_node<'source>(
     }
 
     for (child_name, child_project_node) in &node.children {
-        if let Some(child) = snapshot_project_node(imfs, child_project_node, Cow::Owned(child_name.clone()))? {
+        if let Some(child) = snapshot_project_node(context, imfs, child_project_node, Cow::Owned(child_name.clone()))? {
             snapshot.children.push(child);
         }
     }
@@ -189,6 +239,7 @@ pub fn snapshot_project_node<'source>(
 }
 
 pub fn snapshot_imfs_path<'source>(
+    context: &SnapshotContext,
     imfs: &'source Imfs,
     path: &Path,
     instance_name: Option<Cow<'source, str>>,
@@ -196,23 +247,25 @@ pub fn snapshot_imfs_path<'source>(
     // If the given path doesn't exist in the in-memory filesystem, we consider
     // that an error.
     match imfs.get(path) {
-        Some(imfs_item) => snapshot_imfs_item(imfs, imfs_item, instance_name),
+        Some(imfs_item) => snapshot_imfs_item(context, imfs, imfs_item, instance_name),
         None => return Err(SnapshotError::DidNotExist(path.to_owned())),
     }
 }
 
 fn snapshot_imfs_item<'source>(
+    context: &SnapshotContext,
     imfs: &'source Imfs,
     item: &'source ImfsItem,
     instance_name: Option<Cow<'source, str>>,
 ) -> SnapshotResult<'source> {
     match item {
-        ImfsItem::File(file) => snapshot_imfs_file(file, instance_name),
+        ImfsItem::File(file) => snapshot_imfs_file(context, file, instance_name),
-        ImfsItem::Directory(directory) => snapshot_imfs_directory(imfs, directory, instance_name),
+        ImfsItem::Directory(directory) => snapshot_imfs_directory(context, imfs, directory, instance_name),
     }
 }
 
 fn snapshot_imfs_directory<'source>(
+    context: &SnapshotContext,
     imfs: &'source Imfs,
     directory: &'source ImfsDirectory,
     instance_name: Option<Cow<'source, str>>,
@@ -229,11 +282,11 @@ fn snapshot_imfs_directory<'source>(
         });
 
     let mut snapshot = if directory.children.contains(&init_path) {
-        snapshot_imfs_path(imfs, &init_path, Some(snapshot_name))?.unwrap()
+        snapshot_imfs_path(context, imfs, &init_path, Some(snapshot_name))?.unwrap()
     } else if directory.children.contains(&init_server_path) {
-        snapshot_imfs_path(imfs, &init_server_path, Some(snapshot_name))?.unwrap()
+        snapshot_imfs_path(context, imfs, &init_server_path, Some(snapshot_name))?.unwrap()
     } else if directory.children.contains(&init_client_path) {
-        snapshot_imfs_path(imfs, &init_client_path, Some(snapshot_name))?.unwrap()
+        snapshot_imfs_path(context, imfs, &init_client_path, Some(snapshot_name))?.unwrap()
     } else {
         RbxSnapshotInstance {
             class_name: Cow::Borrowed("Folder"),
@@ -262,7 +315,7 @@ fn snapshot_imfs_directory<'source>(
                 // them here.
             },
             _ => {
-                if let Some(child) = snapshot_imfs_path(imfs, child_path, None)? {
+                if let Some(child) = snapshot_imfs_path(context, imfs, child_path, None)? {
                     snapshot.children.push(child);
                 }
             },
@@ -273,6 +326,7 @@ fn snapshot_imfs_directory<'source>(
 }
 
 fn snapshot_imfs_file<'source>(
+    context: &SnapshotContext,
     file: &'source ImfsFile,
     instance_name: Option<Cow<'source, str>>,
 ) -> SnapshotResult<'source> {
@@ -308,6 +362,20 @@ fn snapshot_imfs_file<'source>(
         info!("File generated no snapshot: {}", file.path.display());
     }
 
+    if let Some(snapshot) = maybe_snapshot.as_ref() {
+        if let Some(plugin_context) = &context.plugin_context {
+            for plugin in &plugin_context.plugins {
+                let owned_snapshot = snapshot.get_owned();
+                let registry_key = &plugin.callback;
+
+                plugin_context.lua.context(move |context| {
+                    let callback: rlua::Function = context.registry_value(registry_key).unwrap();
+                    callback.call::<_, ()>(LuaRbxSnapshot(owned_snapshot)).unwrap();
+                });
+            }
+        }
+    }
+
     Ok(maybe_snapshot)
 }
 

server/src/snapshot_reconciler.rs

@@ -64,7 +64,7 @@ impl InstanceChanges {
 
 /// A lightweight, hierarchical representation of an instance that can be
 /// applied to the tree.
-#[derive(Debug, PartialEq, Serialize, Deserialize)]
+#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)]
 pub struct RbxSnapshotInstance<'a> {
     pub name: Cow<'a, str>,
     pub class_name: Cow<'a, str>,
@@ -73,6 +73,22 @@ pub struct RbxSnapshotInstance<'a> {
     pub metadata: MetadataPerInstance,
 }
 
+impl<'a> RbxSnapshotInstance<'a> {
+    pub fn get_owned(&'a self) -> RbxSnapshotInstance<'static> {
+        let children: Vec<RbxSnapshotInstance<'static>> = self.children.iter()
+            .map(RbxSnapshotInstance::get_owned)
+            .collect();
+
+        RbxSnapshotInstance {
+            name: Cow::Owned(self.name.clone().into_owned()),
+            class_name: Cow::Owned(self.class_name.clone().into_owned()),
+            properties: self.properties.clone(),
+            children,
+            metadata: self.metadata.clone(),
+        }
+    }
+}
+
 impl<'a> PartialOrd for RbxSnapshotInstance<'a> {
     fn partial_cmp(&self, other: &RbxSnapshotInstance) -> Option<Ordering> {
         Some(self.name.cmp(&other.name)

server/tests/read_projects.rs

@@ -86,6 +86,7 @@ fn single_partition_game() {
         Project {
             name: "single-sync-point".to_string(),
             tree: root_node,
+            plugins: Vec::new(),
             serve_port: None,
             serve_place_ids: None,
             file_location: project_location.join("default.project.json"),

server/tests/snapshots.rs

@@ -8,7 +8,7 @@ use pretty_assertions::assert_eq;
 use librojo::{
     imfs::Imfs,
     project::{Project, ProjectNode},
-    rbx_snapshot::snapshot_project_tree,
+    rbx_snapshot::{SnapshotContext, snapshot_project_tree},
     snapshot_reconciler::{RbxSnapshotInstance},
 };
 
@@ -47,7 +47,11 @@ fn run_snapshot_test(path: &Path) {
     imfs.add_roots_from_project(&project)
         .expect("Could not add IMFS roots to snapshot project");
 
-    let mut snapshot = snapshot_project_tree(&imfs, &project)
+    let context = SnapshotContext {
+        plugin_context: None,
+    };
+
+    let mut snapshot = snapshot_project_tree(&context, &imfs, &project)
         .expect("Could not generate snapshot for snapshot test");
 
     if let Some(snapshot) = snapshot.as_mut() {