User plugin foundations for 0.6.0 (#257)

Starts work on #55.

This is similar to the previous work in #125. It's gated behind a new Cargo
feature, `user-plugins`. This time, the config gate is much smaller. The
`plugins` member of projects is still accessible when plugins aren't enabled,
but is always empty. Additionally, user plugins are only enabled if there's a
Lua state present in the snapshot context when the `SnapshotUserPlugins`
snapshot middleware runs. This not ever the case currently.

This code has very little possibility of rotting while we focus on other work,
since it'll be guaranteed to still compile and can be tested in isolation
without the feature being turned on.

src/change_processor.rs

@@ -12,7 +12,7 @@ use crate::{
     imfs::{Imfs, ImfsEvent, ImfsFetcher},
     message_queue::MessageQueue,
     snapshot::{apply_patch_set, compute_patch_set, AppliedPatchSet, InstigatingSource, RojoTree},
-    snapshot_middleware::snapshot_from_imfs,
+    snapshot_middleware::{snapshot_from_imfs, InstanceSnapshotContext},
 };
 
 pub struct ChangeProcessor {
@@ -99,7 +99,10 @@ impl ChangeProcessor {
                                                 .get(path)
                                                 .expect("could not get instigating path from filesystem");
 
-                                            let snapshot = snapshot_from_imfs(&mut imfs, &entry)
+                                            // TODO: Use persisted snapshot
+                                            // context struct instead of
+                                            // recreating it every time.
+                                            let snapshot = snapshot_from_imfs(&mut InstanceSnapshotContext::default(), &mut imfs, &entry)
                                                 .expect("snapshot failed")
                                                 .expect("snapshot did not return an instance");
 

src/commands/build.rs

@@ -11,7 +11,7 @@ use rbx_dom_weak::RbxInstanceProperties;
 use crate::{
     imfs::{FsError, Imfs, RealFetcher, WatchMode},
     snapshot::{apply_patch_set, compute_patch_set, InstancePropertiesWithMeta, RojoTree},
-    snapshot_middleware::snapshot_from_imfs,
+    snapshot_middleware::{snapshot_from_imfs, InstanceSnapshotContext},
 };
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -96,8 +96,9 @@ pub fn build(options: &BuildOptions) -> Result<(), BuildError> {
         .get(&options.fuzzy_project_path)
         .expect("could not get project path");
 
+    // TODO: Compute snapshot context from project.
     log::trace!("Generating snapshot of instances from IMFS");
-    let snapshot = snapshot_from_imfs(&mut imfs, &entry)
+    let snapshot = snapshot_from_imfs(&mut InstanceSnapshotContext::default(), &mut imfs, &entry)
         .expect("snapshot failed")
         .expect("snapshot did not return an instance");
 

src/commands/upload.rs

@@ -8,7 +8,7 @@ use crate::{
     auth_cookie::get_auth_cookie,
     imfs::{Imfs, RealFetcher, WatchMode},
     snapshot::{apply_patch_set, compute_patch_set, InstancePropertiesWithMeta, RojoTree},
-    snapshot_middleware::snapshot_from_imfs,
+    snapshot_middleware::{snapshot_from_imfs, InstanceSnapshotContext},
 };
 
 #[derive(Debug, Fail)]
@@ -63,8 +63,9 @@ pub fn upload(options: UploadOptions) -> Result<(), UploadError> {
         .get(&options.fuzzy_project_path)
         .expect("could not get project path");
 
+    // TODO: Compute snapshot context from project.
     log::trace!("Generating snapshot of instances from IMFS");
-    let snapshot = snapshot_from_imfs(&mut imfs, &entry)
+    let snapshot = snapshot_from_imfs(&mut InstanceSnapshotContext::default(), &mut imfs, &entry)
         .expect("snapshot failed")
         .expect("snapshot did not return an instance");
 

src/project.rs

@@ -30,6 +30,10 @@ struct SourceProject {
 
     #[serde(skip_serializing_if = "Option::is_none")]
     serve_place_ids: Option<HashSet<u64>>,
+
+    #[serde(default, skip_serializing_if = "Vec::is_empty")]
+    #[cfg_attr(not(feature = "user-plugins"), serde(skip_deserializing))]
+    plugins: Vec<String>,
 }
 
 impl SourceProject {
@@ -37,11 +41,19 @@ impl SourceProject {
     pub fn into_project(self, project_file_location: &Path) -> Project {
         let tree = self.tree.into_project_node(project_file_location);
 
+        let project_folder = project_file_location.parent().unwrap();
+        let plugins = self
+            .plugins
+            .into_iter()
+            .map(|path| project_folder.join(path))
+            .collect();
+
         Project {
             name: self.name,
             tree,
             serve_port: self.serve_port,
             serve_place_ids: self.serve_place_ids,
+            plugins,
             file_location: PathBuf::from(project_file_location),
         }
     }
@@ -318,6 +330,7 @@ pub struct Project {
     pub tree: ProjectNode,
     pub serve_port: Option<u16>,
     pub serve_place_ids: Option<HashSet<u64>>,
+    pub plugins: Vec<PathBuf>,
     pub file_location: PathBuf,
 }
 
@@ -391,6 +404,7 @@ impl Project {
             tree,
             serve_port: None,
             serve_place_ids: None,
+            plugins: Vec::new(),
             file_location: project_path.clone(),
         };
 
@@ -557,10 +571,24 @@ impl Project {
     }
 
     fn to_source_project(&self) -> SourceProject {
+        // TODO: Use path_serializer instead of transforming paths between
+        // String and PathBuf?
+        let plugins = self
+            .plugins
+            .iter()
+            .map(|path| {
+                path.strip_prefix(self.folder_location())
+                    .unwrap()
+                    .display()
+                    .to_string()
+            })
+            .collect();
+
         SourceProject {
             name: self.name.clone(),
             tree: self.tree.to_source_node(&self.file_location),
             serve_port: self.serve_port,
+            plugins,
             serve_place_ids: self.serve_place_ids.clone(),
         }
     }

src/serve_session.rs

@@ -16,7 +16,7 @@ use crate::{
     snapshot::{
         apply_patch_set, compute_patch_set, AppliedPatchSet, InstancePropertiesWithMeta, RojoTree,
     },
-    snapshot_middleware::snapshot_from_imfs,
+    snapshot_middleware::{snapshot_from_imfs, InstanceSnapshotContext},
 };
 
 /// Contains all of the state for a Rojo serve session.
@@ -111,10 +111,12 @@ impl<F: ImfsFetcher + Send + 'static> ServeSession<F> {
         log::trace!("Loading start path: {}", start_path.display());
         let entry = imfs.get(start_path).expect("could not get project path");
 
+        // TODO: Compute snapshot context from project.
         log::trace!("Snapshotting start path");
-        let snapshot = snapshot_from_imfs(&mut imfs, &entry)
-            .expect("snapshot failed")
-            .expect("snapshot did not return an instance");
+        let snapshot =
+            snapshot_from_imfs(&mut InstanceSnapshotContext::default(), &mut imfs, &entry)
+                .expect("snapshot failed")
+                .expect("snapshot did not return an instance");
 
         log::trace!("Computing initial patch set");
         let patch_set = compute_patch_set(&snapshot, &tree, root_id);

src/snapshot_middleware/context.rs

@@ -1,8 +1,12 @@
+use std::{fmt, ops::Deref, path::PathBuf};
+
+use rlua::{Lua, RegistryKey};
+
 #[derive(Debug)]
 pub struct InstanceSnapshotContext {
     /// Empty struct that will be used later to fill out required Lua state for
     /// user plugins.
-    pub plugin_context: Option<()>,
+    pub plugin_context: Option<SnapshotPluginContext>,
 }
 
 impl Default for InstanceSnapshotContext {
@@ -13,4 +17,47 @@ impl Default for InstanceSnapshotContext {
     }
 }
 
+#[derive(Debug)]
+pub struct SnapshotPluginContext {
+    pub state: IgnoreDebug<Lua>,
+
+    /// Paths to the user plugins files. These paths are generated by the root
+    /// project file, if there is one.
+    pub plugin_paths: Vec<PathBuf>,
+
+    /// Lazy-initialized registry keys pointing to the values returned by each
+    /// user plugin. When processing user plugins, these should be applied in
+    /// order.
+    pub plugin_functions: Option<Vec<RegistryKey>>,
+}
+
+impl SnapshotPluginContext {
+    pub fn new(plugin_paths: Vec<PathBuf>) -> Self {
+        Self {
+            state: IgnoreDebug(Lua::new()),
+            plugin_paths,
+            plugin_functions: None,
+        }
+    }
+}
+
+/// Utility type to enable having a field of a struct not implement Debug and
+/// instead show a placeholder.
+#[derive(Clone)]
+pub struct IgnoreDebug<T>(pub T);
+
+impl<T> fmt::Debug for IgnoreDebug<T> {
+    fn fmt(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
+        write!(formatter, "<no debug representation>")
+    }
+}
+
+impl<T> Deref for IgnoreDebug<T> {
+    type Target = T;
+
+    fn deref(&self) -> &T {
+        &self.0
+    }
+}
+
 pub struct ImfsSnapshotContext;

src/snapshot_middleware/dir.rs

@@ -18,7 +18,7 @@ pub struct SnapshotDir;
 
 impl SnapshotMiddleware for SnapshotDir {
     fn from_imfs<F: ImfsFetcher>(
-        _context: &mut InstanceSnapshotContext,
+        context: &mut InstanceSnapshotContext,
         imfs: &mut Imfs<F>,
         entry: &ImfsEntry,
     ) -> SnapshotInstanceResult<'static> {
@@ -31,7 +31,7 @@ impl SnapshotMiddleware for SnapshotDir {
         let mut snapshot_children = Vec::new();
 
         for child in children.into_iter() {
-            if let Some(child_snapshot) = snapshot_from_imfs(imfs, &child)? {
+            if let Some(child_snapshot) = snapshot_from_imfs(context, imfs, &child)? {
                 snapshot_children.push(child_snapshot);
             }
         }

src/snapshot_middleware/error.rs

@@ -16,6 +16,13 @@ impl SnapshotError {
         }
     }
 
+    pub(crate) fn wrap(inner: impl Into<SnapshotErrorDetail>, path: impl Into<PathBuf>) -> Self {
+        SnapshotError {
+            detail: inner.into(),
+            path: Some(path.into()),
+        }
+    }
+
     pub(crate) fn file_did_not_exist(path: impl Into<PathBuf>) -> SnapshotError {
         SnapshotError {
             detail: SnapshotErrorDetail::FileDidNotExist,
@@ -70,27 +77,45 @@ impl From<FsError> for SnapshotError {
     fn from(error: FsError) -> Self {
         let (inner, path) = error.into_raw();
 
-        let detail = SnapshotErrorDetail::IoError { inner };
+        Self::new(inner.into(), Some(path))
+    }
+}
 
-        Self::new(detail, Some(path))
+impl From<rlua::Error> for SnapshotError {
+    fn from(error: rlua::Error) -> Self {
+        Self::new(error.into(), Option::<PathBuf>::None)
     }
 }
 
 #[derive(Debug)]
 pub enum SnapshotErrorDetail {
     IoError { inner: io::Error },
+    Lua { inner: rlua::Error },
     FileDidNotExist,
     FileNameBadUnicode,
     FileContentsBadUnicode { inner: std::str::Utf8Error },
     MalformedProject { inner: serde_json::Error },
 }
 
+impl From<io::Error> for SnapshotErrorDetail {
+    fn from(inner: io::Error) -> Self {
+        SnapshotErrorDetail::IoError { inner }
+    }
+}
+
+impl From<rlua::Error> for SnapshotErrorDetail {
+    fn from(inner: rlua::Error) -> Self {
+        SnapshotErrorDetail::Lua { inner }
+    }
+}
+
 impl SnapshotErrorDetail {
     fn source(&self) -> Option<&(dyn Error + 'static)> {
         use self::SnapshotErrorDetail::*;
 
         match self {
             IoError { inner } => Some(inner),
+            Lua { inner } => Some(inner),
             FileContentsBadUnicode { inner } => Some(inner),
             MalformedProject { inner } => Some(inner),
             _ => None,
@@ -104,6 +129,7 @@ impl fmt::Display for SnapshotErrorDetail {
 
         match self {
             IoError { inner } => write!(formatter, "I/O error: {}", inner),
+            Lua { inner } => write!(formatter, "{}", inner),
             FileDidNotExist => write!(formatter, "file did not exist"),
             FileNameBadUnicode => write!(formatter, "file name had malformed Unicode"),
             FileContentsBadUnicode { inner } => {

src/snapshot_middleware/mod.rs

@@ -16,14 +16,15 @@ mod rbxlx;
 mod rbxm;
 mod rbxmx;
 mod txt;
+mod user_plugins;
 mod util;
 
+pub use self::context::*;
 pub use self::error::*;
 
 use rbx_dom_weak::{RbxId, RbxTree};
 
 use self::{
-    context::InstanceSnapshotContext,
     csv::SnapshotCsv,
     dir::SnapshotDir,
     json_model::SnapshotJsonModel,
@@ -34,6 +35,7 @@ use self::{
     rbxm::SnapshotRbxm,
     rbxmx::SnapshotRbxmx,
     txt::SnapshotTxt,
+    user_plugins::SnapshotUserPlugins,
 };
 use crate::imfs::{Imfs, ImfsEntry, ImfsFetcher};
 
@@ -41,15 +43,14 @@ macro_rules! middlewares {
     ( $($middleware: ident,)* ) => {
         /// Generates a snapshot of instances from the given ImfsEntry.
         pub fn snapshot_from_imfs<F: ImfsFetcher>(
+            context: &mut InstanceSnapshotContext,
             imfs: &mut Imfs<F>,
             entry: &ImfsEntry,
         ) -> SnapshotInstanceResult<'static> {
-            let mut context = InstanceSnapshotContext::default();
-
             $(
                 log::trace!("trying middleware {} on {}", stringify!($middleware), entry.path().display());
 
-                if let Some(snapshot) = $middleware::from_imfs(&mut context, imfs, entry)? {
+                if let Some(snapshot) = $middleware::from_imfs(context, imfs, entry)? {
                     log::trace!("middleware {} success on {}", stringify!($middleware), entry.path().display());
                     return Ok(Some(snapshot));
                 }
@@ -75,6 +76,7 @@ macro_rules! middlewares {
 
 middlewares! {
     SnapshotProject,
+    SnapshotUserPlugins,
     SnapshotJsonModel,
     SnapshotRbxlx,
     SnapshotRbxmx,

src/snapshot_middleware/project.rs

@@ -48,7 +48,8 @@ impl SnapshotMiddleware for SnapshotProject {
 
         // Snapshotting a project should always return an instance, so this
         // unwrap is safe.
-        let mut snapshot = snapshot_project_node(&project.name, &project.tree, imfs)?.unwrap();
+        let mut snapshot =
+            snapshot_project_node(context, &project.name, &project.tree, imfs)?.unwrap();
 
         // Setting the instigating source to the project file path is a little
         // coarse.
@@ -76,6 +77,7 @@ impl SnapshotMiddleware for SnapshotProject {
 }
 
 fn snapshot_project_node<F: ImfsFetcher>(
+    context: &mut InstanceSnapshotContext,
     instance_name: &str,
     node: &ProjectNode,
     imfs: &mut Imfs<F>,
@@ -92,7 +94,7 @@ fn snapshot_project_node<F: ImfsFetcher>(
     if let Some(path) = &node.path {
         let entry = imfs.get(path)?;
 
-        if let Some(snapshot) = snapshot_from_imfs(imfs, &entry)? {
+        if let Some(snapshot) = snapshot_from_imfs(context, imfs, &entry)? {
             // If a class name was already specified, then it'll override the
             // class name of this snapshot ONLY if it's a Folder.
             //
@@ -142,7 +144,7 @@ fn snapshot_project_node<F: ImfsFetcher>(
         .expect("$className or $path must be specified");
 
     for (child_name, child_project_node) in &node.children {
-        if let Some(child) = snapshot_project_node(child_name, child_project_node, imfs)? {
+        if let Some(child) = snapshot_project_node(context, child_name, child_project_node, imfs)? {
             children.push(child);
         }
     }

src/snapshot_middleware/user_plugins.rs

@@ -0,0 +1,114 @@
+use std::{fs, path::Path};
+
+use rlua::{Lua, RegistryKey};
+
+use crate::imfs::{Imfs, ImfsEntry, ImfsFetcher};
+
+use super::{
+    context::InstanceSnapshotContext,
+    error::SnapshotError,
+    middleware::{SnapshotInstanceResult, SnapshotMiddleware},
+};
+
+/// Handles snapshotting of any file that a user plugin wants to handle.
+///
+/// User plugins are specified in the project file, but there are never user
+/// plugins specified unless a Cargo feature is enabled, `user-plugins`.
+/// Additionally, extra data needs to be set up inside the snapshot context
+/// which is not currently wired up.
+pub struct SnapshotUserPlugins;
+
+impl SnapshotMiddleware for SnapshotUserPlugins {
+    fn from_imfs<F: ImfsFetcher>(
+        context: &mut InstanceSnapshotContext,
+        _imfs: &mut Imfs<F>,
+        _entry: &ImfsEntry,
+    ) -> SnapshotInstanceResult<'static> {
+        // User plugins are only enabled if present on the snapshot context.
+        let plugin_context = match &mut context.plugin_context {
+            Some(ctx) => ctx,
+            None => return Ok(None),
+        };
+
+        // If the plugins listed for use haven't been loaded yet, read them into
+        // memory, run them, and keep the result they return as a registry key
+        // into our Lua state.
+        let keys = match &plugin_context.plugin_functions {
+            Some(keys) => keys,
+            None => {
+                plugin_context.plugin_functions = Some(initialize_plugins(
+                    &plugin_context.state,
+                    &plugin_context.plugin_paths,
+                )?);
+                plugin_context.plugin_functions.as_ref().unwrap()
+            }
+        };
+
+        plugin_context.state.context(|lua_context| {
+            lua_context.scope(|_scope| {
+                for _key in keys {
+                    // TODO: Invoke plugin here and get result out.
+
+                    // The current plan for plugins here is to make them work
+                    // like Redux/Rodux middleware. A plugin will be a function
+                    // that accepts the next middleware in the chain as a
+                    // function and the snapshot subject (the IMFS entry).
+                    //
+                    // Plugins can (but don't have to) invoke the next snapshot
+                    // function and may or may not mutate the result. The hope
+                    // is that this model enables the most flexibility possible
+                    // for plugins to modify existing Rojo output, as well as
+                    // generate new outputs.
+                    //
+                    // Open questions:
+                    // * How will middleware be ordered? Does putting user
+                    //   middleware always at the beginning or always at the end
+                    //   of the chain reduce the scope of what that middleware
+                    //   can do?
+                    //
+                    // * Will plugins hurt Rojo's ability to parallelize
+                    //   snapshotting in the future?
+                    //
+                    // * Do the mutable handles to the Imfs and the snapshot
+                    //   context prevent plugins from invoking other plugins
+                    //   indirectly?
+                    //
+                    // * Will there be problems using a single Lua state because
+                    //   of re-entrancy?
+                    //
+                    // * Can the Lua <-> Rojo bindings used for middleware be
+                    //   reused for or from another project like Remodel?
+                }
+            })
+        });
+
+        Ok(None)
+    }
+}
+
+fn initialize_plugins<P: AsRef<Path>>(
+    lua_state: &Lua,
+    plugin_paths: &[P],
+) -> Result<Vec<RegistryKey>, SnapshotError> {
+    plugin_paths
+        .iter()
+        .map(|path| {
+            let path = path.as_ref();
+
+            let content = fs::read_to_string(path).map_err(|err| SnapshotError::wrap(err, path))?;
+
+            lua_state.context(|lua_context| {
+                // Plugins are currently expected to return a function that will
+                // be run when a snapshot needs to be generated.
+                let result = lua_context
+                    .load(&content)
+                    .set_name(&path.display().to_string())?
+                    .call::<_, rlua::Function>(())?;
+
+                let key = lua_context.create_registry_value(result)?;
+
+                Ok(key)
+            })
+        })
+        .collect::<Result<Vec<_>, _>>()
+}