Use SerializationService as a fallback for when patch application fails (#1030)

plugin/src/ApiContext.lua

@@ -10,6 +10,8 @@ local Version = require(script.Parent.Version)
 local validateApiInfo = Types.ifEnabled(Types.ApiInfoResponse)
 local validateApiRead = Types.ifEnabled(Types.ApiReadResponse)
 local validateApiSubscribe = Types.ifEnabled(Types.ApiSubscribeResponse)
+local validateApiSerialize = Types.ifEnabled(Types.ApiSerializeResponse)
+local validateApiRefPatch = Types.ifEnabled(Types.ApiRefPatchResponse)
 
 local function rejectFailedRequests(response)
 	if response.code >= 400 then
@@ -252,4 +254,32 @@ function ApiContext:open(id)
 	end)
 end
 
+function ApiContext:serialize(ids: { string })
+	local url = ("%s/api/serialize/%s"):format(self.__baseUrl, table.concat(ids, ","))
+
+	return Http.get(url):andThen(rejectFailedRequests):andThen(Http.Response.json):andThen(function(body)
+		if body.sessionId ~= self.__sessionId then
+			return Promise.reject("Server changed ID")
+		end
+
+		assert(validateApiSerialize(body))
+
+		return body
+	end)
+end
+
+function ApiContext:refPatch(ids: { string })
+	local url = ("%s/api/ref-patch/%s"):format(self.__baseUrl, table.concat(ids, ","))
+
+	return Http.get(url):andThen(rejectFailedRequests):andThen(Http.Response.json):andThen(function(body)
+		if body.sessionId ~= self.__sessionId then
+			return Promise.reject("Server changed ID")
+		end
+
+		assert(validateApiRefPatch(body))
+
+		return body
+	end)
+end
+
 return ApiContext

plugin/src/App/StatusPages/Settings/init.lua

@@ -181,6 +181,14 @@ function SettingsPage:render()
 				layoutOrder = layoutIncrement(),
 			}),
 
+			EnableSyncFallback = e(Setting, {
+				id = "enableSyncFallback",
+				name = "Enable Sync Fallback",
+				description = "Whether Instances that fail to sync are remade as a fallback. If this is enabled, Instances may be destroyed and remade when syncing.",
+				transparency = self.props.transparency,
+				layoutOrder = layoutIncrement(),
+			}),
+
 			CheckForUpdates = e(Setting, {
 				id = "checkForUpdates",
 				name = "Check For Updates",

plugin/src/App/init.lua

@@ -595,13 +595,13 @@ function App:startSession()
 		twoWaySync = Settings:get("twoWaySync"),
 	})
 
-	self.cleanupPrecommit = serveSession.__reconciler:hookPrecommit(function(patch, instanceMap)
+	self.cleanupPrecommit = serveSession:hookPrecommit(function(patch, instanceMap)
 		-- Build new tree for patch
 		self:setState({
 			patchTree = PatchTree.build(patch, instanceMap, { "Property", "Old", "New" }),
 		})
 	end)
-	self.cleanupPostcommit = serveSession.__reconciler:hookPostcommit(function(patch, instanceMap, unappliedPatch)
+	self.cleanupPostcommit = serveSession:hookPostcommit(function(patch, instanceMap, unappliedPatch)
 		-- Update tree with unapplied metadata
 		self:setState(function(prevState)
 			return {

plugin/src/PatchSet.lua

@@ -282,6 +282,22 @@ function PatchSet.assign(target, ...)
 	return target
 end
 
+function PatchSet.addedIdList(patchSet): { string }
+	local idList = table.create(#patchSet.added)
+	for id in patchSet.added do
+		table.insert(idList, id)
+	end
+	return table.freeze(idList)
+end
+
+function PatchSet.updatedIdList(patchSet): { string }
+	local idList = table.create(#patchSet.updated)
+	for _, item in patchSet.updated do
+		table.insert(idList, item.id)
+	end
+	return table.freeze(idList)
+end
+
 --[[
 	Create a list of human-readable statements summarizing the contents of this
 	patch, intended to be displayed to users.

plugin/src/Reconciler/applyPatch.lua

@@ -5,8 +5,6 @@
 	Patches can come from the server or be generated by the client.
 ]]
 
-local ChangeHistoryService = game:GetService("ChangeHistoryService")
-
 local Packages = script.Parent.Parent.Parent.Packages
 local Log = require(Packages.Log)
 
@@ -20,13 +18,6 @@ local reifyInstance, applyDeferredRefs = reify.reifyInstance, reify.applyDeferre
 local setProperty = require(script.Parent.setProperty)
 
 local function applyPatch(instanceMap, patch)
-	local patchTimestamp = DateTime.now():FormatLocalTime("LTS", "en-us")
-	local historyRecording = ChangeHistoryService:TryBeginRecording("Rojo: Patch " .. patchTimestamp)
-	if not historyRecording then
-		-- There can only be one recording at a time
-		Log.debug("Failed to begin history recording for " .. patchTimestamp .. ". Another recording is in progress.")
-	end
-
 	-- Tracks any portions of the patch that could not be applied to the DOM.
 	local unappliedPatch = PatchSet.newEmpty()
 
@@ -73,9 +64,6 @@ local function applyPatch(instanceMap, patch)
 		if parentInstance == nil then
 			-- This would be peculiar. If you create an instance with no
 			-- parent, were you supposed to create it at all?
-			if historyRecording then
-				ChangeHistoryService:FinishRecording(historyRecording, Enum.FinishRecordingOperation.Commit)
-			end
 			invariant(
 				"Cannot add an instance from a patch that has no parent.\nInstance {} with parent {}.\nState: {:#?}",
 				id,
@@ -244,10 +232,6 @@ local function applyPatch(instanceMap, patch)
 		end
 	end
 
-	if historyRecording then
-		ChangeHistoryService:FinishRecording(historyRecording, Enum.FinishRecordingOperation.Commit)
-	end
-
 	applyDeferredRefs(instanceMap, deferredRefs, unappliedPatch)
 
 	return unappliedPatch

plugin/src/Reconciler/init.lua

@@ -5,9 +5,6 @@
 
 local Rojo = script:FindFirstAncestor("Rojo")
 local Plugin = Rojo.Plugin
-local Packages = Rojo.Packages
-
-local Log = require(Packages.Log)
 
 local Timer = require(Plugin.Timer)
 
@@ -22,78 +19,17 @@ function Reconciler.new(instanceMap)
 	local self = {
 		-- Tracks all of the instances known by the reconciler by ID.
 		__instanceMap = instanceMap,
-		__precommitCallbacks = {},
-		__postcommitCallbacks = {},
 	}
 
 	return setmetatable(self, Reconciler)
 end
 
-function Reconciler:hookPrecommit(callback: (patch: any, instanceMap: any) -> ()): () -> ()
-	table.insert(self.__precommitCallbacks, callback)
-	Log.trace("Added precommit callback: {}", callback)
-
-	return function()
-		-- Remove the callback from the list
-		for i, cb in self.__precommitCallbacks do
-			if cb == callback then
-				table.remove(self.__precommitCallbacks, i)
-				Log.trace("Removed precommit callback: {}", callback)
-				break
-			end
-		end
-	end
-end
-
-function Reconciler:hookPostcommit(callback: (patch: any, instanceMap: any, unappliedPatch: any) -> ()): () -> ()
-	table.insert(self.__postcommitCallbacks, callback)
-	Log.trace("Added postcommit callback: {}", callback)
-
-	return function()
-		-- Remove the callback from the list
-		for i, cb in self.__postcommitCallbacks do
-			if cb == callback then
-				table.remove(self.__postcommitCallbacks, i)
-				Log.trace("Removed postcommit callback: {}", callback)
-				break
-			end
-		end
-	end
-end
-
 function Reconciler:applyPatch(patch)
 	Timer.start("Reconciler:applyPatch")
 
-	Timer.start("precommitCallbacks")
-	-- Precommit callbacks must be serial in order to obey the contract that
-	-- they execute before commit
-	for _, callback in self.__precommitCallbacks do
-		local success, err = pcall(callback, patch, self.__instanceMap)
-		if not success then
-			Log.warn("Precommit hook errored: {}", err)
-		end
-	end
-	Timer.stop()
-
-	Timer.start("apply")
 	local unappliedPatch = applyPatch(self.__instanceMap, patch)
-	Timer.stop()
-
-	Timer.start("postcommitCallbacks")
-	-- Postcommit callbacks can be called with spawn since regardless of firing order, they are
-	-- guaranteed to be called after the commit
-	for _, callback in self.__postcommitCallbacks do
-		task.spawn(function()
-			local success, err = pcall(callback, patch, self.__instanceMap, unappliedPatch)
-			if not success then
-				Log.warn("Postcommit hook errored: {}", err)
-			end
-		end)
-	end
-	Timer.stop()
 
 	Timer.stop()
-
 	return unappliedPatch
 end
 

plugin/src/ServeSession.lua

@@ -1,11 +1,15 @@
 local StudioService = game:GetService("StudioService")
 local RunService = game:GetService("RunService")
+local ChangeHistoryService = game:GetService("ChangeHistoryService")
+local SerializationService = game:GetService("SerializationService")
+local Selection = game:GetService("Selection")
 
 local Packages = script.Parent.Parent.Packages
 local Log = require(Packages.Log)
 local Fmt = require(Packages.Fmt)
 local t = require(Packages.t)
 local Promise = require(Packages.Promise)
+local Timer = require(script.Parent.Timer)
 
 local ChangeBatcher = require(script.Parent.ChangeBatcher)
 local encodePatchUpdate = require(script.Parent.ChangeBatcher.encodePatchUpdate)
@@ -95,6 +99,8 @@ function ServeSession.new(options)
 		__changeBatcher = changeBatcher,
 		__statusChangedCallback = nil,
 		__connections = connections,
+		__precommitCallbacks = {},
+		__postcommitCallbacks = {},
 	}
 
 	setmetatable(self, ServeSession)
@@ -125,12 +131,46 @@ function ServeSession:setConfirmCallback(callback)
 	self.__userConfirmCallback = callback
 end
 
+--[=[
+	Hooks a function to run before patch application.
+	The provided function is called with the incoming patch and an InstanceMap
+	as parameters.
+]=]
 function ServeSession:hookPrecommit(callback)
-	return self.__reconciler:hookPrecommit(callback)
+	table.insert(self.__precommitCallbacks, callback)
+	Log.trace("Added precommit callback: {}", callback)
+
+	return function()
+		-- Remove the callback from the list
+		for i, cb in self.__precommitCallbacks do
+			if cb == callback then
+				table.remove(self.__precommitCallbacks, i)
+				Log.trace("Removed precommit callback: {}", callback)
+				break
+			end
+		end
+	end
 end
 
+--[=[
+	Hooks a function to run after patch application.
+	The provided function is called with the applied patch, the current
+	InstanceMap, and a PatchSet containing any unapplied changes.
+]=]
 function ServeSession:hookPostcommit(callback)
-	return self.__reconciler:hookPostcommit(callback)
+	table.insert(self.__postcommitCallbacks, callback)
+	Log.trace("Added postcommit callback: {}", callback)
+
+	return function()
+		-- Remove the callback from the list
+		for i, cb in self.__postcommitCallbacks do
+			if cb == callback then
+				table.remove(self.__postcommitCallbacks, i)
+				Log.trace("Removed postcommit callback: {}", callback)
+				break
+			end
+		end
+	end
 end
 
 function ServeSession:start()
@@ -206,6 +246,169 @@ function ServeSession:__onActiveScriptChanged(activeScript)
 	self.__apiContext:open(scriptId)
 end
 
+function ServeSession:__replaceInstances(idList)
+	if #idList == 0 then
+		return true, PatchSet.newEmpty()
+	end
+	-- It would be annoying if selection went away, so we try to preserve it.
+	local selection = Selection:Get()
+	local selectionMap = {}
+	for i, instance in selection do
+		selectionMap[instance] = i
+	end
+
+	-- TODO: Should we do this in multiple requests so we can more granularly mark failures?
+	local modelSuccess, replacements = self.__apiContext
+		:serialize(idList)
+		:andThen(function(response)
+			Log.debug("Deserializing results from serialize endpoint")
+			local objects = SerializationService:DeserializeInstancesAsync(response.modelContents)
+			if not objects[1] then
+				return Promise.reject("Serialize endpoint did not deserialize into any Instances")
+			end
+			if #objects[1]:GetChildren() ~= #idList then
+				return Promise.reject("Serialize endpoint did not return the correct number of Instances")
+			end
+
+			local instanceMap = {}
+			for _, item in objects[1]:GetChildren() do
+				instanceMap[item.Name] = item.Value
+			end
+			return instanceMap
+		end)
+		:await()
+
+	local refSuccess, refPatch = self.__apiContext
+		:refPatch(idList)
+		:andThen(function(response)
+			return response.patch
+		end)
+		:await()
+
+	if not (modelSuccess and refSuccess) then
+		return false
+	end
+
+	for id, replacement in replacements do
+		local oldInstance = self.__instanceMap.fromIds[id]
+		self.__instanceMap:insert(id, replacement)
+		Log.trace("Swapping Instance {} out via api/models/ endpoint", id)
+		local oldParent = oldInstance.Parent
+		for _, child in oldInstance:GetChildren() do
+			child.Parent = replacement
+		end
+
+		replacement.Parent = oldParent
+		-- ChangeHistoryService doesn't like it if an Instance has been
+		-- Destroyed. So, we have to accept the potential memory hit and
+		-- just set the parent to `nil`.
+		oldInstance.Parent = nil
+
+		if selectionMap[oldInstance] then
+			-- This is a bit funky, but it saves the order of Selection
+			-- which might matter for some use cases.
+			selection[selectionMap[oldInstance]] = replacement
+		end
+	end
+
+	local patchApplySuccess, unappliedPatch = pcall(self.__reconciler.applyPatch, self.__reconciler, refPatch)
+	if patchApplySuccess then
+		Selection:Set(selection)
+		return true, unappliedPatch
+	else
+		error(unappliedPatch)
+	end
+end
+
+function ServeSession:__applyPatch(patch)
+	local patchTimestamp = DateTime.now():FormatLocalTime("LTS", "en-us")
+	local historyRecording = ChangeHistoryService:TryBeginRecording("Rojo: Patch " .. patchTimestamp)
+	if not historyRecording then
+		-- There can only be one recording at a time
+		Log.debug("Failed to begin history recording for " .. patchTimestamp .. ". Another recording is in progress.")
+	end
+
+	Timer.start("precommitCallbacks")
+	-- Precommit callbacks must be serial in order to obey the contract that
+	-- they execute before commit
+	for _, callback in self.__precommitCallbacks do
+		local success, err = pcall(callback, patch, self.__instanceMap)
+		if not success then
+			Log.warn("Precommit hook errored: {}", err)
+		end
+	end
+	Timer.stop()
+
+	local patchApplySuccess, unappliedPatch = pcall(self.__reconciler.applyPatch, self.__reconciler, patch)
+	if not patchApplySuccess then
+		if historyRecording then
+			ChangeHistoryService:FinishRecording(historyRecording, Enum.FinishRecordingOperation.Commit)
+		end
+		-- This might make a weird stack trace but the only way applyPatch can
+		-- fail is if a bug occurs so it's probably fine.
+		error(unappliedPatch)
+	end
+
+	if PatchSet.isEmpty(unappliedPatch) then
+		if historyRecording then
+			ChangeHistoryService:FinishRecording(historyRecording, Enum.FinishRecordingOperation.Commit)
+		end
+		return
+	end
+
+	local addedIdList = PatchSet.addedIdList(unappliedPatch)
+	local updatedIdList = PatchSet.updatedIdList(unappliedPatch)
+
+	local actualUnappliedPatches = PatchSet.newEmpty()
+	if Settings:get("enableSyncFallback") then
+		Log.debug("ServeSession:__replaceInstances(unappliedPatch.added)")
+		Timer.start("ServeSession:__replaceInstances(unappliedPatch.added)")
+		local addSuccess, unappliedAddedRefs = self:__replaceInstances(addedIdList)
+		Timer.stop()
+
+		Log.debug("ServeSession:__replaceInstances(unappliedPatch.updated)")
+		Timer.start("ServeSession:__replaceInstances(unappliedPatch.updated)")
+		local updateSuccess, unappliedUpdateRefs = self:__replaceInstances(updatedIdList)
+		Timer.stop()
+
+		if addSuccess then
+			table.clear(unappliedPatch.added)
+			PatchSet.assign(actualUnappliedPatches, unappliedAddedRefs)
+		end
+		if updateSuccess then
+			table.clear(unappliedPatch.updated)
+			PatchSet.assign(actualUnappliedPatches, unappliedUpdateRefs)
+		end
+	else
+		Log.debug("Skipping ServeSession:__replaceInstances because of setting")
+	end
+	PatchSet.assign(actualUnappliedPatches, unappliedPatch)
+
+	if not PatchSet.isEmpty(actualUnappliedPatches) then
+		Log.debug(
+			"Could not apply all changes requested by the Rojo server:\n{}",
+			PatchSet.humanSummary(self.__instanceMap, unappliedPatch)
+		)
+	end
+
+	Timer.start("postcommitCallbacks")
+	-- Postcommit callbacks can be called with spawn since regardless of firing order, they are
+	-- guaranteed to be called after the commit
+	for _, callback in self.__postcommitCallbacks do
+		task.spawn(function()
+			local success, err = pcall(callback, patch, self.__instanceMap, actualUnappliedPatches)
+			if not success then
+				Log.warn("Postcommit hook errored: {}", err)
+			end
+		end)
+	end
+	Timer.stop()
+
+	if historyRecording then
+		ChangeHistoryService:FinishRecording(historyRecording, Enum.FinishRecordingOperation.Commit)
+	end
+end
+
 function ServeSession:__initialSync(serverInfo)
 	return self.__apiContext:read({ serverInfo.rootInstanceId }):andThen(function(readResponseBody)
 		-- Tell the API Context that we're up-to-date with the version of
@@ -280,15 +483,7 @@ function ServeSession:__initialSync(serverInfo)
 
 			return self.__apiContext:write(inversePatch)
 		elseif userDecision == "Accept" then
-			local unappliedPatch = self.__reconciler:applyPatch(catchUpPatch)
-
-			if not PatchSet.isEmpty(unappliedPatch) then
-				Log.debug(
-					"Could not apply all changes requested by the Rojo server:\n{}",
-					PatchSet.humanSummary(self.__instanceMap, unappliedPatch)
-				)
-			end
-
+			self:__applyPatch(catchUpPatch)
 			return Promise.resolve()
 		else
 			return Promise.reject("Invalid user decision: " .. userDecision)
@@ -311,14 +506,7 @@ function ServeSession:__mainSyncLoop()
 					Log.trace("Serve session {} retrieved {} messages", tostring(self), #messages)
 
 					for _, message in messages do
-						local unappliedPatch = self.__reconciler:applyPatch(message)
-
-						if not PatchSet.isEmpty(unappliedPatch) then
-							Log.debug(
-								"Could not apply all changes requested by the Rojo server:\n{}",
-								PatchSet.humanSummary(self.__instanceMap, unappliedPatch)
-							)
-						end
+						self:__applyPatch(message)
 					end
 				end)
 				:await()

plugin/src/Settings.lua

@@ -14,6 +14,7 @@ local defaultSettings = {
 	twoWaySync = false,
 	autoReconnect = false,
 	showNotifications = true,
+	enableSyncFallback = true,
 	syncReminderMode = "Notify" :: "None" | "Notify" | "Fullscreen",
 	syncReminderPolling = true,
 	checkForUpdates = true,

plugin/src/Types.lua

@@ -55,6 +55,16 @@ local ApiSubscribeResponse = t.interface({
 	messages = t.array(ApiSubscribeMessage),
 })
 
+local ApiSerializeResponse = t.interface({
+	sessionId = t.string,
+	modelContents = t.buffer,
+})
+
+local ApiRefPatchResponse = t.interface({
+	sessionId = t.string,
+	patch = ApiSubscribeMessage,
+})
+
 local ApiError = t.interface({
 	kind = t.union(t.literal("NotFound"), t.literal("BadRequest"), t.literal("InternalError")),
 	details = t.string,
@@ -82,6 +92,8 @@ return strict("Types", {
 	ApiInstanceUpdate = ApiInstanceUpdate,
 	ApiInstanceMetadata = ApiInstanceMetadata,
 	ApiSubscribeMessage = ApiSubscribeMessage,
+	ApiSerializeResponse = ApiSerializeResponse,
+	ApiRefPatchResponse = ApiRefPatchResponse,
 	ApiValue = ApiValue,
 	RbxId = RbxId,