--[[----------------------------------------------------------------------------

WFCatStuckSyncUtils.lua

--------------------------------------------------------------------------------

ADOBE CONFIDENTIAL
------------------
Copyright 2021 Adobe
All Rights Reserved.

NOTICE: All information contained herein is, and remains
the property of Adobe and its suppliers, if any. The intellectual
and technical concepts contained herein are proprietary to Adobe
and its suppliers and are protected by all applicable intellectual
property laws, including trade secret and copyright laws.
Dissemination of this information or reproduction of this material
is strictly forbidden unless prior written permission is obtained
from Adobe.

----------------------------------------------------------------------------]]--

	-- RxLua
local Rx = require 'Rx'
local Observable = Rx.Observable

	-- WF.core.utils
local KSIdGenerator = require 'KSIdGenerator'
local KSDate = require 'KSDate'
local KSTableUtils = require 'KSTableUtils'
local WFArguments = require 'WFArguments'
local WFLogger = require 'WFLogger'
local WFCapabilitiesCache = require 'WFCapabilitiesCache'
local WFDiagnosticLog = require 'WFDiagnosticLog'
local WFPlatformInfo = require 'WFPlatformInfo'
local WFRxDocValueWatcher = require "WFRxDocValueWatcher"
local WFRxScheduler = require 'WFRxScheduler'
local WFRxThrottledBatchedBulkApplyOverlay = require 'WFRxThrottledBatchedBulkApplyOverlay'
local WFRxThrottledBatchedBulkDocValue = require 'WFRxThrottledBatchedBulkDocValue'
local WFAnalytics = require "WFAnalytics"

	--WF.oz_client.shared.utils
local LrAssetUtils = require 'LrAssetUtils'

	-- WF.managed_catalog
local WFCatBinaryUploader -- lazy-loaded
local WFCatIndexUtils = require 'WFCatIndexUtils'
local WFCatMetadataPusher = require 'WFCatMetadataPusher'
local WFCatOzErrors = require 'WFCatOzErrors'
local WFCatSyncStatus = require 'WFCatSyncStatus'

	-- WF.web_client.json
local KSJsonEncoder = require 'KSJsonEncoder'

--------------------------------------------------------------------------------

local WFArguments_check = assert( WFArguments.check )

local KSTableUtils_derefOpt = assert( KSTableUtils.derefOpt )

--------------------------------------------------------------------------------

local logger = WFLogger 'WFCatStuckSyncUtils'

local kStuckReportingTimeThresholdInSeconds = 60 * 5 -- 5 minutes
	-- The amount of time something has to appear unsynced to be considered stuck.

local kGeneratedRevisionFrequencyInSeconds = 60 * 60 * 4 -- 4 hours
	-- How often we will generate revisions to resolve stuck documents.

local kMaxGeneratedRevisionCount = 100
	-- Maximum number of auto-generated revisions per document.

local kGlobalDocumentId

if WFPlatformInfo.isMobile() then

	local WFOzSyncConstants = require 'WFOzSyncConstants'

	kGlobalDocumentId = WFOzSyncConstants.kGlobalDocumentId

else

	kGlobalDocumentId = '00000000000000000000000000000000'

end

--==============================================================================

local WFCatStuckSyncUtils = {}

--------------------------------------------------------------------------------

local function checkStuckDocumentAgainstThreshold( 
		maybeStuckDocIdsAndTimestamps, stuckTimeThreshold, period )

	-- Every N seconds (period), evaluate the set of stuck doc ids to see if
	-- any of them have been stuck for a time period greater than the
	-- stuck sync time threshold.
	
	if not next( maybeStuckDocIdsAndTimestamps ) then
		return Observable.just( {} )
	end
	
	return Observable.timer( 0, period, WFRxScheduler.sharedScheduler() )
	
		:map( function()
			
			local stuckDocIds = {}

			if next( maybeStuckDocIdsAndTimestamps ) then

				logger:infof( "Now checking %s against the stuck sync threshold of %d", maybeStuckDocIdsAndTimestamps, stuckTimeThreshold )

				local currentTime = KSDate.timeNow()

				for docId, stuckTimeStamp in pairs( maybeStuckDocIdsAndTimestamps ) do

					if currentTime - stuckTimeStamp > stuckTimeThreshold then

						logger:infof( "Asset %s is stuck", docId )
						stuckDocIds[ #stuckDocIds + 1 ] = docId

					else
					
						logger:infof( "Asset %s is not yet considered stuck", docId )
					end

				end

			end

			return stuckDocIds
		
		end )				

end

--------------------------------------------------------------------------------

--[[!

	\brief Get a stream that returns asset document IDs that the system believes
		are in a stuck sync state due to non-uploaded metadata. Calling this
		function multiple times will return the same observable stream.
		
	\param wfCatalog (WFCatalog)
	
	\param isNormalSyncRunningStream (Observable) An observable that fires with true
		when the sync processes are not paused, and the app is not offline. It
		should fire with false otherwise.
	
	\param optCalm (optional number) optional number of seconds indicating how often we
		should check for stuck documents. The default is 10 seconds.
		
	\param optStuckTimeThreshold(optional number) optional number of seconds an asset
		must be appear stuck before being recorded as stuck. Defaults to 180 seconds.
		
	Note: Assets may appear to be stuck in the window in between when data is sent to Oz
	and when a response is received. If this proves problematic, we may have to introduce
	a stuck sync time threshold.
		
	\return <code>Observable<table></code> An observable that fires periodically with a 
		list of document Ids that may be in a stuck sync state.
		
--]]

local instanceOfDocsStuckSyncingMetadataStream

function WFCatStuckSyncUtils.getDocsStuckSyncingMetadataStream( 
		wfCatalog, isNormalSyncRunningStream, optCalm, optStuckReportingTimeThreshold,
		optCapabilities )

	local capabilitiesCache = WFCapabilitiesCache( optCapabilities )
		-- this allows unit tests to enable capabilities locally rather than globally

	if wfCatalog:ozUserId() == kGlobalDocumentId then
		return Observable.never() -- Don't track stuck syncs for freemium users.
	end

	if not instanceOfDocsStuckSyncingMetadataStream then

		local stuckTimeThreshold = optStuckReportingTimeThreshold or kStuckReportingTimeThresholdInSeconds

		local docStore = wfCatalog:docStore()

		local assetsNeedingMetadataUploadSql

		local assetSyncStatusList = WFCatSyncStatus.getAssetSyncStatusList()

		if WFPlatformInfo.isMobile() then -- iOS & Android

			assetsNeedingMetadataUploadSql =
					[[ SELECT docId
							FROM coreInfo
							WHERE syncStatus == 1
						UNION
						SELECT assetId FROM unconfirmedDeletions ]]

		else -- Desktop
		
			assetsNeedingMetadataUploadSql =
					[[ SELECT assetId 
							FROM asset_sync_status_ui
							WHERE keepLocallyConfiguration == ?
								AND syncStatuses & ?
						UNION
						SELECT assetId FROM unconfirmedDeletions ]]

		end

		local assetsNeedingMetadataUploadStream = wfCatalog:indexStore():query( {
				sql = assetsNeedingMetadataUploadSql,
				calm = optCalm or 30,
				singleColumnQuery = true,
				WFCatSyncStatus.getKeepLocallyConfiguration(),
				assetSyncStatusList.NeedsMetadataUploaded
			} )

			:flatMapLatest( function( assetIds )
			
				local assetDocValueStream = WFRxDocValueWatcher.make( Observable.just( assetIds ), docStore, "stuckSync" )
				
					:bufferWithTimeOrCount( optCalm or 1, 100, WFRxScheduler.sharedScheduler() )
					
					:scan( function( docValues, batchOfDocValues )
					
						for _, docValue in ipairs( batchOfDocValues ) do
						
							docValues[ docValue._id ] = docValue
						
						end
					
						return docValues
						
					end, {} )
					
				return assetDocValueStream
			
			end )

		local docsNeedingADSUploadStream = Observable.just( {} )

		if capabilitiesCache:get( 'WFCatBinaryUploader.atomicDevelopSettings' ) then

			-- Asset documents emitted by this query are not considered
			-- stuck, as metadata sync is gated by the atomic upload of
			-- the associated develop settings and auxiliaries.

			-- Develop settings and auxiliaries that need to be uploaded
			-- atomically - before the current document metadata can be considered
			-- synced - will show up in needsBinaryUpload with kReadyToUploadAtomic.
			-- Once the binaries have been uploaded and the ADS temp links are present,
			-- the kReadyToUploadAtomic check will cause the parent documents to no longer
			-- be emitted by this query. Similarly, when the uploads have been confirmed,
			-- the parent documents will no longer show up in this query.

			-- Asset documents can be considered stuck if the metadata appears unsynced
			-- and all ADS binaries have temporary links.

			if not WFCatBinaryUploader then
				WFCatBinaryUploader = require 'WFCatBinaryUploader'
			end

			local kReadyToUploadAtomic = WFCatBinaryUploader.readyToUploadEnum.kReadyToUploadAtomic

			local docsNeedingADSUploadSql =
					[[SELECT DISTINCT docId
						FROM needsBinaryUpload
						WHERE isReadyToUpload & ?
							AND binaryType IN ( 'xmp_develop', 'aux' ) ]]

			docsNeedingADSUploadStream = wfCatalog:indexStore():query {
				sql = docsNeedingADSUploadSql,
				quiesceUntilUpToDate = true,
				singleColumnQuery = true,
				calm = optCalm or 30,
				kReadyToUploadAtomic
			}

			:map( KSTableUtils.invert ) -- Turns result into a map with the docIDs as keys

		end

		local catalogPusherLocalDocValueId = WFCatMetadataPusher.getContinueFromLocalDocValueKey( false )
				
		local catalogPusherChangeCounterStream = docStore:ongoingLocalDocValue( catalogPusherLocalDocValueId )
				:pluck( 'continueFrom' )
		
		local spacePusherLocalDocValueId = WFCatMetadataPusher.getContinueFromLocalDocValueKey( true )
		
		local spacePusherChangeCounterStream = docStore:ongoingLocalDocValue( spacePusherLocalDocValueId )
				:pluck( 'continueFrom' )

		local maybeStuckDocIdsAndTimestamps = {}

		isNormalSyncRunningStream = isNormalSyncRunningStream:distinctUntilChanged()

		instanceOfDocsStuckSyncingMetadataStream = Observable.combineLatest( 
				isNormalSyncRunningStream, assetsNeedingMetadataUploadStream,
				catalogPusherChangeCounterStream, spacePusherChangeCounterStream,
				docsNeedingADSUploadStream )
		
			:filter( function( isSyncRunning )

				-- To reduce false positives, we don't check for stuck assets unless
				-- sync is enabled ("enabled" meaning not paused, and not offline).
			
				if not isSyncRunning then

					maybeStuckDocIdsAndTimestamps = {}

				end

				return isSyncRunning

			end )

			:flatMapLatest( function( _,
					maybeStuckDocValues, catalogPusherChangeCounter,
					spacePusherChangeCounter, docsNeedingADSUpload )

				-- If ever the change counter of the document is smaller than the change counter
				-- of the associated pusher, and the document has unsynced metadata, then the
				-- document may be in a stuck sync state.
				
				-- In order to reduce false positives, we track the timestamp of when the
				-- stuck sync was detected. Only docs that have been stuck for a duration
				-- of time greater than the stuck sync time threshold are emitted by this
				-- code.
			
				catalogPusherChangeCounter = catalogPusherChangeCounter or -1
				spacePusherChangeCounter = spacePusherChangeCounter or -1
				
				for docId, docValue in pairs( maybeStuckDocValues ) do

					if docsNeedingADSUpload[ docId ] then

						-- When ADS is enabled, the asset will remain in a needsMetadataSynced
						-- state until the associated ADS binaries are uploaded and Oz can
						-- assemble the complete revision.

						logger:infof( "Asset %s is not stuck", docId )
						maybeStuckDocIdsAndTimestamps[ docId ] = nil

					elseif not docValue._conflict and not WFCatOzErrors.shouldSkipUploadDueToError( docValue ) then

						-- Make sure we're ignoring assets in conflict, and
						-- other assets that should not be sent to Oz. The index store query
						-- should be filtering these documents out, but it feels safer to a guard
						-- here as well.

						if docValue._spaceId then

							if docValue._changeCounter < spacePusherChangeCounter
									and not LrAssetUtils.isLatestMetadataUploaded( docValue ) then

								if not maybeStuckDocIdsAndTimestamps[ docId ] then

									logger:infof( "SWY asset %s might be stuck", docId )
									maybeStuckDocIdsAndTimestamps[ docId ] = KSDate.timeNow()

								end

							else

								logger:infof( "SWY asset %s is not stuck", docId )
								maybeStuckDocIdsAndTimestamps[ docId ] = nil

							end

						elseif docValue._changeCounter < catalogPusherChangeCounter
								and not LrAssetUtils.isLatestMetadataUploaded( docValue ) then

							if not maybeStuckDocIdsAndTimestamps[ docId ] then

								logger:infof( "Asset %s might be stuck", docId )
								maybeStuckDocIdsAndTimestamps[ docId ] = KSDate.timeNow()

							end

						else

							logger:infof( "Asset %s is not stuck", docId )
							maybeStuckDocIdsAndTimestamps[ docId ] = nil

						end

					end

				end

				-- Stop tracking assets that are no longer stuck

				for docId in pairs( maybeStuckDocIdsAndTimestamps ) do

					if not maybeStuckDocValues[ docId ] then

						maybeStuckDocIdsAndTimestamps[ docId ] = nil
						
						logger:infof( "Asset %s is no longer stuck", docId )
						
					end
					
				end
				
				return checkStuckDocumentAgainstThreshold( maybeStuckDocIdsAndTimestamps, stuckTimeThreshold, optCalm or 10 )
				
			end )

			:replay( nil, 1 )
							
			:refCount()

			:finally( function()
				instanceOfDocsStuckSyncingMetadataStream = nil
			end )

	end
	
	return instanceOfDocsStuckSyncingMetadataStream

end

--------------------------------------------------------------------------------

function WFCatStuckSyncUtils.wasCurrentRevisionGeneratedToResolveStuckSync( docValue )

	local stuckSyncUtilsAnnotation
			= KSTableUtils_derefOpt( docValue, '_localOnly', 'stuckSyncUtils' )

	local lastGeneratedRevIdToResolveStuckSync
			= KSTableUtils_derefOpt( stuckSyncUtilsAnnotation, 'lastGeneratedRevisionId' )

	local generatedRevisionCount
			= KSTableUtils_derefOpt( stuckSyncUtilsAnnotation, 'generatedRevisionCount' ) or 0

	local currentRevisionGeneratedToResolveStuckSync
			= docValue._revId == lastGeneratedRevIdToResolveStuckSync

	return currentRevisionGeneratedToResolveStuckSync, generatedRevisionCount

end

--------------------------------------------------------------------------------

local function wasLastRevisionGeneratedRecently( revisionTimestampInSeconds, recencyWindowInSeconds )

	local timeNSecondsAgo = KSDate.timeNow() - recencyWindowInSeconds

	return revisionTimestampInSeconds >= timeNSecondsAgo

end

--------------------------------------------------------------------------------

local function shouldGenerateRevisionToResolveStuckSync( docValue, testOverrides )

	testOverrides = testOverrides or {}

	local stuckSyncUtilsAnnotation
			= KSTableUtils_derefOpt( docValue, '_localOnly', 'stuckSyncUtils' )

	local lastGeneratedRevisionTimestamp
			= KSTableUtils_derefOpt( stuckSyncUtilsAnnotation, 'lastGeneratedRevisionTimestamp' ) or 0

	local generatedRevisionCount
			= KSTableUtils_derefOpt( stuckSyncUtilsAnnotation, 'generatedRevisionCount' ) or 0

	-- We should only generate a new revision to resolve stuck sync if we haven't
	-- exceeded the maximum number of auto-generated revisions, and if the last
	-- generated revision was not recently created.

	local belowRevisionCountLimit = generatedRevisionCount <
			( testOverrides.maxGeneratedRevisionCount or kMaxGeneratedRevisionCount )

	local lastRevisionGeneratedRecently = wasLastRevisionGeneratedRecently(
			lastGeneratedRevisionTimestamp,
					( testOverrides.generatedRevisionFrequencyInSeconds or kGeneratedRevisionFrequencyInSeconds ) )

	return belowRevisionCountLimit and not lastRevisionGeneratedRecently

end

--------------------------------------------------------------------------------

local function isInvalidAssetDocument( docValue )

	-- The following verifies that the image/video asset has the required
	-- importSource section. We've seen instances of stuck assets that lack
	-- this section, and the theory is that it's related to Cloud Trash operations.
	-- https://jira.corp.adobe.com/browse/WF-1824
	-- https://jira.corp.adobe.com/browse/WF-1824

	local isValid = WFCatIndexUtils.isFileBasedAssetDocument( docValue )
			and KSTableUtils_derefOpt( docValue, 'importSource' )

	return not isValid

end

--------------------------------------------------------------------------------

local function removeDeprecatedAnnotations( overlay )

	KSTableUtils.setNested( overlay, { _delete = true }, '_localOnly', 'lastGeneratedRevIdToResolveStuckSync' )
	KSTableUtils.setNested( overlay, { _delete = true }, '_localOnly', 'lastGeneratedRevIdToResolveStuckSync_new' )

end

--------------------------------------------------------------------------------

local function generateRevisionOverlayToResolveStuckSync( docValue )

	local newRevId = KSIdGenerator.uuidInLowercaseHex()

	local generatedRevisionCount
			= KSTableUtils_derefOpt( docValue, '_localOnly', 'stuckSyncUtils', 'generatedRevisionCount' ) or 0

	local overlay = {

		_id = docValue._id,

		_newRevId = newRevId,
			-- bump the sequence counter in order to trigger a resync.

		_localOnly = {

			-- Track the auto-generated revision info, which is used to prevent loopers.

			stuckSyncUtils = {
				lastGeneratedRevisionId = newRevId, -- Track the latest generated revision ID
				lastGeneratedRevisionTimestamp = KSDate.timeNow(), -- Track the timestamp of the latest generated revision
				generatedRevisionCount = generatedRevisionCount + 1, -- Track the count of generated revisions
			}

		}

	}

	removeDeprecatedAnnotations( overlay )

	return overlay

end

--------------------------------------------------------------------------------

local function trackResolveStuckMetadataDocument( docId )

	local event = {}

	event['event.type'] = 'info'
	event['event.subtype'] = 'resolve-error'
	event['event.appWorkflow'] = 'SYNC'
	event['event.subcategory'] = 'Metadata_pusher'
	event['event.error_type'] = 'MetadataUpload.AutoResolveStuckSync'
	event['content.id'] = docId

	WFAnalytics.trackEvent( event )

end

--------------------------------------------------------------------------------

local haveLoggedFailureToAutoResolveStuckSync = {}
	-- Only log stuck sync info *once* per app session. At most, we will log the
	-- following for a single document:
	--
	-- MetadataUpload.StuckSync w/ hasTriedToAutoResolveStuckSync == false
	-- MetadataUpload.AutoResolveStuckSync
	-- MetadataUpload.StuckSync w/ hasTriedToAutoResolveStuckSync == true

local function maybeTrackStuckMetadataDocument(
		docValue, wasCurrentRevisionGenerated, generatedRevisionCount, isInvalidDocument, isDeleted )

	-- Note: This function only returns a boolean for unit testing purposes.

	local docId = docValue._id

	if not haveLoggedFailureToAutoResolveStuckSync[ docId ] then

		local errorEvent = {}

		errorEvent['event.type'] = 'error'
		errorEvent['event.appWorkflow'] = 'SYNC'
		errorEvent['event.subcategory'] = 'Metadata_pusher'
		errorEvent['event.error_type'] = 'MetadataUpload.StuckSync'
		errorEvent['content.hasTriedToAutoResolveStuckSync'] = wasCurrentRevisionGenerated
		errorEvent['content.generatedRevisionCount'] = generatedRevisionCount
		errorEvent['content.invalidDocument'] = not not isInvalidDocument
		errorEvent['content.isTombstone'] = not not isDeleted
		errorEvent['content.id'] = docId

		if generatedRevisionCount > 10 then

			-- We might have a looper, so extract out even more info.

			-- FIXME [jandrew 2023-08-23]: Lets make this better, and create a utility
			-- function for creating a minimized doc value that can be used for analytics.

			local extra = {
				revisionIds = KSTableUtils_derefOpt( docValue, '_serverProvidedMetadata', 'revision_ids' ) or 'unknown',
				labels = KSTableUtils_derefOpt( docValue, '_labels') or 'unknown'
			}

			errorEvent['content.extra'] = KSJsonEncoder.encodeOzUploadJson( extra )

		end

		WFAnalytics.trackEvent( errorEvent )

		haveLoggedFailureToAutoResolveStuckSync[ docId ] = wasCurrentRevisionGenerated

		return true

	end

	return false

end

WFCatStuckSyncUtils._DEBUG_maybeTrackStuckMetadataDocument = maybeTrackStuckMetadataDocument
	-- Exposed for unit testing

--------------------------------------------------------------------------------

--[[!

	\brief Attempts to resolve stuck syncs for a given list of document IDs. This API
		only works for assets stuck due to non-uploaded metadata.
		
	\param wfCatalog (WFCatalog) 

	\param stuckDocIds (table) a table of document IDs in a stuck sync state.

	\param testOverrides (optional table) a table of overrides, only used by unit tests:
		generatedRevisionFrequencyInSeconds: Used to override kGeneratedRevisionFrequencyInSeconds
		maxGeneratedRevisionCount: Used to override kMaxGeneratedRevisionCount

	\return <code>Observable</code> An observable that completes after
		attempting to resolve the stuck sync.
		
--]]

function WFCatStuckSyncUtils.resolveStuckSync( wfCatalog, stuckDocIds, testOverrides )

	WFArguments_check( 'WFCatalog', 'table', '?table' )

	stuckDocIds = stuckDocIds or {}
	stuckDocIds.simplifyDeletedDocs = false

	local overlayStream =
			WFRxThrottledBatchedBulkDocValue( "resolveStuckSync", wfCatalog:docStore(), stuckDocIds, 100, 10 )

				:scan( function( overlays, batchOfDocValues )

					for _, docValue in ipairs( batchOfDocValues ) do

						local wasCurrentRevisionGenerated, generatedRevisionCount
								= WFCatStuckSyncUtils.wasCurrentRevisionGeneratedToResolveStuckSync( docValue )

						local isInvalidDocument = isInvalidAssetDocument( docValue )

						local isDeleted = docValue._deleted

						maybeTrackStuckMetadataDocument(
								docValue, wasCurrentRevisionGenerated, generatedRevisionCount,
								isInvalidDocument, isDeleted )

						if shouldGenerateRevisionToResolveStuckSync( docValue, testOverrides ) then -- Looper prevention

							trackResolveStuckMetadataDocument( docValue._id )

							local overlay = generateRevisionOverlayToResolveStuckSync( docValue )

							overlays[ #overlays + 1 ] = overlay

						end

					end

					return overlays

				end, {} )

				:takeLast( 1 )

	return overlayStream

		:flatMap( function( overlays )

			return WFRxThrottledBatchedBulkApplyOverlay(
					"resolveStuckSync", wfCatalog:docStore(), overlays, 100, 10 )

				:takeLast( 1 )

		end )

end

--------------------------------------------------------------------------------

function WFCatStuckSyncUtils.addDiagnosticLogSectionFunc( wfCatalog )

	WFArguments_check( 'WFCatalog' )

	local fireWithTrue = Observable.just( true )

	WFDiagnosticLog.addLogSectionFunc( function()
	
		local body = {
		
			_README_ = "Note: Assets may appear to be stuck in the window in between when data is sent to Oz, and when a response is received.",
			
			docsStuckSyncingMetadata
					= WFCatStuckSyncUtils.getDocsStuckSyncingMetadataStream( wfCatalog, fireWithTrue )
						:take( 1 )

		}

		return {

			name = "Stuck Sync Detection",
			id = 'WFCatStuckSyncUtils',
			weight = 510,
			parentSection = WFDiagnosticLog.Sections.SYNC,
			body = body

		}

	end )

end

--------------------------------------------------------------------------------

--[[!

	\brief Clears all error states and retries sync for a list of assets.

	This function removes all error-related annotations and generates new revision IDs
	for the specified assets, effectively resetting their sync state and allowing them
	to retry synchronization.

	\param wfCatalog (WFCatalog) The catalog containing the assets to retry.

	\param assetIds (table) A list of document IDs for assets that need their errors cleared
		and sync retried.

	\return <code>Observable</code> An observable that completes after applying the
		error clearing overlays to all specified assets.

--]]

function WFCatStuckSyncUtils.clearErrorsAndRetrySync( wfCatalog, assetIds )

	local timeStamp = KSDate.formatDateTimeAsUTC( KSDate.timeNow() ) .. "Z"

	local overlays = {}

	for i, docId in ipairs( assetIds ) do

		local newRevId = KSIdGenerator.uuidInLowercaseHex()

		local clearErrorsOverlay = {
			_id = docId,
			_newRevId = newRevId,
			userUpdated = KSDate.timeInIso8601Format(),
			_localOnly = {
				lastGeneratedRevIdToResolveStuckSync = { _delete = true }, -- deprecated annotation
				lastGeneratedRevIdToResolveStuckSync_new = { _delete = true }, -- deprecated annotation
				stuckSyncUtils = { _delete = true },
				ozError = { _delete = true },
				localOriginalError = { _delete = true },
				binaryUploadErrors = { _delete = true },
				binaryDownloadErrors = { _delete = true },
				deniedBy = { _delete = true },
				blacklistedBy = { _delete = true },
				history = { [ timeStamp ] = "User request clearErrorsAndRetrySync" },
			}
		}

		overlays[ i ] = clearErrorsOverlay

	end

	return WFRxThrottledBatchedBulkApplyOverlay( "clearErrorsAndRetrySync", wfCatalog:docStore(), overlays, 100, 10 )

end

--==============================================================================

return WFCatStuckSyncUtils
