--[[----------------------------------------------------------------------------

WFCatErrorHandlerTest.lua

--------------------------------------------------------------------------------

ADOBE CONFIDENTIAL
------------------
Copyright 2016 Adobe Systems Incorporated
All Rights Reserved.

NOTICE: All information contained herein is, and remain the property of
Adobe Systems Incorporated and its suppliers, if any. The intellectual and
technical concepts contained herein are proprietary to Adobe Systems
Incorporated and its suppliers and may be covered by U.S. and Foreign
Patents, patents in process, and are protected by trade secret or copyright
law.  Dissemination of this information or reproduction of this material is
strictly forbidden unless prior written permission is obtained from
Adobe Systems Incorporated.

------------------------------------------------------------------------------]]

local Rx = require 'Rx'

local KSData = require 'KSData'
local KSDate = require "KSDate"
local KSTableUtils = require 'KSTableUtils'
local WFRxTestCase = require 'WFRxTestCase'
local WFRxTestUtils = require 'WFRxTestUtils'

local WFCatAssetAux = require 'WFCatAssetAux'
local WFCatAssetTestUtils = require 'WFCatAssetTestUtils'
local WFCatBinaryUploader = require 'WFCatBinaryUploader'
local WFCatErrorHandler = require 'WFCatErrorHandler'
local WFCatMockCatalog = require 'WFCatMockCatalog'
local WFCatMetadataPuller = require 'WFCatMetadataPuller'
local WFCatMetadataPusher = require 'WFCatMetadataPusher'
local WFCatStacksTest = require 'WFCatStacksTest'
local WFCatStacks = require 'WFCatStacks'
local WFCatOzErrors = require 'WFCatOzErrors'
local WFCatSpecMock = require 'WFCatSpecMock'
local WFCatTestSession = require 'WFCatTestSession'
local WFCatVersionTestUtils = require 'WFCatVersionTestUtils'

local KSIdGenerator = require 'KSIdGenerator'
local KSJsonEncoder = require 'KSJsonEncoder'
local KSJsonDecoder = require 'KSJsonDecoder'

local WFMockHttpRequest = require 'WFMockHttpRequest'
local WFOzDocUtils = require 'WFOzDocUtils'
local WFPlatformInfo = require 'WFPlatformInfo'

local WFArguments = require 'WFArguments'
local KSPathUtils = require 'KSPathUtils'
local WFFileUtils = require 'WFFileUtils'

local WFRxRunUntilDone = require 'WFRxRunUntilDone'

--------------------------------------------------------------------------------

local runUntilDone = WFRxRunUntilDone

local WFRxScheduler = require 'WFRxScheduler'
local WFRxBatchedBulkApplyOverlay = require 'WFRxBatchedBulkApplyOverlay'

--------------------------------------------------------------------------------

local WFCatErrorHandlerTest = {}

local function bulkDelete( catalog, docIds  )

	-- very simple bulkDelete that does not delete album_assets automatically

	local overlays = {}

	for i, docId in ipairs( docIds ) do
		overlays[ #overlays + 1 ] = {
			_id = docId,
			_deleted = true,
			_localOnly = { deletedLocally = true },
		}
	end

	return WFRxBatchedBulkApplyOverlay( "bulkDeleteTest", catalog:docStore(), overlays )

end

--------------------------------------------------------------------------------

local function deleteDoc( catalog, docId )

	return bulkDelete( catalog, { docId } )
end

--------------------------------------------------------------------------------

local function makeExternalizeSettings( sha256 )

	return function( catalog, assetId, maybeLimit )

		local docStore = catalog:docStore()

		return docStore:docValue( assetId )
			:flatMap( function( docValue )
				return docStore:applyOverlay {
					_id = assetId,
					develop = {
						xmpCameraRaw = {
							sha256 = sha256
						}
					}
				}
		end )

	end

end

--------------------------------------------------------------------------------

local function makeAssetDocDisplayable( docValue )

	-- add a local master path, otherwise asset would not be "displayable"
	-- and only displayable assets have a syncStatus.

	if WFPlatformInfo.isDesktop then

		docValue._localOnly = {
			files = {
				original = {
					master = {
						relativePath = "foo",
						volumeGuid = "bar"
					}
				}
			}
		}

	else

		docValue._localOnly = {
			-- iOS
			files = {
				master = {
					path = "..."
				}
			},
			-- Android
			originalFilePath = "...",
		}

	end

	return docValue

end

--------------------------------------------------------------------------------

local options = {

	continuousTaskDelays = {
		progressiveBackoff = false,
		noWorkDelay = 1,
		maxNoWorkDelay = 1,
		errorDelay = 1,
		maxDelay = 1,
	},

	isCatalogPuller = true

}

--------------------------------------------------------------------------------

local errorHandlerOptions = {
	debugMode = true
}

--------------------------------------------------------------------------------

-- @mark maybeMarkDocumentForRetry

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestMaybeMarkDocumentForRetry( rxTest )

	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc that has not been synced to Oz
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			type = 'asset',
			subtype = 'image',
		}
	)

	-- lookup doc from Oz

	local docValue
	local prevChangeCounter

	local function lookupDocValue( expectedSubtype, expectedStatus, expectedRetries, expectedChangeCounter )

		rxTest:waitFor(
			docStore:docValue( docId )
				:map( function( doc )
					docValue = doc
					prevChangeCounter = docValue._changeCounter
				end )
			)

		if expectedSubtype then
			local ozError = KSTableUtils.derefOpt( docValue, '_localOnly', 'ozError' )
			rxTest:expectEqual( expectedSubtype, ozError.subtype )
			rxTest:expectEqual( expectedStatus, ozError.status )
			rxTest:expectEqual( expectedRetries, ozError.retries )
			rxTest:expectEqual( expectedChangeCounter, prevChangeCounter )
		end

	end

	lookupDocValue()

	-- mark with error should be setup for first retry

	rxTest:expectSingleValue(
		WFCatErrorHandler.maybeMarkDocumentForRetry( docStore, docValue, 'ErrorOne', 404 ),
		'MarkedForRetry' )

	lookupDocValue( 'ErrorOne', 404, 1, prevChangeCounter + 1 )

	-- mark with same error should be setup for second retry

	rxTest:expectSingleValue(
		WFCatErrorHandler.maybeMarkDocumentForRetry( docStore, docValue, 'ErrorOne', 404 ),
		'MarkedForRetry' )

	lookupDocValue( 'ErrorOne', 404, 2, prevChangeCounter + 1 )

	-- mark with same error should be setup for third retry

	rxTest:expectSingleValue(
		WFCatErrorHandler.maybeMarkDocumentForRetry( docStore, docValue, 'ErrorOne', 404 ),
		'MarkedForRetry' )

	lookupDocValue( 'ErrorOne', 404, 3, prevChangeCounter + 1 )

	-- mark with same error will not change out of retries

	rxTest:expectSingleValue(
		WFCatErrorHandler.maybeMarkDocumentForRetry( docStore, docValue, 'ErrorOne', 404 ),
		'Ignored_ExceededRetries' )

	lookupDocValue( 'ErrorOne', 404, 3, prevChangeCounter )

	-- mark with same error with optional error doc will not change out of retries

	rxTest:expectSingleValue(
		WFCatErrorHandler.maybeMarkDocumentForRetry( docStore, docValue, 'ErrorOne', 404, { error = 'document' } ),
		'Ignored_ExceededRetries' )

	lookupDocValue( 'ErrorOne', 404, 3, prevChangeCounter )

	-- mark with different error subtype should reset retries

	rxTest:expectSingleValue(
		WFCatErrorHandler.maybeMarkDocumentForRetry( docStore, docValue, 'ErrorTwo', 404 ),
		'MarkedForRetry' )

	lookupDocValue( 'ErrorTwo', 404, 1, prevChangeCounter + 1 )

	-- mark with different error status should reset retries

	rxTest:expectSingleValue(
		WFCatErrorHandler.maybeMarkDocumentForRetry( docStore, docValue, 'ErrorTwo', 403 ),
		'MarkedForRetry' )

	lookupDocValue( 'ErrorTwo', 403, 1, prevChangeCounter + 1 )

	-- if last retry was more than an hour ago, should reset retries

	local hourAgoTime = KSDate.shiftIso8601DateTime( KSDate.timeInIso8601Format(), -(60 * 60 + 1) )

	rxTest:waitFor(
		docStore:applyOverlay {
			_id = docId,
			_localOnly = {
				ozError = {
					lastRetry = hourAgoTime
				}
			}
		}
	)

	lookupDocValue()

	rxTest:expectSingleValue(
		WFCatErrorHandler.maybeMarkDocumentForRetry( docStore, docValue, 'ErrorTwo', 403 ),
		'MarkedForRetry' )

	lookupDocValue( 'ErrorTwo', 403, 1, prevChangeCounter + 1 )

end

--------------------------------------------------------------------------------

-- @mark ResourceNotFoundError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResourceNotFoundErrorSimple( rxTest )

	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc that has not been synced to Oz
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		}
	)

	-- create error doc for asset that indicates the resource is not found
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"unknown error",
					},
				},
				missing_resource_id = docId,
				source = {
					action = "update",
					id = docId,
					revision_id = KSIdGenerator.uuidInLowercaseHex(),
					type = "asset",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- ResourceNotFoundError for asset that is not yet on Oz should be ignored and
	-- we will assume that the condition blocking the resource getting to Oz is separate
	-- and will be cleared up eventually

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	-- asset should not have ResourceNotFoundError annotation, it may have one for the error that
	-- is presenting the asset from getting to Oz at all, but that should not be replaced by a
	-- ResourceNotFoundError
	rxTest:expectSingleValue(
		docStore:docValue( docId )
			:map( function( docValue )
				return KSTableUtils.derefOpt( docValue, '_localOnly', 'ozError', 'subtype' ) ~= 'ResourceNotFoundError'
			end ), true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResourceNotFoundErrorOzDeleted( rxTest )

	-- a ResourceNotFoundError for a document that used to be on Oz should cause that
	-- document to be deleted
	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc that has not been synced to Oz
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			_newRevId = revisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			},
			_serverProvidedMetadata = {
				created = '2016-06-07T15:55:34.218872Z'
			}
		}
	)

	-- create error doc for asset that indicates the resource is not found
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"unknown error",
					},
				},
				missing_resource_id = docId,
				source = {
					action = "update",
					id = docId,
					revision_id = revisionId,
					type = "asset",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- ResourceNotFoundError for asset that is not yet on Oz should be ignored and
	-- we will assume that the condition blocking the resource getting to Oz is separate
	-- and will be cleared up eventually

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	-- a document that Oz responds to with ResourceNotFoundError that is previously known to
	-- have existed on Oz should be removed
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, docId ) )
		:expectDataWithinTime( 3, true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResourceNotFoundErrorCausedByDoubleDelete( rxTest )

	-- a ResourceNotFoundError for a document that used to be on Oz should cause that
	-- document to be deleted
	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	local deletedImageId = KSIdGenerator.uuidInLowercaseHex()

	-- create asset as appears after first delete.
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			_labels = { oz = true },
		}
	)

	-- delete the asset (and create the trash asset)
	rxTest:waitFor(
		docStore:applyOverlay {
			_id = docId,
			_deleted = true,
			_newId = deletedImageId,
			_newIdOverlay = {
				type = 'asset',
				subtype = 'deleted_image',
				userCreated = '2016-06-07T15:55:34.218872Z',
				purgeDate = '2016-06-08T15:55:34.218872Z',
				original = {
					id = docId,
					userCreated = '2016-06-07T15:55:34.218872Z',
				},
			}
		}
	)

	-- create error doc for asset that indicates the resource is not found
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			_labels = { oz = true },
			_localId = 4,
			_serverProvidedMetadata = {
				new_id = deletedImageId,
				errors = {
					asset = {
						"does not exist",
					},
				},
				missing_resource_id = docId,
				source = {
					action = "delete",
					id = docId,
					revision_id = revisionId,
					type = "asset",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
			subtype = "ResourceNotFoundError",
			type = "error",
		}
	)

	-- ResourceNotFoundError for asset that has already been deleted by this
	-- client, as simulated by the conditions above, should be ignored.

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	-- trash asset should remain in the trash after error doc is removed and
	-- error handling is complete
	rxTest:expectSingleValue( docStore:docValue( deletedImageId )
		:map( function( docValue )
			return docValue._deleted == nil and docValue._empty == nil
		end ), true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResourceNotFoundErrorIndirect( rxTest )

	-- ResourceNotFoundErrors that are indirect and not for the doc that was sent to oz (e.g.
	-- getting a ResourceNotFoundError on an album_asset create because the asset in question
	-- does not exist on Oz (yet) should be handled by copying any error on the root asset to
	-- the document that received the error. This allows downstream docs to be cleared when the
	-- root document does

	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local rootDocId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc that has not been synced to Oz
	rxTest:waitFor(
		docStore:setDocValue {
			_id = rootDocId,
			_newRevId = revisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		}
	)

	-- create error doc for asset that would cause a ResourceNotFoundError later
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "StorageExceededError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"unknown error",
					},
				},
				source = {
					action = "create",
					id = rootDocId,
					revision_id = revisionId,
					type = "asset",
				},
				status = 403,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)


	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	-- asset doc should end up with quota exceeded annotation
	rxTest:expectSingleValue(
		docStore:docValue( rootDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasStorageExceededError( docValue )
			end ), true )


	local derivedDocId = KSIdGenerator.uuidInLowercaseHex()
	local derivedRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local derivedErrorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create the album_asset doc
	rxTest:waitFor(
		docStore:setDocValue {
			_id = derivedDocId,
			_newRevId = derivedRevisionId,
			album = {
				id = KSIdGenerator.uuidInLowercaseHex(),
			},
			asset = {
				id = rootDocId,
			},
			type = "album_asset",
		}
	)


	-- set album_asset error doc
	rxTest:waitFor(
		docStore:setDocValue {
			_id = derivedErrorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"resource not found",
					},
				},
				missing_resource_id = rootDocId,
				source = {
					action = "create",
					id = derivedDocId,
					revision_id = derivedRevisionId,
					type = "album_asset",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, derivedErrorDocId ) )
		:expectDataWithinTime( 5, true )

	-- album_asset doc should end up with quota exceeded annotation
	rxTest:expectSingleValue(
		docStore:docValue( derivedDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasStorageExceededError( docValue )
			end ), true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResourceNotFoundErrorContributorDelete( rxTest )

	-- ResourceNotFoundErrors that are indirect and not for the doc that was sent to oz (e.g.
	-- getting a ResourceNotFoundError on an album_asset create because the asset in question
	-- does not exist on Oz (yet) should be handled by copying any error on the root asset to
	-- the document that received the error. This allows downstream docs to be cleared when the
	-- root document does

	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local rootDocId = KSIdGenerator.uuidInLowercaseHex()
	local trashDocId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc (that will be deleted below)
	rxTest:waitFor(
		docStore:setDocValue {
			_id = rootDocId,
			_labels = { oz = true },
		}
	)

	-- delete the asset (and create the trash asset)
	rxTest:waitFor(
		docStore:applyOverlay {
			_id = rootDocId,
			_deleted = true,
			_newId = trashDocId,
			_newIdOverlay = {
				type = 'asset',
				subtype = 'deleted_image',
				userCreated = '2016-06-07T15:55:34.218872Z',
				purgeDate = '2016-06-08T15:55:34.218872Z',
				original = {
					id = rootDocId,
					userCreated = '2016-06-07T15:55:34.218872Z',
				},
				_localOnly = {
					universal = {
						trash = {
							createFrom = {
								id = rootDocId,
								lastAssetId = rootDocId,
								lastRevisionId = revisionId,
								lastTrashPurgeDate = "2019-07-12T21:32:40.415Z",
								lastTrashUserCreated = "2019-05-13T21:32:40.415Z",
								revision_id = revisionId,
								subtype = "image",
							},
							createWithRevId = KSIdGenerator.uuidInLowercaseHex(),
							uploaded = {
								["/rels/master"] = true,
								["/rels/proxy_type/2560"] = true,
								["/rels/xmp"] = true,
							},
						},
					},
				}
			}
		}
	)


	-- create error doc for asset that indicates the resource is not found and
	-- was deleted by another user
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				deleted_by = KSIdGenerator.uuidInLowercaseHex(),
				errors = {
					asset = {
						"unknown error",
					},
				},
				missing_resource_id = rootDocId,
				source = {
					action = "update",
					id = rootDocId,
					revision_id = KSIdGenerator.uuidInLowercaseHex(),
					type = "asset",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	-- a document that Oz responds to with ResourceNotFoundError with a deleted_by
	-- should ensure that any trash documents originating from the document that
	-- was deleted by the contributor are also delted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, rootDocId ) )
		:expectDataWithinTime( 3, true )

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, trashDocId ) )
		:expectDataWithinTime( 3, true )

	-- do the same basic tests again, but push the trash operations out two steps to
	-- make sure we can follow the _newId and properly remove the doc
	rootDocId = KSIdGenerator.uuidInLowercaseHex()
	trashDocId = KSIdGenerator.uuidInLowercaseHex()
	errorDocId = KSIdGenerator.uuidInLowercaseHex()
	local trashDocId2 = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc (that will be deleted below)
	rxTest:waitFor(
		docStore:setDocValue {
			_id = rootDocId,
			_labels = { oz = true },
		}
	)

	-- delete the asset (and create the trash asset)
	rxTest:waitFor(
		docStore:applyOverlay {
			_id = rootDocId,
			_deleted = true,
			_newId = trashDocId,
			_newIdOverlay = {
				type = 'asset',
				subtype = 'deleted_image',
				userCreated = '2016-06-07T15:55:34.218872Z',
				purgeDate = '2016-06-08T15:55:34.218872Z',
				original = {
					id = rootDocId,
					userCreated = '2016-06-07T15:55:34.218872Z',
				},
				_localOnly = {
					universal = {
						trash = {
							createFrom = {
								id = rootDocId,
								lastAssetId = rootDocId,
								lastRevisionId = revisionId,
								lastTrashPurgeDate = "2019-07-12T21:32:40.415Z",
								lastTrashUserCreated = "2019-05-13T21:32:40.415Z",
								revision_id = revisionId,
								subtype = "image",
							},
							createWithRevId = KSIdGenerator.uuidInLowercaseHex(),
							uploaded = {
								["/rels/master"] = true,
								["/rels/proxy_type/2560"] = true,
								["/rels/xmp"] = true,
							},
						},
					},
				}
			}
		}
	)

	-- delete the trashed asset asset and create a new asset, i.e. restore
	rxTest:waitFor(
		docStore:applyOverlay {
			_id = trashDocId,
			_deleted = true,
			_newId = trashDocId2,
			_newIdOverlay = {
				type = 'asset',
				subtype = 'image',
				userCreated = '2016-06-07T15:55:34.218872Z',
				purgeDate = '2016-06-08T15:55:34.218872Z',
				original = {
					id = rootDocId,
					userCreated = '2016-06-07T15:55:34.218872Z',
				},
				_localOnly = {
					universal = {
						trash = {
							createFrom = {
								id = trashDocId,
								lastAssetId = trashDocId,
								lastRevisionId = revisionId,
								lastTrashPurgeDate = "2019-07-12T21:32:40.415Z",
								lastTrashUserCreated = "2019-05-13T21:32:40.415Z",
								revision_id = revisionId,
								subtype = "deleted_image",
							},
							createWithRevId = KSIdGenerator.uuidInLowercaseHex(),
							uploaded = {
								["/rels/master"] = true,
								["/rels/proxy_type/2560"] = true,
								["/rels/xmp"] = true,
							},
						},
					},
				}
			}
		}
	)


	-- create error doc for asset that indicates the resource is not found and
	-- was deleted by another user
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				deleted_by = KSIdGenerator.uuidInLowercaseHex(),
				errors = {
					asset = {
						"unknown error",
					},
				},
				missing_resource_id = rootDocId,
				source = {
					action = "update",
					id = rootDocId,
					revision_id = KSIdGenerator.uuidInLowercaseHex(),
					type = "asset",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	-- a document that Oz responds to with ResourceNotFoundError with a deleted_by
	-- should ensure that any trash documents originating from the document that
	-- was deleted by the contributor are also delted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, rootDocId ) )
		:expectDataWithinTime( 3, true )

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, trashDocId ) )
		:expectDataWithinTime( 3, true )

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, trashDocId2 ) )
		:expectDataWithinTime( 3, true )
end

--------------------------------------------------------------------------------

local indexDefSpec = WFPlatformInfo.isDesktop and 'WFCatIndexDefinitions.lua' or 'LrmCatalogIndexDefinitions.lua'
local catSyncStatus =  WFPlatformInfo.isDesktop and require 'WFCatSyncStatus' or require 'LrmCatSyncStatus'

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResolveResourceNotFoundError_AlbumAsset( rxTest )

	rxTest:setTimeLimit( 15 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, nil, indexDefSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
			getSyncStatusOfAssetsOfAlbumAssets = catSyncStatus.getSyncStatusOfAssetsOfAlbumAssets,
			getSyncStatusOfAlbumsOfAlbumAssets = catSyncStatus.getSyncStatusOfAlbumsOfAlbumAssets,
			getSyncStatusOfParentCollectionOfAlbums = catSyncStatus.getSyncStatusOfParentCollectionOfAlbums,
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local rootDocId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local albumId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc that has not been synced to Oz
	local docValue = {
		_id = rootDocId,
		_newRevId = revisionId,
		type = 'asset',
		subtype = 'image',
		importSource = {
			filePath = '/path/',
			originalHeight = 150,
			originalWidth = 100,
		}
	}

	docValue = makeAssetDocDisplayable( docValue )

	rxTest:waitFor( docStore:setDocValue( docValue ) )

	local derivedDocId = KSIdGenerator.uuidInLowercaseHex()
	local derivedRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local derivedErrorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create the album_asset doc
	rxTest:waitFor(
		docStore:setDocValue {
			_id = derivedDocId,
			_newRevId = derivedRevisionId,
			album = {
				id = albumId,
			},
			asset = {
				id = rootDocId,
			},
			type = "album_asset",
		}
	)

	-- set album_asset error doc
	rxTest:waitFor(
		docStore:setDocValue {
			_id = derivedErrorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"resource not found",
					},
				},
				missing_resource_id = rootDocId,
				source = {
					action = "create",
					id = derivedDocId,
					revision_id = derivedRevisionId,
					type = "album_asset",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, derivedErrorDocId ) )
		:expectDataWithinTime( 5, true )

	rxTest:expectSingleValue(
		docStore:docValue( derivedDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end ), true )

	-- Make the asset appear synced.

	local overlay = {
		_id = rootDocId,
		_serverProvidedMetadata = {
			links = {
				[ '/rels/master'] = {
					href = "/v2/...."
				}
			},
			revision_ids = {}
		}
	}

	rxTest:waitFor( docStore:applyOverlay( overlay ) )

	-- Make the album appear synced.

	rxTest:waitFor(
		docStore:setDocValue {
			_id = albumId,
			cover = {
				id = rootDocId,
			},
			name = "Sync me",
			type = "album",
			subtype = "collection",
			_serverProvidedMetadata = {
				links = {},
				revision_ids = {}
			}
		}
	)

	-- Once the album and asset are synced, the resource not found error should
	-- be removed from the album_asset.

	rxTest:observe( docStore:ongoingDocValue( derivedDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end )
		)
		:expectDataWithinTime( 15, false )
		:dispose()

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResolveResourceNotFoundError_DependentVersion( rxTest )

	rxTest:setTimeLimit( 15 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, nil, indexDefSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
			getSyncStatusOfAssetsOfAlbumAssets = catSyncStatus.getSyncStatusOfAssetsOfAlbumAssets,
			getSyncStatusOfAlbumsOfAlbumAssets = catSyncStatus.getSyncStatusOfAlbumsOfAlbumAssets,
			getSyncStatusOfParentCollectionOfAlbums = catSyncStatus.getSyncStatusOfParentCollectionOfAlbums,
			getSyncStatusOfAssetsOfVersions = catSyncStatus.getSyncStatusOfAssetsOfVersions,
			getSyncStatusOfAssetsOfComments = catSyncStatus.getSyncStatusOfAssetsOfComments,
			getSyncStatusOfAssetsOfFavorites = catSyncStatus.getSyncStatusOfAssetsOfFavorites,

		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local rootDocId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc that has not been synced to Oz
	local docValue = {
		_id = rootDocId,
		_newRevId = revisionId,
		type = 'asset',
		subtype = 'image',
		importSource = {
			filePath = '/path/',
			originalHeight = 150,
			originalWidth = 100,
		}
	}

	docValue = makeAssetDocDisplayable( docValue )

	rxTest:waitFor( docStore:setDocValue( docValue ) )

	local dependentDocId = KSIdGenerator.uuidInLowercaseHex()
	local dependentRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local dependentErrorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create the dependent doc

	rxTest:waitFor(
		docStore:setDocValue {
			_id = dependentDocId,
			_newRevId = dependentRevisionId,
			asset = {
				id = rootDocId,
			},
			type = "version",
			subtype = "edit",
		}
	)

	-- set dependent error doc

	rxTest:waitFor(
		docStore:setDocValue {
			_id = dependentErrorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"resource not found",
					},
				},
				missing_resource_id = rootDocId,
				source = {
					action = "create",
					id = dependentDocId,
					revision_id = dependentRevisionId,
					type = "version",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, dependentErrorDocId ) )
		:expectDataWithinTime( 5, true )

	rxTest:expectSingleValue(
		docStore:docValue( dependentDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end ), true )

	-- Make the asset appear synced.

	local overlay = {
		_id = rootDocId,
		_serverProvidedMetadata = {
			links = {
				[ '/rels/master'] = {
					href = "/v2/...."
				}
			},
			revision_ids = {}
		}
	}

	rxTest:waitFor( docStore:applyOverlay( overlay ) )

	-- Once the asset is synced, the resource not found error should
	-- be removed from the dependent doc.

	rxTest:observe( docStore:ongoingDocValue( dependentDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end )
		)
		:expectDataWithinTime( 15, false )
		:dispose()

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResolveResourceNotFoundError_DependentFavorite( rxTest )

	rxTest:setTimeLimit( 15 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, nil, indexDefSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
			getSyncStatusOfAssetsOfAlbumAssets = catSyncStatus.getSyncStatusOfAssetsOfAlbumAssets,
			getSyncStatusOfAlbumsOfAlbumAssets = catSyncStatus.getSyncStatusOfAlbumsOfAlbumAssets,
			getSyncStatusOfParentCollectionOfAlbums = catSyncStatus.getSyncStatusOfParentCollectionOfAlbums,
			getSyncStatusOfAssetsOfVersions = catSyncStatus.getSyncStatusOfAssetsOfVersions,
			getSyncStatusOfAssetsOfComments = catSyncStatus.getSyncStatusOfAssetsOfComments,
			getSyncStatusOfAssetsOfFavorites = catSyncStatus.getSyncStatusOfAssetsOfFavorites,

		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local rootDocId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc that has not been synced to Oz
	local docValue = {
		_id = rootDocId,
		_newRevId = revisionId,
		type = 'asset',
		subtype = 'image',
		importSource = {
			filePath = '/path/',
			originalHeight = 150,
			originalWidth = 100,
		}
	}

	docValue = makeAssetDocDisplayable( docValue )

	rxTest:waitFor( docStore:setDocValue( docValue ) )

	local dependentDocId = KSIdGenerator.uuidInLowercaseHex()
	local dependentRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local dependentErrorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create the dependent favorite

	rxTest:waitFor(
		docStore:setDocValue {
			_id = dependentDocId,
			_newRevId = dependentRevisionId,
			userCreated = KSDate.timeInIso8601Format(),
			asset = {
				id = rootDocId,
			},
			space = {
				id = KSIdGenerator.uuidInLowercaseHex(),
			},
			type = "favorite",
			subtype = "user",
		}
	)

	-- set dependent error doc

	rxTest:waitFor(
		docStore:setDocValue {
			_id = dependentErrorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"resource not found",
					},
				},
				missing_resource_id = rootDocId,
				source = {
					action = "create",
					id = dependentDocId,
					revision_id = dependentRevisionId,
					type = "favorite",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, dependentErrorDocId ) )
		:expectDataWithinTime( 5, true )

	rxTest:expectSingleValue(
		docStore:docValue( dependentDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end ), true )

	-- Make the asset appear synced.

	local overlay = {
		_id = rootDocId,
		_serverProvidedMetadata = {
			links = {
				[ '/rels/master'] = {
					href = "/v2/...."
				}
			},
			revision_ids = {}
		}
	}

	rxTest:waitFor( docStore:applyOverlay( overlay ) )

	-- Once the asset is synced, the resource not found error should
	-- be removed from the dependent doc.

	rxTest:observe( docStore:ongoingDocValue( dependentDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end )
		)
		:expectDataWithinTime( 15, false )
		:dispose()

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResolveResourceNotFoundError_DependentComment( rxTest )

	rxTest:setTimeLimit( 15 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, nil, indexDefSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
			getSyncStatusOfAssetsOfAlbumAssets = catSyncStatus.getSyncStatusOfAssetsOfAlbumAssets,
			getSyncStatusOfAlbumsOfAlbumAssets = catSyncStatus.getSyncStatusOfAlbumsOfAlbumAssets,
			getSyncStatusOfParentCollectionOfAlbums = catSyncStatus.getSyncStatusOfParentCollectionOfAlbums,
			getSyncStatusOfAssetsOfVersions = catSyncStatus.getSyncStatusOfAssetsOfVersions,
			getSyncStatusOfAssetsOfComments = catSyncStatus.getSyncStatusOfAssetsOfComments,
			getSyncStatusOfAssetsOfFavorites = catSyncStatus.getSyncStatusOfAssetsOfFavorites,

		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local rootDocId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc that has not been synced to Oz
	local docValue = {
		_id = rootDocId,
		_newRevId = revisionId,
		type = 'asset',
		subtype = 'image',
		importSource = {
			filePath = '/path/',
			originalHeight = 150,
			originalWidth = 100,
		}
	}

	docValue = makeAssetDocDisplayable( docValue )

	rxTest:waitFor( docStore:setDocValue( docValue ) )

	local dependentDocId = KSIdGenerator.uuidInLowercaseHex()
	local dependentRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local dependentErrorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create the dependent doc

	rxTest:waitFor(
		docStore:setDocValue {
			_id = dependentDocId,
			_newRevId = dependentRevisionId,
			userCreated = KSDate.timeInIso8601Format(),
			asset = {
				id = rootDocId,
			},
			space = {
				id = KSIdGenerator.uuidInLowercaseHex(),
			},
			type = "comment",
			subtype = "user",
		}
	)

	-- set dependent error doc

	rxTest:waitFor(
		docStore:setDocValue {
			_id = dependentErrorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"resource not found",
					},
				},
				missing_resource_id = rootDocId,
				source = {
					action = "create",
					id = dependentDocId,
					revision_id = dependentRevisionId,
					type = "comment",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, dependentErrorDocId ) )
		:expectDataWithinTime( 5, true )

	rxTest:expectSingleValue(
		docStore:docValue( dependentDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end ), true )

	-- Make the asset appear synced.

	local overlay = {
		_id = rootDocId,
		_serverProvidedMetadata = {
			links = {
				[ '/rels/master'] = {
					href = "/v2/...."
				}
			},
			revision_ids = {}
		}
	}

	rxTest:waitFor( docStore:applyOverlay( overlay ) )

	-- Once the asset is synced, the resource not found error should
	-- be removed from the dependent doc.

	rxTest:observe( docStore:ongoingDocValue( dependentDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end )
		)
		:expectDataWithinTime( 15, false )
		:dispose()

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResolveResourceNotFoundError_DependentDocDeletion( rxTest )

	rxTest:setTimeLimit( 15 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, nil, indexDefSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
			getSyncStatusOfAssetsOfAlbumAssets = catSyncStatus.getSyncStatusOfAssetsOfAlbumAssets,
			getSyncStatusOfAlbumsOfAlbumAssets = catSyncStatus.getSyncStatusOfAlbumsOfAlbumAssets,
			getSyncStatusOfParentCollectionOfAlbums = catSyncStatus.getSyncStatusOfParentCollectionOfAlbums,
			getSyncStatusOfAssetsOfVersions = catSyncStatus.getSyncStatusOfAssetsOfVersions,
			getSyncStatusOfAssetsOfComments = catSyncStatus.getSyncStatusOfAssetsOfComments,
			getSyncStatusOfAssetsOfFavorites = catSyncStatus.getSyncStatusOfAssetsOfFavorites,

		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local rootDocId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc that has not been synced to Oz
	local docValue = {
		_id = rootDocId,
		_newRevId = revisionId,
		type = 'asset',
		subtype = 'image',
		importSource = {
			filePath = '/path/',
			originalHeight = 150,
			originalWidth = 100,
		}
	}

	docValue = makeAssetDocDisplayable( docValue )

	rxTest:waitFor( docStore:setDocValue( docValue ) )

	-- Delete the asset document

	local overlay = {
		_id = rootDocId,
		_deleted = true
	}

	rxTest:waitFor( docStore:applyOverlay( overlay ) )

	-- create the dependent doc

	local dependentDocId = KSIdGenerator.uuidInLowercaseHex()
	local dependentRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local dependentErrorDocId = KSIdGenerator.uuidInLowercaseHex()

	rxTest:waitFor(
		docStore:setDocValue {
			_id = dependentDocId,
			_newRevId = dependentRevisionId,
			asset = {
				id = rootDocId,
			},
			type = "version",
			subtype = "edit",
		}
	)

	-- set dependent error doc

	rxTest:waitFor(
		docStore:setDocValue {
			_id = dependentErrorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"resource not found",
					},
				},
				missing_resource_id = rootDocId,
				source = {
					action = "create",
					id = dependentDocId,
					revision_id = dependentRevisionId,
					type = "version",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc and dependent doc should be deleted

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, dependentErrorDocId ) )
		:expectDataWithinTime( 5, true )

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, dependentDocId ) )
		:expectDataWithinTime( 5, true )

end


--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResolveResourceNotFoundError_Album( rxTest )

	rxTest:setTimeLimit( 15 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, nil, indexDefSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
			getSyncStatusOfAssetsOfAlbumAssets = catSyncStatus.getSyncStatusOfAssetsOfAlbumAssets,
			getSyncStatusOfAlbumsOfAlbumAssets = catSyncStatus.getSyncStatusOfAlbumsOfAlbumAssets,
			getSyncStatusOfParentCollectionOfAlbums = catSyncStatus.getSyncStatusOfParentCollectionOfAlbums,
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local parentId = KSIdGenerator.uuidInLowercaseHex()

	-- create a parent 'collection_set' doc that has not been synced to Oz
	rxTest:waitFor(
		docStore:setDocValue {
			_id = parentId,
			name = "Collection Set",
			type = "album",
			subtype = "collection_set",
		}
	)

	local childId = KSIdGenerator.uuidInLowercaseHex()
	local childRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create the child 'album'
	rxTest:waitFor(
		docStore:setDocValue {
			_id = childId,
			_newRevId = childRevisionId,
			parent = {
				id = parentId,
			},
			name = "Sync me",
			type = "album",
			subtype = "collection",
		}
	)

	-- set child album error doc
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"resource not found",
					},
				},
				missing_resource_id = parentId,
				source = {
					action = "create",
					id = childId,
					revision_id = childRevisionId,
					type = "collection_set",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 5, true )

	rxTest:expectSingleValue(
		docStore:docValue( childId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end ), true )

	-- Make the parent collection_set appear synced.
	rxTest:waitFor(
		docStore:applyOverlay {
			_id = parentId,
			_serverProvidedMetadata = {
				links = {},
				revision_ids = {}
			}
		}
	)

	-- Once the collection_set is synced, the resource not found error should
	-- be removed from the child album.

	rxTest:observe( docStore:ongoingDocValue( childId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end )
		)
		:expectDataWithinTime( 15, false )
		:dispose()

end

--------------------------------------------------------------------------------

local function testResolveResourceNotFoundError_stackCoverMissing( rxTest, hasStackAsset )

	rxTest:setTimeLimit( 15 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, nil, indexDefSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
			getSyncStatusOfAssetsOfAlbumAssets = catSyncStatus.getSyncStatusOfAssetsOfAlbumAssets,
			getSyncStatusOfAlbumsOfAlbumAssets = catSyncStatus.getSyncStatusOfAlbumsOfAlbumAssets,
			getSyncStatusOfParentCollectionOfAlbums = catSyncStatus.getSyncStatusOfParentCollectionOfAlbums,
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local rootDocId = KSIdGenerator.uuidInLowercaseHex()
	local rootRevisionId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc that has not been synced to Oz
	local docValue = {
		_id = rootDocId,
		_newRevId = rootRevisionId,
		type = 'asset',
		subtype = 'image',
		captureDate = "2004-11-02T11:10:09",
		importSource = {
			filePath = '/path/',
			originalHeight = 150,
			originalWidth = 100,
		}
	}

	docValue = makeAssetDocDisplayable( docValue )

	rxTest:waitFor( docStore:setDocValue( docValue ) )


	local stackDocId = KSIdGenerator.uuidInLowercaseHex()
	local stackRevisionId = KSIdGenerator.uuidInLowercaseHex()

	-- scenario: different client deleted the cover asset id while this client created the stack using it while paused
	local missingCoverDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create the asset for the stack doc
	rxTest:waitFor(
		docStore:setDocValue {
			_id = stackDocId,
			_newRevId = stackRevisionId,
			type = "asset",
			subtype = "stack",
			cover = {
				id = missingCoverDocId,
			},
			userCreated = "2011-11-11T11:11:11.111111Z",
			userUpdated = "2011-11-11T11:11:11.111111Z",
		}
	)

	if hasStackAsset then

		local derivedDocId = KSIdGenerator.uuidInLowercaseHex()
		local derivedRevisionId = KSIdGenerator.uuidInLowercaseHex()

		-- create the stack_asset doc
		rxTest:waitFor(
			docStore:setDocValue {
				_id = derivedDocId,
				_newRevId = derivedRevisionId,
				stack = {
					id = stackDocId,
				},
				asset = {
					id = rootDocId,
				},
				type = "stack_asset",
			}
		)

	end

	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- double check stack doc has a cover id (which is invalid)
	rxTest:expectSingleValue(
		docStore:docValue( stackDocId ):pluck( 'cover', 'id' ):map( type ),
		'string'
	)

	-- create the error doc related to stack cover missing
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					cover = {
						"does not exist",
					},
				},
				missing_resource_id = missingCoverDocId,
				source = {
					action = "create",
					id = stackDocId,
					revision_id = stackRevisionId,
					type = "asset",
				},
				status = 404,
				updated = "2024-10-14T21:27:12.863673Z",

				-- The fields below may only be present if the cover photo
				-- was moved to and is still in the trash,.
				deleted_by = "72f774baf53e28980b084ca83fbb9af5",
				new_id = "6d2a8f49ad45c771a23adc3165311e44",
			},
		}
	)

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 5, true )
		:dispose()

	-- Stacks are not implemented on other platforms error will be processed but ignored
	if WFPlatformInfo.isDesktop then

		if hasStackAsset then

			-- Verify the invalid cover id has been replaced with the existing
			-- stack_asset's assetId (rootDocId).
			rxTest:observe( docStore:ongoingDocValue( stackDocId )
					:pluck( 'cover', 'id' )
				)
				:expectDataWithinTime( 15, rootDocId )
				:dispose()

		else

			-- Verify the stack has been removed as there are not stack_assets left for the cover.
			rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, stackDocId ) )
				:expectDataWithinTime( 3, true )
				:dispose()

		end

	end

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResolveResourceNotFoundError_stackCoverMissing_hasStackAsset( rxTest )

	testResolveResourceNotFoundError_stackCoverMissing( rxTest, true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResolveResourceNotFoundError_stackCoverMissing_noStackAsset( rxTest )

	testResolveResourceNotFoundError_stackCoverMissing( rxTest, false )

end

--------------------------------------------------------------------------------

-- @mark InlineXMPDevelopSettingTooBigError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestInlineXmpDevelopSettingTooBigError( rxTest )

	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local sha256 = KSIdGenerator.uuidInLowercaseHex() .. KSIdGenerator.uuidInLowercaseHex()
	local errorHandler = WFCatErrorHandler( catalog, {
			externalizeSettings = makeExternalizeSettings( sha256 ),
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			_newRevId = revisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			},
			develop = {
				xmpCameraRaw = '<xmpmeta />',
			},
		}
	)

	-- create error doc for asset
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "InlineXmpDevelopSettingTooBigError",
			type = "error",
			_serverProvidedMetadata = {
				limit = 10240,
				errors = {
					xmpCameraRaw = {
						"is too big",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = revisionId,
					type = "asset",
				},
				status = 400,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	-- asset doc should end up with develop settings as sha256
	rxTest:expectSingleValue(
		docStore:docValue( docId )
			:map( function( docValue )
				return ( KSTableUtils.derefOpt( docValue, 'develop', 'xmpCameraRaw', 'sha256' ) == sha256 )
			end ), true )

end

--------------------------------------------------------------------------------


function WFCatErrorHandlerTest.rxTestInlineXmpDevelopSettingTooBigError_Version( rxTest )

	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local sha256 = KSIdGenerator.uuidInLowercaseHex() .. KSIdGenerator.uuidInLowercaseHex()
	local errorHandler = WFCatErrorHandler( catalog, {
		externalizeSettings = makeExternalizeSettings( sha256 ),
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'version' doc
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			_newRevId = revisionId,
			type = 'version',
			subtype = 'edit',
			develop = {
				xmpCameraRaw = '<xmpmeta />',
			},
		}
	)

	-- create error doc for version
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "InlineXmpDevelopSettingTooBigError",
			type = "error",
			_serverProvidedMetadata = {
				limit = 10240,
				errors = {
					xmpCameraRaw = {
						"is too big",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = revisionId,
					type = "version",
				},
				status = 400,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	-- version doc should end up with develop settings as sha256
	rxTest:expectSingleValue(
		docStore:docValue( docId )
			:map( function( docValue )
				return ( KSTableUtils.derefOpt( docValue, 'develop', 'xmpCameraRaw', 'sha256' ) == sha256 )
			end ),
		true
	)

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestInlineXmpDevelopSettingTooBigError_testNightly( rxTest )

	rxTest:setTimeLimit( 60 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog

	rxTest:waitFor(
		catalogTestSession:catalog()
			:tap( function( _catalog )
				catalog = _catalog
			end )
	)

	local docStore = catalog:docStore()

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )

	local sha256 = KSIdGenerator.uuidInLowercaseHex() .. KSIdGenerator.uuidInLowercaseHex()
	local errorHandler = WFCatErrorHandler( catalog, {
		externalizeSettings = makeExternalizeSettings( sha256 ),
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
		},
		errorHandlerOptions )

	rxTest:finally( function()
		errorHandler:close()
		puller:close()
		pusher:close()

	end )

	puller:getUpdatesContinuously()
	rxTest:waitFor( pusher:sendUpdatesContinuously() )

	local assetId = KSIdGenerator.uuidInLowercaseHex()
	rxTest:waitFor(
		docStore:setDocValue {
			_id = assetId,
			type = 'asset',
			subtype = 'image',
			captureDate = "0000-00-00T00:00:00",
			importSource = {
				importTimestamp = '2017-03-14T10:50:59.000Z', -- seconds has been adjusted
				filePath = '/path/',
				fileName = 'fileName.jpg',
				importedBy = KSIdGenerator.uuidInLowercaseHex(),
				originalHeight = 150,
				originalWidth = 100,
			},
			ratings = {
				[ KSIdGenerator.uuidInLowercaseHex() ] = {
					rating = 5,
				}
			},
			develop = {
				processingModel = 'lightroom',
				userUpdated = "2017-02-22T16:29:42.616Z",
				-- this is 11KB of settings
				xmpCameraRaw = [[<x:xmpmeta xmlns:x="adobe:ns:meta/" x:xmptk="Adobe XMP Core 5.6-c128 79.159124, 2016/03/18-14:01:55        ">
										 <rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
										  <rdf:Description rdf:about=""
										    xmlns:crs="http://ns.adobe.com/camera-raw-settings/1.0/"
										   crs:Version="9.9"
										   crs:ProcessVersion="5.7"
										   crs:WhiteBalance="Custom"
										   crs:AutoWhiteVersion="134348800"
										   crs:Temperature="3250"
										   crs:Tint="-31"
										   crs:Exposure="+0.25"
										   crs:Shadows="5"
										   crs:Brightness="+50"
										   crs:Contrast="+25"
										   crs:Saturation="0"
										   crs:Sharpness="25"
										   crs:LuminanceSmoothing="0"
										   crs:ColorNoiseReduction="25"
										   crs:VignetteAmount="0"
										   crs:ShadowTint="0"
										   crs:RedHue="0"
										   crs:RedSaturation="0"
										   crs:GreenHue="0"
										   crs:GreenSaturation="0"
										   crs:BlueHue="0"
										   crs:BlueSaturation="0"
										   crs:FillLight="0"
										   crs:Vibrance="0"
										   crs:HighlightRecovery="0"
										   crs:Clarity="0"
										   crs:HueAdjustmentRed="0"
										   crs:HueAdjustmentOrange="0"
										   crs:HueAdjustmentYellow="0"
										   crs:HueAdjustmentGreen="0"
										   crs:HueAdjustmentAqua="0"
										   crs:HueAdjustmentBlue="0"
										   crs:HueAdjustmentPurple="0"
										   crs:HueAdjustmentMagenta="0"
										   crs:SaturationAdjustmentRed="0"
										   crs:SaturationAdjustmentOrange="0"
										   crs:SaturationAdjustmentYellow="0"
										   crs:SaturationAdjustmentGreen="0"
										   crs:SaturationAdjustmentAqua="0"
										   crs:SaturationAdjustmentBlue="0"
										   crs:SaturationAdjustmentPurple="0"
										   crs:SaturationAdjustmentMagenta="0"
										   crs:LuminanceAdjustmentRed="0"
										   crs:LuminanceAdjustmentOrange="0"
										   crs:LuminanceAdjustmentYellow="0"
										   crs:LuminanceAdjustmentGreen="0"
										   crs:LuminanceAdjustmentAqua="0"
										   crs:LuminanceAdjustmentBlue="0"
										   crs:LuminanceAdjustmentPurple="0"
										   crs:LuminanceAdjustmentMagenta="0"
										   crs:SplitToningShadowHue="0"
										   crs:SplitToningShadowSaturation="0"
										   crs:SplitToningHighlightHue="0"
										   crs:SplitToningHighlightSaturation="0"
										   crs:SplitToningBalance="0"
										   crs:ParametricShadows="0"
										   crs:ParametricDarks="0"
										   crs:ParametricLights="0"
										   crs:ParametricHighlights="0"
										   crs:ParametricShadowSplit="25"
										   crs:ParametricMidtoneSplit="50"
										   crs:ParametricHighlightSplit="75"
										   crs:SharpenRadius="+1.0"
										   crs:SharpenDetail="25"
										   crs:SharpenEdgeMasking="0"
										   crs:PostCropVignetteAmount="0"
										   crs:GrainAmount="0"
										   crs:ColorNoiseReductionDetail="50"
										   crs:ColorNoiseReductionSmoothness="50"
										   crs:LensProfileEnable="0"
										   crs:LensManualDistortionAmount="0"
										   crs:PerspectiveVertical="0"
										   crs:PerspectiveHorizontal="0"
										   crs:PerspectiveRotate="0.0"
										   crs:PerspectiveScale="100"
										   crs:PerspectiveAspect="0"
										   crs:PerspectiveUpright="0"
										   crs:PerspectiveX="0.00"
										   crs:PerspectiveY="0.00"
										   crs:AutoLateralCA="0"
										   crs:DefringePurpleAmount="0"
										   crs:DefringePurpleHueLo="30"
										   crs:DefringePurpleHueHi="70"
										   crs:DefringeGreenAmount="0"
										   crs:DefringeGreenHueLo="40"
										   crs:DefringeGreenHueHi="60"
										   crs:ConvertToGrayscale="False"
										   crs:ToneCurveName="Medium Contrast"
										   crs:ToneCurveName2012="Linear"
										   crs:CameraProfile="Adobe Standard"
										   crs:CameraProfileDigest="730B5111DF7449A8EE0790633B8D3861"
										   crs:LensProfileSetup="LensDefaults"
										   crs:UprightVersion="151388160"
										   crs:UprightCenterMode="0"
										   crs:UprightCenterNormX="0.5"
										   crs:UprightCenterNormY="0.5"
										   crs:UprightFocalMode="0"
										   crs:UprightFocalLength35mm="35"
										   crs:UprightPreview="False"
										   crs:UprightTransformCount="6"
										   crs:UprightFourSegmentsCount="0"
										   crs:HasSettings="True"
										   crs:AlreadyApplied="False">
										   <crs:ToneCurve>
										    <rdf:Seq>
										     <rdf:li>0, 0</rdf:li>
										     <rdf:li>32, 22</rdf:li>
										     <rdf:li>64, 56</rdf:li>
										     <rdf:li>128, 128</rdf:li>
										     <rdf:li>192, 196</rdf:li>
										     <rdf:li>255, 255</rdf:li>
										    </rdf:Seq>
										   </crs:ToneCurve>
										   <crs:ToneCurveRed>
										    <rdf:Seq>
										     <rdf:li>0, 0</rdf:li>
										     <rdf:li>255, 255</rdf:li>
										    </rdf:Seq>
										   </crs:ToneCurveRed>
										   <crs:ToneCurveGreen>
										    <rdf:Seq>
										     <rdf:li>0, 0</rdf:li>
										     <rdf:li>255, 255</rdf:li>
										    </rdf:Seq>
										   </crs:ToneCurveGreen>
										   <crs:ToneCurveBlue>
										    <rdf:Seq>
										     <rdf:li>0, 0</rdf:li>
										     <rdf:li>255, 255</rdf:li>
										    </rdf:Seq>
										   </crs:ToneCurveBlue>
										   <crs:ToneCurvePV2012>
										    <rdf:Seq>
										     <rdf:li>0, 0</rdf:li>
										     <rdf:li>255, 255</rdf:li>
										    </rdf:Seq>
										   </crs:ToneCurvePV2012>
										   <crs:ToneCurvePV2012Red>
										    <rdf:Seq>
										     <rdf:li>0, 0</rdf:li>
										     <rdf:li>255, 255</rdf:li>
										    </rdf:Seq>
										   </crs:ToneCurvePV2012Red>
										   <crs:ToneCurvePV2012Green>
										    <rdf:Seq>
										     <rdf:li>0, 0</rdf:li>
										     <rdf:li>255, 255</rdf:li>
										    </rdf:Seq>
										   </crs:ToneCurvePV2012Green>
										   <crs:ToneCurvePV2012Blue>
										    <rdf:Seq>
										     <rdf:li>0, 0</rdf:li>
										     <rdf:li>255, 255</rdf:li>
										    </rdf:Seq>
										   </crs:ToneCurvePV2012Blue>
										   <crs:PaintBasedCorrections>
										    <rdf:Seq>
										     <rdf:li>
										      <rdf:Description
										       crs:What="Correction"
										       crs:CorrectionAmount="1.000000"
										       crs:CorrectionActive="true"
										       crs:LocalExposure="0.552941"
										       crs:LocalSaturation="0.000000"
										       crs:LocalContrast="0.000000"
										       crs:LocalClarity="0.000000"
										       crs:LocalSharpness="0.000000"
										       crs:LocalBrightness="0.000000"
										       crs:LocalToningHue="0.000000"
										       crs:LocalToningSaturation="0.000000">
										      <crs:CorrectionMasks>
										       <rdf:Seq>
										        <rdf:li>
										         <rdf:Description
										          crs:What="Mask/Paint"
										          crs:MaskValue="0.597701"
										          crs:Radius="0.294229"
										          crs:Flow="0.374138"
										          crs:CenterWeight="0.336813">
										         <crs:Dabs>
										          <rdf:Seq>
										           <rdf:li>d 0.583691 0.603004</rdf:li>
										           <rdf:li>d 0.506847 0.542476</rdf:li>
										           <rdf:li>d 0.445992 0.503752</rdf:li>
										           <rdf:li>d 0.510425 0.594056</rdf:li>
										           <rdf:li>d 0.589165 0.649437</rdf:li>
										           <rdf:li>d 0.590581 0.680258</rdf:li>
										           <rdf:li>d 0.503070 0.664129</rdf:li>
										           <rdf:li>d 0.417757 0.631625</rdf:li>
										           <rdf:li>d 0.345171 0.564655</rdf:li>
										           <rdf:li>d 0.366658 0.567833</rdf:li>
										           <rdf:li>d 0.436027 0.649538</rdf:li>
										           <rdf:li>d 0.501659 0.737556</rdf:li>
										           <rdf:li>d 0.571560 0.818406</rdf:li>
										           <rdf:li>d 0.612505 0.805913</rdf:li>
										           <rdf:li>d 0.543371 0.723593</rdf:li>
										           <rdf:li>d 0.475483 0.639351</rdf:li>
										           <rdf:li>d 0.451223 0.540773</rdf:li>
										           <rdf:li>d 0.525914 0.601557</rdf:li>
										           <rdf:li>d 0.544495 0.599863</rdf:li>
										           <rdf:li>d 0.490855 0.494746</rdf:li>
										           <rdf:li>d 0.427792 0.403438</rdf:li>
										           <rdf:li>d 0.358655 0.321123</rdf:li>
										           <rdf:li>d 0.289519 0.238807</rdf:li>
										           <rdf:li>d 0.220372 0.156511</rdf:li>
										           <rdf:li>d 0.151199 0.074263</rdf:li>
										           <rdf:li>d 0.079237 -0.002181</rdf:li>
										           <rdf:li>d 0.005210 -0.074281</rdf:li>
										           <rdf:li>d -0.070480 -0.142402</rdf:li>
										           <rdf:li>d -0.102730 -0.153072</rdf:li>
										           <rdf:li>d -0.038006 -0.063046</rdf:li>
										           <rdf:li>d 0.026719 0.026980</rdf:li>
										           <rdf:li>d 0.091444 0.117007</rdf:li>
										           <rdf:li>d 0.156168 0.207033</rdf:li>
										           <rdf:li>d 0.213520 0.287557</rdf:li>
										           <rdf:li>d 0.249205 0.406728</rdf:li>
										           <rdf:li>d 0.309851 0.502384</rdf:li>
										           <rdf:li>d 0.386433 0.567872</rdf:li>
										           <rdf:li>d 0.466134 0.623738</rdf:li>
										           <rdf:li>d 0.434471 0.647575</rdf:li>
										           <rdf:li>d 0.352438 0.611033</rdf:li>
										           <rdf:li>d 0.294644 0.564284</rdf:li>
										           <rdf:li>d 0.352680 0.663384</rdf:li>
										           <rdf:li>d 0.402755 0.772418</rdf:li>
										           <rdf:li>d 0.464043 0.866000</rdf:li>
										           <rdf:li>d 0.550064 0.883831</rdf:li>
										           <rdf:li>d 0.638275 0.879062</rdf:li>
										           <rdf:li>d 0.622170 0.812559</rdf:li>
										           <rdf:li>d 0.542796 0.755269</rdf:li>
										           <rdf:li>d 0.457879 0.722551</rdf:li>
										           <rdf:li>d 0.370080 0.708936</rdf:li>
										           <rdf:li>d 0.282253 0.695747</rdf:li>
										           <rdf:li>d 0.194528 0.682390</rdf:li>
										           <rdf:li>d 0.251015 0.681170</rdf:li>
										           <rdf:li>d 0.339248 0.684909</rdf:li>
										           <rdf:li>d 0.427482 0.688648</rdf:li>
										           <rdf:li>d 0.509635 0.735799</rdf:li>
										           <rdf:li>d 0.591826 0.783998</rdf:li>
										           <rdf:li>d 0.664021 0.855778</rdf:li>
										           <rdf:li>d 0.665281 0.832685</rdf:li>
										           <rdf:li>d 0.601375 0.741388</rdf:li>
										           <rdf:li>d 0.525838 0.674864</rdf:li>
										           <rdf:li>d 0.446731 0.616125</rdf:li>
										           <rdf:li>d 0.367624 0.557387</rdf:li>
										           <rdf:li>d 0.288517 0.498648</rdf:li>
										           <rdf:li>d 0.209410 0.439909</rdf:li>
										           <rdf:li>d 0.127992 0.391444</rdf:li>
										           <rdf:li>d 0.040708 0.372114</rdf:li>
										           <rdf:li>d 0.072242 0.419354</rdf:li>
										           <rdf:li>d 0.148677 0.485448</rdf:li>
										           <rdf:li>d 0.225726 0.550039</rdf:li>
										           <rdf:li>d 0.303516 0.612609</rdf:li>
										           <rdf:li>d 0.381858 0.672870</rdf:li>
										           <rdf:li>d 0.456527 0.743481</rdf:li>
										           <rdf:li>d 0.531195 0.814091</rdf:li>
										           <rdf:li>d 0.608030 0.832618</rdf:li>
										           <rdf:li>d 0.601863 0.768196</rdf:li>
										          </rdf:Seq>
										         </crs:Dabs>
										         </rdf:Description>
										        </rdf:li>
										        <rdf:li>
										         <rdf:Description
										          crs:What="Mask/Paint"
										          crs:MaskValue="0.597701"
										          crs:Radius="0.294229"
										          crs:Flow="0.374138"
										          crs:CenterWeight="0.336813">
										         <crs:Dabs>
										          <rdf:Seq>
										           <rdf:li>d 0.806867 0.881974</rdf:li>
										           <rdf:li>d 0.813094 0.752712</rdf:li>
										           <rdf:li>d 0.798072 0.622405</rdf:li>
										           <rdf:li>d 0.784627 0.492781</rdf:li>
										           <rdf:li>d 0.725046 0.398217</rdf:li>
										           <rdf:li>d 0.659724 0.309852</rdf:li>
										           <rdf:li>d 0.586062 0.238182</rdf:li>
										           <rdf:li>d 0.508032 0.178908</rdf:li>
										           <rdf:li>d 0.426434 0.132882</rdf:li>
										          </rdf:Seq>
										         </crs:Dabs>
										         </rdf:Description>
										        </rdf:li>
										        <rdf:li>
										         <rdf:Description
										          crs:What="Mask/Paint"
										          crs:MaskValue="0.597701"
										          crs:Radius="0.294229"
										          crs:Flow="0.374138"
										          crs:CenterWeight="0.336813">
										         <crs:Dabs>
										          <rdf:Seq>
										           <rdf:li>d 0.868383 0.993562</rdf:li>
										           <rdf:li>d 0.857835 0.862179</rdf:li>
										           <rdf:li>d 0.838056 0.733487</rdf:li>
										           <rdf:li>d 0.802776 0.612341</rdf:li>
										           <rdf:li>d 0.768633 0.490447</rdf:li>
										           <rdf:li>d 0.727548 0.373261</rdf:li>
										           <rdf:li>d 0.689283 0.254060</rdf:li>
										           <rdf:li>d 0.624340 0.167826</rdf:li>
										           <rdf:li>d 0.544827 0.110515</rdf:li>
										           <rdf:li>d 0.462461 0.063275</rdf:li>
										          </rdf:Seq>
										         </crs:Dabs>
										         </rdf:Description>
										        </rdf:li>
										       </rdf:Seq>
										      </crs:CorrectionMasks>
										      </rdf:Description>
										     </rdf:li>
										    </rdf:Seq>
										   </crs:PaintBasedCorrections>
										  </rdf:Description>
										 </rdf:RDF>
										</x:xmpmeta>
										]],
			},
		}
	)


	-- asset doc should end up with develop settings as sha256
	local assetSha256 = rxTest:observe(
							docStore:ongoingDocValue( assetId )
								:map( function( docValue )
									return ( KSTableUtils.derefOpt( docValue, 'develop', 'xmpCameraRaw', 'sha256' ) == sha256 )
								end ) )

	assetSha256:expectDataWithinTime( 40, true )

	local assetOnOz = rxTest:observe(
							docStore:ongoingDocValue( assetId )
								:map( function( docValue )
									return docValue._serverProvidedMetadata ~= nil
								end ) )

	assetOnOz:expectDataWithinTime( 40, true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestInlineXmpDevelopSettingTooBigError_Version_testNightly( rxTest )

	rxTest:setTimeLimit( 60 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog

	rxTest:waitFor(
		catalogTestSession:catalog()
			:tap( function( _catalog )
				catalog = _catalog
			end )
	)

	local docStore = catalog:docStore()

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )

	local sha256 = KSIdGenerator.uuidInLowercaseHex() .. KSIdGenerator.uuidInLowercaseHex()
	local errorHandler = WFCatErrorHandler( catalog, {
		externalizeSettings = makeExternalizeSettings( sha256 ),
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
		},
		errorHandlerOptions )

	rxTest:finally( function()
		errorHandler:close()
		puller:close()
		pusher:close()

	end )

	puller:getUpdatesContinuously()
	rxTest:waitFor( pusher:sendUpdatesContinuously() )

	local originalSha256 = KSIdGenerator.uuidInLowercaseHex() .. KSIdGenerator.uuidInLowercaseHex()
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	rxTest:waitFor(
		docStore:setDocValue {
			_id = assetId,
			type = 'asset',
			subtype = 'image',
			captureDate = "0000-00-00T00:00:00",
			importSource = {
				importTimestamp = '2017-03-14T10:50:59.000Z', -- seconds has been adjusted
				filePath = '/path/',
				fileName = 'fileName.jpg',
				importedBy = KSIdGenerator.uuidInLowercaseHex(),
				sha256 = originalSha256,
				originalHeight = 150,
				originalWidth = 100,
			},
			ratings = {
				[ KSIdGenerator.uuidInLowercaseHex() ] = {
					rating = 5,
				}
			},
			develop = {
				processingModel = 'lightroom',
				userUpdated = "2017-02-22T16:29:42.616Z",
				xmpCameraRaw = "<xmp/>",
			},
		}
	)

	local versionId = KSIdGenerator.uuidInLowercaseHex()
	rxTest:waitFor(
		docStore:setDocValue {
			_id = versionId,
			type = 'version',
			subtype = 'edit',
			asset = {
				id = assetId,
				sha256 = originalSha256,
			},
			userCreated = KSDate.timeInIso8601Format(),
			userUpdated = KSDate.timeInIso8601Format(),
			develop = {
				processingModel = 'lightroom',
				userUpdated = "2017-02-22T16:29:42.616Z",
				-- this is 11KB of settings
				xmpCameraRaw = [[<x:xmpmeta xmlns:x="adobe:ns:meta/" x:xmptk="Adobe XMP Core 5.6-c128 79.159124, 2016/03/18-14:01:55        ">
									 <rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
									  <rdf:Description rdf:about=""
									    xmlns:crs="http://ns.adobe.com/camera-raw-settings/1.0/"
									   crs:Version="9.9"
									   crs:ProcessVersion="5.7"
									   crs:WhiteBalance="Custom"
									   crs:AutoWhiteVersion="134348800"
									   crs:Temperature="3250"
									   crs:Tint="-31"
									   crs:Exposure="+0.25"
									   crs:Shadows="5"
									   crs:Brightness="+50"
									   crs:Contrast="+25"
									   crs:Saturation="0"
									   crs:Sharpness="25"
									   crs:LuminanceSmoothing="0"
									   crs:ColorNoiseReduction="25"
									   crs:VignetteAmount="0"
									   crs:ShadowTint="0"
									   crs:RedHue="0"
									   crs:RedSaturation="0"
									   crs:GreenHue="0"
									   crs:GreenSaturation="0"
									   crs:BlueHue="0"
									   crs:BlueSaturation="0"
									   crs:FillLight="0"
									   crs:Vibrance="0"
									   crs:HighlightRecovery="0"
									   crs:Clarity="0"
									   crs:HueAdjustmentRed="0"
									   crs:HueAdjustmentOrange="0"
									   crs:HueAdjustmentYellow="0"
									   crs:HueAdjustmentGreen="0"
									   crs:HueAdjustmentAqua="0"
									   crs:HueAdjustmentBlue="0"
									   crs:HueAdjustmentPurple="0"
									   crs:HueAdjustmentMagenta="0"
									   crs:SaturationAdjustmentRed="0"
									   crs:SaturationAdjustmentOrange="0"
									   crs:SaturationAdjustmentYellow="0"
									   crs:SaturationAdjustmentGreen="0"
									   crs:SaturationAdjustmentAqua="0"
									   crs:SaturationAdjustmentBlue="0"
									   crs:SaturationAdjustmentPurple="0"
									   crs:SaturationAdjustmentMagenta="0"
									   crs:LuminanceAdjustmentRed="0"
									   crs:LuminanceAdjustmentOrange="0"
									   crs:LuminanceAdjustmentYellow="0"
									   crs:LuminanceAdjustmentGreen="0"
									   crs:LuminanceAdjustmentAqua="0"
									   crs:LuminanceAdjustmentBlue="0"
									   crs:LuminanceAdjustmentPurple="0"
									   crs:LuminanceAdjustmentMagenta="0"
									   crs:SplitToningShadowHue="0"
									   crs:SplitToningShadowSaturation="0"
									   crs:SplitToningHighlightHue="0"
									   crs:SplitToningHighlightSaturation="0"
									   crs:SplitToningBalance="0"
									   crs:ParametricShadows="0"
									   crs:ParametricDarks="0"
									   crs:ParametricLights="0"
									   crs:ParametricHighlights="0"
									   crs:ParametricShadowSplit="25"
									   crs:ParametricMidtoneSplit="50"
									   crs:ParametricHighlightSplit="75"
									   crs:SharpenRadius="+1.0"
									   crs:SharpenDetail="25"
									   crs:SharpenEdgeMasking="0"
									   crs:PostCropVignetteAmount="0"
									   crs:GrainAmount="0"
									   crs:ColorNoiseReductionDetail="50"
									   crs:ColorNoiseReductionSmoothness="50"
									   crs:LensProfileEnable="0"
									   crs:LensManualDistortionAmount="0"
									   crs:PerspectiveVertical="0"
									   crs:PerspectiveHorizontal="0"
									   crs:PerspectiveRotate="0.0"
									   crs:PerspectiveScale="100"
									   crs:PerspectiveAspect="0"
									   crs:PerspectiveUpright="0"
									   crs:PerspectiveX="0.00"
									   crs:PerspectiveY="0.00"
									   crs:AutoLateralCA="0"
									   crs:DefringePurpleAmount="0"
									   crs:DefringePurpleHueLo="30"
									   crs:DefringePurpleHueHi="70"
									   crs:DefringeGreenAmount="0"
									   crs:DefringeGreenHueLo="40"
									   crs:DefringeGreenHueHi="60"
									   crs:ConvertToGrayscale="False"
									   crs:ToneCurveName="Medium Contrast"
									   crs:ToneCurveName2012="Linear"
									   crs:CameraProfile="Adobe Standard"
									   crs:CameraProfileDigest="730B5111DF7449A8EE0790633B8D3861"
									   crs:LensProfileSetup="LensDefaults"
									   crs:UprightVersion="151388160"
									   crs:UprightCenterMode="0"
									   crs:UprightCenterNormX="0.5"
									   crs:UprightCenterNormY="0.5"
									   crs:UprightFocalMode="0"
									   crs:UprightFocalLength35mm="35"
									   crs:UprightPreview="False"
									   crs:UprightTransformCount="6"
									   crs:UprightFourSegmentsCount="0"
									   crs:HasSettings="True"
									   crs:AlreadyApplied="False">
									   <crs:ToneCurve>
									    <rdf:Seq>
									     <rdf:li>0, 0</rdf:li>
									     <rdf:li>32, 22</rdf:li>
									     <rdf:li>64, 56</rdf:li>
									     <rdf:li>128, 128</rdf:li>
									     <rdf:li>192, 196</rdf:li>
									     <rdf:li>255, 255</rdf:li>
									    </rdf:Seq>
									   </crs:ToneCurve>
									   <crs:ToneCurveRed>
									    <rdf:Seq>
									     <rdf:li>0, 0</rdf:li>
									     <rdf:li>255, 255</rdf:li>
									    </rdf:Seq>
									   </crs:ToneCurveRed>
									   <crs:ToneCurveGreen>
									    <rdf:Seq>
									     <rdf:li>0, 0</rdf:li>
									     <rdf:li>255, 255</rdf:li>
									    </rdf:Seq>
									   </crs:ToneCurveGreen>
									   <crs:ToneCurveBlue>
									    <rdf:Seq>
									     <rdf:li>0, 0</rdf:li>
									     <rdf:li>255, 255</rdf:li>
									    </rdf:Seq>
									   </crs:ToneCurveBlue>
									   <crs:ToneCurvePV2012>
									    <rdf:Seq>
									     <rdf:li>0, 0</rdf:li>
									     <rdf:li>255, 255</rdf:li>
									    </rdf:Seq>
									   </crs:ToneCurvePV2012>
									   <crs:ToneCurvePV2012Red>
									    <rdf:Seq>
									     <rdf:li>0, 0</rdf:li>
									     <rdf:li>255, 255</rdf:li>
									    </rdf:Seq>
									   </crs:ToneCurvePV2012Red>
									   <crs:ToneCurvePV2012Green>
									    <rdf:Seq>
									     <rdf:li>0, 0</rdf:li>
									     <rdf:li>255, 255</rdf:li>
									    </rdf:Seq>
									   </crs:ToneCurvePV2012Green>
									   <crs:ToneCurvePV2012Blue>
									    <rdf:Seq>
									     <rdf:li>0, 0</rdf:li>
									     <rdf:li>255, 255</rdf:li>
									    </rdf:Seq>
									   </crs:ToneCurvePV2012Blue>
									   <crs:PaintBasedCorrections>
									    <rdf:Seq>
									     <rdf:li>
									      <rdf:Description
									       crs:What="Correction"
									       crs:CorrectionAmount="1.000000"
									       crs:CorrectionActive="true"
									       crs:LocalExposure="0.552941"
									       crs:LocalSaturation="0.000000"
									       crs:LocalContrast="0.000000"
									       crs:LocalClarity="0.000000"
									       crs:LocalSharpness="0.000000"
									       crs:LocalBrightness="0.000000"
									       crs:LocalToningHue="0.000000"
									       crs:LocalToningSaturation="0.000000">
									      <crs:CorrectionMasks>
									       <rdf:Seq>
									        <rdf:li>
									         <rdf:Description
									          crs:What="Mask/Paint"
									          crs:MaskValue="0.597701"
									          crs:Radius="0.294229"
									          crs:Flow="0.374138"
									          crs:CenterWeight="0.336813">
									         <crs:Dabs>
									          <rdf:Seq>
									           <rdf:li>d 0.583691 0.603004</rdf:li>
									           <rdf:li>d 0.506847 0.542476</rdf:li>
									           <rdf:li>d 0.445992 0.503752</rdf:li>
									           <rdf:li>d 0.510425 0.594056</rdf:li>
									           <rdf:li>d 0.589165 0.649437</rdf:li>
									           <rdf:li>d 0.590581 0.680258</rdf:li>
									           <rdf:li>d 0.503070 0.664129</rdf:li>
									           <rdf:li>d 0.417757 0.631625</rdf:li>
									           <rdf:li>d 0.345171 0.564655</rdf:li>
									           <rdf:li>d 0.366658 0.567833</rdf:li>
									           <rdf:li>d 0.436027 0.649538</rdf:li>
									           <rdf:li>d 0.501659 0.737556</rdf:li>
									           <rdf:li>d 0.571560 0.818406</rdf:li>
									           <rdf:li>d 0.612505 0.805913</rdf:li>
									           <rdf:li>d 0.543371 0.723593</rdf:li>
									           <rdf:li>d 0.475483 0.639351</rdf:li>
									           <rdf:li>d 0.451223 0.540773</rdf:li>
									           <rdf:li>d 0.525914 0.601557</rdf:li>
									           <rdf:li>d 0.544495 0.599863</rdf:li>
									           <rdf:li>d 0.490855 0.494746</rdf:li>
									           <rdf:li>d 0.427792 0.403438</rdf:li>
									           <rdf:li>d 0.358655 0.321123</rdf:li>
									           <rdf:li>d 0.289519 0.238807</rdf:li>
									           <rdf:li>d 0.220372 0.156511</rdf:li>
									           <rdf:li>d 0.151199 0.074263</rdf:li>
									           <rdf:li>d 0.079237 -0.002181</rdf:li>
									           <rdf:li>d 0.005210 -0.074281</rdf:li>
									           <rdf:li>d -0.070480 -0.142402</rdf:li>
									           <rdf:li>d -0.102730 -0.153072</rdf:li>
									           <rdf:li>d -0.038006 -0.063046</rdf:li>
									           <rdf:li>d 0.026719 0.026980</rdf:li>
									           <rdf:li>d 0.091444 0.117007</rdf:li>
									           <rdf:li>d 0.156168 0.207033</rdf:li>
									           <rdf:li>d 0.213520 0.287557</rdf:li>
									           <rdf:li>d 0.249205 0.406728</rdf:li>
									           <rdf:li>d 0.309851 0.502384</rdf:li>
									           <rdf:li>d 0.386433 0.567872</rdf:li>
									           <rdf:li>d 0.466134 0.623738</rdf:li>
									           <rdf:li>d 0.434471 0.647575</rdf:li>
									           <rdf:li>d 0.352438 0.611033</rdf:li>
									           <rdf:li>d 0.294644 0.564284</rdf:li>
									           <rdf:li>d 0.352680 0.663384</rdf:li>
									           <rdf:li>d 0.402755 0.772418</rdf:li>
									           <rdf:li>d 0.464043 0.866000</rdf:li>
									           <rdf:li>d 0.550064 0.883831</rdf:li>
									           <rdf:li>d 0.638275 0.879062</rdf:li>
									           <rdf:li>d 0.622170 0.812559</rdf:li>
									           <rdf:li>d 0.542796 0.755269</rdf:li>
									           <rdf:li>d 0.457879 0.722551</rdf:li>
									           <rdf:li>d 0.370080 0.708936</rdf:li>
									           <rdf:li>d 0.282253 0.695747</rdf:li>
									           <rdf:li>d 0.194528 0.682390</rdf:li>
									           <rdf:li>d 0.251015 0.681170</rdf:li>
									           <rdf:li>d 0.339248 0.684909</rdf:li>
									           <rdf:li>d 0.427482 0.688648</rdf:li>
									           <rdf:li>d 0.509635 0.735799</rdf:li>
									           <rdf:li>d 0.591826 0.783998</rdf:li>
									           <rdf:li>d 0.664021 0.855778</rdf:li>
									           <rdf:li>d 0.665281 0.832685</rdf:li>
									           <rdf:li>d 0.601375 0.741388</rdf:li>
									           <rdf:li>d 0.525838 0.674864</rdf:li>
									           <rdf:li>d 0.446731 0.616125</rdf:li>
									           <rdf:li>d 0.367624 0.557387</rdf:li>
									           <rdf:li>d 0.288517 0.498648</rdf:li>
									           <rdf:li>d 0.209410 0.439909</rdf:li>
									           <rdf:li>d 0.127992 0.391444</rdf:li>
									           <rdf:li>d 0.040708 0.372114</rdf:li>
									           <rdf:li>d 0.072242 0.419354</rdf:li>
									           <rdf:li>d 0.148677 0.485448</rdf:li>
									           <rdf:li>d 0.225726 0.550039</rdf:li>
									           <rdf:li>d 0.303516 0.612609</rdf:li>
									           <rdf:li>d 0.381858 0.672870</rdf:li>
									           <rdf:li>d 0.456527 0.743481</rdf:li>
									           <rdf:li>d 0.531195 0.814091</rdf:li>
									           <rdf:li>d 0.608030 0.832618</rdf:li>
									           <rdf:li>d 0.601863 0.768196</rdf:li>
									          </rdf:Seq>
									         </crs:Dabs>
									         </rdf:Description>
									        </rdf:li>
									        <rdf:li>
									         <rdf:Description
									          crs:What="Mask/Paint"
									          crs:MaskValue="0.597701"
									          crs:Radius="0.294229"
									          crs:Flow="0.374138"
									          crs:CenterWeight="0.336813">
									         <crs:Dabs>
									          <rdf:Seq>
									           <rdf:li>d 0.806867 0.881974</rdf:li>
									           <rdf:li>d 0.813094 0.752712</rdf:li>
									           <rdf:li>d 0.798072 0.622405</rdf:li>
									           <rdf:li>d 0.784627 0.492781</rdf:li>
									           <rdf:li>d 0.725046 0.398217</rdf:li>
									           <rdf:li>d 0.659724 0.309852</rdf:li>
									           <rdf:li>d 0.586062 0.238182</rdf:li>
									           <rdf:li>d 0.508032 0.178908</rdf:li>
									           <rdf:li>d 0.426434 0.132882</rdf:li>
									          </rdf:Seq>
									         </crs:Dabs>
									         </rdf:Description>
									        </rdf:li>
									        <rdf:li>
									         <rdf:Description
									          crs:What="Mask/Paint"
									          crs:MaskValue="0.597701"
									          crs:Radius="0.294229"
									          crs:Flow="0.374138"
									          crs:CenterWeight="0.336813">
									         <crs:Dabs>
									          <rdf:Seq>
									           <rdf:li>d 0.868383 0.993562</rdf:li>
									           <rdf:li>d 0.857835 0.862179</rdf:li>
									           <rdf:li>d 0.838056 0.733487</rdf:li>
									           <rdf:li>d 0.802776 0.612341</rdf:li>
									           <rdf:li>d 0.768633 0.490447</rdf:li>
									           <rdf:li>d 0.727548 0.373261</rdf:li>
									           <rdf:li>d 0.689283 0.254060</rdf:li>
									           <rdf:li>d 0.624340 0.167826</rdf:li>
									           <rdf:li>d 0.544827 0.110515</rdf:li>
									           <rdf:li>d 0.462461 0.063275</rdf:li>
									          </rdf:Seq>
									         </crs:Dabs>
									         </rdf:Description>
									        </rdf:li>
									       </rdf:Seq>
									      </crs:CorrectionMasks>
									      </rdf:Description>
									     </rdf:li>
									    </rdf:Seq>
									   </crs:PaintBasedCorrections>
									  </rdf:Description>
									 </rdf:RDF>
									</x:xmpmeta>
									]],
			}
		}
	)

	-- version doc should end up with develop settings as sha256
	local versionSha256 = rxTest:observe(
							docStore:ongoingDocValue( versionId )
								:map( function( docValue )
									return ( KSTableUtils.derefOpt( docValue, 'develop', 'xmpCameraRaw', 'sha256' ) == sha256 )
								end ) )

	versionSha256:expectDataWithinTime( 40, true )

	local versionOnOz = rxTest:observe(
							docStore:ongoingDocValue( versionId )
								:map( function( docValue )
									return docValue._serverProvidedMetadata ~= nil
								end ) )

	versionOnOz:expectDataWithinTime( 40, true )

end

--------------------------------------------------------------------------------

-- @mark StorageExceededError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestStorageExceededError( rxTest )

	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			_newRevId = revisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		}
	)

	-- create error doc for asset
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "StorageExceededError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"quota exceeded",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = revisionId,
					type = "asset",
				},
				status = 403,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	-- asset doc should end up with quota exceeded annotation
	rxTest:expectSingleValue(
		docStore:docValue( docId )
			:map( function( docValue )
				return WFCatOzErrors.hasStorageExceededError( docValue )
			end ), true )

end

--------------------------------------------------------------------------------

local function testStorageExceededError_restoreModify( rxTest, options )

	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	local deletedImageId = KSIdGenerator.uuidInLowercaseHex()
	local deletedRevisionId = KSIdGenerator.uuidInLowercaseHex()

	local maybeModifiedRevId = revisionId
	if options.modifyAfterRestore then
		maybeModifiedRevId = KSIdGenerator.uuidInLowercaseHex()
	end

	-- create restored 'asset' doc
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			_newRevId = maybeModifiedRevId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			},
			_localOnly = {
				universal = {
					trash = {
						createFrom = {
							id = deletedImageId,
							lastAssetId = deletedImageId,
							lastRevisionId = deletedRevisionId,
							lastTrashPurgeDate = "2019-07-12T21:32:40.415Z",
							lastTrashUserCreated = "2019-05-13T21:32:40.415Z",
							revision_id = deletedRevisionId,
							subtype = "deleted_image",
						},
						createWithRevId = revisionId,
						uploaded = {
							["/rels/master"] = true,
							["/rels/proxy_type/2560"] = true,
							["/rels/xmp"] = true,
						},
					},
				},
			}
		}
	)

	-- create error doc for asset
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "StorageExceededError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"quota exceeded",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = revisionId,
					type = "asset",
				},
				status = 403,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	-- asset doc should end up with quota exceeded annotation
	rxTest:expectSingleValue(
		docStore:docValue( docId )
			:map( function( docValue )
				return WFCatOzErrors.hasStorageExceededError( docValue )
			end ), true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestStorageExceededError_restore( rxTest )
	testStorageExceededError_restoreModify( rxTest, {} )
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestStorageExceededError_restoreModify( rxTest )
	testStorageExceededError_restoreModify( rxTest, { modifyAfterRestore = true } )
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestStorageExceededError_restoreDelete( rxTest, options )

	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	local deletedImageId = KSIdGenerator.uuidInLowercaseHex()
	local deletedRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local deletedImageId2 = KSIdGenerator.uuidInLowercaseHex()

	-- create restored 'asset' doc (that will be deleted below)
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			_labels = { oz = true },
		}
	)

	-- delete the restored asset (and create the trash asset)
	rxTest:waitFor(
		docStore:applyOverlay {
			_id = docId,
			_deleted = true,
			_newId = deletedImageId2,
			_newIdOverlay = {
				type = 'asset',
				subtype = 'deleted_image',
				userCreated = '2016-06-07T15:55:34.218872Z',
				purgeDate = '2016-06-08T15:55:34.218872Z',
				original = {
					id = docId,
					userCreated = '2016-06-07T15:55:34.218872Z',
				},
				_localOnly = {
					universal = {
						trash = {
							createFrom = {
								id = deletedImageId,
								lastAssetId = deletedImageId,
								lastRevisionId = deletedRevisionId,
								lastTrashPurgeDate = "2019-07-12T21:32:40.415Z",
								lastTrashUserCreated = "2019-05-13T21:32:40.415Z",
								revision_id = deletedRevisionId,
								subtype = "deleted_image",
							},
							createWithRevId = revisionId,
							uploaded = {
								["/rels/master"] = true,
								["/rels/proxy_type/2560"] = true,
								["/rels/xmp"] = true,
							},
						},
					},
				}
			}
		}
	)

	-- create error doc for asset
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "StorageExceededError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"quota exceeded",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = revisionId,
					type = "asset",
				},
				status = 403,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	-- The newly created trash asset should end up with quota exceeded annotation
	rxTest:expectSingleValue(
		docStore:docValue( deletedImageId2 )
			:map( function( docValue )
				return WFCatOzErrors.hasStorageExceededError( docValue )
			end ), true )

end

--------------------------------------------------------------------------------

-- @mark TrialOrSubscriptionExpiredError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestTrialOrSubscriptionExpiredError( rxTest )

	rxTest:setTimeLimit( 30 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			_newRevId = revisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		}
	)

	-- create error doc for asset
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			_newRevId = revisionId,
			subtype = "TrialOrSubscriptionExpiredError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"quota exceeded",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = revisionId,
					type = "asset",
				},
				status = 403,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	-- asset doc should end up with quota exceeded annotation
	rxTest:expectSingleValue(
		docStore:docValue( docId )
			:map( function( docValue )
				return WFCatOzErrors.hasTrialOrSubscriptionExpiredError( docValue )
			end ), true )

end

--------------------------------------------------------------------------------

-- @mark TrialOrSubscriptionExpiredError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestEntitlementExpiredError( rxTest )

	rxTest:setTimeLimit( 30 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			_newRevId = revisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		}
	)

	-- create error doc for asset
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			_newRevId = revisionId,
			subtype = "EntitlementExpiredError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"entitlement expired",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = revisionId,
					type = "asset",
				},
				status = 403,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	-- asset doc should end up with quota exceeded annotation
	rxTest:expectSingleValue(
		docStore:docValue( docId )
			:map( function( docValue )
				return WFCatOzErrors.hasEntitlementExpiredError( docValue )
			end ), true )

end

--------------------------------------------------------------------------------

-- @mark MultipartCompleteError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestMultipartCompleteError( rxTest )

	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )
	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			_newRevId = revisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		}
	)

	-- create error doc for asset
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "MultipartCompleteError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"Multipart complete was unsuccessful",
					},
				},
				source = {
					asset_id = docId,
					revision_id = revisionId,
					type = "master",
					id = "master",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	-- asset doc should end up with quota exceeded annotation
	rxTest:expectSingleValue(
		docStore:docValue( docId )
			:map( function( docValue )
				return WFCatOzErrors.hasMultipartCompleteError( docValue )
			end ), true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestMultipartCompleteError_testNightly( rxTest )

	rxTest:setTimeLimit( 600 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog

	rxTest:waitFor(
		catalogTestSession:catalog()
			:tap( function( _catalog )
				catalog = _catalog
			end )
	)

	local docStore = catalog:docStore()

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )

	local catalogSpec

	rxTest:waitFor(
		catalogTestSession:catalogSpec()
			:tap( function( _catalogSpec )
				WFArguments.check( 'WFCatSpec' )
				catalogSpec = _catalogSpec
			end )
	)

	local assetId = KSIdGenerator.uuidInLowercaseHex()

	local testDataPath = KSPathUtils.getStandardFilePath( "testdata" )
	testDataPath = KSPathUtils.child( testDataPath, 'assets' )
	local imageChunkedPath = KSPathUtils.child( testDataPath, '12MB.jpg' )
	local fileAttributes = WFFileUtils.fileAttributes( imageChunkedPath )
	local imageFileSize = fileAttributes.fileSize

	local revId

	rxTest:finally( function()

		errorHandler:close()
		puller:close()
		pusher:close()

	end )

	-- start the sync processes and track error message
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT count(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )

	puller:getUpdatesContinuously()
	rxTest:waitFor( pusher:sendUpdatesContinuously() )

	-- We push one asset to Oz and upload file with finalUrl different sessionId.

	-- Prime the updates feed pump.

	rxTest:waitFor(
		docStore:setDocValue {
			_id = assetId,
			_localOnly = {
				mumble = "stuff",
				originalFilePath = imageChunkedPath,
			},
			captureDate = "2004-11-02T11:10:09",
			importSource = {
				fileName = "test.jpeg",
				fileSize = imageFileSize,
				contentType = 'image/jpeg',
				importedBy = catalogSpec:ozUserId(),
				importTimestamp = "2015-11-02T08:37:24.513Z",
				originalHeight = 16,
				originalWidth = 16,
			},
			subtype = "image",
			type = "asset",
		}
		:tap( function( _revId )
			revId = _revId
		end )
	)

	local masterCreateUrl

	local assetCreatedOnOz = rxTest:observe(
							docStore:ongoingDocValue( assetId )
								:map( function( docValue )
									masterCreateUrl = KSTableUtils.derefOpt( docValue, '_serverProvidedMetadata', 'links', '/rels/master_create', 'href' )

									return docValue._serverProvidedMetadata ~= nil and masterCreateUrl ~= nil
								end ) )

	assetCreatedOnOz:expectDataWithinTime( 40, true )

	-- multipart upload initiate

	local ozUser = catalogSpec:wfOzUser()

	masterCreateUrl = masterCreateUrl:replace( '{revision_id}', revId )

	rxTest:waitFor(
		docStore:docValue( assetId )
			:map( function( docValue )

				local url = masterCreateUrl .. '/parts'
				local putArgs = {
					requestHeaders = {
						{
							header = 'Content-Type',
							value = 'image/jpeg'
						},
						{
							header = 'Accept',
							value = 'application/json'
						},
						{
							header = 'X-Generate-Proxies',
							value = 'all'
						},
						{
							header = 'X-Generate-Renditions',
							value = 'all'
						}
					},
					timeout = 60,
					returnHttpErrorsAsData = true,
					bypassRateLimiter = true,
				}

				return ozUser
					:put( url, "", putArgs )
					:map( function( data, headers )

						if headers and headers.status_code == "201" then

							local results = KSJsonDecoder.decodeWebSafeJson_stripNull_ignoreEmpty( data )

							local docStatusOverlay = {
								_id = assetId,
								type = "asset",
								_localOnly = {
									chunkedUpload = {
										parts = {

										},
										part_template = KSTableUtils.derefOpt( results, "links", "/rels/create_part", "href" ),
										finish_href = KSTableUtils.derefOpt( results, "links", "self", "href" ),
									},
								},
							}

							return docStore
								:applyOverlay( docStatusOverlay )

						end
					end )
					:call( runUntilDone )

			end )
	)

	local multipartInitSucceed = rxTest:observe(
							docStore:ongoingDocValue( assetId )
								:map( function( docValue )
									local chunkedUploadParts = KSTableUtils.derefOpt( docValue, '_localOnly', 'chunkedUpload', 'parts' )
									local finishUrl = KSTableUtils.derefOpt( docValue, '_localOnly', 'chunkedUpload', 'finish_href' )

									return chunkedUploadParts ~= nil and finishUrl ~= nil
								end ) )

	multipartInitSucceed:expectDataWithinTime( 10, true )



	rxTest:waitFor(
		docStore:docValue( assetId )
			:flatMap( function( docValue )

				local putArgs = {
					requestHeaders = {
						{
							header = 'Content-Type',
							value = 'image/jpeg'
						},
						{
							header = 'Accept',
							value = 'application/json'
						},
						{
							header = 'X-Generate-Proxies',
							value = 'all'
						},
						{
							header = 'X-Generate-Renditions',
							value = 'all'
						}
					},
					timeout = 60,
					returnHttpErrorsAsData = true,
					bypassRateLimiter = true,
				}

				local finishUrl = KSTableUtils.derefOpt( docValue, '_localOnly', 'chunkedUpload', 'finish_href' )

				local jsonData = KSJsonEncoder.encodeOzUploadJson({
					parts = {
						[ 1 ] = {
							part_number = 1,
							etag = "123",
						},
						[ 2 ] = {
							part_number = 2,
							etag = "456",
						}
					}
				})

				putArgs.requestMethod = "POST"


				local found = false
				local headerName = 'Content-Type'
				local headerValue = 'application/json'
				for index, header in ipairs( putArgs.requestHeaders ) do
					if header.header == headerName then
						if headerValue ~= nil then
							header.value = headerValue
						else
							table.remove( putArgs.requestHeaders, index )
						end
						found = true
						break
					end
				end
				if not found then
					putArgs.requestHeaders[ #putArgs.requestHeaders + 1 ] = {
						header = headerName,
						value = headerValue,
					}
				end


				return ozUser
					:putBinary( finishUrl, jsonData, putArgs )
					:flatMap( function( data, headers )

						if headers and headers.status_code == "202" then

							local partialMasterUrl = finishUrl:gsub( "^.*//([^/]+", "" )
							local endTime = KSDate.timeInIso8601Format()

							local docStatusOverlay = {
								_id = docValue._id,
								type = "asset",
								_localOnly = {
									chunkedUpload = {
										_delete = true,
									},
									original = {
										ozUrl = partialMasterUrl,
										doNotBackUpFlagSet = true,
										uploaded = endTime,
									 },
								},
							}

							return docStore
								:applyOverlay( docStatusOverlay )

						end

					end )

			end )
	)

	-- we should get an error from Oz
	outstandingErrors:expectDataWithinTime( 40, 1 )

	-- the error handler should eventually clear the error
	outstandingErrors:expectDataWithinTime( 40, 0 )

	local multipartCompleteErrorFromOz = rxTest:observe(
							docStore:ongoingDocValue( assetId )
								:map( function( docValue )
									local ozMultipartCompleteError = KSTableUtils.derefOpt( docValue, '_localOnly', 'ozError', 'subtype' )
									return ozMultipartCompleteError == 'MultipartCompleteError'
								end ) )

	multipartCompleteErrorFromOz:expectDataWithinTime( 10, true )

end

--------------------------------------------------------------------------------

-- @mark PayloadValidationError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetPayloadValidationError( rxTest )

	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local revId = KSIdGenerator.uuidInLowercaseHex()
	local auxId = KSIdGenerator.uuidInLowercaseHex()
	local auxDigest = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc that has not been synced to Oz
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			_newRevId = revId,
			type = 'asset',
			subtype = 'image',
			-- invalid date
			captureDate = false,
			importSource = {
				importTimestamp = '2017-03-14T10:50:60.000Z', -- note seconds are 60
				filePath = '/path/',
				originalHeight = 150,
				-- fractional width
				originalWidth = 100.5,
			},
			ratings = {
				[ userId ] = {
					date = '0000-00-00T00:00:00',
					rating = 5,
				}
			},
			develop = {
				-- disallowed field
				_merge = 'atomic',
				-- invalid TZ
				userUpdated = "2017-02-22T18:29:42.616+02:00",
				xmpCameraRaw = '<xmpmeta /',
				-- processingModel is missing
				profiles = {},
				processVersion = 1
			},
			aux = {
				[ auxId ] = {
					digest = auxDigest
				}
			},
			video = {
			 frameRate = {
			     6.9130254280137e+015,
			     4.5035996273705e+015
			 },
			},
			['0'] = 'totally made up',
			xmp = {
				exif = {
					-- ISOSpeedRatings must be a single number
					ISOSpeedRatings = { 50, 100 },
					-- wrong key name
					fnumber = { 1, 2 },
					-- illegal value
					FocalLength = { 'bad' },
				}
			}
		}
	)

	-- create error doc for asset that indicates the resource is not found
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "PayloadValidationError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					['payload.develop._merge' ] = { 'disallowed' },
					['payload.develop.userUpdated' ] = { 'does not match pattern' },
					['payload.develop.processingModel'] = { 'required' },
					['payload.develop.processVersion' ] = { 'dependency_required' },
					['payload.aux' ] = { 'cannot be deleted' },
					['payload.captureDate' ] = { 'invalid' },
					['payload.BadKey' ] = { 'invalid' },
					['payload.ratings.'..userId..'.date'] = { 'does not match pattern' },
					['payload.xmp.exif.ISOSpeedRatings' ] = { 'invalid' },
					['payload.xmp.exif.FocalLength' ] = { 'invalid' },
					['payload.video.frameRate.0' ] = { 'invalid' },
					['payload.video.frameRate.1' ] = { 'invalid' },
					['payload.0' ] = { 'invalid' },
					['payload.develop.profiles' ] = { 'invalid' },
					['payload.importSource.importTimestamp'] = { 'does not match pattern' },
					['payload.importSource.originalWidth'] = { 'invalid type' },
					-- NMB-11045 this field doesn't exist in initial doc and we shouldn't create an empty hash trail to
					-- it by attempting to remove something that doesn't exist
					['payload.xmp.nmb111045.subject' ] = { 'minimum number of properties not met' },
				},
				missing_resource_id = docId,
				source = {
					action = "create",
					id = docId,
					revision_id = revId,
					type = "asset",
				},
				status = 400,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	-- asset should not have ResourceNotFoundError annotation, it may have one for the error that
	-- is presenting the asset from getting to Oz at all, but that should not be replaced by a
	-- ResourceNotFoundError
	rxTest:expectSingleValue(
		docStore:docValue( docId )
			:map( function( docValue )

				docValue = KSTableUtils.deepCopy( docValue )
				docValue._localId = nil
				docValue._changeCounter = nil
				docValue._revId = nil

				return docValue

			end ), {
				_id = docId,
				type = 'asset',
				subtype = 'image',
				captureDate = "0000-00-00T00:00:00",
				importSource = {
					importTimestamp = '2017-03-14T10:50:59.000Z', -- seconds has been adjusted
					filePath = '/path/',
					originalHeight = 150,
					originalWidth = 100,
				},
				ratings = {
					[ userId ] = {
						rating = 5,
					}
				},
				develop = {
					userUpdated = "2017-02-22T16:29:42.616Z",
					xmpCameraRaw = '<xmpmeta /',
					processingModel = 'lightroom',
					processVersion = 1
				},
				aux = {
					[ auxId ] = {
						digest = auxDigest
					}
				},
				video = {
				     frameRate = {
				     },
				 },
				xmp = {
					exif = {
						fnumber = { 1, 2 }
					}
				}
			} )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetPayloadValidationError_DeleteOrRestoreFromOz_Create( rxTest )
	
	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	local docStore = catalog:docStore()
	
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local smartAlbumId = KSIdGenerator.uuidInLowercaseHex()
	local smartAlbumRevId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	
	rxTest:waitFor( docStore:bulkApplyOverlay( {
		{
			_id = smartAlbumId,
			_newRevId = smartAlbumRevId,
			type = 'album',
			subtype = 'smart',
			name = 'fooo',
			version = 1,
			predicates = {
				class = "and",
				terms = {
					{
						class = 'folder',
						terms = { op = 'is', value = 'uuid' }
					}
				}
			}
		},
		{
			_id = errorDocId,
			type = 'error',
			subtype = 'PayloadValidationError',
			_serverProvidedMetadata = {
				source = {
					id = smartAlbumId,
					revision_id = smartAlbumRevId,
					action = "create"
				},
				errors = {
		            ['payload.predicates.terms.0' ] = { 'invalid' },
		        },
			}
		},
	} ) )
	
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	-- smart album doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, smartAlbumId ) )
		:expectDataWithinTime( 3, true )
	
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetPayloadValidationError_DeleteOrRestoreFromOz_Update( rxTest )
	
	rxTest:setTimeLimit( 5 )
	
	local smartAlbumId = KSIdGenerator.uuidInLowercaseHex()
	local smartAlbumRevId1 = KSIdGenerator.uuidInLowercaseHex()
	local smartAlbumRevId2 = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	
	local albumUrl = string.format( '/v2/catalogs/%s/albums/%s', catalogId, smartAlbumId )
	
	local expectedResponses = {
		{
			url = albumUrl,
			
			response = string.format( [[{
				"base": "https://photos.adobe.io/v2/catalogs/%s",
				"payload": {
					"name": "foo",
					"version": 1,
					"predicates": {
						"class": "and",
						"terms": [{
							"class": "keyword",
							"terms": { "op": "is", "value": "key1" }
						}]
					}
				},
				"id": "%s",
				"type": "album",
				"subtype": "smart",
				"revision_ids": [
					"%s"
				]
			}]], catalogId, smartAlbumId, smartAlbumRevId1 ),
			
			headers = {
				status_code = '200',
			},
		}
	}
	
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest, expectedResponses )
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	local docStore = catalog:docStore()
	
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	-- create 'smart album' doc with valid predicate
	rxTest:waitFor(
		docStore:setDocValue {
			_id = smartAlbumId,
			_newRevId = smartAlbumRevId1,
			type = 'album',
			subtype = 'smart',
			name = 'fooo',
			version = 1,
			predicates = {
				class = "and",
				terms = {
					{
						class = 'keyword',
						terms = { op = 'is', value = 'key1' }
					}
				}
			},
			_serverProvidedMetadata = {
				links = {
					["self"] = {
						href = albumUrl,
					},
				}
			}
		}
	)
	
	-- update the document with an invalid predicate
	rxTest:waitFor( docStore:bulkApplyOverlay( {
		{
			_id = smartAlbumId,
			_newRevId = smartAlbumRevId2,
			predicates = {
				class = "and",
				terms = {
					{
						class = 'folder',
						terms = { op = 'is', value = 'uuid' }
					}
				}
			}
		},
		{
			_id = errorDocId,
			type = 'error',
			subtype = 'PayloadValidationError',
			_serverProvidedMetadata = {
				source = {
					id = smartAlbumId,
					revision_id = smartAlbumRevId2,
					action = "update"
				},
				errors = {
		            ['payload.predicates.terms.0' ] = { 'invalid' },
		        },
			}
		},
	} ) )
	
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
	
	-- verify the smart album was restored from the oz version
	local albumOnOz = rxTest:observe(
							docStore:ongoingDocValue( smartAlbumId )
								:map( function( docValue )
									return KSTableUtils.derefOpt( docValue, 'predicates', 'terms', 1, 'class' )

								end ) )
	
	albumOnOz:expectDataWithinTime( 3, 'keyword' )
	
end

----------------------------------------------------------------------------------------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetPayloadValidationErrorMarkUnknown( rxTest )

	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local assetRevId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	rxTest:waitFor( catalog:docStore():bulkApplyOverlay( {
		{
			_id = assetId,
			_newRevId = assetRevId,
			type = 'asset',
			subtype = 'image',
			name = 'fooo',
		},
		{
			_id = errorDocId,
			type = 'error',
			subtype = 'PayloadValidationError',
			_serverProvidedMetadata = {
				source = {
					id = assetId,
					revision_id = assetRevId
				},
				errors = {
		            file_extension = {
		                "unsupported",
		            },
		        },
			}
		},
	} ) )

	rxTest:sleep( 3 )

	rxTest:expectSingleValue(
		catalog:indexStore():query( {
			sql = 'SELECT COUNT(docId) FROM errors'
		} )
		:take( 1 )
		:map( function( rows )

			return rows[ 1 ][ 1 ]

		end ), 0 )

	rxTest:expectSingleValue(
		catalog:docStore():docValue( errorDocId )
			:map( function( docValue )

				return KSTableUtils.derefOpt( docValue, '_localOnly', 'unknownError' )

			end ), true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetPayloadValidationError_testNightly( rxTest )

	rxTest:setTimeLimit( 120 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog
	local assetId = KSIdGenerator.uuidInLowercaseHex()

	rxTest:waitFor(
		catalogTestSession:catalog()
			:tap( function( _catalog )
				catalog = _catalog
			end )
	)

	local docStore = catalog:docStore()

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:finally( function()
		errorHandler:close()
		puller:close()
		pusher:close()

	end )

	-- watch the errors table, to ensure we actually do get errors
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT count(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )

	-- start the sync processes
	puller:getUpdatesContinuously()
	rxTest:waitFor( pusher:sendUpdatesContinuously() )

	rxTest:waitFor(
		docStore:setDocValue {
			_id = assetId,
			type = 'asset',
			subtype = 'image',
			-- captureDate = '610626-01-19T07:14:54.560+07:36:42',
			importSource = {
				fileName = '/path/',
				originalHeight = 150,
				originalWidth = 100.5,
				importTimestamp = '2017-03-14T10:50:60.000Z', -- note seconds are 60
				importedBy = KSIdGenerator.uuidInLowercaseHex(),
			},
			develop = {
				_merge = 'atomic',
				userUpdated = "2017-02-22T18:29:42.616+02:00",
				processingModel = 'lightroom',
				xmpCameraRaw = '<xmpmeta /',
			},
			ratings = {
				[ KSIdGenerator.uuidInLowercaseHex() ] = {
					date = "0000-00-00T00:00:00",
					rating = 5,
				}
			},
			xmp = {
				exif = {
					ISOSpeedRatings = 50,
					fnumber = { 1, 2 }
				}
			}
		}
	)

	-- we should get an error from Oz
	outstandingErrors:expectDataWithinTime( 40, 1 )

	-- the error handler should eventually clear the error
	outstandingErrors:expectDataWithinTime( 10, 0 )

	-- Verify asset was uploaded
	local assetOnOz = rxTest:observe(
							docStore:ongoingDocValue( assetId )
								:map( function( docValue )
									return docValue._serverProvidedMetadata ~= nil
								end ) )

	assetOnOz:expectDataWithinTime( 40, true )

	-- Test video validation for 'userUpdated'
	assetId = KSIdGenerator.uuidInLowercaseHex()

	rxTest:waitFor(
		docStore:setDocValue {
			_id = assetId,
			type = 'asset',
			subtype = 'video',
			importSource = {
				fileName = '/path/',
				originalHeight = 150,
				originalWidth = 100.5,
				importTimestamp = '2017-03-14T10:50:60.000Z',
				importedBy = KSIdGenerator.uuidInLowercaseHex(),
			},
			develop = {
				-- This is not supported for videos. We should be able to recover
				version = 1,
				processVersion = 1,
				userUpdated = "2017-02-22T18:29:42.616+02:00",
				croppedWidth = 1000,
				croppedHeight = 1000
			},
		}
	)

	-- we should get an error from Oz
	outstandingErrors:expectDataWithinTime( 40, 1 )

	-- the error handler should eventually clear the error
	outstandingErrors:expectDataWithinTime( 10, 0 )

	-- Verify asset was uploaded
	assetOnOz = rxTest:observe(
							docStore:ongoingDocValue( assetId )
								:map( function( docValue )
									return docValue._serverProvidedMetadata ~= nil
								end ) )

	assetOnOz:expectDataWithinTime( 40, true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetPayloadValidationErrorTwoTries_testNightly( rxTest )

	rxTest:setTimeLimit( 120 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog
	local assetId = KSIdGenerator.uuidInLowercaseHex()

	rxTest:waitFor(
		catalogTestSession:catalog()
			:tap( function( _catalog )
				catalog = _catalog
			end )
	)

	local docStore = catalog:docStore()

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:finally( function()
		errorHandler:close()
		puller:close()
		pusher:close()
	end )

	-- watch the errors table, to ensure we actually do get errors
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT count(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )

	-- start the sync processes
	puller:getUpdatesContinuously()
	rxTest:waitFor( pusher:sendUpdatesContinuously() )

	rxTest:waitFor(
		docStore:setDocValue {
			_id = assetId,
			type = 'asset',
			subtype = 'image',
			captureDate = '610626-01-19T07:14:54.560+07:36:42',
			importSource = {
				fileName = '/path/',
				originalHeight = 150,
				originalWidth = 100.5,
				importTimestamp = '2017-03-14T10:50:60.000Z', -- note seconds are 60
				importedBy = KSIdGenerator.uuidInLowercaseHex(),
			},
			develop = {
				_merge = 'atomic',
				userUpdated = "2017-02-22T18:29:42.616+02:00",
				processingModel = 'lightroom',
				xmpCameraRaw = '<xmpmeta /',
			},
			-- we should need two attempts to get this cleaned up.
			-- the first should remove the empty date, the second the empty table
			ratings = {
				[ KSIdGenerator.uuidInLowercaseHex() ] = {
					date = "0000-00-00T00:00:00",
				}
			},
			xmp = {
				exif = {
					ISOSpeedRatings = 50,
					fnumber = { 1, 2 }
				},
				tiff = {
					InvalidField = 'ForceTwoPass'
				}
			}
		}
	)

	-- we should get an error from Oz
	outstandingErrors:expectDataWithinTime( 60, 1 )

	-- the error handler should eventually clear the error
	outstandingErrors:expectDataWithinTime( 10, 0 )

	-- we should get a second error from Oz
	outstandingErrors:expectDataWithinTime( 60, 1 )

	-- the error handler should eventually clear the error
	outstandingErrors:expectDataWithinTime( 10, 0 )

	-- Verify asset was uploaded
	local assetOnOz = rxTest:observe(
							docStore:ongoingDocValue( assetId )
								:map( function( docValue )
									return docValue._serverProvidedMetadata ~= nil
								end ) )

	assetOnOz:expectDataWithinTime( 60, true )

end

--------------------------------------------------------------------------------

-- @mark InputValidationError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestPersonPrimaryInputValidationError( rxTest )

	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local revId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			_newRevId = revId,
			type = 'asset',
			subtype = 'person',
			primary = "d58059e3898ee16896d490a9ea3c8888",
			name = "hari"
		}
	)

	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "InputValidationError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					['payload.primary' ] = { 'primary asset should not have a primary' },
				},
				source = {
					action = "update",
					id = docId,
					revision_id = revId,
					type = "asset",
				},
				status = 400,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	rxTest:expectSingleValue(
		docStore:docValue( docId )
		:map( function( docValue )
			docValue = KSTableUtils.deepCopy( docValue )
			docValue._localId = nil
			docValue._changeCounter = nil
			docValue._revId = nil

			return docValue
		end ),
		{
			_id = docId,
			type = 'asset',
			subtype = 'person',
			name = "hari"
		} )
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestInputValidationError_ifAssetDoesNotExistError( rxTest )

	rxTest:setTimeLimit( 5 )
	
	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	-- Create document
	
	local docId = KSIdGenerator.uuidInLowercaseHex()
	local docValue = WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, docId )
	local revId = docValue._newRevId

	rxTest:waitFor( docStore:setDocValue( docValue ) )
		
	-- Create error document
	
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "InputValidationError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = { 
						 "does not exist"
					},
				},
				source = {
					action = "update",
					id = docId,
					revision_id = revId,
					type = "asset",
				},
				status = 400,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- Error handler should process the InputValidationError and delete the asset
	
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, docId ) )
		:expectDataWithinTime( 3, true )
	
	-- Error doc should be deleted
		
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

end
--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestInputValidationError_retryIfAssetDoesNotExistAndHasNotBeenHandled( rxTest )

	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	-- Create first document 

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local docValue = WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, docId )
	local revId = docValue._newRevId
	
	rxTest:waitFor( docStore:setDocValue( docValue ) )

	-- Create error document

	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "InputValidationError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = { 
						"does not exist"
					},
				},
				source = {
					action = "update",
					id = docId,
					revision_id = revId,
					type = "asset",
				},
				status = 400,
				updated = "2016-06-07T15:55:34.218872Z",
			},
			_localOnly = {
				localError = {
					InputValidationError = {
						handlingFailed = false
					},
				},
				unknownError = true,
			}
		}
	)
	
	-- Make sure second asset is deleted
	
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, docId ) )
		:expectDataWithinTime( 3, true )
	
	-- Error doc should be deleted, this mimics the document being reindex back and making sure it's 
	-- being handled by the new handler. 
		
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestInputValidationError_spaceDoc_testNightly( rxTest )

	rxTest:setTimeLimit( 120 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog
	local spaceId = KSIdGenerator.uuidInLowercaseHex()

	rxTest:waitFor(
		catalogTestSession:catalog()
			:tap( function( _catalog )
				catalog = _catalog
			end )
	)

	local docStore = catalog:docStore()

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )

	rxTest:finally( function()
		errorHandler:close()
		puller:close()
		pusher:close()

	end )

	-- watch the errors table, to ensure we actually do get errors
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT count(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )

	-- start the sync processes
	puller:getUpdatesContinuously()
	rxTest:waitFor( pusher:sendUpdatesContinuously() )

	rxTest:waitFor(
		docStore:setDocValue {
			_id = spaceId,
			type = 'space',
			subtype = 'shared',
			location = false,
			metadata = false,
			private = false,
		}
	)

	-- we should get an error from Oz
	outstandingErrors:expectDataWithinTime( 40, 1 )

	-- the error handler should eventually clear the error
	outstandingErrors:expectDataWithinTime( 10, 0 )

	-- Verify asset was uploaded
	local spaceOnOz = rxTest:observe(
							docStore:ongoingDocValue( spaceId )
								:map( function( docValue )
									return docValue._serverProvidedMetadata ~= nil
								end ) )

	spaceOnOz:expectDataWithinTime( 40, true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestInputValidationError_ifMatch_testNightly( rxTest )

	rxTest:setTimeLimit( 120 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog
	local assetId = KSIdGenerator.uuidInLowercaseHex()

	rxTest:waitFor(
		catalogTestSession:catalog()
			:tap( function( _catalog )
				catalog = _catalog
			end )
	)

	local docStore = catalog:docStore()

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )

	rxTest:finally( function()
		errorHandler:close()
		puller:close()
		pusher:close()

	end )

	puller:getUpdatesContinuously()
	rxTest:waitFor( pusher:sendUpdatesContinuously() )

	-- create a document and sync it to Oz
	rxTest:waitFor(
		docStore:setDocValue {
			_id = assetId,
			type = 'asset',
			subtype = 'image',
			captureDate = '2016-01-19T07:14:54.560Z',
			importSource = {
				fileName = '/path/',
				originalHeight = 150,
				originalWidth = 100,
				importTimestamp = '2017-03-14T10:50:59.000Z',
				importedBy = KSIdGenerator.uuidInLowercaseHex(),
			},
			develop = {
				userUpdated = "2017-02-22T18:29:42.616Z",
				processingModel = 'lightroom',
				xmpCameraRaw = '<xmpmeta /',
			},
			xmp = {
				dc = {
					title = 'WF UNIT TEST'
				}
			}
		}
	)

	-- watch the errors table, to ensure we actually do get errors
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT count(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )

	-- sync it
	rxTest:expectSingleValue( docStore:ongoingDocValue( assetId )
				:filter( function( docValue )
					return not not KSTableUtils.derefOpt( docValue, '_serverProvidedMetadata', 'links' )
				end )
				:map( function( docValue )
					-- we've got a links section so it got to Oz
					return true
				end )
				:take( 1 ), true )

	-- now we're going to fake things going off the rails
	-- first update the document and also delete the revision_ids in the local document
	rxTest:waitFor(
		docStore:docValue( assetId )
			:flatMap( function( docValue )
				return docStore:applyOverlay(
					{
						_id = assetId,
						_serverProvidedMetadata = {
							revision_ids = { _delete = true }
						},
						xmp = {
							dc = {
								title = 'WF UNIT TEST UPDATE'
							}
						}
					} )
			end )
	)

	-- we should get an InputValidationError from Oz
	outstandingErrors:expectDataWithinTime( 40, 1 )

	-- the error handler should clear the error
	outstandingErrors:expectDataWithinTime( 10, 0 )

	-- but clearing the error will result in a conflict so we'll need to ensure that gets cleared
	local conflictResolver = require 'WFCatConflictResolver'( docStore, catalog:indexStore() )
	rxTest:addCloseTarget( conflictResolver )

	-- wait until the latest doc value has both the updated title and the 'oz' label
	-- and 2 revisions in the revision tree
	rxTest:expectSingleValue( Rx.Observable.timer( 0, 1, WFRxScheduler.sharedScheduler() )
		:flatMap( function()
			return docStore:docValue( assetId )
				:map( function( docValue )

					local labels = docValue._labels or {}
					local revisions = KSTableUtils.derefOpt( docValue, '_serverProvidedMetadata', 'revision_ids' ) or {}
					local title = KSTableUtils.derefOpt( docValue, 'xmp', 'dc', 'title' )

					return labels.oz and ( title == 'WF UNIT TEST UPDATE' ) and ( #revisions == 2 )
				end )
		end )
		:filter( function( value ) return not not value end )
		:take( 1 ), true )

	-- Verify how the error was resolved
	rxTest:expectEqual( {
		InputValidationError = {
			count = 1,
			resolution = {
				Fixed = {
					count = 1,
					sourceDocIds = {
						assetId,
					},
				},
			},
		},
	}, errorHandler:DEBUG_errorsResolved() )

end

--------------------------------------------------------------------------------

local function observeIsDeletedOnOz( rxTest, catalog, docId )

	rxTest:logf( "Waiting for %s to be deleted on Oz", docId )

	return rxTest:observe( catalog:docStore():ongoingDocValue( docId )
		:map( function( docValue )

			local isDeleted = not not ( docValue._deleted or docValue._empty )
			local isLabeledOz = not not ( docValue._labels and docValue._labels.oz )

			return isDeleted and isLabeledOz

		end )
	)

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestInputValidationError_ifMatch_deleted_testNightly( rxTest )

	rxTest:setTimeLimit( 120 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog
	local assetId = KSIdGenerator.uuidInLowercaseHex()

	rxTest:waitFor(
		catalogTestSession:catalog()
			:tap( function( _catalog )
				catalog = _catalog
			end )
	)

	local docStore = catalog:docStore()

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )

	rxTest:finally( function()
		errorHandler:close()
		puller:close()
		pusher:close()

	end )

	puller:getUpdatesContinuously()
	rxTest:waitFor( pusher:sendUpdatesContinuously() )

	-- create a document and sync it to Oz
	rxTest:waitFor(
		docStore:setDocValue {
			_id = assetId,
			type = 'asset',
			subtype = 'image',
			captureDate = '2016-01-19T07:14:54.560Z',
			importSource = {
				fileName = '/path/',
				originalHeight = 150,
				originalWidth = 100,
				importTimestamp = '2017-03-14T10:50:59.000Z',
				importedBy = KSIdGenerator.uuidInLowercaseHex(),
			},
			develop = {
				userUpdated = "2017-02-22T18:29:42.616Z",
				processingModel = 'lightroom',
				xmpCameraRaw = '<xmpmeta /',
			},
			xmp = {
				dc = {
					title = 'WF UNIT TEST'
				}
			}
		}
	)

	-- watch the errors table, to ensure we actually do get errors
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT count(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )

	local fullDocValue
	-- sync it
	rxTest:expectSingleValue( docStore:ongoingDocValue( assetId )
				:filter( function( docValue )
					return not not KSTableUtils.derefOpt( docValue, '_serverProvidedMetadata', 'links' )
				end )
				:map( function( docValue )
					fullDocValue = docValue
					-- we've got a links section so it got to Oz
					return true
				end )
				:take( 1 ), true )

	-- now we're going to fake things going off the rails
	-- first delete the document and make sure the deletion syncs
	rxTest:waitFor(
		docStore:applyOverlay( {
				_id = assetId,
				_deleted = true
			} )
	)

	local observeAssetDeletedOnOz = observeIsDeletedOnOz( rxTest, catalog, assetId )
	observeAssetDeletedOnOz:expectDataWithinTime( 40, true )
	observeAssetDeletedOnOz:dispose()

	-- add the document back via setDocValue which will overwrite the
	-- deleted doc value with the 'old results'
	fullDocValue._serverProvidedMetadata = { something = true } -- necessary to be able ensure we push the update to Oz
	fullDocValue._changeCounter = nil
	fullDocValue._revId = nil
	fullDocValue._labels = nil
	fullDocValue.xmp.dc.title = 'WF UNIT TEST UPDATE'
	rxTest:waitFor( docStore:setDocValue( fullDocValue ) )

	local watchForDocDeletion = rxTest:observe( docStore:ongoingDocValue( assetId )
		:map( function( docValue )
			return not not docValue._empty
		end ) )

	watchForDocDeletion:expectDataWithinTime( 1, false )

	-- we should get an InputValidationError from Oz
	outstandingErrors:expectDataWithinTime( 40, 1 )

	-- the error handler should clear the error
	outstandingErrors:expectDataWithinTime( 40, 0 )

	-- but clearing the error will result in a conflict so we'll need to ensure that gets cleared
	local conflictResolver = require 'WFCatConflictResolver'( docStore, catalog:indexStore() )
	rxTest:addCloseTarget( conflictResolver )

	watchForDocDeletion:expectDataWithinTime( 15, true )

end

--------------------------------------------------------------------------------

local function testStackCoverInputValidationError( rxTest, coverError, hasStackAssets )

	rxTest:setTimeLimit( 60 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()

	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	local docStore = catalog:docStore()

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local optArgs = { docId = '1234' }
	local image2localId = KSIdGenerator.uuidInLowercaseHex()
	local image3localId = KSIdGenerator.uuidInLowercaseHex()

	WFCatStacksTest.addTestAsset( rxTest, catalog, 1, '2012-08-15T15:22:29', optArgs )

	optArgs = { docId = image2localId }

	WFCatStacksTest.addTestAsset( rxTest, catalog, 2, '2012-08-15T15:22:29.123', optArgs )

	optArgs = { docId = image3localId }

	WFCatStacksTest.addTestAsset( rxTest, catalog, 3, '2012-08-15T15:22:29.456', optArgs )

	-- create a valid stack

	local createStack = WFCatStacks.createStack( catalog, { 1, 2, 3 } )

	local stackId, _ = rxTest:getFirstValue( createStack )

	-- get revision Id

	local getRevId = docStore:docValue( stackId )

	local stackRevId = rxTest:getFirstValue( getRevId
		:map( function( docValue )
			return docValue._revId
		end ) 
	)

	if not hasStackAssets then
		rxTest:waitFor( docStore:deleteDoc( 1 ) )
		rxTest:waitFor( docStore:deleteDoc( 2 ) )
		rxTest:waitFor( docStore:deleteDoc( 3 ) )
	end

	-- Add error document

	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "InputValidationError",
			type = "error",
			_serverProvidedMetadata = {
				base = "link",
				errors = {
					cover = { coverError },
				},
				source = {
					action = "create",
					id = stackId,
					type = "asset",
					revision_id = stackRevId
				},
				status = 400,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 10, true )
		:dispose()


	-- Stacks are not implemented on other platforms error will be processed but ignored
	if WFPlatformInfo.isDesktop then

		if hasStackAssets then

			-- Verify the invalid cover id has been replaced with the existing
			-- stack_asset's assetId
			rxTest:expectSingleValue( docStore:docValue( stackId )
				:pluck( 'cover', 'id' ):map( type ),
				'string' )

		else

			-- Verify the stack has been removed as there are not stack_assets left for the cover.
			rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, stackId ) )
				:expectDataWithinTime( 3, true )
				:dispose()

		end

	end

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestStackCoverInputValidationError_invalidGUID_hasAssets( rxTest )

	testStackCoverInputValidationError( rxTest, "must be a valid GUID", true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestStackCoverInputValidationError_invalidGUID_NoAssets( rxTest )

	testStackCoverInputValidationError( rxTest, "must be a valid GUID", false )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestStackCoverInputValidationError_mustHaveAnAssetId_hasAssets( rxTest )

	testStackCoverInputValidationError( rxTest, "must have an asset ID", true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestStackCoverInputValidationError_mustHaveAnAssetId_NoAssets( rxTest )

	testStackCoverInputValidationError( rxTest, "must have an asset ID", false )

end

--------------------------------------------------------------------------------

-- @mark NotHeadRevisionPreconditionError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestNotHeadRevisionPreconditionError_testNightly( rxTest )

	rxTest:setTimeLimit( 700 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog
	local assetId = KSIdGenerator.uuidInLowercaseHex()

	rxTest:waitFor(
		catalogTestSession:catalog()
			:tap( function( _catalog )
				catalog = _catalog
			end )
	)

	local docStore = catalog:docStore()

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )

	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )

	rxTest:finally( function()
		errorHandler:close()
		puller:close()
		pusher:close()

	end )

	puller:getUpdatesContinuously()
	rxTest:waitFor( pusher:sendUpdatesContinuously() )

	-- create a document and sync it to Oz
	rxTest:waitFor(
		docStore:setDocValue {
			_id = assetId,
			type = 'asset',
			subtype = 'image',
			captureDate = '2016-01-19T07:14:54.560Z',
			importSource = {
				fileName = '/path/',
				originalHeight = 150,
				originalWidth = 100,
				importTimestamp = '2017-03-14T10:50:59.000Z',
				importedBy = KSIdGenerator.uuidInLowercaseHex(),
			},
			develop = {
				userUpdated = "2017-02-22T18:29:42.616Z",
				processingModel = 'lightroom',
				xmpCameraRaw = '<xmpmeta /',
			},
			xmp = {
				dc = {
					title = 'WF UNIT TEST'
				}
			}
		}
	)

	-- watch the errors table, to ensure we actually do get errors
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT count(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )

	-- sync it
	rxTest:observe( docStore:ongoingDocValue( assetId )
		:pluck( '_serverProvidedMetadata', 'links' )
		:map( function( links )
			-- we've got a links section so it got to Oz
			return links ~= nil
		end ) )
	:expectDataWithinTime( 30, true )
	:dispose()


	-- add another revision to the document
	rxTest:waitFor(
		docStore:docValue( assetId )
			:flatMap( function( docValue )

				return docStore:applyOverlay(
					{
						_id = assetId,
						xmp = {
							dc = {
								title = 'WF FIRST UNIT TEST UPDATE'
							}
						}
					} )
			end )
	)

	-- wait for it to sync
	rxTest:observe( docStore:ongoingDocValue( assetId )
		:pluck( '_serverProvidedMetadata', 'revision_ids' )
		:map( function( revision_ids )
			revision_ids = revision_ids or {}
			return #revision_ids > 1
		end ) )
	:expectDataWithinTime( 30, true )
	:dispose()

	pusher:addRequestHeader( 'X-Quicker-Retries', 'true' )

	-- first update the document and also delete the revision_ids in the local document
	rxTest:waitFor(
		docStore:docValue( assetId )
			:flatMap( function( docValue )
				return docStore:applyOverlay(
					{
						_id = assetId,
						_serverProvidedMetadata = {
							revision_ids = {
								[2] = { _delete = true },
							}
						},
						xmp = {
							dc = {
								title = 'WF SECOND UNIT TEST UPDATE'
							}
						}
					} )
			end )
	)

	-- We should get a NotHeadRevisionPreconditionError from Oz.
	outstandingErrors:expectDataWithinTime( 600, 1 )

	-- the error handler should clear the error
	outstandingErrors:expectDataWithinTime( 10, 0 )

	-- but clearing the error will result in a conflict so we'll need to ensure that gets cleared
	local conflictResolver = require 'WFCatConflictResolver'( docStore, catalog:indexStore() )
	rxTest:addCloseTarget( conflictResolver )

	-- wait until the latest doc value has both the updated title and the 'oz' label
	-- and 2 revisions in the revision tree
	rxTest:expectSingleValue( Rx.Observable.timer( 0, 1, WFRxScheduler.sharedScheduler() )
		:flatMap( function()
			return docStore:docValue( assetId )
				:map( function( docValue )

					local labels = docValue._labels or {}
					local title = KSTableUtils.derefOpt( docValue, 'xmp', 'dc', 'title' )

					return labels.oz and ( title == 'WF SECOND UNIT TEST UPDATE' )
				end )
		end )
		:filter( function( value ) return not not value end )
		:take( 1 ), true )

	-- Verify how the error was resolved
	rxTest:expectEqual( {
		NotHeadRevisionPreconditionError = {
			count = 1,
			resolution = {
				FixByGettingOzDoc = {
					count = 1,
					sourceDocIds = {
						assetId,
					},
				},
			},
		},
	}, errorHandler:DEBUG_errorsResolved() )

end

--------------------------------------------------------------------------------

-- @mark ResourceExistsError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetResourceExistsError_Deleted( rxTest )

	rxTest:setTimeLimit( 30 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local docId = KSIdGenerator.uuidInLowercaseHex()
	local docRevId = KSIdGenerator.uuidInLowercaseHex()

	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest, {
		{
			url = string.format( "/v2/catalogs/%s/updates/%s", catalogId, docId ),
			headers = {
				status_code = '200',
			},
			response = [[while (1) {} {"base":"https://photos-stage.adobe.io/v2/catalogs/]]..catalogId..[[/","id":"]]..KSIdGenerator.uuidInLowercaseHex()..[[",
						"deleted":true,"type":"asset","updated":"2017-04-26T20:33:53.650275Z","deleted_by":"]]..userId..[["}]]

		} } )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

		-- create basic 'asset' doc that has not been synced to Oz
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			_newRevId = docRevId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		}
	)

	-- create error doc for asset that indicates the resource is not found
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			_serverProvidedMetadata = {
				base = string.format( "https://photos-stage.adobe.io/v2/catalogs/%s/", catalogId ),
				errors = {
					asset = {
						"already exists",
					},
				},
				source = {
					action = "create",
					id = docId,
					links = {
						["self"] = {
							href = string.format( "/v2/catalogs/%s/updates/%s", catalogId, docId ),
						},
					},
					revision_id = docRevId,
					type = "asset",
				},
				status = 403,
				updated = "2017-04-26T19:27:11.417509Z",
			},
			subtype = "ResourceExistsError",
			type = "error",
		}
	)

	-- the error handler should process the ResourceExistsError and delete
	-- the asset
	local isDocValueDeleted = rxTest:observe( docStore:ongoingDocValue( docId )
								:map( function( docValue )
									return docValue._empty
								end ) )

	isDocValueDeleted:expectDataWithinTime( 40, true )

	-- check index store for error documents, there shouldn't be any any more
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT count(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )

	outstandingErrors:expectDataWithinTime( 10, 0 )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetResourceExistsError_NotDeleted( rxTest )

	rxTest:setTimeLimit( 20 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local docId = KSIdGenerator.uuidInLowercaseHex()
	local docRevId

	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest, {
		{
			url = string.format( "/v2/catalogs/%s/updates/%s", catalogId, docId ),
			headers = {
				status_code = '200',
			},
			response = [[while (1) {} {"base":"https://photos-stage.adobe.io/v2/catalogs/]]..catalogId..[[/","id":"]]..KSIdGenerator.uuidInLowercaseHex()..[[",
						"type":"asset","updated":"2017-04-26T20:33:53.650275Z","deleted_by":"]]..userId..[["}]]

		} } )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

		-- create basic 'asset' doc that has not been synced to Oz
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		}
		:map( function( revisionId )
			docRevId = revisionId
			return revisionId
		end )
	)

	-- create error doc for asset that indicates the resource is not found
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			_serverProvidedMetadata = {
				base = string.format( "https://photos-stage.adobe.io/v2/catalogs/%s/", catalogId ),
				errors = {
					asset = {
						"already exists",
					},
				},
				source = {
					action = "create",
					id = docId,
					links = {
						["self"] = {
							href = string.format( "/v2/catalogs/%s/updates/%s", catalogId, docId ),
						},
					},
					revision_id = docRevId,
					type = "asset",
				},
				status = 403,
				updated = "2017-04-26T19:27:11.417509Z",
			},
			subtype = "ResourceExistsError",
			type = "error",
		}
	)

	-- check index store for error documents, there shouldn't be any any more
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT * FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:map( function( rows )
			return #rows
		end ) )

	outstandingErrors:expectDataWithinTime( 10, 0 )

	-- the error handler should process the ResourceExistsError but not delete
	-- the asset since it does exist and we should eventually resolve through
	-- the normal sync process
	rxTest:expectSingleValue(
		docStore:docValue( docId )
			:map( function( docValue )
				return not not docValue._empty
			end ),
		false
	)

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetResourceExistsError_NotDeletedError( rxTest )

	rxTest:setTimeLimit( 20 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local docId = KSIdGenerator.uuidInLowercaseHex()
	local docRevId

	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest, {
		{
			url = string.format( "/v2/catalogs/%s/updates/%s", catalogId, docId ),
			headers = {
				status_code = '404',
			},
			fail = {
				{
					code = 1000,
					description = "Resource not found",
					errors = {
						revision = {
							"does not exist",
						},
					},
					subtype = "ResourceNotFoundError",
				},
				{
					status_code = '404',
					[ 'X-Result-Code' ] = '1000',
				}
			}

		} } )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

		-- create basic 'asset' doc that has not been synced to Oz
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		}
		:map( function( revisionId )
			docRevId = revisionId
			return revisionId
		end )
	)

	-- create error doc for asset that indicates the resource is not found
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			_serverProvidedMetadata = {
				base = string.format( "https://photos-stage.adobe.io/v2/catalogs/%s/", catalogId ),
				errors = {
					asset = {
						"already exists",
					},
				},
				source = {
					action = "create",
					id = docId,
					links = {
						["self"] = {
							href = string.format( "/v2/catalogs/%s/updates/%s", catalogId, docId ),
						},
					},
					revision_id = docRevId,
					type = "asset",
				},
				status = 403,
				updated = "2017-04-26T19:27:11.417509Z",
			},
			subtype = "ResourceExistsError",
			type = "error",
		}
	)

	-- check index store for error documents, there shouldn't be any any more
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT * FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:map( function( rows )
			return #rows
		end ) )

	outstandingErrors:expectDataWithinTime( 10, 0 )

	-- the error handler should process the ResourceExistsError but not delete
	-- the asset since it does exist and we should eventually resolve through
	-- the normal sync process
	rxTest:expectSingleValue(
		docStore:docValue( docId )
			:map( function( docValue )
				return not not docValue._empty
			end ),
		false
	)

end


--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestFavoriteResourceExistsError( rxTest )

	rxTest:setTimeLimit( 20 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local docId1 = KSIdGenerator.uuidInLowercaseHex()	-- local doc that'll result in error
	local docId2 = KSIdGenerator.uuidInLowercaseHex()	-- doc available on Oz that's returned as duplicate
	local fullSpaceId = "randomSpaceId"
	local fullAssetId = "randomAssetId"
	local docRevId

	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest, {
		{
			url = string.format( "/v2/catalogs/%s/updates/%s", catalogId, docId2 ),
			headers = {
				status_code = '200',
			},
			response = [[while (1) {} {"base":"https://photos-stage.adobe.io/v2/catalogs/]]..catalogId..[[/","id":"]]..docId2..[[",
						"type":"favorite", "subtype":"user", "updated":"2017-04-26T20:33:53.650275Z" }]]

		} } )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

		-- create basic 'favorite' doc that has not been synced to Oz
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId1,
			type = 'favorite',
			subtype = 'user',
			space = { id = fullSpaceId },
			asset = { id = fullAssetId },
			userCreated = "2011-11-11T11:11:11.111111Z",
			userUpdated = "2011-11-11T11:11:11.111111Z",

		}
		:map( function( revisionId )
			docRevId = revisionId
			return revisionId
		end )
	)

	-- create error doc for asset that indicates the resource is not found
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			_serverProvidedMetadata = {
				base = string.format( "https://photos-stage.adobe.io/v2/catalogs/%s/", catalogId ),
				errors = {
					favorite = {
						"favorite by user already exists on asset in space",
					},
				},
				source = {
					action = "create",
					id = docId1,
					links = {
						["self"] = {
							href = string.format( "/v2/catalogs/%s/updates/%s", catalogId, docId2 ),
						},
					},
					revision_id = docRevId,
					type = "favorite",
				},
				status = 403,
				updated = "2017-04-26T19:27:11.417509Z",
			},
			subtype = "ResourceExistsError",
			type = "error",
		}
	)

	-- check index store for error documents, there shouldn't be any any more
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT * FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:map( function( rows )
			return #rows
		end ) )

	outstandingErrors:expectDataWithinTime( 10, 0 )

	-- the error handler should process the ResourceExistsError and delete
	-- the favorite document since we would have synced the duplicate
	-- document through normal sync
	rxTest:expectSingleValue(
		docStore:docValue( docId1 )
			:map( function( docValue )
				return not not docValue._empty
			end ),
		true
	)

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetResourceExistsError_testNightly( rxTest )

	rxTest:setTimeLimit( 60 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog

	rxTest:waitFor( catalogTestSession:catalog()
		:tap( function( _catalog )
			catalog = _catalog
		end )
	)


	local assetId = KSIdGenerator.uuidInLowercaseHex()

	local payload = {
		captureDate = "0000-00-00T00:00:00",
		importSource = {
			fileName = 'test.jpg',
			filePath = '/path/',
			originalHeight = 150,
			originalWidth = 100,
			importedBy = KSIdGenerator.uuidInLowercaseHex(),
			importTimestamp = KSDate.timeInIso8601Format(),
		},
		develop = {
			processingModel = 'lightroom',
			xmpCameraRaw = '<xmpmeta />',
		},
	}

	local assetDocumentForRestAPI =  {
		type = 'asset',
		subtype = 'image',
		payload = payload,
	}

	local assetDocumentForSyncFeed = KSTableUtils.deepCopy( payload )
	assetDocumentForSyncFeed._id = assetId
	assetDocumentForSyncFeed.type = 'asset'
	assetDocumentForSyncFeed.subtype = 'image'


	-- create the asset via the rest API
	local assetUrl = string.format( '/v2/catalogs/%s/assets/%s/revisions/%s', catalog:ozCatalogId(), assetId, KSIdGenerator.uuidInLowercaseHex() )
	rxTest:waitFor(
		catalog:ozCatalogSpec():put( assetUrl, KSJsonEncoder.encodeOzUploadJson( assetDocumentForRestAPI ),
			{
				requestMethod = 'PUT',
				requestHeaders = {
					{ header = 'Content-Type', value = 'application/json' },
				},
			}
		)
	)
	rxTest:sleep( 2 )

	-- delete the asset via the REST API
	local assetDeleteUrl = string.format( '/v2/catalogs/%s/assets/%s', catalog:ozCatalogId(), assetId )
	rxTest:expectSingleValue(
		catalog:ozCatalogSpec():put( assetDeleteUrl, '', { requestMethod = 'DELETE', returnHttpErrorsAsData = true } )
			:map( function( result, headers )
				return headers.status_code == '204'
			end ),
		true
	)

	rxTest:sleep( 5 )

	local docStore = catalog:docStore()
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:finally( function()
		puller:close()
		pusher:close()
		errorHandler:close()
	end )

	-- start the sync processes
	rxTest:waitFor( pusher:sendUpdatesContinuously() )
	puller:getUpdatesContinuously()

	-- now try to create the asset again
	rxTest:waitFor( docStore:setDocValue( assetDocumentForSyncFeed ) )

	-- the error handler should process the ResourceExistsError and delete
	-- the asset

	local isDocValueDeleted = rxTest:observe( docStore:ongoingDocValue( assetId )
								:map( function( docValue )
									return docValue._empty
								end ) )

	isDocValueDeleted:expectDataWithinTime( 40, true )

	-- check index store for error documents, there shouldn't be any anymore
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT * FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:map( function( rows )
			return #rows
		end ) )

	outstandingErrors:expectDataWithinTime( 10, 0 )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResourceNotFoundErrorCausedByDoubleRestore( rxTest )

	-- a ResourceNotFoundError for a document that used to be on Oz should cause that
	-- document to be deleted
	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	local deletedImageId = KSIdGenerator.uuidInLowercaseHex()
	local deletedRevisionId = KSIdGenerator.uuidInLowercaseHex()

	-- create asset as it appears after first restore.
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			_labels = { oz = true },
			type = 'asset',
			subtype = 'image',
		}
	)

	rxTest:waitFor(
		docStore:applyOverlay {
			_id = docId,
			_localId = 4,
			_localOnly = {
				binaryUpload = {
				},
				files = {
					original = {
						master = {
							relativePath = "Users/chet/Pictures/Adobe Revel/Adobe Revel Library.rvl/rxTest_CT_OzDoubleRestore_testNightly-671336d875a543e0911284f0020a4b5e/IMG_7763.JPG",
							size = 2072173,
							volumeGuid = "B90F598C-3328-3F75-B314-035FD0C39493",
						},
					},
				},
				universal = {
					trash = {
						createFrom = {
							id = deletedImageId,
							lastAssetId = deletedImageId,
							lastRevisionId = deletedRevisionId,
							lastTrashPurgeDate = "2019-07-12T21:32:40.415Z",
							lastTrashUserCreated = "2019-05-13T21:32:40.415Z",
							revision_id = deletedRevisionId,
							subtype = "deleted_image",
						},
						createWithRevId = KSIdGenerator.uuidInLowercaseHex(),
						uploaded = {
							["/rels/master"] = true,
							["/rels/proxy_type/2560"] = true,
							["/rels/xmp"] = true,
						},
					},
				},
				xmpSidecarExists = false,
			},
			_serverProvidedMetadata = {
				base = string.format("https://photos-stage.adobe.io/v2/catalogs/%s/", catalogId),
				created = "2019-05-13T21:32:47.862020Z",
				original = {
					id = KSIdGenerator.uuidInLowercaseHex(),
				},
				updated = "2019-05-13T21:32:50.143303Z",
			},
		}
	)

	-- create error doc for asset that indicates the resource is not found
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			_labels = { oz = true },
			_serverProvidedMetadata = {
				new_id = docId,
				errors = {
					asset = {
						"does not exist",
					},
				},
				missing_resource_id = deletedImageId,
				source = {
					action = "create",
					id = docId,
					revision_id = revisionId,
					type = "asset",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
			subtype = "ResourceNotFoundError",
			type = "error",
		}
	)

	-- ResourceNotFoundError for asset that has already been restored by this
	-- client, as simulated by the conditions above, should be ignored.

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	-- restored asset should remain restored after error doc is removed and
	-- error handling is complete
	rxTest:expectSingleValue(
		docStore:docValue( docId )
		 :map( function( docValue )
				return docValue._deleted == nil and docValue._empty == nil
			end ),
		true
	)

end

--------------------------------------------------------------------------------

local function prefetchReqResp( catalogId )

	return WFMockHttpRequest()
		:get( "/v2/catalogs/" .. catalogId .. "/albums?subtype=collection%3Bcollection_set" )
		:respond( '200', [[{
			"base": "https://photos-stage.adobe.io/v2/catalogs/]]..catalogId ..[[/",
			"resources": []
			}]] )

end

--------------------------------------------------------------------------------

local function catalogPrefetchReqResp( catalogId )

	return WFMockHttpRequest()
		:get( "/v2/catalogs/" .. catalogId )
		:respond( '200', [[{
			"base": "https://photos-stage.adobe.io/v2/",
			"id": "]] .. catalogId .. [[",
			"type": "catalog"
			}]] )

end

--------------------------------------------------------------------------------

local function getResourceExistsJsonForCatalogAndAsset( catalogId, assetId, revisionId )
	return [[{"base":"https://photos-stage.adobe.io/v2/catalogs/]]..catalogId..[[/",
		"updates":[{"id":"6d7e7425358aa4177ed4469b134a01f1","updated":"2017-09-06T14:54:48.042095Z",
		"type":"error","status":403,"subtype":"ResourceExistsError","errors":{"asset":["already exists"]},
		"source":{"id":"]]..assetId..[[","action":"create","type":"asset",
		"revision_id":"]]..revisionId..[[","links":{"self":
		{"href":"/v2/catalogs/]]..catalogId..[[/updates/]]..assetId..[["}}}}],
		"links":{"next":{"href":"/v2/catalogs/]]..catalogId..[[/updates?updated_since=2017-09-06T14%3A54%3A48.042095Z"}}}]]
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetResourceExistsError_missingConfirmation_mock( rxTest )

	rxTest:setTimeLimit( 60 )

	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local catalogId = KSIdGenerator.uuidInLowercaseHex()

	local catalogSpec = WFCatSpecMock( catalogId, 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	catalogSpec:expectHttpRequest( prefetchReqResp( catalogId ) )
	catalogSpec:expectHttpRequest( catalogPrefetchReqResp( catalogId ) )
	catalogSpec:expectHttpRequest(
		WFMockHttpRequest()
			:get( string.format( "/v2/catalogs/%s/updates", catalogId ) )
			:respond( '200', [[{
				"base": "https://photos-stage.adobe.io/v2/catalogs/]]..catalogId..[[/",
				"updates": [],
				"links": {
					"next": {
						"href": "/v2/catalogs/]]..catalogId..[[/updates?updated_since=pull1"
					}
				}
			}]] ) )

	local docStore = catalog:docStore()
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )
	local conflictResolver = require 'WFCatConflictResolver'( catalog:docStore(), catalog:indexStore() )

	rxTest:finally( function()
		puller:close()
		pusher:close()
		errorHandler:close()
		conflictResolver:close()
	end )

	-- do an initial pull from Oz to get the prefetch out of the way and empty feed pull
	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	local oldRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local payload = {
		_id = assetId,
		_newRevId = oldRevisionId,
		type = 'asset',
		subtype = 'image',
		captureDate = "0000-00-00T00:00:00",
		importSource = {
			fileName = 'test.jpg',
			filePath = '/path/',
			originalHeight = 150,
			originalWidth = 100,
			importedBy = KSIdGenerator.uuidInLowercaseHex(),
			importTimestamp = KSDate.timeInIso8601Format(),
		},
		develop = {
			processingModel = 'lightroom',
			xmpCameraRaw = '<xmpmeta />',
		},
	}

	-- create the asset document
	rxTest:waitFor( docStore:setDocValue( payload ) )

	-- expect the pusher's updates feed request
	catalogSpec:expectHttpRequest( WFMockHttpRequest()
		:post( string.format( "/v2/catalogs/%s/updates", catalogId ) )
		:validateRequestData( function( str )
					return str:find( [["action":"create"]] ) and
						str:find( [["fileName":"test.jpg"]] ) and
						str:find( [["type":"asset"]] )
				end )
		:respond( '200', [[]] ) )

	-- send the updates with the pusher

	local sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()

	-- modify the asset document (pusher should send another create to Oz)
	rxTest:waitFor(
		docStore:applyOverlay {
			_id = assetId,
			_newRevId = revisionId,
			importSource = {
				fileName = 'test2.jpg'
			}
		}
	)

	-- expect the pusher's updates feed request (fileName should be changed)
	catalogSpec:expectHttpRequest( WFMockHttpRequest()
		:post( string.format( "/v2/catalogs/%s/updates", catalogId ) )
		:validateRequestData( function( str )
					return str:find( [["action":"create"]] ) and
						str:find( [["fileName":"test2.jpg"]] ) and
						str:find( [["type":"asset"]] )
				end )
		:respond( '200', [[]] ) )

	-- send the update to Oz - should be a create here still that will generate a ResourceExistsError
	sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()

	-- updates feed pull (no asset confirmation, just ResourceExistsError)
	catalogSpec:expectHttpRequest( WFMockHttpRequest()
		:get( string.format( "/v2/catalogs/%s/updates?updated_since=pull1", catalogId ) )
		:respond( '200', getResourceExistsJsonForCatalogAndAsset(catalogId, assetId, revisionId ) ) )

	-- after the updates feed pull the error handler should explicitly request the asset from Oz
	catalogSpec:expectHttpRequest( WFMockHttpRequest()
		:get( string.format( "/v2/catalogs/%s/updates/%s", catalogId, assetId ) )
		:respond( '200', [[{"base":"https://photos-stage.adobe.io/v2/catalogs/]]..catalogId..[[/","id":"]]..assetId..[[",
							"type":"asset","updated":"2017-09-06T18:15:10.138376Z","subtype":"image","created":"2017-09-06T18:15:10.138376Z",
							"revision_ids":["]]..oldRevisionId..[["], "links":{"self":{"href":"assets/]]..assetId..[["},
							"/rels/proxy_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/proxies/{proxy_id}","templated":true},
							"/rels/rendition_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/renditions/{rendition_id}",
							"templated":true},"/rels/master_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/master","templated":true},
							"/rels/xmp_develop_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/xmp/develop","templated":true}},
							"payload":{"captureDate":"0000-00-00T00:00:00","importSource":{"originalHeight":150,"filePath":"/path/","importTimestamp":"2017-09-06T18:15:09.432Z",
							"importedBy":"]]..KSIdGenerator.uuidInLowercaseHex()..[[","fileName":"test.jpg","originalWidth":100},"develop":{"processingModel":"lightroom",
							"xmpCameraRaw":"<xmpmeta />"}}}]] ) )

	-- get docs from Oz, but should only get a ResourceExistsError for second create, no confirmation for
	-- first create
	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	-- the error handler should process the ResourceExistsError and the asset info from Oz should
	-- now exist in the
	local doesDocValueHaveLinks = rxTest:observe( docStore:ongoingDocValue( assetId )
								:map( function( docValue )
									return not not KSTableUtils.derefOpt( docValue, '_serverProvidedMetadata', 'links' )
								end ) )

	doesDocValueHaveLinks:expectDataWithinTime( 40, true )

	-- check index store for error documents, there shouldn't be any anymore
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT * FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:map( function( rows )
			return #rows
		end ) )

	outstandingErrors:expectDataWithinTime( 10, 0 )

	-- give the conflict resolver a chance to resolve the doc from Oz
	rxTest:sleep( 3 )

	-- after getting the latest from Oz, the local modification (filename) should be resent to Oz as an update
	catalogSpec:expectHttpRequest( WFMockHttpRequest()
		:post( string.format( "/v2/catalogs/%s/updates", catalogId ) )
		:validateRequestData( function( str )
					return str:find( [["action":"update"]] ) and
						str:find( [["fileName":"test2.jpg"]] ) and
						str:find( [["type":"asset"]] )
				end )
		:respond( '200', [[]] ) )

	--rxTest:expectSingleValue( catalog:docStore():docValue( assetId ), {} )
	sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetResourceExistsError_confirmationAndErrorCoincidental_mock( rxTest )

	rxTest:setTimeLimit( 60 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()

	local catalogSpec = WFCatSpecMock( catalogId, 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local oldRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local payload = {
		_id = assetId,
		_newRevId = oldRevisionId,
		type = 'asset',
		subtype = 'image',
		captureDate = "0000-00-00T00:00:00",
		importSource = {
			fileName = 'test.jpg',
			filePath = '/path/',
			originalHeight = 150,
			originalWidth = 100,
			importedBy = KSIdGenerator.uuidInLowercaseHex(),
			importTimestamp = KSDate.timeInIso8601Format(),
		},
		develop = {
			processingModel = 'lightroom',
			xmpCameraRaw = '<xmpmeta />',
		},
	}

	local docStore = catalog:docStore()
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )
	local conflictResolver = require 'WFCatConflictResolver'( catalog:docStore(), catalog:indexStore() )

	rxTest:finally( function()
		puller:close()
		pusher:close()
		errorHandler:close()
		conflictResolver:close()
	end )

	-- do the first pull to get the album prefetch, catalog prefetch, and an empty pull) out of the way
	catalogSpec:expectHttpRequest( prefetchReqResp( catalogId ) )
	catalogSpec:expectHttpRequest( catalogPrefetchReqResp( catalogId ) )
	catalogSpec:expectHttpRequest(
		WFMockHttpRequest()
			:get( string.format( "/v2/catalogs/%s/updates", catalogId ) )
			:respond( '200', [[{
				"base": "https://photos-stage.adobe.io/v2/catalogs/]]..catalogId..[[/",
				"updates": [],
				"links": {
					"next": {
						"href": "/v2/catalogs/]]..catalogId..[[/updates?updated_since=pull1"
					}
				}
			}]] ) )
	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	-- create the asset document
	rxTest:waitFor( docStore:setDocValue( payload ) )

	-- send it
	catalogSpec:expectHttpRequest(
		WFMockHttpRequest()
			:post( string.format( "/v2/catalogs/%s/updates", catalogId ) )
			:validateRequestData( function( str )
					return str:find( [["action":"create"]] ) and
						str:find( [["fileName":"test.jpg"]] ) and
						str:find( [["type":"asset"]] )
				end )
			:respond( '200', [[]] ) )

	local sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()

	-- modify the asset document (pusher should send another create to Oz since we haven't received a confirmation)
	rxTest:waitFor(
		docStore:applyOverlay {
			_id = assetId,
			_newRevId = revisionId,
			importSource = {
				fileName = 'test2.jpg'
			}
		}
	)

	-- send the update to Oz - should be another create here that should generate a ResourceExistsError
	catalogSpec:expectHttpRequest(
		WFMockHttpRequest()
			:post( string.format( "/v2/catalogs/%s/updates", catalogId ) )
			:validateRequestData( function( str )
					return str:find( [["action":"create"]] ) and
						str:find( [["fileName":"test2.jpg"]] ) and
						str:find( [["type":"asset"]] )
				end )
			:respond( '200', [[]] ) )

	sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()

	-- we'll pull sync feed from Oz and get the original confirmation and a ResourceExistsError for second create, once we have
	-- the ResourceExistsError, the error handler should process it and it should do a get request to get the existing doc on Oz
	catalogSpec:expectHttpRequest(
		WFMockHttpRequest()
			:get( string.format( "/v2/catalogs/%s/updates?updated_since=pull1", catalogId ) )
			:respond( '200',  [[{"base":"https://photos-stage.adobe.io/v2/catalogs/]]..catalogId..[[/",
								"updates":[]]..
								-- asset confirmation
								[[{"id":"]]..assetId..[[",
								"type":"asset","updated":"2017-09-06T18:15:10.138376Z","subtype":"image","created":"2017-09-06T18:15:10.138376Z",
								"revision_ids":["]]..oldRevisionId..[["], "links":{"self":{"href":"assets/]]..assetId..[["},
								"/rels/proxy_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/proxies/{proxy_id}","templated":true},
								"/rels/rendition_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/renditions/{rendition_id}",
								"templated":true},"/rels/master_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/master","templated":true},
								"/rels/xmp_develop_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/xmp/develop","templated":true}},
								"payload":{"captureDate":"0000-00-00T00:00:00","importSource":{"originalHeight":150,"filePath":"/path/","importTimestamp":"2017-09-06T18:15:09.432Z",
								"importedBy":"]]..KSIdGenerator.uuidInLowercaseHex()..[[","fileName":"test.jpg","originalWidth":100},"develop":{"processingModel":"lightroom",
								"xmpCameraRaw":"<xmpmeta />"}}},]]..
								--ResourceExistsError
								[[{"id":"6d7e7425358aa4177ed4469b134a01f1","updated":"2017-09-06T14:54:48.042095Z",
								"type":"error","status":403,"subtype":"ResourceExistsError","errors":{"asset":["already exists"]},
								"source":{"id":"]]..assetId..[[","action":"create","type":"asset",
								"revision_id":"]]..revisionId..[[","links":{"self":
								{"href":"/v2/catalogs/]]..catalogId..[[/updates/]]..assetId..[["}}}}],
								"links":{"next":{"href":"/v2/catalogs/]]..catalogId..[[/updates?updated_since=2017-09-06T14%3A54%3A48.042095Z"}}}]] ) )


	catalogSpec:expectHttpRequest(
		WFMockHttpRequest()
			:get( string.format( "/v2/catalogs/%s/updates/%s", catalogId, assetId ) )
			:respond( '200', [[{"base":"https://photos-stage.adobe.io/v2/catalogs/]]..catalogId..[[/","id":"]]..assetId..[[",
							"type":"asset","updated":"2017-09-06T18:15:10.138376Z","subtype":"image","created":"2017-09-06T18:15:10.138376Z",
							"revision_ids":["]]..oldRevisionId..[["], "links":{"self":{"href":"assets/]]..assetId..[["},
							"/rels/proxy_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/proxies/{proxy_id}","templated":true},
							"/rels/rendition_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/renditions/{rendition_id}",
							"templated":true},"/rels/master_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/master","templated":true},
							"/rels/xmp_develop_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/xmp/develop","templated":true}},
							"payload":{"captureDate":"0000-00-00T00:00:00","importSource":{"originalHeight":150,"filePath":"/path/","importTimestamp":"2017-09-06T18:15:09.432Z",
							"importedBy":"]]..KSIdGenerator.uuidInLowercaseHex()..[[","fileName":"test.jpg","originalWidth":100},"develop":{"processingModel":"lightroom",
							"xmpCameraRaw":"<xmpmeta />"}}}]] ) )

	-- pull
	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	-- the error handler should process the ResourceExistsError and the asset info from Oz should
	-- now exist in the
	local doesDocValueHaveLinks = rxTest:observe( docStore:ongoingDocValue( assetId )
								:map( function( docValue )
									return not not KSTableUtils.derefOpt( docValue, '_serverProvidedMetadata', 'links' )
								end ) )

	doesDocValueHaveLinks:expectDataWithinTime( 40, true )

	-- check index store for error documents, there shouldn't be any anymore
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT * FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:map( function( rows )
			return #rows
		end ) )

	outstandingErrors:expectDataWithinTime( 10, 0 )

	-- give the conflict resolver a chance to resolve the doc from Oz
	rxTest:sleep( 3 )

	catalogSpec:expectHttpRequest(
		WFMockHttpRequest()
			:post( string.format( "/v2/catalogs/%s/updates", catalogId ) )
			:validateRequestData( function( str )
					return str:find( [["action":"update"]] ) and
						str:find( [["fileName":"test2.jpg"]] ) and
						str:find( [["type":"asset"]] )
				end )
			:respond( '200', [[]] ) )

	-- we should have updates to push to Oz here
	sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAlbumResourceExistsError_testNightly( rxTest )

	rxTest:setTimeLimit( 120 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog

	rxTest:waitFor( catalogTestSession:catalog()
		:tap( function( _catalog )
			catalog = _catalog
		end )
	)


	local albumId = KSIdGenerator.uuidInLowercaseHex()

	local payload = {
		name = 'ResourceExistsError Album'
	}

	local albumDocumentForRestAPI =  {
		type = 'album',
		subtype = 'collection',
		payload = payload,
	}

	local albumDocumentForSyncFeed = KSTableUtils.deepCopy( payload )
	albumDocumentForSyncFeed._id = albumId
	albumDocumentForSyncFeed.type = 'album'
	albumDocumentForSyncFeed.subtype = 'collection'


	-- create the album via the rest API
	local albumUrl = string.format( '/v2/catalogs/%s/albums/%s', catalog:ozCatalogId(), albumId )
	rxTest:waitFor(
		catalog:ozCatalogSpec():put( albumUrl, KSJsonEncoder.encodeOzUploadJson( albumDocumentForRestAPI ),
			{
				requestMethod = 'PUT',
				requestHeaders = {
					{ header = 'Content-Type', value = 'application/json' },
				},
			}
		)
	)
	rxTest:sleep( 2 )

	-- delete the album via the REST API
	rxTest:expectSingleValue(
		catalog:ozCatalogSpec():put( albumUrl, '', { requestMethod = 'DELETE', returnHttpErrorsAsData = true } )
			:map( function( result, headers )
				return headers.status_code == '204'
			end ),
		true
	)

	rxTest:sleep( 5 )

	local docStore = catalog:docStore()
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:finally( function()
		puller:close()
		pusher:close()
		errorHandler:close()
	end )

	-- now try to create the album again locally
	rxTest:waitFor( docStore:setDocValue( albumDocumentForSyncFeed ) )

	local sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()

	puller:getUpdatesContinuously()

	-- the error handler should process the ResourceExistsError and delete
	-- the album
	local isDocValueDeleted = rxTest:observe( docStore:ongoingDocValue( albumId )
								:map( function( docValue )
									return docValue._empty
								end ) )

	isDocValueDeleted:expectDataWithinTime( 40, true )

	-- check index store for error documents, there shouldn't be any anymore
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT COUNT(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )

	outstandingErrors:expectDataWithinTime( 10, 0 )

end

--------------------------------------------------------------------------------
-- SPAM [matthewj 2017-04-20]: Oz currently does not send a ResourceExistsError if an
-- album_asset id is reused after deletion and instead allows this to work. If that is
-- changed, this test would cover the ResourceExistsError handling in that case.
--function WFCatErrorHandlerTest.rxTestAlbumAssetResourceExistsError_testNightly( rxTest )
--
--	rxTest:setTimeLimit( 120 )
--
--	local catalogTestSession = WFCatTestSession( rxTest )
--
--	local catalog
--
--	rxTest:waitFor( catalogTestSession:catalog()
--		:tap( function( _catalog )
--			catalog = _catalog
--		end )
--	)
--
--
--	local albumId = KSIdGenerator.uuidInLowercaseHex()
--	local albumDocumentForRestAPI =  {
--		type = 'album',
--		subtype = 'collection',
--		payload = {
--			name = 'ResourceExistsError AlbumAsset'
--		},
--	}
--
--	-- create the album via the rest API
--	local albumUrl = string.format( '/v2/catalogs/%s/albums/%s', catalog:ozCatalogId(), albumId )
--	rxTest:waitFor(
--		catalog:ozCatalogSpec():put( albumUrl, KSJsonEncoder.encodeOzUploadJson( albumDocumentForRestAPI ),
--			{
--				requestMethod = 'PUT',
--				requestHeaders = {
--					{ header = 'Content-Type', value = 'application/json' },
--				},
--			}
--		)
--	)
--	rxTest:sleep( 2 )
--
--	-- create an asset via the rest API
--	local assetId = KSIdGenerator.uuidInLowercaseHex()
--	local assetDocumentForRestAPI = {
--		type = 'asset',
--		subtype = 'image',
--		payload = {
--			captureDate = "0000-00-00T00:00:00",
--			importSource = {
--				fileName = 'test.jpg',
--				filePath = '/path/',
--				originalHeight = 150,
--				originalWidth = 100,
--				importedBy = KSIdGenerator.uuidInLowercaseHex(),
--				importTimestamp = KSDate.timeInIso8601Format(),
--			},
--			develop = {
--				processingModel = 'lightroom',
--				xmpCameraRaw = '<xmpmeta />',
--			},
--		},
--	}
--	local assetUrl = string.format( '/v2/catalogs/%s/assets/%s/revisions/%s', catalog:ozCatalogId(), assetId, KSIdGenerator.uuidInLowercaseHex() )
--	rxTest:waitFor(
--		catalog:ozCatalogSpec():put( assetUrl, KSJsonEncoder.encodeOzUploadJson( assetDocumentForRestAPI ),
--			{
--				requestMethod = 'PUT',
--				requestHeaders = {
--					{ header = 'Content-Type', value = 'application/json' },
--				},
--			}
--		)
--	)
--
--	rxTest:sleep( 2 )
--
--	-- add the asset to the album via the rest API
--	local albumAssetId = KSIdGenerator.uuidInLowercaseHex()
--	local albumAssetForRestAPI = {
--			resources = {
--				{
--					id = assetId,
--					album_asset_id = albumAssetId,
--				}
--			}
--		}
--	local albumAssetUrl = string.format( '/v2/catalogs/%s/albums/%s/assets', catalog:ozCatalogId(), albumId )
--	rxTest:waitFor(
--		catalog:ozCatalogSpec():put( albumAssetUrl, KSJsonEncoder.encodeOzUploadJson( albumAssetForRestAPI ),
--			{
--				requestMethod = 'PUT',
--				requestHeaders = {
--					{ header = 'Content-Type', value = 'application/json' },
--				},
--			}
--		)
--	)
--
--
--	rxTest:sleep( 2 )
--
--	-- delete the album_asset via the REST API
--	local albumAssetDeleteUrl = string.format( '/v2/catalogs/%s/albums/%s/assets/%s', catalog:ozCatalogId(), albumId, assetId )
--	rxTest:expectSingleValue(
--		catalog:ozCatalogSpec():put( albumAssetDeleteUrl, '', { requestMethod = 'DELETE', returnHttpErrorsAsData = true } )
--			:map( function( result, headers )
--				return headers.status_code == '204'
--			end ),
--		true
--	)
--
--	rxTest:sleep( 2 )
--
--	-- now try to add the album_asset back with the same albumAssetId via the sync feed
--	local docStore = catalog:docStore()
--	local puller = WFCatMetadataPuller( catalog )
--	local pusher = WFCatMetadataPusher( catalog, options )
--	local errorHandler = WFCatErrorHandler( catalog, {
--		deleteAssets = bulkDelete,
--		deleteIndividualAlbum = deleteDoc } )
--
--	rxTest:finally( function()
--		puller:close()
--		pusher:close()
--		errorHandler:close()
--	end )
--
--	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )
--
--	puller:getUpdatesContinuously()
--
--	rxTest:waitFor(
--		docStore:setDocValue( {
--			_id = albumAssetId,
--			type = 'album_asset',
--			album = {
--				id = albumId,
--			},
--			asset = {
--				id = assetId,
--			}
--		} )
--	)
--
--	local sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
--	sendNUpdates:expectCompletedWithinTime( 30 )
--  sendNUpdates:dispose()
--
--	local isDocValueDeleted = rxTest:observe( docStore:ongoingDocValue( albumAssetId )
--								:map( function( docValue )
--									return docValue._empty
--								end ) )
--
--	isDocValueDeleted:expectDataWithinTime( 40, true )
--
--	local albumAssetCount = rxTest:observe( catalog:indexStore():query {
--			sql = [[SELECT COUNT(*) FROM album_asset_v2]],
--			quiesceUntilUpToDate = true,
--		}
--		:pluck( 1, 1 ) )
--
--	albumAssetCount:expectDataWithinTime( 10, 0 )
--
--	-- check index store for error documents, there shouldn't be any anymore
--	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
--			sql = [[SELECT COUNT(*) FROM errors]],
--			quiesceUntilUpToDate = true,
--		}
--		:pluck( 1, 1 ) )
--
--	outstandingErrors:expectDataWithinTime( 10, 0 )
--
--end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestDuplicateAlbumAssetResourceExistsError_testNightly( rxTest )
	-- tests that we delete an album_asset record if we try to send a duplicate album_asset record
	-- with a different album_asset id than the one that already exists on Oz

	rxTest:setTimeLimit( 120 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog

	rxTest:waitFor( catalogTestSession:catalog()
		:tap( function( _catalog )
			catalog = _catalog
		end )
	)


	local albumId = KSIdGenerator.uuidInLowercaseHex()
	local albumDocumentForRestAPI =  {
		type = 'album',
		subtype = 'collection',
		payload = {
			name = 'ResourceExistsError AlbumAsset'
		},
	}

	-- create the album via the rest API
	local albumUrl = string.format( '/v2/catalogs/%s/albums/%s', catalog:ozCatalogId(), albumId )
	rxTest:waitFor(
		catalog:ozCatalogSpec():put( albumUrl, KSJsonEncoder.encodeOzUploadJson( albumDocumentForRestAPI ),
			{
				requestMethod = 'PUT',
				requestHeaders = {
					{ header = 'Content-Type', value = 'application/json' },
				},
			}
		)
	)
	rxTest:sleep( 2 )

	-- create an asset via the rest API
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local assetDocumentForRestAPI = {
		type = 'asset',
		subtype = 'image',
		payload = {
			captureDate = "0000-00-00T00:00:00",
			importSource = {
				fileName = 'test.jpg',
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
				importedBy = KSIdGenerator.uuidInLowercaseHex(),
				importTimestamp = KSDate.timeInIso8601Format(),
			},
			develop = {
				processingModel = 'lightroom',
				xmpCameraRaw = '<xmpmeta />',
			},
		},
	}
	local assetUrl = string.format( '/v2/catalogs/%s/assets/%s/revisions/%s', catalog:ozCatalogId(), assetId, KSIdGenerator.uuidInLowercaseHex() )
	rxTest:waitFor(
		catalog:ozCatalogSpec():put( assetUrl, KSJsonEncoder.encodeOzUploadJson( assetDocumentForRestAPI ),
			{
				requestMethod = 'PUT',
				requestHeaders = {
					{ header = 'Content-Type', value = 'application/json' },
				},
			}
		)
	)

	rxTest:sleep( 2 )

	-- add the asset to the album via the rest API
	local albumAssetId = KSIdGenerator.uuidInLowercaseHex()
	local albumAssetForRestAPI = {
			resources = {
				{
					id = assetId,
					album_asset_id = albumAssetId,
				}
			}
		}
	local albumAssetUrl = string.format( '/v2/catalogs/%s/albums/%s/assets', catalog:ozCatalogId(), albumId )
	rxTest:waitFor(
		catalog:ozCatalogSpec():put( albumAssetUrl, KSJsonEncoder.encodeOzUploadJson( albumAssetForRestAPI ),
			{
				requestMethod = 'PUT',
				requestHeaders = {
					{ header = 'Content-Type', value = 'application/json' },
				},
			}
		)
	)


	rxTest:sleep( 2 )

	-- now try to add the album_asset back with the a different albumAssetId via the sync feed
	local docStore = catalog:docStore()
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:finally( function()
		puller:close()
		pusher:close()
		errorHandler:close()
	end )

	local duplicateAlbumAssetId = KSIdGenerator.uuidInLowercaseHex()
	rxTest:waitFor(
		docStore:setDocValue( {
			_id = duplicateAlbumAssetId, -- give the album_asset an id different from the existing one.
			type = 'album_asset',
			album = {
				id = albumId,
			},
			asset = {
				id = assetId,
			}
		} )
	)

	local sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()

	puller:getUpdatesContinuously()

	local isDocValueDeleted = rxTest:observe( docStore:ongoingDocValue( duplicateAlbumAssetId )
								:map( function( docValue )
									return docValue._empty
								end ) )

	isDocValueDeleted:expectDataWithinTime( 40, true )

	local albumAssetCount = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT COUNT(*) FROM album_asset_v2]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )

	albumAssetCount:expectDataWithinTime( 10, 1 )

	-- check index store for error documents, there shouldn't be any anymore
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT COUNT(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )

	outstandingErrors:expectDataWithinTime( 10, 0 )

end

--------------------------------------------------------------------------------

-- @mark RestoreAssetError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestRestoreAssetError_originatingClient_partialRestore( rxTest )

	-- A RestoreAssetError for a restore operation that originated on the current client. When
	-- a client's restore operation fails with a RestoreAssetError, the client may have already
	-- received an asset update from Oz that would otherwise indicate to the client that Oz
	-- successfully processed the restore. The RestoreAssetError indicates it ultimately failed
	-- and that the client needs to retry. In order to do that, the client error handler needs
	-- to clean up the data that makes it look to the client like it succeeded and ensure that
	-- the asset doc is updated so that the WFCatMetadataPusher will pick it up again and resend
	-- the restore.
	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()


	local trashAssetDocId = KSIdGenerator.uuidInLowercaseHex()
	local trashAssetRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local restoredDocId = KSIdGenerator.uuidInLowercaseHex()
	local restoredRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create a basic restored asset doc locally which includes a createFrom section indicating it's
	-- a restore initiated from this client
	rxTest:waitFor(
		docStore:setDocValue {
			_id = restoredDocId,
			_newRevId = restoredRevisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			},
			_localOnly = {
				universal = {
					trash = {
						createFrom = {
							id = trashAssetDocId,
							lastAssetId = trashAssetDocId,
							lastRevisionId = trashAssetRevisionId,
							lastTrashPurgeDate = "2019-07-12T21:32:40.415Z",
							lastTrashUserCreated = "2019-05-13T21:32:40.415Z",
							revision_id = trashAssetRevisionId,
							subtype = "deleted_image",
						},
						createWithRevId = restoredRevisionId,
					}
				}
			}
		}
	)

	-- update with partially restored doc from Oz
	-- oz labeled doc

	rxTest:waitFor(
		docStore:setDocValue {
			_labels = {
				oz = true
			},
			_id = restoredDocId,
			_newRevId = restoredRevisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			},
			_serverProvidedMetadata = {
				links = {
					[ "/rels/master" ] = "fakelink"
				}
			}
		}
	)

	-- make sure we have a doc labeled 'oz'
	rxTest:expectSingleValue( docStore:getRevisionForLabel( restoredDocId, 'oz' ), restoredRevisionId )

	local prevDocChangeCounter = 0
	rxTest:expectSingleValue( docStore:docValue( restoredDocId )
		:map( function( docValue )
			prevDocChangeCounter = docValue._changeCounter
			return true
		end ), true )

	-- add RestoreAssetError doc for asset that indicates the restore failed and asset is not
	-- actually on Oz
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "RestoreAssetError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"restore failed",
					},
				},
				source = {
					action = "create",
					id = restoredDocId,
					revision_id = restoredRevisionId,
					type = "asset",
					restore_from = {
						id = trashAssetDocId
					}
				},
				status = 502,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )


	-- restore asset doc needs to be retried, so we need to make sure it has new sequence +
	-- no Oz label so the pusher will resend it to Oz as a create
	rxTest:expectSingleValue( docStore:getRevisionForLabel( restoredDocId, 'oz' ):catch( Rx.Observable.just(nil) ), nil )

	local docChangeCounter = rxTest:observe( docStore:ongoingDocValue( restoredDocId )
		:map( function( restoredDocValue )
			return restoredDocValue._changeCounter > prevDocChangeCounter
		end ) )
	docChangeCounter:expectDataWithinTime( 2, true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestRestoreAssetError_originatingClient_noPartialRestore( rxTest )

	-- A RestoreAssetError for a restore operation that originated on the current
	-- client.

	-- This test validates restoreAssetError handler properly handles the case
	-- where a doc update with a partial restore of the asset was not pulled
	-- down.

	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local trashAssetDocId = KSIdGenerator.uuidInLowercaseHex()
	local trashAssetRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local restoredDocId = KSIdGenerator.uuidInLowercaseHex()
	local restoredRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create a basic restored asset doc locally which includes a createFrom section indicating it's
	-- a restore initiated from this client
	rxTest:waitFor(
		docStore:setDocValue {
			_id = restoredDocId,
			_newRevId = restoredRevisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			},
			_localOnly = {
				universal = {
					trash = {
						createFrom = {
							id = trashAssetDocId,
							lastAssetId = trashAssetDocId,
							lastRevisionId = trashAssetRevisionId,
							lastTrashPurgeDate = "2019-07-12T21:32:40.415Z",
							lastTrashUserCreated = "2019-05-13T21:32:40.415Z",
							revision_id = trashAssetRevisionId,
							subtype = "deleted_image",
						},
						createWithRevId = restoredRevisionId,
					}
				}
			}
		}
	)

	local prevDocChangeCounter = 0
	rxTest:expectSingleValue( docStore:docValue( restoredDocId )
		:map( function( docValue )
			prevDocChangeCounter = docValue._changeCounter
			return true
		end ), true )

	-- Add RestoreAssetError doc for asset that indicates the restore failed.
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "RestoreAssetError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"restore failed",
					},
				},
				source = {
					action = "create",
					id = restoredDocId,
					revision_id = restoredRevisionId,
					type = "asset",
					restore_from = {
						id = trashAssetDocId
					}
				},
				status = 502,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
		:dispose()

	-- restore asset doc needs to be retried, so we need to make sure it has new sequence +
	-- no Oz label so the pusher will resend it to Oz as a create
	rxTest:expectSingleValue( docStore:getRevisionForLabel( restoredDocId, 'oz' ):catch( Rx.Observable.just(nil) ), nil )

	rxTest:observe( docStore:ongoingDocValue( restoredDocId )
		:map( function( restoredDocValue )
			return restoredDocValue._changeCounter > prevDocChangeCounter
		end ) )
	:expectDataWithinTime( 2, true )
	:dispose()

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestRestoreAssetError_otherClient_partialRestore( rxTest )
	-- A RestoreAssetError for a restore operation that originated on a different client.
	-- When Oz attempts a restore initiated by another client, it may get partway through
	-- the restore and then encounter an error. Due to the current implementation on Oz,
	-- this may result in an asset doc being sent to all clients (which may include binary
	-- links) that is then removed from Oz because the restore failed. In these cases, Oz
	-- does not send a tombstone for the partially restored doc but instead sends a
	-- RestoreAssetError. Because may have sent a partial document and no tombstone clients
	-- that did not initiate the restore need to treat the RestoreAssetError somewhat like a
	-- tombstone. But instead of issuing a delete (the restored asset may eventually come down
	-- correctly from Oz) we need to remove any _serverProvidedMetadata that indicates it
	-- may have data on Oz.
	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()


	local trashAssetDocId = KSIdGenerator.uuidInLowercaseHex()
	local restoredDocId = KSIdGenerator.uuidInLowercaseHex()
	local restoredRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- update with partially restored doc from Oz
	-- oz labeled doc

	rxTest:waitFor(
		docStore:setDocValue {
			_labels = {
				oz = true
			},
			_id = restoredDocId,
			_newRevId = restoredRevisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			},
			_serverProvidedMetadata = {
				links = {
					[ "/rels/master" ] = "fakelink"
				}
			}
		}
	)

	-- make sure we have a doc labeled 'oz'
	rxTest:expectSingleValue( docStore:getRevisionForLabel( restoredDocId, 'oz' ), restoredRevisionId )

	-- add RestoreAssetError doc for asset that indicates the restore failed and asset is not
	-- actually on Oz
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "RestoreAssetError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"restore failed",
					},
				},
				source = {
					action = "create",
					id = restoredDocId,
					revision_id = restoredRevisionId,
					type = "asset",
					restore_from = {
						id = trashAssetDocId
					}
				},
				status = 502,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )


	-- Partially restored asset doc needs to be cleaned up so that it doesn't have enough info to be shown in the client. This
	-- means removing the _serverProvidedMetadata. Unlike the originating client though, we want to keep the 'oz' label or
	-- else the current client is going to try to send this new doc back to Oz as a create (not a restore) and it has none of the
	-- binaries for the asset so it can't create anything real.
	rxTest:expectSingleValue( docStore:getRevisionForLabel( restoredDocId, 'oz' ), restoredRevisionId )

	local restoredDocValueHasNoOzData = rxTest:observe( docStore:ongoingDocValue( restoredDocId )
		:map( function( restoredDocValue )
			return restoredDocValue._serverProvidedMetadata == nil
		end ) )
	restoredDocValueHasNoOzData:expectDataWithinTime( 2, true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestRestoreAssetError_otherClient_noPartialRestore( rxTest )

	-- A RestoreAssetError for a restore operation that originated on a different client.

	-- This test validates restoreAssetError handler properly handles the case
	-- where a doc update with a partial restore of the asset was not pulled
	-- down.

	rxTest:setTimeLimit( 5 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
	{
		deleteAssets = bulkDelete,
		deleteIndividualAlbum = deleteDoc
	},
	errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local trashAssetDocId = KSIdGenerator.uuidInLowercaseHex()
	local restoredDocId = KSIdGenerator.uuidInLowercaseHex()
	local restoredRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- Add RestoreAssetError doc for asset that indicates a restore failed.
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "RestoreAssetError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"restore failed",
					},
				},
				source = {
					action = "create",
					id = restoredDocId,
					revision_id = restoredRevisionId,
					type = "asset",
					restore_from = {
						id = trashAssetDocId
					}
				},
				status = 502,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- Error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
	:expectDataWithinTime( 3, true )
	:dispose()

	-- Still shouldn't have a restored doc value.
	rxTest:expectSingleValue(
		docStore:docValue( restoredDocId ),
		{
			_empty = true,
			_id = restoredDocId,
		}
	)

end

--------------------------------------------------------------------------------

-- @mark Aux and Xmp Errors

--------------------------------------------------------------------------------

local function testSha256MismatchError( rxTest, catalog, errorName, docId, revisionId, sha256, binaryPath, errorCheckFn, docType )

	docType = docType or 'asset'

	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	local expectedSha256 = KSIdGenerator.uuidInLowercaseHex() .. KSIdGenerator.uuidInLowercaseHex()
	local actualSha256 = KSIdGenerator.uuidInLowercaseHex() .. KSIdGenerator.uuidInLowercaseHex()

	local docStore = catalog:docStore()

	-- used to wait until error document is deleted from errors table before refreshing the
	-- doc store with a new error for the same asset with the same id. Because of the way
	-- that Oz generates errors, this is actually something that can happen, so good to test!

	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT count(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )

	-- create error doc for asset with sha256 that is different than the local sha256 (error should be ignored)

	local errorDoc = {
		_id = errorDocId,
		subtype = errorName,
		type = "error",
		_serverProvidedMetadata = {
			errors = {
				[ expectedSha256 ] = { "sha256 mismatch", },
			},
			source = {
				id = docId,
				type = docType,
				expected_sha256 = expectedSha256,
				actual_sha256 = actualSha256,
				revision_id = revisionId
			},
			status = 403,
			updated = "2016-06-07T15:55:34.218872Z",
		}
	}
	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
		:dispose()

	-- entry should be removed from errors table
	outstandingErrors:expectDataWithinTime( 3, 0 )

	-- doc should NOT end up with an invalid aux/xmpDevelop error because the
	-- error doc refers to a binary that the local document isn't aware of.
	rxTest:expectSingleValue( docStore:docValue( docId ):map( errorCheckFn ), false )

	-- apply a new revision to asset to test that sha256 mismatch error is still
	-- handled even when WF has revisions not yet syncd to Oz.
	rxTest:waitFor( docStore:applyOverlay {
		_id = docId,
		_newRevId = KSIdGenerator.uuidInLowercaseHex(),
		ratings = {
			[ KSIdGenerator.uuidInLowercaseHex() ] = {
				rating = 3,
			},
		},
	} )


	-- modify error doc so it has the same sha256 stored locally... now it should
	-- be ignored because local sha256 is correct and retry should happen

	errorDoc._serverProvidedMetadata.source.expected_sha256 = sha256
	errorDoc._serverProvidedMetadata.errors = { [ sha256 ] = { "sha256 mismatch" } }
	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
		:dispose()

	-- entry should be removed from errors table
	outstandingErrors:expectDataWithinTime( 3, 0 )

	-- doc should NOT end up with an invalid aux/xmpDevelop error because the
	-- the sha256 is correct locally.
	rxTest:expectSingleValue( docStore:docValue( docId ):map( errorCheckFn ), false )

	-- confirm that binary still exists in same location... if sha256 did not match,
	-- then error handler would have renamed file
	rxTest:expectEqual( 'file', WFFileUtils.exists( binaryPath ) )


	-- Now make the local binary appear corrupt by modifying it
	WFFileUtils.delete( binaryPath )
	local corruptData = KSData.make( KSIdGenerator.uuidInLowercaseHex() )
	KSData.saveToFile( corruptData, binaryPath  )
	corruptData:close()

	-- same error should now trigger error because local sha256 is not as expected
	-- and file is assuemd to be corrupted

	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
		:dispose()

	-- doc should be removed from errors table
	outstandingErrors:expectDataWithinTime( 3, 0 )

	-- doc should end up with an invalid aux/xmpDevelop error
	rxTest:expectSingleValue( docStore:docValue( docId ):map( errorCheckFn ), true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAuxSha256MismatchError( rxTest )

	rxTest:setTimeLimit( 10 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )
	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()

	-- create asset doc
	local overrides = { hasMasks = true, noMaskLinks = true }
	local docValue, expectedData = WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, docId, overrides )
	local revisionId = docValue._newRevId

	rxTest:waitFor( docStore:setDocValue( docValue ) )

	testSha256MismatchError( rxTest, catalog, "AuxSha256MismatchError", docId, revisionId,
		expectedData.mask1Sha256, expectedData.mask1FilePath,
		WFCatAssetAux.hasLocalAuxErrorForCurrentDevelopSettings )

	-- check specific mask aux 1 is marked with error
	rxTest:expectSingleValue(
		docStore:docValue( docId )
			:map( function( docValue )
				return WFCatAssetAux.hasLocalAuxError( docValue, expectedData.mask1AuxId )
			end ), true )

	-- check specific mask aux 1 is NOT marked with error
	rxTest:expectSingleValue(
		docStore:docValue( docId )
			:map( function( docValue )
				return WFCatAssetAux.hasLocalAuxError( docValue, expectedData.mask2AuxId )
			end ), false )

	-- The mask aux 1 file should have been moved
	rxTest:expectEqual( false, WFFileUtils.exists( expectedData.mask1FilePath ) )
	local invalidPath = string.format( "%s_invalidAux", expectedData.mask1FilePath )
	rxTest:expectEqual( 'file', WFFileUtils.exists( invalidPath ) )

	-- The mask aux 2 file should NOT have been moved
	rxTest:expectEqual( 'file', WFFileUtils.exists( expectedData.mask2FilePath ) )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestXmpDevelopSha256MismatchError( rxTest )

	rxTest:setTimeLimit( 10 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local optOverrides = {
		noDevelopSettingsLink = true,
	}

	-- for asset

	local docId = KSIdGenerator.uuidInLowercaseHex()

	-- create asset doc
	local docValue, expectedData = WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, docId, optOverrides )
	local revisionId = docValue._newRevId

	rxTest:waitFor( docStore:setDocValue( docValue ) )

	local function errorCheck(docValue)
		return not not KSTableUtils.derefOpt( docValue, '_localOnly', 'files', 'xmp_develop', 'error' )
	end

	testSha256MismatchError( rxTest, catalog, "XmpDevelopSha256MismatchError", docId, revisionId,
		expectedData.developSettingsSha256, expectedData.xmpDevelopSettingsFilePath, errorCheck )

	-- The xmpDevelop file should have been moved
	rxTest:expectEqual( false, WFFileUtils.exists( expectedData.xmpDevelopSettingsFilePath ) )
	local invalidPath = string.format( "%s_invalidXmpDevelop", expectedData.xmpDevelopSettingsFilePath )
	rxTest:expectEqual( 'file', WFFileUtils.exists( invalidPath ) )

	-- for version

	docId = KSIdGenerator.uuidInLowercaseHex()

	-- create version doc
	docValue, expectedData = WFCatVersionTestUtils.createVersionDocument( rxTest, catalog, docId, optOverrides )
	revisionId = nil

	rxTest:waitFor( docStore:setDocValue( docValue ) )

	testSha256MismatchError( rxTest, catalog, "XmpDevelopSha256MismatchError", docId, revisionId,
		expectedData.developSettingsSha256, expectedData.xmpDevelopSettingsFilePath, errorCheck, 'version' )

	-- The xmpDevelop file should have been moved
	rxTest:expectEqual( false, WFFileUtils.exists( expectedData.xmpDevelopSettingsFilePath ) )
	invalidPath = string.format( "%s_invalidXmpDevelop", expectedData.xmpDevelopSettingsFilePath )
	rxTest:expectEqual( 'file', WFFileUtils.exists( invalidPath ) )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAuxMissingError( rxTest )

	rxTest:setTimeLimit( 10 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc

	local overrides = {
		hasMasks = true,
		noMaskLinks = true
	}

	local docValue = WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, docId, overrides )
	revisionId = docValue._newRevId

	rxTest:waitFor( docStore:setDocValue( docValue ) )

	-- create error doc for asset

	local missingAuxId1 = KSIdGenerator.uuidInLowercaseHex()
	local missingAuxId2 = KSIdGenerator.uuidInLowercaseHex()
	local errorDoc = {
		_id = errorDocId,
		subtype = "AuxMissingError",
		type = "error",
		_serverProvidedMetadata = {
			errors = {
				missing_aux_ids = {
					missingAuxId1, missingAuxId2,
				},
			},
			source = {
				id = docId,
				type = "asset",
				revision_id = revisionId
			},
			status = 403,
			updated = "2016-06-07T15:55:34.218872Z",
		}
	}

	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- The error doc should be deleted.

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	-- The error doc should not add any error annotations to the asset.

	rxTest:expectSingleValue(
		docStore:docValue( docId )
			:map( function( docValue )
				return WFCatAssetAux.hasLocalAuxErrorForCurrentDevelopSettings( docValue )
			end ), false )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestReferencedAuxMissingError_Basic( rxTest )

	rxTest:setTimeLimit( 15 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, nil, indexDefSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
			getSyncStatusOfAssetsOfAlbumAssets = catSyncStatus.getSyncStatusOfAssetsOfAlbumAssets,
			getSyncStatusOfAlbumsOfAlbumAssets = catSyncStatus.getSyncStatusOfAlbumsOfAlbumAssets,
			getSyncStatusOfParentCollectionOfAlbums = catSyncStatus.getSyncStatusOfParentCollectionOfAlbums,
			getSyncStatusOfAssetsOfVersions = catSyncStatus.getSyncStatusOfAssetsOfVersions,
			getSyncStatusOfAssetsOfComments = catSyncStatus.getSyncStatusOfAssetsOfComments,
			getSyncStatusOfAssetsOfFavorites = catSyncStatus.getSyncStatusOfAssetsOfFavorites,
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local rootDocId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc that has not been synced to Oz. It does not have
	-- any auxiliaries.

	local docValue = {
		_id = rootDocId,
		_newRevId = revisionId,
		type = 'asset',
		subtype = 'image',
		importSource = {
			filePath = '/path/',
			originalHeight = 150,
			originalWidth = 100,
		}
	}

	docValue = makeAssetDocDisplayable( docValue )

	rxTest:waitFor( docStore:setDocValue( docValue ) )

	local dependentDocId = KSIdGenerator.uuidInLowercaseHex()
	local dependentRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local dependentErrorDocId = KSIdGenerator.uuidInLowercaseHex()
	local auxId1 = KSIdGenerator.uuidInLowercaseHex()
	local auxDigest1 = KSIdGenerator.uuidInLowercaseHex()
	local auxId2 = KSIdGenerator.uuidInLowercaseHex()
	local auxDigest2 = KSIdGenerator.uuidInLowercaseHex()

	-- create the dependent doc that has auxiliaries

	rxTest:waitFor(
		docStore:setDocValue {
			_id = dependentDocId,
			_newRevId = dependentRevisionId,
			asset = {
				id = rootDocId,
			},
			develop = {
				masks = {
					[ auxId1 ] = { digest = auxDigest1 },
					[ auxId2 ] = { digest = auxDigest2 },
				}
			},
			type = "version",
			subtype = "edit",
		} )

	-- set dependent error doc

	rxTest:waitFor(
		docStore:setDocValue {
			_id = dependentErrorDocId,
			subtype = "ReferencedAuxMissingError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					aux_ids = {
						'aaa',
						'bbb'
					},
				},
				missing_resource_id = rootDocId,
				source = {
					action = "create",
					id = dependentDocId,
					revision_id = dependentRevisionId,
					type = "version",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		} )

	-- error doc should be deleted

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, dependentErrorDocId ) )
		:expectDataWithinTime( 5, true )

	-- Verify the Version now has the ReferencedAuxMissingError annotation. This
	-- error is not recoverable.

	rxTest:expectSingleValue(
		docStore:docValue( dependentDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasReferencedAuxMissingError( docValue )
			end ), true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestReferencedAuxMissingError_NoMissingAux( rxTest )

	rxTest:setTimeLimit( 15 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, nil, indexDefSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
			getSyncStatusOfAssetsOfAlbumAssets = catSyncStatus.getSyncStatusOfAssetsOfAlbumAssets,
			getSyncStatusOfAlbumsOfAlbumAssets = catSyncStatus.getSyncStatusOfAlbumsOfAlbumAssets,
			getSyncStatusOfParentCollectionOfAlbums = catSyncStatus.getSyncStatusOfParentCollectionOfAlbums,
			getSyncStatusOfAssetsOfVersions = catSyncStatus.getSyncStatusOfAssetsOfVersions,
			getSyncStatusOfAssetsOfComments = catSyncStatus.getSyncStatusOfAssetsOfComments,
			getSyncStatusOfAssetsOfFavorites = catSyncStatus.getSyncStatusOfAssetsOfFavorites,
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local rootDocId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local auxId1 = KSIdGenerator.uuidInLowercaseHex()
	local auxDigest1 = KSIdGenerator.uuidInLowercaseHex()
	local auxId2 = KSIdGenerator.uuidInLowercaseHex()
	local auxDigest2 = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc that has not been synced to Oz. It has
	-- auxiliaries aaa and bbb.

	local docValue = {
		_id = rootDocId,
		_newRevId = revisionId,
		type = 'asset',
		subtype = 'image',
		importSource = {
			filePath = '/path/',
			originalHeight = 150,
			originalWidth = 100,
		},
		aux = {
			[ auxId1 ] = { digest = auxDigest1 },
			[ auxId2 ] = { digest = auxDigest2 },
		}
	}

	docValue = makeAssetDocDisplayable( docValue )

	rxTest:waitFor( docStore:setDocValue( docValue ) )

	local dependentDocId = KSIdGenerator.uuidInLowercaseHex()
	local dependentRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local dependentErrorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create the dependent doc that has auxiliaries 'aaa' and 'bbb'

	rxTest:waitFor(
		docStore:setDocValue {
			_id = dependentDocId,
			_newRevId = dependentRevisionId,
			asset = {
				id = rootDocId,
			},
			develop = {
				masks = {
					[ auxId1 ] = { digest = auxDigest1 },
					[ auxId2 ] = { digest = auxDigest2 },
				}
			},
			type = "version",
			subtype = "edit",
		} )

	-- set dependent error doc

	rxTest:waitFor(
		docStore:setDocValue {
			_id = dependentErrorDocId,
			subtype = "ReferencedAuxMissingError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					aux_ids = { auxId1, auxId2, },
				},
				missing_resource_id = rootDocId,
				source = {
					action = "create",
					id = dependentDocId,
					revision_id = dependentRevisionId,
					type = "version",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		} )

	-- error doc should be deleted

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, dependentErrorDocId ) )
		:expectDataWithinTime( 5, true )

	-- Now the Version should have a ResourceNotFoundError error. This is recoverable.

	rxTest:expectSingleValue(
		docStore:docValue( dependentDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end ), true )

	-- Make the asset appear synced.

	local overlay = {
		_id = rootDocId,
		_serverProvidedMetadata = {
			links = {
				[ '/rels/master'] = {
					href = "/v2/...."
				}
			},
			revision_ids = {}
		}
	}

	rxTest:waitFor( docStore:applyOverlay( overlay ) )

	-- Once the asset is synced, the resource not found error should
	-- be removed from the dependent doc.

	rxTest:observe( docStore:ongoingDocValue( dependentDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end )
		)
		:expectDataWithinTime( 15, false )
		:dispose()

end

--------------------------------------------------------------------------------

-- FIXME [jandrew 2025-03-11]: Disabled due to instability.

function WFCatErrorHandlerTest.DISABLED_rxTestBinaryMissingError( rxTest )

	rxTest:setTimeLimit( 10 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, nil, indexDefSpec )

	local errorHandler = WFCatErrorHandler( catalog, {},
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local assetDoc = WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, assetId )
	rxTest:waitFor( docStore:setDocValue( assetDoc ) )

	rxTest:expectSingleValue( docStore:docValue( assetId )
		:map( function( docValue )
			return KSTableUtils.derefOpt( docValue, '_localOnly', 'ozError' )
		end ),
	nil )

	-- create error BinaryMissingError

	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	local errorDoc = {
		_id = errorDocId,
		subtype = 'BinaryMissingError',
		type = "error",
		_serverProvidedMetadata = {
			errors = {
				missing_aux_ids = { KSIdGenerator.uuidInLowercaseHex() },
			},
			source = {
				id = assetId,
				action = "update",
				type = "asset",
				revision_id = assetDoc._newRevId,
			},
			status = 404,
		},
	}
	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- error doc should be deleted

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 5, true )

	-- the asset should have had error annotation set in _localOnly to trigger
	-- metadata pusher to send to Oz again

	assetDoc = rxTest:getFirstValue( docStore:docValue( assetId ) )
	rxTest:expectEqual( "BinaryMissingError", assetDoc._localOnly.ozError.subtype )
	rxTest:expectEqual( 404, assetDoc._localOnly.ozError.status )
	rxTest:expectEqual( 1, assetDoc._localOnly.ozError.retries )

	-- grab asset's _changeCounter so we can confirm it has or has not changed

	local assetChangeCounter = rxTest:getFirstValue( docStore:docValue( assetId )
		:pluck( '_changeCounter' ) )

	-- now test with a version document

	local versionId = KSIdGenerator.uuidInLowercaseHex()
	local versionDoc, localOnly = WFCatVersionTestUtils.createVersionDocument( rxTest, catalog, versionId )
	versionDoc.asset.id = assetId
	rxTest:waitFor( docStore:setDocValue( versionDoc ) )

	-- create error BinaryMissingError

	local sha256 = KSIdGenerator.uuidInLowercaseHex() .. KSIdGenerator.uuidInLowercaseHex()
	errorDocId = KSIdGenerator.uuidInLowercaseHex()
	errorDoc._id = errorDocId
	local spm = errorDoc._serverProvidedMetadata
	spm.errors = {
		[ "xmp/develop" ] = { { sha256 = sha256 } }
	}
	spm.source = {
		id = versionId,
		action = "create",
		type = "version",
	}
	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- error doc should be deleted

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 5, true )

	-- the version should have had error annotation set in _localOnly to trigger
	-- metadata pusher to send to Oz again

	versionDoc = rxTest:getFirstValue( docStore:docValue( versionId ) )
	rxTest:expectEqual( "BinaryMissingError", versionDoc._localOnly.ozError.subtype )
	rxTest:expectEqual( 404, versionDoc._localOnly.ozError.status )
	rxTest:expectEqual( 1, versionDoc._localOnly.ozError.retries )

	-- asset doc should not have changed

	rxTest:expectSingleValue(
		docStore:docValue( assetId )
			:pluck( '_changeCounter' ),
		assetChangeCounter )

	-- missing_aux_ids should not trigger extra processing if version has no aux

	errorDocId = KSIdGenerator.uuidInLowercaseHex()
	errorDoc._id = errorDocId
	local auxIds = { KSIdGenerator.uuidInLowercaseHex(), KSIdGenerator.uuidInLowercaseHex() }
	spm.errors = {
		missing_aux_ids = auxIds
	}
	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- error doc should be deleted

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 5, true )

	-- asset doc should not have changed

	rxTest:expectSingleValue(
		docStore:docValue( assetId )
			:pluck( '_changeCounter' ),
		assetChangeCounter )

	-- missing_aux_ids should not trigger extra processing if version is missing aux or
	-- missing local only info for aux

	local groupDigest = KSIdGenerator.uuidInLowercaseHex()

	rxTest:waitFor( docStore:applyOverlay {
		_id = versionId,
		develop = {
			masks = {
				[ auxIds[ 1 ] ] = { digest = KSIdGenerator.uuidInLowercaseHex() }
			},
			filters = {
				[ groupDigest ] = {
					[ auxIds[ 2 ] ] = {
						digest = KSIdGenerator.uuidInLowercaseHex(),
						sourceLongEdge = 6000,
                    	crsSizeX = 1024,
						crsSizeY = 768
					}
				}
			}
		}
	} )

	errorDocId = KSIdGenerator.uuidInLowercaseHex()
	errorDoc._id = errorDocId
	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- error doc should be deleted

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 5, true )

	-- asset doc should not have changed

	rxTest:expectSingleValue(
		docStore:docValue( assetId )
			:pluck( '_changeCounter' ),
		assetChangeCounter )

	-- missing_aux_ids should not trigger extra processing if version has aux
	-- info including local file path info but local file is missing for any
	-- missing aux

	versionId = KSIdGenerator.uuidInLowercaseHex()
	versionDoc, localOnly = WFCatVersionTestUtils.createVersionDocument( rxTest, catalog, versionId, { hasDevelopAux = true } )
	versionDoc.asset.id = assetId
	rxTest:waitFor( docStore:setDocValue( versionDoc ) )

	WFFileUtils.deleteFile( localOnly.mask1FilePath )
	--WFFileUtils.deleteFile( localOnly.filter1FilePath )

	errorDocId = KSIdGenerator.uuidInLowercaseHex()
	errorDoc._id = errorDocId
	spm.errors = {
		missing_aux_ids = { localOnly.mask1AuxId, localOnly.mask2AuxId, localOnly.filter1AuxId, localOnly.filter2AuxId }
	}
	spm.source.id = versionId

	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- error doc should be deleted

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 5, true )

	-- asset doc should not have changed

	rxTest:expectSingleValue(
		docStore:docValue( assetId )
			:pluck( '_changeCounter' ),
		assetChangeCounter )

	-- missing_aux_ids should not trigger extra processing if version has complete
	-- data for only one aux and no data on other aux

	rxTest:waitFor( docStore:applyOverlay {
		_id = versionId,
		develop = {
			masks = {
				[ localOnly.mask1AuxId ] = { _delete = true },
			},
			filters = {
				[ localOnly.filterGroupFingerprint ] = {
					[ localOnly.filter1AuxId ] = { _delete = true },
				}
			}
		}
	} )

	errorDocId = KSIdGenerator.uuidInLowercaseHex()
	errorDoc._id = errorDocId
	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- error doc should be deleted

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 5, true )

	-- asset doc should not have changed

	rxTest:expectSingleValue(
		docStore:docValue( assetId )
			:pluck( '_changeCounter' ),
		assetChangeCounter )

	-- missing_aux_ids should try to look up asset locally and
	-- fail cleanly if asset does not exist locally

	versionId = KSIdGenerator.uuidInLowercaseHex()
	versionDoc, localOnly = WFCatVersionTestUtils.createVersionDocument( rxTest, catalog, versionId, { hasDevelopAux = true } )
	rxTest:waitFor( docStore:setDocValue( versionDoc ) )

	errorDocId = KSIdGenerator.uuidInLowercaseHex()
	errorDoc._id = errorDocId
	spm.errors = {
		missing_aux_ids = { localOnly.mask1AuxId, localOnly.mask2AuxId, localOnly.filter1AuxId, localOnly.filter2AuxId }
	}
	spm.source.id = versionId
	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- error doc should be deleted

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 5, true )

	-- asset should continue not to exist

	rxTest:expectSingleValue(
		docStore:docValue( versionDoc.asset.id )
			:map( function( emptyDoc )
				return emptyDoc._empty
			end ), true )

	-- missing_aux_ids should copy to asset if all info is available
	-- and asset does not have any aux info

	rxTest:waitFor( docStore:applyOverlay {
		_id = versionId,
		asset = { id = assetId }
	} )

	errorDocId = KSIdGenerator.uuidInLowercaseHex()
	errorDoc._id = errorDocId

	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- Verify the missing aux have been copied to the asset.

	rxTest:expectSingleValue(
		docStore:ongoingDocValue( assetId )
			:map( function( docValue )
				local aux = docValue.aux or {}
				return aux[ localOnly.mask1AuxId ] and aux[ localOnly.mask2AuxId ] 
					and aux[ localOnly.filter1AuxId ] and aux[ localOnly.filter2AuxId ] 
					and true
			end )
			:filter( function( missingAuxPresent )
				return missingAuxPresent
			end )
			:take( 1 ),
		true )

	-- Now make the aux on the asset appear synced, as the error
	-- document won't be deleted until this is true.

	local assetOverlay = {
		_id = assetId
	}

	local aux1Rels = string.format( "/rels/aux/%s", localOnly.mask1AuxId )
	local aux2Rels = string.format( "/rels/aux/%s", localOnly.mask2AuxId )
	local aux3Rels = string.format( "/rels/aux/%s", localOnly.filter1AuxId )
	local aux4Rels = string.format( "/rels/aux/%s", localOnly.filter2AuxId )

	KSTableUtils.setNested( assetOverlay, 'LINK', '_serverProvidedMetadata', 'links', aux1Rels, 'href' )
	KSTableUtils.setNested( assetOverlay, 'LINK', '_serverProvidedMetadata', 'links', aux2Rels, 'href' )
	KSTableUtils.setNested( assetOverlay, 'LINK', '_serverProvidedMetadata', 'links', aux3Rels, 'href' )
	KSTableUtils.setNested( assetOverlay, 'LINK', '_serverProvidedMetadata', 'links', aux4Rels, 'href' )

	rxTest:waitFor( docStore:applyOverlay( assetOverlay ) )

	local versionChangeCounter = rxTest:getFirstValue( docStore:docValue( versionId )
		:pluck( '_changeCounter' ) )

	-- error doc should be deleted

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 5, true )

	-- asset doc should have updated

	local function assetUpdatedAndHasAllDevelopAux( docValue )

		local aux1 = KSTableUtils.derefOpt( docValue, 'aux', localOnly.mask1AuxId )
		local aux2 = KSTableUtils.derefOpt( docValue, 'aux', localOnly.mask2AuxId )
		local aux3 = KSTableUtils.derefOpt( docValue, 'aux', localOnly.filter1AuxId )
		local aux4 = KSTableUtils.derefOpt( docValue, 'aux', localOnly.filter2AuxId )
		if aux1 == nil or aux2 == nil or aux3 == nil or aux4 == nil then
			return false
		end

		return docValue._changeCounter ~= assetChangeCounter and

			aux1.digest == localOnly.mask1Fingerprint and
			aux1.fileSize == localOnly.mask1FileSize and
			aux1.sha256 == localOnly.mask1Sha256 and
			WFCatAssetAux.getAuxLocalOnlyFilePath( docValue, aux1.digest ) ==
				WFCatAssetAux.getAuxLocalOnlyFilePath( versionDoc, aux1.digest ) and

			aux2.digest == localOnly.mask2Fingerprint and
			aux2.fileSize == localOnly.mask2FileSize and
			aux2.sha256 == localOnly.mask2Sha256 and
			WFCatAssetAux.getAuxLocalOnlyFilePath( docValue, aux2.digest ) ==
				WFCatAssetAux.getAuxLocalOnlyFilePath( versionDoc, aux2.digest ) and 

			aux3.digest == localOnly.filter1Fingerprint and
			aux3.fileSize == localOnly.filter1FileSize and
			aux3.sha256 == localOnly.filter1Sha256 and
			WFCatAssetAux.getAuxLocalOnlyFilePath( docValue, aux3.digest ) ==
				WFCatAssetAux.getAuxLocalOnlyFilePath( versionDoc, aux3.digest ) and 

			aux4.digest == localOnly.filter2Fingerprint and
			aux4.fileSize == localOnly.filter2FileSize and
			aux4.sha256 == localOnly.filter2Sha256 and
			WFCatAssetAux.getAuxLocalOnlyFilePath( docValue, aux4.digest ) ==
				WFCatAssetAux.getAuxLocalOnlyFilePath( versionDoc, aux4.digest )

	end


	rxTest:observe( docStore:ongoingDocValue( assetId )
		:map( assetUpdatedAndHasAllDevelopAux )
	)
		:expectDataWithinTime( 5, true )
		:dispose()

	-- Version doc should have been updated.

	rxTest:expectSingleValue(
		docStore:docValue( versionId )
			:pluck( '_changeCounter' )
			:map( function( changeCounter )
				return changeCounter > versionChangeCounter
			end ),
		true )

	-- missing_aux_ids should update asset for any aux that are missing from
	-- asset if it has some of the aux but not all

	assetId = KSIdGenerator.uuidInLowercaseHex()
	assetDoc = WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, assetId )

	versionId = KSIdGenerator.uuidInLowercaseHex()
	versionDoc, localOnly = WFCatVersionTestUtils.createVersionDocument( rxTest, catalog, versionId, { hasDevelopAux = true } )
	versionDoc.asset.id = assetId
	rxTest:waitFor( docStore:setDocValue( versionDoc ) )

	assetDoc.aux = {
		[ localOnly.mask1AuxId ] = {
			digest = localOnly.mask1Fingerprint,
			fileSize = localOnly.mask1FileSize,
			sha256 = localOnly.mask1Sha256,
		},
		[ localOnly.filter1AuxId ] = {
			digest = localOnly.filter1Fingerprint,
			fileSize = localOnly.filter1FileSize,
			sha256 = localOnly.filter1Sha256,
		},
	}
	assetDoc._localOnly.files.aux = {
		[ localOnly.mask1Fingerprint ] = {
			relativePath = localOnly.mask1RelativeFilePath,
		},
		[ localOnly.filter1Fingerprint ] = {
			relativePath = localOnly.filter1RelativeFilePath,
		}
	}
	rxTest:waitFor( docStore:setDocValue( assetDoc ) )

	errorDocId = KSIdGenerator.uuidInLowercaseHex()
	errorDoc._id = errorDocId
	spm.errors = {
		missing_aux_ids = { localOnly.mask1AuxId, localOnly.mask2AuxId, localOnly.filter1AuxId, localOnly.filter2AuxId }
	}
	spm.source.id = versionId

	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- Verify the missing aux have been copied to the asset.

	rxTest:expectSingleValue(
		docStore:ongoingDocValue( assetId )
			:map( function( docValue )
				local aux = docValue.aux or {}
				return aux[ localOnly.mask1AuxId ] and aux[ localOnly.mask2AuxId ] 
					and aux[ localOnly.filter1AuxId ] and aux[ localOnly.filter2AuxId ]
					and true
			end )
			:filter( function( missingAuxPresent )
				return missingAuxPresent
			end )
			:take( 1 ),
		true )

	-- Make sure the asset appears to have synced aux, as the error
	-- document won't be deleted until this is true.

	assetOverlay = {
		_id = assetId
	}

	aux1Rels = string.format( "/rels/aux/%s", localOnly.mask1AuxId )
	aux2Rels = string.format( "/rels/aux/%s", localOnly.mask2AuxId )
	aux3Rels = string.format( "/rels/aux/%s", localOnly.filter1AuxId )
	aux4Rels = string.format( "/rels/aux/%s", localOnly.filter2AuxId )

	KSTableUtils.setNested( assetOverlay, 'LINK', '_serverProvidedMetadata', 'links', aux1Rels, 'href' )
	KSTableUtils.setNested( assetOverlay, 'LINK', '_serverProvidedMetadata', 'links', aux2Rels, 'href' )
	KSTableUtils.setNested( assetOverlay, 'LINK', '_serverProvidedMetadata', 'links', aux3Rels, 'href' )
	KSTableUtils.setNested( assetOverlay, 'LINK', '_serverProvidedMetadata', 'links', aux4Rels, 'href' )

	rxTest:waitFor( docStore:applyOverlay( assetOverlay ) )

	versionChangeCounter = rxTest:getFirstValue( docStore:docValue( versionId )
		:pluck( '_changeCounter' ) )

	-- error doc should be deleted

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 5, true )

	-- asset doc should have updated

	rxTest:observe( docStore:ongoingDocValue( assetId )
			:map( assetUpdatedAndHasAllDevelopAux ) )
		:expectDataWithinTime( 5, true )
		:dispose()

	-- Version doc should have been updated.

	rxTest:expectSingleValue(
		docStore:docValue( versionId )
			:pluck( '_changeCounter' )
			:map( function( changeCounter )
				return changeCounter > versionChangeCounter
			end ),
		true )

	-- missing_aux_ids should not trigger a version update if one
	-- or more of the aux are corrupt locally. In this case, we ignore
	-- the error, as there's nothing we can do to recover from it.

	assetId = KSIdGenerator.uuidInLowercaseHex()
	assetDoc = WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, assetId )

	versionId = KSIdGenerator.uuidInLowercaseHex()
	versionDoc, localOnly = WFCatVersionTestUtils.createVersionDocument( rxTest, catalog, versionId, { hasDevelopAux = true } )
	versionDoc.asset.id = assetId
	rxTest:waitFor( docStore:setDocValue( versionDoc ) )

	assetDoc.aux = {
		[ localOnly.mask1AuxId ] = {
			digest = localOnly.mask1Fingerprint,
			fileSize = localOnly.mask1FileSize,
			sha256 = localOnly.mask1Sha256,
		},
		[ localOnly.filter1AuxId ] = {
			digest = localOnly.filter1Fingerprint,
			fileSize = localOnly.filter1FileSize,
			sha256 = localOnly.filter1Sha256,
		},
	}
	assetDoc._localOnly.files.aux = {
		[ localOnly.mask1Fingerprint ] = {
			relativePath = localOnly.mask1RelativeFilePath,
			error = true
		},
		[ localOnly.filter1Fingerprint ] = {
			relativePath = localOnly.filter1RelativeFilePath,
			error = true
		},
	}

	rxTest:waitFor( docStore:setDocValue( assetDoc ) )

	errorDocId = KSIdGenerator.uuidInLowercaseHex()
	errorDoc._id = errorDocId
	spm.errors = {
		missing_aux_ids = { localOnly.mask1AuxId, localOnly.mask2AuxId, localOnly.filter1AuxId, localOnly.filter2AuxId }
	}
	spm.source.id = versionId

	versionChangeCounter = rxTest:getFirstValue( docStore:docValue( versionId )
		:pluck( '_changeCounter' ) )

	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- error doc should be deleted

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 5, true )

	-- asset doc should have updated

	rxTest:observe( docStore:ongoingDocValue( assetId )
			:map( assetUpdatedAndHasAllDevelopAux ) )
		:expectDataWithinTime( 5, true )
		:dispose()

	-- Any update to a version will bump the change counter, and trigger the re-upload.
	-- So specifics of HOW a version was updated are particularly important, we just
	-- need to make sure the change counter, aka the sequence number, did not change.

	rxTest:expectSingleValue(
		docStore:docValue( versionId )
			:pluck( '_changeCounter' ),
		versionChangeCounter )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTest_Version_BinaryMissingError_Basic( rxTest )

	rxTest:setTimeLimit( 10 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, nil, indexDefSpec )

	local errorHandler = WFCatErrorHandler( catalog, {},
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	-- Create an Asset and related Version. The Asset has unsynced binaries.

	local assetOptions = {
		hasDevelopAux = true,
		noDevelopAuxLinks = true
	}

	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local assetDoc, localOnly = WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, assetId, assetOptions )
	rxTest:waitFor( docStore:setDocValue( assetDoc ) )

	local versionId = KSIdGenerator.uuidInLowercaseHex()
	local versionDoc = WFCatVersionTestUtils.createVersionDocument( rxTest, catalog, versionId )
	versionDoc.asset.id = assetId
	versionDoc.asset.sha256 = assetDoc.importSource.sha256
	versionDoc._serverProvidedMetadata = nil -- unsynced

	-- Copy all Aux from Asset to Version

	versionDoc.develop.masks = {}
	versionDoc.develop.filters = {}
	versionDoc._localOnly.files.aux = {}

	local missingAuxIds = {}

	versionDoc.develop.masks[ localOnly.mask1AuxId ] = {
		digest = localOnly.mask1Fingerprint,
	}

	versionDoc.develop.masks[ localOnly.mask2AuxId ] = {
		digest = localOnly.mask2fingerprint,
	}

	versionDoc.develop.filters[ localOnly.filterGroupFingerprint ] = {
		[ localOnly.filter1AuxId ] = {
			digest = localOnly.filter1Fingerprint,
			sourceLongEdge = 6000,
			crsSizeX = 1024,
			crsSizeY = 768
		},
		[ localOnly.filter2AuxId ] = {
			digest = localOnly.filter2Fingerprint,
			targetLongEdge = 2560,
			crsSizeX = 437,
			crsSizeY = 328
		}
	}

	for auxId, auxInfo in pairs( assetDoc.aux ) do

		versionDoc._localOnly.files.aux[ auxInfo.digest ] =
			assetDoc._localOnly.files.aux[ auxInfo.digest ]

		missingAuxIds[ #missingAuxIds + 1 ] = auxId

	end

	rxTest:waitFor( docStore:setDocValue( versionDoc ) )

	-- Make sure the asset is not fully synced:

	rxTest:expectSingleValue( WFOzDocUtils.areAssetsFullySyncedWithOz( catalog, { assetId } ), false )

	-- Let's pretend we're trying to sync the Version metdata. The version
	-- doc will be rejected by Oz because the Asset has unsynced auxiliaries:
	-- Create error BinaryMissingError for Version

	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	local errorDoc = {
		_id = errorDocId,
		subtype = 'BinaryMissingError',
		type = "error",
		_serverProvidedMetadata = {
			errors = {
				missing_aux_ids = missingAuxIds,
			},
			source = {
				id = versionId,
				action = "create",
				type = "version",
				revision_id = versionDoc._newRevId,
			},
			status = 404,
		},
	}

	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- Now make the aux on the asset appear synced, as the error document
	-- won't be resolved until this is true.

	local assetOverlay = {
		_id = assetId
	}

	local aux1Rels = string.format( "/rels/aux/%s", localOnly.mask1AuxId )
	local aux2Rels = string.format( "/rels/aux/%s", localOnly.mask2AuxId )
	local aux3Rels = string.format( "/rels/aux/%s", localOnly.filter1AuxId )
	local aux4Rels = string.format( "/rels/aux/%s", localOnly.filter2AuxId )

	KSTableUtils.setNested( assetOverlay, 'LINK', '_serverProvidedMetadata', 'links', aux1Rels, 'href' )
	KSTableUtils.setNested( assetOverlay, 'LINK', '_serverProvidedMetadata', 'links', aux2Rels, 'href' )
	KSTableUtils.setNested( assetOverlay, 'LINK', '_serverProvidedMetadata', 'links', aux3Rels, 'href' )
	KSTableUtils.setNested( assetOverlay, 'LINK', '_serverProvidedMetadata', 'links', aux4Rels, 'href' )

	rxTest:waitFor( docStore:applyOverlay( assetOverlay ) )

	-- Make sure the asset is fully synced:

	rxTest:expectSingleValue( WFOzDocUtils.areAssetsFullySyncedWithOz( catalog, { assetId } ), true )

	-- Error doc should now be deleted

	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 5, true )

	-- The Version should have had error annotation set in _localOnly to trigger
	-- metadata pusher to send to Oz again

	versionDoc = rxTest:getFirstValue( docStore:docValue( versionId ) )
	rxTest:expectEqual( "BinaryMissingError", versionDoc._localOnly.ozError.subtype )
	rxTest:expectEqual( 404, versionDoc._localOnly.ozError.status )
	rxTest:expectEqual( 1, versionDoc._localOnly.ozError.retries )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestBinaryMissingError_testNightly( rxTest )

	-- this tests copying missing aux data over from the version to the asset

	rxTest:setTimeLimit( 700 )

	local catalogTestSession = WFCatTestSession( rxTest )
	local catalog

	rxTest:waitFor(
		catalogTestSession:catalog()
			:map( function( _catalog )
				catalog = _catalog
			end )
	)

	local opts = KSTableUtils.merge( {}, options, { capabilities = {
			[ 'WFCatBinaryUploader.atomicDevelopSettings' ] = true, } } )
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, opts )
	local binaryUploader = WFCatBinaryUploader( catalog, nil, nil, nil, opts )

	rxTest:finally( function()

		puller:stopContinuousUpdates()
		pusher:stopContinuousUpdates()
		binaryUploader:stopContinuousUploads()

		binaryUploader:close()
		puller:close()
		pusher:close()

	end )

	-- Create the initial revision of the asset document.

	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local assetOptions = { hasMasks = false }
	local assetDoc = WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, assetId, assetOptions )
	assetDoc._serverProvidedMetadata = nil

	-- start syncing with quicker retries, both metadata and binaries.

	pusher:addRequestHeader( 'X-Quicker-Retries', 'true' )

	puller:getUpdatesContinuously()
	rxTest:waitFor( pusher:sendUpdatesContinuously() )
	binaryUploader:startContinuousUploads()

	-- Sync the first revision of the asset and wait for the binaries to upload.

	rxTest:waitFor( catalog:docStore():setDocValue( assetDoc ) )

	rxTest:waitFor( WFOzDocUtils.waitUntilAssetsAreFullySyncedWithOz( catalog, { assetId } ) )

	-- Stop the binary uploader, we'll resume it later. We do this to make
	-- the test ordering more predictable.

	binaryUploader:stopContinuousUploads()

	-- Now create a version doc, and a new revision of the asset.

	local versionId = KSIdGenerator.uuidInLowercaseHex()
	local versionDoc = WFCatVersionTestUtils.createVersionDocument( rxTest, catalog, versionId )
	versionDoc.asset.id = assetId
	versionDoc.asset.sha256 = assetDoc.importSource.sha256
	versionDoc._serverProvidedMetadata = nil

	assetOptions = { hasMasks = true }
	local assetDocNewRev = WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, assetId, assetOptions )

	-- copy all aux from asset to version and remove one aux from the new revision of the asset.

	local missingAuxId, missingAuxDigest, missingAuxSha256, missingAuxRelPath, missingAuxFileSize

	versionDoc.develop.masks = {}
	versionDoc._localOnly.files.aux = {}

	for auxId, auxInfo in pairs( assetDocNewRev.aux ) do

		versionDoc.develop.masks[ auxId ] = {
			digest = auxInfo.digest,
		}

		versionDoc._localOnly.files.aux[ auxInfo.digest ] =
			assetDocNewRev._localOnly.files.aux[ auxInfo.digest ]

		if not missingAuxId then

			missingAuxId = auxId

			missingAuxDigest = assetDocNewRev.aux[ auxId ].digest
			missingAuxSha256 = assetDocNewRev.aux[ auxId ].sha256
			missingAuxFileSize = assetDocNewRev.aux[ auxId ].fileSize
			assetDocNewRev.aux[ auxId ] = nil
			assetDocNewRev.develop.masks[ auxId ] = nil

			missingAuxRelPath = assetDocNewRev._localOnly.files.aux[ missingAuxDigest ].relativePath
			assetDocNewRev._localOnly.files.aux[ missingAuxDigest ] = nil

		end

	end

	-- Apply the new asset revision and create the version.

	local overlay = {
		_id = assetId,
		aux = assetDocNewRev.aux,
		develop = assetDocNewRev.develop,
		_localOnly = {
			files = {
				aux = assetDocNewRev._localOnly.files.aux,
				xmp_develop = assetDocNewRev._localOnly.files.xmp_develop
			}
		}
	}

	rxTest:waitFor( catalog:docStore():applyOverlay( overlay ) )

	rxTest:waitFor( catalog:docStore():setDocValue( versionDoc ) )

	-- We should receive two BinaryMissingErrors from Oz. One for the asset,
	-- and one for the version.

	rxTest:observe( catalog:indexStore():query {
			sql = [[ SELECT docId FROM errors WHERE type = 'BinaryMissingError' ]],
			quiesceUntilUpToDate = true,
			singleColumnQuery = true
		}
		:map( function( docIds )
			return #docIds
		end ) )
		:expectDataWithinTime( 600, 2 )
		:dispose()

	-- Now attach the error handler. We do this here, as opposed to at the beginning
	-- of the test, to make the ordering of the handling easier to test.

	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )

	rxTest:finally( function()
		errorHandler:close()
	end )

	-- Wait for the asset doc error to be cleared.

	rxTest:observe( catalog:indexStore():query {
			sql = [[ SELECT count(*) FROM errors WHERE type = 'BinaryMissingError' ]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )
		:expectDataWithinTime( 600, 1 )
		:dispose()

	-- asset doc should have BinaryMissingError annotation
	-- and have missing aux applied

	assetDoc = rxTest:getFirstValue( catalog:docStore():docValue( assetId ) )
	rxTest:expectEqual( "BinaryMissingError", assetDoc._localOnly.ozError.subtype )
	rxTest:expectEqual( 404, assetDoc._localOnly.ozError.status )

	rxTest:expectEqual( missingAuxDigest, assetDoc.aux[ missingAuxId ].digest )
	rxTest:expectEqual( missingAuxSha256, assetDoc.aux[ missingAuxId ].sha256 )
	rxTest:expectEqual( missingAuxFileSize, assetDoc.aux[ missingAuxId ].fileSize )
	rxTest:expectEqual( missingAuxRelPath, assetDoc._localOnly.files.aux[ missingAuxDigest ].relativePath )

	rxTest:expectEqual( nil, assetDoc.develop.masks[ missingAuxId ] )
		-- does not update the asset develop masks

	-- Reenable the Binary Uploader

	binaryUploader:startContinuousUploads()

	-- Wait for the version error to be cleared.

	rxTest:observe( catalog:indexStore():query {
			sql = [[ SELECT count(*) FROM errors WHERE type = 'BinaryMissingError' ]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )
		:expectDataWithinTime( 600, 0 )
		:dispose()

	-- Version doc should have BinaryMissingErrors annotation

	versionDoc = rxTest:getFirstValue( catalog:docStore():docValue( versionId ) )
	rxTest:expectEqual( "BinaryMissingError", versionDoc._localOnly.ozError.subtype )
	rxTest:expectEqual( 404, versionDoc._localOnly.ozError.status )

end

--------------------------------------------------------------------------------

-- @mark Unknown Errors

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestUnknownError_Not500( rxTest )

	-- unhandled errors are logged and then marked as unknown so that we can ignore it for now
	-- but possibly go back to it if we come up with a way to handle and reindex it later
	rxTest:setTimeLimit( 15 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )
	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local docRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			_newRevId = docRevisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		}
	)

	-- create error doc for asset
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ThisIsAnUnknownErrorSubtype",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"unknown error",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = docRevisionId,
					type = "asset",
				},
				status = 400,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be annotated that the error was unknown
	rxTest:observe( docStore:ongoingDocValue( errorDocId )
						:map( function( docValue )
							return KSTableUtils.derefOpt( docValue, '_localOnly', 'unknownError' )
						end ) )
		:expectDataWithinTime( 3, true )

	-- since we've marked the error as unknown, we shouldn't see it in the errors table anymore
	rxTest:observe( catalog:indexStore():query {
							sql = [[SELECT count( docId ) FROM errors]],
						}
						:pluck( 1, 1 ) )
		:expectDataWithinTime( 2, 0 )


	-- asset should have no error annotation since we don't know how to clear an unknown error
	rxTest:expectSingleValue(
		docStore:docValue( docId )
			:map( function( docValue )
				return not not KSTableUtils.derefOpt( docValue, '_localOnly', 'ozError' )
			end ), false )

end


--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestUnknownError_500_Retry( rxTest )

	-- test that with unknown errors with a status code of 5XX that we retry the document
	-- automatically
	rxTest:setTimeLimit( 15 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local assetDoc = WFCatAssetTestUtils.createAssetDocument( rxTest, catalog )
	rxTest:waitFor( docStore:setDocValue( assetDoc ) )

	local docId = assetDoc._id
	local docRevisionId = assetDoc._newRevId

	local initialChangeCounter
	rxTest:waitFor( docStore:docValue( docId )
		:map( function( docValue )
			initialChangeCounter = docValue._changeCounter
			return docValue
		end )
	)

	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	local errorDoc = {
		_id = errorDocId,
		subtype = "ThisIsAnUnknownErrorSubtype",
		type = "error",
		_serverProvidedMetadata = {
			errors = {
				asset = {
					"unknown error",
				},
			},
			source = {
				action = "create",
				id = docId,
				revision_id = docRevisionId,
				type = "asset",
			},
			status = 500,
			updated = "2016-06-07T15:55:34.218872Z",
		},
	}

	-- create error doc for asset
	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- error doc should be deleted after handling it (by marking the asset for retry
	rxTest:observe( docStore:ongoingDocValue( errorDocId )
		:map( function( docValue )
			return docValue._empty
		end ) )
		:expectDataWithinTime( 3, true )

	-- since we've deleted the error we shouldn't see it in the errors table anymore
	rxTest:observe( catalog:indexStore():query {
		sql = [[SELECT count( docId ) FROM errors]],
		}
		:pluck( 1, 1 ) )
		:expectDataWithinTime( 2, 0 )

	-- asset should have an updated change counter after error processing
	rxTest:expectSingleValue( docStore:docValue( docId )
		:map( function( docValue )
			return docValue._changeCounter > initialChangeCounter
		end ),
		true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestUnknownError_500_Retry_NoLooper( rxTest )

	-- test that with unknown errors with a status code of 5XX that we retry the document
	-- automatically but that we don't retry forever and don't create a looper
	rxTest:setTimeLimit( 15 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )
	rxTest:addCloseTarget( errorHandler )

	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT count(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )

	local docStore = catalog:docStore()

	local assetDoc = WFCatAssetTestUtils.createAssetDocument( rxTest, catalog )
	rxTest:waitFor( docStore:setDocValue( assetDoc ) )

	local docId = assetDoc._id
	local docRevisionId = assetDoc._newRevId

	local changeCounter
	rxTest:waitFor( docStore:docValue( docId )
		:map( function( docValue )
			changeCounter = docValue._changeCounter
			return docValue
		end )
	)

	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	local errorDoc = {
		_id = errorDocId,
		subtype = "ThisIsAnUnknownErrorSubtype",
		type = "error",
		_serverProvidedMetadata = {
			errors = {
				asset = {
					"unknown error",
				},
			},
			source = {
				action = "create",
				id = docId,
				revision_id = docRevisionId,
				type = "asset",
			},
			status = 500,
			updated = "2016-06-07T15:55:34.218872Z",
		},
	}

	-- create 1st error doc for asset
	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- error doc should be deleted after handling it (by marking the asset for retry
	rxTest:observe( docStore:ongoingDocValue( errorDocId )
		:map( function( docValue )
			return docValue._empty
		end ) )
		:expectDataWithinTime( 3, true )

	-- error doc should be removed from errors index store table
	-- (need this check because it will be recreated and reprocessed
	-- with same _id, but not if we do not wait for it to be removed
	-- from table first.)
	outstandingErrors:expectDataWithinTime( 5, 0 )

	-- asset should have an updated change counter after error processing
	rxTest:expectSingleValue( docStore:docValue( docId )
		:map( function( docValue )
			local isChangeCounterHigher = docValue._changeCounter > changeCounter
			changeCounter = docValue._changeCounter
			return isChangeCounterHigher
		end ),
		true )

	-- create error document on asset a 2nd time
	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- error doc should be deleted after handling it (by marking the asset for retry
	rxTest:observe( docStore:ongoingDocValue( errorDocId )
		:map( function( docValue )
			return docValue._empty
		end ) )
		:expectDataWithinTime( 3, true )

	-- error doc should be removed from errors index store table
	outstandingErrors:expectDataWithinTime( 5, 0 )

	-- asset should have an updated change counter after error processing
	rxTest:expectSingleValue( docStore:docValue( docId )
		:map( function( docValue )
			local isChangeCounterHigher = docValue._changeCounter > changeCounter
			changeCounter = docValue._changeCounter
			return isChangeCounterHigher
		end ),
		true )

	-- create error document on asset a 3nd time
	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- error doc should be deleted after handling it (by marking the asset for retry
	rxTest:observe( docStore:ongoingDocValue( errorDocId )
		:map( function( docValue )
			return docValue._empty
		end ) )
		:expectDataWithinTime( 3, true )

	-- error doc should be removed from errors index store table
	outstandingErrors:expectDataWithinTime( 5, 0 )

	-- asset should have an updated change counter after error processing
	rxTest:expectSingleValue( docStore:docValue( docId )
		:map( function( docValue )
			local isChangeCounterHigher = docValue._changeCounter > changeCounter
			changeCounter = docValue._changeCounter
			return isChangeCounterHigher
		end ),
		true )

	-- create error document on asset a 4th time
	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- error doc should be deleted after handling it (by marking the asset for retry
	rxTest:observe( docStore:ongoingDocValue( errorDocId )
		:map( function( docValue )
			return docValue._empty
		end ) )
		:expectDataWithinTime( 3, true )

	-- asset should not have an updated change counter after error processing, we can't
	-- keep sending an asset update to Oz or that will be a looper
	local lastRetryTime
	rxTest:expectSingleValue( docStore:docValue( docId )
		:map( function( docValue )
			local isChangeCounterHigher = docValue._changeCounter > changeCounter
			changeCounter = docValue._changeCounter
			lastRetryTime = KSTableUtils.derefOpt( docValue, '_localOnly', 'ozError', 'lastRetry' )
			return isChangeCounterHigher
		end ),
		false )

	-- now that we've stopped the possibility of a bad looper, we can't let this error
	-- stop retries forever if this same asset hits another 500 error. We'll make a change
	-- to make the lastRetry time a bit over an hour old. Then send another error to see if we
	-- properly reset the retries and update the change counter so that it will be resent
	-- to Oz

	local retryTimeSinceReference = KSDate.iso8601ToTimeIntervalSinceReference( lastRetryTime )
	retryTimeSinceReference = retryTimeSinceReference - 5400 -- (1.5 hours)
	local newLastRetryTime = KSDate.timeInIso8601Format( retryTimeSinceReference )

	rxTest:waitFor(
		docStore:applyOverlay {
			_id = docId,
			_localOnly = {
				ozError = {
					lastRetry = newLastRetryTime,
				}
			}
		}
	)

	-- after the overlay, get the latest change counter (it should also be higher than the last time)
	rxTest:expectSingleValue( docStore:docValue( docId )
		:map( function( docValue )
			local isChangeCounterHigher = docValue._changeCounter > changeCounter
			changeCounter = docValue._changeCounter
			docRevisionId = docValue._revId
			return isChangeCounterHigher
		end ),
		true )

	-- create error document a 5th time with a new id this time
	errorDocId = KSIdGenerator.uuidInLowercaseHex()
	errorDoc._id = errorDocId
	rxTest:waitFor( docStore:setDocValue( errorDoc ) )

	-- error doc should be deleted after handling it (by marking the asset for retry
	rxTest:observe( docStore:ongoingDocValue( errorDocId )
		:map( function( docValue )
			return docValue._empty
		end ) )
		:expectDataWithinTime( 3, true )

	-- asset should not have an updated change counter after error processing, we can't
	-- keep sending an asset update to Oz or that will be a looper
	rxTest:expectSingleValue( docStore:docValue( docId )
		:map( function( docValue )
			local isChangeCounterHigher = docValue._changeCounter > changeCounter
			changeCounter = docValue._changeCounter
			return isChangeCounterHigher
		end ),
		true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestErrorNotThisClient( rxTest )

	-- the sync feed will receive errors that may have been generated from other clients. when this happens, the error
	-- handler should ignore the error and delete it straight away.
	rxTest:setTimeLimit( 15 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )
	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		}
	)

	-- create error doc for random asset
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "InputValidationError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"unknown error",
					},
				},
				source = {
					action = "create",
					id = KSIdGenerator.uuidInLowercaseHex(),
					revision_id = KSIdGenerator.uuidInLowercaseHex(),
					type = "asset",
				},
				status = 493,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- since we've deleted the error as not from this client, we shouldn't see it in the errors table anymore
	rxTest:observe( catalog:indexStore():query {
							sql = [[SELECT count( docId ) FROM errors]],
						}
						:pluck( 1, 1 ) )
		:expectDataWithinTime( 2, 0 )

	-- error document should be marked as deleted
	local errorDocValue = rxTest:observe( docStore:ongoingDocValue( errorDocId )
			:map( function( docValue )
				return docValue._empty
			end ) )

	errorDocValue:expectDataWithinTime( 5, true )

	-- create error doc for asset we have but different revision (i.e. not this client or, at least, not the
	-- latest version of the document on this client)
	errorDocId = KSIdGenerator.uuidInLowercaseHex()
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "InputValidationError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"unknown error",
					},
				},
				source = {
					action = "update",
					id = docId,
					revision_id = KSIdGenerator.uuidInLowercaseHex(),
					type = "asset",
				},
				status = 493,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- since we've deleted the error as not from this client, we shouldn't see it in the errors table anymore
	rxTest:observe( catalog:indexStore():query {
							sql = [[SELECT count( docId ) FROM errors]],
						}
						:pluck( 1, 1 ) )
		:expectDataWithinTime( 2, 0 )

	-- error document should be marked as deleted
	errorDocValue = rxTest:observe( docStore:ongoingDocValue( errorDocId )
			:map( function( docValue )
				return docValue._empty
			end ) )

	errorDocValue:expectDataWithinTime( 5, true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestUnhandledError_MissingSourceSection( rxTest )

	-- we can't verify whether this error is for this client, so we should log it and mark as unknown so if we ever figure it out
	-- we can possibly reattempt to process it
		-- unhandled errors are logged and then marked as unknown so that we can ignore it for now
	-- but possibly go back to it if we come up with a way to handle and reindex it later
	rxTest:setTimeLimit( 15 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )
	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local docRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create basic 'asset' doc
	rxTest:waitFor(
		docStore:setDocValue {
			_id = docId,
			_newRevId = docRevisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		}
	)

	-- create error doc for asset
	rxTest:waitFor(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ThisIsAnUnknownErrorSubtype",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"unknown error",
					},
				},
				status = 400,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		}
	)

	-- error doc should be annotated that the error was unknown
	rxTest:observe( docStore:ongoingDocValue( errorDocId )
						:map( function( docValue )
							return KSTableUtils.derefOpt( docValue, '_localOnly', 'unknownError' )
						end ) )
		:expectDataWithinTime( 3, true )

	-- since we've marked the error as unknown, we shouldn't see it in the errors table anymore
	rxTest:observe( catalog:indexStore():query {
							sql = [[SELECT count( docId ) FROM errors]],
						}
						:pluck( 1, 1 ) )
		:expectDataWithinTime( 2, 0 )


	-- asset should have no error annotation since we don't know how to clear an unknown error
	rxTest:expectSingleValue(
		docStore:docValue( docId )
			:map( function( docValue )
				return not not KSTableUtils.derefOpt( docValue, '_localOnly', 'ozError' )
			end ), false )

end

--------------------------------------------------------------------------------

local function httpMockDataTransformation( data, requestInfoData )

	data = KSJsonDecoder.decodeWebSafeJson( data )
	requestInfoData = KSJsonDecoder.decodeWebSafeJson( requestInfoData )

	return data, requestInfoData

end

--------------------------------------------------------------------------------

if WFPlatformInfo.isDesktop then

-- The following test relies on WFCatSyncStatus, which is Desktop only.
-- TODO [jandrew 2024-10-02]: In general, there are a lot of dependencies in this test
-- suite on Desktop-only code, even though this is a cross-platform file. This should
-- be cleaned up.

function WFCatErrorHandlerTest.rxTestResourceNotFoundError_ErrorOnDelete_mock( rxTest )
	-- this test case is based on the behavior seen in NMB-9050
	rxTest:setTimeLimit( 15 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	catalogSpec:expectHttpRequest( prefetchReqResp( catalogId ) )
	catalogSpec:expectHttpRequest( catalogPrefetchReqResp( catalogId ) )
	catalogSpec:expectHttpRequest(
		WFMockHttpRequest()
			:get( string.format( "/v2/catalogs/%s/updates", catalogId ) )
			:respond( '200', [[{
				"base": "https://photos-stage.adobe.io/v2/catalogs/]]..catalogId..[[/",
				"updates": [],
				"links": {
					"next": {
						"href": "/v2/catalogs/]]..catalogId..[[/updates?updated_since=pull1"
					}
				}
			}]] ) )

	local docStore = catalog:docStore()
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )

	rxTest:finally( function()
		puller:close()
		pusher:close()
		errorHandler:close()
	end )

	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local initialRevisionId = KSIdGenerator.uuidInLowercaseHex()

	local isSyncPending = rxTest:observe( require 'WFCatSyncStatus'.syncPendingForAnyAsset( catalog ) )

	isSyncPending:expectDataWithinTime( 5, false )

	local docValue = {
		_id = assetId,
		_newRevId = initialRevisionId,
		type = 'asset',
		subtype = 'image',
		importSource = {
			fileName = 'test.jpg',
			filePath = '/path/',
			originalHeight = 150,
			originalWidth = 100,
		}
	}

	docValue = makeAssetDocDisplayable( docValue )

	-- create basic 'asset' doc
	rxTest:waitFor( docStore:setDocValue( docValue ) )

	isSyncPending:expectDataWithinTime( 5, true )

	-- expect the pusher's updates feed create
	catalogSpec:expectHttpRequest( WFMockHttpRequest()
		:post( string.format( "/v2/catalogs/%s/updates", catalogId ) )
		:validateRequestData( function( str )
					return str:find( [["action":"create"]] ) and
						str:find( [["fileName":"test.jpg"]] ) and
						str:find( [["type":"asset"]] )
				end )
		:respond( '200', [[]] ) )

	local sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()

	-- expect the acceptance of the create
	catalogSpec:expectHttpRequest( WFMockHttpRequest()
		:get( string.format( "/v2/catalogs/%s/updates?updated_since=pull1", catalogId ) )
		:respond( '200', [[
			{
			  "base": "https://photos-stage.adobe.io/v2/catalogs/]]..catalogId..[[/",
			  "updates": [
				{
				  "id": "]]..assetId..[[",
				  "type": "asset",
				  "updated": "2017-09-06T18:15:10.138376Z",
				  "subtype": "image",
				  "created": "2017-09-06T18:15:10.138376Z",
				  "revision_ids": [
					"]]..initialRevisionId..[["
				  ],
				  "links": {
					"self": {
					  "href": "assets/]]..assetId..[["
					},
					"/rels/proxy_create": {
					  "href": "assets/]]..assetId..[[/revisions/{revision_id}/proxies/{proxy_id}",
					  "templated": true
					},
					"/rels/rendition_create": {
					  "href": "assets/]]..assetId..[[/revisions/{revision_id}/renditions/{rendition_id}",
					  "templated": true
					},
					"/rels/master_create": {
					  "href": "assets/]]..assetId..[[/revisions/{revision_id}/master",
					  "templated": true
					},
					"/rels/master": {
					  "href": "assets/]]..assetId..[[/revisions/{revision_id}/master"
					},
					"/rels/xmp_develop_create": {
					  "href": "assets/]]..assetId..[[/revisions/{revision_id}/xmp/develop",
					  "templated": true
					}
				  },
				  "payload": {
					"captureDate": "0000-00-00T00:00:00",
					"importSource": {
					  "originalHeight": 150,
					  "filePath": "/path/",
					  "importTimestamp": "2017-09-06T18:15:09.432Z",
					  "importedBy": "]]..KSIdGenerator.uuidInLowercaseHex()..[[",
					  "fileName": "test.jpg",
					  "originalWidth": 100
					},
					"develop": {
					  "processingModel": "lightroom",
					  "xmpCameraRaw": "<xmpmeta />"
					}
				  }
				}
			  ],
			  "links": {
				"next": {
				  "href": "/v2/catalogs/]]..catalogId..[[/updates?updated_since=pull2"
				}
			  }
			}
		  ]] ) )

	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	isSyncPending:expectDataWithinTime( 5, false )

	local newRevisionId = KSIdGenerator.uuidInLowercaseHex()

	-- delete doc
	rxTest:waitFor(
		docStore:applyOverlay( {
			_id = assetId,
			_newRevId = newRevisionId,
			_deleted = true
		} )
	)

	-- expect push of deletion
	catalogSpec:expectHttpRequest( WFMockHttpRequest()
		:post( string.format( "/v2/catalogs/%s/updates", catalogId ) )
		:expectRequestData( [[[{"id":"]]..assetId..[[","type":"asset","action":"delete","revision_id":"]]..newRevisionId..'"}]' )
		:requestDataTransformation( httpMockDataTransformation )
		:respond( '200', [[]] ) )

	sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()

	-- get RNF on pull rather than tombstone
	catalogSpec:expectHttpRequest( WFMockHttpRequest()
		:get( string.format( "/v2/catalogs/%s/updates?updated_since=pull2", catalogId ) )
		:respond( '200', [[{"base":"https://photos-stage.adobe.io/v2/catalogs/]]..catalogId..[[/",
								"updates": []]..
								-- ResourceNotFoundError
								[[{"id":"6d7e7425358aa4177ed4469b134a01f1","updated":"2017-09-06T14:54:48.042095Z",
								"type":"error","status":404,"subtype":"ResourceNotFoundError","errors":{"asset":["does not exist"]},
								"source":{"id":"]]..assetId..[[","action":"delete","type":"asset",
								"revision_id":"]]..newRevisionId..[["}}],
								"links": {
									"next": {
										"href": "/v2/catalogs/]]..catalogId..[[/updates?updated_since=pull2"
									}
								}
						} ]] ) )

	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	-- let error handler handle the error


	-- doc should be marked as deleted on Oz (and no need to sync)
	isSyncPending:expectDataWithinTime( 15, false )

end

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestInvalidAccessRequestError( rxTest )

	rxTest:setTimeLimit( 20 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, KSIdGenerator.uuidInLowercaseHex(), rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local inviteId = KSIdGenerator.uuidInLowercaseHex()
	local docRevId

		-- create basic 'asset' doc that has not been synced to Oz
	rxTest:waitFor(
		docStore:setDocValue {
			_id = inviteId,
			type = 'spaceinvite',
			subtype = 'request',
			status = 'approved',
			role = 'viewer',
			space = {
				id = KSIdGenerator.uuidInLowercaseHex(),
			},
			user = {
				id = KSIdGenerator.uuidInLowercaseHex(),
			},
		}
		:map( function( revisionId )
			docRevId = revisionId
			return revisionId
		end )
	)

	-- create error doc for asset that indicates the resource is not found
	rxTest:waitFor(
		docStore:setDocValue {
			_id = KSIdGenerator.uuidInLowercaseHex(),
			_serverProvidedMetadata = {
				base = string.format( "https://photos-stage.adobe.io/v2/catalogs/%s/", catalogId ),
				source = {
					action = "update",
					id = inviteId,
					links = {
						["self"] = {
							href = string.format( "/v2/catalogs/%s/updates/%s", catalogId, inviteId ),
						},
					},
					revision_id = docRevId,
					type = "spaceinvite",
				},
				status = 403,
				updated = "2017-04-26T19:27:11.417509Z",
			},
			subtype = "InvalidAccessRequestError",
			type = "error",
		}
	)

	-- check index store for error documents, there shouldn't be any any more
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT * FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:map( function( rows )
			return #rows
		end ) )

	outstandingErrors:expectDataWithinTime( 10, 0 )

	-- the error handler should process the ResourceExistsError but not delete
	-- the space_invite
	rxTest:expectSingleValue(
		docStore:docValue( inviteId )
			:map( function( docValue )
				return not not docValue._empty
		end ),
		false
	)

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestSpaceUserExistsError( rxTest )

	rxTest:setTimeLimit( 20 )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, KSIdGenerator.uuidInLowercaseHex(), rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local inviteId = KSIdGenerator.uuidInLowercaseHex()
	local docRevId

		-- create basic 'asset' doc that has not been synced to Oz
	rxTest:waitFor(
		docStore:setDocValue {
			_id = inviteId,
			type = 'spaceinvite',
			subtype = 'request',
			status = 'approved',
			role = 'viewer',
			space = {
				id = KSIdGenerator.uuidInLowercaseHex(),
			},
			user = {
				id = KSIdGenerator.uuidInLowercaseHex(),
			},
		}
		:map( function( revisionId )
			docRevId = revisionId
			return revisionId
		end )
	)

	-- create error doc for asset that indicates the resource is not found
	rxTest:waitFor(
		docStore:setDocValue {
			_id = KSIdGenerator.uuidInLowercaseHex(),
			_serverProvidedMetadata = {
				base = string.format( "https://photos-stage.adobe.io/v2/catalogs/%s/", catalogId ),
				errors = {
					space_user = {
						"already exists for space",
					},
				},
				source = {
					action = "update",
					id = inviteId,
					links = {
						["self"] = {
							href = string.format( "/v2/catalogs/%s/updates/%s", catalogId, inviteId ),
						},
					},
					revision_id = docRevId,
					type = "spaceinvite",
				},
				status = 403,
				updated = "2017-04-26T19:27:11.417509Z",
			},
			subtype = "SpaceUserExistsError",
			type = "error",
		}
	)

	-- check index store for error documents, there shouldn't be any any more
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT * FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:map( function( rows )
			return #rows
		end ) )

	outstandingErrors:expectDataWithinTime( 10, 0 )

	-- the error handler should process the SpaceUserExistsError and also delete
	-- the offending space_invite
	rxTest:expectSingleValue(
		docStore:docValue( inviteId )
			:map( function( docValue )
				return docValue._deleted
			end ),
		true
	)

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestRecoverMissingAuxOverlay( rxTest )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	-- Create test document with aux files

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local docValue, expectedData
			= WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, docId, { hasMasks = true } )

	-- Test 1: No-op: Recover aux files, but local-only annotations already exist.

	local overlay = WFCatErrorHandler.recoverMissingAuxOverlay( catalog, docValue )

	rxTest:expectEqual( {}, overlay )

	-- Test 2: Recover aux files, where local-only annotations do not exist.

	docValue._localOnly = nil

	overlay = WFCatErrorHandler.recoverMissingAuxOverlay( catalog, docValue )

	rxTest:expectEqual( docId, overlay._id )

	local auxPath1 = WFCatAssetAux.getAuxLocalOnlyFilePath( overlay, expectedData.mask1Fingerprint )
	local auxPath2 = WFCatAssetAux.getAuxLocalOnlyFilePath( overlay, expectedData.mask2Fingerprint )

	rxTest:expectEqual( 'string', type( auxPath1 ) )
	rxTest:expectEqual( 'string', type( auxPath2 ) )

	-- Test 3: Recover specific aux IDs

	local specificAuxIds = { expectedData.mask1AuxId }
	overlay = WFCatErrorHandler.recoverMissingAuxOverlay( catalog, docValue, specificAuxIds )

	rxTest:expectEqual( docId, overlay._id )

	auxPath1 = WFCatAssetAux.getAuxLocalOnlyFilePath( overlay, expectedData.mask1Fingerprint )
	auxPath2 = WFCatAssetAux.getAuxLocalOnlyFilePath( overlay, expectedData.mask2Fingerprint )

	rxTest:expectEqual( 'string', type( auxPath1 ) )
	rxTest:expectEqual( nil, auxPath2 )

end

--------------------------------------------------------------------------------

local function testRecoverOrRemoveMissingAux_setup( rxTest )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	local docStore = catalog:docStore()

	return catalog, docStore

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestRecoverOrRemoveMissingAux_Noop( rxTest )

	local catalog, docStore = testRecoverOrRemoveMissingAux_setup( rxTest )

	-- Create test document with aux files

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local docValue
			= WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, docId, { hasMasks = true } )
	rxTest:waitFor( docStore:setDocValue( docValue ) )

	-- All aux files exist

	local didApplyOverlay = false

	rxTest:expectSingleValue(
			WFCatErrorHandler._DEBUG_recoverOrRemoveMissingAux( catalog, docValue ), didApplyOverlay )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestRecoverOrRemoveMissingAux_MissingAux_NoOzRevision( rxTest )

	local catalog, docStore = testRecoverOrRemoveMissingAux_setup( rxTest )

	-- Create test document with aux files

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local docValue, expectedData
			= WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, docId, { hasMasks = true } )

	-- Make sure the revision does not have an Oz label.

	rxTest:waitFor( docStore:bulkSetDocValue( {
			label = 'foo',
			docValue
		} ) )

	-- Remove Aux from disk

	WFFileUtils.delete( expectedData.mask1FilePath )

	local missingAuxIds = { expectedData.mask1AuxId, expectedData.mask2AuxId }

	-- When there is no Oz revision, we expect nothing to be removed.

	local didApplyOverlay = false

	rxTest:expectSingleValue(
			WFCatErrorHandler._DEBUG_recoverOrRemoveMissingAux( catalog, docValue, missingAuxIds ),
			didApplyOverlay )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestRecoverOrRemoveMissingAux_MissingAux_OzReference( rxTest )

	local catalog, docStore = testRecoverOrRemoveMissingAux_setup( rxTest )

	-- Create test document with aux files

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local docValue, expectedData
			= WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, docId, { hasMasks = true } )

	-- Make sure the revision has an Oz label.

	rxTest:waitFor( docStore:bulkSetDocValue( {
			label = 'oz',
			docValue
		} ) )

	-- Create a new revision, with no label.

	local overlay = {
		_id = docId,
		_newRevId = KSIdGenerator.uuidInLowercaseHex(),
		_localOnly = {
			foo = true
		}
	}

	rxTest:waitFor( docStore:applyOverlay( overlay ) )

	-- Remove Aux from disk

	WFFileUtils.delete( expectedData.mask1FilePath )

	local missingAuxIds = { expectedData.mask1AuxId, expectedData.mask2AuxId }

	-- When there is an Oz revision that references the aux, it's a no-op.

	local didApplyOverlay = false

	rxTest:expectSingleValue(
			WFCatErrorHandler._DEBUG_recoverOrRemoveMissingAux( catalog, docValue, missingAuxIds ),
			didApplyOverlay )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestRecoverOrRemoveMissingAux_MissingAux_NoOzReference( rxTest )

	local catalog, docStore = testRecoverOrRemoveMissingAux_setup( rxTest )

	-- Create a doc revision with no aux files

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local docValue
			= WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, docId )

	-- Make sure the revision has an Oz label.

	rxTest:waitFor( docStore:bulkSetDocValue( {
			label = 'oz',
			docValue
		} ) )

	-- Create a new revision that HAS aux references, and with no Oz label.

	local newDocRevision, expectedData
			= WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, docId, { hasDevelopAux = true } )

	local overlay = {
		_id = docId,
		develop = newDocRevision.develop,
		aux = newDocRevision.aux
	}

	local localOnlyAuxInfo = KSTableUtils.derefOpt( newDocRevision, '_localOnly', 'files', 'aux' )
	KSTableUtils.setNested( overlay, localOnlyAuxInfo, '_localOnly', 'files', 'aux' )

	-- Remove Mask Aux 1 from disk. It should have it's references removed from the doc.

	WFFileUtils.delete( expectedData.mask1FilePath )
	
	-- Remove Filter Aux 1 from disk. It should have it's references removed from the doc.

	WFFileUtils.delete( expectedData.filter1FilePath )

	-- Apply the overlay and refresh the docValue

	rxTest:waitFor( docStore:applyOverlay( overlay ) )

	docValue = rxTest:getFirstValue( docStore:docValue( docId ) )

	-- When the Oz revision does not reference the aux, then the aux references
	-- should be removed.

	local missingAuxIds = {
		expectedData.mask1AuxId,
		expectedData.mask2AuxId,
		expectedData.filter1AuxId,
		expectedData.filter2AuxId
	}

	local didApplyOverlay = true

	rxTest:expectSingleValue( WFCatErrorHandler._DEBUG_recoverOrRemoveMissingAux( catalog, docValue, missingAuxIds )

		:map( function( response )

			return not not response

		end ), didApplyOverlay )

	-- Make sure the aux references were removed.

	rxTest:waitFor( docStore:docValue( docId )

		:map( function( updatedDoc )

			-- Missing mask aux reference should have been removed

			local hasAuxReferenceForMissingAux = not not updatedDoc.aux[ expectedData.mask1AuxId ]
			rxTest:expectEqual( false, hasAuxReferenceForMissingAux )

			local hasDevelopReferenceForMissingAux = not not updatedDoc.develop.masks[ expectedData.mask1AuxId ]
			rxTest:expectEqual( false, hasDevelopReferenceForMissingAux )

			-- Other mask aux reference should still be present

			local hasAuxReferenceForOtherAux = not not updatedDoc.aux[ expectedData.mask2AuxId ]
			rxTest:expectEqual( true, hasAuxReferenceForOtherAux )

			local hasDevelopReferenceForOtherAux = not not updatedDoc.develop.masks[ expectedData.mask2AuxId ]
			rxTest:expectEqual( true, hasDevelopReferenceForOtherAux )

			-- Missing filter aux reference should have been removed
			
			local hasAuxReferenceForMissingFilterAux = not not updatedDoc.aux[ expectedData.filter1AuxId ]
			rxTest:expectEqual( false, hasAuxReferenceForMissingFilterAux )

			local hasDevelopReferenceForMissingFilterAux = not not updatedDoc.develop.filters[ expectedData.filterGroupFingerprint ][ expectedData.filter1AuxId ]
			rxTest:expectEqual( false, hasDevelopReferenceForMissingFilterAux )

			-- Other filter aux reference should still be present

			local hasAuxReferenceForOtherFilterAux = not not updatedDoc.aux[ expectedData.filter2AuxId ]
			rxTest:expectEqual( true, hasAuxReferenceForOtherFilterAux )

			local hasDevelopReferenceForOtherFilterAux = not not updatedDoc.develop.filters[ expectedData.filterGroupFingerprint ][ expectedData.filter2AuxId ]
			rxTest:expectEqual( true, hasDevelopReferenceForOtherFilterAux )

		end ) )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestRecoverOrRemoveMissingAux_MissingAux_NoOzReferenceAndRecover( rxTest )

	local catalog, docStore = testRecoverOrRemoveMissingAux_setup( rxTest )

	-- Create a doc revision with no aux files

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local docValue
			= WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, docId, { hasMasks = false } )

	-- Make sure the revision has an Oz label.

	rxTest:waitFor( docStore:bulkSetDocValue( {
			label = 'oz',
			docValue
		} ) )

	-- Create a new revision that HAS aux references, and with no Oz label.

	local newDocRevision, expectedData
			= WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, docId, { hasMasks = true } )

	local overlay = newDocRevision

	-- Remove Aux 1 from disk. It should have it's references removed from the doc.

	WFFileUtils.delete( expectedData.mask1FilePath )

	-- Remove the local only annotation for Aux 2. It should be recovered.

	KSTableUtils.setNested( overlay, nil, '_localOnly', 'files', 'aux', expectedData.mask2Fingerprint )

	-- Apply the overlay and refresh the docValue

	rxTest:waitFor( docStore:applyOverlay( overlay ) )

	docValue = rxTest:getFirstValue( docStore:docValue( docId ) )

	-- When the Oz revision does not reference the aux, then the aux references
	-- should be removed.

	local missingAuxIds = { expectedData.mask1AuxId, expectedData.mask2AuxId }

	local didApplyOverlay = true

	rxTest:expectSingleValue( WFCatErrorHandler._DEBUG_recoverOrRemoveMissingAux( catalog, docValue, missingAuxIds )

		:map( function( response )

			return not not response

		end ), didApplyOverlay )

	-- Make sure the aux references were removed.

	rxTest:waitFor( docStore:docValue( docId )

		:map( function( updatedDoc )

			-- Missing mask aux reference should have been removed

			local hasAuxReferenceForMissingAux = not not updatedDoc.aux[ expectedData.mask1AuxId ]
			rxTest:expectEqual( false, hasAuxReferenceForMissingAux )

			local hasDevelopReferenceForMissingAux = not not updatedDoc.develop.masks[ expectedData.mask1AuxId ]
			rxTest:expectEqual( false, hasDevelopReferenceForMissingAux )

			-- Other mask aux reference should still be present

			local hasAuxReferenceForOtherAux = not not updatedDoc.aux[ expectedData.mask2AuxId ]
			rxTest:expectEqual( true, hasAuxReferenceForOtherAux )

			local hasDevelopReferenceForOtherAux = not not updatedDoc.develop.masks[ expectedData.mask2AuxId ]
			rxTest:expectEqual( true, hasDevelopReferenceForOtherAux )

			-- Other mask aux file should have been recovered, since the annotation was previously missing.

			local hasLocalOnlyAuxInfoForOtherAux
					= not not KSTableUtils.derefOpt( updatedDoc, '_localOnly', 'files', 'aux', expectedData.mask2Fingerprint )
			rxTest:expectEqual( true, hasLocalOnlyAuxInfoForOtherAux )

		end ) )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestRecoverOrRemoveMissingAux_MissingAux_NoOzReferenceAndVersions( rxTest )

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, nil, indexDefSpec )
	local docStore = catalog:docStore()

	-- Create a doc revision with no aux files

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local docValue
			= WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, docId )

	-- Make sure the revision has an Oz label.

	rxTest:waitFor( docStore:bulkSetDocValue( {
			label = 'oz',
			docValue
		} ) )

	-- Create a new revision that HAS aux references, and with no Oz label.

	local settings = {
		hasDevelopAux = true,
		sharedMasks = true,
		sharedFilters = true
	}

	local newDocRevision, localOnly
			= WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, docId, settings )

	rxTest:waitFor( docStore:applyOverlay( newDocRevision ) )

	-- Create a version that references the same aux.

	local versionId = KSIdGenerator.uuidInLowercaseHex()
	local versionDoc = WFCatVersionTestUtils.createVersionDocument( rxTest, catalog, versionId, settings )
	versionDoc.asset.id = docId
	versionDoc.asset.sha256 = newDocRevision.importSource.sha256
	versionDoc._serverProvidedMetadata = nil -- unsynced

	rxTest:waitFor( docStore:setDocValue( versionDoc ) )

	-- Remove Mask Aux 1 from disk. It should have it's references removed from the doc.

	WFFileUtils.delete( localOnly.mask1FilePath )

	-- Remove Filter Aux 1 from disk. It should have it's references removed from the doc.

	WFFileUtils.delete( localOnly.filter1FilePath )

	-- When the Oz revision does not reference the aux, then the aux references
	-- should be removed from the asset and the version documents.

	docValue = rxTest:getFirstValue( docStore:docValue( docId ) )

	local missingAuxIds = {
		localOnly.mask1AuxId,
		localOnly.mask2AuxId,
		localOnly.filter1AuxId,
		localOnly.filter2AuxId
	}

	local didApplyOverlay = true

	rxTest:expectSingleValue( WFCatErrorHandler._DEBUG_recoverOrRemoveMissingAux( catalog, docValue, missingAuxIds )

		:map( function( response )

			return not not response

		end ), didApplyOverlay )

	-- Make sure the aux references were removed from the Asset

	rxTest:waitFor( docStore:docValue( docId )

		:map( function( updatedDoc )

			-- Missing mask aux reference should have been removed

			local hasAuxReferenceForMissingAux = not not updatedDoc.aux[ localOnly.mask1AuxId ]
			rxTest:expectEqual( false, hasAuxReferenceForMissingAux )

			local hasDevelopReferenceForMissingAux = not not updatedDoc.develop.masks[ localOnly.mask1AuxId ]
			rxTest:expectEqual( false, hasDevelopReferenceForMissingAux )

			-- Other mask aux reference should still be present

			local hasAuxReferenceForOtherAux = not not updatedDoc.aux[ localOnly.mask2AuxId ]
			rxTest:expectEqual( true, hasAuxReferenceForOtherAux )

			local hasDevelopReferenceForOtherAux = not not updatedDoc.develop.masks[ localOnly.mask2AuxId ]
			rxTest:expectEqual( true, hasDevelopReferenceForOtherAux )
			
			-- Missing filter aux reference should have been removed

			local hasAuxReferenceForMissingFilterAux = not not updatedDoc.aux[ localOnly.filter1AuxId ]
			rxTest:expectEqual( false, hasAuxReferenceForMissingFilterAux )

			local hasDevelopReferenceForMissingFilterAux = not not updatedDoc.develop.filters[ localOnly.filterGroupFingerprint ][ localOnly.filter1AuxId ]
			rxTest:expectEqual( false, hasDevelopReferenceForMissingFilterAux )

			-- Other filter aux reference should still be present

			local hasAuxReferenceForOtherFilterAux = not not updatedDoc.aux[ localOnly.filter2AuxId ]
			rxTest:expectEqual( true, hasAuxReferenceForOtherFilterAux )

			local hasDevelopReferenceForOtherFilterAux = not not updatedDoc.develop.filters[ localOnly.filterGroupFingerprint ][ localOnly.filter2AuxId ]
			rxTest:expectEqual( true, hasDevelopReferenceForOtherFilterAux )

		end ) )
		
	-- Make sure the aux references were removed from the Version

	rxTest:waitFor( docStore:docValue( versionId )

		:map( function( updatedDoc )

			-- Missing mask aux reference should have been removed

			local hasDevelopReferenceForMissingAux = not not updatedDoc.develop.masks[ localOnly.mask1AuxId ]
			rxTest:expectEqual( false, hasDevelopReferenceForMissingAux )

			-- Other mastk aux reference should still be present

			local hasDevelopReferenceForOtherAux = not not updatedDoc.develop.masks[ localOnly.mask2AuxId ]
			rxTest:expectEqual( true, hasDevelopReferenceForOtherAux )

			-- Missing filter aux reference should have been removed

			local hasDevelopReferenceForMissingFilterAux = not not updatedDoc.develop.filters[ localOnly.filterGroupFingerprint ][ localOnly.filter1AuxId ]
			rxTest:expectEqual( false, hasDevelopReferenceForMissingFilterAux )

			-- Other filter aux reference should still be present

			local hasDevelopReferenceForOtherFilterAux = not not updatedDoc.develop.filters[ localOnly.filterGroupFingerprint ][ localOnly.filter2AuxId ]
			rxTest:expectEqual( true, hasDevelopReferenceForOtherFilterAux )

		end ) )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestRefineOverlayForRemovedAux_removesEmptySections( rxTest )

	-- Create a test catalog and asset doc with masks, filters, and payload auxes

	local catalogId = KSIdGenerator.uuidInLowercaseHex()
	local userId = KSIdGenerator.uuidInLowercaseHex()
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest )
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, nil, indexDefSpec )

	local docId = KSIdGenerator.uuidInLowercaseHex()

	local docValue, localOnly = WFCatAssetTestUtils.createAssetDocument( rxTest, catalog, docId, {
		hasMasks = true,
		hasFilters = true
	} )

	-- Add a non-develop aux to the doc

	local nonDevelopAuxId = KSIdGenerator.uuidInLowercaseHex()

	docValue.aux[ nonDevelopAuxId ] = {
		digest = KSIdGenerator.uuidInLowercaseHex(),
		type = 'foo'
	}

	-- Build normalizedAuxInfoList as in production

	local normalizedAuxInfoList = WFCatAssetAux.getAuxInfoList( docValue )

	-- Remove Nothing

	local overlay = { _id = docId }

	local masksOverlay = {}
	local filtersOverlay = {}
	local payloadAuxOverlay = {}

	local removedAuxIds = {}

	local result = WFCatErrorHandler._DEBUG_refineOverlayForRemovedAux(
		docValue, normalizedAuxInfoList, removedAuxIds, KSTableUtils.deepCopy( overlay ),
		masksOverlay, filtersOverlay, payloadAuxOverlay )

	rxTest:expectEqual( false, not not KSTableUtils.derefOpt( result, 'develop', 'masks', '_delete' ) )
	rxTest:expectEqual( false, not not KSTableUtils.derefOpt( result, 'develop', 'filters', '_delete' ) )
	rxTest:expectEqual( false, not not KSTableUtils.derefOpt( result, 'aux', '_delete' ) )

	-- Build some sample overlays to test

	KSTableUtils.setNested( masksOverlay, { _delete = true }, 'develop', 'masks', localOnly.mask1AuxId )
	KSTableUtils.setNested( filtersOverlay, { _delete = true }, 'develop', 'filters', localOnly.filterGroupFingerprint, localOnly.filter1AuxId )
	KSTableUtils.setNested( payloadAuxOverlay, { _delete = true }, 'aux', nonDevelopAuxId )

	removedAuxIds = {
		[ localOnly.mask1AuxId ] = true,
		[ localOnly.filter1AuxId ] = true,
		[ nonDevelopAuxId ] = true
	}

	result = WFCatErrorHandler._DEBUG_refineOverlayForRemovedAux(
		docValue, normalizedAuxInfoList, removedAuxIds, KSTableUtils.deepCopy( overlay ),
		masksOverlay, filtersOverlay, payloadAuxOverlay )

	rxTest:expectEqual( false, not not KSTableUtils.derefOpt( result, 'develop', 'masks', '_delete' ) )
	rxTest:expectEqual( false, not not KSTableUtils.derefOpt( result, 'develop', 'filters', '_delete' ) )
	rxTest:expectEqual( false, not not KSTableUtils.derefOpt( result, 'aux', '_delete' ) )

	-- Remove the rest of the masks and filters.

	KSTableUtils.setNested( masksOverlay, { _delete = true }, 'develop', 'masks', localOnly.mask2AuxId )
	KSTableUtils.setNested( filtersOverlay, { _delete = true }, 'develop', 'filters', localOnly.filterGroupFingerprint, localOnly.filter2AuxId )

	removedAuxIds = {
		[ localOnly.mask1AuxId ] = true,
		[ localOnly.mask2AuxId ] = true,
		[ localOnly.filter1AuxId ] = true,
		[ localOnly.filter2AuxId ] = true,
		[ nonDevelopAuxId ] = true
	}

	result = WFCatErrorHandler._DEBUG_refineOverlayForRemovedAux(
		docValue, normalizedAuxInfoList, removedAuxIds, KSTableUtils.deepCopy( overlay ),
		masksOverlay, filtersOverlay, payloadAuxOverlay )

	rxTest:expectEqual( true, not not KSTableUtils.derefOpt( result, 'develop', 'masks', '_delete' ) )
	rxTest:expectEqual( true, not not KSTableUtils.derefOpt( result, 'develop', 'filters', '_delete' ) )
	rxTest:expectEqual( true, not not KSTableUtils.derefOpt( result, 'aux', '_delete' ) )

end

--------------------------------------------------------------------------------

-- SPAM [jandrew 2020-11-24]: The InsufficientPermission Error Handler is tested
-- within WFCatSpaceFeedsManagerTest.

--------------------------------------------------------------------------------

return WFRxTestCase.wrapSuite( 'WFCatErrorHandlerTest', WFCatErrorHandlerTest )
