--[[----------------------------------------------------------------------------

WFCatErrorHandlerTest.lua

--------------------------------------------------------------------------------

ADOBE CONFIDENTIAL
------------------
Copyright 2016 Adobe Systems Incorporated
All Rights Reserved.

NOTICE: All information contained herein is, and remain the property of
Adobe Systems Incorporated and its suppliers, if any. The intellectual and
technical concepts contained herein are proprietary to Adobe Systems
Incorporated and its suppliers and may be covered by U.S. and Foreign
Patents, patents in process, and are protected by trade secret or copyright
law.  Dissemination of this information or reproduction of this material is
strictly forbidden unless prior written permission is obtained from
Adobe Systems Incorporated.

------------------------------------------------------------------------------]]

local Rx = require 'Rx'

local KSDate = require "KSDate"
local KSTableUtils = require 'KSTableUtils'
local WFRxTestCase = require 'WFRxTestCase'
local WFRxTestUtils = require 'WFRxTestUtils'
local WFCatSpecMock = require 'WFCatSpecMock'
local WFCatMockCatalog = require 'WFCatMockCatalog'

local WFCatErrorHandler = require 'WFCatErrorHandler'
local WFCatOzErrors = require 'WFCatOzErrors'

local WFCatTestSession = require 'WFCatTestSession'
local KSIdGenerator = require 'KSIdGenerator'
local KSJsonEncoder = require 'KSJsonEncoder'
local KSJsonDecoder = require 'KSJsonDecoder'
local WFCatMetadataPuller = require 'WFCatMetadataPuller'
local WFCatMetadataPusher = require 'WFCatMetadataPusher'
local WFMockHttpRequest = require 'WFMockHttpRequest'
local WFPlatformInfo = require 'WFPlatformInfo'

local WFArguments = require 'WFArguments'
local KSPathUtils = require 'KSPathUtils'
local WFFileUtils = require 'WFFileUtils'

local WFRxRunUntilDone = require 'WFRxRunUntilDone'

--------------------------------------------------------------------------------

local runUntilDone = WFRxRunUntilDone

local WFRxScheduler = require 'WFRxScheduler'
local WFRxBatchedBulkApplyOverlay = require 'WFRxBatchedBulkApplyOverlay'

--------------------------------------------------------------------------------

local WFCatErrorHandlerTest = {}

local function bulkDelete( catalog, docIds  )

	-- very simple bulkDelete that does not delete album_assets automatically

	local overlays = {}

	for i, docId in ipairs( docIds ) do
		overlays[ #overlays + 1 ] = {
			_id = docId,
			_deleted = true,
			_localOnly = { deletedLocally = true },
		}
	end
	
	return WFRxBatchedBulkApplyOverlay( "bulkDeleteTest", catalog:docStore(), overlays )
	
end

--------------------------------------------------------------------------------

local function deleteDoc( catalog, docId )

	return bulkDelete( catalog, { docId } )
end

--------------------------------------------------------------------------------

local function makeExternalizeSettings( sha256 ) 

	return function( catalog, assetId, maybeLimit )

		local docStore = catalog:docStore()

		return docStore:docValue( assetId )
			:flatMap( function( docValue )
				return docStore:applyOverlay {
					_id = assetId,
					develop = {
						xmpCameraRaw = {
							sha256 = sha256
						}
					}
				}
		end )

	end

end

--------------------------------------------------------------------------------

local options = {

	continuousTaskDelays = {
		progressiveBackoff = false,
		noWorkDelay = 1,
		maxNoWorkDelay = 1,
		errorDelay = 1,
		maxDelay = 1,
	},
	
	isCatalogPuller = true
	
}

--------------------------------------------------------------------------------

local errorHandlerOptions = {
	debugMode = true
}

--------------------------------------------------------------------------------

-- @mark ResourceNotFoundError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResourceNotFoundErrorSimple( rxTest )
	
	rxTest:setTimeLimit( 5 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog, 
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )
		
	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local docId = '7206c3ae9a094d69861ee8983e95085d'
	local errorDocId = 'd58059e3898ee16896d490a9ea3c6027'
	
	-- create basic 'asset' doc that has not been synced to Oz
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		},
		':ignore:' )
	
	-- create error doc for asset that indicates the resource is not found
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"unknown error",
					},
				},
				missing_resource_id = docId,
				source = {
					action = "update",
					id = docId,
					revision_id = "c1122c33d44e4477a584e494f4c57b75",
					type = "asset",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- ResourceNotFoundError for asset that is not yet on Oz should be ignored and
	-- we will assume that the condition blocking the resource getting to Oz is separate
	-- and will be cleared up eventually
	
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
	
	-- asset should not have ResourceNotFoundError annotation, it may have one for the error that
	-- is presenting the asset from getting to Oz at all, but that should not be replaced by a
	-- ResourceNotFoundError
	rxTest:expectSingleValue( 
		docStore:docValue( docId )
			:map( function( docValue )
				return KSTableUtils.derefOpt( docValue, '_localOnly', 'ozError', 'subtype' ) ~= 'ResourceNotFoundError'
			end ), true )
	
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResourceNotFoundErrorOzDeleted( rxTest )

	-- a ResourceNotFoundError for a document that used to be on Oz should cause that
	-- document to be deleted
	rxTest:setTimeLimit( 5 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local docId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	
	-- create basic 'asset' doc that has not been synced to Oz
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId,
			_newRevId = revisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			},
			_serverProvidedMetadata = {
				created = '2016-06-07T15:55:34.218872Z'
			}
		},
		':ignore:' )
	
	-- create error doc for asset that indicates the resource is not found
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"unknown error",
					},
				},
				missing_resource_id = docId,
				source = {
					action = "update",
					id = docId,
					revision_id = revisionId,
					type = "asset",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- ResourceNotFoundError for asset that is not yet on Oz should be ignored and
	-- we will assume that the condition blocking the resource getting to Oz is separate
	-- and will be cleared up eventually
	
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
	
	-- a document that Oz responds to with ResourceNotFoundError that is previously known to 
	-- have existed on Oz should be removed
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, docId ) )
		:expectDataWithinTime( 3, true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResourceNotFoundErrorCausedByDoubleDelete( rxTest )

	-- a ResourceNotFoundError for a document that used to be on Oz should cause that
	-- document to be deleted
	rxTest:setTimeLimit( 5 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local docId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	local deletedImageId = KSIdGenerator.uuidInLowercaseHex()
	
	-- create asset as appears after first delete.
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId,
			_labels = { oz = true },
		},
		':ignore:' )
	
	-- delete the asset (and create the trash asset)
	rxTest:expectSingleValue(
		docStore:applyOverlay {
			_id = docId,
			_deleted = true,
			_newId = deletedImageId,
			_newIdOverlay = { 
				type = 'asset',
				subtype = 'deleted_image',
				userCreated = '2016-06-07T15:55:34.218872Z',
				purgeDate = '2016-06-08T15:55:34.218872Z',
				original = {
					id = docId,
					userCreated = '2016-06-07T15:55:34.218872Z',
				},
			}
		},
		':ignore:' )
		
	-- create error doc for asset that indicates the resource is not found
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			_labels = { oz = true },
			_localId = 4,
			_serverProvidedMetadata = {
				new_id = deletedImageId,
				errors = {
					asset = {
						"does not exist",
					},
				},
				missing_resource_id = docId,
				source = {
					action = "delete",
					id = docId,
					revision_id = revisionId,
					type = "asset",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
			subtype = "ResourceNotFoundError",
			type = "error",
		},
		':ignore:' )
	
	-- ResourceNotFoundError for asset that has already been deleted by this
	-- client, as simulated by the conditions above, should be ignored.
	
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
	
	-- trash asset should remain in the trash after error doc is removed and
	-- error handling is complete
	rxTest:expectSingleValue( docStore:docValue( deletedImageId )
		:map( function( docValue ) 
			return docValue._deleted == nil and docValue._empty == nil
		end ), true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResourceNotFoundErrorIndirect( rxTest )
	
	-- ResourceNotFoundErrors that are indirect and not for the doc that was sent to oz (e.g.
	-- getting a ResourceNotFoundError on an album_asset create because the asset in question 
	-- does not exist on Oz (yet) should be handled by copying any error on the root asset to 
	-- the document that received the error. This allows downstream docs to be cleared when the
	-- root document does
	
	rxTest:setTimeLimit( 5 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local rootDocId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	
	-- create basic 'asset' doc that has not been synced to Oz
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = rootDocId,
			_newRevId = revisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		},
		':ignore:' )
	
	-- create error doc for asset that would cause a ResourceNotFoundError later
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "StorageExceededError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"unknown error",
					},
				},
				source = {
					action = "create",
					id = rootDocId,
					revision_id = revisionId,
					type = "asset",
				},
				status = 403,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
	
	-- asset doc should end up with quota exceeded annotation
	rxTest:expectSingleValue( 
		docStore:docValue( rootDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasStorageExceededError( docValue )
			end ), true )
			
	
	local derivedDocId = '69861ee8983e95085d7206c3ae9a094d'
	local derivedRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local derivedErrorDocId = '6896d490a9ea3c6027d58059e3898ee1'
	
	-- create the album_asset doc
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = derivedDocId,
			_newRevId = derivedRevisionId,
			album = {
				id = "12c35e04fa2b4d5b9e97b2b8423205aa",
			},
			asset = {
				id = rootDocId,
			},
			type = "album_asset",
		}, ':ignore:' )
		
		
	-- set album_asset error doc
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = derivedErrorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"resource not found",
					},
				},
				missing_resource_id = rootDocId,
				source = {
					action = "create",
					id = derivedDocId,
					revision_id = derivedRevisionId,
					type = "album_asset",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
		
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, derivedErrorDocId ) )
		:expectDataWithinTime( 5, true )
	
	-- album_asset doc should end up with quota exceeded annotation
	rxTest:expectSingleValue( 
		docStore:docValue( derivedDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasStorageExceededError( docValue )
			end ), true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResourceNotFoundErrorContributorDelete( rxTest )
	
	-- ResourceNotFoundErrors that are indirect and not for the doc that was sent to oz (e.g.
	-- getting a ResourceNotFoundError on an album_asset create because the asset in question 
	-- does not exist on Oz (yet) should be handled by copying any error on the root asset to 
	-- the document that received the error. This allows downstream docs to be cleared when the
	-- root document does
	
	rxTest:setTimeLimit( 5 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local rootDocId = KSIdGenerator.uuidInLowercaseHex()
	local trashDocId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	
	-- create basic 'asset' doc (that will be deleted below)
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = rootDocId,
			_labels = { oz = true },
		},
		':ignore:' )

	-- delete the asset (and create the trash asset)
	rxTest:expectSingleValue(
		docStore:applyOverlay {
			_id = rootDocId,
			_deleted = true,
			_newId = trashDocId,
			_newIdOverlay = {
				type = 'asset',
				subtype = 'deleted_image',
				userCreated = '2016-06-07T15:55:34.218872Z',
				purgeDate = '2016-06-08T15:55:34.218872Z',
				original = {
					id = rootDocId,
					userCreated = '2016-06-07T15:55:34.218872Z',
				},
				_localOnly = {
					universal = {
						trash = {
							createFrom = {
								id = rootDocId,
								lastAssetId = rootDocId,
								lastRevisionId = "d4a62b52683d171ee6f0197f359a4af0",
								lastTrashPurgeDate = "2019-07-12T21:32:40.415Z",
								lastTrashUserCreated = "2019-05-13T21:32:40.415Z",
								revision_id = "d4a62b52683d171ee6f0197f359a4af0",
								subtype = "image",
							},
							createWithRevId = KSIdGenerator.uuidInLowercaseHex(),
							uploaded = {
								["/rels/master"] = true,
								["/rels/proxy_type/2560"] = true,
								["/rels/xmp"] = true,
							},
						},
					},
				}
			}
		},
		':ignore:' )
	
	
	-- create error doc for asset that indicates the resource is not found and
	-- was deleted by another user
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				deleted_by = KSIdGenerator.uuidInLowercaseHex(),
				errors = {
					asset = {
						"unknown error",
					},
				},
				missing_resource_id = rootDocId,
				source = {
					action = "update",
					id = rootDocId,
					revision_id = KSIdGenerator.uuidInLowercaseHex(),
					type = "asset",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
	
	-- a document that Oz responds to with ResourceNotFoundError with a deleted_by 
	-- should ensure that any trash documents originating from the document that
	-- was deleted by the contributor are also delted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, rootDocId ) )
		:expectDataWithinTime( 3, true )
	
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, trashDocId ) )
		:expectDataWithinTime( 3, true )
	
	-- do the same basic tests again, but push the trash operations out two steps to 
	-- make sure we can follow the _newId and properly remove the doc
	rootDocId = KSIdGenerator.uuidInLowercaseHex()
	trashDocId = KSIdGenerator.uuidInLowercaseHex()
	errorDocId = KSIdGenerator.uuidInLowercaseHex()
	local trashDocId2 = KSIdGenerator.uuidInLowercaseHex()
	
	-- create basic 'asset' doc (that will be deleted below)
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = rootDocId,
			_labels = { oz = true },
		},
		':ignore:' )

	-- delete the asset (and create the trash asset)
	rxTest:expectSingleValue(
		docStore:applyOverlay {
			_id = rootDocId,
			_deleted = true,
			_newId = trashDocId,
			_newIdOverlay = {
				type = 'asset',
				subtype = 'deleted_image',
				userCreated = '2016-06-07T15:55:34.218872Z',
				purgeDate = '2016-06-08T15:55:34.218872Z',
				original = {
					id = rootDocId,
					userCreated = '2016-06-07T15:55:34.218872Z',
				},
				_localOnly = {
					universal = {
						trash = {
							createFrom = {
								id = rootDocId,
								lastAssetId = rootDocId,
								lastRevisionId = "d4a62b52683d171ee6f0197f359a4af0",
								lastTrashPurgeDate = "2019-07-12T21:32:40.415Z",
								lastTrashUserCreated = "2019-05-13T21:32:40.415Z",
								revision_id = "d4a62b52683d171ee6f0197f359a4af0",
								subtype = "image",
							},
							createWithRevId = KSIdGenerator.uuidInLowercaseHex(),
							uploaded = {
								["/rels/master"] = true,
								["/rels/proxy_type/2560"] = true,
								["/rels/xmp"] = true,
							},
						},
					},
				}
			}
		},
		':ignore:' )
		
	-- delete the trashed asset asset and create a new asset, i.e. restore
	rxTest:expectSingleValue(
		docStore:applyOverlay {
			_id = trashDocId,
			_deleted = true,
			_newId = trashDocId2,
			_newIdOverlay = {
				type = 'asset',
				subtype = 'image',
				userCreated = '2016-06-07T15:55:34.218872Z',
				purgeDate = '2016-06-08T15:55:34.218872Z',
				original = {
					id = rootDocId,
					userCreated = '2016-06-07T15:55:34.218872Z',
				},
				_localOnly = {
					universal = {
						trash = {
							createFrom = {
								id = trashDocId,
								lastAssetId = trashDocId,
								lastRevisionId = "d4a62b52683d171ee6f0197f359a4af0",
								lastTrashPurgeDate = "2019-07-12T21:32:40.415Z",
								lastTrashUserCreated = "2019-05-13T21:32:40.415Z",
								revision_id = "d4a62b52683d171ee6f0197f359a4af0",
								subtype = "deleted_image",
							},
							createWithRevId = KSIdGenerator.uuidInLowercaseHex(),
							uploaded = {
								["/rels/master"] = true,
								["/rels/proxy_type/2560"] = true,
								["/rels/xmp"] = true,
							},
						},
					},
				}
			}
		},
		':ignore:' )
	
	
	-- create error doc for asset that indicates the resource is not found and
	-- was deleted by another user
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				deleted_by = KSIdGenerator.uuidInLowercaseHex(),
				errors = {
					asset = {
						"unknown error",
					},
				},
				missing_resource_id = rootDocId,
				source = {
					action = "update",
					id = rootDocId,
					revision_id = KSIdGenerator.uuidInLowercaseHex(),
					type = "asset",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
	
	-- a document that Oz responds to with ResourceNotFoundError with a deleted_by 
	-- should ensure that any trash documents originating from the document that
	-- was deleted by the contributor are also delted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, rootDocId ) )
		:expectDataWithinTime( 3, true )
	
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, trashDocId ) )
		:expectDataWithinTime( 3, true )
		
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, trashDocId2 ) )
		:expectDataWithinTime( 3, true )
end

--------------------------------------------------------------------------------

local indexDefSpec = WFPlatformInfo.isDesktop and 'WFCatIndexDefinitions.lua' or 'LrmCatalogIndexDefinitions.lua'
local catSyncStatus =  WFPlatformInfo.isDesktop and require 'WFCatSyncStatus' or require 'LrmCatSyncStatus'

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResolveResourceNotFoundError_AlbumAsset( rxTest )
	
	rxTest:setTimeLimit( 15 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, nil, indexDefSpec )
	
	local errorHandler = WFCatErrorHandler( catalog, 
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
			getSyncStatusOfAssetsOfAlbumAssets = catSyncStatus.getSyncStatusOfAssetsOfAlbumAssets,
			getSyncStatusOfAlbumsOfAlbumAssets = catSyncStatus.getSyncStatusOfAlbumsOfAlbumAssets,
			getSyncStatusOfParentCollectionOfAlbums = catSyncStatus.getSyncStatusOfParentCollectionOfAlbums,
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local rootDocId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local albumId = KSIdGenerator.uuidInLowercaseHex()
	
	-- create basic 'asset' doc that has not been synced to Oz
	local docValue = {
		_id = rootDocId,
		_newRevId = revisionId,
		type = 'asset',
		subtype = 'image',
		importSource = {
			filePath = '/path/',
			originalHeight = 150,
			originalWidth = 100,
		}
	}

	-- Mobile: add a local master path, otherwise asset would not be "displayable"
	-- and only displayable assets have a syncStatus.
	if not WFPlatformInfo.desktop then
		docValue._localOnly = {
			-- iOS
			files = {
				master = {
					path = "..."
				}
			},
			-- Android
			originalFilePath = "...",
		}
	end

	rxTest:expectSingleValue(
		docStore:setDocValue( docValue ),
		':ignore:' )
	
	local derivedDocId = '69861ee8983e95085d7206c3ae9a094d'
	local derivedRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local derivedErrorDocId = '6896d490a9ea3c6027d58059e3898ee1'
	
	-- create the album_asset doc
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = derivedDocId,
			_newRevId = derivedRevisionId,
			album = {
				id = albumId,
			},
			asset = {
				id = rootDocId,
			},
			type = "album_asset",
		}, ':ignore:' )
		
	-- set album_asset error doc
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = derivedErrorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"resource not found",
					},
				},
				missing_resource_id = rootDocId,
				source = {
					action = "create",
					id = derivedDocId,
					revision_id = derivedRevisionId,
					type = "album_asset",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
		
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, derivedErrorDocId ) )
		:expectDataWithinTime( 5, true )
	
	rxTest:expectSingleValue( 
		docStore:docValue( derivedDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end ), true )
			
	-- Make the asset appear synced.
	local overlay = {
		_id = rootDocId,
		_serverProvidedMetadata = {
			links = {},
			revision_ids = {}
		}
	}

	-- Mobile: add a master link to let asset appear synced
	-- TODO: unify desktop and mobiles notion about what is "synced"
	if not WFPlatformInfo.isDesktop then
		overlay._serverProvidedMetadata.links = {
			[ '/rels/master'] = {
				href = "/v2/...."
			},
		}
	end

	rxTest:expectSingleValue( 
		docStore:applyOverlay( overlay ),
		':ignore:' )
			
	-- Make the album appear synced.
	
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = albumId,
			cover = {
				id = rootDocId,
			},
			name = "Sync me",
			type = "album",
			subtype = "collection",
			_serverProvidedMetadata = {
				links = {},
				revision_ids = {}
			}
		},
		':ignore:' )
		
	-- Once the album and asset are synced, the resource not found error should
	-- be removed from the album_asset.
	
	rxTest:observe( docStore:ongoingDocValue( derivedDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end )
		)
		:expectDataWithinTime( 15, false )
		:dispose()			

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResolveResourceNotFoundError_DependentVersion( rxTest )
	
	rxTest:setTimeLimit( 15 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, nil, indexDefSpec )
	
	local errorHandler = WFCatErrorHandler( catalog, 
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
			getSyncStatusOfAssetsOfAlbumAssets = catSyncStatus.getSyncStatusOfAssetsOfAlbumAssets,
			getSyncStatusOfAlbumsOfAlbumAssets = catSyncStatus.getSyncStatusOfAlbumsOfAlbumAssets,
			getSyncStatusOfParentCollectionOfAlbums = catSyncStatus.getSyncStatusOfParentCollectionOfAlbums,
			getSyncStatusOfAssetsOfVersions = catSyncStatus.getSyncStatusOfAssetsOfVersions,
			getSyncStatusOfAssetsOfComments = catSyncStatus.getSyncStatusOfAssetsOfComments,
			getSyncStatusOfAssetsOfFavorites = catSyncStatus.getSyncStatusOfAssetsOfFavorites,
			
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local rootDocId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	
	-- create basic 'asset' doc that has not been synced to Oz
	local docValue = {
		_id = rootDocId,
		_newRevId = revisionId,
		type = 'asset',
		subtype = 'image',
		importSource = {
			filePath = '/path/',
			originalHeight = 150,
			originalWidth = 100,
		}
	}

	-- Mobile: add a local master path, otherwise asset would not be "displayable"
	-- and only displayable assets have a syncStatus.
	if not WFPlatformInfo.desktop then
		docValue._localOnly = {
			-- iOS
			files = {
				master = {
					path = "..."
				}
			},
			-- Android
			originalFilePath = "...",
		}
	end

	rxTest:expectSingleValue(
		docStore:setDocValue( docValue ),
		':ignore:' )
	
	local dependentDocId = '69861ee8983e95085d7206c3ae9a094d'
	local dependentRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local dependentErrorDocId = '6896d490a9ea3c6027d58059e3898ee1'
	
	-- create the dependent doc
	
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = dependentDocId,
			_newRevId = dependentRevisionId,
			asset = {
				id = rootDocId,
			},
			type = "version",
			subtype = "edit",
		}, ':ignore:' )
		
	-- set dependent error doc
	
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = dependentErrorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"resource not found",
					},
				},
				missing_resource_id = rootDocId,
				source = {
					action = "create",
					id = dependentDocId,
					revision_id = dependentRevisionId,
					type = "version",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
		
	-- error doc should be deleted
	
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, dependentErrorDocId ) )
		:expectDataWithinTime( 5, true )
	
	rxTest:expectSingleValue( 
		docStore:docValue( dependentDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end ), true )
			
	-- Make the asset appear synced.
	
	local overlay = {
		_id = rootDocId,
		_serverProvidedMetadata = {
			links = {},
			revision_ids = {}
		}
	}

	-- Mobile: add a master link to let asset appear synced
	-- TODO: unify desktop and mobiles notion about what is "synced"
	if not WFPlatformInfo.isDesktop then
		overlay._serverProvidedMetadata.links = {
			[ '/rels/master'] = {
				href = "/v2/...."
			},
		}
	end

	rxTest:expectSingleValue( 
		docStore:applyOverlay( overlay ),
		':ignore:' )
		
	-- Once the asset is synced, the resource not found error should
	-- be removed from the dependent doc.
	
	rxTest:observe( docStore:ongoingDocValue( dependentDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end )
		)
		:expectDataWithinTime( 15, false )
		:dispose()			

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResolveResourceNotFoundError_DependentFavorite( rxTest )
	
	rxTest:setTimeLimit( 15 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, nil, indexDefSpec )
	
	local errorHandler = WFCatErrorHandler( catalog, 
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
			getSyncStatusOfAssetsOfAlbumAssets = catSyncStatus.getSyncStatusOfAssetsOfAlbumAssets,
			getSyncStatusOfAlbumsOfAlbumAssets = catSyncStatus.getSyncStatusOfAlbumsOfAlbumAssets,
			getSyncStatusOfParentCollectionOfAlbums = catSyncStatus.getSyncStatusOfParentCollectionOfAlbums,
			getSyncStatusOfAssetsOfVersions = catSyncStatus.getSyncStatusOfAssetsOfVersions,
			getSyncStatusOfAssetsOfComments = catSyncStatus.getSyncStatusOfAssetsOfComments,
			getSyncStatusOfAssetsOfFavorites = catSyncStatus.getSyncStatusOfAssetsOfFavorites,
			
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local rootDocId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	
	-- create basic 'asset' doc that has not been synced to Oz
	local docValue = {
		_id = rootDocId,
		_newRevId = revisionId,
		type = 'asset',
		subtype = 'image',
		importSource = {
			filePath = '/path/',
			originalHeight = 150,
			originalWidth = 100,
		}
	}

	-- Mobile: add a local master path, otherwise asset would not be "displayable"
	-- and only displayable assets have a syncStatus.
	if not WFPlatformInfo.desktop then
		docValue._localOnly = {
			-- iOS
			files = {
				master = {
					path = "..."
				}
			},
			-- Android
			originalFilePath = "...",
		}
	end

	rxTest:expectSingleValue(
		docStore:setDocValue( docValue ),
		':ignore:' )
	
	local dependentDocId = '69861ee8983e95085d7206c3ae9a094d'
	local dependentRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local dependentErrorDocId = '6896d490a9ea3c6027d58059e3898ee1'
	
	-- create the dependent favorite
	
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = dependentDocId,
			_newRevId = dependentRevisionId,
			userCreated = KSDate.timeInIso8601Format(),
			asset = {
				id = rootDocId,
			},
			space = {
				id = KSIdGenerator.uuidInLowercaseHex(),
			},
			type = "favorite",
			subtype = "user",
		}, ':ignore:' )
		
	-- set dependent error doc
	
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = dependentErrorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"resource not found",
					},
				},
				missing_resource_id = rootDocId,
				source = {
					action = "create",
					id = dependentDocId,
					revision_id = dependentRevisionId,
					type = "favorite",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
		
	-- error doc should be deleted
	
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, dependentErrorDocId ) )
		:expectDataWithinTime( 5, true )
	
	rxTest:expectSingleValue( 
		docStore:docValue( dependentDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end ), true )
			
	-- Make the asset appear synced.
	
	local overlay = {
		_id = rootDocId,
		_serverProvidedMetadata = {
			links = {},
			revision_ids = {}
		}
	}

	-- Mobile: add a master link to let asset appear synced
	-- TODO: unify desktop and mobiles notion about what is "synced"
	if not WFPlatformInfo.isDesktop then
		overlay._serverProvidedMetadata.links = {
			[ '/rels/master'] = {
				href = "/v2/...."
			},
		}
	end

	rxTest:expectSingleValue( 
		docStore:applyOverlay( overlay ),
		':ignore:' )
		
	-- Once the asset is synced, the resource not found error should
	-- be removed from the dependent doc.
	
	rxTest:observe( docStore:ongoingDocValue( dependentDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end )
		)
		:expectDataWithinTime( 15, false )
		:dispose()			

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResolveResourceNotFoundError_DependentComment( rxTest )
	
	rxTest:setTimeLimit( 15 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, nil, indexDefSpec )
	
	local errorHandler = WFCatErrorHandler( catalog, 
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
			getSyncStatusOfAssetsOfAlbumAssets = catSyncStatus.getSyncStatusOfAssetsOfAlbumAssets,
			getSyncStatusOfAlbumsOfAlbumAssets = catSyncStatus.getSyncStatusOfAlbumsOfAlbumAssets,
			getSyncStatusOfParentCollectionOfAlbums = catSyncStatus.getSyncStatusOfParentCollectionOfAlbums,
			getSyncStatusOfAssetsOfVersions = catSyncStatus.getSyncStatusOfAssetsOfVersions,
			getSyncStatusOfAssetsOfComments = catSyncStatus.getSyncStatusOfAssetsOfComments,
			getSyncStatusOfAssetsOfFavorites = catSyncStatus.getSyncStatusOfAssetsOfFavorites,
			
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local rootDocId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	
	-- create basic 'asset' doc that has not been synced to Oz
	local docValue = {
		_id = rootDocId,
		_newRevId = revisionId,
		type = 'asset',
		subtype = 'image',
		importSource = {
			filePath = '/path/',
			originalHeight = 150,
			originalWidth = 100,
		}
	}

	-- Mobile: add a local master path, otherwise asset would not be "displayable"
	-- and only displayable assets have a syncStatus.
	if not WFPlatformInfo.desktop then
		docValue._localOnly = {
			-- iOS
			files = {
				master = {
					path = "..."
				}
			},
			-- Android
			originalFilePath = "...",
		}
	end

	rxTest:expectSingleValue(
		docStore:setDocValue( docValue ),
		':ignore:' )
	
	local dependentDocId = '69861ee8983e95085d7206c3ae9a094d'
	local dependentRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local dependentErrorDocId = '6896d490a9ea3c6027d58059e3898ee1'
	
	-- create the dependent doc
	
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = dependentDocId,
			_newRevId = dependentRevisionId,
			userCreated = KSDate.timeInIso8601Format(),
			asset = {
				id = rootDocId,
			},
			space = {
				id = KSIdGenerator.uuidInLowercaseHex(),
			},
			type = "comment",
			subtype = "user",
		}, ':ignore:' )
		
	-- set dependent error doc
	
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = dependentErrorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"resource not found",
					},
				},
				missing_resource_id = rootDocId,
				source = {
					action = "create",
					id = dependentDocId,
					revision_id = dependentRevisionId,
					type = "comment",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
		
	-- error doc should be deleted
	
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, dependentErrorDocId ) )
		:expectDataWithinTime( 5, true )
	
	rxTest:expectSingleValue( 
		docStore:docValue( dependentDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end ), true )
			
	-- Make the asset appear synced.
	
	local overlay = {
		_id = rootDocId,
		_serverProvidedMetadata = {
			links = {},
			revision_ids = {}
		}
	}

	-- Mobile: add a master link to let asset appear synced
	-- TODO: unify desktop and mobiles notion about what is "synced"
	if not WFPlatformInfo.isDesktop then
		overlay._serverProvidedMetadata.links = {
			[ '/rels/master'] = {
				href = "/v2/...."
			},
		}
	end

	rxTest:expectSingleValue( 
		docStore:applyOverlay( overlay ),
		':ignore:' )
		
	-- Once the asset is synced, the resource not found error should
	-- be removed from the dependent doc.
	
	rxTest:observe( docStore:ongoingDocValue( dependentDocId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end )
		)
		:expectDataWithinTime( 15, false )
		:dispose()			

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResolveResourceNotFoundError_DependentDocDeletion( rxTest )
	
	rxTest:setTimeLimit( 15 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, nil, indexDefSpec )
	
	local errorHandler = WFCatErrorHandler( catalog, 
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
			getSyncStatusOfAssetsOfAlbumAssets = catSyncStatus.getSyncStatusOfAssetsOfAlbumAssets,
			getSyncStatusOfAlbumsOfAlbumAssets = catSyncStatus.getSyncStatusOfAlbumsOfAlbumAssets,
			getSyncStatusOfParentCollectionOfAlbums = catSyncStatus.getSyncStatusOfParentCollectionOfAlbums,
			getSyncStatusOfAssetsOfVersions = catSyncStatus.getSyncStatusOfAssetsOfVersions,
			getSyncStatusOfAssetsOfComments = catSyncStatus.getSyncStatusOfAssetsOfComments,
			getSyncStatusOfAssetsOfFavorites = catSyncStatus.getSyncStatusOfAssetsOfFavorites,
			
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local rootDocId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	
	-- create basic 'asset' doc that has not been synced to Oz
	local docValue = {
		_id = rootDocId,
		_newRevId = revisionId,
		type = 'asset',
		subtype = 'image',
		importSource = {
			filePath = '/path/',
			originalHeight = 150,
			originalWidth = 100,
		}
	}

	-- Mobile: add a local master path, otherwise asset would not be "displayable"
	-- and only displayable assets have a syncStatus.
	if not WFPlatformInfo.desktop then
		docValue._localOnly = {
			-- iOS
			files = {
				master = {
					path = "..."
				}
			},
			-- Android
			originalFilePath = "...",
		}
	end

	rxTest:expectSingleValue(
		docStore:setDocValue( docValue ),
		':ignore:' )
		
	-- Delete the asset document
	
	local overlay = {
		_id = rootDocId,
		_deleted = true
	}
	
	rxTest:expectSingleValue(
		docStore:applyOverlay( overlay ),
		':ignore:' )
	
	-- create the dependent doc
	
	local dependentDocId = '69861ee8983e95085d7206c3ae9a094d'
	local dependentRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local dependentErrorDocId = '6896d490a9ea3c6027d58059e3898ee1'

	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = dependentDocId,
			_newRevId = dependentRevisionId,
			asset = {
				id = rootDocId,
			},
			type = "version",
			subtype = "edit",
		}, ':ignore:' )
		
	-- set dependent error doc
	
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = dependentErrorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"resource not found",
					},
				},
				missing_resource_id = rootDocId,
				source = {
					action = "create",
					id = dependentDocId,
					revision_id = dependentRevisionId,
					type = "version",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
		
	-- error doc and dependent doc should be deleted
	
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, dependentErrorDocId ) )
		:expectDataWithinTime( 5, true )
	
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, dependentDocId ) )
		:expectDataWithinTime( 5, true )	

end


--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResolveResourceNotFoundError_Album( rxTest )
	
	rxTest:setTimeLimit( 15 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, nil, indexDefSpec )
	
	local errorHandler = WFCatErrorHandler( catalog, 
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
			getSyncStatusOfAssetsOfAlbumAssets = catSyncStatus.getSyncStatusOfAssetsOfAlbumAssets,
			getSyncStatusOfAlbumsOfAlbumAssets = catSyncStatus.getSyncStatusOfAlbumsOfAlbumAssets,
			getSyncStatusOfParentCollectionOfAlbums = catSyncStatus.getSyncStatusOfParentCollectionOfAlbums,
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local parentId = KSIdGenerator.uuidInLowercaseHex()
	
	-- create a parent 'collection_set' doc that has not been synced to Oz
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = parentId,
			name = "Collection Set",
			type = "album",
			subtype = "collection_set",
		},
		':ignore:' )
	
	local childId = '69861ee8983e95085d7206c3ae9a094d'
	local childRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = '6896d490a9ea3c6027d58059e3898ee1'
	
	-- create the child 'album'
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = childId,
			_newRevId = childRevisionId,
			parent = {
				id = parentId,
			},
			name = "Sync me",
			type = "album",
			subtype = "collection",
		}, ':ignore:' )
		
	-- set child album error doc
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ResourceNotFoundError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"resource not found",
					},
				},
				missing_resource_id = parentId,
				source = {
					action = "create",
					id = childId,
					revision_id = childRevisionId,
					type = "collection_set",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
		
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 5, true )
	
	rxTest:expectSingleValue( 
		docStore:docValue( childId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end ), true )
			
	-- Make the parent collection_set appear synced.
	rxTest:expectSingleValue( 
		docStore:applyOverlay {
			_id = parentId,
			_serverProvidedMetadata = {
				links = {},
				revision_ids = {}
			}
		},
		':ignore:' )

	-- Once the collection_set is synced, the resource not found error should
	-- be removed from the child album.
	
	rxTest:observe( docStore:ongoingDocValue( childId )
			:map( function( docValue )
				return WFCatOzErrors.hasResourceNotFoundError( docValue )
			end )
		)
		:expectDataWithinTime( 15, false )
		:dispose()
		
end

--------------------------------------------------------------------------------

-- @mark InlineXMPDevelopSettingTooBigError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestInlineXmpDevelopSettingTooBigError( rxTest )
	
	rxTest:setTimeLimit( 5 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog, { 
			externalizeSettings = makeExternalizeSettings( "665407789296dc485e3e623a022615d7fd06f0ea9d5e81406f267ecf2a921749" ),
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
		}, 
		errorHandlerOptions )
		
	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local docId = '7206c3ae9a094d69861ee8983e95085d'
	local errorDocId = 'd58059e3898ee16896d490a9ea3c6027'
	
	-- create basic 'asset' doc
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId,
			_newRevId = "c1122c33d44e4477a584e494f4c57b75",
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			},
			develop = {
				xmpCameraRaw = '<xmpmeta />',
			},
		},
		':ignore:' )
	
	-- create error doc for asset
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "InlineXmpDevelopSettingTooBigError",
			type = "error",
			_serverProvidedMetadata = {
				limit = 10240,
				errors = {
					xmpCameraRaw = {
						"is too big",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = "c1122c33d44e4477a584e494f4c57b75",
					type = "asset",
				},
				status = 400,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
	
	-- asset doc should end up with develop settings as sha256
	rxTest:expectSingleValue( 
		docStore:docValue( docId )
			:map( function( docValue )
				return ( KSTableUtils.derefOpt( docValue, 'develop', 'xmpCameraRaw', 'sha256' ) == '665407789296dc485e3e623a022615d7fd06f0ea9d5e81406f267ecf2a921749' )
			end ), true )
	 
end

--------------------------------------------------------------------------------


function WFCatErrorHandlerTest.rxTestInlineXmpDevelopSettingTooBigError_Version( rxTest )
	
	rxTest:setTimeLimit( 5 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog, { 
		externalizeSettings = makeExternalizeSettings( "665407789296dc485e3e623a022615d7fd06f0ea9d5e81406f267ecf2a921749" ),
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
		}, 
		errorHandlerOptions )
	
	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local docId = '7206c3ae9a094d69861ee8983e95085d'
	local errorDocId = 'd58059e3898ee16896d490a9ea3c6027'
	
	-- create basic 'version' doc
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId,
			_newRevId = "c1122c33d44e4477a584e494f4c57b75",
			type = 'version',
			subtype = 'edit',
			develop = {
				xmpCameraRaw = '<xmpmeta />',
			},
		},
		':ignore:' )
	
	-- create error doc for version
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "InlineXmpDevelopSettingTooBigError",
			type = "error",
			_serverProvidedMetadata = {
				limit = 10240,
				errors = {
					xmpCameraRaw = {
						"is too big",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = "c1122c33d44e4477a584e494f4c57b75",
					type = "version",
				},
				status = 400,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
	
	-- version doc should end up with develop settings as sha256
	rxTest:expectSingleValue( 
		docStore:docValue( docId )
			:map( function( docValue )
				return ( KSTableUtils.derefOpt( docValue, 'develop', 'xmpCameraRaw', 'sha256' ) == '665407789296dc485e3e623a022615d7fd06f0ea9d5e81406f267ecf2a921749' )
			end ), true )
	 
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestInlineXmpDevelopSettingTooBigError_testNightly( rxTest )

	rxTest:setTimeLimit( 60 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog
	
	rxTest:expectSingleValue(

		catalogTestSession:catalog()
			:tap( function( _catalog )
				catalog = _catalog
			end ),

		':ignore:' )

	local docStore = catalog:docStore()
	
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	
	local errorHandler = WFCatErrorHandler( catalog, {
		externalizeSettings = makeExternalizeSettings( "665407789296dc485e3e623a022615d7fd06f0ea9d5e81406f267ecf2a921749" ),
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
		}, 
		errorHandlerOptions )

	rxTest:finally( function()
		errorHandler:close()
		puller:close()
		pusher:close()
		
	end )
	
	puller:getUpdatesContinuously()
	pusher:sendUpdatesContinuously()
	
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	rxTest:expectSingleValue(
		docStore:setDocValue {
				_id = assetId,
				type = 'asset',
				subtype = 'image',
				captureDate = "0000-00-00T00:00:00",
				importSource = {
					importTimestamp = '2017-03-14T10:50:59.000Z', -- seconds has been adjusted
					filePath = '/path/',
					fileName = 'fileName.jpg',
					importedBy = KSIdGenerator.uuidInLowercaseHex(),
					originalHeight = 150,
					originalWidth = 100,
				},
				ratings = {
					[ 'aec4af47d8114158ac494aa530848430' ] = {
						rating = 5,
					}
				},
				develop = {
					processingModel = 'lightroom',
					userUpdated = "2017-02-22T16:29:42.616Z",
					-- this is 11KB of settings
					xmpCameraRaw = [[<x:xmpmeta xmlns:x="adobe:ns:meta/" x:xmptk="Adobe XMP Core 5.6-c128 79.159124, 2016/03/18-14:01:55        ">
										 <rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
										  <rdf:Description rdf:about=""
										    xmlns:crs="http://ns.adobe.com/camera-raw-settings/1.0/"
										   crs:Version="9.9"
										   crs:ProcessVersion="5.7"
										   crs:WhiteBalance="Custom"
										   crs:AutoWhiteVersion="134348800"
										   crs:Temperature="3250"
										   crs:Tint="-31"
										   crs:Exposure="+0.25"
										   crs:Shadows="5"
										   crs:Brightness="+50"
										   crs:Contrast="+25"
										   crs:Saturation="0"
										   crs:Sharpness="25"
										   crs:LuminanceSmoothing="0"
										   crs:ColorNoiseReduction="25"
										   crs:VignetteAmount="0"
										   crs:ShadowTint="0"
										   crs:RedHue="0"
										   crs:RedSaturation="0"
										   crs:GreenHue="0"
										   crs:GreenSaturation="0"
										   crs:BlueHue="0"
										   crs:BlueSaturation="0"
										   crs:FillLight="0"
										   crs:Vibrance="0"
										   crs:HighlightRecovery="0"
										   crs:Clarity="0"
										   crs:HueAdjustmentRed="0"
										   crs:HueAdjustmentOrange="0"
										   crs:HueAdjustmentYellow="0"
										   crs:HueAdjustmentGreen="0"
										   crs:HueAdjustmentAqua="0"
										   crs:HueAdjustmentBlue="0"
										   crs:HueAdjustmentPurple="0"
										   crs:HueAdjustmentMagenta="0"
										   crs:SaturationAdjustmentRed="0"
										   crs:SaturationAdjustmentOrange="0"
										   crs:SaturationAdjustmentYellow="0"
										   crs:SaturationAdjustmentGreen="0"
										   crs:SaturationAdjustmentAqua="0"
										   crs:SaturationAdjustmentBlue="0"
										   crs:SaturationAdjustmentPurple="0"
										   crs:SaturationAdjustmentMagenta="0"
										   crs:LuminanceAdjustmentRed="0"
										   crs:LuminanceAdjustmentOrange="0"
										   crs:LuminanceAdjustmentYellow="0"
										   crs:LuminanceAdjustmentGreen="0"
										   crs:LuminanceAdjustmentAqua="0"
										   crs:LuminanceAdjustmentBlue="0"
										   crs:LuminanceAdjustmentPurple="0"
										   crs:LuminanceAdjustmentMagenta="0"
										   crs:SplitToningShadowHue="0"
										   crs:SplitToningShadowSaturation="0"
										   crs:SplitToningHighlightHue="0"
										   crs:SplitToningHighlightSaturation="0"
										   crs:SplitToningBalance="0"
										   crs:ParametricShadows="0"
										   crs:ParametricDarks="0"
										   crs:ParametricLights="0"
										   crs:ParametricHighlights="0"
										   crs:ParametricShadowSplit="25"
										   crs:ParametricMidtoneSplit="50"
										   crs:ParametricHighlightSplit="75"
										   crs:SharpenRadius="+1.0"
										   crs:SharpenDetail="25"
										   crs:SharpenEdgeMasking="0"
										   crs:PostCropVignetteAmount="0"
										   crs:GrainAmount="0"
										   crs:ColorNoiseReductionDetail="50"
										   crs:ColorNoiseReductionSmoothness="50"
										   crs:LensProfileEnable="0"
										   crs:LensManualDistortionAmount="0"
										   crs:PerspectiveVertical="0"
										   crs:PerspectiveHorizontal="0"
										   crs:PerspectiveRotate="0.0"
										   crs:PerspectiveScale="100"
										   crs:PerspectiveAspect="0"
										   crs:PerspectiveUpright="0"
										   crs:PerspectiveX="0.00"
										   crs:PerspectiveY="0.00"
										   crs:AutoLateralCA="0"
										   crs:DefringePurpleAmount="0"
										   crs:DefringePurpleHueLo="30"
										   crs:DefringePurpleHueHi="70"
										   crs:DefringeGreenAmount="0"
										   crs:DefringeGreenHueLo="40"
										   crs:DefringeGreenHueHi="60"
										   crs:ConvertToGrayscale="False"
										   crs:ToneCurveName="Medium Contrast"
										   crs:ToneCurveName2012="Linear"
										   crs:CameraProfile="Adobe Standard"
										   crs:CameraProfileDigest="730B5111DF7449A8EE0790633B8D3861"
										   crs:LensProfileSetup="LensDefaults"
										   crs:UprightVersion="151388160"
										   crs:UprightCenterMode="0"
										   crs:UprightCenterNormX="0.5"
										   crs:UprightCenterNormY="0.5"
										   crs:UprightFocalMode="0"
										   crs:UprightFocalLength35mm="35"
										   crs:UprightPreview="False"
										   crs:UprightTransformCount="6"
										   crs:UprightFourSegmentsCount="0"
										   crs:HasSettings="True"
										   crs:AlreadyApplied="False">
										   <crs:ToneCurve>
										    <rdf:Seq>
										     <rdf:li>0, 0</rdf:li>
										     <rdf:li>32, 22</rdf:li>
										     <rdf:li>64, 56</rdf:li>
										     <rdf:li>128, 128</rdf:li>
										     <rdf:li>192, 196</rdf:li>
										     <rdf:li>255, 255</rdf:li>
										    </rdf:Seq>
										   </crs:ToneCurve>
										   <crs:ToneCurveRed>
										    <rdf:Seq>
										     <rdf:li>0, 0</rdf:li>
										     <rdf:li>255, 255</rdf:li>
										    </rdf:Seq>
										   </crs:ToneCurveRed>
										   <crs:ToneCurveGreen>
										    <rdf:Seq>
										     <rdf:li>0, 0</rdf:li>
										     <rdf:li>255, 255</rdf:li>
										    </rdf:Seq>
										   </crs:ToneCurveGreen>
										   <crs:ToneCurveBlue>
										    <rdf:Seq>
										     <rdf:li>0, 0</rdf:li>
										     <rdf:li>255, 255</rdf:li>
										    </rdf:Seq>
										   </crs:ToneCurveBlue>
										   <crs:ToneCurvePV2012>
										    <rdf:Seq>
										     <rdf:li>0, 0</rdf:li>
										     <rdf:li>255, 255</rdf:li>
										    </rdf:Seq>
										   </crs:ToneCurvePV2012>
										   <crs:ToneCurvePV2012Red>
										    <rdf:Seq>
										     <rdf:li>0, 0</rdf:li>
										     <rdf:li>255, 255</rdf:li>
										    </rdf:Seq>
										   </crs:ToneCurvePV2012Red>
										   <crs:ToneCurvePV2012Green>
										    <rdf:Seq>
										     <rdf:li>0, 0</rdf:li>
										     <rdf:li>255, 255</rdf:li>
										    </rdf:Seq>
										   </crs:ToneCurvePV2012Green>
										   <crs:ToneCurvePV2012Blue>
										    <rdf:Seq>
										     <rdf:li>0, 0</rdf:li>
										     <rdf:li>255, 255</rdf:li>
										    </rdf:Seq>
										   </crs:ToneCurvePV2012Blue>
										   <crs:PaintBasedCorrections>
										    <rdf:Seq>
										     <rdf:li>
										      <rdf:Description
										       crs:What="Correction"
										       crs:CorrectionAmount="1.000000"
										       crs:CorrectionActive="true"
										       crs:LocalExposure="0.552941"
										       crs:LocalSaturation="0.000000"
										       crs:LocalContrast="0.000000"
										       crs:LocalClarity="0.000000"
										       crs:LocalSharpness="0.000000"
										       crs:LocalBrightness="0.000000"
										       crs:LocalToningHue="0.000000"
										       crs:LocalToningSaturation="0.000000">
										      <crs:CorrectionMasks>
										       <rdf:Seq>
										        <rdf:li>
										         <rdf:Description
										          crs:What="Mask/Paint"
										          crs:MaskValue="0.597701"
										          crs:Radius="0.294229"
										          crs:Flow="0.374138"
										          crs:CenterWeight="0.336813">
										         <crs:Dabs>
										          <rdf:Seq>
										           <rdf:li>d 0.583691 0.603004</rdf:li>
										           <rdf:li>d 0.506847 0.542476</rdf:li>
										           <rdf:li>d 0.445992 0.503752</rdf:li>
										           <rdf:li>d 0.510425 0.594056</rdf:li>
										           <rdf:li>d 0.589165 0.649437</rdf:li>
										           <rdf:li>d 0.590581 0.680258</rdf:li>
										           <rdf:li>d 0.503070 0.664129</rdf:li>
										           <rdf:li>d 0.417757 0.631625</rdf:li>
										           <rdf:li>d 0.345171 0.564655</rdf:li>
										           <rdf:li>d 0.366658 0.567833</rdf:li>
										           <rdf:li>d 0.436027 0.649538</rdf:li>
										           <rdf:li>d 0.501659 0.737556</rdf:li>
										           <rdf:li>d 0.571560 0.818406</rdf:li>
										           <rdf:li>d 0.612505 0.805913</rdf:li>
										           <rdf:li>d 0.543371 0.723593</rdf:li>
										           <rdf:li>d 0.475483 0.639351</rdf:li>
										           <rdf:li>d 0.451223 0.540773</rdf:li>
										           <rdf:li>d 0.525914 0.601557</rdf:li>
										           <rdf:li>d 0.544495 0.599863</rdf:li>
										           <rdf:li>d 0.490855 0.494746</rdf:li>
										           <rdf:li>d 0.427792 0.403438</rdf:li>
										           <rdf:li>d 0.358655 0.321123</rdf:li>
										           <rdf:li>d 0.289519 0.238807</rdf:li>
										           <rdf:li>d 0.220372 0.156511</rdf:li>
										           <rdf:li>d 0.151199 0.074263</rdf:li>
										           <rdf:li>d 0.079237 -0.002181</rdf:li>
										           <rdf:li>d 0.005210 -0.074281</rdf:li>
										           <rdf:li>d -0.070480 -0.142402</rdf:li>
										           <rdf:li>d -0.102730 -0.153072</rdf:li>
										           <rdf:li>d -0.038006 -0.063046</rdf:li>
										           <rdf:li>d 0.026719 0.026980</rdf:li>
										           <rdf:li>d 0.091444 0.117007</rdf:li>
										           <rdf:li>d 0.156168 0.207033</rdf:li>
										           <rdf:li>d 0.213520 0.287557</rdf:li>
										           <rdf:li>d 0.249205 0.406728</rdf:li>
										           <rdf:li>d 0.309851 0.502384</rdf:li>
										           <rdf:li>d 0.386433 0.567872</rdf:li>
										           <rdf:li>d 0.466134 0.623738</rdf:li>
										           <rdf:li>d 0.434471 0.647575</rdf:li>
										           <rdf:li>d 0.352438 0.611033</rdf:li>
										           <rdf:li>d 0.294644 0.564284</rdf:li>
										           <rdf:li>d 0.352680 0.663384</rdf:li>
										           <rdf:li>d 0.402755 0.772418</rdf:li>
										           <rdf:li>d 0.464043 0.866000</rdf:li>
										           <rdf:li>d 0.550064 0.883831</rdf:li>
										           <rdf:li>d 0.638275 0.879062</rdf:li>
										           <rdf:li>d 0.622170 0.812559</rdf:li>
										           <rdf:li>d 0.542796 0.755269</rdf:li>
										           <rdf:li>d 0.457879 0.722551</rdf:li>
										           <rdf:li>d 0.370080 0.708936</rdf:li>
										           <rdf:li>d 0.282253 0.695747</rdf:li>
										           <rdf:li>d 0.194528 0.682390</rdf:li>
										           <rdf:li>d 0.251015 0.681170</rdf:li>
										           <rdf:li>d 0.339248 0.684909</rdf:li>
										           <rdf:li>d 0.427482 0.688648</rdf:li>
										           <rdf:li>d 0.509635 0.735799</rdf:li>
										           <rdf:li>d 0.591826 0.783998</rdf:li>
										           <rdf:li>d 0.664021 0.855778</rdf:li>
										           <rdf:li>d 0.665281 0.832685</rdf:li>
										           <rdf:li>d 0.601375 0.741388</rdf:li>
										           <rdf:li>d 0.525838 0.674864</rdf:li>
										           <rdf:li>d 0.446731 0.616125</rdf:li>
										           <rdf:li>d 0.367624 0.557387</rdf:li>
										           <rdf:li>d 0.288517 0.498648</rdf:li>
										           <rdf:li>d 0.209410 0.439909</rdf:li>
										           <rdf:li>d 0.127992 0.391444</rdf:li>
										           <rdf:li>d 0.040708 0.372114</rdf:li>
										           <rdf:li>d 0.072242 0.419354</rdf:li>
										           <rdf:li>d 0.148677 0.485448</rdf:li>
										           <rdf:li>d 0.225726 0.550039</rdf:li>
										           <rdf:li>d 0.303516 0.612609</rdf:li>
										           <rdf:li>d 0.381858 0.672870</rdf:li>
										           <rdf:li>d 0.456527 0.743481</rdf:li>
										           <rdf:li>d 0.531195 0.814091</rdf:li>
										           <rdf:li>d 0.608030 0.832618</rdf:li>
										           <rdf:li>d 0.601863 0.768196</rdf:li>
										          </rdf:Seq>
										         </crs:Dabs>
										         </rdf:Description>
										        </rdf:li>
										        <rdf:li>
										         <rdf:Description
										          crs:What="Mask/Paint"
										          crs:MaskValue="0.597701"
										          crs:Radius="0.294229"
										          crs:Flow="0.374138"
										          crs:CenterWeight="0.336813">
										         <crs:Dabs>
										          <rdf:Seq>
										           <rdf:li>d 0.806867 0.881974</rdf:li>
										           <rdf:li>d 0.813094 0.752712</rdf:li>
										           <rdf:li>d 0.798072 0.622405</rdf:li>
										           <rdf:li>d 0.784627 0.492781</rdf:li>
										           <rdf:li>d 0.725046 0.398217</rdf:li>
										           <rdf:li>d 0.659724 0.309852</rdf:li>
										           <rdf:li>d 0.586062 0.238182</rdf:li>
										           <rdf:li>d 0.508032 0.178908</rdf:li>
										           <rdf:li>d 0.426434 0.132882</rdf:li>
										          </rdf:Seq>
										         </crs:Dabs>
										         </rdf:Description>
										        </rdf:li>
										        <rdf:li>
										         <rdf:Description
										          crs:What="Mask/Paint"
										          crs:MaskValue="0.597701"
										          crs:Radius="0.294229"
										          crs:Flow="0.374138"
										          crs:CenterWeight="0.336813">
										         <crs:Dabs>
										          <rdf:Seq>
										           <rdf:li>d 0.868383 0.993562</rdf:li>
										           <rdf:li>d 0.857835 0.862179</rdf:li>
										           <rdf:li>d 0.838056 0.733487</rdf:li>
										           <rdf:li>d 0.802776 0.612341</rdf:li>
										           <rdf:li>d 0.768633 0.490447</rdf:li>
										           <rdf:li>d 0.727548 0.373261</rdf:li>
										           <rdf:li>d 0.689283 0.254060</rdf:li>
										           <rdf:li>d 0.624340 0.167826</rdf:li>
										           <rdf:li>d 0.544827 0.110515</rdf:li>
										           <rdf:li>d 0.462461 0.063275</rdf:li>
										          </rdf:Seq>
										         </crs:Dabs>
										         </rdf:Description>
										        </rdf:li>
										       </rdf:Seq>
										      </crs:CorrectionMasks>
										      </rdf:Description>
										     </rdf:li>
										    </rdf:Seq>
										   </crs:PaintBasedCorrections>
										  </rdf:Description>
										 </rdf:RDF>
										</x:xmpmeta>
										]],
				},
			}, ':ignore:' )
	
	
	-- asset doc should end up with develop settings as sha256
	local assetSha256 = rxTest:observe( 
							docStore:ongoingDocValue( assetId )
								:map( function( docValue )
									return ( KSTableUtils.derefOpt( docValue, 'develop', 'xmpCameraRaw', 'sha256' ) == '665407789296dc485e3e623a022615d7fd06f0ea9d5e81406f267ecf2a921749' )
								end ) )
								
	assetSha256:expectDataWithinTime( 40, true )
	
	local assetOnOz = rxTest:observe( 
							docStore:ongoingDocValue( assetId )
								:map( function( docValue )
									return docValue._serverProvidedMetadata ~= nil
								end ) )
								
	assetOnOz:expectDataWithinTime( 40, true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestInlineXmpDevelopSettingTooBigError_Version_testNightly( rxTest )

	rxTest:setTimeLimit( 60 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog
	
	rxTest:expectSingleValue(

		catalogTestSession:catalog()
			:tap( function( _catalog )
				catalog = _catalog
			end ),

		':ignore:' )

	local docStore = catalog:docStore()
	
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	
	local errorHandler = WFCatErrorHandler( catalog, {
		externalizeSettings = makeExternalizeSettings( "665407789296dc485e3e623a022615d7fd06f0ea9d5e81406f267ecf2a921749" ),
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc,
		}, 
		errorHandlerOptions )

	rxTest:finally( function()
		errorHandler:close()
		puller:close()
		pusher:close()
		
	end )
	
	puller:getUpdatesContinuously()
	pusher:sendUpdatesContinuously()
	
	local sha256 = 'f9b430859b70d4d2673051172ac9bf8861ea398cbe7ee413d5a47fa49b7855c1'
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	rxTest:expectSingleValue(
		docStore:setDocValue {
				_id = assetId,
				type = 'asset',
				subtype = 'image',
				captureDate = "0000-00-00T00:00:00",
				importSource = {
					importTimestamp = '2017-03-14T10:50:59.000Z', -- seconds has been adjusted
					filePath = '/path/',
					fileName = 'fileName.jpg',
					importedBy = KSIdGenerator.uuidInLowercaseHex(),
					sha256 = sha256,
					originalHeight = 150,
					originalWidth = 100,
				},
				ratings = {
					[ 'aec4af47d8114158ac494aa530848430' ] = {
						rating = 5,
					}
				},
				develop = {
					processingModel = 'lightroom',
					userUpdated = "2017-02-22T16:29:42.616Z",
					xmpCameraRaw = "<xmp/>",
				},
			}, ':ignore:' )
			
	local versionId = KSIdGenerator.uuidInLowercaseHex()
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = versionId,
			type = 'version',
			subtype = 'edit',
			asset = {
				id = assetId,
				sha256 = sha256,
			},
			userCreated = KSDate.timeInIso8601Format(),
			userUpdated = KSDate.timeInIso8601Format(),
			develop = {
				processingModel = 'lightroom',
				userUpdated = "2017-02-22T16:29:42.616Z",
				-- this is 11KB of settings
				xmpCameraRaw = [[<x:xmpmeta xmlns:x="adobe:ns:meta/" x:xmptk="Adobe XMP Core 5.6-c128 79.159124, 2016/03/18-14:01:55        ">
									 <rdf:RDF xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#">
									  <rdf:Description rdf:about=""
									    xmlns:crs="http://ns.adobe.com/camera-raw-settings/1.0/"
									   crs:Version="9.9"
									   crs:ProcessVersion="5.7"
									   crs:WhiteBalance="Custom"
									   crs:AutoWhiteVersion="134348800"
									   crs:Temperature="3250"
									   crs:Tint="-31"
									   crs:Exposure="+0.25"
									   crs:Shadows="5"
									   crs:Brightness="+50"
									   crs:Contrast="+25"
									   crs:Saturation="0"
									   crs:Sharpness="25"
									   crs:LuminanceSmoothing="0"
									   crs:ColorNoiseReduction="25"
									   crs:VignetteAmount="0"
									   crs:ShadowTint="0"
									   crs:RedHue="0"
									   crs:RedSaturation="0"
									   crs:GreenHue="0"
									   crs:GreenSaturation="0"
									   crs:BlueHue="0"
									   crs:BlueSaturation="0"
									   crs:FillLight="0"
									   crs:Vibrance="0"
									   crs:HighlightRecovery="0"
									   crs:Clarity="0"
									   crs:HueAdjustmentRed="0"
									   crs:HueAdjustmentOrange="0"
									   crs:HueAdjustmentYellow="0"
									   crs:HueAdjustmentGreen="0"
									   crs:HueAdjustmentAqua="0"
									   crs:HueAdjustmentBlue="0"
									   crs:HueAdjustmentPurple="0"
									   crs:HueAdjustmentMagenta="0"
									   crs:SaturationAdjustmentRed="0"
									   crs:SaturationAdjustmentOrange="0"
									   crs:SaturationAdjustmentYellow="0"
									   crs:SaturationAdjustmentGreen="0"
									   crs:SaturationAdjustmentAqua="0"
									   crs:SaturationAdjustmentBlue="0"
									   crs:SaturationAdjustmentPurple="0"
									   crs:SaturationAdjustmentMagenta="0"
									   crs:LuminanceAdjustmentRed="0"
									   crs:LuminanceAdjustmentOrange="0"
									   crs:LuminanceAdjustmentYellow="0"
									   crs:LuminanceAdjustmentGreen="0"
									   crs:LuminanceAdjustmentAqua="0"
									   crs:LuminanceAdjustmentBlue="0"
									   crs:LuminanceAdjustmentPurple="0"
									   crs:LuminanceAdjustmentMagenta="0"
									   crs:SplitToningShadowHue="0"
									   crs:SplitToningShadowSaturation="0"
									   crs:SplitToningHighlightHue="0"
									   crs:SplitToningHighlightSaturation="0"
									   crs:SplitToningBalance="0"
									   crs:ParametricShadows="0"
									   crs:ParametricDarks="0"
									   crs:ParametricLights="0"
									   crs:ParametricHighlights="0"
									   crs:ParametricShadowSplit="25"
									   crs:ParametricMidtoneSplit="50"
									   crs:ParametricHighlightSplit="75"
									   crs:SharpenRadius="+1.0"
									   crs:SharpenDetail="25"
									   crs:SharpenEdgeMasking="0"
									   crs:PostCropVignetteAmount="0"
									   crs:GrainAmount="0"
									   crs:ColorNoiseReductionDetail="50"
									   crs:ColorNoiseReductionSmoothness="50"
									   crs:LensProfileEnable="0"
									   crs:LensManualDistortionAmount="0"
									   crs:PerspectiveVertical="0"
									   crs:PerspectiveHorizontal="0"
									   crs:PerspectiveRotate="0.0"
									   crs:PerspectiveScale="100"
									   crs:PerspectiveAspect="0"
									   crs:PerspectiveUpright="0"
									   crs:PerspectiveX="0.00"
									   crs:PerspectiveY="0.00"
									   crs:AutoLateralCA="0"
									   crs:DefringePurpleAmount="0"
									   crs:DefringePurpleHueLo="30"
									   crs:DefringePurpleHueHi="70"
									   crs:DefringeGreenAmount="0"
									   crs:DefringeGreenHueLo="40"
									   crs:DefringeGreenHueHi="60"
									   crs:ConvertToGrayscale="False"
									   crs:ToneCurveName="Medium Contrast"
									   crs:ToneCurveName2012="Linear"
									   crs:CameraProfile="Adobe Standard"
									   crs:CameraProfileDigest="730B5111DF7449A8EE0790633B8D3861"
									   crs:LensProfileSetup="LensDefaults"
									   crs:UprightVersion="151388160"
									   crs:UprightCenterMode="0"
									   crs:UprightCenterNormX="0.5"
									   crs:UprightCenterNormY="0.5"
									   crs:UprightFocalMode="0"
									   crs:UprightFocalLength35mm="35"
									   crs:UprightPreview="False"
									   crs:UprightTransformCount="6"
									   crs:UprightFourSegmentsCount="0"
									   crs:HasSettings="True"
									   crs:AlreadyApplied="False">
									   <crs:ToneCurve>
									    <rdf:Seq>
									     <rdf:li>0, 0</rdf:li>
									     <rdf:li>32, 22</rdf:li>
									     <rdf:li>64, 56</rdf:li>
									     <rdf:li>128, 128</rdf:li>
									     <rdf:li>192, 196</rdf:li>
									     <rdf:li>255, 255</rdf:li>
									    </rdf:Seq>
									   </crs:ToneCurve>
									   <crs:ToneCurveRed>
									    <rdf:Seq>
									     <rdf:li>0, 0</rdf:li>
									     <rdf:li>255, 255</rdf:li>
									    </rdf:Seq>
									   </crs:ToneCurveRed>
									   <crs:ToneCurveGreen>
									    <rdf:Seq>
									     <rdf:li>0, 0</rdf:li>
									     <rdf:li>255, 255</rdf:li>
									    </rdf:Seq>
									   </crs:ToneCurveGreen>
									   <crs:ToneCurveBlue>
									    <rdf:Seq>
									     <rdf:li>0, 0</rdf:li>
									     <rdf:li>255, 255</rdf:li>
									    </rdf:Seq>
									   </crs:ToneCurveBlue>
									   <crs:ToneCurvePV2012>
									    <rdf:Seq>
									     <rdf:li>0, 0</rdf:li>
									     <rdf:li>255, 255</rdf:li>
									    </rdf:Seq>
									   </crs:ToneCurvePV2012>
									   <crs:ToneCurvePV2012Red>
									    <rdf:Seq>
									     <rdf:li>0, 0</rdf:li>
									     <rdf:li>255, 255</rdf:li>
									    </rdf:Seq>
									   </crs:ToneCurvePV2012Red>
									   <crs:ToneCurvePV2012Green>
									    <rdf:Seq>
									     <rdf:li>0, 0</rdf:li>
									     <rdf:li>255, 255</rdf:li>
									    </rdf:Seq>
									   </crs:ToneCurvePV2012Green>
									   <crs:ToneCurvePV2012Blue>
									    <rdf:Seq>
									     <rdf:li>0, 0</rdf:li>
									     <rdf:li>255, 255</rdf:li>
									    </rdf:Seq>
									   </crs:ToneCurvePV2012Blue>
									   <crs:PaintBasedCorrections>
									    <rdf:Seq>
									     <rdf:li>
									      <rdf:Description
									       crs:What="Correction"
									       crs:CorrectionAmount="1.000000"
									       crs:CorrectionActive="true"
									       crs:LocalExposure="0.552941"
									       crs:LocalSaturation="0.000000"
									       crs:LocalContrast="0.000000"
									       crs:LocalClarity="0.000000"
									       crs:LocalSharpness="0.000000"
									       crs:LocalBrightness="0.000000"
									       crs:LocalToningHue="0.000000"
									       crs:LocalToningSaturation="0.000000">
									      <crs:CorrectionMasks>
									       <rdf:Seq>
									        <rdf:li>
									         <rdf:Description
									          crs:What="Mask/Paint"
									          crs:MaskValue="0.597701"
									          crs:Radius="0.294229"
									          crs:Flow="0.374138"
									          crs:CenterWeight="0.336813">
									         <crs:Dabs>
									          <rdf:Seq>
									           <rdf:li>d 0.583691 0.603004</rdf:li>
									           <rdf:li>d 0.506847 0.542476</rdf:li>
									           <rdf:li>d 0.445992 0.503752</rdf:li>
									           <rdf:li>d 0.510425 0.594056</rdf:li>
									           <rdf:li>d 0.589165 0.649437</rdf:li>
									           <rdf:li>d 0.590581 0.680258</rdf:li>
									           <rdf:li>d 0.503070 0.664129</rdf:li>
									           <rdf:li>d 0.417757 0.631625</rdf:li>
									           <rdf:li>d 0.345171 0.564655</rdf:li>
									           <rdf:li>d 0.366658 0.567833</rdf:li>
									           <rdf:li>d 0.436027 0.649538</rdf:li>
									           <rdf:li>d 0.501659 0.737556</rdf:li>
									           <rdf:li>d 0.571560 0.818406</rdf:li>
									           <rdf:li>d 0.612505 0.805913</rdf:li>
									           <rdf:li>d 0.543371 0.723593</rdf:li>
									           <rdf:li>d 0.475483 0.639351</rdf:li>
									           <rdf:li>d 0.451223 0.540773</rdf:li>
									           <rdf:li>d 0.525914 0.601557</rdf:li>
									           <rdf:li>d 0.544495 0.599863</rdf:li>
									           <rdf:li>d 0.490855 0.494746</rdf:li>
									           <rdf:li>d 0.427792 0.403438</rdf:li>
									           <rdf:li>d 0.358655 0.321123</rdf:li>
									           <rdf:li>d 0.289519 0.238807</rdf:li>
									           <rdf:li>d 0.220372 0.156511</rdf:li>
									           <rdf:li>d 0.151199 0.074263</rdf:li>
									           <rdf:li>d 0.079237 -0.002181</rdf:li>
									           <rdf:li>d 0.005210 -0.074281</rdf:li>
									           <rdf:li>d -0.070480 -0.142402</rdf:li>
									           <rdf:li>d -0.102730 -0.153072</rdf:li>
									           <rdf:li>d -0.038006 -0.063046</rdf:li>
									           <rdf:li>d 0.026719 0.026980</rdf:li>
									           <rdf:li>d 0.091444 0.117007</rdf:li>
									           <rdf:li>d 0.156168 0.207033</rdf:li>
									           <rdf:li>d 0.213520 0.287557</rdf:li>
									           <rdf:li>d 0.249205 0.406728</rdf:li>
									           <rdf:li>d 0.309851 0.502384</rdf:li>
									           <rdf:li>d 0.386433 0.567872</rdf:li>
									           <rdf:li>d 0.466134 0.623738</rdf:li>
									           <rdf:li>d 0.434471 0.647575</rdf:li>
									           <rdf:li>d 0.352438 0.611033</rdf:li>
									           <rdf:li>d 0.294644 0.564284</rdf:li>
									           <rdf:li>d 0.352680 0.663384</rdf:li>
									           <rdf:li>d 0.402755 0.772418</rdf:li>
									           <rdf:li>d 0.464043 0.866000</rdf:li>
									           <rdf:li>d 0.550064 0.883831</rdf:li>
									           <rdf:li>d 0.638275 0.879062</rdf:li>
									           <rdf:li>d 0.622170 0.812559</rdf:li>
									           <rdf:li>d 0.542796 0.755269</rdf:li>
									           <rdf:li>d 0.457879 0.722551</rdf:li>
									           <rdf:li>d 0.370080 0.708936</rdf:li>
									           <rdf:li>d 0.282253 0.695747</rdf:li>
									           <rdf:li>d 0.194528 0.682390</rdf:li>
									           <rdf:li>d 0.251015 0.681170</rdf:li>
									           <rdf:li>d 0.339248 0.684909</rdf:li>
									           <rdf:li>d 0.427482 0.688648</rdf:li>
									           <rdf:li>d 0.509635 0.735799</rdf:li>
									           <rdf:li>d 0.591826 0.783998</rdf:li>
									           <rdf:li>d 0.664021 0.855778</rdf:li>
									           <rdf:li>d 0.665281 0.832685</rdf:li>
									           <rdf:li>d 0.601375 0.741388</rdf:li>
									           <rdf:li>d 0.525838 0.674864</rdf:li>
									           <rdf:li>d 0.446731 0.616125</rdf:li>
									           <rdf:li>d 0.367624 0.557387</rdf:li>
									           <rdf:li>d 0.288517 0.498648</rdf:li>
									           <rdf:li>d 0.209410 0.439909</rdf:li>
									           <rdf:li>d 0.127992 0.391444</rdf:li>
									           <rdf:li>d 0.040708 0.372114</rdf:li>
									           <rdf:li>d 0.072242 0.419354</rdf:li>
									           <rdf:li>d 0.148677 0.485448</rdf:li>
									           <rdf:li>d 0.225726 0.550039</rdf:li>
									           <rdf:li>d 0.303516 0.612609</rdf:li>
									           <rdf:li>d 0.381858 0.672870</rdf:li>
									           <rdf:li>d 0.456527 0.743481</rdf:li>
									           <rdf:li>d 0.531195 0.814091</rdf:li>
									           <rdf:li>d 0.608030 0.832618</rdf:li>
									           <rdf:li>d 0.601863 0.768196</rdf:li>
									          </rdf:Seq>
									         </crs:Dabs>
									         </rdf:Description>
									        </rdf:li>
									        <rdf:li>
									         <rdf:Description
									          crs:What="Mask/Paint"
									          crs:MaskValue="0.597701"
									          crs:Radius="0.294229"
									          crs:Flow="0.374138"
									          crs:CenterWeight="0.336813">
									         <crs:Dabs>
									          <rdf:Seq>
									           <rdf:li>d 0.806867 0.881974</rdf:li>
									           <rdf:li>d 0.813094 0.752712</rdf:li>
									           <rdf:li>d 0.798072 0.622405</rdf:li>
									           <rdf:li>d 0.784627 0.492781</rdf:li>
									           <rdf:li>d 0.725046 0.398217</rdf:li>
									           <rdf:li>d 0.659724 0.309852</rdf:li>
									           <rdf:li>d 0.586062 0.238182</rdf:li>
									           <rdf:li>d 0.508032 0.178908</rdf:li>
									           <rdf:li>d 0.426434 0.132882</rdf:li>
									          </rdf:Seq>
									         </crs:Dabs>
									         </rdf:Description>
									        </rdf:li>
									        <rdf:li>
									         <rdf:Description
									          crs:What="Mask/Paint"
									          crs:MaskValue="0.597701"
									          crs:Radius="0.294229"
									          crs:Flow="0.374138"
									          crs:CenterWeight="0.336813">
									         <crs:Dabs>
									          <rdf:Seq>
									           <rdf:li>d 0.868383 0.993562</rdf:li>
									           <rdf:li>d 0.857835 0.862179</rdf:li>
									           <rdf:li>d 0.838056 0.733487</rdf:li>
									           <rdf:li>d 0.802776 0.612341</rdf:li>
									           <rdf:li>d 0.768633 0.490447</rdf:li>
									           <rdf:li>d 0.727548 0.373261</rdf:li>
									           <rdf:li>d 0.689283 0.254060</rdf:li>
									           <rdf:li>d 0.624340 0.167826</rdf:li>
									           <rdf:li>d 0.544827 0.110515</rdf:li>
									           <rdf:li>d 0.462461 0.063275</rdf:li>
									          </rdf:Seq>
									         </crs:Dabs>
									         </rdf:Description>
									        </rdf:li>
									       </rdf:Seq>
									      </crs:CorrectionMasks>
									      </rdf:Description>
									     </rdf:li>
									    </rdf:Seq>
									   </crs:PaintBasedCorrections>
									  </rdf:Description>
									 </rdf:RDF>
									</x:xmpmeta>
									]],
			}	
		}, ':ignore:' )
	
	
	-- version doc should end up with develop settings as sha256
	local versionSha256 = rxTest:observe( 
							docStore:ongoingDocValue( versionId )
								:map( function( docValue )
									return ( KSTableUtils.derefOpt( docValue, 'develop', 'xmpCameraRaw', 'sha256' ) == '665407789296dc485e3e623a022615d7fd06f0ea9d5e81406f267ecf2a921749' )
								end ) )
								
	versionSha256:expectDataWithinTime( 40, true )
	
	local versionOnOz = rxTest:observe( 
							docStore:ongoingDocValue( versionId )
								:map( function( docValue )
									return docValue._serverProvidedMetadata ~= nil
								end ) )
								
	versionOnOz:expectDataWithinTime( 40, true )

end

--------------------------------------------------------------------------------

-- @mark StorageExceededError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestStorageExceededError( rxTest )
	
	rxTest:setTimeLimit( 5 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )
	
	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local docId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	
	-- create basic 'asset' doc
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId,
			_newRevId = revisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		},
		':ignore:' )
	
	-- create error doc for asset
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "StorageExceededError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"quota exceeded",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = revisionId,
					type = "asset",
				},
				status = 403,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
	
	-- asset doc should end up with quota exceeded annotation
	rxTest:expectSingleValue( 
		docStore:docValue( docId )
			:map( function( docValue )
				return WFCatOzErrors.hasStorageExceededError( docValue )
			end ), true )
	
	
end

--------------------------------------------------------------------------------

local function testStorageExceededError_restoreModify( rxTest, options )

	rxTest:setTimeLimit( 5 )

	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	local deletedImageId = KSIdGenerator.uuidInLowercaseHex()

	local maybeModifiedRevId = revisionId
	if options.modifyAfterRestore then
		maybeModifiedRevId = KSIdGenerator.uuidInLowercaseHex()
	end

	-- create restored 'asset' doc
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = docId,
			_newRevId = maybeModifiedRevId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			},
			_localOnly = {
				universal = {
					trash = {
						createFrom = {
							id = deletedImageId,
							lastAssetId = deletedImageId,
							lastRevisionId = "d4a62b52683d171ee6f0197f359a4af0",
							lastTrashPurgeDate = "2019-07-12T21:32:40.415Z",
							lastTrashUserCreated = "2019-05-13T21:32:40.415Z",
							revision_id = "d4a62b52683d171ee6f0197f359a4af0",
							subtype = "deleted_image",
						},
						createWithRevId = revisionId,
						uploaded = {
							["/rels/master"] = true,
							["/rels/proxy_type/2560"] = true,
							["/rels/xmp"] = true,
						},
					},
				},
			}
		},
		':ignore:' )

	-- create error doc for asset
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "StorageExceededError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"quota exceeded",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = revisionId,
					type = "asset",
				},
				status = 403,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	-- asset doc should end up with quota exceeded annotation
	rxTest:expectSingleValue(
		docStore:docValue( docId )
			:map( function( docValue )
				return WFCatOzErrors.hasStorageExceededError( docValue )
			end ), true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestStorageExceededError_restore( rxTest )
	testStorageExceededError_restoreModify( rxTest, {} )
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestStorageExceededError_restoreModify( rxTest )
	testStorageExceededError_restoreModify( rxTest, { modifyAfterRestore = true } )
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestStorageExceededError_restoreDelete( rxTest, options )

	rxTest:setTimeLimit( 5 )

	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local docId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	local deletedImageId = KSIdGenerator.uuidInLowercaseHex()
	local deletedImageId2 = KSIdGenerator.uuidInLowercaseHex()

	-- create restored 'asset' doc (that will be deleted below)
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = docId,
			_labels = { oz = true },
		},
		':ignore:' )

	-- delete the restored asset (and create the trash asset)
	rxTest:expectSingleValue(
		docStore:applyOverlay {
			_id = docId,
			_deleted = true,
			_newId = deletedImageId2,
			_newIdOverlay = {
				type = 'asset',
				subtype = 'deleted_image',
				userCreated = '2016-06-07T15:55:34.218872Z',
				purgeDate = '2016-06-08T15:55:34.218872Z',
				original = {
					id = docId,
					userCreated = '2016-06-07T15:55:34.218872Z',
				},
				_localOnly = {
					universal = {
						trash = {
							createFrom = {
								id = deletedImageId,
								lastAssetId = deletedImageId,
								lastRevisionId = "d4a62b52683d171ee6f0197f359a4af0",
								lastTrashPurgeDate = "2019-07-12T21:32:40.415Z",
								lastTrashUserCreated = "2019-05-13T21:32:40.415Z",
								revision_id = "d4a62b52683d171ee6f0197f359a4af0",
								subtype = "deleted_image",
							},
							createWithRevId = revisionId,
							uploaded = {
								["/rels/master"] = true,
								["/rels/proxy_type/2560"] = true,
								["/rels/xmp"] = true,
							},
						},
					},
				}
			}
		},
		':ignore:' )

	-- create error doc for asset
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "StorageExceededError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"quota exceeded",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = revisionId,
					type = "asset",
				},
				status = 403,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )

	-- The newly created trash asset should end up with quota exceeded annotation
	rxTest:expectSingleValue(
		docStore:docValue( deletedImageId2 )
			:map( function( docValue )
				return WFCatOzErrors.hasStorageExceededError( docValue )
			end ), true )

end

--------------------------------------------------------------------------------

-- @mark TrialOrSubscriptionExpiredError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestTrialOrSubscriptionExpiredError( rxTest )
	
	rxTest:setTimeLimit( 30 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )
	
	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local docId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	
	-- create basic 'asset' doc
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId,
			_newRevId = revisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		},
		':ignore:' )
	
	-- create error doc for asset
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			_newRevId = revisionId,
			subtype = "TrialOrSubscriptionExpiredError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"quota exceeded",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = revisionId,
					type = "asset",
				},
				status = 403,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
	
	-- asset doc should end up with quota exceeded annotation
	rxTest:expectSingleValue( 
		docStore:docValue( docId )
			:map( function( docValue )
				return WFCatOzErrors.hasTrialOrSubscriptionExpiredError( docValue )
			end ), true )
	
	
end

--------------------------------------------------------------------------------

-- @mark TrialOrSubscriptionExpiredError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestEntitlementExpiredError( rxTest )
	
	rxTest:setTimeLimit( 30 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )
	
	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local docId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	
	-- create basic 'asset' doc
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId,
			_newRevId = revisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		},
		':ignore:' )
	
	-- create error doc for asset
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			_newRevId = revisionId,
			subtype = "EntitlementExpiredError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"entitlement expired",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = revisionId,
					type = "asset",
				},
				status = 403,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
	
	-- asset doc should end up with quota exceeded annotation
	rxTest:expectSingleValue( 
		docStore:docValue( docId )
			:map( function( docValue )
				return WFCatOzErrors.hasEntitlementExpiredError( docValue )
			end ), true )
	
	
end

--------------------------------------------------------------------------------

-- @mark MultipartCompleteError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestMultipartCompleteError( rxTest )
	
	rxTest:setTimeLimit( 5 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )
	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local docId = '7206c3ae9a094d69861ee8983e95085d'
	local errorDocId = 'd58059e3898ee16896d490a9ea3c6027'
	local revisionId = 'c1122c33d44e4477a584e494f4c57b75'
	
	-- create basic 'asset' doc
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId,
			_newRevId = revisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		},
		':ignore:' )
	
	-- create error doc for asset
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "MultipartCompleteError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"Multipart complete was unsuccessful",
					},
				},
				source = {
					asset_id = docId,
					revision_id = revisionId,
					type = "master",
					id = "master",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
	
	-- asset doc should end up with quota exceeded annotation
	rxTest:expectSingleValue( 
		docStore:docValue( docId )
			:map( function( docValue )
				return WFCatOzErrors.hasMultipartCompleteError( docValue )
			end ), true )
	
	
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestMultipartCompleteError_testNightly( rxTest )

	rxTest:setTimeLimit( 600 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog
	
	rxTest:expectSingleValue(

		catalogTestSession:catalog()
			:tap( function( _catalog )
				catalog = _catalog
			end ),

		':ignore:' )

	local docStore = catalog:docStore()
	
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )

	local catalogSpec

	rxTest:expectSingleValue(

		catalogTestSession:catalogSpec()
			:tap( function( _catalogSpec )
				WFArguments.check( 'WFCatSpec' )
				catalogSpec = _catalogSpec
			end ),

		':ignore:' 
	)

	
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	
	local testDataPath = KSPathUtils.getStandardFilePath( "testdata" )
	testDataPath = KSPathUtils.child( testDataPath, 'assets' )
	local imageChunkedPath = KSPathUtils.child( testDataPath, '12MB.jpg' )
	local fileAttributes = WFFileUtils.fileAttributes( imageChunkedPath )
	local imageFileSize = fileAttributes.fileSize
	
	local revId

	rxTest:finally( function()
	
		errorHandler:close()
		puller:close()
		pusher:close()
		
	end )
	
	-- start the sync processes and track error message
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT count(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )	
	
	puller:getUpdatesContinuously()
	pusher:sendUpdatesContinuously()
	
	-- We push one asset to Oz and upload file with finalUrl different sessionId.
	
	-- Prime the updates feed pump.
	
	rxTest:expectSingleValue(

		docStore:setDocValue {
			_id = assetId,
			_localOnly = {
				mumble = "stuff",
				originalFilePath = imageChunkedPath,
			},
			captureDate = "2004-11-02T11:10:09",
			importSource = {
				fileName = "test.jpeg",
				fileSize = imageFileSize,
				contentType = 'image/jpeg',
				importedBy = catalogSpec:ozUserId(),
				importTimestamp = "2015-11-02T08:37:24.513Z",
				originalHeight = 16,
				originalWidth = 16,
			},
			subtype = "image",
			type = "asset",
		}
		:tap( function( _revId )
			revId = _revId
		end ),

		':ignore:' )
	
	local masterCreateUrl 
	
	local assetCreatedOnOz = rxTest:observe( 
							docStore:ongoingDocValue( assetId )
								:map( function( docValue )
									masterCreateUrl = KSTableUtils.derefOpt( docValue, '_serverProvidedMetadata', 'links', '/rels/master_create', 'href' )
				
									return docValue._serverProvidedMetadata ~= nil and masterCreateUrl ~= nil
								end ) )
								
	assetCreatedOnOz:expectDataWithinTime( 40, true )

	-- multipart upload initiate

	local ozUser = catalogSpec:wfOzUser()
	
	masterCreateUrl = masterCreateUrl:replace( '{revision_id}', revId )
	
	rxTest:expectSingleValue( 
		
		docStore:docValue( assetId )
			:map( function( docValue )
			
				local url = masterCreateUrl .. '/parts'	
				local putArgs = {
					requestHeaders = {
						{
							header = 'Content-Type',
							value = 'image/jpeg'
						},
						{
							header = 'Accept',
							value = 'application/json'
						},
						{ 
							header = 'X-Generate-Proxies', 
							value = 'all' 
						},
						{ 
							header = 'X-Generate-Renditions', 
							value = 'all' 
						}
					},
					timeout = 60,
					returnHttpErrorsAsData = true,
					bypassRateLimiter = true,
				}
	
				return ozUser
					:put( url, "", putArgs )
					:map( function( data, headers )
					
						if headers and headers.status_code == "201" then
						
							local results = KSJsonDecoder.decodeWebSafeJson_stripNull_ignoreEmpty( data )
						
							local docStatusOverlay = {
								_id = assetId,
								type = "asset",
								_localOnly = {
									chunkedUpload = {
										parts = {
										
										},
										part_template = KSTableUtils.derefOpt( results, "links", "/rels/create_part", "href" ),
										finish_href = KSTableUtils.derefOpt( results, "links", "self", "href" ),
									},
								},
							}
							
							return docStore
								:applyOverlay( docStatusOverlay )
							
						end
					end )
					:call( runUntilDone )
				
			end ),
			
		':ignore:'
	)
	
	local multipartInitSucceed = rxTest:observe( 
							docStore:ongoingDocValue( assetId )
								:map( function( docValue )
									local chunkedUploadParts = KSTableUtils.derefOpt( docValue, '_localOnly', 'chunkedUpload', 'parts' )
									local finishUrl = KSTableUtils.derefOpt( docValue, '_localOnly', 'chunkedUpload', 'finish_href' )
				
									return chunkedUploadParts ~= nil and finishUrl ~= nil
								end ) )
								
	multipartInitSucceed:expectDataWithinTime( 10, true )
	


	rxTest:expectSingleValue( 
		
		docStore:docValue( assetId )
			:flatMap( function( docValue )
			
				local putArgs = {
					requestHeaders = {
						{
							header = 'Content-Type',
							value = 'image/jpeg'
						},
						{
							header = 'Accept',
							value = 'application/json'
						},
						{ 
							header = 'X-Generate-Proxies', 
							value = 'all' 
						},
						{ 
							header = 'X-Generate-Renditions', 
							value = 'all' 
						}
					},
					timeout = 60,
					returnHttpErrorsAsData = true,
					bypassRateLimiter = true,
				}
				
				local finishUrl = KSTableUtils.derefOpt( docValue, '_localOnly', 'chunkedUpload', 'finish_href' )
			
				local jsonData = KSJsonEncoder.encodeOzUploadJson({ 
					parts = {
						[ 1 ] = {
							part_number = 1,
							etag = "123",
						},
						[ 2 ] = {
							part_number = 2,
							etag = "456",
						}
					} 
				})
				
				putArgs.requestMethod = "POST"


				local found = false
				local headerName = 'Content-Type'
				local headerValue = 'application/json'
				for index, header in ipairs( putArgs.requestHeaders ) do
					if header.header == headerName then
						if headerValue ~= nil then
							header.value = headerValue
						else
							table.remove( putArgs.requestHeaders, index )
						end
						found = true
						break
					end
				end
				if not found then
					putArgs.requestHeaders[ #putArgs.requestHeaders + 1 ] = {
						header = headerName,
						value = headerValue,
					}
				end
				
				
				return ozUser
					:putBinary( finishUrl, jsonData, putArgs )
					:flatMap( function( data, headers )
					
						if headers and headers.status_code == "202" then
						
							local partialMasterUrl = finishUrl:gsub( "^.*//([^/]+", "" )
							local endTime = KSDate.timeInIso8601Format()
							
							local docStatusOverlay = {
								_id = docValue._id,
								type = "asset",
								_localOnly = {
									chunkedUpload = {
										_delete = true,
									},
									original = {
										ozUrl = partialMasterUrl,
										doNotBackUpFlagSet = true,
										uploaded = endTime,
									 },
								},
							}
						
							return docStore
								:applyOverlay( docStatusOverlay )
						
						end
						
					end )
				
			end ),
			
		':ignore:'
	)

	-- we should get an error from Oz
	outstandingErrors:expectDataWithinTime( 40, 1 )
	
	-- the error handler should eventually clear the error
	outstandingErrors:expectDataWithinTime( 40, 0 )
		
	local multipartCompleteErrorFromOz = rxTest:observe( 
							docStore:ongoingDocValue( assetId )
								:map( function( docValue )
									local ozMultipartCompleteError = KSTableUtils.derefOpt( docValue, '_localOnly', 'ozError', 'subtype' )
									return ozMultipartCompleteError == 'MultipartCompleteError'
								end ) )
								
	multipartCompleteErrorFromOz:expectDataWithinTime( 10, true )
		
end

--------------------------------------------------------------------------------

-- @mark PayloadValidationError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetPayloadValidationError( rxTest )
	
	rxTest:setTimeLimit( 5 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local docId = '7206c3ae9a094d69861ee8983e95085d'
	local revId = '6206c3ae9a094d69861ee8983e95085e'
	local errorDocId = 'd58059e3898ee16896d490a9ea3c6027'
	
	-- create basic 'asset' doc that has not been synced to Oz
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId,
			_newRevId = revId,
			type = 'asset',
			subtype = 'image',
			-- invalid date
			captureDate = false,
			importSource = {
				importTimestamp = '2017-03-14T10:50:60.000Z', -- note seconds are 60
				filePath = '/path/',
				originalHeight = 150,
				-- fractional width
				originalWidth = 100.5,
			},
			ratings = {
				[ 'aec4af47d8114158ac494aa530848430' ] = {
					date = '0000-00-00T00:00:00',
					rating = 5,
				}
			},
			develop = {
				-- disallowed field
				_merge = 'atomic',
				-- invalid TZ
				userUpdated = "2017-02-22T18:29:42.616+02:00",
				xmpCameraRaw = '<xmpmeta /',
				-- processingModel is missing
				profiles = {},
			},
			video = {
			 frameRate = {
			     6.9130254280137e+015,
			     4.5035996273705e+015
			 },
			},
			['0'] = 'totally made up',
			xmp = {
				exif = {
					-- ISOSpeedRatings must be a single number
					ISOSpeedRatings = { 50, 100 },
					-- wrong key name
					fnumber = { 1, 2 },
					-- illegal value
					FocalLength = { 'bad' },
				}
			}
		},
		':ignore:' )
	
	-- create error doc for asset that indicates the resource is not found
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "PayloadValidationError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					['payload.develop._merge' ] = { 'disallowed' },
					['payload.develop.userUpdated' ] = { 'does not match pattern' },
					['payload.develop.processingModel'] = { 'required' },
					['payload.captureDate' ] = { 'invalid' },
					['payload.BadKey' ] = { 'invalid' },
					['payload.ratings.aec4af47d8114158ac494aa530848430.date'] = { 'does not match pattern' },
					['payload.xmp.exif.ISOSpeedRatings' ] = { 'invalid' },
					['payload.xmp.exif.FocalLength' ] = { 'invalid' },
					['payload.video.frameRate.0' ] = { 'invalid' },
					['payload.video.frameRate.1' ] = { 'invalid' },
					['payload.0' ] = { 'invalid' },
					['payload.develop.profiles' ] = { 'invalid' },
					['payload.importSource.importTimestamp'] = { 'does not match pattern' },
					['payload.importSource.originalWidth'] = { 'invalid type' },
					-- NMB-11045 this field doesn't exist in initial doc and we shouldn't create an empty hash trail to
					-- it by attempting to remove something that doesn't exist
					['payload.xmp.nmb111045.subject' ] = { 'minimum number of properties not met' },
				},
				missing_resource_id = docId,
				source = {
					action = "create",
					id = docId,
					revision_id = revId,
					type = "asset",
				},
				status = 400,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
	
	-- asset should not have ResourceNotFoundError annotation, it may have one for the error that
	-- is presenting the asset from getting to Oz at all, but that should not be replaced by a
	-- ResourceNotFoundError
	rxTest:expectSingleValue( 
		docStore:docValue( docId )
			:map( function( docValue )
			
				docValue = KSTableUtils.deepCopy( docValue )
				docValue._localId = nil
				docValue._changeCounter = nil
				docValue._revId = nil
				
				return docValue
					
			end ), {
				_id = docId,
				type = 'asset',
				subtype = 'image',
				captureDate = "0000-00-00T00:00:00",
				importSource = {
					importTimestamp = '2017-03-14T10:50:59.000Z', -- seconds has been adjusted
					filePath = '/path/',
					originalHeight = 150,
					originalWidth = 100,
				},
				ratings = {
					[ 'aec4af47d8114158ac494aa530848430' ] = {
						rating = 5,
					}
				},
				develop = {
					userUpdated = "2017-02-22T16:29:42.616Z",
					xmpCameraRaw = '<xmpmeta /',
					processingModel = 'lightroom',
				},
				video = {
				     frameRate = {
				     },
				 },
				xmp = {
					exif = {
						fnumber = { 1, 2 }
					}
				}
			} )
	
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetPayloadValidationErrorMarkUnknown( rxTest )

	rxTest:setTimeLimit( 5 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local assetRevId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	
	rxTest:expectSingleValue( catalog:docStore():bulkApplyOverlay( {
		{
			_id = assetId,
			_newRevId = assetRevId,
			type = 'asset',
			subtype = 'image',
			name = 'fooo',
		},
		{
			_id = errorDocId,
			type = 'error',
			subtype = 'PayloadValidationError',
			_serverProvidedMetadata = {
				source = {
					id = assetId,
					revision_id = assetRevId
				},
				errors = {
		            file_extension = {
		                "unsupported",
		            },
		        },
			}
		},
	} ), ':ignore:' )
	
	rxTest:sleep( 3 )
	
	rxTest:expectSingleValue( 
		catalog:indexStore():query( {
			sql = 'SELECT COUNT(docId) FROM errors'
		} )
		:take( 1 )
		:map( function( rows )
		
			return rows[ 1 ][ 1 ]
			
		end ), 0 )
		
	rxTest:expectSingleValue(
		catalog:docStore():docValue( errorDocId )
			:map( function( docValue )
			
				return KSTableUtils.derefOpt( docValue, '_localOnly', 'unknownError' )
				
			end ), true )
	
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetPayloadValidationError_testNightly( rxTest )

	rxTest:setTimeLimit( 120 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	
	rxTest:expectSingleValue(

		catalogTestSession:catalog()
			:tap( function( _catalog )
				catalog = _catalog
			end ),

		':ignore:' )

	local docStore = catalog:docStore()
	
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:finally( function()
		errorHandler:close()
		puller:close()
		pusher:close()
		
	end )
	
	-- watch the errors table, to ensure we actually do get errors
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT count(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )
		
	-- start the sync processes
	puller:getUpdatesContinuously()
	pusher:sendUpdatesContinuously()
	
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = assetId,
			type = 'asset',
			subtype = 'image',
			-- captureDate = '610626-01-19T07:14:54.560+07:36:42',
			importSource = {
				fileName = '/path/',
				originalHeight = 150,
				originalWidth = 100.5,
				importTimestamp = '2017-03-14T10:50:60.000Z', -- note seconds are 60
				importedBy = KSIdGenerator.uuidInLowercaseHex(),
			},
			develop = {
				_merge = 'atomic',
				userUpdated = "2017-02-22T18:29:42.616+02:00",
				processingModel = 'lightroom',
				xmpCameraRaw = '<xmpmeta /',
			},
			ratings = {
				[ 'aec4af47d8114158ac494aa530848430' ] = {
					date = "0000-00-00T00:00:00",
					rating = 5,
				}
			},
			xmp = {
				exif = {
					ISOSpeedRatings = 50,
					fnumber = { 1, 2 }
				}
			}
		},
		':ignore:' )
	
	-- we should get an error from Oz
	outstandingErrors:expectDataWithinTime( 40, 1 )
	
	-- the error handler should eventually clear the error
	outstandingErrors:expectDataWithinTime( 10, 0 )
		
	-- Verify asset was uploaded
	local assetOnOz = rxTest:observe( 
							docStore:ongoingDocValue( assetId )
								:map( function( docValue )
									return docValue._serverProvidedMetadata ~= nil
								end ) )
								
	assetOnOz:expectDataWithinTime( 40, true )
		
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetPayloadValidationErrorTwoTries_testNightly( rxTest )

	rxTest:setTimeLimit( 120 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	
	rxTest:expectSingleValue(

		catalogTestSession:catalog()
			:tap( function( _catalog )
				catalog = _catalog
			end ),

		':ignore:' )

	local docStore = catalog:docStore()
	
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )
	
	rxTest:finally( function()
		errorHandler:close()
		puller:close()
		pusher:close()
	end )
	
	-- watch the errors table, to ensure we actually do get errors
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT count(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )
	
	-- start the sync processes
	puller:getUpdatesContinuously()
	pusher:sendUpdatesContinuously()

	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = assetId,
			type = 'asset',
			subtype = 'image',
			captureDate = '610626-01-19T07:14:54.560+07:36:42',
			importSource = {
				fileName = '/path/',
				originalHeight = 150,
				originalWidth = 100.5,
				importTimestamp = '2017-03-14T10:50:60.000Z', -- note seconds are 60
				importedBy = KSIdGenerator.uuidInLowercaseHex(),
			},
			develop = {
				_merge = 'atomic',
				userUpdated = "2017-02-22T18:29:42.616+02:00",
				processingModel = 'lightroom',
				xmpCameraRaw = '<xmpmeta /',
			},
			-- we should need two attempts to get this cleaned up.
			-- the first should remove the empty date, the second the empty table
			ratings = {
				[ 'aec4af47d8114158ac494aa530848430' ] = {
					date = "0000-00-00T00:00:00",
				}
			},
			xmp = {
				exif = {
					ISOSpeedRatings = 50,
					fnumber = { 1, 2 }
				},
				tiff = {
					InvalidField = 'ForceTwoPass'
				}
			}
		},
		':ignore:' )
	
	-- we should get an error from Oz
	outstandingErrors:expectDataWithinTime( 60, 1 )
	
	-- the error handler should eventually clear the error
	outstandingErrors:expectDataWithinTime( 10, 0 )
	
	-- we should get a second error from Oz
	outstandingErrors:expectDataWithinTime( 60, 1 )
	
	-- the error handler should eventually clear the error
	outstandingErrors:expectDataWithinTime( 10, 0 )
	
	-- Verify asset was uploaded
	local assetOnOz = rxTest:observe( 
							docStore:ongoingDocValue( assetId )
								:map( function( docValue )
									return docValue._serverProvidedMetadata ~= nil
								end ) )
								
	assetOnOz:expectDataWithinTime( 60, true )
		
end

--------------------------------------------------------------------------------

-- @mark InputValidationError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestPersonPrimaryInputValidationError( rxTest )
	
	rxTest:setTimeLimit( 5 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local docId = '7206c3ae9a094d69861ee8983e95085d'
	local revId = '6206c3ae9a094d69861ee8983e95085e'
	local errorDocId = 'd58059e3898ee16896d490a9ea3c6027'
	
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId,
			_newRevId = revId,
			type = 'asset',
			subtype = 'person',
			primary = "d58059e3898ee16896d490a9ea3c8888",
			name = "hari"
		},
		':ignore:' )
	
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "InputValidationError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					['payload.primary' ] = { 'primary asset should not have a primary' },
				},
				source = {
					action = "update",
					id = docId,
					revision_id = revId,
					type = "asset",
				},
				status = 400,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
		
	rxTest:expectSingleValue( 
		docStore:docValue( docId )
		:map( function( docValue )
			docValue = KSTableUtils.deepCopy( docValue )
			docValue._localId = nil
			docValue._changeCounter = nil
			docValue._revId = nil
			
			return docValue
		end ), 
		{
			_id = docId,
			type = 'asset',
			subtype = 'person',
			name = "hari"
		} )
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestInputValidationError_spaceDoc_testNightly( rxTest )

	rxTest:setTimeLimit( 120 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog
	local spaceId = KSIdGenerator.uuidInLowercaseHex()
	
	rxTest:expectSingleValue(

		catalogTestSession:catalog()
			:tap( function( _catalog )
				catalog = _catalog
			end ),

		':ignore:' )

	local docStore = catalog:docStore()
	
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )
	
	rxTest:finally( function()
		errorHandler:close()
		puller:close()
		pusher:close()
		
	end )
	
	-- watch the errors table, to ensure we actually do get errors
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT count(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )
		
	-- start the sync processes
	puller:getUpdatesContinuously()
	pusher:sendUpdatesContinuously()
	
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = spaceId,
			type = 'space',
			subtype = 'shared',
			location = false,
			metadata = false,
			private = false,
		},
		':ignore:' )
	
	-- we should get an error from Oz
	outstandingErrors:expectDataWithinTime( 40, 1 )
	
	-- the error handler should eventually clear the error
	outstandingErrors:expectDataWithinTime( 10, 0 )
		
	-- Verify asset was uploaded
	local spaceOnOz = rxTest:observe( 
							docStore:ongoingDocValue( spaceId )
								:map( function( docValue )
									return docValue._serverProvidedMetadata ~= nil
								end ) )
								
	spaceOnOz:expectDataWithinTime( 40, true )
		
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestInputValidationError_ifMatch_testNightly( rxTest )
	
	rxTest:setTimeLimit( 120 )
	
	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	
	rxTest:expectSingleValue(

		catalogTestSession:catalog()
			:tap( function( _catalog )
				catalog = _catalog
			end ),

		':ignore:' )

	local docStore = catalog:docStore()
	
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )
	
	rxTest:finally( function()
		errorHandler:close()
		puller:close()
		pusher:close()
		
	end )
	
	puller:getUpdatesContinuously()
	pusher:sendUpdatesContinuously()
	
	-- create a document and sync it to Oz
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = assetId,
			type = 'asset',
			subtype = 'image',
			captureDate = '2016-01-19T07:14:54.560Z',
			importSource = {
				fileName = '/path/',
				originalHeight = 150,
				originalWidth = 100,
				importTimestamp = '2017-03-14T10:50:59.000Z',
				importedBy = KSIdGenerator.uuidInLowercaseHex(),
			},
			develop = {
				userUpdated = "2017-02-22T18:29:42.616Z",
				processingModel = 'lightroom',
				xmpCameraRaw = '<xmpmeta /',
			},
			xmp = {
				dc = {
					title = 'WF UNIT TEST'
				}
			}
		},
		':ignore:' )
	
	-- watch the errors table, to ensure we actually do get errors
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT count(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )
		
	-- sync it
	rxTest:expectSingleValue( docStore:ongoingDocValue( assetId )
				:filter( function( docValue )
					return not not KSTableUtils.derefOpt( docValue, '_serverProvidedMetadata', 'links' )
				end )
				:map( function( docValue )
					-- we've got a links section so it got to Oz
					return true
				end )
				:take( 1 ), true )
	
	-- now we're going to fake things going off the rails
	-- first update the document and also delete the revision_ids in the local document 
	rxTest:expectSingleValue( 
		docStore:docValue( assetId )
			:flatMap( function( docValue )
				return docStore:applyOverlay(
					{
						_id = assetId,
						_serverProvidedMetadata = {
							revision_ids = { _delete = true }
						},
						xmp = {
							dc = {
								title = 'WF UNIT TEST UPDATE'
							}
						} 
					} )
			end ), ':ignore:' )
		
	-- we should get an InputValidationError from Oz
	outstandingErrors:expectDataWithinTime( 40, 1 )
	
	-- the error handler should clear the error
	outstandingErrors:expectDataWithinTime( 10, 0 )
	
	-- but clearing the error will result in a conflict so we'll need to ensure that gets cleared
	local conflictResolver = require 'WFCatConflictResolver'( docStore, catalog:indexStore() )
	rxTest:addCloseTarget( conflictResolver )	
	
	-- wait until the latest doc value has both the updated title and the 'oz' label
	-- and 2 revisions in the revision tree
	rxTest:expectSingleValue( Rx.Observable.timer( 0, 1, WFRxScheduler.sharedScheduler() )
		:flatMap( function()
			return docStore:docValue( assetId )
				:map( function( docValue )
			
					local labels = docValue._labels or {}
					local revisions = KSTableUtils.derefOpt( docValue, '_serverProvidedMetadata', 'revision_ids' ) or {}
					local title = KSTableUtils.derefOpt( docValue, 'xmp', 'dc', 'title' )
				
					return labels.oz and ( title == 'WF UNIT TEST UPDATE' ) and ( #revisions == 2 )
				end )
		end )
		:filter( function( value ) return not not value end )
		:take( 1 ), true )
end

--------------------------------------------------------------------------------

local function observeIsDeletedOnOz( rxTest, catalog, docId )

	rxTest:logf( "Waiting for %s to be deleted on Oz", docId )

	return rxTest:observe( catalog:docStore():ongoingDocValue( docId )
		:map( function( docValue )

			local isDeleted = not not ( docValue._deleted or docValue._empty )
			local isLabeledOz = not not ( docValue._labels and docValue._labels.oz )
			
			return isDeleted and isLabeledOz

		end )
	)

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestInputValidationError_ifMatch_deleted_testNightly( rxTest )
	
	rxTest:setTimeLimit( 120 )
	
	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	
	rxTest:expectSingleValue(

		catalogTestSession:catalog()
			:tap( function( _catalog )
				catalog = _catalog
			end ),

		':ignore:' )

	local docStore = catalog:docStore()
	
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )
	
	rxTest:finally( function()
		errorHandler:close()
		puller:close()
		pusher:close()
		
	end )
	
	puller:getUpdatesContinuously()
	pusher:sendUpdatesContinuously()
	
	-- create a document and sync it to Oz
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = assetId,
			type = 'asset',
			subtype = 'image',
			captureDate = '2016-01-19T07:14:54.560Z',
			importSource = {
				fileName = '/path/',
				originalHeight = 150,
				originalWidth = 100,
				importTimestamp = '2017-03-14T10:50:59.000Z',
				importedBy = KSIdGenerator.uuidInLowercaseHex(),
			},
			develop = {
				userUpdated = "2017-02-22T18:29:42.616Z",
				processingModel = 'lightroom',
				xmpCameraRaw = '<xmpmeta /',
			},
			xmp = {
				dc = {
					title = 'WF UNIT TEST'
				}
			}
		},
		':ignore:' )
	
	-- watch the errors table, to ensure we actually do get errors
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT count(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )
	
	local fullDocValue
	-- sync it
	rxTest:expectSingleValue( docStore:ongoingDocValue( assetId )
				:filter( function( docValue )
					return not not KSTableUtils.derefOpt( docValue, '_serverProvidedMetadata', 'links' )
				end )
				:map( function( docValue )
					fullDocValue = docValue
					-- we've got a links section so it got to Oz
					return true
				end )
				:take( 1 ), true )
	
	-- now we're going to fake things going off the rails
	-- first delete the document and make sure the deletion syncs
	rxTest:expectSingleValue( 
		docStore:applyOverlay( {
				_id = assetId,
				_deleted = true
			} ), ':ignore:' )
	
	local observeAssetDeletedOnOz = observeIsDeletedOnOz( rxTest, catalog, assetId )
	observeAssetDeletedOnOz:expectDataWithinTime( 40, true )
	observeAssetDeletedOnOz:dispose()
	
	-- add the document back via setDocValue which will overwrite the 
	-- deleted doc value with the 'old results'
	fullDocValue._serverProvidedMetadata = { something = true } -- necessary to be able ensure we push the update to Oz
	fullDocValue._changeCounter = nil
	fullDocValue._revId = nil
	fullDocValue._labels = nil
	fullDocValue.xmp.dc.title = 'WF UNIT TEST UPDATE'
	rxTest:expectSingleValue( docStore:setDocValue( fullDocValue ), ':ignore:' )
	
	local watchForDocDeletion = rxTest:observe( docStore:ongoingDocValue( assetId )
		:map( function( docValue )
			return not not docValue._empty
		end ) )
		
	watchForDocDeletion:expectDataWithinTime( 1, false )
	
	-- we should get an InputValidationError from Oz
	outstandingErrors:expectDataWithinTime( 40, 1 )
	
	-- the error handler should clear the error
	outstandingErrors:expectDataWithinTime( 40, 0 )
	
	-- but clearing the error will result in a conflict so we'll need to ensure that gets cleared
	local conflictResolver = require 'WFCatConflictResolver'( docStore, catalog:indexStore() )
	rxTest:addCloseTarget( conflictResolver )	
	
	watchForDocDeletion:expectDataWithinTime( 15, true )
	
end

--------------------------------------------------------------------------------

-- @mark ResourceExistsError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetResourceExistsError_Deleted( rxTest )
	
	rxTest:setTimeLimit( 30 )
	
	local catalogId = '99999999999999999999999999999999'
	local userId =    '44444444444444444444444444444444'
	local docId = KSIdGenerator.uuidInLowercaseHex()
	local docRevId = KSIdGenerator.uuidInLowercaseHex()
	
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest, { 
		{ 
			url = "/v2/catalogs/99999999999999999999999999999999/updates/" .. docId,
			headers = {
				status_code = '200',
			},
			response = [[while (1) {} {"base":"https://photos-stage.adobe.io/v2/catalogs/de7a06ae654a4fc98ca96b3dbf6a03df/","id":"339acfbddbe6489cba429b223e3654bc",
						"deleted":true,"type":"asset","updated":"2017-04-26T20:33:53.650275Z","deleted_by":"995b4f462a3067e04b79192440133171"}]]
			
		} } )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	
		-- create basic 'asset' doc that has not been synced to Oz
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId,
			_newRevId = docRevId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		},
		':ignore:' )
	
	-- create error doc for asset that indicates the resource is not found
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			_serverProvidedMetadata = {
				base = "https://photos-stage.adobe.io/v2/catalogs/99999999999999999999999999999999/",
				errors = {
					asset = {
						"already exists",
					},
				},
				source = {
					action = "create",
					id = docId,
					links = {
						["self"] = {
							href = "/v2/catalogs/99999999999999999999999999999999/updates/" .. docId,
						},
					},
					revision_id = docRevId,
					type = "asset",
				},
				status = 403,
				updated = "2017-04-26T19:27:11.417509Z",
			},
			subtype = "ResourceExistsError",
			type = "error",
		},
		':ignore:' )
	
	-- the error handler should process the ResourceExistsError and delete
	-- the asset
	local isDocValueDeleted = rxTest:observe( docStore:ongoingDocValue( docId )
								:map( function( docValue ) 
									return docValue._empty
								end ) )
								
	isDocValueDeleted:expectDataWithinTime( 40, true )
	
	-- check index store for error documents, there shouldn't be any any more
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT count(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )
	
	outstandingErrors:expectDataWithinTime( 10, 0 )
	
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetResourceExistsError_NotDeleted( rxTest )
	
	rxTest:setTimeLimit( 20 )
	
	local catalogId = '99999999999999999999999999999999'
	local userId =    '44444444444444444444444444444444'
	local docId = 'b848bb70f55c4a208e9ab0e1ae42b04f'
	local docRevId
	
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest, { 
		{ 
			url = "/v2/catalogs/99999999999999999999999999999999/updates/" .. docId,
			headers = {
				status_code = '200',
			},
			response = [[while (1) {} {"base":"https://photos-stage.adobe.io/v2/catalogs/de7a06ae654a4fc98ca96b3dbf6a03df/","id":"339acfbddbe6489cba429b223e3654bc",
						"type":"asset","updated":"2017-04-26T20:33:53.650275Z","deleted_by":"995b4f462a3067e04b79192440133171"}]]
			
		} } )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	
		-- create basic 'asset' doc that has not been synced to Oz
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		}
		:map( function( revisionId )
			docRevId = revisionId
			return revisionId
		end ),
		':ignore:' )
	
	-- create error doc for asset that indicates the resource is not found
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			_serverProvidedMetadata = {
				base = "https://photos-stage.adobe.io/v2/catalogs/99999999999999999999999999999999/",
				errors = {
					asset = {
						"already exists",
					},
				},
				source = {
					action = "create",
					id = docId,
					links = {
						["self"] = {
							href = "/v2/catalogs/99999999999999999999999999999999/updates/" .. docId,
						},
					},
					revision_id = docRevId,
					type = "asset",
				},
				status = 403,
				updated = "2017-04-26T19:27:11.417509Z",
			},
			subtype = "ResourceExistsError",
			type = "error",
		},
		':ignore:' )
	
	-- check index store for error documents, there shouldn't be any any more
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT * FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:map( function( rows )
			return #rows 
		end ) )
	
	outstandingErrors:expectDataWithinTime( 10, 0 )
	
	-- the error handler should process the ResourceExistsError but not delete
	-- the asset since it does exist and we should eventually resolve through
	-- the normal sync process
	rxTest:expectSingleValue( docStore:docValue( docId )
								:map( function( docValue ) 
									return not not docValue._empty
								end ), false )
								
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetResourceExistsError_NotDeletedError( rxTest )
	
	rxTest:setTimeLimit( 20 )
	
	local catalogId = '99999999999999999999999999999999'
	local userId =    '44444444444444444444444444444444'
	local docId = 'b848bb70f55c4a208e9ab0e1ae42b04f'
	local docRevId
	
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest, { 
		{ 
			url = "/v2/catalogs/99999999999999999999999999999999/updates/" .. docId,
			headers = {
				status_code = '404',
			},
			fail = {
				{
					code = 1000,
					description = "Resource not found",
					errors = {
						revision = {
							"does not exist",
						},
					},
					subtype = "ResourceNotFoundError",
				},
				{
					status_code = '404',
					[ 'X-Result-Code' ] = '1000',
				}
			}
			
		} } )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	
		-- create basic 'asset' doc that has not been synced to Oz
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		}
		:map( function( revisionId )
			docRevId = revisionId
			return revisionId
		end ),
		':ignore:' )
	
	-- create error doc for asset that indicates the resource is not found
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			_serverProvidedMetadata = {
				base = "https://photos-stage.adobe.io/v2/catalogs/99999999999999999999999999999999/",
				errors = {
					asset = {
						"already exists",
					},
				},
				source = {
					action = "create",
					id = docId,
					links = {
						["self"] = {
							href = "/v2/catalogs/99999999999999999999999999999999/updates/" .. docId,
						},
					},
					revision_id = docRevId,
					type = "asset",
				},
				status = 403,
				updated = "2017-04-26T19:27:11.417509Z",
			},
			subtype = "ResourceExistsError",
			type = "error",
		},
		':ignore:' )
	
	-- check index store for error documents, there shouldn't be any any more
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT * FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:map( function( rows )
			return #rows 
		end ) )
	
	outstandingErrors:expectDataWithinTime( 10, 0 )
	
	-- the error handler should process the ResourceExistsError but not delete
	-- the asset since it does exist and we should eventually resolve through
	-- the normal sync process
	rxTest:expectSingleValue( docStore:docValue( docId )
								:map( function( docValue ) 
									return not not docValue._empty
								end ), false )
								
end


--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestFavoriteResourceExistsError( rxTest )
	
	rxTest:setTimeLimit( 20 )
	
	local catalogId = '99999999999999999999999999999999'
	local userId =    '44444444444444444444444444444444'
	local docId1 = 'aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'	-- local doc that'll result in error
	local docId2 = 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb'	-- doc available on Oz that's returned as duplicate
	local fullSpaceId = "randomSpaceId"
	local fullAssetId = "randomAssetId"
	local docRevId
	
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest, { 
		{ 
			url = "/v2/catalogs/99999999999999999999999999999999/updates/" .. docId2,
			headers = {
				status_code = '200',
			},
			response = [[while (1) {} {"base":"https://photos-stage.adobe.io/v2/catalogs/de7a06ae654a4fc98ca96b3dbf6a03df/","id":"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb",
						"type":"favorite", "subtype":"user", "updated":"2017-04-26T20:33:53.650275Z" }]]
			
		} } )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	
		-- create basic 'favorite' doc that has not been synced to Oz
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId1,
			type = 'favorite',
			subtype = 'user',
			space = { id = fullSpaceId },
			asset = { id = fullAssetId },
			userCreated = "2011-11-11T11:11:11.111111Z",
			userUpdated = "2011-11-11T11:11:11.111111Z",
			
		}
		:map( function( revisionId )
			docRevId = revisionId
			return revisionId
		end ),
		':ignore:' )
	
	-- create error doc for asset that indicates the resource is not found
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			_serverProvidedMetadata = {
				base = "https://photos-stage.adobe.io/v2/catalogs/99999999999999999999999999999999/",
				errors = {
					favorite = {
						"favorite by user already exists on asset in space",
					},
				},
				source = {
					action = "create",
					id = docId1,
					links = {
						["self"] = {
							href = "/v2/catalogs/99999999999999999999999999999999/updates/" .. docId2,
						},
					},
					revision_id = docRevId,
					type = "favorite",
				},
				status = 403,
				updated = "2017-04-26T19:27:11.417509Z",
			},
			subtype = "ResourceExistsError",
			type = "error",
		},
		':ignore:' )
	
	-- check index store for error documents, there shouldn't be any any more
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT * FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:map( function( rows )
			return #rows 
		end ) )
	
	outstandingErrors:expectDataWithinTime( 10, 0 )
	
	-- the error handler should process the ResourceExistsError and delete
	-- the favorite document since we would have synced the duplicate
	-- document through normal sync
	rxTest:expectSingleValue( docStore:docValue( docId1 )
								:map( function( docValue ) 
									return not not docValue._empty
								end ), true )
								
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetResourceExistsError_testNightly( rxTest )
	
	rxTest:setTimeLimit( 60 )

	local catalogTestSession = WFCatTestSession( rxTest )
	
	local catalog
	
	rxTest:expectSingleValue( catalogTestSession:catalog()
		:tap( function( _catalog )
			catalog = _catalog
		end ), ':ignore:' )


	local assetId = KSIdGenerator.uuidInLowercaseHex()
	
	local payload = {
		captureDate = "0000-00-00T00:00:00",
		importSource = {
			fileName = 'test.jpg',
			filePath = '/path/',
			originalHeight = 150,
			originalWidth = 100,
			importedBy = KSIdGenerator.uuidInLowercaseHex(),
			importTimestamp = KSDate.timeInIso8601Format(),
		},
		develop = {
			processingModel = 'lightroom',
			xmpCameraRaw = '<xmpmeta />',
		},
	}
	
	local assetDocumentForRestAPI =  {
		type = 'asset',
		subtype = 'image',
		payload = payload,
	}

	local assetDocumentForSyncFeed = KSTableUtils.deepCopy( payload )
	assetDocumentForSyncFeed._id = assetId
	assetDocumentForSyncFeed.type = 'asset'
	assetDocumentForSyncFeed.subtype = 'image'
	

	-- create the asset via the rest API
	local assetUrl = string.format( '/v2/catalogs/%s/assets/%s/revisions/%s', catalog:ozCatalogId(), assetId, KSIdGenerator.uuidInLowercaseHex() )
	rxTest:expectSingleValue( catalog:ozCatalogSpec():put( assetUrl, KSJsonEncoder.encodeOzUploadJson( assetDocumentForRestAPI ), 
																{ 	requestMethod = 'PUT', 
																	requestHeaders = {
																		{ header = 'Content-Type', value = 'application/json' },
																	},
																} ), ':ignore:', ':ignore:' )
	rxTest:sleep( 2 )
	
	-- delete the asset via the REST API
	local assetDeleteUrl = string.format( '/v2/catalogs/%s/assets/%s', catalog:ozCatalogId(), assetId )
	rxTest:expectSingleValue( catalog:ozCatalogSpec():put( assetDeleteUrl, '', { requestMethod = 'DELETE',
																 				 returnHttpErrorsAsData = true } )
															:map( function( result, headers )
																return headers.status_code == '204'
															end ), true )
	
	rxTest:sleep( 5 )
	
	local docStore = catalog:docStore()
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )
	
	rxTest:finally( function()
		puller:close()
		pusher:close()
		errorHandler:close()
	end )
	
	-- start the sync processes
	pusher:sendUpdatesContinuously()
	puller:getUpdatesContinuously()
	
	-- now try to create the asset again
	rxTest:expectSingleValue( docStore:setDocValue( assetDocumentForSyncFeed ), ':ignore:' )
	
	-- the error handler should process the ResourceExistsError and delete
	-- the asset

	local isDocValueDeleted = rxTest:observe( docStore:ongoingDocValue( assetId )
								:map( function( docValue ) 
									return docValue._empty
								end ) )
								
	isDocValueDeleted:expectDataWithinTime( 40, true )
	
	-- check index store for error documents, there shouldn't be any anymore
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT * FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:map( function( rows )
			return #rows 
		end ) )
	
	outstandingErrors:expectDataWithinTime( 10, 0 )
	
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResourceNotFoundErrorCausedByDoubleRestore( rxTest )

	-- a ResourceNotFoundError for a document that used to be on Oz should cause that
	-- document to be deleted
	rxTest:setTimeLimit( 5 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local docId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	local deletedImageId = KSIdGenerator.uuidInLowercaseHex()
	
	-- create asset as it appears after first restore.
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId,
			_labels = { oz = true },
			type = 'asset',
			subtype = 'image',
		},
		':ignore:' )
		
	rxTest:expectSingleValue(
		docStore:applyOverlay {
			_id = docId,
			_localId = 4,
			_localOnly = {
				binaryUpload = {
				},
				files = {
					original = {
						master = {
							relativePath = "Users/chet/Pictures/Adobe Revel/Adobe Revel Library.rvl/rxTest_CT_OzDoubleRestore_testNightly-671336d875a543e0911284f0020a4b5e/IMG_7763.JPG",
							size = 2072173,
							volumeGuid = "B90F598C-3328-3F75-B314-035FD0C39493",
						},
					},
				},
				universal = {
					trash = {
						createFrom = {
							id = deletedImageId,
							lastAssetId = deletedImageId,
							lastRevisionId = "d4a62b52683d171ee6f0197f359a4af0",
							lastTrashPurgeDate = "2019-07-12T21:32:40.415Z",
							lastTrashUserCreated = "2019-05-13T21:32:40.415Z",
							revision_id = "d4a62b52683d171ee6f0197f359a4af0",
							subtype = "deleted_image",
						},
						createWithRevId = "5a651d49386c4090a59497d3dca2958d",
						uploaded = {
							["/rels/master"] = true,
							["/rels/proxy_type/2560"] = true,
							["/rels/xmp"] = true,
						},
					},
				},
				xmpSidecarExists = false,
			},
			_serverProvidedMetadata = {
				base = "https://photos-stage.adobe.io/v2/catalogs/2331adb32fdb4004922c5e53e3c7ed75/",
				created = "2019-05-13T21:32:47.862020Z",
				original = {
					id = "f0f041d35ae146aeb70ed527d62d409e",
				},
				updated = "2019-05-13T21:32:50.143303Z",
			},

		},
		':ignore:' )
		
	-- create error doc for asset that indicates the resource is not found
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			_labels = { oz = true },
			_serverProvidedMetadata = {
				new_id = docId,
				errors = {
					asset = {
						"does not exist",
					},
				},
				missing_resource_id = deletedImageId,
				source = {
					action = "create",
					id = docId,
					revision_id = revisionId,
					type = "asset",
				},
				status = 404,
				updated = "2016-06-07T15:55:34.218872Z",
			},
			subtype = "ResourceNotFoundError",
			type = "error",
		},
		':ignore:' )
	
	-- ResourceNotFoundError for asset that has already been restored by this
	-- client, as simulated by the conditions above, should be ignored.
	
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
	
	-- restored asset should remain restored after error doc is removed and
	-- error handling is complete
	rxTest:expectSingleValue( docStore:docValue( docId )
		:map( function( docValue ) 
			return docValue._deleted == nil and docValue._empty == nil
		end ), true )

end

--------------------------------------------------------------------------------

local function prefetchReqResp( catalogId )
	
	return WFMockHttpRequest()
		:get( "/v2/catalogs/" .. catalogId .. "/albums?subtype=collection%3Bcollection_set" )
		:respond( '200', [[{
			"base": "https://photos-stage.adobe.io/v2/catalogs/]]..catalogId ..[[/",
			"resources": []
			}]] )
			
end

--------------------------------------------------------------------------------

local function catalogPrefetchReqResp( catalogId )
	
	return WFMockHttpRequest()
		:get( "/v2/catalogs/" .. catalogId )
		:respond( '200', [[{
			"base": "https://photos-stage.adobe.io/v2/",
			"id": "]] .. catalogId .. [[",
			"type": "catalog"
			}]] )

end

--------------------------------------------------------------------------------

local function getResourceExistsJsonForCatalogAndAsset( catalogId, assetId, revisionId )
	return [[{"base":"https://photos-stage.adobe.io/v2/catalogs/]]..catalogId..[[/",
		"updates":[{"id":"6d7e7425358aa4177ed4469b134a01f1","updated":"2017-09-06T14:54:48.042095Z",
		"type":"error","status":403,"subtype":"ResourceExistsError","errors":{"asset":["already exists"]},
		"source":{"id":"]]..assetId..[[","action":"create","type":"asset",
		"revision_id":"]]..revisionId..[[","links":{"self":
		{"href":"/v2/catalogs/]]..catalogId..[[/updates/]]..assetId..[["}}}}],
		"links":{"next":{"href":"/v2/catalogs/]]..catalogId..[[/updates?updated_since=2017-09-06T14%3A54%3A48.042095Z"}}}]]
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetResourceExistsError_missingConfirmation_mock( rxTest )
	
	rxTest:setTimeLimit( 60 )
	
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	local catalogId = '98db98bfd554470dab2f862a4b853c40'
	
	local catalogSpec = WFCatSpecMock( catalogId, 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	catalogSpec:expectHttpRequest( prefetchReqResp( catalogId ) )
	catalogSpec:expectHttpRequest( catalogPrefetchReqResp( catalogId ) )
	catalogSpec:expectHttpRequest( 
		WFMockHttpRequest()
			:get( "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates" )
			:respond( '200', [[{
				"base": "https://photos-stage.adobe.io/v2/catalogs/98db98bfd554470dab2f862a4b853c40/",
				"updates": [],
				"links": {
					"next": {
						"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1"
					}
				}
			}]] ) )
	
	local docStore = catalog:docStore()
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )
	local conflictResolver = require 'WFCatConflictResolver'( catalog:docStore(), catalog:indexStore() )
	
	rxTest:finally( function()
		puller:close()
		pusher:close()
		errorHandler:close()
		conflictResolver:close()
	end )
	
	-- do an initial pull from Oz to get the prefetch out of the way and empty feed pull
	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )
	
	local payload = {
		_id = assetId,
		_newRevId = 'a98382e8ceab4b39b1283259e77c1768',
		type = 'asset',
		subtype = 'image',
		captureDate = "0000-00-00T00:00:00",
		importSource = {
			fileName = 'test.jpg',
			filePath = '/path/',
			originalHeight = 150,
			originalWidth = 100,
			importedBy = KSIdGenerator.uuidInLowercaseHex(),
			importTimestamp = KSDate.timeInIso8601Format(),
		},
		develop = {
			processingModel = 'lightroom',
			xmpCameraRaw = '<xmpmeta />',
		},
	}
	
	-- create the asset document
	rxTest:expectSingleValue( docStore:setDocValue( payload ), ':ignore:' )
	
	-- expect the pusher's updates feed request
	catalogSpec:expectHttpRequest( WFMockHttpRequest()
		:post( "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates" )
		:validateRequestData( function( str )
					return str:find( [["action":"create"]] ) and
						str:find( [["fileName":"test.jpg"]] ) and
						str:find( [["type":"asset"]] )
				end )
		:respond( '200', [[]] ) )
		
	-- send the updates with the pusher
	
	local sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()
	
	-- modify the asset document (pusher should send another create to Oz)
	rxTest:expectSingleValue( docStore:applyOverlay { 
										_id = assetId,
										_newRevId = revisionId,
										importSource = {
											fileName = 'test2.jpg' 
										}
									}, ':ignore:' )
	
	-- expect the pusher's updates feed request (fileName should be changed)
	catalogSpec:expectHttpRequest( WFMockHttpRequest()
		:post( "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates" )
		:validateRequestData( function( str )
					return str:find( [["action":"create"]] ) and
						str:find( [["fileName":"test2.jpg"]] ) and
						str:find( [["type":"asset"]] )
				end )
		:respond( '200', [[]] ) )
		
	-- send the update to Oz - should be a create here still that will generate a ResourceExistsError
	sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()
	
	-- updates feed pull (no asset confirmation, just ResourceExistsError)
	catalogSpec:expectHttpRequest( WFMockHttpRequest()
		:get( "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1" )
		:respond( '200', getResourceExistsJsonForCatalogAndAsset('98db98bfd554470dab2f862a4b853c40', assetId, revisionId ) ) )
	
	-- after the updates feed pull the error handler should explicitly request the asset from Oz
	catalogSpec:expectHttpRequest( WFMockHttpRequest()
		:get( "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates/"..assetId )
		:respond( '200', [[{"base":"https://photos-stage.adobe.io/v2/catalogs/98db98bfd554470dab2f862a4b853c40/","id":"]]..assetId..[[",
							"type":"asset","updated":"2017-09-06T18:15:10.138376Z","subtype":"image","created":"2017-09-06T18:15:10.138376Z",
							"revision_ids":["a98382e8ceab4b39b1283259e77c1768"], "links":{"self":{"href":"assets/]]..assetId..[["},
							"/rels/proxy_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/proxies/{proxy_id}","templated":true},
							"/rels/rendition_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/renditions/{rendition_id}",
							"templated":true},"/rels/master_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/master","templated":true},
							"/rels/xmp_develop_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/xmp/develop","templated":true}},
							"payload":{"captureDate":"0000-00-00T00:00:00","importSource":{"originalHeight":150,"filePath":"/path/","importTimestamp":"2017-09-06T18:15:09.432Z",
							"importedBy":"81473e13354f414b9eb0f119f24884ac","fileName":"test.jpg","originalWidth":100},"develop":{"processingModel":"lightroom",
							"xmpCameraRaw":"<xmpmeta />"}}}]] ) )
		
	-- get docs from Oz, but should only get a ResourceExistsError for second create, no confirmation for
	-- first create
	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )
	
	-- the error handler should process the ResourceExistsError and the asset info from Oz should
	-- now exist in the 
	local doesDocValueHaveLinks = rxTest:observe( docStore:ongoingDocValue( assetId )
								:map( function( docValue ) 
									return not not KSTableUtils.derefOpt( docValue, '_serverProvidedMetadata', 'links' )
								end ) )
								
	doesDocValueHaveLinks:expectDataWithinTime( 40, true )
	
	-- check index store for error documents, there shouldn't be any anymore
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT * FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:map( function( rows )
			return #rows 
		end ) )
	
	outstandingErrors:expectDataWithinTime( 10, 0 )
	
	-- give the conflict resolver a chance to resolve the doc from Oz
	rxTest:sleep( 3 )
	
	-- after getting the latest from Oz, the local modification (filename) should be resent to Oz as an update
	catalogSpec:expectHttpRequest( WFMockHttpRequest()
		:post( "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates" )
		:validateRequestData( function( str )
					return str:find( [["action":"update"]] ) and
						str:find( [["fileName":"test2.jpg"]] ) and
						str:find( [["type":"asset"]] )
				end )
		:respond( '200', [[]] ) )
	
	--rxTest:expectSingleValue( catalog:docStore():docValue( assetId ), {} )
	sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()
	
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAssetResourceExistsError_confirmationAndErrorCoincidental_mock( rxTest )
	
	rxTest:setTimeLimit( 60 )
	
	local catalogId = '98db98bfd554470dab2f862a4b853c40'
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	
	local catalogSpec = WFCatSpecMock( catalogId, 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
	
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local payload = {
		_id = assetId,
		_newRevId = 'a98382e8ceab4b39b1283259e77c1768',
		type = 'asset',
		subtype = 'image',
		captureDate = "0000-00-00T00:00:00",
		importSource = {
			fileName = 'test.jpg',
			filePath = '/path/',
			originalHeight = 150,
			originalWidth = 100,
			importedBy = KSIdGenerator.uuidInLowercaseHex(),
			importTimestamp = KSDate.timeInIso8601Format(),
		},
		develop = {
			processingModel = 'lightroom',
			xmpCameraRaw = '<xmpmeta />',
		},
	}
	
	local docStore = catalog:docStore()
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )
	local conflictResolver = require 'WFCatConflictResolver'( catalog:docStore(), catalog:indexStore() )
	
	rxTest:finally( function()
		puller:close()
		pusher:close()
		errorHandler:close()
		conflictResolver:close()
	end )
	
	-- do the first pull to get the album prefetch, catalog prefetch, and an empty pull) out of the way
	catalogSpec:expectHttpRequest( prefetchReqResp( catalogId ) )
	catalogSpec:expectHttpRequest( catalogPrefetchReqResp( catalogId ) )
	catalogSpec:expectHttpRequest( 
		WFMockHttpRequest()
			:get( "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates" )
			:respond( '200', [[{
				"base": "https://photos-stage.adobe.io/v2/catalogs/98db98bfd554470dab2f862a4b853c40/",
				"updates": [],
				"links": {
					"next": {
						"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1"
					}
				}
			}]] ) )
	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )
	
	-- create the asset document
	rxTest:expectSingleValue( docStore:setDocValue( payload ), ':ignore:' )
	
	-- send it
	catalogSpec:expectHttpRequest( 
		WFMockHttpRequest()
			:post( "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates" )
			:validateRequestData( function( str )
					return str:find( [["action":"create"]] ) and
						str:find( [["fileName":"test.jpg"]] ) and
						str:find( [["type":"asset"]] )
				end )
			:respond( '200', [[]] ) )

	local sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()
	
	-- modify the asset document (pusher should send another create to Oz since we haven't received a confirmation)
	rxTest:expectSingleValue( docStore:applyOverlay { 
										_id = assetId,
										_newRevId = revisionId,
										importSource = {
											fileName = 'test2.jpg' 
										}
									}, ':ignore:' )
	
	-- send the update to Oz - should be another create here that should generate a ResourceExistsError
	catalogSpec:expectHttpRequest( 
		WFMockHttpRequest()
			:post( "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates" )
			:validateRequestData( function( str )
					return str:find( [["action":"create"]] ) and
						str:find( [["fileName":"test2.jpg"]] ) and
						str:find( [["type":"asset"]] )
				end )
			:respond( '200', [[]] ) )
			
	sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()
	
	-- we'll pull sync feed from Oz and get the original confirmation and a ResourceExistsError for second create, once we have
	-- the ResourceExistsError, the error handler should process it and it should do a get request to get the existing doc on Oz 
	catalogSpec:expectHttpRequest(
		WFMockHttpRequest()
			:get( "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1" )
			:respond( '200',  [[{"base":"https://photos-stage.adobe.io/v2/catalogs/]]..catalogId..[[/",
								"updates":[]]..
								-- asset confirmation
								[[{"id":"]]..assetId..[[",
								"type":"asset","updated":"2017-09-06T18:15:10.138376Z","subtype":"image","created":"2017-09-06T18:15:10.138376Z",
								"revision_ids":["a98382e8ceab4b39b1283259e77c1768"], "links":{"self":{"href":"assets/]]..assetId..[["},
								"/rels/proxy_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/proxies/{proxy_id}","templated":true},
								"/rels/rendition_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/renditions/{rendition_id}",
								"templated":true},"/rels/master_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/master","templated":true},
								"/rels/xmp_develop_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/xmp/develop","templated":true}},
								"payload":{"captureDate":"0000-00-00T00:00:00","importSource":{"originalHeight":150,"filePath":"/path/","importTimestamp":"2017-09-06T18:15:09.432Z",
								"importedBy":"81473e13354f414b9eb0f119f24884ac","fileName":"test.jpg","originalWidth":100},"develop":{"processingModel":"lightroom",
								"xmpCameraRaw":"<xmpmeta />"}}},]]..
								--ResourceExistsError
								[[{"id":"6d7e7425358aa4177ed4469b134a01f1","updated":"2017-09-06T14:54:48.042095Z",
								"type":"error","status":403,"subtype":"ResourceExistsError","errors":{"asset":["already exists"]},
								"source":{"id":"]]..assetId..[[","action":"create","type":"asset",
								"revision_id":"]]..revisionId..[[","links":{"self":
								{"href":"/v2/catalogs/]]..catalogId..[[/updates/]]..assetId..[["}}}}],
								"links":{"next":{"href":"/v2/catalogs/]]..catalogId..[[/updates?updated_since=2017-09-06T14%3A54%3A48.042095Z"}}}]] ) )
		

	catalogSpec:expectHttpRequest(
		WFMockHttpRequest()
			:get( "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates/" .. assetId )
			:respond( '200', [[{"base":"https://photos-stage.adobe.io/v2/catalogs/98db98bfd554470dab2f862a4b853c40/","id":"]]..assetId..[[",
							"type":"asset","updated":"2017-09-06T18:15:10.138376Z","subtype":"image","created":"2017-09-06T18:15:10.138376Z",
							"revision_ids":["a98382e8ceab4b39b1283259e77c1768"], "links":{"self":{"href":"assets/]]..assetId..[["},
							"/rels/proxy_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/proxies/{proxy_id}","templated":true},
							"/rels/rendition_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/renditions/{rendition_id}",
							"templated":true},"/rels/master_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/master","templated":true},
							"/rels/xmp_develop_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/xmp/develop","templated":true}},
							"payload":{"captureDate":"0000-00-00T00:00:00","importSource":{"originalHeight":150,"filePath":"/path/","importTimestamp":"2017-09-06T18:15:09.432Z",
							"importedBy":"81473e13354f414b9eb0f119f24884ac","fileName":"test.jpg","originalWidth":100},"develop":{"processingModel":"lightroom",
							"xmpCameraRaw":"<xmpmeta />"}}}]] ) )
	
	-- pull
	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )
	
	-- the error handler should process the ResourceExistsError and the asset info from Oz should
	-- now exist in the 
	local doesDocValueHaveLinks = rxTest:observe( docStore:ongoingDocValue( assetId )
								:map( function( docValue ) 
									return not not KSTableUtils.derefOpt( docValue, '_serverProvidedMetadata', 'links' )
								end ) )
					
	doesDocValueHaveLinks:expectDataWithinTime( 40, true )
	
	-- check index store for error documents, there shouldn't be any anymore
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT * FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:map( function( rows )
			return #rows 
		end ) )
	
	outstandingErrors:expectDataWithinTime( 10, 0 )
	
	-- give the conflict resolver a chance to resolve the doc from Oz
	rxTest:sleep( 3 )
	
	catalogSpec:expectHttpRequest( 
		WFMockHttpRequest()
			:post( "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates" )
			:validateRequestData( function( str )
					return str:find( [["action":"update"]] ) and
						str:find( [["fileName":"test2.jpg"]] ) and
						str:find( [["type":"asset"]] )
				end )
			:respond( '200', [[]] ) )

	-- we should have updates to push to Oz here
	sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()
	
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestAlbumResourceExistsError_testNightly( rxTest )

	rxTest:setTimeLimit( 120 )

	local catalogTestSession = WFCatTestSession( rxTest )
	
	local catalog
	
	rxTest:expectSingleValue( catalogTestSession:catalog()
		:tap( function( _catalog )
			catalog = _catalog
		end ), ':ignore:' )


	local albumId = KSIdGenerator.uuidInLowercaseHex()
	
	local payload = {
		name = 'ResourceExistsError Album'
	}
	
	local albumDocumentForRestAPI =  {
		type = 'album',
		subtype = 'collection',
		payload = payload,
	}

	local albumDocumentForSyncFeed = KSTableUtils.deepCopy( payload )
	albumDocumentForSyncFeed._id = albumId
	albumDocumentForSyncFeed.type = 'album'
	albumDocumentForSyncFeed.subtype = 'collection'
	

	-- create the album via the rest API
	local albumUrl = string.format( '/v2/catalogs/%s/albums/%s', catalog:ozCatalogId(), albumId )
	rxTest:expectSingleValue( catalog:ozCatalogSpec():put( albumUrl, KSJsonEncoder.encodeOzUploadJson( albumDocumentForRestAPI ), 
																{ 	requestMethod = 'PUT', 
																	requestHeaders = {
																		{ header = 'Content-Type', value = 'application/json' },
																	},
																} ), ':ignore:', ':ignore:' )
	rxTest:sleep( 2 )
	
	-- delete the album via the REST API
	rxTest:expectSingleValue( catalog:ozCatalogSpec():put( albumUrl, '', { requestMethod = 'DELETE',
																 							 returnHttpErrorsAsData = true } )
															:map( function( result, headers )
																return headers.status_code == '204'
															end ), true )
	
	rxTest:sleep( 5 )
	
	local docStore = catalog:docStore()
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )
	
	rxTest:finally( function()
		puller:close()
		pusher:close()
		errorHandler:close()
	end )
	
	-- now try to create the album again locally
	rxTest:expectSingleValue( docStore:setDocValue( albumDocumentForSyncFeed ), ':ignore:' )
	
	local sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()
	
	puller:getUpdatesContinuously()
	
	-- the error handler should process the ResourceExistsError and delete
	-- the album
	local isDocValueDeleted = rxTest:observe( docStore:ongoingDocValue( albumId )
								:map( function( docValue ) 
									return docValue._empty
								end ) )
								
	isDocValueDeleted:expectDataWithinTime( 40, true )
	
	-- check index store for error documents, there shouldn't be any anymore
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT COUNT(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )
	 
	outstandingErrors:expectDataWithinTime( 10, 0 )
	
end

--------------------------------------------------------------------------------
-- SPAM [matthewj 2017-04-20]: Oz currently does not send a ResourceExistsError if an 
-- album_asset id is reused after deletion and instead allows this to work. If that is 
-- changed, this test would cover the ResourceExistsError handling in that case.
--function WFCatErrorHandlerTest.rxTestAlbumAssetResourceExistsError_testNightly( rxTest )
--	
--	rxTest:setTimeLimit( 120 )
--
--	local catalogTestSession = WFCatTestSession( rxTest )
--	
--	local catalog
--	
--	rxTest:expectSingleValue( catalogTestSession:catalog()
--		:tap( function( _catalog )
--			catalog = _catalog
--		end ), ':ignore:' )
--
--
--	local albumId = KSIdGenerator.uuidInLowercaseHex()
--	local albumDocumentForRestAPI =  {
--		type = 'album',
--		subtype = 'collection',
--		payload = {
--			name = 'ResourceExistsError AlbumAsset'
--		},
--	}
--
--	-- create the album via the rest API
--	local albumUrl = string.format( '/v2/catalogs/%s/albums/%s', catalog:ozCatalogId(), albumId )
--	rxTest:expectSingleValue( catalog:ozCatalogSpec():put( albumUrl, KSJsonEncoder.encodeOzUploadJson( albumDocumentForRestAPI ), 
--																{ 	requestMethod = 'PUT', 
--																	requestHeaders = {
--																		{ header = 'Content-Type', value = 'application/json' },
--																	},
--																} ), ':ignore:', ':ignore:' )
--	rxTest:sleep( 2 )
--	
--	-- create an asset via the rest API
--	local assetId = KSIdGenerator.uuidInLowercaseHex()
--	local assetDocumentForRestAPI = {
--		type = 'asset',
--		subtype = 'image',
--		payload = {
--			captureDate = "0000-00-00T00:00:00",
--			importSource = {
--				fileName = 'test.jpg',
--				filePath = '/path/',
--				originalHeight = 150,
--				originalWidth = 100,
--				importedBy = KSIdGenerator.uuidInLowercaseHex(),
--				importTimestamp = KSDate.timeInIso8601Format(),
--			},
--			develop = {
--				processingModel = 'lightroom',
--				xmpCameraRaw = '<xmpmeta />',
--			},
--		},
--	}
--	local assetUrl = string.format( '/v2/catalogs/%s/assets/%s/revisions/%s', catalog:ozCatalogId(), assetId, KSIdGenerator.uuidInLowercaseHex() )
--	rxTest:expectSingleValue( catalog:ozCatalogSpec():put( assetUrl, KSJsonEncoder.encodeOzUploadJson( assetDocumentForRestAPI ), 
--																{ 	requestMethod = 'PUT', 
--																	requestHeaders = {
--																		{ header = 'Content-Type', value = 'application/json' },
--																	},
--																} ), ':ignore:', ':ignore:' )
--																
--	rxTest:sleep( 2 )
--	
--	-- add the asset to the album via the rest API
--	local albumAssetId = KSIdGenerator.uuidInLowercaseHex()
--	local albumAssetForRestAPI = {
--			resources = {
--				{ 
--					id = assetId,
--					album_asset_id = albumAssetId,
--				}
--			}
--		}
--	local albumAssetUrl = string.format( '/v2/catalogs/%s/albums/%s/assets', catalog:ozCatalogId(), albumId )
--	rxTest:expectSingleValue( catalog:ozCatalogSpec():put( albumAssetUrl, KSJsonEncoder.encodeOzUploadJson( albumAssetForRestAPI ),
--															{ 	requestMethod = 'PUT', 
--																	requestHeaders = {
--																		{ header = 'Content-Type', value = 'application/json' },
--																	},
--																} ), ':ignore:', ':ignore:' )
--	
--	
--	rxTest:sleep( 2 )
--	
--	-- delete the album_asset via the REST API
--	local albumAssetDeleteUrl = string.format( '/v2/catalogs/%s/albums/%s/assets/%s', catalog:ozCatalogId(), albumId, assetId )
--	rxTest:expectSingleValue( catalog:ozCatalogSpec():put( albumAssetDeleteUrl, '', { requestMethod = 'DELETE',
--																 							 returnHttpErrorsAsData = true } )
--															:map( function( result, headers )
--																return headers.status_code == '204'
--															end ), true )
--	
--	rxTest:sleep( 2 )
--	
--	-- now try to add the album_asset back with the same albumAssetId via the sync feed
--	local docStore = catalog:docStore()
--	local puller = WFCatMetadataPuller( catalog )
--	local pusher = WFCatMetadataPusher( catalog, options )
--	local errorHandler = WFCatErrorHandler( catalog, {
--		deleteAssets = bulkDelete,
--		deleteIndividualAlbum = deleteDoc } )
--	
--	rxTest:finally( function()
--		puller:close()
--		pusher:close()
--		errorHandler:close()
--	end )
--	
--	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )
--	
--	puller:getUpdatesContinuously()
--	
--	rxTest:expectSingleValue( docStore:setDocValue( {
--					_id = albumAssetId,
--					type = 'album_asset',
--					album = {
--						id = albumId,
--					},
--					asset = {
--						id = assetId,
--					}
--				} ), ':ignore:' )
--	
--	local sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
--	sendNUpdates:expectCompletedWithinTime( 30 )
--  sendNUpdates:dispose()
--
--	local isDocValueDeleted = rxTest:observe( docStore:ongoingDocValue( albumAssetId )
--								:map( function( docValue ) 
--									return docValue._empty
--								end ) )
--								
--	isDocValueDeleted:expectDataWithinTime( 40, true )
--	
--	local albumAssetCount = rxTest:observe( catalog:indexStore():query {
--			sql = [[SELECT COUNT(*) FROM album_asset_v2]],
--			quiesceUntilUpToDate = true,
--		}
--		:pluck( 1, 1 ) )
--	
--	albumAssetCount:expectDataWithinTime( 10, 0 )
--	
--	-- check index store for error documents, there shouldn't be any anymore
--	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
--			sql = [[SELECT COUNT(*) FROM errors]],
--			quiesceUntilUpToDate = true,
--		}
--		:pluck( 1, 1 ) )
--	 
--	outstandingErrors:expectDataWithinTime( 10, 0 )
--	
--end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestDuplicateAlbumAssetResourceExistsError_testNightly( rxTest )
	-- tests that we delete an album_asset record if we try to send a duplicate album_asset record 
	-- with a different album_asset id than the one that already exists on Oz
	
	rxTest:setTimeLimit( 120 )

	local catalogTestSession = WFCatTestSession( rxTest )
	
	local catalog
	
	rxTest:expectSingleValue( catalogTestSession:catalog()
		:tap( function( _catalog )
			catalog = _catalog
		end ), ':ignore:' )


	local albumId = KSIdGenerator.uuidInLowercaseHex()
	local albumDocumentForRestAPI =  {
		type = 'album',
		subtype = 'collection',
		payload = {
			name = 'ResourceExistsError AlbumAsset'
		},
	}

	-- create the album via the rest API
	local albumUrl = string.format( '/v2/catalogs/%s/albums/%s', catalog:ozCatalogId(), albumId )
	rxTest:expectSingleValue( catalog:ozCatalogSpec():put( albumUrl, KSJsonEncoder.encodeOzUploadJson( albumDocumentForRestAPI ), 
																{ 	requestMethod = 'PUT', 
																	requestHeaders = {
																		{ header = 'Content-Type', value = 'application/json' },
																	},
																} ), ':ignore:', ':ignore:' )
	rxTest:sleep( 2 )
	
	-- create an asset via the rest API
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local assetDocumentForRestAPI = {
		type = 'asset',
		subtype = 'image',
		payload = {
			captureDate = "0000-00-00T00:00:00",
			importSource = {
				fileName = 'test.jpg',
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
				importedBy = KSIdGenerator.uuidInLowercaseHex(),
				importTimestamp = KSDate.timeInIso8601Format(),
			},
			develop = {
				processingModel = 'lightroom',
				xmpCameraRaw = '<xmpmeta />',
			},
		},
	}
	local assetUrl = string.format( '/v2/catalogs/%s/assets/%s/revisions/%s', catalog:ozCatalogId(), assetId, KSIdGenerator.uuidInLowercaseHex() )
	rxTest:expectSingleValue( catalog:ozCatalogSpec():put( assetUrl, KSJsonEncoder.encodeOzUploadJson( assetDocumentForRestAPI ), 
																{ 	requestMethod = 'PUT', 
																	requestHeaders = {
																		{ header = 'Content-Type', value = 'application/json' },
																	},
																} ), ':ignore:', ':ignore:' )
																
	rxTest:sleep( 2 )
	
	-- add the asset to the album via the rest API
	local albumAssetId = KSIdGenerator.uuidInLowercaseHex()
	local albumAssetForRestAPI = {
			resources = {
				{ 
					id = assetId,
					album_asset_id = albumAssetId,
				}
			}
		}
	local albumAssetUrl = string.format( '/v2/catalogs/%s/albums/%s/assets', catalog:ozCatalogId(), albumId )
	rxTest:expectSingleValue( catalog:ozCatalogSpec():put( albumAssetUrl, KSJsonEncoder.encodeOzUploadJson( albumAssetForRestAPI ),
															{ 	requestMethod = 'PUT', 
																	requestHeaders = {
																		{ header = 'Content-Type', value = 'application/json' },
																	},
																} ), ':ignore:', ':ignore:' )
	
	
	rxTest:sleep( 2 )
	
	-- now try to add the album_asset back with the a different albumAssetId via the sync feed
	local docStore = catalog:docStore()
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )
	
	rxTest:finally( function()
		puller:close()
		pusher:close()
		errorHandler:close()
	end )
	
	local duplicateAlbumAssetId = KSIdGenerator.uuidInLowercaseHex()
	rxTest:expectSingleValue( docStore:setDocValue( {
					_id = duplicateAlbumAssetId, -- give the album_asset an id different from the existing one.
					type = 'album_asset',
					album = {
						id = albumId,
					},
					asset = {
						id = assetId,
					}
				} ), ':ignore:' )
	
	local sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()
	
	puller:getUpdatesContinuously()
		
	local isDocValueDeleted = rxTest:observe( docStore:ongoingDocValue( duplicateAlbumAssetId )
								:map( function( docValue ) 
									return docValue._empty
								end ) )
								
	isDocValueDeleted:expectDataWithinTime( 40, true )
	
	local albumAssetCount = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT COUNT(*) FROM album_asset_v2]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )
	
	albumAssetCount:expectDataWithinTime( 10, 1 )
	
	-- check index store for error documents, there shouldn't be any anymore
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT COUNT(*) FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:pluck( 1, 1 ) )
	 
	outstandingErrors:expectDataWithinTime( 10, 0 )
	
end

--------------------------------------------------------------------------------

-- @mark RestoreAssetError

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestRestoreAssetError_originatingClient_partialRestore( rxTest )

	-- A RestoreAssetError for a restore operation that originated on the current client. When 
	-- a client's restore operation fails with a RestoreAssetError, the client may have already
	-- received an asset update from Oz that would otherwise indicate to the client that Oz 
	-- successfully processed the restore. The RestoreAssetError indicates it ultimately failed
	-- and that the client needs to retry. In order to do that, the client error handler needs
	-- to clean up the data that makes it look to the client like it succeeded and ensure that
	-- the asset doc is updated so that the WFCatMetadataPusher will pick it up again and resend
	-- the restore.
	rxTest:setTimeLimit( 5 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog, 
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc 
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	
	local trashAssetDocId = KSIdGenerator.uuidInLowercaseHex()
	local restoredDocId = KSIdGenerator.uuidInLowercaseHex()
	local restoredRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	
	-- create a basic restored asset doc locally which includes a createFrom section indicating it's
	-- a restore initiated from this client
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = restoredDocId,
			_newRevId = restoredRevisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			},
			_localOnly = {
				universal = {
					trash = {
						createFrom = {
							id = trashAssetDocId,
							lastAssetId = trashAssetDocId,
							lastRevisionId = "d4a62b52683d171ee6f0197f359a4af0",
							lastTrashPurgeDate = "2019-07-12T21:32:40.415Z",
							lastTrashUserCreated = "2019-05-13T21:32:40.415Z",
							revision_id = "d4a62b52683d171ee6f0197f359a4af0",
							subtype = "deleted_image",
						},
						createWithRevId = restoredRevisionId,
					}
				}
			}			
		},
		':ignore:' )
	
	-- update with partially restored doc from Oz
	-- oz labeled doc
	
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_labels = {
				oz = true
			},
			_id = restoredDocId,
			_newRevId = restoredRevisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			},
			_serverProvidedMetadata = {
				links = {
					[ "/rels/master" ] = "fakelink"
				}
			}
		},
		':ignore:' )
	
	-- make sure we have a doc labeled 'oz'
	rxTest:expectSingleValue( docStore:getRevisionForLabel( restoredDocId, 'oz' ), restoredRevisionId )
	
	local prevDocChangeCounter = 0
	rxTest:expectSingleValue( docStore:docValue( restoredDocId )
		:map( function( docValue )
			prevDocChangeCounter = docValue._changeCounter
			return true
		end ), true )
		
	-- add RestoreAssetError doc for asset that indicates the restore failed and asset is not 
	-- actually on Oz
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "RestoreAssetError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"restore failed",
					},
				},
				source = {
					action = "create",
					id = restoredDocId,
					revision_id = restoredRevisionId,
					type = "asset",
					restore_from = {
						id = trashAssetDocId
					}
				},
				status = 502,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
	
	
	-- restore asset doc needs to be retried, so we need to make sure it has new sequence +
	-- no Oz label so the pusher will resend it to Oz as a create
	rxTest:expectSingleValue( docStore:getRevisionForLabel( restoredDocId, 'oz' ):catch( Rx.Observable.just(nil) ), nil )
	
	local docChangeCounter = rxTest:observe( docStore:ongoingDocValue( restoredDocId )
		:map( function( restoredDocValue )
			return restoredDocValue._changeCounter > prevDocChangeCounter
		end ) )
	docChangeCounter:expectDataWithinTime( 2, true )
	
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestRestoreAssetError_originatingClient_noPartialRestore( rxTest )

	-- A RestoreAssetError for a restore operation that originated on the current
	-- client.

	-- This test validates restoreAssetError handler properly handles the case
	-- where a doc update with a partial restore of the asset was not pulled
	-- down.

	rxTest:setTimeLimit( 5 )

	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local errorHandler = WFCatErrorHandler( catalog, 
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc 
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local trashAssetDocId = KSIdGenerator.uuidInLowercaseHex()
	local restoredDocId = KSIdGenerator.uuidInLowercaseHex()
	local restoredRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- create a basic restored asset doc locally which includes a createFrom section indicating it's
	-- a restore initiated from this client
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = restoredDocId,
			_newRevId = restoredRevisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			},
			_localOnly = {
				universal = {
					trash = {
						createFrom = {
							id = trashAssetDocId,
							lastAssetId = trashAssetDocId,
							lastRevisionId = "d4a62b52683d171ee6f0197f359a4af0",
							lastTrashPurgeDate = "2019-07-12T21:32:40.415Z",
							lastTrashUserCreated = "2019-05-13T21:32:40.415Z",
							revision_id = "d4a62b52683d171ee6f0197f359a4af0",
							subtype = "deleted_image",
						},
						createWithRevId = restoredRevisionId,
					}
				}
			}
		},
		':ignore:' )

	local prevDocChangeCounter = 0
	rxTest:expectSingleValue( docStore:docValue( restoredDocId )
		:map( function( docValue )
			prevDocChangeCounter = docValue._changeCounter
			return true
		end ), true )

	-- Add RestoreAssetError doc for asset that indicates the restore failed.
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "RestoreAssetError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"restore failed",
					},
				},
				source = {
					action = "create",
					id = restoredDocId,
					revision_id = restoredRevisionId,
					type = "asset",
					restore_from = {
						id = trashAssetDocId
					}
				},
				status = 502,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )

	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
		:dispose()

	-- restore asset doc needs to be retried, so we need to make sure it has new sequence +
	-- no Oz label so the pusher will resend it to Oz as a create
	rxTest:expectSingleValue( docStore:getRevisionForLabel( restoredDocId, 'oz' ):catch( Rx.Observable.just(nil) ), nil )

	rxTest:observe( docStore:ongoingDocValue( restoredDocId )
		:map( function( restoredDocValue )
			return restoredDocValue._changeCounter > prevDocChangeCounter
		end ) )
	:expectDataWithinTime( 2, true )
	:dispose()

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestRestoreAssetError_otherClient_partialRestore( rxTest )
	-- A RestoreAssetError for a restore operation that originated on a different client.
	-- When Oz attempts a restore initiated by another client, it may get partway through 
	-- the restore and then encounter an error. Due to the current implementation on Oz, 
	-- this may result in an asset doc being sent to all clients (which may include binary 
	-- links) that is then removed from Oz because the restore failed. In these cases, Oz 
	-- does not send a tombstone for the partially restored doc but instead sends a 
	-- RestoreAssetError. Because may have sent a partial document and no tombstone clients
	-- that did not initiate the restore need to treat the RestoreAssetError somewhat like a 
	-- tombstone. But instead of issuing a delete (the restored asset may eventually come down
	-- correctly from Oz) we need to remove any _serverProvidedMetadata that indicates it 
	-- may have data on Oz.
	rxTest:setTimeLimit( 5 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog, 
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc 
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	
	local trashAssetDocId = KSIdGenerator.uuidInLowercaseHex()
	local restoredDocId = KSIdGenerator.uuidInLowercaseHex()
	local restoredRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	
	-- update with partially restored doc from Oz
	-- oz labeled doc
	
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_labels = {
				oz = true
			},
			_id = restoredDocId,
			_newRevId = restoredRevisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			},
			_serverProvidedMetadata = {
				links = {
					[ "/rels/master" ] = "fakelink"
				}
			}
		},
		':ignore:' )
	
	-- make sure we have a doc labeled 'oz'
	rxTest:expectSingleValue( docStore:getRevisionForLabel( restoredDocId, 'oz' ), restoredRevisionId )
		
	-- add RestoreAssetError doc for asset that indicates the restore failed and asset is not 
	-- actually on Oz
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "RestoreAssetError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"restore failed",
					},
				},
				source = {
					action = "create",
					id = restoredDocId,
					revision_id = restoredRevisionId,
					type = "asset",
					restore_from = {
						id = trashAssetDocId
					}
				},
				status = 502,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
		:expectDataWithinTime( 3, true )
	
	
	-- Partially restored asset doc needs to be cleaned up so that it doesn't have enough info to be shown in the client. This 
	-- means removing the _serverProvidedMetadata. Unlike the originating client though, we want to keep the 'oz' label or 
	-- else the current client is going to try to send this new doc back to Oz as a create (not a restore) and it has none of the
	-- binaries for the asset so it can't create anything real.
	rxTest:expectSingleValue( docStore:getRevisionForLabel( restoredDocId, 'oz' ), restoredRevisionId )
	
	local restoredDocValueHasNoOzData = rxTest:observe( docStore:ongoingDocValue( restoredDocId )
		:map( function( restoredDocValue )
			return restoredDocValue._serverProvidedMetadata == nil
		end ) )
	restoredDocValueHasNoOzData:expectDataWithinTime( 2, true )

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestRestoreAssetError_otherClient_noPartialRestore( rxTest )

	-- A RestoreAssetError for a restore operation that originated on a different client.

	-- This test validates restoreAssetError handler properly handles the case
	-- where a doc update with a partial restore of the asset was not pulled
	-- down.

	rxTest:setTimeLimit( 5 )

	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

		local errorHandler = WFCatErrorHandler( catalog, 
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc 
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )

	local docStore = catalog:docStore()

	local trashAssetDocId = KSIdGenerator.uuidInLowercaseHex()
	local restoredDocId = KSIdGenerator.uuidInLowercaseHex()
	local restoredRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()

	-- Add RestoreAssetError doc for asset that indicates a restore failed.
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "RestoreAssetError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"restore failed",
					},
				},
				source = {
					action = "create",
					id = restoredDocId,
					revision_id = restoredRevisionId,
					type = "asset",
					restore_from = {
						id = trashAssetDocId
					}
				},
				status = 502,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )

	-- Error doc should be deleted
	rxTest:observe( WFRxTestUtils.ongoingDocValueIsEmpty( docStore, errorDocId ) )
	:expectDataWithinTime( 3, true )
	:dispose()

	-- Still shouldn't have a restored doc value.
	rxTest:expectSingleValue( docStore:docValue( restoredDocId ), {
		_empty = true,
		_id = restoredDocId,
	} )

end

--------------------------------------------------------------------------------

-- @mark Unknown Errors

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestUnknownError_Not500( rxTest )
	
	-- unhandled errors are logged and then marked as unknown so that we can ignore it for now
	-- but possibly go back to it if we come up with a way to handle and reindex it later
	rxTest:setTimeLimit( 15 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )
	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local docId = KSIdGenerator.uuidInLowercaseHex()
	local docRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = 'd58059e3898ee16896d490a9ea3c6027'
	
	-- create basic 'asset' doc
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId,
			_newRevId = docRevisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		},
		':ignore:' )
	
	-- create error doc for asset
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ThisIsAnUnknownErrorSubtype",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"unknown error",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = docRevisionId,
					type = "asset",
				},
				status = 400,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- error doc should be annotated that the error was unknown
	rxTest:observe( docStore:ongoingDocValue( errorDocId )
						:map( function( docValue )
							return KSTableUtils.derefOpt( docValue, '_localOnly', 'unknownError' )
						end ) )
		:expectDataWithinTime( 3, true )
		
	-- since we've marked the error as unknown, we shouldn't see it in the errors table anymore
	rxTest:observe( catalog:indexStore():query {
							sql = [[SELECT count( docId ) FROM errors]],
						}
						:pluck( 1, 1 ) )
		:expectDataWithinTime( 2, 0 )
						
	
	-- asset should have no error annotation since we don't know how to clear an unknown error
	rxTest:expectSingleValue( 
		docStore:docValue( docId )
			:map( function( docValue )
				return not not KSTableUtils.derefOpt( docValue, '_localOnly', 'ozError' )
			end ), false )

end


--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestUnknownError_500_Retry( rxTest )
	
	-- test that with unknown errors with a status code of 5XX that we retry the document
	-- automatically
	rxTest:setTimeLimit( 15 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog,
		{
			deleteAssets = bulkDelete,
			deleteIndividualAlbum = deleteDoc
		},
		errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local docId = KSIdGenerator.uuidInLowercaseHex()
	local docRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = 'd58059e3898ee16896d490a9ea3c6027'
	
	-- create basic 'asset' doc
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId,
			_newRevId = docRevisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		},
		':ignore:' )
		
	local initialChangeCounter
	rxTest:expectSingleValue( docStore:docValue( docId )
		:map( function( docValue )
			initialChangeCounter = docValue._changeCounter
			return docValue
		end ),
		':ignore:' )
	
	-- create error doc for asset
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ThisIsAnUnknownErrorSubtype",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"unknown error",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = docRevisionId,
					type = "asset",
				},
				status = 500,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- error doc should be deleted after handling it (by marking the asset for retry
	rxTest:observe( docStore:ongoingDocValue( errorDocId )
						:map( function( docValue )
							return docValue._empty
						end ) )
		:expectDataWithinTime( 3, true )
		
	-- since we've deleted the error we shouldn't see it in the errors table anymore
	rxTest:observe( catalog:indexStore():query {
							sql = [[SELECT count( docId ) FROM errors]],
						}
						:pluck( 1, 1 ) )
		:expectDataWithinTime( 2, 0 )
						
	
	-- asset should have an updated change counter after error processing
	rxTest:expectSingleValue( docStore:docValue( docId )
		:map( function( docValue )
			return docValue._changeCounter > initialChangeCounter
		end ),
		true )
	

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestUnknownError_500_Retry_NoLooper( rxTest )
	
	-- test that with unknown errors with a status code of 5XX that we retry the document
	-- automatically but that we don't retry forever and don't create a looper
	rxTest:setTimeLimit( 15 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )
	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local docId = KSIdGenerator.uuidInLowercaseHex()
	local docRevisionId = KSIdGenerator.uuidInLowercaseHex()
	
	-- create basic 'asset' doc
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId,
			_newRevId = docRevisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		},
		':ignore:' )
		
	local changeCounter
	rxTest:expectSingleValue( docStore:docValue( docId )
		:map( function( docValue )
			changeCounter = docValue._changeCounter
			return docValue
		end ),
		':ignore:' )
	
	-- create 1st error doc for asset
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ThisIsAnUnknownErrorSubtype",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"unknown error",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = docRevisionId,
					type = "asset",
				},
				status = 500,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- error doc should be deleted after handling it (by marking the asset for retry
	rxTest:observe( docStore:ongoingDocValue( errorDocId )
						:map( function( docValue )
							return docValue._empty
						end ) )
		:expectDataWithinTime( 3, true )
		
	-- asset should have an updated change counter after error processing
	rxTest:expectSingleValue( docStore:docValue( docId )
		:map( function( docValue )
			local isChangeCounterHigher = docValue._changeCounter > changeCounter
			changeCounter = docValue._changeCounter
			return isChangeCounterHigher
		end ),
		true )
		
	-- create 2nd error document on asset
	errorDocId = KSIdGenerator.uuidInLowercaseHex()
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ThisIsAnUnknownErrorSubtype",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"unknown error",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = docRevisionId,
					type = "asset",
				},
				status = 503,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- error doc should be deleted after handling it (by marking the asset for retry
	rxTest:observe( docStore:ongoingDocValue( errorDocId )
						:map( function( docValue )
							return docValue._empty
						end ) )
		:expectDataWithinTime( 3, true )
		
	-- asset should have an updated change counter after error processing
	rxTest:expectSingleValue( docStore:docValue( docId )
		:map( function( docValue )
			local isChangeCounterHigher = docValue._changeCounter > changeCounter
			changeCounter = docValue._changeCounter
			return isChangeCounterHigher
		end ),
		true )
		
	-- create 3rd error document on asset
	errorDocId = KSIdGenerator.uuidInLowercaseHex()
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ThisIsAnUnknownErrorSubtype",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"unknown error",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = docRevisionId,
					type = "asset",
				},
				status = 512,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- error doc should be deleted after handling it (by marking the asset for retry
	rxTest:observe( docStore:ongoingDocValue( errorDocId )
						:map( function( docValue )
							return docValue._empty
						end ) )
		:expectDataWithinTime( 3, true )
		
	-- asset should have an updated change counter after error processing
	rxTest:expectSingleValue( docStore:docValue( docId )
		:map( function( docValue )
			local isChangeCounterHigher = docValue._changeCounter > changeCounter
			changeCounter = docValue._changeCounter
			return isChangeCounterHigher
		end ),
		true )
	
	-- create 4th error document on asset
	errorDocId = KSIdGenerator.uuidInLowercaseHex()
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ThisIsAnUnknownErrorSubtype",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"unknown error",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = docRevisionId,
					type = "asset",
				},
				status = 504,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- error doc should be deleted after handling it (by marking the asset for retry
	rxTest:observe( docStore:ongoingDocValue( errorDocId )
						:map( function( docValue )
							return docValue._empty
						end ) )
		:expectDataWithinTime( 3, true )
		
	-- asset should not have an updated change counter after error processing, we can't
	-- keep sending an asset update to Oz or that will be a looper
	local lastRetryTime
	rxTest:expectSingleValue( docStore:docValue( docId )
		:map( function( docValue )
			local isChangeCounterHigher = docValue._changeCounter > changeCounter
			changeCounter = docValue._changeCounter
			lastRetryTime = KSTableUtils.derefOpt( docValue, '_localOnly', 'ozError', 'lastRetry' )
			return isChangeCounterHigher
		end ),
		false )
	
	-- now that we've stopped the possibility of a bad looper, we can't let this error
	-- stop retries forever if this same asset hits another 500 error. We'll make a change
	-- to make the lastRetry time a bit over an hour old. Then send another error to see if we 
	-- properly reset the retries and update the change counter so that it will be resent
	-- to Oz
	
	local retryTimeSinceReference = KSDate.iso8601ToTimeIntervalSinceReference( lastRetryTime )
	retryTimeSinceReference = retryTimeSinceReference - 5400 -- (1.5 hours)
	local newLastRetryTime = KSDate.timeInIso8601Format( retryTimeSinceReference )
	
	rxTest:expectSingleValue( 
		docStore:applyOverlay {
			_id = docId,
			_localOnly = {
				ozError = {
					lastRetry = newLastRetryTime,
				}
			}
		}, ':ignore:' )
	
	-- after the overlay, get the latest change counter (it should also be higher than the last time)
	rxTest:expectSingleValue( docStore:docValue( docId )
		:map( function( docValue )
			local isChangeCounterHigher = docValue._changeCounter > changeCounter
			changeCounter = docValue._changeCounter
			docRevisionId = docValue._revId
			return isChangeCounterHigher
		end ),
		true )
		
	-- create 5th error document
	errorDocId = KSIdGenerator.uuidInLowercaseHex()
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ThisIsAnUnknownErrorSubtype",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"unknown error",
					},
				},
				source = {
					action = "create",
					id = docId,
					revision_id = docRevisionId,
					type = "asset",
				},
				status = 503,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- error doc should be deleted after handling it (by marking the asset for retry
	rxTest:observe( docStore:ongoingDocValue( errorDocId )
						:map( function( docValue )
							return docValue._empty
						end ) )
		:expectDataWithinTime( 3, true )
		
	-- asset should not have an updated change counter after error processing, we can't
	-- keep sending an asset update to Oz or that will be a looper
	rxTest:expectSingleValue( docStore:docValue( docId )
		:map( function( docValue )
			local isChangeCounterHigher = docValue._changeCounter > changeCounter
			changeCounter = docValue._changeCounter
			return isChangeCounterHigher
		end ),
		true )
		
	
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestErrorNotThisClient( rxTest )
	
	-- the sync feed will receive errors that may have been generated from other clients. when this happens, the error 
	-- handler should ignore the error and delete it straight away. 
	rxTest:setTimeLimit( 15 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )
	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local docId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = KSIdGenerator.uuidInLowercaseHex()
	
	-- create basic 'asset' doc
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		},
		':ignore:' )
	
	-- create error doc for random asset
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "InputValidationError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"unknown error",
					},
				},
				source = {
					action = "create",
					id = KSIdGenerator.uuidInLowercaseHex(),
					revision_id = KSIdGenerator.uuidInLowercaseHex(),
					type = "asset",
				},
				status = 493,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- since we've deleted the error as not from this client, we shouldn't see it in the errors table anymore
	rxTest:observe( catalog:indexStore():query {
							sql = [[SELECT count( docId ) FROM errors]],
						}
						:pluck( 1, 1 ) )
		:expectDataWithinTime( 2, 0 )
						
	
	-- error document should be marked as deleted
	local errorDocValue = rxTest:observe( docStore:ongoingDocValue( errorDocId )
			:map( function( docValue )
				return docValue._empty
			end ) )
			
	errorDocValue:expectDataWithinTime( 5, true )
	
	-- create error doc for asset we have but different revision (i.e. not this client or, at least, not the 
	-- latest version of the document on this client)
	errorDocId = KSIdGenerator.uuidInLowercaseHex()
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "InputValidationError",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"unknown error",
					},
				},
				source = {
					action = "update",
					id = docId,
					revision_id = KSIdGenerator.uuidInLowercaseHex(),
					type = "asset",
				},
				status = 493,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- since we've deleted the error as not from this client, we shouldn't see it in the errors table anymore
	rxTest:observe( catalog:indexStore():query {
							sql = [[SELECT count( docId ) FROM errors]],
						}
						:pluck( 1, 1 ) )
		:expectDataWithinTime( 2, 0 )
	
	-- error document should be marked as deleted
	errorDocValue = rxTest:observe( docStore:ongoingDocValue( errorDocId )
			:map( function( docValue )
				return docValue._empty
			end ) )
			
	errorDocValue:expectDataWithinTime( 5, true )
	
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestUnhandledError_MissingSourceSection( rxTest )
	
	-- we can't verify whether this error is for this client, so we should log it and mark as unknown so if we ever figure it out
	-- we can possibly reattempt to process it
		-- unhandled errors are logged and then marked as unknown so that we can ignore it for now
	-- but possibly go back to it if we come up with a way to handle and reindex it later
	rxTest:setTimeLimit( 15 )
	
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )
	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local docId = KSIdGenerator.uuidInLowercaseHex()
	local docRevisionId = KSIdGenerator.uuidInLowercaseHex()
	local errorDocId = 'd58059e3898ee16896d490a9ea3c6027'
	
	-- create basic 'asset' doc
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = docId,
			_newRevId = docRevisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		},
		':ignore:' )
	
	-- create error doc for asset
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = errorDocId,
			subtype = "ThisIsAnUnknownErrorSubtype",
			type = "error",
			_serverProvidedMetadata = {
				errors = {
					asset = {
						"unknown error",
					},
				},
				status = 400,
				updated = "2016-06-07T15:55:34.218872Z",
			},
		},
		':ignore:' )
	
	-- error doc should be annotated that the error was unknown
	rxTest:observe( docStore:ongoingDocValue( errorDocId )
						:map( function( docValue )
							return KSTableUtils.derefOpt( docValue, '_localOnly', 'unknownError' )
						end ) )
		:expectDataWithinTime( 3, true )
		
	-- since we've marked the error as unknown, we shouldn't see it in the errors table anymore
	rxTest:observe( catalog:indexStore():query {
							sql = [[SELECT count( docId ) FROM errors]],
						}
						:pluck( 1, 1 ) )
		:expectDataWithinTime( 2, 0 )
						
	
	-- asset should have no error annotation since we don't know how to clear an unknown error
	rxTest:expectSingleValue( 
		docStore:docValue( docId )
			:map( function( docValue )
				return not not KSTableUtils.derefOpt( docValue, '_localOnly', 'ozError' )
			end ), false )

end

--------------------------------------------------------------------------------

local function httpMockDataTransformation( data, requestInfoData )

	data = KSJsonDecoder.decodeWebSafeJson( data )
	requestInfoData = KSJsonDecoder.decodeWebSafeJson( requestInfoData )

	return data, requestInfoData

end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestResourceNotFoundError_ErrorOnDelete_mock( rxTest )
	-- this test case is based on the behavior seen in NMB-9050
	rxTest:setTimeLimit( 15 )
	
	local catalogId = '98db98bfd554470dab2f862a4b853c40'
	
	local catalogSpec = WFCatSpecMock( catalogId, 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )
	
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	catalogSpec:expectHttpRequest( prefetchReqResp( catalogId ) )
	catalogSpec:expectHttpRequest( catalogPrefetchReqResp( catalogId ) )
	catalogSpec:expectHttpRequest( 
		WFMockHttpRequest()
			:get( "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates" )
			:respond( '200', [[{
				"base": "https://photos-stage.adobe.io/v2/catalogs/98db98bfd554470dab2f862a4b853c40/",
				"updates": [],
				"links": {
					"next": {
						"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1"
					}
				}
			}]] ) )
	
	local docStore = catalog:docStore()
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	local pusher = WFCatMetadataPusher( catalog, options )
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )
	
	rxTest:finally( function()
		puller:close()
		pusher:close()
		errorHandler:close()
	end )
	
	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )
	
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local initialRevisionId = KSIdGenerator.uuidInLowercaseHex()
	
	local isSyncPending = rxTest:observe( require 'WFCatSyncStatus'.syncPendingForAnyAsset( catalog ) )
	
	isSyncPending:expectDataWithinTime( 5, false )
	
	-- create basic 'asset' doc
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = assetId,
			_newRevId = initialRevisionId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				fileName = 'test.jpg',
				filePath = '/path/',
				originalHeight = 150,
				originalWidth = 100,
			}
		},
		':ignore:' )
	
	isSyncPending:expectDataWithinTime( 5, true )
	
	-- expect the pusher's updates feed create
	catalogSpec:expectHttpRequest( WFMockHttpRequest()
		:post( "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates" )
		:validateRequestData( function( str )
					return str:find( [["action":"create"]] ) and
						str:find( [["fileName":"test.jpg"]] ) and
						str:find( [["type":"asset"]] )
				end )
		:respond( '200', [[]] ) )
	
	local sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )	
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()
	
	-- expect the acceptance of the create	
	catalogSpec:expectHttpRequest( WFMockHttpRequest()
		:get( "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1" )
		:respond( '200', [[ {"base":"https://photos-stage.adobe.io/v2/catalogs/98db98bfd554470dab2f862a4b853c40/",
							"updates": [
									{ "id":"]]..assetId..[[","type":"asset","updated":"2017-09-06T18:15:10.138376Z","subtype":"image","created":"2017-09-06T18:15:10.138376Z",
								"revision_ids":["]]..initialRevisionId..[["], "links":{"self":{"href":"assets/]]..assetId..[["},
								"/rels/proxy_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/proxies/{proxy_id}","templated":true},
								"/rels/rendition_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/renditions/{rendition_id}",
								"templated":true},"/rels/master_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/master","templated":true},
								"/rels/xmp_develop_create":{"href":"assets/]]..assetId..[[/revisions/{revision_id}/xmp/develop","templated":true}},
								"payload":{"captureDate":"0000-00-00T00:00:00","importSource":{"originalHeight":150,"filePath":"/path/","importTimestamp":"2017-09-06T18:15:09.432Z",
								"importedBy":"81473e13354f414b9eb0f119f24884ac","fileName":"test.jpg","originalWidth":100},"develop":{"processingModel":"lightroom",
								"xmpCameraRaw":"<xmpmeta />"}}} 
								],
								"links": {
									"next": {
										"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
									}
								}
							} ]] ) )

	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )
	
	isSyncPending:expectDataWithinTime( 5, false )
	
	local newRevisionId = KSIdGenerator.uuidInLowercaseHex()
	
	-- delete doc
	rxTest:expectSingleValue( docStore:applyOverlay( {
		_id = assetId,
		_newRevId = newRevisionId,
		_deleted = true
		} ), ':ignore:' )
	
	-- expect push of deletion
	catalogSpec:expectHttpRequest( WFMockHttpRequest()
		:post( "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates" )
		:expectRequestData( [[[{"id":"]]..assetId..[[","type":"asset","action":"delete","revision_id":"]]..newRevisionId..'"}]' )
		:requestDataTransformation( httpMockDataTransformation )
		:respond( '200', [[]] ) )
		
	sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()
	
	-- get RNF on pull rather than tombstone
	catalogSpec:expectHttpRequest( WFMockHttpRequest()
		:get( "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2" )
		:respond( '200', [[{"base":"https://photos-stage.adobe.io/v2/catalogs/98db98bfd554470dab2f862a4b853c40/",
								"updates": []]..
								-- ResourceNotFoundError
								[[{"id":"6d7e7425358aa4177ed4469b134a01f1","updated":"2017-09-06T14:54:48.042095Z",
								"type":"error","status":404,"subtype":"ResourceNotFoundError","errors":{"asset":["does not exist"]},
								"source":{"id":"]]..assetId..[[","action":"delete","type":"asset",
								"revision_id":"]]..newRevisionId..[["}}],
								"links": {
									"next": {
										"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
									}
								}
						} ]] ) )
	
	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )
	
	-- let error handler handle the error
	
	
	-- doc should be marked as deleted on Oz (and no need to sync)
	isSyncPending:expectDataWithinTime( 15, false )
	
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestInvalidAccessRequestError( rxTest )
	
	rxTest:setTimeLimit( 20 )
	
	local catalogSpec = WFCatSpecMock( '99999999999999999999999999999999', '44444444444444444444444444444444', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local inviteId = KSIdGenerator.uuidInLowercaseHex()
	local docRevId
	
		-- create basic 'asset' doc that has not been synced to Oz
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = inviteId,
			type = 'spaceinvite',
			subtype = 'request',
			status = 'approved',
			role = 'viewer',
			space = {
				id = KSIdGenerator.uuidInLowercaseHex(),
			},
			user = {
				id = KSIdGenerator.uuidInLowercaseHex(),
			},
		}
		:map( function( revisionId )
			docRevId = revisionId
			return revisionId
		end ), ':ignore:' )
	
	-- create error doc for asset that indicates the resource is not found
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = KSIdGenerator.uuidInLowercaseHex(),
			_serverProvidedMetadata = {
				base = "https://photos-stage.adobe.io/v2/catalogs/99999999999999999999999999999999/",
				source = {
					action = "update",
					id = inviteId,
					links = {
						["self"] = {
							href = "/v2/catalogs/99999999999999999999999999999999/updates/" .. inviteId,
						},
					},
					revision_id = docRevId,
					type = "spaceinvite",
				},
				status = 403,
				updated = "2017-04-26T19:27:11.417509Z",
			},
			subtype = "InvalidAccessRequestError",
			type = "error",
		},
		':ignore:' )
	
	-- check index store for error documents, there shouldn't be any any more
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT * FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:map( function( rows )
			return #rows 
		end ) )
	
	outstandingErrors:expectDataWithinTime( 10, 0 )
	
	-- the error handler should process the ResourceExistsError but not delete
	-- the space_invite
	rxTest:expectSingleValue( docStore:docValue( inviteId )
								:map( function( docValue ) 
									return not not docValue._empty
								end ), false )
								
end

--------------------------------------------------------------------------------

function WFCatErrorHandlerTest.rxTestSpaceUserExistsError( rxTest )
	
	rxTest:setTimeLimit( 20 )
	
	local catalogSpec = WFCatSpecMock( '99999999999999999999999999999999', '44444444444444444444444444444444', rxTest )
		
	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )
	
	local errorHandler = WFCatErrorHandler( catalog, {}, errorHandlerOptions )

	rxTest:addCloseTarget( errorHandler )
	
	local docStore = catalog:docStore()
	
	local inviteId = KSIdGenerator.uuidInLowercaseHex()
	local docRevId
	
		-- create basic 'asset' doc that has not been synced to Oz
	rxTest:expectSingleValue( 
		docStore:setDocValue {
			_id = inviteId,
			type = 'spaceinvite',
			subtype = 'request',
			status = 'approved',
			role = 'viewer',
			space = {
				id = KSIdGenerator.uuidInLowercaseHex(),
			},
			user = {
				id = KSIdGenerator.uuidInLowercaseHex(),
			},
		}
		:map( function( revisionId )
			docRevId = revisionId
			return revisionId
		end ), ':ignore:' )
	
	-- create error doc for asset that indicates the resource is not found
	rxTest:expectSingleValue(
		docStore:setDocValue {
			_id = KSIdGenerator.uuidInLowercaseHex(),
			_serverProvidedMetadata = {
				base = "https://photos-stage.adobe.io/v2/catalogs/99999999999999999999999999999999/",
				errors = {
					space_user = {
						"already exists for space",
					},
				},
				source = {
					action = "update",
					id = inviteId,
					links = {
						["self"] = {
							href = "/v2/catalogs/99999999999999999999999999999999/updates/" .. inviteId,
						},
					},
					revision_id = docRevId,
					type = "spaceinvite",
				},
				status = 403,
				updated = "2017-04-26T19:27:11.417509Z",
			},
			subtype = "SpaceUserExistsError",
			type = "error",
		},
		':ignore:' )
	
	-- check index store for error documents, there shouldn't be any any more
	local outstandingErrors = rxTest:observe( catalog:indexStore():query {
			sql = [[SELECT * FROM errors]],
			quiesceUntilUpToDate = true,
		}
		:map( function( rows )
			return #rows 
		end ) )
	
	outstandingErrors:expectDataWithinTime( 10, 0 )
	
	-- the error handler should process the SpaceUserExistsError and also delete
	-- the offending space_invite
	rxTest:expectSingleValue( docStore:docValue( inviteId )
								:map( function( docValue ) 
									return docValue._deleted
								end ), true )
								
end

--------------------------------------------------------------------------------

-- SPAM [jandrew 2020-11-24]: The InsufficientPermission Error Handler is tested
-- within WFCatSpaceFeedsManagerTest.

--------------------------------------------------------------------------------

return WFRxTestCase.wrapSuite( WFCatErrorHandlerTest )
