--[[----------------------------------------------------------------------------

WFCatTrashMultipleDevicesBasicOtherRestoreTest.lua

--------------------------------------------------------------------------------
ADOBE CONFIDENTIAL
------------------
Copyright 2023 Adobe
All Rights Reserved.

NOTICE: All information contained herein is, and remains
the property of Adobe and its suppliers, if any. The intellectual
and technical concepts contained herein are proprietary to Adobe
and its suppliers and are protected by all applicable intellectual
property laws, including trade secret and copyright laws.
Dissemination of this information or reproduction of this material
is strictly forbidden unless prior written permission is obtained
from Adobe.

----------------------------------------------------------------------------]]--

	-- WF.core.self_test
local WFRxTestCase = require 'WFRxTestCase'
local WFTestFileUtils = require 'WFTestFileUtils'

	-- WF.core.os
local KSDate = require 'KSDate'
local WFFileUtils = require 'WFFileUtils'

	-- WF.core.utils
local KSIdGenerator = require 'KSIdGenerator'
local KSTableUtils = require 'KSTableUtils'
local WFCatAssetUtils = require 'WFCatAssetUtils'
local WFCatAssetTestUtils = require 'WFCatAssetTestUtils'

	-- WF.oz_client.managed_catalog
local WFCatTrash = require 'WFCatTrash'
local WFCatTrashTestUtils = require 'WFCatTrashTestUtils'
local WFCatTrashUtils = require 'WFCatTrashUtils'

	-- WF.oz_client.shared.utils
local LrAssetUtils = require 'LrAssetUtils'

	-- WF.web.http_client
local WFMockHttpRequest = require 'WFMockHttpRequest'

	-- WF.web.json
local KSJsonDecoder = require 'KSJsonDecoder'
local KSJsonEncoder = require 'KSJsonEncoder'

--------------------------------------------------------------------------------

local coveragePath

local derefOpt = KSTableUtils.derefOpt

local failStubbedTests = false

local kOzWaitMultiplier = 30
local kTimeoutForSendingUpdates = 5

--==============================================================================

local WFCatTrashMultipleDevicesBasicOtherRestoreTest = {}

local kApplyTintXmp = [[
	<x:xmpmeta xmlns:x='adobe:ns:meta/' x:xmptk='XMP toolkit 3.0-28, framework 1.6'>
	<rdf:RDF xmlns:rdf='http://www.w3.org/1999/02/22-rdf-syntax-ns#' xmlns:iX='http://ns.adobe.com/iX/1.0/'>
	 <rdf:Description rdf:about=''
	  xmlns:crs='http://ns.adobe.com/camera-raw-settings/1.0/'
		crs:Version='11.1'
		crs:ProcessVersion='11.0'
		crs:Temperature='50000'
		crs:Tint='+8'
		crs:Saturation='0'
	 </rdf:Description>
	</rdf:RDF>
	</x:xmpmeta>
	]]

--------------------------------------------------------------------------------

local function setTimeoutAndRetryLimit( rxTest, options, timeoutValue )

	rxTest:setTimeLimit( timeoutValue )

	if not options.mock then
		rxTest:setRetryLimit( 3, 30 )
	end

end

--==============================================================================
-- @mark tests that involve other client actions
--==============================================================================

local function testAddDeleteSyncOtherRestore( rxTest, options )

	options = options or {}

	local waitMultiplier = options.mock and 1 or kOzWaitMultiplier

	setTimeoutAndRetryLimit( rxTest, options, 15 * waitMultiplier )

	local testSession, catalog, catalogSpec, pusher, puller, uploader, pc = WFCatTrashTestUtils.setupCloudTrashTestSession( rxTest, options )
	assert( testSession )

	local userCreated = '2018-04-15T18:18:18Z'
	local importTimestamp = '2018-04-15T19:19:19Z'
	local assetCreated = '2018-04-15T21:21:21Z'
	local trashPurgeDate

	rxTest:expectSingleValue( catalog:ozUser():purgeAssetsAfterValue()
		:map( function( purgeAssetsAfter )
			trashPurgeDate = KSDate.shiftIso8601DateTime( assetCreated, purgeAssetsAfter )
			return true
		end ),
	true )

	local subtype = 'image'
	local fileName = "IMG_7763.JPG"
	local sha256 = "3bd8b936cc646bf1296e5ba4a76c410a27beb80809f525a8f5072f0115c54ccc"

	fileName = options.fileName or fileName
	subtype = options.subtype or subtype
	local trashAssetSubtype = 'deleted_' .. subtype

	local expectedUserCreated = userCreated

	local catalogId = catalog:ozCatalogId()
	local userId = catalog:ozUserId()
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId

	local tempTestFolder = WFTestFileUtils.createTestFolder( rxTest )
	local originalFilePath = WFCatTrashTestUtils.getOriginalFilePath( rxTest, tempTestFolder, fileName )

	-- First pull gets the album prefetch, catalog prefetch, and an empty pull
	catalogSpec:expectHttpRequest( WFCatTrashTestUtils.prefetchReqResp( catalogId ) )

	if options.mock then
		catalogSpec:expectHttpRequest( WFCatTrashTestUtils.catalogPrefetchReqResp( catalogId ) )
	end

	catalogSpec:expectHttpRequest( WFCatTrashTestUtils.emptyPullReqResp( catalogId ) )
	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	local importSource = WFCatTrashTestUtils.makeImportSource( catalog, fileName, importTimestamp, sha256 )

	-- Add an image asset
	rxTest:expectSingleValue( WFCatTrashTestUtils.addDocumentWithLocalMaster( catalog, catalogId, assetId, originalFilePath, subtype, userCreated, importSource )
		:map( function( revId )
			revisionId = revId
			return revId
		end ),
	':ignore:' )

	-- Delete the asset before syncing up anything related to it.
	rxTest:expectOnlyCompleted( WFCatTrash.moveAssetsToTrash( catalog, { assetId }, pc ) )

	-- Make sure a deleted_image was created with the correct original info and type/subtype
	local trashAssetId, trashRevId, trashDocValue = WFCatTrashTestUtils.getTrashAssetForDeletedAsset( rxTest, catalog, waitMultiplier, assetId )
	rxTest:expectNotEqual( assetId, trashAssetId ) -- should not have the original asset id
	rxTest:expectEqual( 'asset', trashDocValue.type )
	rxTest:expectEqual( trashAssetSubtype, trashDocValue.subtype )
	rxTest:expectEqual( assetId, derefOpt( trashDocValue, 'original', 'id' ) )
	rxTest:expectEqual( expectedUserCreated, derefOpt( trashDocValue, 'original', 'userCreated' )  )

	-- Sync up the deleted asset
	catalogSpec:expectHttpRequest( WFCatTrashTestUtils.pushCreateAssetReqResp( catalogId, fileName, trashAssetSubtype ) )

	rxTest:expectOnlyCompleted( pusher:_DEBUG_sendNUpdates( 1, kTimeoutForSendingUpdates ) )

	-- Get the trashed asset back in the sync feed with the links to upload the original
	rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 5 * waitMultiplier, catalogSpec, catalogId, puller, {
		WFCatTrashTestUtils.trashAssetUpdateResponse( trashAssetId, trashRevId, fileName, trashAssetSubtype, assetCreated, trashPurgeDate, userCreated, importSource, assetId, false, false ),
	}, WFCatTrashTestUtils.stripForAssetUpdate( false ) ), 'success' )

	-- Update the trashRevId with the latest revision id before issuing the WFCatTrashTestUtils.createMasterReqResp
	trashRevId = WFCatTrashTestUtils.getLatestRevisionId( rxTest, catalog, trashAssetId )

	-- Upload the master
	catalogSpec:expectHttpRequest( WFCatTrashTestUtils.createMasterReqResp( catalogId, trashAssetId, trashRevId, originalFilePath ) )
	uploader:startContinuousUploads()

	-- Wait for the upload master to complete, (it will upload after the xmp_develop if there is one.
	rxTest:observe( WFCatTrashTestUtils.pollUploadCount( uploader, 'master' ))
	:expectDataWithinTime( 5 * waitMultiplier, 1)
	:dispose()

	-- Wait for oz to generate the proxy and renditions before requesting next update
	rxTest:expectSingleValue( WFCatTrashTestUtils.pollUntilHasRequiredLinks( 5 * waitMultiplier, catalogSpec, catalogId, trashAssetId, trashRevId, trashAssetSubtype ), true )

	-- Get the metadata for the newly added asset that includes the links for uploading binaries.
	rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 5 * waitMultiplier, catalogSpec, catalogId, puller, {
		WFCatTrashTestUtils.trashAssetUpdateResponse( trashAssetId, trashRevId, fileName, trashAssetSubtype, assetCreated, trashPurgeDate, userCreated, importSource, assetId, true, true )
	}, WFCatTrashTestUtils.stripForAssetUpdate( true ) ), 'success' )

	-- Make sure the original asset was deleted
	rxTest:expectSingleValue( catalog:docStore():docValue( assetId )
		:pluck( '_deleted' ),
	true )

	-- Restore asset on other client
	local restoredAssetId = KSIdGenerator.uuidInLowercaseHex()
	local restoredAssetRevision = KSIdGenerator.uuidInLowercaseHex()

	WFCatTrashTestUtils.simulateOtherClientAssetRestore {
		rxTest = rxTest,
		catalog = catalog,
		catalogSpec = catalogSpec,
		assetId = restoredAssetId,
		revisionId = restoredAssetRevision,
		trashAssetId = trashAssetId,
		trashAssetRevId = trashRevId,
		userUpdated = userCreated,
		options = options,
	}

	if options.splitResponseTombstoneFirst then

		rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 5 * waitMultiplier, catalogSpec, catalogId, puller, {
			WFCatTrashTestUtils.assetDeletedResponse( trashAssetId, restoredAssetId, userId ),
		} ), 'success' )

		rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 5 * waitMultiplier, catalogSpec, catalogId, puller, {
			WFCatTrashTestUtils.restoredAssetUpdateResponse( restoredAssetId, restoredAssetRevision, fileName, subtype, assetCreated, userCreated, importSource, assetId, true, true ),
		} ), 'success' )

		-- Wait the conflict to be resolved by conflict resolver
		rxTest:observe( catalog:docStore():ongoingDocValue( restoredAssetId ):pluck( '_conflict' ) )
		:expectDataWithinTime( 5 * waitMultiplier, nil )
		:dispose()

	elseif options.splitResponseUpdateFirst then

		rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 5 * waitMultiplier, catalogSpec, catalogId, puller, {
			WFCatTrashTestUtils.restoredAssetUpdateResponse( restoredAssetId, restoredAssetRevision, fileName, subtype, assetCreated, userCreated, importSource, assetId, true, true ),
		} ), 'success' )

		rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 5 * waitMultiplier, catalogSpec, catalogId, puller, {
			WFCatTrashTestUtils.assetDeletedResponse( trashAssetId, restoredAssetId, userId ),
		} ), 'success' )

	else

		rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 5 * waitMultiplier, catalogSpec, catalogId, puller, {
			WFCatTrashTestUtils.restoredAssetUpdateResponse( restoredAssetId, restoredAssetRevision, fileName, subtype, assetCreated, userCreated, importSource, assetId, true, true ),
			WFCatTrashTestUtils.assetDeletedResponse( trashAssetId, restoredAssetId, userId ),
		} ), 'success' )

	end

	-- Check that trash asset is now deleted.
	rxTest:expectSingleValue( catalog:docStore():docValue( trashAssetId ):pluck( '_deleted' ), true )

	-- Check that there is no conflict.
	rxTest:expectSingleValue( catalog:docStore():docValue( restoredAssetId ):pluck( '_conflict' ), nil )

	-- Make sure the restored asset has a local master
	rxTest:expectSingleValue( catalog:docStore():docValue( restoredAssetId ):map( LrAssetUtils.hasMasterLocally ), true )

	-- check that we think the current revision exists on oz to ensure we don't have a stuck sync
	rxTest:expectSingleValue( catalog:docStore():docValue( restoredAssetId ):map( LrAssetUtils.isLatestMetadataUploaded ), true )

	-- check that there is nothing to push
	rxTest:expectOnlyCompleted( pusher:_DEBUG_sendNUpdates( 0, kTimeoutForSendingUpdates ) )

end

--------------------------------------------------------------------------------

function WFCatTrashMultipleDevicesBasicOtherRestoreTest.rxTest_CT_MockOtherRestore( rxTest )

-- Test that it properly syncs restored trash assets that were restored by
-- another client and that it can download thumbnails.

	testAddDeleteSyncOtherRestore( rxTest, { mock = true } )

end

--------------------------------------------------------------------------------

function WFCatTrashMultipleDevicesBasicOtherRestoreTest.rxTest_CT_MockOtherRestore_splitResponse_tombstone( rxTest )

-- Test that it properly syncs restored trash assets that were restored by
-- another client and that it can download thumbnails.

	testAddDeleteSyncOtherRestore( rxTest, { mock = true, splitResponseTombstoneFirst = true } )

end

--------------------------------------------------------------------------------

function WFCatTrashMultipleDevicesBasicOtherRestoreTest.rxTest_CT_MockOtherRestore_splitResponse_update( rxTest )

-- Test that it properly syncs restored trash assets that were restored by
-- another client and that it can download thumbnails.

	testAddDeleteSyncOtherRestore( rxTest, { mock = true, splitResponseUpdateFirst = true } )

end

--------------------------------------------------------------------------------

function WFCatTrashMultipleDevicesBasicOtherRestoreTest.rxTest_CT_OzOtherRestore_testNightly( rxTest )

-- Test that it properly syncs restored trash assets that were restored by
-- another client and that it can download thumbnails.

	testAddDeleteSyncOtherRestore( rxTest )

end

--------------------------------------------------------------------------------

local function testAddSyncRestoreOtherRestoreSync( rxTest, options )

-- NMB-10313
-- This tests the two client restore scenario where two clients restore the same asset.
-- One of these will 'win' and get to Oz first and it's up to the other client to ensure
-- that it handles the error messages from Oz appropriately, ensuring that no changes from
-- its local restore are lost even if the actual asset restored locally is removed in favor
-- of the restored asset from the other client.

	options = options or {}

	local waitMultiplier = options.mock and 1 or kOzWaitMultiplier

	setTimeoutAndRetryLimit( rxTest, options, 15 * waitMultiplier )

	local testSession, catalog, catalogSpec, pusher, puller, uploader, pc, errorHandler = WFCatTrashTestUtils.setupCloudTrashTestSession( rxTest, options )
	assert( testSession )

	local userCreated = '2018-04-15T18:18:18Z'
	local importTimestamp = '2018-04-15T19:19:19Z'
	local assetCreated = '2018-04-15T20:20:20Z'

	local subtype = 'image'
	local fileName = "IMG_7763.JPG"
	local sha256 = "3bd8b936cc646bf1296e5ba4a76c410a27beb80809f525a8f5072f0115c54ccc"
	local proxyFileName = "IMG_7763.DNG"

	fileName = options.fileName or fileName
	subtype = options.subtype or subtype
	local trashAssetSubtype = 'deleted_' .. subtype

	local catalogId = catalog:ozCatalogId()
	local userId = catalog:ozUserId()
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId
	local albumId = KSIdGenerator.uuidInLowercaseHex()
	local albumAssetId = KSIdGenerator.uuidInLowercaseHex()
	local albumAssetRevId
	local spaceId = KSIdGenerator.uuidInLowercaseHex()
	local spaceAlbumId = KSIdGenerator.uuidInLowercaseHex()
	local commentId = KSIdGenerator.uuidInLowercaseHex()
	local commentRevId
	local favoriteId = KSIdGenerator.uuidInLowercaseHex()
	local favoriteRevId

	-- First pull gets the album prefetch, catalog prefetch, and an empty pull
	catalogSpec:expectHttpRequest( WFCatTrashTestUtils.prefetchReqResp( catalogId ) )

	if options.mock then
		catalogSpec:expectHttpRequest( WFCatTrashTestUtils.catalogPrefetchReqResp( catalogId ) )
	end

	catalogSpec:expectHttpRequest( WFCatTrashTestUtils.emptyPullReqResp( catalogId ) )
	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	local importSource = WFCatTrashTestUtils.makeImportSource( catalog, fileName, importTimestamp, sha256 )

-- Step 1: Add an image asset
	local tempTestFolder = WFTestFileUtils.createTestFolder( rxTest )

	local filePath, proxyFilePath
	if options.proxyOnlyAsset then
		proxyFilePath = WFCatTrashTestUtils.getProxyFilePath( rxTest, tempTestFolder, proxyFileName )

		rxTest:expectSingleValue( WFCatTrashTestUtils.addDocumentWithProxyOnly( catalog, catalogId, assetId, proxyFilePath, subtype, userCreated, importSource )
			:tap( function( revId )
				revisionId = revId
			end ),
			':ignore:' )

	else
		filePath = WFCatTrashTestUtils.getOriginalFilePath( rxTest, tempTestFolder, fileName )
		proxyFilePath = WFCatTrashTestUtils.getProxyFilePath( rxTest, tempTestFolder, proxyFileName )

		rxTest:expectSingleValue( WFCatTrashTestUtils.addDocumentWithLocalMasterAndProxy( catalog, catalogId, assetId, filePath, proxyFilePath, subtype, userCreated, importSource )
			:tap( function( revId )
				revisionId = revId
			end ),
		':ignore:' )
	end

	-- these are the /rels/* links we currently expect in the received document
	local linksExpectedInDoc = { master = false, proxy = false, renditions = false }

	-- Send the metadata for the newly added asset
	catalogSpec:expectHttpRequest( WFCatTrashTestUtils.pushCreateAssetReqResp( catalogId, fileName, subtype ) )
	rxTest:expectOnlyCompleted( pusher:_DEBUG_sendNUpdates( 1, kTimeoutForSendingUpdates ) )

	-- Get the metadata for the newly added asset that includes the links for uploading binaries.
	rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 5 * waitMultiplier, catalogSpec, catalogId, puller, {
		WFCatTrashTestUtils.assetUpdateResponse( assetId, revisionId, fileName, subtype, assetCreated, userCreated, importSource, linksExpectedInDoc )
	}, WFCatTrashTestUtils.stripForAssetUpdate( false ) ), 'success' )

	if options.includeAssociatedDocs then
		-- add any associated docs
		-- album
		local albumRevId
		local albumName = 'Album1'
		rxTest:expectSingleValue( catalog:docStore():applyOverlay {
				_id = albumId,
				type = 'album',
				subtype = 'collection',
				name = albumName,
			}
			:map( function( revId )
				albumRevId = revId
				return revId
			end )
		, ':ignore:' )

		-- sync up the album
		catalogSpec:expectHttpRequest( WFCatTrashTestUtils.pushCreateAlbumReqResp( catalogId, albumId, albumName ) )
		rxTest:expectOnlyCompleted( pusher:_DEBUG_sendNUpdates( 1, kTimeoutForSendingUpdates ) )

		-- album_asset
		rxTest:expectSingleValue( catalog:docStore():applyOverlay {
				_id = albumAssetId,
				type = 'album_asset',
				album = { id = albumId },
				asset = { id = assetId }
			}
			:map( function( revId )
				albumAssetRevId = revId
				return revId
			end )
		, ':ignore:' )

		-- Sync up the album_asset
		catalogSpec:expectHttpRequest( WFCatTrashTestUtils.pushCreateAlbumAssetReqResp( catalogId, albumAssetId, albumId, assetId ) )
		rxTest:expectOnlyCompleted( pusher:_DEBUG_sendNUpdates( 1, kTimeoutForSendingUpdates ) )

		-- Sync down the album and album_asset
		rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 10 * waitMultiplier, catalogSpec, catalogId, puller, {
			WFCatTrashTestUtils.albumAssetUpdateResponse( albumAssetId, albumAssetRevId, albumId, assetId ),
			WFCatTrashTestUtils.albumUpdateResponse( catalogId, albumId, albumRevId, albumName )
		} ), 'success' )

		-- This first part of this step is contrived as we aren't the other client. But
		-- we want to add the space_album, and comment documents here so when we run
		-- this test against oz it will have the documents as well and move them around
		-- during the delete and restore process.
		local now = KSDate.timeInIso8601Format()

		local spaceAlbumRevId
		rxTest:expectSingleValue( catalog:docStore():bulkApplyOverlay(
			{
				label = 'oz',
				{
					_id = spaceId,
					userCreated = now,
					userUpdated = now,
					type = 'space',
					subtype = 'shared',
					private = false,
					download = false,
					metadata = false,
					location = false,
					presentationFilters = true,
				},
				{
					_id = spaceAlbumId,
					type = 'space_album',
					space = { id = spaceId },
					album = { id = albumId },
				}
			} ) :map( function( overlayResults )
					spaceAlbumRevId = derefOpt( overlayResults, 2, 'revId' )
					return overlayResults
				end )
		, ':ignore:' )

		-- Now create the space via a direct put as MetadataPusher doesn't handles 'space' docs.
		local createSpaceUrl = string.format( '%s/spaces/%s', catalogSpec:catalogHref(), spaceId )
		local createSpaceOptions = {
			requestMethod = 'PUT',
			requestHeaders = {
				{ header = 'Content-Type', value = 'application/json' },
			},
		}
		local createSpaceData = KSJsonEncoder.encodeOzUploadJson( {
			payload = {
				private = false,
				download = false,
				metadata = false,
				location = false,
				presentationFilters = true,
			},
			subtype = 'shared',
			resources = {
				{
					id = albumId,
					type = 'album',
					space_album_id = spaceAlbumId,
				}
			}
		} )

		catalogSpec:expectHttpRequest( WFCatTrashTestUtils.putCreateSpaceReqResp( createSpaceUrl, createSpaceData, createSpaceOptions ) )
		rxTest:expectSingleValue(
			catalogSpec:put( createSpaceUrl, createSpaceData, createSpaceOptions )
			:map( function( responseData, headers )
				return headers.status_code
			end )
		, '201' )

		rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 5 * waitMultiplier, catalogSpec, catalogId, puller, {
			WFCatTrashTestUtils.spaceAlbumUpdateResponse( catalogId, spaceAlbumId, spaceAlbumRevId, spaceId, albumId ),
			WFCatTrashTestUtils.spaceUpdateResponse( catalogId, spaceId )
		}), 'success' )

	end

	revisionId = WFCatTrashTestUtils.getLatestRevisionId( rxTest, catalog, assetId )

	-- push and validate for the binary upload
	local expectedOriginalUploadCount = 0
	local expectedProxyUploadCount = 0
	if options.proxyOnlyAsset then
		expectedProxyUploadCount = expectedProxyUploadCount + 1

		-- Upload the proxy (and generate renditions)
		catalogSpec:expectHttpRequest( WFCatTrashTestUtils.createProxyReqResp( catalogId, assetId, revisionId, KSIdGenerator.uuidInLowercaseHex(), proxyFilePath ) )
		uploader:startContinuousUploads()

		-- wait for the uploader to issue the create proxy request
		rxTest:observe( WFCatTrashTestUtils.pollUploadCount( uploader, 'proxy' ))
		:expectDataWithinTime( 5 * waitMultiplier, expectedProxyUploadCount )
		:dispose()
	else
		expectedOriginalUploadCount = expectedOriginalUploadCount + 1

		-- Upload the master
		catalogSpec:expectHttpRequest( WFCatTrashTestUtils.createMasterReqResp( catalogId, assetId, revisionId, filePath ) )
		uploader:startContinuousUploads()

		-- Wait for the uploader to issue the create master request
		rxTest:observe( WFCatTrashTestUtils.pollUploadCount( uploader, 'master' ))
		:expectDataWithinTime( 5 * waitMultiplier, expectedOriginalUploadCount )
		:dispose()
	end
	uploader:stopContinuousUploads()

	-- Wait for oz to generate the renditions before requesting next update
	rxTest:expectSingleValue( WFCatTrashTestUtils.pollUntilHasRequiredLinks( 10 * waitMultiplier, catalogSpec, catalogId, assetId, revisionId, subtype, { proxyOnlyAsset = true } ), true )

	-- Get the updated metadata with the binary link.
	rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 5 * waitMultiplier, catalogSpec, catalogId, puller, {
		WFCatTrashTestUtils.assetUpdateResponse( assetId, revisionId, fileName, subtype, assetCreated, userCreated, importSource, { master = not options.proxyOnlyAsset, proxy = options.proxyOnlyAsset, renditions = true }, nil, true )
	}, WFCatTrashTestUtils.stripForAssetUpdate( true ) ), 'success' )

	-- Wait for the update to get applied to the assets docValue
	local binaryRelsKey = options.proxyOnlyAsset and '/rels/proxy_type/2560' or '/rels/master'
	rxTest:observe( catalog:docStore():ongoingDocValue( assetId )
		:pluck( '_serverProvidedMetadata', 'links', binaryRelsKey, 'href' )
		:map( type ))
	:expectDataWithinTime( 5 * waitMultiplier, 'string' )
	:dispose()


-- Step 2: Maybe remove some of the local binaries for the asset to test different restore scenarios
	if options.dropLocalMaster then
		rxTest:expectSingleValue( catalog:docStore():applyOverlay(
			WFCatAssetTestUtils.getOverlayToRemoveLocalOriginal( assetId ) ),
			':ignore:' )
	end

	if options.dropLocalProxy then
		rxTest:expectSingleValue( catalog:docStore():applyOverlay(
			WFCatAssetTestUtils.getOverlayToRemoveLocalProxy( assetId ) ),
			':ignore:' )
	end

-- Step 3: Move the asset to trash locally and send that to Oz

	-- Delete the asset which is on Oz
	rxTest:expectOnlyCompleted( WFCatTrash.moveAssetsToTrash( catalog, { assetId }, pc ) )

	-- Make sure the original asset was deleted, and get it's new revision id
	rxTest:expectSingleValue( catalog:docStore():docValue( assetId )
	:tap( function( docValue )
		revisionId = docValue._revId
	end )
	:pluck( '_deleted' )
	, true )

	-- Make sure a deleted_image was created
	local trashAssetId, trashRevId = WFCatTrashTestUtils.getTrashAssetForDeletedAsset( rxTest, catalog, waitMultiplier, assetId )
	rxTest:expectEqual( 'string', type( trashAssetId ) )
	rxTest:expectEqual( 'string', type( trashRevId ) )

	local trashUserCreated, trashPurgeDate
	rxTest:expectSingleValue( catalog:docStore():docValue( trashAssetId )
		:tap( function( docValue )
			trashUserCreated = docValue.userCreated
			trashPurgeDate = docValue.purgeDate
		end )
	, ':ignore:' )

	-- Verify we only send up the delete, the trash asset should not be sent if the original asset was already synced up to oz
	catalogSpec:expectHttpRequest( WFCatTrashTestUtils.pushDeleteAssetReqResp( catalogId, assetId, revisionId, trashAssetId, trashRevId ) )
	rxTest:expectOnlyCompleted( pusher:_DEBUG_sendNUpdates( 1, kTimeoutForSendingUpdates ) )

	local waitForRenditions = not options.noWaitForRenditions

	-- Get the trashed asset back in the sync feed with the links to necessary binaries
	local expectedUpdates = {
		WFCatTrashTestUtils.trashAssetUpdateResponse2( trashAssetId, trashRevId, fileName, trashAssetSubtype, trashUserCreated, trashPurgeDate, userCreated, importSource, assetId, { master = true, renditions = waitForRenditions, xmpDevelop = options.xmpDevelop } ),
		WFCatTrashTestUtils.assetDeletedResponse( assetId, trashAssetId, userId ),
	}

	if options.includeAssociatedDocs then
		expectedUpdates[ #expectedUpdates + 1 ] = WFCatTrashTestUtils.albumAssetUpdateResponse( albumAssetId, albumAssetRevId, albumId, assetId ) -- new unknown id and revId will be created by oz
		expectedUpdates[ #expectedUpdates + 1 ] = WFCatTrashTestUtils.albumAssetDeletedResponse( albumAssetId, userId, KSIdGenerator.uuidInLowercaseHex() )
	end

	rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 5 * waitMultiplier, catalogSpec, catalogId, puller, expectedUpdates, WFCatTrashTestUtils.transformResponseWithUnknownNewIds ), 'success' )

-- Other Client ------------------------------------------------------------

-- Step 4: On a second client, restore the asset from the trash on that second client
-- so that it's a new asset on Oz

	-- Send an update with a restore trash asset without our docstore knowing about it
	-- so it appears as if a different client issued the restore.
	local otherRestoredAssetId = KSIdGenerator.uuidInLowercaseHex()
	local otherRestoredRevId = KSIdGenerator.uuidInLowercaseHex()

	WFCatTrashTestUtils.simulateOtherClientAssetRestore {
		rxTest = rxTest,
		catalog = catalog,
		catalogSpec = catalogSpec,
		assetId = otherRestoredAssetId,
		revisionId = otherRestoredRevId,
		trashAssetId = trashAssetId,
		trashAssetRevId = trashRevId,
		userUpdated = trashUserCreated,
		options = options,
	}

	rxTest:expectSingleValue( WFCatTrashTestUtils.pollUntilDeleted( 10 * waitMultiplier, catalogSpec, catalogId, trashAssetId, trashRevId, options ), true )

-- Original Client ------------------------------------------------------------

-- Step 5: Attempt to restore the asset locally

	-- Restore the trash asset
	rxTest:expectOnlyCompleted( WFCatTrash.restoreTrashAssets( catalog, { trashAssetId }, pc ) )

	-- Make sure the trash asset was deleted
	rxTest:expectSingleValue( catalog:docStore():docValue( trashAssetId )
		:pluck( '_deleted' ),
	true )

	local restoredAssetId, restoredRevId, restoredDocValue = WFCatTrashTestUtils.getNewlyCreatedAsset( rxTest, catalog, waitMultiplier, { assetId, trashAssetId } )
	rxTest:expectEqual( 'asset', restoredDocValue.type )
	rxTest:expectEqual( subtype, restoredDocValue.subtype )
	rxTest:expectEqual( nil, derefOpt( restoredDocValue, 'original' ) )

	-- Make sure the last trash userCreated and purgeDate data is correctly stored in restored asset createFrom field
	rxTest:expectSingleValue( catalog:docStore():docValue( restoredAssetId )
		:map( function( docValue )
			local createFromInfo = WFCatTrashUtils.createFromInfo( docValue )
			return createFromInfo.lastTrashUserCreated == trashUserCreated and createFromInfo.lastTrashPurgeDate == trashPurgeDate
		end )
	, true )

-- Step 6 (optional): Add a comment/favorite to the asset to ensure these get transferred to the Oz restored asset
	local commentBody = "Comment1 Mumble, mumble, mumble"
	if options.includeAssociatedDocs then
		-- Add comment
		rxTest:expectSingleValue( catalog:docStore():applyOverlay {
				type = 'comment',
				subtype = 'user',
				_id = commentId,
				body = commentBody,
				space = { id = spaceId },
				asset = { id = restoredAssetId },
				_localOnly = { created = '2018-12-15T21:21:21Z' },
			}
			:map( function( revId )
				commentRevId = revId
				return revId
			end )
		, ':ignore:' )

		-- Add favorite
		rxTest:expectSingleValue( catalog:docStore():applyOverlay {
				type = 'favorite',
				subtype = 'user',
				_id = favoriteId,
				space = { id = spaceId },
				asset = { id = restoredAssetId },
				_localOnly = { created = '2018-12-15T21:21:21Z' },
			}
			:map( function( revId )
				favoriteRevId = revId
				return revId
			end )
		, ':ignore:' )

	end

-- Step 7 (optional): Modify the restored asset locally

	local developSettings = {
		processingModel = "lightroom",
		xmpCameraRaw = kApplyTintXmp
	}

	local userUpdated = restoredDocValue.userUpdated

	if options.modifyAfterRestore then
		userUpdated = KSDate.timeInIso8601Format()
		rxTest:expectSingleValue( catalog:docStore():applyOverlay {
				_id = restoredAssetId,
				develop = developSettings,
				userUpdated = userUpdated,
			}
		, ':ignore:' )
	end

	-- Send create restored asset from original assets
	if options.includeAssociatedDocs then

		catalogSpec:expectHttpRequest( WFCatTrashTestUtils.pushRestoreAssetAndCommentFavoriteDocsReqResp( catalogId, restoredAssetId, restoredRevId, trashAssetId, trashRevId,
										spaceId, commentId, commentBody, favoriteId ) )
		rxTest:expectOnlyCompleted( pusher:_DEBUG_sendNUpdates( 3, kTimeoutForSendingUpdates ) )

	else

		catalogSpec:expectHttpRequest( WFCatTrashTestUtils.pushRestoreAsset2ReqResp( catalogId, restoredAssetId, restoredRevId, trashAssetId, trashRevId, userUpdated ) )
		rxTest:expectOnlyCompleted( pusher:_DEBUG_sendNUpdates( 1, kTimeoutForSendingUpdates ) )

	end

	-- Make sure the trash asset was deleted locally
	rxTest:expectSingleValue( catalog:docStore():docValue( trashAssetId )
		:pluck( '_deleted' ),
	true )

	expectedUpdates = {
			WFCatTrashTestUtils.trashAssetDeletedResponse( trashAssetId, userId, otherRestoredAssetId ),
			WFCatTrashTestUtils.restoredAssetUpdateResponse( otherRestoredAssetId, otherRestoredRevId, fileName, subtype, assetCreated, userCreated, importSource, assetId, true, waitForRenditions ),
			WFCatTrashTestUtils.resourceNotFoundErrorUpdateResponse( "create", restoredAssetId, restoredRevId, "asset", trashAssetId, "deleted_asset", otherRestoredAssetId, userId ),
		}

	if options.includeAssociatedDocs then
		-- unknown id will be created by Oz and stripped when we receive the response here
		expectedUpdates[ #expectedUpdates + 1 ] = WFCatTrashTestUtils.albumAssetUpdateResponse( albumAssetId, albumAssetRevId, albumId, assetId )
		expectedUpdates[ #expectedUpdates + 1 ] = WFCatTrashTestUtils.albumAssetDeletedResponse( albumAssetId, userId, KSIdGenerator.uuidInLowercaseHex() )
	end

	rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 10 * waitMultiplier, catalogSpec, catalogId, puller, expectedUpdates, WFCatTrashTestUtils.transformResponseWithUnknownNewIds ), 'success' )

-- Step 8: The error handler should process the RNFE and turn the restore into a delete of the locally restored asset (which will not sync to Oz) and, if applicable,
-- an update of the oz restored asset
	-- check that the locally restored asset is deleted
	rxTest:observe( catalog:docStore():ongoingDocValue( restoredAssetId ):pluck( '_empty' ) )
		:expectDataWithinTime( 2 * waitMultiplier, true )

	-- any binaries that were on the locally restored asset should now be on the restored asset from Oz
	if not ( options.dropLocalMaster and options.dropLocalProxy ) then
		-- Make sure the restored asset has the expected set of local files
		rxTest:expectSingleValue( catalog:docStore():docValue( otherRestoredAssetId )
			:map( function( docValue )
				local hasBinary = 'unknown'
				if options.proxyOnlyAsset or options.dropLocalMaster then
					local proxyPath = WFCatAssetUtils.getFullProxyPathFromDocValue( docValue )
					hasBinary = ( proxyPath and WFFileUtils.exists( proxyPath ) == 'file' ) and 'hasProxy' or 'no proxy'
				else
					local originalPath = WFCatAssetUtils.getFullOriginalPathFromDocValue( docValue )
					hasBinary = ( originalPath and WFFileUtils.exists( originalPath ) == 'file' ) and 'hasOriginal' or 'no original'
				end

				return hasBinary
			end ),
		(options.proxyOnlyAsset or options.dropLocalMaster) and 'hasProxy' or 'hasOriginal' )

	end

	if options.modifyAfterRestore then

		-- check that the restored asset has the changes from the locally restored asset that we expect
		catalogSpec:expectHttpRequest( WFCatTrashTestUtils.pushUpdateRestoredReqResp( catalogId, otherRestoredAssetId, otherRestoredRevId, restoredDocValue.subtype, false, developSettings.xmpCameraRaw ) )
		rxTest:expectOnlyCompleted( pusher:_DEBUG_sendNUpdates( 1, kTimeoutForSendingUpdates * 3 ) )


		-- pull from Oz to get the asset update response
		rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 5 * waitMultiplier, catalogSpec, catalogId, puller, {
				WFCatTrashTestUtils.restoredAssetUpdateResponse( otherRestoredAssetId, otherRestoredRevId, fileName, subtype, assetCreated, userCreated, importSource, assetId, true, true, false, true, developSettings ),
			}, WFCatTrashTestUtils.transformOrderAndStripAssetUpdate( false )), 'success' )

	elseif not options.includeAssociatedDocs then

		-- The error handler is updated the otherRestoredAssetId with info from our restored asset, so send it up in case there are changes.
		catalogSpec:expectHttpRequest( WFCatTrashTestUtils.pushUpdateRestoredReqResp( catalogId, otherRestoredAssetId, otherRestoredRevId, restoredDocValue.subtype, false ) )
		rxTest:expectOnlyCompleted( pusher:_DEBUG_sendNUpdates( 1, kTimeoutForSendingUpdates * 3 ) )

		-- pull from Oz to get the asset update response
		rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 5 * waitMultiplier, catalogSpec, catalogId, puller, {
				WFCatTrashTestUtils.restoredAssetUpdateResponse( otherRestoredAssetId, otherRestoredRevId, fileName, subtype, assetCreated, userCreated, importSource, assetId, true, true, false ),
			}, WFCatTrashTestUtils.transformOrderAndStripAssetUpdate( false )), 'success' )

	end

	if options.includeAssociatedDocs then

		-- check that we send the comment and favorite for the oz restored asset id
		local updateRequest = WFMockHttpRequest()
			:post( "/v2/catalogs/" .. catalogId .."/updates" )
			:expectRequestDataAfterTransformation( 'success', function( str )

				local updatesBeingPushed = KSJsonDecoder.decodeWebSafeJson_stripNull_ignoreEmpty( str )

				local commentUpdate, favoriteUpdate
				for _, update in ipairs( updatesBeingPushed ) do
					if update.type == 'comment' then
						commentUpdate = update
					elseif update.type == 'favorite' then
						favoriteUpdate = update
					end
				end

				local correctCommentUpdate = WFCatTrashTestUtils.hasCorrectCommentUpdate( commentUpdate, commentId, spaceId, otherRestoredAssetId, commentBody )
				local correctFavoriteUpdate = WFCatTrashTestUtils.hasCorrectFavoriteUpdate( favoriteUpdate, favoriteId, spaceId, otherRestoredAssetId )

				if correctCommentUpdate and correctFavoriteUpdate then
					return 'success'
				end

				return str

			end )
			:respond( '202', WFCatTrashTestUtils.futureUpdatesResponse( catalogId ) )

		catalogSpec:expectHttpRequest( updateRequest )
		rxTest:expectOnlyCompleted( pusher:_DEBUG_sendNUpdates( 3, kTimeoutForSendingUpdates * 3 ) )

	end

	-- Check that there is nothing to send
	rxTest:expectOnlyCompleted( pusher:_DEBUG_sendNUpdates( 0, kTimeoutForSendingUpdates ) )

	-- Check to see how the ResourceNotFoundErrors were resolved
	rxTest:expectEqual( {
		ResourceNotFoundError = {
			count = 1,
			resolution = {
				CreatedOverlaysToMergeTheRestoredAssetsDeletingLocal_LocalRestore = {
					count = 1,
					sourceDocIds = {
						restoredAssetId,
					},
				},
			},
		},
	}, errorHandler:DEBUG_errorsResolved() )

end

--------------------------------------------------------------------------------

function WFCatTrashMultipleDevicesBasicOtherRestoreTest.rxTest_CT_MockRestore_OtherRestore( rxTest )
-- NMB-10313
-- If another client beats us to a restore, we want to resolve the two
-- restored assets such that any local changes we have are moved to the
-- restored asset that Oz accepted.
	testAddSyncRestoreOtherRestoreSync( rxTest, { mock = true } )
end

--------------------------------------------------------------------------------

function WFCatTrashMultipleDevicesBasicOtherRestoreTest.rxTest_CT_MockRestore_OtherRestore_NoLocalMaster( rxTest )
-- NMB-10313
-- If another client beats us to a restore, we want to resolve the two
-- restored assets such that any local changes we have are moved to the
-- restored asset that Oz accepted.
	testAddSyncRestoreOtherRestoreSync( rxTest, { mock = true, dropLocalMaster = true } )
end

--------------------------------------------------------------------------------

function WFCatTrashMultipleDevicesBasicOtherRestoreTest.rxTest_CT_MockRestore_OtherRestore_NoLocalBinaries( rxTest )
-- NMB-10313
-- If another client beats us to a restore, we want to resolve the two
-- restored assets such that any local changes we have are moved to the
-- restored asset that Oz accepted.
	testAddSyncRestoreOtherRestoreSync( rxTest, { mock = true, dropLocalMaster = true, dropLocalProxy = true } )
end


--------------------------------------------------------------------------------

function WFCatTrashMultipleDevicesBasicOtherRestoreTest.rxTest_CT_MockRestore_OtherRestore_assocDocs( rxTest )
-- NMB-10313
-- If another client beats us to a restore, we want to resolve the two
-- restored assets such that any local changes we have are moved to the
-- restored asset that Oz accepted.
	testAddSyncRestoreOtherRestoreSync( rxTest, { mock = true, includeAssociatedDocs = true } )
end


--------------------------------------------------------------------------------

function WFCatTrashMultipleDevicesBasicOtherRestoreTest.rxTest_CT_Restore_OtherRestore_assocDocs_testNightly( rxTest )
-- NMB-10313
-- If another client beats us to a restore, we want to resolve the two
-- restored assets such that any local changes we have are moved to the
-- restored asset that Oz accepted.
	testAddSyncRestoreOtherRestoreSync( rxTest, { includeAssociatedDocs = true } )
end

--------------------------------------------------------------------------------

function WFCatTrashMultipleDevicesBasicOtherRestoreTest.rxTest_CT_Restore_OtherRestore_testNightly( rxTest )
-- NMB-10313
-- If we had the original locally then it could potentially create a new asset
-- This should follow the same rules RestoreOtherDelTrash so there would be
-- May want to add the _Master, _NoMaster, _NoProxy versions as well.
	testAddSyncRestoreOtherRestoreSync( rxTest )
end

--------------------------------------------------------------------------------

function WFCatTrashMultipleDevicesBasicOtherRestoreTest.rxTest_CT_MockRestoreModify_OtherRestore( rxTest )
-- NMB-10313
-- If we had the original locally then it could potentially create a new asset
-- This should follow the same rules RestoreOtherDelTrash so there would be
-- May want to add the _Master, _NoMaster, _NoProxy versions as well.
	testAddSyncRestoreOtherRestoreSync( rxTest, { mock = true, modifyAfterRestore = true } )
end

--------------------------------------------------------------------------------

function WFCatTrashMultipleDevicesBasicOtherRestoreTest.rxTest_CT_RestoreModify_OtherRestore_testNightly( rxTest )
-- NMB-10313
-- If we had the original locally then it could potentially create a new asset
-- This should follow the same rules RestoreOtherDelTrash so there would be
-- May want to add the _Master, _NoMaster, _NoProxy versions as well.
	testAddSyncRestoreOtherRestoreSync( rxTest, { modifyAfterRestore = true } )
end

--------------------------------------------------------------------------------

function WFCatTrashMultipleDevicesBasicOtherRestoreTest.rxTest_CT_DelTrash_OtherRestore( rxTest )
-- This test should probably just fail with a message saying the asset was purged
-- If we had the original locally then it could potentially create a new asset
	rxTest:expectEqual( false, failStubbedTests )
end

--------------------------------------------------------------------------------

function WFCatTrashMultipleDevicesBasicOtherRestoreTest.rxTest_CT_DelTrash_OtherDelTrash( rxTest )
-- This test should probably just fail with a message saying the asset was purged
-- If we had the original locally then it could potentially create a new asset
	rxTest:expectEqual( false, failStubbedTests )
end

--------------------------------------------------------------------------------

local function testAddDeleteSyncRestoreDeleteOtherRestoreSync( rxTest, options )

-- NMB-13896 #1 / NMB-14702
-- Scenario:
-- Steps:
--		1) Add photo.
--		2) Delete the photo.
--		3) Sync, Pause
--      4) Other Client Restore
-- 		5) Restore and Delete the photo.
--		6) Resume Sync
--
-- Expected: Photo in Trash
--   Should get two ResourceNotFoundError, the first of which should update the
--   trash createFrom info to restore from the oz asset and the second should be
--   ignored.

	options = options or {}

	local waitMultiplier = options.mock and 1 or kOzWaitMultiplier

	setTimeoutAndRetryLimit( rxTest, options, 15 * waitMultiplier )

	local testSession, catalog, catalogSpec, pusher, puller, uploader, pc, errorHandler = WFCatTrashTestUtils.setupCloudTrashTestSession( rxTest, options )
	assert( testSession )

	local userCreated = '2018-04-15T18:18:18Z'
	local importTimestamp = '2018-04-15T19:19:19Z'
	local assetCreated = '2018-04-15T20:20:20Z'

	local subtype = 'image'
	local fileName = "IMG_7763.JPG"
	local sha256 = "3bd8b936cc646bf1296e5ba4a76c410a27beb80809f525a8f5072f0115c54ccc"
	local proxyFileName = "IMG_7763.DNG"

	fileName = options.fileName or fileName
	subtype = options.subtype or subtype
	local trashAssetSubtype = 'deleted_' .. subtype

	local catalogId = catalog:ozCatalogId()
	local userId = catalog:ozUserId()
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId

	-- First pull gets the album prefetch, catalog prefetch, and an empty pull
	catalogSpec:expectHttpRequest( WFCatTrashTestUtils.prefetchReqResp( catalogId ) )

	if options.mock then
		catalogSpec:expectHttpRequest( WFCatTrashTestUtils.catalogPrefetchReqResp( catalogId ) )
	end

	catalogSpec:expectHttpRequest( WFCatTrashTestUtils.emptyPullReqResp( catalogId ) )
	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	local importSource = WFCatTrashTestUtils.makeImportSource( catalog, fileName, importTimestamp, sha256 )

-- Step 1: Add photo
	local tempTestFolder = WFTestFileUtils.createTestFolder( rxTest )

	local filePath, proxyFilePath
	if options.proxyOnlyAsset then
		proxyFilePath = WFCatTrashTestUtils.getProxyFilePath( rxTest, tempTestFolder, proxyFileName )

		rxTest:expectSingleValue( WFCatTrashTestUtils.addDocumentWithProxyOnly( catalog, catalogId, assetId, proxyFilePath, subtype, userCreated, importSource )
			:tap( function( revId )
				revisionId = revId
			end ),
			':ignore:' )

	else
		filePath = WFCatTrashTestUtils.getOriginalFilePath( rxTest, tempTestFolder, fileName )
		proxyFilePath = WFCatTrashTestUtils.getProxyFilePath( rxTest, tempTestFolder, proxyFileName )

		rxTest:expectSingleValue( WFCatTrashTestUtils.addDocumentWithLocalMasterAndProxy( catalog, catalogId, assetId, filePath, proxyFilePath, subtype, userCreated, importSource )
			:tap( function( revId )
				revisionId = revId
			end ),
		':ignore:' )
	end

	-- Send the metadata for the newly added asset
	catalogSpec:expectHttpRequest( WFCatTrashTestUtils.pushCreateAssetReqResp( catalogId, fileName, subtype ) )
	rxTest:expectOnlyCompleted( pusher:_DEBUG_sendNUpdates( 1, kTimeoutForSendingUpdates ) )

	-- these are the /rels/* links we currently expect in the received document
	local linksExpectedInDoc = { master = false, proxy = false, renditions = false }

	-- Get the metadata for the newly added asset that includes the links for uploading binaries.
	rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 5 * waitMultiplier, catalogSpec, catalogId, puller, {
		WFCatTrashTestUtils.assetUpdateResponse( assetId, revisionId, fileName, subtype, assetCreated, userCreated, importSource, linksExpectedInDoc )
	}, WFCatTrashTestUtils.stripForAssetUpdate( false ) ), 'success' )

	revisionId = WFCatTrashTestUtils.getLatestRevisionId( rxTest, catalog, assetId )

	-- push and validate for the binary upload
	local expectedOriginalUploadCount = 0
	local expectedProxyUploadCount = 0
	if options.proxyOnlyAsset then
		expectedProxyUploadCount = expectedProxyUploadCount + 1

		-- Upload the proxy (and generate renditions)
		catalogSpec:expectHttpRequest( WFCatTrashTestUtils.createProxyReqResp( catalogId, assetId, revisionId, KSIdGenerator.uuidInLowercaseHex(), proxyFilePath ) )
		uploader:startContinuousUploads()

		-- wait for the uploader to issue the create proxy request
		rxTest:observe( WFCatTrashTestUtils.pollUploadCount( uploader, 'proxy' ))
		:expectDataWithinTime( 5 * waitMultiplier, expectedProxyUploadCount )
		:dispose()
	else
		expectedOriginalUploadCount = expectedOriginalUploadCount + 1

		-- Upload the master
		catalogSpec:expectHttpRequest( WFCatTrashTestUtils.createMasterReqResp( catalogId, assetId, revisionId, filePath ) )
		uploader:startContinuousUploads()

		-- Wait for the uploader to issue the create master request
		rxTest:observe( WFCatTrashTestUtils.pollUploadCount( uploader, 'master' ))
		:expectDataWithinTime( 5 * waitMultiplier, expectedOriginalUploadCount )
		:dispose()
	end
	uploader:stopContinuousUploads()

	-- Wait for oz to generate the renditions before requesting next update
	rxTest:expectSingleValue( WFCatTrashTestUtils.pollUntilHasRequiredLinks( 10 * waitMultiplier, catalogSpec, catalogId, assetId, revisionId, subtype, { proxyOnlyAsset = true } ), true )

	-- Get the updated metadata with the binary link.
	rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 5 * waitMultiplier, catalogSpec, catalogId, puller, {
		WFCatTrashTestUtils.assetUpdateResponse( assetId, revisionId, fileName, subtype, assetCreated, userCreated, importSource, { master = not options.proxyOnlyAsset, proxy = options.proxyOnlyAsset, renditions = true }, nil, true )
	}, WFCatTrashTestUtils.stripAssetAndLocation ), 'success' )

	-- Wait for the update to get applied to the assets docValue
	local binaryRelsKey = options.proxyOnlyAsset and '/rels/proxy_type/2560' or '/rels/master'
	rxTest:observe( catalog:docStore():ongoingDocValue( assetId )
		:pluck( '_serverProvidedMetadata', 'links', binaryRelsKey, 'href' )
		:map( type ))
	:expectDataWithinTime( 5 * waitMultiplier, 'string' )
	:dispose()

-- Step 2: Move the asset to trash and sync it

	-- Delete the asset which is on Oz
	rxTest:expectOnlyCompleted( WFCatTrash.moveAssetsToTrash( catalog, { assetId }, pc ) )

	-- Make sure the original asset was deleted, and get it's new revision id
	rxTest:expectSingleValue( catalog:docStore():docValue( assetId )
	:tap( function( docValue )
		revisionId = docValue._revId
	end )
	:pluck( '_deleted' )
	, true )

	-- Make sure a deleted_image was created
	local trashAssetId, trashRevId = WFCatTrashTestUtils.getTrashAssetForDeletedAsset( rxTest, catalog, waitMultiplier, assetId )
	rxTest:expectEqual( 'string', type( trashAssetId ) )
	rxTest:expectEqual( 'string', type( trashRevId ) )

	local trashUserCreated, trashPurgeDate
	rxTest:expectSingleValue( catalog:docStore():docValue( trashAssetId )
		:tap( function( docValue )
			trashUserCreated = docValue.userCreated
			trashPurgeDate = docValue.purgeDate
		end )
	, ':ignore:' )

	-- Send up the Delete request
	catalogSpec:expectHttpRequest( WFCatTrashTestUtils.pushDeleteAssetReqResp( catalogId, assetId, revisionId, trashAssetId, trashRevId ) )
	rxTest:expectOnlyCompleted( pusher:_DEBUG_sendNUpdates( 1, kTimeoutForSendingUpdates ) )

	-- Get the trashed asset back in the sync feed with the links to necessary binaries
	rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 5 * waitMultiplier, catalogSpec, catalogId, puller, {
		WFCatTrashTestUtils.trashAssetUpdateResponse2( trashAssetId, trashRevId, fileName, trashAssetSubtype, trashUserCreated, trashPurgeDate, userCreated, importSource, assetId, { master = true, renditions = true } ),
		WFCatTrashTestUtils.assetDeletedResponse( assetId, trashAssetId, userId ),
	}, WFCatTrashTestUtils.transformOrderAndStripAssetUpdate( true ) ), 'success' )

-- Step 3 Pause Sync - this test issues separate sync actions so need to pause.
	pusher:stopContinuousUpdates()

-- Other Client ------------------------------------------------------------

-- Step 4: On a second client, restore the trash asset

	-- Restore asset on other client
	local otherClientRestoredAssetId = KSIdGenerator.uuidInLowercaseHex()
	local otherClientRestoredRevId = KSIdGenerator.uuidInLowercaseHex()

	WFCatTrashTestUtils.simulateOtherClientAssetRestore {
		rxTest = rxTest,
		catalog = catalog,
		catalogSpec = catalogSpec,
		assetId = otherClientRestoredAssetId,
		revisionId = otherClientRestoredRevId,
		trashAssetId = trashAssetId,
		trashAssetRevId = trashRevId,
		userUpdated = userCreated,
		options = options,
	}

-- Original Client ------------------------------------------------------------

-- Step 5: Restore and Delete the asset locally

	-- Restore the trash asset
	rxTest:expectOnlyCompleted( WFCatTrash.restoreTrashAssets( catalog, { trashAssetId }, pc ) )

	-- Make sure the trash asset was deleted
	rxTest:expectSingleValue( catalog:docStore():docValue( trashAssetId )
		:pluck( '_deleted' ),
	true )

	local restoredAssetId, restoredRevId, restoredDocValue = WFCatTrashTestUtils.getNewlyCreatedAsset( rxTest, catalog, waitMultiplier, { assetId, trashAssetId } )
	rxTest:expectEqual( 'asset', restoredDocValue.type )
	rxTest:expectEqual( subtype, restoredDocValue.subtype )
	rxTest:expectEqual( nil, derefOpt( restoredDocValue, 'original' ) )

	-- Make sure the last trash userCreated and purgeDate data is correctly stored in restored asset createFrom field
	rxTest:expectSingleValue( catalog:docStore():docValue( restoredAssetId )
		:map( function( docValue )
			local createFromInfo = WFCatTrashUtils.createFromInfo( docValue )
			return createFromInfo.lastTrashUserCreated == trashUserCreated and createFromInfo.lastTrashPurgeDate == trashPurgeDate
		end )
	, true )

	-- Delete the locally restored asset.
	rxTest:expectOnlyCompleted( WFCatTrash.moveAssetsToTrash( catalog, { restoredAssetId }, pc ) )

	-- Make sure the restored asset asset was deleted, and get it's new revision id
	rxTest:expectSingleValue( catalog:docStore():docValue( restoredAssetId )
	:tap( function( docValue )
		revisionId = docValue._revId
	end )
	:pluck( '_deleted' )
	, true )

	-- Get the new trash asset id
	local trashAssetId2, trashRevId2 = WFCatTrashTestUtils.getTrashAssetForDeletedAsset( rxTest, catalog, waitMultiplier, restoredAssetId )
	rxTest:expectEqual( 'string', type( trashAssetId2 ) )
	rxTest:expectEqual( 'string', type( trashRevId2 ) )

-- Step 6: Turn on Sync
	-- we have to give Oz staging a bit of time to process the other restore - it's not always as fast as we would like
	-- if we don't we get different documents passed back and forth than what this test is expecting. It should still lead
	-- to the same end result, but this test is covering one specific path
	if not options.mock then
		rxTest:expectSingleValue( WFCatTrashTestUtils.pollUntilAssetExistsWithLinks( 3 * waitMultiplier, catalogSpec, catalogId, otherClientRestoredAssetId, WFCatTrashTestUtils.expectedBinaryLinksForImage() ), true )
	end

	-- Send up the Restore/Delete request
	catalogSpec:expectHttpRequest( WFCatTrashTestUtils.pushRestoreDeleteAssetReqResp( catalogId, trashAssetId, trashRevId, restoredAssetId, restoredRevId, trashAssetId2, trashRevId2 ) )
	rxTest:expectOnlyCompleted( pusher:_DEBUG_sendNUpdates( 2, kTimeoutForSendingUpdates ) )

	-- Get the other client's updates from the sync feed.
	rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 7 * waitMultiplier, catalogSpec, catalogId, puller, {
		WFCatTrashTestUtils.assetDeletedResponse( trashAssetId, otherClientRestoredAssetId, userId ),
		WFCatTrashTestUtils.restoredAssetUpdateResponse( otherClientRestoredAssetId, otherClientRestoredRevId, fileName, subtype, assetCreated, userCreated, importSource, assetId, true, true ),
	}, WFCatTrashTestUtils.transformOrderAndStripAssetUpdate( true ) ), 'success' )

	rxTest:expectSingleValue( catalog:docStore():docValue( otherClientRestoredAssetId )
	:map( function( otherClientDocValue )
		otherClientRestoredRevId = otherClientDocValue._revId
		return otherClientDocValue._deleted
	end ), nil )


	if options.splitErrors then

		-- Get the errors with respect to what we tried to copy from as they are no longer there.
		rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 10 * waitMultiplier, catalogSpec, catalogId, puller, {
			WFCatTrashTestUtils.resourceNotFoundErrorUpdateResponse( "create", restoredAssetId, restoredRevId, "asset", trashAssetId, "deleted_asset", otherClientRestoredAssetId, userId ),
		}, WFCatTrashTestUtils.stripUpdatesAndLinks ), 'success' )

	else

		-- Get the errors with respect to what we tried to copy from as they are no longer there.
		rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 10 * waitMultiplier, catalogSpec, catalogId, puller, {
			WFCatTrashTestUtils.resourceNotFoundErrorUpdateResponse( "create", restoredAssetId, restoredRevId, "asset", trashAssetId, "deleted_asset", otherClientRestoredAssetId, userId ),
			WFCatTrashTestUtils.resourceNotFoundErrorUpdateResponse( "delete", restoredAssetId, restoredRevId, "asset", restoredAssetId, "asset", nil, userId ),
		}, WFCatTrashTestUtils.stripUpdatesAndLinks ), 'success' )

	end

	-- Verify WFCatErrorHandler will update the createFrom trash info
	-- so that we attempt to create from the oz trash asset even though
	-- it is not there anymore, we will re-issue the restore request
	-- and get a new error later.
	rxTest:observe( catalog:docStore():ongoingDocValue( trashAssetId2 )
	:map( WFCatTrashUtils.getCreateFromId ) )
	:expectDataWithinTime( 5 * waitMultiplier, otherClientRestoredAssetId )
	:dispose()

	rxTest:expectSingleValue( catalog:docStore():docValue( trashAssetId2 ):pluck('_empty'), nil )
	rxTest:expectSingleValue( catalog:docStore():docValue( otherClientRestoredAssetId ):pluck('_empty'), nil )

	-- Send up a new Delete request referring to the other client's restored asset
	catalogSpec:expectHttpRequest( WFCatTrashTestUtils.pushDeleteAssetReqResp( catalogId, otherClientRestoredAssetId, otherClientRestoredRevId, trashAssetId2, trashRevId2 ) )
	rxTest:expectOnlyCompleted( pusher:_DEBUG_sendNUpdates( 1, kTimeoutForSendingUpdates ) )


	if options.splitErrors then
		-- Success! Get the tombstone for otherClientRestoredAssetId and the new trashAssetId2.
		rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 10 * waitMultiplier, catalogSpec, catalogId, puller, {
			WFCatTrashTestUtils.assetDeletedResponse( otherClientRestoredAssetId, trashAssetId2, userId ),
			WFCatTrashTestUtils.trashAssetUpdateResponse2( trashAssetId2, trashRevId2, fileName, trashAssetSubtype, trashUserCreated, trashPurgeDate, userCreated, importSource, otherClientRestoredAssetId, { master = true, renditions = true } ),
			WFCatTrashTestUtils.resourceNotFoundErrorUpdateResponse( "delete", restoredAssetId, restoredRevId, "asset", restoredAssetId, "asset", nil, userId),
		}, WFCatTrashTestUtils.transformOrderAndStripAssetUpdate( false )), 'success' )
	else
		-- Success! Get the tombstone for otherClientRestoredAssetId and the new trashAssetId2.
		rxTest:expectSingleValue( WFCatTrashTestUtils.pollForUpdateWith( 10 * waitMultiplier, catalogSpec, catalogId, puller, {
			WFCatTrashTestUtils.assetDeletedResponse( otherClientRestoredAssetId, trashAssetId2, userId ),
			WFCatTrashTestUtils.trashAssetUpdateResponse2( trashAssetId2, trashRevId2, fileName, trashAssetSubtype, trashUserCreated, trashPurgeDate, userCreated, importSource, otherClientRestoredAssetId, { master = true, renditions = true } ),
		}, WFCatTrashTestUtils.transformOrderAndStripAssetUpdate( false )), 'success' )
	end

	rxTest:observe( catalog:docStore():ongoingDocValue( otherClientRestoredAssetId ):pluck('_empty') )
	:expectDataWithinTime( 5 * waitMultiplier, true )
	:dispose()

	-- Wait for the errors to be processed if present.
	rxTest:observe( WFCatTrashTestUtils.pollErrorHandlerErrorsProcessed( errorHandler, 'ResourceNotFoundError' ))
	:expectDataWithinTime( 5 * waitMultiplier, 2, false )
	:dispose()

	-- Check that there is nothing else to send.
	rxTest:expectOnlyCompleted( pusher:_DEBUG_sendNUpdates( 0, kTimeoutForSendingUpdates ) )

	-- Check to see how the ResourceNotFoundErrors were resolved
	rxTest:expectEqual( {
		ResourceNotFoundError = {
			count = 2,
			resolution = {
				IgnoredRevisionDoesNotMatch = {
					count = 1,
					sourceDocIds = {
						restoredAssetId,
					},
				},
				UpdatedLocalCreateFromTargetingOzDoc_LocalRestoreDeleted = {
					count = 1,
					sourceDocIds = {
						restoredAssetId,
					},
				},
			},
		},
	}, errorHandler:DEBUG_errorsResolved() )

end

--------------------------------------------------------------------------------

function WFCatTrashMultipleDevicesBasicOtherRestoreTest.rxTest_CT_MockRestoreDelete_OtherRestore( rxTest )
-- NMB-13896 #1 / NMB-14702
	testAddDeleteSyncRestoreDeleteOtherRestoreSync( rxTest, { mock = true } )
end

--------------------------------------------------------------------------------

function WFCatTrashMultipleDevicesBasicOtherRestoreTest.rxTest_CT_OzRestoreDelete_OtherRestore_testNightly( rxTest )
-- NMB-13896 #1 / NMB-14702
	testAddDeleteSyncRestoreDeleteOtherRestoreSync( rxTest )
end

--------------------------------------------------------------------------------

function WFCatTrashMultipleDevicesBasicOtherRestoreTest.rxTest_CT_MockRestoreDelete_OtherRestore_splitErrors( rxTest )
-- NMB-13896 #1 / NMB-14702
	testAddDeleteSyncRestoreDeleteOtherRestoreSync( rxTest, { mock = true, splitErrors = true } )
end

--------------------------------------------------------------------------------

function WFCatTrashMultipleDevicesBasicOtherRestoreTest.rxTest_CT_OzRestoreDelete_OtherRestore_splitErrors_testNightly( rxTest )
-- NMB-13896 #1 / NMB-14702
	testAddDeleteSyncRestoreDeleteOtherRestoreSync( rxTest, { splitErrors = true } )
end

--==============================================================================
-- @mark setup
--==============================================================================

function WFCatTrashMultipleDevicesBasicOtherRestoreTest.setUp( inCoveragePath )

	coveragePath = inCoveragePath

end

--------------------------------------------------------------------------------

return WFRxTestCase.wrapSuite( 'WFCatTrashMultipleDevicesBasicOtherRestoreTest', WFCatTrashMultipleDevicesBasicOtherRestoreTest )
