--[[----------------------------------------------------------------------------

ADOBE CONFIDENTIAL
------------------
Copyright 2016 Adobe Systems Incorporated
All Rights Reserved.

NOTICE: All information contained herein is, and remain the property of
Adobe Systems Incorporated and its suppliers, if any. The intellectual and
technical concepts contained herein are proprietary to Adobe Systems
Incorporated and its suppliers and may be covered by U.S. and Foreign
Patents, patents in process, and are protected by trade secret or copyright
law.  Dissemination of this information or reproduction of this material is
strictly forbidden unless prior written permission is obtained from
Adobe Systems Incorporated.

----------------------------------------------------------------------------]]--

	-- WF.core.os
local WFPlatformInfo = require 'WFPlatformInfo'

if WFPlatformInfo.isMac or WFPlatformInfo.isAndroid or WFPlatformInfo.isWindows then
       return {}
end

	-- WF.core.rxlua
local WFRxTestCase = require 'WFRxTestCase'

	-- WF.core.utils
local KSIdGenerator = require 'KSIdGenerator'
local KSTableUtils = require 'KSTableUtils'
local KSDate = require "KSDate"

	-- WF.core.lua
local KSData = require 'KSData'

	-- WF.core.os
local WFFileUtils = require 'WFFileUtils'

local LrmConfig = require "LrmConfig"

local LrmCatalogTestHelper = require 'LrmCatalogTestHelper'
local LrmOzPurgeAgent = require 'LrmOzPurgeAgent'

local KSPathUtils = require "KSPathUtils"

local OzFilePaths = require "OzFilePaths"

--------------------------------------------------------------------------------

local LriOzPurgeAgentTest = {}

local purgeAgent, catalog

local TEST_TIMEOUT = 60 -- TODO [immisch 2016-06-09]: debugging only

local deref = KSTableUtils.derefOpt

--------------------------------------------------------------------------------

local function writeTestData( filePath, optData )

	local testData = KSData.make( optData or 'foo' )
	KSData.saveToUniquelyNamedFile( testData, filePath )
	
	local size = KSData.length( testData )
	testData:close()

	return size
	
end

--------------------------------------------------------------------------------

local function makeBlob( rxTest, optData )

	local testData = KSData.make( optData or 'foo' )
	local size = KSData.length( testData )

	local catalogPath = OzFilePaths.getCatalogBasePath( catalog._catalogId )
	local fileName = KSIdGenerator.uuidInLowercaseHex()
	local filePath = KSPathUtils.child( catalogPath, fileName )

	KSData.saveToFile( testData, filePath )
	
	rxTest:finally( function()
		WFFileUtils.delete( filePath )
	end )
	
	testData:close()

	return {
		path = fileName,
		fileSize = size
	}
	
end

--------------------------------------------------------------------------------

local function blobFileExists( blob )

	local catalogPath = OzFilePaths.getCatalogBasePath( catalog._catalogId )

	return WFFileUtils.exists( KSPathUtils.child( catalogPath, blob.path ) )	

end

--------------------------------------------------------------------------------

local function generateAssetWithLocalOriginal_Uploaded( optData, optUserUpdated )

	optUserUpdated = optUserUpdated or KSDate.timeInIso8601Format()

	local testFilePath = LrmCatalogTestHelper.makeTestFilePath()	
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local testDataSize = writeTestData( testFilePath, optData )

	return {
		type = 'asset',
		subtype = 'image',
		_id = assetId,
		userUpdated = optUserUpdated,
		importSource = {
			originalWidth = 100,
			originalHeight = 100,
		},
		_localOnly = {
			files = {
				master = {
					path = testFilePath,
					fileSize = testDataSize,
				}
			}
		},
		_serverProvidedMetadata = {
			links = {
				[ '/rels/master' ] = { 
					href = assetId
				}
			}
		}
	}

end

--------------------------------------------------------------------------------

local function generateVersionWithLocalXmpForAsset_Uploaded( rxTest, optData, assetId )

	local versionId = KSIdGenerator.uuidInLowercaseHex()

	local blob = makeBlob( rxTest, optData )

	return {
		_id = versionId,
		created = "2018-08-23T20:25:31.964992Z",
		updated = "2018-08-23T20:25:31.964992Z",
		type = "version",
		subtype = "edit",
		name = 'foo',
		develop = {
			sha256 = KSIdGenerator.uuidInLowercaseHex()
		},
		asset = {
			id = assetId,
		},
		_localOnly = {
			files = {
				xmp_develop = blob
			}
		},
		_serverProvidedMetadata = {
			links = {
				["/rels/xmp/develop"] = {
					fileSize = 26104,
					href = "<<stripped>>",
				}
			}
		}
	}, blob

end

--------------------------------------------------------------------------------

local function generateAlbumAsset( albumId, assetId )

	local albumAssetId = KSIdGenerator.uuidInLowercaseHex()

	return {

		type = 'album_asset',
		_id = albumAssetId,
		album = {
				id = albumId,
			},
			asset = {
				id = assetId,
			}
		}
		
end

--------------------------------------------------------------------------------

function LriOzPurgeAgentTest.setUp()

	catalog = LrmCatalogTestHelper.makeCatalogService()
	
	local options = {
		delay = 0.1
	}
	
	purgeAgent = LrmOzPurgeAgent( catalog, LrmConfig.StrategyPreferMaster, nil, options )
	
	purgeAgent:mayBeStartPurging()
	
	local catalogPath = OzFilePaths.getCatalogBasePath( catalog._catalogId )

	WFFileUtils.mkdir( catalogPath )
		
end

--------------------------------------------------------------------------------

function LriOzPurgeAgentTest.tearDown()

	if purgeAgent then
	
		purgeAgent:close()
		purgeAgent = nil
		
	end
	
	if catalog then
	
		catalog:close()
		catalog = nil
	
	end
	
end

--------------------------------------------------------------------------------

function LriOzPurgeAgentTest.rxTestPurgeAlbumWhileOffline( rxTest )
	-- Purge should not work when network is offline
	
	rxTest:setTimeLimit( TEST_TIMEOUT )

	local albumId = KSIdGenerator.uuidInLowercaseHex()
	
	purgeAgent:mayBeStopPurging()

	local asset = generateAssetWithLocalOriginal_Uploaded( 'foo' )
	rxTest:expectSingleValue( catalog:docStore():applyOverlay( asset ),
		':ignore:' )

	local asset2 = generateAssetWithLocalOriginal_Uploaded( 'foo2' )
	rxTest:expectSingleValue( catalog:docStore():applyOverlay( asset2 ),
		':ignore:' )
		
	rxTest:expectSingleValue( purgeAgent:purgeForAlbum( albumId, LrmConfig.StrategyPreferMaster ), 0 )
	
	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
	
			type = 'album',
			subtype = 'collection',
			_id = albumId,
			name = 'foo',
			_localOnly = {
				useOffline = true,
			}
		},
		':ignore:')
		
	rxTest:expectSingleValue( catalog:docStore():applyOverlay( generateAlbumAsset( albumId, asset._id ) ),
		':ignore:' )

	rxTest:expectSingleValue( catalog:docStore():applyOverlay( generateAlbumAsset( albumId, asset2._id ) ),
		':ignore:' )

	rxTest:expectSingleValue( purgeAgent:purgeForAlbum( albumId, LrmConfig.StrategyPreferMaster ), 0 )
	
	rxTest:expectSingleValue( catalog:indexStore():queryOnce{
			sql = 'SELECT docId FROM cacheReferences ORDER BY size',
			quiesceUntilUpToDate = true 		
		},
		{
			{ asset._id },
			{ asset2._id },
		} ) 

	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
			
			type = 'album',
			subtype = 'collection',
			_id = albumId,
			name = 'foo',
			_localOnly = {
				useOffline = false,
			}
		},
		':ignore:' )


	rxTest:expectSingleValue( purgeAgent:purgeForAlbum( albumId, LrmConfig.StrategyPreferMaster ), 0)

	rxTest:expectSingleValue( catalog:indexStore():queryOnce{
			sql = 'SELECT docId FROM cacheReferences ORDER BY size',
			quiesceUntilUpToDate = true 		
		},
		{
			{ asset._id },
			{ asset2._id },
		} ) 


	purgeAgent:mayBeStartPurging()

end


function LriOzPurgeAgentTest.rxTestPurgeAlbum( rxTest )

	rxTest:setTimeLimit( TEST_TIMEOUT )

	local albumId = KSIdGenerator.uuidInLowercaseHex()

	local asset = generateAssetWithLocalOriginal_Uploaded( 'foo' )
	rxTest:expectSingleValue( catalog:docStore():applyOverlay( asset ),
		':ignore:' )

	local asset2 = generateAssetWithLocalOriginal_Uploaded( 'foo2' )
	rxTest:expectSingleValue( catalog:docStore():applyOverlay( asset2 ),
		':ignore:' )
		
	local version, xmpDevelop = generateVersionWithLocalXmpForAsset_Uploaded( rxTest, 'hi', asset._id )
	rxTest:expectSingleValue( catalog:docStore():applyOverlay( version ),
		':ignore:' )
	
	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
	
			type = 'album',
			subtype = 'collection',
			_id = albumId,
			name = 'foo',
			_localOnly = {
				useOffline = true,
			}
		},
		':ignore:')
		
	rxTest:expectSingleValue( catalog:docStore():applyOverlay( generateAlbumAsset( albumId, asset._id ) ),
		':ignore:' )

	rxTest:expectSingleValue( catalog:docStore():applyOverlay( generateAlbumAsset( albumId, asset2._id ) ),
		':ignore:' )

	rxTest:expectSingleValue( purgeAgent:purgeForAlbum( albumId, LrmConfig.StrategyPreferMaster ), 0 )
	
	rxTest:expectSingleValue( catalog:indexStore():queryOnce{
			sql = 'SELECT docId FROM cacheReferences ORDER BY size',
			quiesceUntilUpToDate = true 		
		},
		{
			{ version._id },
			{ asset._id },
			{ asset2._id },
		} ) 

	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
			
			type = 'album',
			subtype = 'collection',
			_id = albumId,
			name = 'foo',
			_localOnly = {
				useOffline = false,
			}
		},
		':ignore:' )

	rxTest:expectSingleValue( purgeAgent:purgeForAlbum( albumId, LrmConfig.StrategyPreferMaster ), 3 )

	local exists = blobFileExists( xmpDevelop )
	rxTest:expectEqual(	exists, false )

	rxTest:expectSingleValue( catalog:indexStore():queryOnce{
			sql = 'SELECT docId FROM cacheReferences',
			quiesceUntilUpToDate = true 		
		},
		{ } )
	
end

--------------------------------------------------------------------------------

function LriOzPurgeAgentTest.rxTestPurgeImmediately1k( rxTest )

	rxTest:setTimeLimit( 120 )

	local albumId = KSIdGenerator.uuidInLowercaseHex()

	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
	
			type = 'album',
			subtype = 'collection',
			_id = albumId,
			name = 'foo',
		},
		':ignore:')

	local assetCount = 0

	-- do chunks of 100 for the bulk documents
	for i=1, 10 do

		local assets = {}
		local albumAssets = {}
		
		for j=1, 100 do

			local asset = generateAssetWithLocalOriginal_Uploaded()
			
			assets[ j ] = asset
			albumAssets[ j ] = generateAlbumAsset( albumId, asset._id ) 
						
		end
		
		rxTest:expectSingleValue( catalog:docStore():bulkApplyOverlay( assets ),
				':ignore:' )

		rxTest:expectSingleValue( catalog:docStore():bulkApplyOverlay( albumAssets ),
				':ignore:' )

		assetCount = assetCount + #assets

	end

	rxTest:expectSingleValue( purgeAgent:purgeImmediately( LrmConfig.StrategyPreferMaster, true ), assetCount )

	rxTest:expectSingleValue( catalog:indexStore():queryOnce{
			sql = 'SELECT docId FROM cacheReferences',
			quiesceUntilUpToDate = true 		
		},
		{ } )
	
end

--------------------------------------------------------------------------------

function LriOzPurgeAgentTest.rxTestPurgeNoPurgeLimit1k( rxTest )

	rxTest:setTimeLimit( 120 )

	local albumId = KSIdGenerator.uuidInLowercaseHex()

	rxTest:expectSingleValue( catalog:docStore():applyOverlay {

			type = 'album',
			subtype = 'collection',
			_id = albumId,
			name = 'foo',
		},
		':ignore:')

	local assetCount = 0

	-- do chunks of 100 for the bulk documents
	for i=1, 10 do

		local assets = {}
		local albumAssets = {}

		for j=1, 100 do

			local asset = generateAssetWithLocalOriginal_Uploaded()
			asset._localOnly.files.master.fileSize = 1000000 -- fake a considerable size

			assets[ j ] = asset
			albumAssets[ j ] = generateAlbumAsset( albumId, asset._id ) 

		end

		rxTest:expectSingleValue( catalog:docStore():bulkApplyOverlay( assets ),
				':ignore:' )

		rxTest:expectSingleValue( catalog:docStore():bulkApplyOverlay( albumAssets ),
				':ignore:' )

		assetCount = assetCount + #assets

	end

	rxTest:expectSingleValue( purgeAgent:purgeImmediately( LrmConfig.StrategyPreferMaster, true ), assetCount )

	rxTest:expectSingleValue( catalog:indexStore():queryOnce{
			sql = 'SELECT docId FROM cacheReferences',
			quiesceUntilUpToDate = true
		},
		{ } )

end

--------------------------------------------------------------------------------

function LriOzPurgeAgentTest.rxTestPurgeAllOriginal( rxTest )

	rxTest:setTimeLimit( TEST_TIMEOUT )

	-- pretend we always need disk space
	purgeAgent.private._getDiskSpaceNeeded = function()
	
		return 1234567
	
	end
	
	local assetId = KSIdGenerator.uuidInLowercaseHex()

	local master = makeBlob( rxTest )

	rxTest:expectSingleValue( catalog:docStore()
		:applyOverlay {	
			_id = assetId,
			type = 'asset',
			subtype = 'image',
			importSource = {
				originalWidth = 100,
				originalHeight = 100,
			},
			_localOnly = {
				files = {
					thumbnail2x = makeBlob( rxTest ),
					master = master
				}
			}
		},
		':ignore:' )
	
	rxTest:expectSingleValue( catalog:indexStore():queryOnce 
		{ 
			sql = [[SELECT docId FROM cacheReferences WHERE hasUploaded == 1]],
		},
		{}
	)
		
	rxTest:expectSingleValue( catalog:docStore():applyOverlay {			
			_id = assetId,
			_serverProvidedMetadata = {
				links = {
					[ '/rels/master' ] = {
						href = '/v1.0/123..'
					}
				}
			}
		},
		':ignore:' )

	-- Ensure indexStore is up-to-date. Note that the purgeAll query
	-- is no longer 'quiesceUntilUpToDate'.
	rxTest:expectSingleValue( catalog:indexStore():queryOnce
		{
			sql = [[SELECT docId FROM cacheReferences WHERE hasUploaded == 1]],
			quiesceUntilUpToDate = true,
		},
		':ignore:' )

	rxTest:expectSingleValue( purgeAgent:purgeImmediately( LrmConfig.StrategyPreferMaster ), 1 )
	
	local exists = blobFileExists( master )
	rxTest:expectEqual(	exists, false )
		
	rxTest:expectSingleValue( catalog:docStore():docValue( assetId )
			
		:map( function( docValue )
			return deref( docValue, '_localOnly', 'files', 'master' )					
		end ),
		nil )

end

--------------------------------------------------------------------------------

function LriOzPurgeAgentTest.rxTestPurgeAllOriginalOfflineEditing( rxTest )

	rxTest:setTimeLimit( TEST_TIMEOUT )

	-- pretend we always need disk space
	purgeAgent.private._getDiskSpaceNeeded = function()
	
		return 1234567
	
	end
	
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local albumId = KSIdGenerator.uuidInLowercaseHex()
	
	local master = makeBlob( rxTest )
	
	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
			
			type = 'asset',
			subtype = 'image',
			_id = assetId,
			importSource = {
				originalWidth = 100,
				originalHeight = 100,
			},
			_localOnly = {
				files = {
					thumbnail2x = makeBlob( rxTest ),
					master = master
				}
			},
			_serverProvidedMetadata = {
				links = {
					[ '/rels/master' ] = {
						href = '/v1.0/123..'
					}
				}
			}
		}, 
		':ignore:' )
		
	rxTest:expectSingleValue( catalog:docStore():applyOverlay( generateAlbumAsset( albumId, assetId ) ),
		':ignore:' )
	
	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
			
			type = 'album',
			subtype = 'collection',
			_id = albumId,
			name = 'foo',
			_localOnly = {
				useOffline = true,
			}
		},
		':ignore:' )

	-- Ensure indexStore is up-to-date. Note that the purgeAll query
	-- is no longer 'quiesceUntilUpToDate'.
	rxTest:expectSingleValue( catalog:indexStore():queryOnce
		{
			sql = [[SELECT docId FROM cacheReferences WHERE hasUploaded == 1]],
			quiesceUntilUpToDate = true,
		},
		':ignore:' )

		
	rxTest:expectSingleValue( purgeAgent:purgeImmediately( LrmConfig.StrategyPreferMaster ), 0 )

	local exists = blobFileExists( master )
	rxTest:expectEqual(	'file', exists )		

	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
			
			type = 'album',
			_id = albumId,
			name = 'foo',
			_localOnly = {
				useOffline = false,
			}
		},
		':ignore:')
		
	rxTest:expectSingleValue( catalog:indexStore():queryOnce
		{
			sql = [[SELECT docId FROM cacheReferences WHERE hasUploaded == 1]],
			quiesceUntilUpToDate = true,
		},
		':ignore:' )

	rxTest:expectSingleValue( purgeAgent:purgeImmediately( LrmConfig.StrategyPreferMaster ), 1 )
		
	exists = blobFileExists( master )
		
	rxTest:expectEqual(	false, exists )
		
end

--------------------------------------------------------------------------------

function LriOzPurgeAgentTest.rxTestPurgeAllOfflineEditing( rxTest )
 
	rxTest:setTimeLimit( TEST_TIMEOUT )

	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local albumId = KSIdGenerator.uuidInLowercaseHex()
	local albumAssetId = KSIdGenerator.uuidInLowercaseHex()
	
	local master = makeBlob( rxTest )
	local proxy = makeBlob( rxTest )
	
	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
 			
		type = 'asset',
			_id = assetId,
			importSource = {
				originalWidth = 100,
				originalHeight = 100,
			},
			_localOnly = {
				files = {
					thumbnail2x = makeBlob( rxTest ),
					master = master,
					proxy = proxy
				}
			},
			_serverProvidedMetadata = {
				links = {
					[ '/rels/master' ] = {
						href = '/v1.0/123..'
					}
				}
			}
		},
		':ignore:' )

 		
 	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
 			
			_id = albumId,
			type = 'album',
			subtype = 'collection',
			name = 'foo',
			_localOnly = {
				useOffline = true,
			}
		},
		':ignore:' )
 			

	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
		
			type = 'album_asset',
			_id = albumAssetId,
			album = {
				id = albumId,
			},
			asset = {
				id = assetId,
			}
		},
		':ignore:' )
 	
	local version, versionBlob = generateVersionWithLocalXmpForAsset_Uploaded( rxTest, 'hi', assetId )
	rxTest:expectSingleValue( catalog:docStore():applyOverlay( version ),
		':ignore:' )
	
	-- StrategyProxy (or StrategyMaster) should not delete any original or proxy in offline albums
	rxTest:expectSingleValue( purgeAgent:purgeImmediately( LrmConfig.StrategyProxy ), 0 )
 		
	local exists = blobFileExists( master )

	rxTest:expectEqual(	'file', exists )

	exists = blobFileExists( proxy )
 		
	rxTest:expectEqual(	'file', exists )

	exists = blobFileExists( versionBlob )
 		
	rxTest:expectEqual(	'file', exists )

 end

--------------------------------------------------------------------------------

function LriOzPurgeAgentTest.rxTestPurgeContinuously( rxTest )

	rxTest:setTimeLimit( TEST_TIMEOUT )

	-- pretend we always need disk space
	purgeAgent.private._getDiskSpaceNeeded = function()
	
		return 1234567
	
	end

	local sum = 0
	
	local obs = rxTest:observe( purgeAgent:startContinuousPurge( LrmConfig.StrategyPreferMaster )
		:map( function( count )
			sum = sum + count
			return sum
		end ) )

	local asset = generateAssetWithLocalOriginal_Uploaded( 'foo' )
	rxTest:expectSingleValue( catalog:docStore():applyOverlay( asset ),
		':ignore:' )

	local asset2 = generateAssetWithLocalOriginal_Uploaded( 'fooo' )
	rxTest:expectSingleValue( catalog:docStore():applyOverlay( asset2 ),
		':ignore:' )

	local asset3 = generateAssetWithLocalOriginal_Uploaded( 'foooo' )
	rxTest:expectSingleValue( catalog:docStore():applyOverlay( asset3 ),
		':ignore:' )

	obs:expectDataWithinTime( TEST_TIMEOUT / 2, 2 )

	rxTest:expectSingleValue( catalog:indexStore():queryOnce{
			sql = 'SELECT docId FROM cacheReferences ORDER BY size',
			quiesceUntilUpToDate = true
		},
		{
			{ asset3._id },
		} ) 
	
end
 
--------------------------------------------------------------------------------

function LriOzPurgeAgentTest.rxTestPurgeContinuouslyRestarted( rxTest )

	rxTest:setTimeLimit( TEST_TIMEOUT )

	-- pretend we always need disk space
	purgeAgent.private._getDiskSpaceNeeded = function()

		return 1234567

	end

	local sum = 0

	purgeAgent:mayBeStopPurging()

	local obs = rxTest:observe( purgeAgent:startContinuousPurge( LrmConfig.StrategyPreferMaster )
		:map( function( count )
			sum = sum + count
			return sum
		end ) )

	local asset = generateAssetWithLocalOriginal_Uploaded( 'foo' )
	rxTest:expectSingleValue( catalog:docStore():applyOverlay( asset ),
		':ignore:' )

	local asset2 = generateAssetWithLocalOriginal_Uploaded( 'fooo' )
	rxTest:expectSingleValue( catalog:docStore():applyOverlay( asset2 ),
		':ignore:' )

	local asset3 = generateAssetWithLocalOriginal_Uploaded( 'foooo' )
	rxTest:expectSingleValue( catalog:docStore():applyOverlay( asset3 ),
		':ignore:' )

	obs:expectDataWithinTime( TEST_TIMEOUT / 2, 0 )

	purgeAgent:mayBeStartPurging()

	obs:expectDataWithinTime( TEST_TIMEOUT / 2, 2 )

	rxTest:expectSingleValue( catalog:indexStore():queryOnce{
			sql = 'SELECT docId FROM cacheReferences ORDER BY size',
			quiesceUntilUpToDate = true
		},
		{
			{ asset3._id },
		} )

end

--------------------------------------------------------------------------------

function LriOzPurgeAgentTest.rxTestPurgeContinuouslyWithFileLock( rxTest )

	rxTest:setTimeLimit( TEST_TIMEOUT * 2 )

	-- pretend we always need disk space
	purgeAgent.private._getDiskSpaceNeeded = function()

		return 1234567

	end

	local asset = generateAssetWithLocalOriginal_Uploaded( 'foo' )
	rxTest:waitFor( catalog:docStore():applyOverlay( asset ) )

	-- an asset with a higher epoch
	local asset2 = generateAssetWithLocalOriginal_Uploaded( 'fooo' )
	rxTest:waitFor( catalog:docStore():applyOverlay( asset2 ) )

	local asset3 = generateAssetWithLocalOriginal_Uploaded( 'foooo' )
	rxTest:waitFor( catalog:docStore():applyOverlay( asset3 ) )

	purgeAgent:lockAssetForPurge( asset._id )

	local totalPurgedDocIds = {}
	local sum = 0
	
	local obs = rxTest:observe( purgeAgent:startContinuousPurge( LrmConfig.StrategyPreferMaster )
	
			:map( function( _, _, purgedDocIds )
			
				for _, docId in ipairs( purgedDocIds ) do
				
					if not totalPurgedDocIds[ docId ] then
						sum = sum + 1
					end
				
					totalPurgedDocIds[ docId ] = true
										
				end
				
				return sum
				
			end ) )

	obs:expectDataWithinTime( TEST_TIMEOUT / 2, 1 )

	rxTest:expectSingleValue( catalog:indexStore():queryOnce{
			sql = 'SELECT docId FROM cacheReferences ORDER BY size',
			quiesceUntilUpToDate = true
		},
		{
			{ asset._id },
			{ asset3._id },
		} )

	purgeAgent:unlockAssetForPurge( asset._id )

	-- unlockAssetForPurge will not kick off another purge cycle
	-- we need to add another asset to reliably get the purge query to fire again 
	local asset4 = generateAssetWithLocalOriginal_Uploaded( 'fooooo' )
	rxTest: waitFor( catalog:docStore():applyOverlay( asset4 ) )
		
	obs:expectDataWithinTime( TEST_TIMEOUT, 3 )

	rxTest:expectSingleValue( catalog:indexStore():queryOnce{
			sql = 'SELECT docId FROM cacheReferences ORDER BY size',
			quiesceUntilUpToDate = true
		},
		{
			{ asset4._id },
		} )

end

--------------------------------------------------------------------------------

function LriOzPurgeAgentTest.rxTestPurgeDeletedDocument( rxTest )

	rxTest:setTimeLimit( TEST_TIMEOUT )

	local assetId = KSIdGenerator.uuidInLowercaseHex()
	
	local master = makeBlob( rxTest )
	local proxy = makeBlob( rxTest )
	local thumbnail2x = makeBlob( rxTest )
	local xmp_develop = makeBlob( rxTest, '<xmp />' )

	-- create an asset with a thumbnail and external develop settings
	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
			
			type = 'asset',
			subtype = 'image',
			_id = assetId,
			importSource = {
				originalWidth = 100,
				originalHeight = 100,
			},
			_localOnly = {
				files = {
					master = master,
					proxy = proxy,
					thumbnail2x = thumbnail2x,
					xmp_develop = xmp_develop
				},
			},
		}, 
		':ignore:' )

	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
			_id = assetId,
			_deleted = true,
		},
		':ignore:' )

	local obs = rxTest:observe( purgeAgent._purgeOnDeleteChain )
	obs:expectDataWithinTime( 10, { assetId } ) 
	obs:dispose()
	
	-- make sure external develop is gone
	rxTest:expectEqual( false, blobFileExists( master ) )
	rxTest:expectEqual( false, blobFileExists( proxy ) )
	rxTest:expectEqual( false, blobFileExists( thumbnail2x ) )
	rxTest:expectEqual( false, blobFileExists( xmp_develop ) )

end

--------------------------------------------------------------------------------

function LriOzPurgeAgentTest.rxTestPurgeAlbumAssets( rxTest )
 
	rxTest:setTimeLimit( TEST_TIMEOUT )

	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local albumId = KSIdGenerator.uuidInLowercaseHex()
	local albumAssetId = KSIdGenerator.uuidInLowercaseHex()

	local testFilePath = LrmCatalogTestHelper.makeTestFilePath()
	local proxyFilePath = LrmCatalogTestHelper.makeTestFilePath()
	
	local testData = KSData.make( 'foo' )
	
	KSData.saveToUniquelyNamedFile( testData, testFilePath )
	KSData.saveToUniquelyNamedFile( testData, proxyFilePath )
	
	local testDataSize = KSData.length( testData )
	testData:close()
	
	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
 			
		type = 'asset',
			_id = assetId,
			importSource = {
				originalWidth = 100,
				originalHeight = 100,
			},
			_localOnly = {
				files = {
					thumbnail2x = {
						path = 'foo',
					},
					master = {
						path = testFilePath,
						fileSize = testDataSize,
					},
					proxy = {
						path = proxyFilePath,
						fileSize = testDataSize,
						epoch = 1,
					}
				}
			},
			_serverProvidedMetadata = {
				links = {
					[ '/rels/master' ] = {
						href = '/v1.0/123..'
					}
				}
			}
		},
		':ignore:' )

 		
 	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
 			
			_id = albumId,
			type = 'album',
			subtype = 'collection',
			name = 'foo',
			_localOnly = {
				useOffline = true,
			}
		},
		':ignore:' )
 			

	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
		
			type = 'album_asset',
			_id = albumAssetId,
			album = {
				id = albumId,
			},
			asset = {
				id = assetId,
			}
		},
		':ignore:' )
 	
	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
			_id = albumId,
			_deleted = true,
		},
		':ignore:' )

	local obs = rxTest:observe( catalog:indexStore():query{
			sql = 'SELECT docId FROM cacheReferences WHERE docId = ?',
			quiesceUntilUpToDate = true,
			albumId
		} )
	
	obs:expectDataWithinTime( 10, {} )
	
	obs:dispose()
	
	obs = rxTest:observe( catalog:indexStore():query{
			sql = 'SELECT COUNT(docId) FROM album_asset WHERE albumId = ?',
			quiesceUntilUpToDate = true,
			albumId
		} )
		
	obs:expectDataWithinTime( 10, { { 0 } } )
	
	obs:dispose()
	
 end

--------------------------------------------------------------------------------

function LriOzPurgeAgentTest.rxTestPurgeSpaceAssets( rxTest )
 
	rxTest:setTimeLimit( TEST_TIMEOUT )

	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local spaceId = KSIdGenerator.uuidInLowercaseHex()
	local spaceFavoriteId = KSIdGenerator.uuidInLowercaseHex()

	local testFilePath = LrmCatalogTestHelper.makeTestFilePath()
	local proxyFilePath = LrmCatalogTestHelper.makeTestFilePath()
	
	local testData = KSData.make( 'foo' )
	
	KSData.saveToUniquelyNamedFile( testData, testFilePath )
	KSData.saveToUniquelyNamedFile( testData, proxyFilePath )
	
	local testDataSize = KSData.length( testData )
	testData:close()
	
	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
 			
		type = 'asset',
			_id = assetId,
			importSource = {
				originalWidth = 100,
				originalHeight = 100,
			},
			_localOnly = {
				files = {
					thumbnail2x = {
						path = 'foo',
					},
					master = {
						path = testFilePath,
						fileSize = testDataSize,
					},
					proxy = {
						path = proxyFilePath,
						fileSize = testDataSize,
					}
				}
			},
			_serverProvidedMetadata = {
				links = {
					[ '/rels/master' ] = {
						href = '/v1.0/123..'
					}
				}
			}
		},
		':ignore:' )

 		
 	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
 			
			_id = spaceId,
			type = 'space',
			subtype = 'shared',
			download = false,
			private = false,
			location = false,
			metadata = false,
		},
		':ignore:' )


	for i = 1, 101 do

		local spaceCommentId = KSIdGenerator.uuidInLowercaseHex()

		rxTest:expectSingleValue( catalog:docStore():applyOverlay {
			
				type = 'comment',
				subtype = 'user',
				_id = spaceCommentId,
				space = {
					id = spaceId,
				},
				asset = {
					id = assetId,
				},
				body = string.format( '/* no comment %d */', i ),
				_localOnly = {
					created = KSDate.timeInIso8601Format(),
				}
			},
			':ignore:' )
	end

	rxTest:expectSingleValue( catalog:indexStore():queryOnce{
			sql = 'SELECT COUNT(docId) FROM comment WHERE spaceId = ?',
			quiesceUntilUpToDate = true,
			spaceId
		},
		{ { 101 } } )
	
	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
		
			type = 'favorite',
			subtype = 'user',
			_id = spaceFavoriteId,
			space = {
				id = spaceId,
			},
			asset = {
				id = assetId,
			},
			_localOnly = {
				created = "2016-05-11T20:23:22.018642Z",
			}
		},
		':ignore:' )

	rxTest:expectSingleValue( catalog:indexStore():queryOnce{
			sql = 'SELECT COUNT(docId) FROM spacefavorite WHERE spaceId = ?',
			quiesceUntilUpToDate = true,
			spaceId
		},
		{ { 1 } } )

 	
	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
			_id = spaceId,
			_deleted = true,
		},
		':ignore:' )

	local obs = rxTest:observe( catalog:indexStore():query{
			sql = 'SELECT COUNT(docId) FROM comment WHERE spaceId = ?',
			quiesceUntilUpToDate = true,
			spaceId
		} )
	
	obs:expectDataWithinTime( 10, { { 0 } } )
	
	obs:dispose()
	
	obs = rxTest:observe( catalog:indexStore():query{
			sql = 'SELECT COUNT(docId) FROM spacefavorite WHERE spaceId = ?',
			quiesceUntilUpToDate = true,
			spaceId
		} )
		
	obs:expectDataWithinTime( 10, { { 0 } } )
	
	obs:dispose()
	
 end

--------------------------------------------------------------------------------

function LriOzPurgeAgentTest.rxTestAssetDeletionWithLocalNewId( rxTest )
 
	rxTest:setTimeLimit( TEST_TIMEOUT )

	local assetId = KSIdGenerator.uuidInLowercaseHex()
	
	local master = makeBlob( rxTest )
	local proxy = makeBlob( rxTest )
	local thumbnail2x = makeBlob( rxTest )
	local xmp_develop = makeBlob( rxTest, '<xmp />' )

	-- create an asset with master, proxy, thumbnail and external develop settings
	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
			
			type = 'asset',
			subtype = 'image',
			_id = assetId,
			importSource = {
				originalWidth = 100,
				originalHeight = 100,
			},
			_localOnly = {
				files = {
					master = master,
					proxy = proxy,
					thumbnail2x = thumbnail2x,
					xmp_develop = xmp_develop
				},
			},
		}, 
		':ignore:' )

	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
			_id = assetId,
			_deleted = true,
			_newId = KSIdGenerator.uuidInLowercaseHex()
		},
		':ignore:' )

	local obs = rxTest:observe( purgeAgent._purgeOnDeleteChain )
	obs:expectData( { assetId } ) 
	
	-- make sure files are not purged
	rxTest:expectEqual( 'file', blobFileExists( master ) )
	rxTest:expectEqual( 'file', blobFileExists( proxy ) )
	rxTest:expectEqual( 'file', blobFileExists( thumbnail2x ) )
	rxTest:expectEqual( 'file', blobFileExists( xmp_develop ) )
	
end

--------------------------------------------------------------------------------

function LriOzPurgeAgentTest.rxTestAssetDeletionWithNewIdFromOz( rxTest )
 
	rxTest:setTimeLimit( TEST_TIMEOUT )

	local assetId = KSIdGenerator.uuidInLowercaseHex()

	local master = makeBlob( rxTest )
	local proxy = makeBlob( rxTest )

	-- create an asset with master, proxy, thumbnail and external develop settings
	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
			
			type = 'asset',
			subtype = 'image',
			_id = assetId,
			importSource = {
				originalWidth = 100,
				originalHeight = 100,
			},
			_localOnly = {
				files = {
					master = master,
					proxy = proxy,
				},
			},
		}, 
		':ignore:' )
		
	local new_id = KSIdGenerator.uuidInLowercaseHex()
	
	-- Delete asset with new_id from Oz
	rxTest:expectSingleValue( catalog:docStore():applyOverlay {
	
			_id = assetId,
			_deleted = true,
			
			_serverProvidedMetadata = {
				new_id = new_id
			}
		}, 
		':ignore:' )
	
	local obs = rxTest:observe( purgeAgent._purgeOnDeleteChain )
	obs:expectData( { assetId } ) 
	
	-- make sure new asset is with master info
	rxTest:observe( catalog:docStore():ongoingDocValue( new_id ) 
			:map( function( docValue ) 
				
				return deref( docValue, '_localOnly', 'files', 'master' )
				
			end )
		)
		:expectData( master )
		:dispose()
	
	-- make sure files are not purged
	rxTest:expectEqual( 'file', blobFileExists( master ) )
	rxTest:expectEqual( 'file', blobFileExists( proxy ) )
	
end

--------------------------------------------------------------------------------

function LriOzPurgeAgentTest.rxTestPreparePurgeImp( rxTest )

	rxTest:setTimeLimit( TEST_TIMEOUT )

	local asset = generateAssetWithLocalOriginal_Uploaded( 'foo' )
	local id1 = asset._id
	
	local asset2 = generateAssetWithLocalOriginal_Uploaded( 'foo2' )
	
	KSTableUtils.setNested( asset2, "does not match",
				'importSource',
				'status',
				'error',
				'all',
				'reason',
				'sha256' )
	
	local id2 = asset2._id
	
	local spaceNeeded = 1000000
	
	local docValuesMap = {}
	docValuesMap[ id1 ] = asset
	docValuesMap[ id2 ] = asset2
	
	local victims = {
		{
			id1,
			id1,
			'original',
			"2021-11-15T18:58:31.754Z",
			50000
		},
		{
			id2,
			id2,
			'original',
			"2021-11-15T18:58:31.754Z",
			10000
		}
	}
	
	local expected = {
		debugDocIds = {
			id1,
		},
		developSettingsRelPathsToDelete = {},
		docs = {
			[ id1 ] = {
				_id = id1,
				_localOnly = {
					files = {
						master = {
							_delete = true,
						},
					},
					originalFilePath = {
						_delete = true,
					},
				},
			},
		},
		purged = 1,
		removeCache = {},
		removeFiles = {
			"2021-11-15T18:58:31.754Z",
		},
		spaceReclaimed = 50000,
	}
	
	rxTest:expectEqual( expected, purgeAgent.private._preparePurgeImp(
			victims, spaceNeeded, {}, docValuesMap ) )
	
end

--------------------------------------------------------------------------------

return WFRxTestCase.wrapSuite( LriOzPurgeAgentTest )
