--[[----------------------------------------------------------------------------

WFCatDelete.lua

--------------------------------------------------------------------------------

ADOBE CONFIDENTIAL
------------------
Copyright 2015 Adobe Systems Incorporated
All Rights Reserved.

NOTICE: All information contained herein is, and remains the property of
Adobe Systems Incorporated and its suppliers, if any. The intellectual and
technical concepts contained herein are proprietary to Adobe Systems
Incorporated and its suppliers and are protected by all applicable
intellectual property laws, including trade secret and copyright laws.
Dissemination of this information or reproduction of this material is
strictly forbidden unless prior written permission is obtained from
Adobe Systems Incorporated.

----------------------------------------------------------------------------]]--

	-- RxLua
local Rx = require 'Rx'
local Observable = Rx.Observable

	-- WF.core.logger
local WFLogger = require 'WFLogger'
local logger = WFLogger( 'WFCatDelete' )

	-- WF.core.rxlua
local WFRxRunUntilDone = require 'WFRxRunUntilDone'
local WFRxScheduler = require 'WFRxScheduler'

	-- WF.core.utils
local KSPathUtils = require 'KSPathUtils'
local KSStringUtils = require 'KSStringUtils'
local KSTableUtils = require 'KSTableUtils'
local WFArguments = require 'WFArguments'
local WFCloseSupport = require 'WFCloseSupport'
local WFFileUtils = require 'WFFileUtils'
local WFRxThrottledBatchedBulkApplyOverlay = require 'WFRxThrottledBatchedBulkApplyOverlay'
local WFRxThrottledBatchedApplyOverlayToIds = require 'WFRxThrottledBatchedApplyOverlayToIds'
local KSDate = require 'KSDate'

	-- WF.oz_client.managed_catalog
local NmbLocalRenditionDigest = require 'NmbLocalRenditionDigest'
local WFCatFileSpec = require 'WFCatFileSpec'
local WFCatIdUtils = require 'WFCatIdUtils'
local WFCatIndexUtils = require 'WFCatIndexUtils'
local WFCatTrash = require 'WFCatTrash'
local WFRxIndexStoreSchema = require 'WFRxIndexStoreSchema'

--==============================================================================

local WFCatDelete = {}

local runUntilDone = WFRxRunUntilDone

local parent = KSPathUtils.parent

local derefOpt = KSTableUtils.derefOpt

local check = assert( WFArguments.check )

local NmbLocalRenditionDigest_maybeComputeFromDocValue = assert( NmbLocalRenditionDigest.maybeComputeFromDocValue )
local WFCatIndexUtils_isValidDisplayableAssetDocument = assert( WFCatIndexUtils.isValidDisplayableAssetDocument )
local WFCatIndexUtils_getDuration = assert( WFCatIndexUtils.getDuration )
local WFCatIndexUtils_getOrientationWidthAndHeight = assert( WFCatIndexUtils.getOrientationWidthAndHeight )

--------------------------------------------------------------------------------

local allTrashAssetRequiredFields = "docId, width, height, orientation, assetId, subtype, duration, fileName, purgeDate"
local purgeDateIndexInQueryFields = 9

local function getTrashAssetInfoDetails( currentAssetInfo , additionalDataForAsset)
	local assetInfo = 
		{
			id = currentAssetInfo[ 1 ],
			width = currentAssetInfo[ 2 ],
			height = currentAssetInfo[ 3 ],
			orientation = currentAssetInfo[ 4 ],
			aspectRatio = currentAssetInfo[ 2 ] / currentAssetInfo[ 3 ],
			subtype = currentAssetInfo[ 6 ],
			duration = currentAssetInfo[ 7 ],
			fileName = currentAssetInfo[ 8 ],
			purgeDate = currentAssetInfo[ 9 ]
		}
		
	return assetInfo
end

--------------------------------------------------------------------------------

local function maybeConvertTimestampToUTC( timestamp )

	if timestamp:find( "Z$" ) then
		-- If all sync clients play nice, this should be the normal case.
		return timestamp
	end

	local secs = KSDate.iso8601ToTimeIntervalSinceReference( timestamp )
	return KSDate.formatDateTimeAsUTC( secs )

end

--------------------------------------------------------------------------------

function WFCatDelete.generateIndexSchema( indexStoreSchema )

	check( 'WFRxIndexStoreSchema' )

	indexStoreSchema:addTable {

		sql = [[CREATE TABLE trashAssets(
					width INTEGER,
					height INTEGER,
					orientation INTEGER,
					purgeDate,
					captureDate,
					filename,
					filenameLC,
					assetId,
					subtype,
					originalFileSize,
					duration INTEGER )]],

		relevantTypeSubtypeTuples = {
			{ 'asset', 'deleted_image' },
			{ 'asset', 'deleted_video' },
		},

		map = function( docValue, emit )

			local subtype = docValue.subtype

			local isDisplayable, displayableDocValue = WFCatIndexUtils_isValidDisplayableAssetDocument( docValue )
			if isDisplayable then

				local orientation, width, height = WFCatIndexUtils_getOrientationWidthAndHeight( displayableDocValue )

				local filename = derefOpt( displayableDocValue, 'importSource', 'fileName' )
				local filesize = derefOpt( displayableDocValue, 'importSource', 'fileSize' )

				local filenameLC = ""
				if filename and type( filename ) == "string" then
					filenameLC = KSStringUtils.lower( filename )
					filenameLC = KSStringUtils.trim( filenameLC )
				end

				-- For deleted assets we are only interested in the time of delete which is stored in userCreated.
				local purgeDate = derefOpt( displayableDocValue, 'purgeDate' )
				local captureDate = derefOpt( displayableDocValue, 'captureDate' )

				local duration =  WFCatIndexUtils_getDuration( displayableDocValue )
				
				if width and height and purgeDate then

					-- Ensure purge date is GMT.
					maybeConvertTimestampToUTC( purgeDate )

					emit( width, height, orientation, purgeDate, captureDate, filename, filenameLC, docValue._id, subtype, filesize, duration )

				end

			end

		end,

		useLocalId = true,
		hasOneRowPerDocument = true,
		includeDocumentsFromSource = WFRxIndexStoreSchema.sourceCatalogDocuments,
		
	}

	indexStoreSchema:addIndex(
		[[CREATE INDEX trashAssets_assetIddocId ON trashAssets( assetId, docId )]] )
	indexStoreSchema:addIndex(
		[[CREATE INDEX trashAssets_purgeDateCaptureDateFilenameLCassetId ON trashAssets( purgeDate, captureDate, filenameLC COLLATE i18n_collate, assetId ) ]] )


	-- The 'assetsPendingRestore' table is needed for identifying and counting assets pending restoration
	-- from cloud trash. This information is needed for the purpose of displaying a 'pancake' message
	-- reminding the user of potential data loss after restoring assets while out of Oz storage or
	-- in an expired subscription state. The purgeDate is needed for telling the user how long is left
	-- before the data loss (purging of originals on Oz) will occur.
	
	indexStoreSchema:addTable {

		sql = [[CREATE TABLE assetsPendingRestore(
					purgeDate )]],

		relevantTypeSubtypeTuples = {
			{ 'asset', 'image' },
			{ 'asset', 'video' },
		},

		map = function( docValue, emit )

			local isDisplayable, displayableDocValue = WFCatIndexUtils_isValidDisplayableAssetDocument( docValue )
			if isDisplayable then
				local localTrashInfo = derefOpt( displayableDocValue, '_localOnly', 'universal', 'trash', 'createFrom' )
				local spm = derefOpt( displayableDocValue, '_serverProvidedMetadata' )
				
				if localTrashInfo and not spm then
					local lastTrashPurgeDate = localTrashInfo.lastTrashPurgeDate
					if lastTrashPurgeDate then
						emit( lastTrashPurgeDate )
					end
				end
			end

		end,

		useLocalId = true,
		hasOneRowPerDocument = true,
		includeDocumentsFromSource = WFRxIndexStoreSchema.sourceCatalogDocuments,
		
	}

	indexStoreSchema:addIndex(
		[[CREATE INDEX assetsPendingRestore_purgeDateDocId ON assetsPendingRestore( purgeDate, docId )]] )

end

--------------------------------------------------------------------------------

local _deletionStream = Rx.Subject()
-- Stream which fires notification when
-- any file or folder is deleted

--------------------------------------------------------------------------------

local function deleteFileOrFolder( path, onlyIfEmpty )

	local ok, err = pcall( WFFileUtils.delete, path, onlyIfEmpty )
	_deletionStream:onNext( ok, path )
	
	return ok, err
end

--------------------------------------------------------------------------------

local function attemptToDeleteFolder( path, onlyIfEmpty, extraDesc )

	local ok, err = deleteFileOrFolder( path, onlyIfEmpty )
	
	if not ok then 
		logger:warnf( "Unable to delete %s folder at %s, error %s.", extraDesc, path, err )
	end
	
end

--------------------------------------------------------------------------------

local function attemptToDeleteParentFolders( path, optDeferredDirsCallback )

	local parentFolder = parent( path )
	local grandparentFolder = parent( parentFolder )
	
	if optDeferredDirsCallback then 
		optDeferredDirsCallback( parentFolder, grandparentFolder )
	else
	
		-- Maybe delete the parent folder, if empty. For proxies, this is a 3-letter
		-- folder. For originals, this is the date folder. ( Example: "2012-09-11" ).
		
		if parentFolder and parentFolder ~= '' then
			attemptToDeleteFolder( parentFolder, true, "parent" )
		end
		
		-- Maybe delete the grandparent folder, if empty. For proxies, this is the
		-- proxy size folder ( Examples: "2560", "pano", etc ). For originals, this 
		-- is the year folder ( Examples: "2016", "2015", etc ).
		
		if grandparentFolder and grandparentFolder  ~= '' then
			attemptToDeleteFolder( grandparentFolder, true, "grandparent" )
		end
		
	end

end

--------------------------------------------------------------------------------

function WFCatDelete.deleteAssetAndMaybeParentFolders( volumeGuid, relativePath, optDeletionBacklog, optDeferredDirsCallback )
	
	check( 'string', 'string', '?NmbMRUDatabase', '?function' )
	
	-- Build the asset path
	
	local path = WFFileUtils.createFullPathFromRelativePathAndGuid( relativePath, volumeGuid )
	
	local deleteSuccessful, err

	if path then
	
		-- Delete the asset
	
		deleteSuccessful, err = deleteFileOrFolder( path )
		attemptToDeleteParentFolders( path, optDeferredDirsCallback )
		
	end
	
	if not path or not deleteSuccessful then
		
		logger:warnf( "Unable to delete asset at %s:%s, error %s.", volumeGuid, relativePath, err )
		
		if optDeletionBacklog then
		
			-- Since we were unable to delete the asset, store it in the asset deletion backlog. It will
			-- be processed and deleted at a later time.
		
			optDeletionBacklog:addToDeletionBacklog( volumeGuid, relativePath )
			
		end
		
	end
	
	return deleteSuccessful
	
end

--------------------------------------------------------------------------------

function WFCatDelete.deleteAssetAndMaybeParentFoldersByAbsolutePath( path, optDeletionBacklog, optDeferredDirsCallback )
	
	check( 'string', '?NmbMRUDatabase', '?function' )
	
	-- Delete the asset
	
	local deleteSuccessful, maybeErrorMessage = deleteFileOrFolder( path )
	
	if not deleteSuccessful then 
	
		logger:warnf( "Unable to delete asset at %s, error %s.", path, maybeErrorMessage )
		
		if optDeletionBacklog then
		
			-- Since we were unable to delete the asset, store it in the asset deletion backlog. It will
			-- be processed and deleted at a later time.
		
			local volumeGuid = path and WFFileUtils.getVolumeGuidForPath( path )
			
			local relativePath = path and WFFileUtils.removeVolumeFromPath( path )
		
			if volumeGuid and relativePath then
				optDeletionBacklog:addToDeletionBacklog( volumeGuid, relativePath )
			end

		end
		
	end
	
	attemptToDeleteParentFolders( path, optDeferredDirsCallback )

	return deleteSuccessful, maybeErrorMessage
	
end

--------------------------------------------------------------------------------

function WFCatDelete.filePathsToDeleteAndOverlays( docValue, binaryTypes )

	local localFilePaths = {}
	local deleteFilesArray = {}

	for _, binaryType in pairs( binaryTypes ) do
	
		local volumeGuid, relativePath, absolutePath, size, hasPathAnnotation
		
		local fileCategory = binaryType
		
		if binaryType == 'original' then
			volumeGuid = WFCatFileSpec.getOriginalVolumeGuidFromDocValue( docValue )
			relativePath = WFCatFileSpec.getOriginalRelativePathFromDocValue( docValue )
			hasPathAnnotation = not not volumeGuid
			size = 'master'
		elseif binaryType == 'proxy' then
			volumeGuid, size = WFCatFileSpec.getProxyVolumeGuidFromDocValue( docValue )
			relativePath = WFCatFileSpec.getProxyRelativePathFromDocValue( docValue )
			hasPathAnnotation = not not volumeGuid
		elseif binaryType == 'xmp' then
			absolutePath = WFCatFileSpec.getXmpSidecarPathFromDocValue( docValue )
			hasPathAnnotation = not not absolutePath
			fileCategory = 'original'
			size = 'xmp'
		end
		
		if hasPathAnnotation then
		
			localFilePaths[ binaryType ] = {
				localId = docValue._localId,
				binaryType = binaryType,
				absolutePath = absolutePath,
				volumeGuid = volumeGuid,
				relativePath = relativePath
			}
		
			local deletePathOverlay = { 
				path = { _delete = true },
				relativePath = { _delete = true },
				volumeGuid = { _delete = true },
				sha256 = { _delete = true },
			}
			
			if not deleteFilesArray[ fileCategory ] then
				deleteFilesArray[ fileCategory ] = {}
			end
		
			deleteFilesArray[ fileCategory ][ size ] = deletePathOverlay
		
		end	
		
	end
	
	local assetOverlay
	
	if next( deleteFilesArray ) then
	
		assetOverlay = {
			_id = docValue._id,
			_localOnly = {  
				files = deleteFilesArray
			}
		}
		
	end
	
	return localFilePaths, assetOverlay

end

--------------------------------------------------------------------------------

local function deleteAssetsFromDocStore( docStore, docValues, binaryType, maybeDeletionBacklog )

	local localFilePaths = {}
	local assetOverlays = {}
	
	for _, docValue in pairs( docValues ) do

		local filePaths, assetOverlay = WFCatDelete.filePathsToDeleteAndOverlays( docValue, { binaryType } )
		
		if next( filePaths ) and filePaths[ binaryType ] then
			localFilePaths[ #localFilePaths + 1 ] = filePaths[ binaryType ]
		end
		
		if assetOverlay then
			assetOverlays[ #assetOverlays + 1 ] = assetOverlay
		end
			
	end
			
	if #assetOverlays > 0 then
		
		local maybeSaveInDeletionBacklog = Observable.just( true )
		
		if maybeDeletionBacklog then
		
			maybeSaveInDeletionBacklog = maybeDeletionBacklog:bulkAddToDeletionBacklog( localFilePaths )
				:map( function()
				
					logger:info( "Ensuring deletion backlog is up to date." )
					return maybeDeletionBacklog:bringDeletionBacklogUpToDate()
					
				end )
			
		end
		
		-- Delete all the file annotations from the doc store
		
		logger:infof( "Caching %d file deletions in the backlog.", #localFilePaths )
		
		return maybeSaveInDeletionBacklog
		
			:flatMapLatest( function()
			
				logger:infof( "Deleting %d file annotations from the doc store.", #assetOverlays )
			
				return docStore:bulkApplyOverlay( assetOverlays )
				
			end )

			:map( function()
			
				return localFilePaths
			
			end )
		
	end
	
	return Observable.just( false )

end

--------------------------------------------------------------------------------

local function getBatch( t, first, last )
	
	local sub = {}
	
	for i = first, last do
		sub[ #sub + 1 ] = t[ i ]
	end
	
	return sub
	
end

--------------------------------------------------------------------------------

local function deleteLocalAssetsByDocValues( docStore, docValues, binaryType, maybeDeletionBacklog )

	local count = #docValues
	
	-- If there's nothing to delete, return.
	
	if count < 1 then
		return Observable.just( true )
	end
	
	-- Perform deletions in batches of 100.
	
	local batchSize = 100
	local totalBatches = math.ceil( count / batchSize )
	
	local success = true
	
	local localFilePathsToDelete = {}
	
	local batchOfDocValuesStream = Rx.ReplaySubject( totalBatches )
	
	local batchDeletionStream = batchOfDocValuesStream
	
		:map( function( batchOfDocValues )
		
			local shuttingDown = WFCloseSupport.hasBeenOrWillBeClosed( docStore )
		
			if shuttingDown then
				return Observable.just( localFilePathsToDelete )
			end
		
			return deleteAssetsFromDocStore( docStore, batchOfDocValues, binaryType, maybeDeletionBacklog )
					
					:map( function( localFiles )
					
						return KSTableUtils.appendArray( localFilePathsToDelete, localFiles )
					
					end )
					
					:catch( function( maybeErrorMessage )

						if maybeErrorMessage then
							logger:errorf( "deleteLocalAssetsByDocValues failed with error: %s" , maybeErrorMessage )
						end

						success = false
						
						return Observable.just( localFilePathsToDelete )
						
					end )

		end )
		
		:flatMerge( 1 ) -- Process the batches one at a time.
		
		:takeLast( 1 ) -- Wait for the last batch to complete.
		
		:flatMap( function( localFilePathsToDelete )
		
			local maybeTimer = Observable.timer( 2, WFRxScheduler.sharedScheduler() )
				-- SPAM [jandrew 2016-11-09]: There's a possible race condition where
				-- we're in the middle of saving a negative, have written the annotation
				-- to disk, and are about to move the negative to the correct location on disk.
				-- If we delete the annotation via the above `bulkApplyOverlay`, and then immediately
				-- try to delete the files on disk, it's possible we will leak some untracked files.
				-- By sleeping for a few seconds after the annotation deletion, we give the PP a chance
				-- to finish saving files that we might intend on deleting. This delay should not
				-- impact perceived performance, since deleting the annotation is what updates the UI.
				
			local shuttingDown = WFCloseSupport.hasBeenOrWillBeClosed( docStore )
				
			if shuttingDown or #localFilePathsToDelete == 0 then
				maybeTimer = Observable.just( true )
			end
				
			return maybeTimer
			
				:map( function()
					return localFilePathsToDelete
				end )
			
		end )
		
		:take( 1 )
			
		:map( function( localFilePathsToDelete )
		
			-- Remove the local files from disk
		
			logger:infof( "Now deleting %d files from disk.", #localFilePathsToDelete )
		
			for _, entry in ipairs( localFilePathsToDelete ) do	
			
				local absolutePath = entry.absolutePath
					-- XMP sidecar files are stored via absolute paths.
					
				local volumeGuid = entry.volumeGuid
				local relativePath = entry.relativePath
					-- Originals and proxies are stored via relative paths.
					
				local deleteSuccessful = false
					
				if volumeGuid and relativePath then
		
					deleteSuccessful = WFCatDelete.deleteAssetAndMaybeParentFolders( volumeGuid, relativePath )
				
				elseif absolutePath then
			
					deleteSuccessful = WFCatDelete.deleteAssetAndMaybeParentFoldersByAbsolutePath( absolutePath )
					
					if deleteSuccessful then
						volumeGuid = WFFileUtils.getVolumeGuidForPath( absolutePath )
						relativePath = WFFileUtils.removeVolumeFromPath( absolutePath )
					end
					
				end
				
				if deleteSuccessful
					and maybeDeletionBacklog
					and volumeGuid 
					and relativePath then
					maybeDeletionBacklog:removeFromDeletionBacklog( volumeGuid, relativePath )
				end
				
			end
		
			return success
		
		end )
	
	local batchCount = 0
	local batchOffset = 1
	
	while batchCount < totalBatches do

		logger:infof( "Now processing assets of range %d through %d for deletion.", batchOffset, batchOffset + batchSize - 1 )

		local batchOfDocValues = getBatch( docValues, batchOffset, batchOffset + batchSize - 1 )
		
		batchOffset = batchOffset + batchSize
		
		batchOfDocValuesStream:onNext( batchOfDocValues )
		
		batchCount = batchCount + 1
		
	end
	
	batchOfDocValuesStream:onCompleted()
		
	return batchDeletionStream:call( runUntilDone )

end

--------------------------------------------------------------------------------

local function deleteLocalAssetByDocId( docStore, docId, binaryType, maybeDeletionBacklog )

	return docStore:docValue( docId )

		:flatMapLatest( function( docValue )

			return deleteLocalAssetsByDocValues( docStore, { docValue }, binaryType, maybeDeletionBacklog )

		end )
		
		:call( runUntilDone )

end

--------------------------------------------------------------------------------

local function deleteLocalAssetsByDocIds( docStore, docIds, binaryType, maybeDeletionBacklog )

	return docStore:bulkDocValue( docIds )

		:flatMapLatest( function( docValues )

			return deleteLocalAssetsByDocValues( docStore, docValues, binaryType, maybeDeletionBacklog )

		end )
		
		:call( runUntilDone )

end

--------------------------------------------------------------------------------

function WFCatDelete.deleteLocalOriginalByDocId( docStore, docId, optDeletionBacklog )

	check( 'WFRxDocStore', '+string|number', '?NmbMRUDatabase' )

	return deleteLocalAssetByDocId( docStore, docId, 'original', optDeletionBacklog )

end

--------------------------------------------------------------------------------

function WFCatDelete.deleteLocalOriginalsByDocValues( docStore, docValues, optDeletionBacklog )

	check( 'WFRxDocStore', 'table', '?NmbMRUDatabase' )

	return deleteLocalAssetsByDocValues( docStore, docValues, 'original', optDeletionBacklog )
		:flatMap( function( originalResult )
			return deleteLocalAssetsByDocValues( docStore, docValues, 'xmp', optDeletionBacklog )
				:map( function( xmpResult )
					return originalResult
				end )
		end )

end

--------------------------------------------------------------------------------

function WFCatDelete.deleteLocalProxiesByDocValues( docStore, docValues, optDeletionBacklog )

	check( 'WFRxDocStore', 'table', '?NmbMRUDatabase' )

	return deleteLocalAssetsByDocValues( docStore, docValues, 'proxy', optDeletionBacklog )

end

--------------------------------------------------------------------------------

function WFCatDelete.deleteLocalOriginalsByDocIds( docStore, docIds, optDeletionBacklog )

	check( 'WFRxDocStore', 'table', '?NmbMRUDatabase' )

	return deleteLocalAssetsByDocIds( docStore, docIds, 'original', optDeletionBacklog )
		:flatMap( function( originalResult )
			return deleteLocalAssetsByDocIds( docStore, docIds, 'xmp', optDeletionBacklog )
				:map( function( xmpResult )
					return originalResult
				end )
		end )

end

--------------------------------------------------------------------------------

function WFCatDelete.deleteLocalProxiesByDocIds( docStore, docIds, optDeletionBacklog )

	check( 'WFRxDocStore', 'table', '?NmbMRUDatabase' )

	return deleteLocalAssetsByDocIds( docStore, docIds, 'proxy', optDeletionBacklog )

end

--------------------------------------------------------------------------------

function WFCatDelete.deleteDocStoreFileArray( fileArray, optDeletionBacklog, optDeferredDirsCallback )

	check( 'table', '?NmbMRUDatabase', '?function' )
	
	for fileCategory, fileCategoryArray in pairs( fileArray ) do
		
		if type( fileCategoryArray ) == 'table' then
			
			-- xmp_develop deletion won't be handled here
			-- PixelProviderFileCachePurgeWorker would take care of xmp_develop deletion
			
			for fileSubCategory, fileSubCategoryInfo in pairs( fileCategoryArray ) do
				
				if type( fileSubCategoryInfo ) == 'table' then
				
					local filePath = fileSubCategoryInfo.path or fileSubCategoryInfo.absolutePath
						-- Renditions and Sidecar XMP are stored via absolute paths.
						
					local relativePath = fileSubCategoryInfo.relativePath
					local volumeGuid = fileSubCategoryInfo.volumeGuid
						-- Originals and proxies are stored via relative paths.
						
					if relativePath and volumeGuid then
			
						WFCatDelete.deleteAssetAndMaybeParentFolders(
							volumeGuid, relativePath, optDeletionBacklog, optDeferredDirsCallback )
					
					elseif filePath then
				
						WFCatDelete.deleteAssetAndMaybeParentFoldersByAbsolutePath(
							filePath, optDeletionBacklog, optDeferredDirsCallback )
						
					end
					
				end
				
			end
			
		end
		
	end

	return true

end

--------------------------------------------------------------------------------

local function deleteAssetsFromDisk( localFiles, maybeDeletionBacklog )
	
	local parents = { }
	local grandParents = { }
	
	for _, fileArray in ipairs( localFiles ) do
	
		WFCatDelete.deleteDocStoreFileArray( fileArray, maybeDeletionBacklog, function( parentFolder, grandParentFolder )
			
			if parentFolder then
				parents[ parentFolder ] = true
			end
			
			if grandParentFolder then
				grandParents[ grandParentFolder ] = true
			end
			
		end )
	
	end

	for dir in pairs( parents ) do
		attemptToDeleteFolder( dir, true, "parent" )
	end

	for dir in pairs( grandParents ) do
		attemptToDeleteFolder( dir, true, "grandparent" )
	end
	
	return true

end

--------------------------------------------------------------------------------

function WFCatDelete.deleteDocs( docStore, docIds, dontNotifyOz )

	if not docIds or #docIds < 1 then
		return Observable.just( true )
	end

	local overlay = {
		_newLabel = dontNotifyOz and '_dontsendtooz' or nil,
		_deleted = true
	}

	return WFRxThrottledBatchedApplyOverlayToIds( "WFCatDelete_deleteDocs", docStore, overlay, docIds, 100, 10 )

end

--------------------------------------------------------------------------------

function WFCatDelete.deleteDocsFromQueryResult( docStore, rows, dontNotifyOz )

	if #rows < 1 then
		return Observable.just( true )
	end

	local docIds = {}
	for i, row in ipairs( rows ) do
	
		docIds[ i ] = row[ 1 ]
			
	end
	
	return WFCatDelete.deleteDocs( docStore, docIds, dontNotifyOz )

end

--------------------------------------------------------------------------------

local function deleteAlbumAssets( wfCatalog, assetIds )

	return wfCatalog:indexStore():queryOnce {
		-- get all the album_assets
		sql = [[SELECT aa.docId FROM album_asset_v2 AS aa JOIN CARRAY( ?, ?, 'char*' ) AS assetIds ON assetIds.value = aa.assetId]],
		assetIds,
		#assetIds,
		quiesceUntilUpToDate = true,
		singleColumnQuery = true
	}
	:flatMap( function( docIds )
		return WFCatDelete.deleteDocs( wfCatalog:docStore(), docIds, true )
	end )
	
end

--------------------------------------------------------------------------------

local function assetOverlaysAndFileDeletions( wfCatalog, assetIds )

	local fieldSpec = { _id = true, _localOnly = { files = true  } }

	-- Gather the local file arrays for all the assets and create the asset overlays.
	return wfCatalog:docStore():bulkDocValue( assetIds, fieldSpec )
	
		:map( function( docValues )

			local localFiles = {}
			local assetOverlays = {}

			for i, docValue in ipairs( docValues ) do
			
				localFiles[ i ] = derefOpt( docValue, '_localOnly', 'files' )

				assetOverlays[ i ] = {
					_id = docValue._id,
					_deleted = true,
					_localOnly = {
						deletedLocally = true -- signals that file deletions are handled
					}
				}
					
			end

			return assetOverlays, localFiles

		end )
end

--------------------------------------------------------------------------------

local function stackCleanupOverlays( wfCatalog, stacksThatNeedAssetCleanup, stackAssetsBeingRemoved )
	
	if #stacksThatNeedAssetCleanup < 1 then
		return Observable.just( {} )
	end
	
	return wfCatalog:indexStore():queryOnce {
			sql = [[SELECT sa.docId FROM stack_assets AS sa JOIN CARRAY( ?, ?, 'char*' ) AS stacksThatNeedAssetCleanup ON stacksThatNeedAssetCleanup.value == sa.stackId]],
			stacksThatNeedAssetCleanup,
			#stacksThatNeedAssetCleanup,
			quiesceUntilUpToDate = true,
			singleColumnQuery = true
		}
	:map( function( docIds )
		local overlays = {}
		
		for _, docId in ipairs( docIds ) do

			if not stackAssetsBeingRemoved[ docId ] then
				overlays[ #overlays + 1 ] = {
					_id = docId,
					_deleted = true,
				}
			end
			
		end
		
		return overlays
	end )

end
	
--------------------------------------------------------------------------------

local function stackRelatedOverlays( wfCatalog, assetIds )
	
	local indexStore = wfCatalog:indexStore()
	
	local maybeAddStackAssetDeleteOverlay = function( overlays, stackAssetId, alreadyRemoved )
		
		if not alreadyRemoved[ stackAssetId ] then
			
			alreadyRemoved[ stackAssetId ] = true
			
			overlays[ #overlays + 1 ] = {
				_id = stackAssetId,
				_deleted = true,
			}
		end
		
	end
	
	local stacksBeingRemoved = indexStore:queryOnce {
			sql = [[SELECT sa.docId, sa.stackId FROM stack_assets AS sa JOIN CARRAY( ?, ?, 'char*' ) AS assetIds ON assetIds.value = stackId]],
			quiesceUntilUpToDate = true,
			assetIds,
			#assetIds,
		}
		
	local stackAssetsAndCoversToRemove = indexStore:queryOnce {
			sql = [[SELECT s.stackId, sa.docId, s.ozCoverId == sa.assetId FROM stacks AS s 
													JOIN stack_assets AS sa USING( stackId )
													JOIN CARRAY( ?, ?, 'char*' ) AS assetIds ON assetIds.value = sa.assetId]],
			quiesceUntilUpToDate = true,
			assetIds,
			#assetIds,
		}
		
	local stackCountsForAllStacks = indexStore:queryOnce {
			sql = [[SELECT stackId, count( assetId ) FROM stack_assets GROUP BY stackId]],
			quiesceUntilUpToDate = true,
			tempIds = assetIds,
		}
		:map( function( rows )
			local results = {}
			for _, row in ipairs( rows ) do
				local stackId = row[ 1 ]
				local stackCount = row[ 2 ]
				if stackId then
					results[ stackId ] = stackCount
				end
			end
			return results
		end )

	local stackRemovalCounts = indexStore:queryOnce {
			sql = [[SELECT stackId, count( assetId ) FROM CARRAY( ?, ?, 'char*' ) AS assetIds
						JOIN stack_assets AS sa ON assetIds.value = sa.assetId GROUP BY stackId]],
			quiesceUntilUpToDate = true,
			assetIds,
			#assetIds,
		}
		:map( function( rows )
			local results = {}
			for _, row in ipairs( rows ) do
				local stackId = row[ 1 ]
				local stackCount = row[ 2 ]
				if stackId then
					results[ stackId ] = stackCount
				end
			end
			return results
		end )
			
	local stackCountsRemainingForTouchedStacks = Observable.combineLatest( stackRemovalCounts, stackCountsForAllStacks )
		:map( function( stackRemovalCounts, stackCountsForAllStacks )
			local countsRemaining = {}
			for stackId, numberRemoved in pairs( stackRemovalCounts ) do
				local fullStackCount = stackCountsForAllStacks[ stackId ]
				if fullStackCount then
					countsRemaining[ stackId ] = math.max( fullStackCount - numberRemoved, 0 )
				end
			end 
			return countsRemaining
		end )
		
		
	return Observable.combineLatest( stacksBeingRemoved, stackAssetsAndCoversToRemove, stackCountsRemainingForTouchedStacks )
		:map( function( stackAssetsForStacksBeingRemoved, stackAssetsBeingRemovedDirectly, stackCountsRemaining )
			
			local stacksBeingCompletelyRemoved = {} -- deletes for these will come from the regular asset delete code
			local stackAssetsBeingRemoved = {} -- track which stack_assets we've already deleted so we don't add one multiple times
			
			local stackRemovalOverlays = {} -- additional stack overlays that should go to Oz
			local stackAssetLocalRemovalOverlays = { label = '_dontsendtooz' } -- stack_asset overlays that shouldn't go to Oz
			local stacksThatNeedNewCover = {}
			local stacksThatNeedAssetCleanup = {}
			
			-- stack_assets for stacks that are being explicitly removed shouldn't be sent to Oz
			for _, row in ipairs( stackAssetsForStacksBeingRemoved ) do
				local saDocId = row[ 1 ]
				local stackId = row[ 2 ] 
				
				stacksBeingCompletelyRemoved[ stackId ] = true
				
				maybeAddStackAssetDeleteOverlay( stackAssetLocalRemovalOverlays, saDocId, stackAssetsBeingRemoved )
			end
			
			-- determine which stacks have < 2 assets. if we're removing assets from them, we want to clean them up
			-- if we're not removing assets from them, we'll leave them as it's safer that way
			local stacksThatAreTooSmall = {}
			for stackId, countRemaining in pairs( stackCountsRemaining ) do
				if countRemaining < 2 and ( not stacksBeingCompletelyRemoved[ stackId ] ) then
					stacksThatAreTooSmall[ stackId ] = true
				end
			end
			
			-- for those assets in stacks that are being removed directly (the asset is being deleted, not the stack)
			-- * if the stack is being explicitly deleted, add a local stack_asset removal (if we don't already have it)
			-- * otherwise remove the stack_asset locally and if the stack would be too small, remove the stack
			-- we also need to make sure that if we're removing a cover, we replace it
			for _, row in ipairs( stackAssetsBeingRemovedDirectly ) do
				local stackId = row[ 1 ]
				local saDocId = row[ 2 ]
				local isCover = tonumber( row[ 3 ] ) ~= 0
				
				if stacksThatAreTooSmall[ stackId ] then
					
					-- mark the stack for removal
					if not stacksBeingCompletelyRemoved[ stackId ] then
						stacksBeingCompletelyRemoved[ stackId ] = true
						
						stackRemovalOverlays[ #stackRemovalOverlays + 1 ] = {
								_id = stackId,
								_deleted = true
							}
							
						stacksThatNeedAssetCleanup[ #stacksThatNeedAssetCleanup + 1 ] = stackId
						
					end
					
					
				else
				
					-- remove the stack asset
					maybeAddStackAssetDeleteOverlay( stackAssetLocalRemovalOverlays, saDocId, stackAssetsBeingRemoved )
					
					-- if we're removing the cover track it so we can replace it
					if isCover then
						stacksThatNeedNewCover[ #stacksThatNeedNewCover + 1 ] = stackId
					end
					
				end
			end
			
			return stackRemovalOverlays, stackAssetLocalRemovalOverlays, assetIds, stacksThatNeedNewCover, stacksThatNeedAssetCleanup, stackAssetsBeingRemoved
			
		end )
		:flatMap( function( ozOverlays, localOverlays, assetIds, stacksThatNeedNewCover, stacksThatNeedAssetCleanup, stackAssetsBeingRemoved )
			-- get new cover for stack if cover was removed but stack will still stick around
			return Observable.combineLatest(  
					require 'WFCatStacks'.getOverlaysForNewCovers( wfCatalog, stacksThatNeedNewCover, assetIds ),
					stackCleanupOverlays( wfCatalog, stacksThatNeedAssetCleanup, stackAssetsBeingRemoved ) )
					
				:map( function( coverOverlays, cleanupOverlays )
					
					for _, overlay in ipairs( coverOverlays ) do
						ozOverlays[ #ozOverlays + 1 ] = overlay
					end
					
					for _, overlay in ipairs( cleanupOverlays ) do
						localOverlays[ #localOverlays + 1 ] = overlay
					end
					
					return { [ 'oz' ] = ozOverlays, [ 'local' ] = localOverlays }
				end )
		end )
	
end

--------------------------------------------------------------------------------

local function getAssetUuidsFromImageIds( wfCatalog, imageIds, assetIdFromLocalIdFn )

	-- return Observable which fires only once with the list of oz-asset-ids
	-- converting local ids to oz-asset-ids if needed.

	-- Also ensures imageIds are either all local or all oz-asset-ids.

	local assetUuids

	if type( imageIds[ 1 ] ) == 'number' then

		for _, imageId in ipairs( imageIds ) do
			assert( type( imageId ) == 'number', 'All asset ids must be either local or global ids but not a mix' )
		end

		assetUuids = assetIdFromLocalIdFn( wfCatalog, imageIds )

	elseif type( imageIds[ 1 ] ) == 'string' then

		for _, imageId in ipairs( imageIds ) do
			assert( type( imageId ) == 'string', 'All asset ids must be either local or global ids but not a mix' )
		end

		assetUuids = Observable.just( imageIds )

	else

		assert( false, 'Asset ids must all be uuid strings, or all be non-zero positive integers' )

	end
	
	return assetUuids

end

--------------------------------------------------------------------------------

local function deleteAssetsImmediately( wfCatalog, imageIds, optPixelProviderProcessor, optDeletionBacklog )

	check( 'table', 'table' )

	local assetUuids = getAssetUuidsFromImageIds( wfCatalog, imageIds, WFCatIdUtils.getOzAssetIdsForLocalIds )

	local docStore = wfCatalog:docStore()

	return assetUuids:flatMap( function( assetIds )

		return Observable.combineLatest(
			Observable.just( assetIds ),
			stackRelatedOverlays( wfCatalog, assetIds ),
			assetOverlaysAndFileDeletions( wfCatalog, assetIds ) )

	end )

	-- Delete all the assets from the doc store

	:flatMap( function( assetIds, stackOverlays, assetOverlays, localFiles )

		local stackOverlaysOz = stackOverlays[ 'oz' ] or {}
		local stackOverlaysLocal = stackOverlays[ 'local' ] or {}

		-- combine the stackOverlays and the assetOverlays, making sure that the
		-- stackOverlays are first, then apply the assets...
		local overlayStacksOz = #stackOverlaysOz > 0 and
				WFRxThrottledBatchedBulkApplyOverlay( "deleteAssetsImmediately_stacksOz", docStore, stackOverlaysOz, 100, 10 ) or
				Observable.just( {} )

		local overlayStacksLocal = #stackOverlaysLocal > 0 and
				WFRxThrottledBatchedBulkApplyOverlay( "deleteAssetsImmediately_stacksLocal", docStore, stackOverlaysLocal, 100, 10 ) or
				Observable.just( {} )

		local overlayAssets = #assetOverlays > 0 and
				WFRxThrottledBatchedBulkApplyOverlay( "deleteAssetsImmediately_assets", docStore, assetOverlays, 100, 10 ) or
				Observable.just( {} )

		return overlayStacksOz
			:concat( overlayStacksLocal )
			:concat( overlayAssets )
			:flatMap( function()
				return deleteAlbumAssets( wfCatalog, assetIds )
			end )
			:flatMap( function()
				return Observable.just( localFiles, assetOverlays )
			end )

	end )

	-- Remove the local files from disk

	:tap( function( localFiles, assetOverlays )

		if localFiles then

			deleteAssetsFromDisk( localFiles, optDeletionBacklog )

			if optPixelProviderProcessor then
				local docIds = { }
				for i, overlay in ipairs( assetOverlays ) do
					docIds[ i ] = overlay._id
				end

				optPixelProviderProcessor:purgeRenditionsForDeletedAssets( docIds ):call( runUntilDone )

			end

		end

	end)

	:takeLast( 0 )

end

--------------------------------------------------------------------------------

function WFCatDelete.deleteStackAssets( wfCatalog, stackIds, optPixelProviderProcessor, optDeletionBacklog )

	-- Stacks currently cannot be moved into the trash so just delete them immediately.

	return deleteAssetsImmediately( wfCatalog, stackIds, optPixelProviderProcessor, optDeletionBacklog )

end

--------------------------------------------------------------------------------

local function getPusherControlFuncs( wfCatSynchronizer )
	return {
		pauseReturningCounter = function( ... )
			return wfCatSynchronizer._metadataSyncClient:pausePusherReturningCounter( ... )
		end,
		resume = function()
			wfCatSynchronizer._metadataSyncClient:resumePusher()
				:call( runUntilDone )
		end,
	}
end

--------------------------------------------------------------------------------

function WFCatDelete.gatherBinaryFilesToMove( wfCatalog, srcDocValue, dstDocValue, batchOfFilesToMoveArray )

	-- Original, Proxy, and xmp related paths are part of _localOnly.files which
	-- is copied so no need to move those binaries.

	local srcXmpCameraRawMd5 = NmbLocalRenditionDigest_maybeComputeFromDocValue( srcDocValue )
	local dstXmpCameraRawMd5 = NmbLocalRenditionDigest_maybeComputeFromDocValue( dstDocValue )

	if srcXmpCameraRawMd5 and dstXmpCameraRawMd5 then

		for _, longEdge in ipairs( { 320, 640, 1280, 2048 } ) do
			local srcPath = wfCatalog:fileSpec():pathForPreviewDigestAndLongEdge( srcDocValue._id, srcXmpCameraRawMd5, longEdge, false )
			if WFFileUtils.exists( srcPath ) == "file" then
				local dstPath = wfCatalog:fileSpec():pathForPreviewDigestAndLongEdge( dstDocValue._id, dstXmpCameraRawMd5, longEdge, true )
				if dstPath ~= srcPath then
					table.insert( batchOfFilesToMoveArray, { srcFilePath = srcPath, dstFilePath = dstPath } )
				end
			end
		end

	end

end

local gatherBinaryFilesToMove = WFCatDelete.gatherBinaryFilesToMove

--------------------------------------------------------------------------------

function WFCatDelete.deleteAssets( wfCatalog, imageIds, maybeSynchronizer, optPixelProviderProcessor, optDeletionBacklog )

	local pusherControl = maybeSynchronizer and getPusherControlFuncs( maybeSynchronizer )
	return WFCatTrash.moveAssetsToTrash( wfCatalog, imageIds, pusherControl, stackRelatedOverlays, gatherBinaryFilesToMove )

end

--------------------------------------------------------------------------------

function WFCatDelete.getDeletionStream()

	return _deletionStream

end

--------------------------------------------------------------------------------

function WFCatDelete.restoreTrashAssets( wfCatalog, trashImageIds, wfCatSynchronizer )

	local pusherControl = getPusherControlFuncs( wfCatSynchronizer )
	return WFCatTrash.restoreTrashAssets( wfCatalog, trashImageIds, pusherControl, false, nil, gatherBinaryFilesToMove )

end

--------------------------------------------------------------------------------

function WFCatDelete.deleteTrashAssets( wfCatalog, imageIds, maybeSynchronizer, optPixelProviderProcessor, optDeletionBacklog )

	check( 'table', 'table' )

	local function fileDeletionHandler( docIds, localFiles )

		-- Remove the local files from disk

		deleteAssetsFromDisk( localFiles, optDeletionBacklog )

		if optPixelProviderProcessor then
			optPixelProviderProcessor:purgeRenditionsForDeletedAssets( docIds ):call( runUntilDone )
		end

	end

	local function addToFilesToDelete( filesToDeleteList, docValue )
		filesToDeleteList[ #filesToDeleteList + 1 ] = derefOpt( docValue, '_localOnly', 'files' )
	end

	local pusherControl = maybeSynchronizer and getPusherControlFuncs( maybeSynchronizer )

	return WFCatTrash.deleteTrashAssets( wfCatalog, imageIds, pusherControl,
		addToFilesToDelete, fileDeletionHandler )

end

--------------------------------------------------------------------------------

function WFCatDelete.trashAssetListModel( wfCatalogStream )

	check( 'table' )

	local value = wfCatalogStream
		:filter( function( wfCatalog )
			return wfCatalog ~= nil
		end )
		:flatMap( function( wfCatalog )
			
			local getMostImmediatePurgeDateFunction = function( rows, ascDesc )
				
				local mostImmediatePurgeDate
				
				if rows[ 1 ] then
					local mostImmediatePurgeDateRowIndex = ascDesc == 'ASC' and 1 or #rows
					local mostImmediatePurgeDateRow = rows[ mostImmediatePurgeDateRowIndex ]
					mostImmediatePurgeDate = mostImmediatePurgeDateRow[ purgeDateIndexInQueryFields ]
				end
				
				return mostImmediatePurgeDate
				
			end
			
			return WFCatTrash.trashAssetList( wfCatalog, allTrashAssetRequiredFields, getMostImmediatePurgeDateFunction )
			
			:map( function( rows )
			
				logger:infof( "WFCatDelete.trashAssetListModel: %s rows with purge date after now", #rows )

				local assetInfos = {}

				for i, row in ipairs( rows or {} ) do
					assetInfos[ i ] =  getTrashAssetInfoDetails( row )
				end
				
				return assetInfos

			end )

		end )

	return {

		value = value,

		events = {
			setSortOrder = function( model, sortAscending )
				WFCatTrash.setTrashAssetListSortOrder( sortAscending and 'ASC' or 'DESC' )
			end,
		}

	}

end

--------------------------------------------------------------------------------

function WFCatDelete.count( wfCatalog )

	check( 'table' )

	return WFCatTrash.countTrashAssets( wfCatalog )
		:pluck( 1, 1 )

end

--------------------------------------------------------------------------------

function WFCatDelete.trashAssetCountModel( wfCatalogStream )

	local value = wfCatalogStream

		:filter( function( wfCatalog )
			return wfCatalog ~= nil
		end )

		:flatMapLatest( function( wfCatalog )

			return WFCatDelete.count( wfCatalog )

		end )

	return {
		value = value,
	}

end

--------------------------------------------------------------------------------

function WFCatDelete.stackRelatedOverlays( wfCatalog, assetIds )

	return stackRelatedOverlays( wfCatalog, assetIds )
	
end

--==============================================================================

return WFCatDelete
