--[[----------------------------------------------------------------------------

NmbLeakedFileDetector.lua

--------------------------------------------------------------------------------

ADOBE CONFIDENTIAL
------------------
Copyright 2017 Adobe Systems Incorporated
All Rights Reserved.

NOTICE: All information contained herein is, and remain the property of
Adobe Systems Incorporated and its suppliers, if any. The intellectual and
technical concepts contained herein are proprietary to Adobe Systems
Incorporated and its suppliers and may be covered by U.S. and Foreign
Patents, patents in process, and are protected by trade secret or copyright
law.  Dissemination of this information or reproduction of this material is
strictly forbidden unless prior written permission is obtained from
Adobe Systems Incorporated.

------------------------------------------------------------------------------]]

	-- Rx
local Rx = require 'Rx'

	-- WF.core.logger
local WFLogger = require 'WFLogger'
local NmbTrackedDataBroadcaster = require 'NmbTrackedDataBroadcaster'

	-- WF.core.utils
local WFArguments = require 'WFArguments'
local KSData = require 'KSData'
local WFFolderWalker = require 'WFFolderWalker'
local WFRxRunUntilDone = require 'WFRxRunUntilDone'
local WFIdle = require 'WFIdle'
local WFFileUtils = require 'WFFileUtils'
local KSPathUtils = require 'KSPathUtils'

	-- WF.oz_client.managed_catalog
local WFCatDelete = require 'WFCatDelete'

--------------------------------------------------------------------------------

local check = assert( WFArguments.check )
KSData.registerArgumentChecker()

local runUntilDone = WFRxRunUntilDone

local logger = WFLogger 'NmbLeakedFileDetector'

local kStandard = 'standard'
local kArchived = 'archived'
local kNeedle = '::'

--============================================================================--

-- @mark NmbLeakedFileDetector namespace

local NmbLeakedFileDetector = {}

--------------------------------------------------------------------------------

local function createKey( volumeGuid, relativePath )

	if volumeGuid and relativePath then

		return string.format( "%s%s%s", volumeGuid, kNeedle, relativePath )

	end
	
end

--------------------------------------------------------------------------------

local function findUntrackedFiles( resultStream, queryType, indexStore, binaryType, untrackedBinaries, offset )

	check( 'table', 'string', 'WFRxIndexStore', 'string', 'table', '?number' )

	-- The first time this is called, untrackedBinaries contains every single file
	-- in the specified parent directory. This function calls itself recursively
	-- and will remove any tracked files from untrackedBinaries. When this work is
	-- complete, resultStream fires with the remaining untracked binaries.

	if next( untrackedBinaries ) == nil then
	
		-- There are no untracked binaries. We're done!
	
		resultStream:onNext( false )
		resultStream:onCompleted()
	
		return
	
	end

	local throttleSize = 500
	
	offset = offset or 0

	logger:infof( "findUntrackedFiles %s size: %d, offset %d.", queryType, throttleSize, offset )

	local trackedAssetsQuery

	if queryType == kStandard then

		local selectStatement = 'SELECT localId, originalVolumeGuid, originalRelativePath '
		local whereClause = ' AND originalVolumeGuid IS NOT NULL AND originalRelativePath IS NOT NULL '
		
		if binaryType == 'proxy' then
		
			selectStatement = 'SELECT localId, proxyVolumeGuid, proxyRelativePath '
			whereClause = ' AND proxyVolumeGuid IS NOT NULL AND proxyRelativePath IS NOT NULL '
			
		end

		trackedAssetsQuery = indexStore:queryOnce {
			sql = selectStatement ..
					[[FROM assetPathsAndInfo
					WHERE localId > ?]]
					.. whereClause ..
					[[ORDER BY localId ASC
					LIMIT ?]], 
				offset,
				throttleSize,
			quiesceUntilUpToDate = true
		}

	else -- kArchived

		trackedAssetsQuery = indexStore:queryOnce {
			sql = [[SELECT localId, archivedVolumeGuid, archivedRelativePath
					FROM archivedFiles
					WHERE localId > ?
					AND binaryType == ?
					ORDER BY localId ASC
					LIMIT ?]],
				offset,
				binaryType,
				throttleSize,
			quiesceUntilUpToDate = true,
		}
	
	end
	
	trackedAssetsQuery
	
		:map( function( rows )
		
			for _, row in ipairs( rows ) do
			
				local localId = row[ 1 ]
				local volumeGuid = row[ 2 ]
				local relativePath = row[ 3 ]

				local key = createKey( volumeGuid, relativePath )
	
				-- Since this file _is_ tracked, remove it from the list of untracked
				-- binaries.
	
				if key then
					untrackedBinaries[ key ] = nil
				end
				
				offset = localId
	
			end
			
			return #rows, offset

		end )
		
		:tap( function( numberOfRows, newOffset )
		
			if numberOfRows > 0 then
			
				WFIdle.doLater( function()
					findUntrackedFiles( resultStream, queryType, indexStore, binaryType, untrackedBinaries, newOffset )
				end )
				
			else
		
				-- No more work to do, we're done!
			
				resultStream:onNext( untrackedBinaries )
				resultStream:onCompleted()
			
			end
		
		end )

		:catch( function( err )

			logger:warnf( "findUntrackedFiles failed: %s", err )

			resultStream:onNext( false )
			resultStream:onCompleted()

			return Rx.Observable.just( {} )

		end )

		:call( runUntilDone )

end

--------------------------------------------------------------------------------

local function walkFolders( pathsToRootBinaryFolders, binaryType )

	check( 'table' )

	local localBinaries = {}

	local expectedFolderName

	if binaryType == 'original' then
		expectedFolderName = 'originals'
	else
		expectedFolderName = 'proxies'
	end

	for _, folder in ipairs( pathsToRootBinaryFolders ) do

		-- If the path does not end in the expected folder name, don't process
		-- those assets. This is more of a security check to prevent someone from
		-- calling this function on a bad path and wiping away some unrelated
		-- directory.

		if KSPathUtils.leafName( folder ) == expectedFolderName then

			logger:infof( "Finding all assets in %s", folder ) 

			local volumeGuid

			for path in WFFolderWalker( folder ) do
			
				volumeGuid = volumeGuid or WFFileUtils.getVolumeGuidForPath( path )
				local relativePath = WFFileUtils.removeVolumeFromPath( path )
			
				local key = createKey( volumeGuid, relativePath )
				
				if key then
					localBinaries[ key ] = true
				end
				
			end
		
		end
	
	end
	
	return localBinaries
	
end

--------------------------------------------------------------------------------

local function leakCheck( indexStore, binaryType, pathsToRootBinaryFolders, performDelete )

	local resultStream = Rx.Subject()

	logger:infof( "Checking local %s files...", binaryType ) 

	local localBinaries = walkFolders( pathsToRootBinaryFolders, binaryType )
		-- TODO [jandrew 2017-10-24]: Should maybe move this to a different thread?

	findUntrackedFiles( resultStream, kStandard, indexStore, binaryType, localBinaries )
		-- Rule out binaries that are tracked in the doc store.
		
	return resultStream
	
		:flatMapLatest( function( untrackedBinaries )
		
			if untrackedBinaries then
			
				local resultStream2 = Rx.Subject()
				
				findUntrackedFiles( resultStream2, kArchived, indexStore, binaryType, untrackedBinaries )
					-- Rule out binaries that are archived in the doc store.
					
				return resultStream2
				
			else
			
				return Rx.Observable.just( {} )
			
			end
			
		end )
		
		:map( function( untrackedBinaries )
		
			local untrackedCount = 0
		
			untrackedBinaries = untrackedBinaries or {}
		
			for key, _ in pairs( untrackedBinaries ) do	
					
				if performDelete then	
						
					local volumeGuid, relativePath = key:match( "(.+)" .. kNeedle .. "(.+)" )
					
					if volumeGuid and relativePath then
						WFCatDelete.deleteAssetAndMaybeParentFolders( volumeGuid, relativePath )
					end

				end

				untrackedCount = untrackedCount + 1

			end
		
			if performDelete then
				logger:infof( "Found and deleted %d untracked %s files!", untrackedCount, binaryType )
			else
				logger:infof( "Found %d untracked %s files!", untrackedCount, binaryType )
			end
			
			local leakedFileStats = {
				binaryType = binaryType,
				numberOfLeakedFiles = untrackedCount,
			}
			
			NmbTrackedDataBroadcaster.broadcastMessage( 'LeakedFileStats', leakedFileStats )
			
			return leakedFileStats
			
		end )
		
		:tapOnCompleted( function()
		
			logger:infof( "NmbLeakedFileDetector completed for %s files.", binaryType ) 
		
		end )
		
end
		
--------------------------------------------------------------------------------
		
function NmbLeakedFileDetector.cleanUpLeakedFiles( indexStore, binaryType, pathsToRootBinaryFolders )

	check( 'WFRxIndexStore', 'string', 'table' )

	return leakCheck( indexStore, binaryType, pathsToRootBinaryFolders, true )
		
		:takeLast( 0 )
		
		:call( runUntilDone )

end

--------------------------------------------------------------------------------
		
function NmbLeakedFileDetector.checkForLeakedFiles( indexStore, binaryType, pathsToRootBinaryFolders )

	check( 'WFRxIndexStore', 'string', 'table' )

	return leakCheck( indexStore, binaryType, pathsToRootBinaryFolders, false )

end

--------------------------------------------------------------------------------

return NmbLeakedFileDetector
