--[[----------------------------------------------------------------------------

PixelProviderExternalXmpDevelopDownloadTask.lua

--------------------------------------------------------------------------------

ADOBE CONFIDENTIAL
------------------
Copyright 2022 Adobe
All Rights Reserved.

NOTICE: All information contained herein is, and remains
the property of Adobe and its suppliers, if any. The intellectual
and technical concepts contained herein are proprietary to Adobe
and its suppliers and are protected by all applicable intellectual
property laws, including trade secret and copyright laws.
Dissemination of this information or reproduction of this material
is strictly forbidden unless prior written permission is obtained
from Adobe.
------------------------------------------------------------------------------]]

	-- Rx
local Rx = require "Rx"

	-- WF.core.utils
local WFFunctionUtils = require 'WFFunctionUtils'
local KSPathUtils = require "KSPathUtils"
local KSTableUtils = require "KSTableUtils"
local WFFileUtils = require 'WFFileUtils'
local WFAsyncFileUtils = require 'WFAsyncFileUtils'

	-- WF.oz_client.managed_catalog.pixel_provider
local WFCatAssetAux = require 'WFCatAssetAux'

	-- pixel_provider
local PixelPathProviderStateMachine = require 'PixelPathProviderStateMachine'
local NmbDocValueToRenderInfo = require 'NmbDocValueToRenderInfo'
local PixelPathProviderSchedulerPrivateMethods =
		require "PixelPathProviderSchedulerPrivateMethods"

--------------------------------------------------------------------------------

local logger = require 'WFLogger'( 'PixelProviderLocalBinaryPathCheckTask' )

--------------------------------------------------------------------------------

local function reduceT( acc, f, t, ... )

	for k, v in pairs( t ) do

		acc = f( acc, k, v, ... )

	end

	return acc

end

--------------------------------------------------------------------------------

local function countT( t )

	return reduceT( 0, function( count ) return count + 1 end, t )

end

--------------------------------------------------------------------------------

local _auxFileHttpDownloadOptions = {
		requestHeaders = {
			{
				header = "Content-Type",
				value = "application/octet-stream",
			}
		}
	}

--------------------------------------------------------------------------------

local function makeSingleResultStreamWithDefault( stream, defaultVal )

	return stream
			:startWith( defaultVal )
			:takeLast( 1 )

end

--------------------------------------------------------------------------------

local function ensureParentFolderExists( filePath )

	local parentPath = KSPathUtils.parent( filePath )

	return pcall( WFFileUtils.mkdir, parentPath )

end

--------------------------------------------------------------------------------

local function makeSaveBlobToPathResultStream( 
				saveToFileFunc, makeParentFolderFunc, localAssetId, data,
				filePath )

	if not data then

		return Rx.Observable.just( false )

	end

	local ok, maybeErr = makeParentFolderFunc( filePath ) 

	if not ok then

		logger:warnf( "ensureParentFolderExists( %s ) failed: %s", 
						filePath, maybeErr )

		return Rx.Observable.just( false )

	end

	local rawSaveFileResultStream =
			Rx.Observable.concat(
				-- ignore any results, WFAsyncFileUtils.saveToFile will
				-- throw if it fails
				saveToFileFunc( data, filePath ):takeLast( 0 ),
				Rx.Observable.just( true ) )

	local saveFileResultStream =
			rawSaveFileResultStream
				:catch(
					function( saveError, platformError )

						logger:warnf( "auxFile for assetId %s. saveError = %s",
										localAssetId, saveError )

						logger:warnf( "auxFile for assetId %s. platformError = %s",
										localAssetId, platformError )

						return Rx.Observable.just( false )

					end )

	return saveFileResultStream

end

--------------------------------------------------------------------------------
	
local _staleDataException = {}

--------------------------------------------------------------------------------

local function areExternalDevelopFilesChanged( 
				baseRenderInfoCache, localAssetId, origExternalDevelopFiles )

	local currBaseRenderInfo = baseRenderInfoCache:peekCacheItem( localAssetId )

	if not currBaseRenderInfo then

		return true

	end

	return not NmbDocValueToRenderInfo.areExternalDevelopFilesEqual(
				currBaseRenderInfo.externalDevelopFiles,
				origExternalDevelopFiles )
	
end

--------------------------------------------------------------------------------

local function makeFileSaveResultStream(
				uri, filePath, wfCatalog, saveToFileFunc, makeParentFolderFunc,
				baseRenderInfoCache, localAssetId, origExternalDevelopFiles )

	if not uri then

		return Rx.Observable.just( false )

	end

	local blobStream =
			wfCatalog
				:ozCatalogSpec()
				:getBinary( uri, _auxFileHttpDownloadOptions )
				:catch(
					function( errStr )

						logger:warnf( "unexpected exception with getBinary( %s ) for localAssetId %s: %s",
										uri, localAssetId, errStr )

						return Rx.Observable.just( nil )

					end )


	local rawFileSaveResultStream =
			blobStream	
				:flatMap(
					function( blob )

						if areExternalDevelopFilesChanged( 
								baseRenderInfoCache, localAssetId,
								origExternalDevelopFiles ) then

							return Rx.Observable.throw( _staleDataException )

						end

						return makeSaveBlobToPathResultStream( 
								saveToFileFunc, makeParentFolderFunc,
								localAssetId, blob, filePath )

					end )

	local fileSaveResultStream =
			makeSingleResultStreamWithDefault( rawFileSaveResultStream, false )

	return fileSaveResultStream

end

--------------------------------------------------------------------------------

local function getDeleteAuxFileOverlay( digest )

	local deleteOverlayForAuxFile = {}
	
	if not KSTableUtils.setNested( deleteOverlayForAuxFile, 
			{ _delete = true }, 
			'_localOnly', 'files', 'aux', digest ) then

		return {}

	end

	return deleteOverlayForAuxFile

end

--------------------------------------------------------------------------------
-- makeAuxFileSyncTaskResultStream emits exactly one tuple of form:
--
-- 	auxFileExists, auxFileLocalFileOverlay

local function makeAuxFileSyncTaskResultStream( 
				missingAuxFileInfo, wfCatalog, saveToFileFunc,
				makeParentFolderFunc, baseRenderInfoCache, localAssetId,
				origExternalDevelopFiles )

	local digest, uri, filePath, relativePath = unpack( missingAuxFileInfo, 1, 4 )

	local fileSaveResultStream =
			makeFileSaveResultStream(
				uri, filePath, wfCatalog, saveToFileFunc, makeParentFolderFunc,
				baseRenderInfoCache, localAssetId, origExternalDevelopFiles )

	-- fileSaveOverlayResultStream emits a single result tuple:
	-- 	true|false, fileOverlay
	local fileSaveOverlayResultStream =
			fileSaveResultStream
				:flatMap(
					function( didSave )

						if areExternalDevelopFilesChanged( 
								baseRenderInfoCache, localAssetId,
								origExternalDevelopFiles ) then

							return Rx.Observable.throw( _staleDataException )

						end

						local fileOverlay = 
								didSave and
									WFCatAssetAux.createAuxLocalOnlyOverlay( 
										digest, relativePath ) or
									getDeleteAuxFileOverlay( digest )

						return Rx.Observable.just( didSave, fileOverlay )

					end )

	return fileSaveOverlayResultStream

end

--------------------------------------------------------------------------------

local function makeApplyOverlayResultStream( docStore, localAssetId, overlay )

	if not next( overlay ) then

		return Rx.Observable.empty()

	end

	overlay._id = localAssetId

	return docStore
				:applyOverlay( overlay )
				:catch(
					function( errStr )

						logger:warnf( "error applyOverlay( %s ): %s",
										localAssetId, errStr )

						return Rx.Observable.empty()

					end )

end

--------------------------------------------------------------------------------

local function makeExternalDevelopFileSyncTasksResultStream( 
				wfCatalog, saveToFileFunc, makeParentFolderFunc,
				baseRenderInfoCache, localAssetId, missingDevelopAuxFiles,
				origExternalDevelopFiles )

	local auxFileSyncTaskResultStreams = 
		WFFunctionUtils.map(
			makeAuxFileSyncTaskResultStream,
				missingDevelopAuxFiles, wfCatalog, saveToFileFunc, makeParentFolderFunc,
				baseRenderInfoCache, localAssetId, origExternalDevelopFiles )

	-- rawAllAuxFileSyncTaskResultsStream fires off all the sync tasks
	-- at once and combines the results into a stream of tuples
	--
	-- 	auxFile1Exists, auxFile1Overlay
	-- 	auxFile2Exists, auxFile2Overlay
	-- 	...
	-- 	auxFileNExists, auxFileNOverlay
	local auxFileExistsOverlayTupleStream =
			Rx.Observable.from( auxFileSyncTaskResultStreams )
				:flatMergeAll()

	-- :scan() plus :takeLast( 1 ) combines all of the
	-- auxFileExistsOverlayTupleStream tuples into a single result tuple stream
	--   allAuxFilesExist, combinedFileOverlay 
	--
	local combinedOverlayTupleStream = 
			auxFileExistsOverlayTupleStream
			:scan( 
				function( currAllAuxFilesExist, currCombinedAuxFileOverlay, auxFileExists, auxFileOverlay )

					local allAuxFilesExist = currAllAuxFilesExist and auxFileExists

					local combinedAuxFileOverlay =
							KSTableUtils.merge( 
								{}, 
								currCombinedAuxFileOverlay, 
								auxFileOverlay )

					return allAuxFilesExist, combinedAuxFileOverlay

				end, 
				true, {} )
			:takeLast( 1 )

	-- Given a single tuple stream emit a single state machine transition
	--
	-- This may have the side effect of updating the docValue _localOnly
	-- section, but that update is not required to succeed
	local stateMachineTransitionStream =
			combinedOverlayTupleStream
				:flatMap(
					function( allAuxFilesExist, combinedAuxFileOverlay )

						local applyOverlayResultStream = 
								makeApplyOverlayResultStream( 
									wfCatalog:docStore(), localAssetId, 
									combinedAuxFileOverlay )


						return Rx.Observable.concat(
								-- :takeLast( 0 ) means run to completion
								-- ignoring all results
								applyOverlayResultStream:takeLast( 0 ),
								Rx.Observable.create( 
									function( observer )

										-- check one last time for stale data
										-- after overlays applied

										if areExternalDevelopFilesChanged( 
												baseRenderInfoCache, localAssetId,
												origExternalDevelopFiles ) then

											observer:onError( _staleDataException )

											return

										end

										-- otherwise, we are done 
										local transition = 
												allAuxFilesExist and
													'all_external_develop_xmp_are_downloaded' or
													'failed_to_download_one_or_more_external_develop_uri'

										observer:onNext( transition )
										observer:onCompleted()

									end ) )

					end )
									

	return stateMachineTransitionStream
			:catch(
				function( ... )

					local err = ...

					-- for stale data, return the correct state machine transition
					if err == _staleDataException then

						return Rx.Observable.just( 'external_develop_xmp_data_changed' )

					end

					-- otherwise re-throw
					return Rx.Observable.throw( ... )

				end )

end

--------------------------------------------------------------------------------

local PixelPathProviderSchedulerPrivateMethods_maybeApplyTransitionForLocalAssetId =
		assert( PixelPathProviderSchedulerPrivateMethods.maybeApplyTransitionForLocalAssetId,
				"missing PixelPathProviderSchedulerPrivateMethods.maybeApplyTransitionForLocalAssetId" )

--------------------------------------------------------------------------------

local function startAuxFileSyncTaskForRequest( 
				externalDevelopFileSyncTasksPerAssetId, request, scheduler,
				wfCatalog, saveToFileFunc, makeParentFolderFunc,
				baseRenderInfoCache, needDevelopAuxFileDownloadCache )

	local pixStreamId, localAssetId = unpack( request )

	local needAuxFileDownloadInfo =
			needDevelopAuxFileDownloadCache
				:peekCacheItem( localAssetId )

	if not needAuxFileDownloadInfo or 
			not needAuxFileDownloadInfo.externalDevelopFiles or
			#needAuxFileDownloadInfo.externalDevelopFiles == 0 or
			not needAuxFileDownloadInfo.missingDevelopAuxFiles or
			#needAuxFileDownloadInfo.missingDevelopAuxFiles == 0 then

		logger:warnf( "Request %s:%s is missing needAuxFileDownloadInfo",
						pixStreamId, localAssetId )

		scheduler:_applyTransitionToRequest( 
			request, 'missing_need_xmp_develop_download_cache_data' )

		return externalDevelopFileSyncTasksPerAssetId

	end

	if externalDevelopFileSyncTasksPerAssetId[ localAssetId ] then

		-- an existing request is already happening, apply start transition to
		-- this one so it waits for the same results as the other request
		scheduler:_applyTransitionToRequest( 
			request, 'start_xmp_develop_download' )

		return externalDevelopFileSyncTasksPerAssetId

	end

	-- set up an external develop file sync task for this localAssetId
	externalDevelopFileSyncTasksPerAssetId[ localAssetId ] =
			makeExternalDevelopFileSyncTasksResultStream( 
				wfCatalog, saveToFileFunc, makeParentFolderFunc,
				baseRenderInfoCache, localAssetId,
				needAuxFileDownloadInfo.missingDevelopAuxFiles,
				needAuxFileDownloadInfo.externalDevelopFiles )
			:subscribe(
				function( externalDevelopFileSyncResultTransition )

					PixelPathProviderSchedulerPrivateMethods_maybeApplyTransitionForLocalAssetId(
						scheduler, localAssetId,
						externalDevelopFileSyncResultTransition )

				end,
				function( errStr )

					logger:warnf( "unexpected exception syncing external develop files for %s: %s",
									localAssetId, errStr )

					PixelPathProviderSchedulerPrivateMethods_maybeApplyTransitionForLocalAssetId(
						scheduler, localAssetId, 
						'unexpected_external_develop_file_check_exception' )

					externalDevelopFileSyncTasksPerAssetId[ localAssetId ] = nil
					
				end,
				function()

					externalDevelopFileSyncTasksPerAssetId[ localAssetId ] = nil

				end)

	scheduler:_applyTransitionToRequest( 
			request, 'start_xmp_develop_download' )

	return externalDevelopFileSyncTasksPerAssetId

end

--------------------------------------------------------------------------------

local _maxActiveExternalDevelopFileSyncTasks = 4

--------------------------------------------------------------------------------

local _needExternalDevelopAuxFileSyncStates = {

	PixelPathProviderStateMachine.DownloadStates.NeedExternalDevelopAuxFilesDownload, 
	PixelPathProviderStateMachine.DownloadStates.RenderNeedExternalDevelopAuxFilesDownload, 

}

--------------------------------------------------------------------------------

local function maybeApplyStartDownloadTransition( 
				request, scheduler, externalDevelopFileSyncTasksPerAssetId )

	local localAssetId = unpack( request )

	if not externalDevelopFileSyncTasksPerAssetId[ localAssetId ] then

		return

	end

	scheduler
		:_applyTransitionToRequest( request, 'start_xmp_develop_download' )

end

--------------------------------------------------------------------------------

local function processRequests( scheduler, taskState, baseRenderInfoCache,
									needDevelopAuxFileDownloadCache )

	repeat 

		-- first join any incoming requests to inflight downloads
		scheduler
			:_mapAllRequestsWithStates( _needExternalDevelopAuxFileSyncStates,
				maybeApplyStartDownloadTransition,
					scheduler,
					taskState.externalDevelopFileSyncTasksPerAssetId )

		-- now start scheduling new downloads if we can
		local unusedExternalDevelopFileSyncTasks =
			_maxActiveExternalDevelopFileSyncTasks - 
				countT( taskState.externalDevelopFileSyncTasksPerAssetId )
					
		if unusedExternalDevelopFileSyncTasks <= 0 then

			return

		end
		
		local request =
			{ PixelPathProviderSchedulerPrivateMethods.getNextRequestWithState( 
						scheduler,  unpack( _needExternalDevelopAuxFileSyncStates ) ) }

		if not next( request ) then

			return

		end

		taskState.externalDevelopFileSyncTasksPerAssetId =
			startAuxFileSyncTaskForRequest(
				taskState.externalDevelopFileSyncTasksPerAssetId, request,
				scheduler, taskState.wfCatalog, taskState.saveToFileFunc,
				taskState.makeParentFolderFunc, baseRenderInfoCache,
				needDevelopAuxFileDownloadCache )

	until false
	
end

--------------------------------------------------------------------------------

local function setupTask( wfCatalog )

	return {

		wfCatalog = wfCatalog,

		-- parametize saveToFileFunc so we can emulate failures via unit
		-- test taskState manipulation
		saveToFileFunc = WFAsyncFileUtils.saveToFile,

		-- parametize makeParentFolderFunc so we can emulate failures via unit
		-- test taskState manipulation
		makeParentFolderFunc = ensureParentFolderExists,

		externalDevelopFileSyncTasksPerAssetId = {}
		
	}

end

--------------------------------------------------------------------------------

local function cleanup( taskState )

	for _, subscr in pairs( taskState.externalDevelopFileSyncTasksPerAssetId ) do

		subscr:dispose()

	end

	taskState.externalDevelopFileSyncTasksPerAssetId = {}

end

--==============================================================================

return {

	processRequests = processRequests,

	setupTask = setupTask,

	cleanup = cleanup,

	privateUnitTestExport = {

		needExternalDevelopAuxFileSyncStates = _needExternalDevelopAuxFileSyncStates,

		maxActiveExternalDevelopFileSyncTasks = 
			_maxActiveExternalDevelopFileSyncTasks,

	}

}
