--[[----------------------------------------------------------------------------

PixelProviderLocalBinaryPathCheckTask.lua

--------------------------------------------------------------------------------

ADOBE CONFIDENTIAL
------------------
Copyright 2016 Adobe Systems Incorporated
All Rights Reserved.

NOTICE: All information contained herein is, and remain the property of
Adobe Systems Incorporated and its suppliers, if any. The intellectual and
technical concepts contained herein are proprietary to Adobe Systems
Incorporated and its suppliers and may be covered by U.S. and Foreign
Patents, patents in process, and are protected by trade secret or copyright
law. Dissemination of this information or reproduction of this material is
strictly forbidden unless prior written permission is obtained from
Adobe Systems Incorporated.

------------------------------------------------------------------------------]]

local WFConfiguration = require 'WFConfiguration'

	-- Rx
local Rx = require "Rx"
local Observable = Rx.Observable

	-- WF.core.utils
local WFRxRunUntilDone = require 'WFRxRunUntilDone'
local KSDate = require 'KSDate'
local KSTableUtils = require 'KSTableUtils'
local WFFileUtils = require 'WFFileUtils'
local WFCrypto = require 'WFCrypto'

	-- WF.oz_client.utils
local WFOzDocUtils = require 'WFOzDocUtils'
local WFCatFileSpec = require 'WFCatFileSpec'

	-- WF.oz_client.managed_catalog
local WFCatBinaryUploader = require 'WFCatBinaryUploader'

	-- WF.oz_client.managed_catalog.pixel_provider
local PixelPathProviderStateMachine = require 'PixelPathProviderStateMachine'
local PixelProviderSharedTaskUtils = require 'PixelProviderSharedTaskUtils'
local PixelProviderDownloadToFileUtils = require 'PixelProviderDownloadToFileUtils'
local WFCatPixelFileCacheCheck = require 'WFCatPixelFileCacheCheck'
local NmbPathRestorer = require 'NmbPathRestorer'
local PixelPathProviderSchedulerPrivateMethods =
		require "PixelPathProviderSchedulerPrivateMethods"

--------------------------------------------------------------------------------

local DISABLE_DOWNLOAD_TO_FILE = WFConfiguration.PixelProvider.enableDownloadToFile == false

local ENABLE_DOWNLOAD_TO_FILE_FOR_ALL_FILE_TYPES = 
		WFConfiguration.PixelProvider.enableDownloadToFile == "all"

--------------------------------------------------------------------------------

local logger = require 'WFLogger'( 'PixelProviderLocalBinaryPathCheckTask' )

local runUntilDone = WFRxRunUntilDone

local KSTableUtils_derefOpt = KSTableUtils.derefOpt

--------------------------------------------------------------------------------

local LocalBinaryPathCheckStates = 
		PixelPathProviderStateMachine.LocalBinaryPathCheckStates

--------------------------------------------------------------------------------

local function deleteLocalFileAnnotation( docStore, localId, binaryType )

	-- SPAM [jandrew 8/18/2016]:
	-- The doc store annotation was innaccurate, so we should
	-- remove the annotation. This allows us to redownload
	-- the asset as needed.
	
	-- TODO: Move this annotation deletion to a new task. Something like:
	-- PixelProviderPixelClearLocalBinaryPathInDocStoreTask
	
	local deleteFilesArray = {}
	deleteFilesArray[ binaryType ] = { _delete = true }

	return docStore:applyOverlay(
			{	
				_id = localId,
				_localOnly = {  
					files = deleteFilesArray
				}
			} )
								
end

--------------------------------------------------------------------------------

local function errorLocalFileAnnotation( docStore, localId )

	return docStore:applyOverlay(
			{	
				_id = localId,
				_localOnly = {  
					localOriginalError = 'InvalidLocalOriginal'
				}
			} )
								
end

--------------------------------------------------------------------------------

local function getNeedDownloadTransition( isNonVideo, binaryType )

	local isVideo = not isNonVideo

	local fileTypeUsesDownloadToFile = 
			isVideo or ENABLE_DOWNLOAD_TO_FILE_FOR_ALL_FILE_TYPES

	local binaryTypeUsesDownloadToFile = 
			( binaryType == 'proxy' or binaryType == 'original' )

	local expectDownloadToFileTransition =
			not DISABLE_DOWNLOAD_TO_FILE and
			fileTypeUsesDownloadToFile and
			binaryTypeUsesDownloadToFile

	local downloadToFileTransition =
			expectDownloadToFileTransition and
				PixelProviderDownloadToFileUtils.maybeGetNeedDownloadTransitionForBinaryType( 
					binaryType ) or
				nil

	if downloadToFileTransition then

		return downloadToFileTransition

	end

	if expectDownloadToFileTransition then

		logger:errorf( "Don't know how to download to file for binaryType %s for %svideo file", 
						binaryType, isNonVideo and "" or "non-" )

		-- try other forms of downloading before giving up

	end

	if binaryType == 'proxy' then

		return 'need_download_proxy'

	end

	if binaryType == 'original' then

		return 'need_download_original'

	end

	if binaryType == 'xmp_sidecar' then

		return 'need_download_xmp_sidecar'

	end

	logger:errorf( "Don't know how to download binaryType %s", binaryType )

	return 'cannot_download_binary_type'

end

--------------------------------------------------------------------------------

local function createNeedsDownloadHandler( scheduler, canDownloadBinary, binaryType, 
											ozBinarySubtype, pixStreamId, localAssetId, 
											state, isNonVideo )

	return function()
		
		if canDownloadBinary then

			logger:infof( "Negative not found locally for %s|%s, but there is a download URI", binaryType, ozBinarySubtype )

			scheduler
				:_maybeSetNextState( 
					pixStreamId, localAssetId, 
					getNeedDownloadTransition( isNonVideo, binaryType ) )

			return

		end
			
		logger:infof( "Negative not found locally, nor do we have a download link for %s|%s", 
				binaryType, ozBinarySubtype )
				
		local transition = PixelPathProviderSchedulerPrivateMethods.stateRespondsToTransition( scheduler, state, 'no_downloadable_orig_uri' ) 
						   and 'no_downloadable_orig_uri' or 'no_downloadable_uri'

		scheduler:_maybeSetNextState( pixStreamId, localAssetId, transition )

	end

end

--------------------------------------------------------------------------------

local function createErrorHandler( streamName, scheduler, pixStreamId, localAssetId )

	return function( errStr )

		logger:errorf( "error in %s: %s", streamName, errStr )
			
		scheduler:_maybeSetNextState( pixStreamId, localAssetId, 'no_downloadable_uri' )
		
	end
		
end

--------------------------------------------------------------------------------

local _videoProxyOzSubtypesArray = { "1080p", "1080p709", "720p", "360p" }

--------------------------------------------------------------------------------

local _videoProxyOzSubtypes = KSTableUtils.mapArrayToSet( 
								_videoProxyOzSubtypesArray )

--------------------------------------------------------------------------------

local function maybeGetOzSubtypeFromPreferredSize( preferredSize )

	if not _videoProxyOzSubtypes[ preferredSize ] then

		return

	end

	return preferredSize

end

--------------------------------------------------------------------------------

local function checkForBinaryAndMaybeDeleteAnnotation( docStore, cacheChecker, maybeBinaryPath, 
										localAssetId, binaryType )

	if not maybeBinaryPath then
		return Observable.just( false )
	end

	return cacheChecker:checkPath( maybeBinaryPath )
							
		:flatMapLatest( function( localPathExists, localPathIsOnOfflineVolume )

			if localPathExists then
				return Observable.just( maybeBinaryPath )
			end

			if localPathIsOnOfflineVolume then
				return Observable.just( false, 'offline' )
			end

			-- must either validate we have correct video proxy path or if volume is offline
			-- need docValue in either case.

			return docStore:docValue( localAssetId )
			
				:flatMapLatest( function( docValue )
					
					-- If the local path doesn't exist, the asset may be on an offline volume.
					-- If that's the case, don't delete the annotation.

					if binaryType == 'original' then

						-- If the asset is not synced yet, then we should produce an error but keep the
						-- annotation visible

						local availableOnOz = WFOzDocUtils.isValidOriginalAvailableOnOz( docValue )

						-- If asset is from LrClassic then we don't have the original locally or on Oz.
						local willNotUploadMaster = KSTableUtils.derefOpt( docValue, 'importSource', 'willNotUploadMaster' )

						if not availableOnOz and not WFCatBinaryUploader.canUploadOriginalViaCopy( docValue ) then

							if not willNotUploadMaster then

								return errorLocalFileAnnotation( docStore, localAssetId, binaryType )
									:map( function()
										return false
									end )

							else

								return Observable.just( false )

							end

						end

					elseif binaryType == 'proxy' then

						local availableOnOz = WFOzDocUtils.isProxyAvailableOnOz( docValue )

						if not availableOnOz then

							-- If the asset is not synced yet, then we should produce an error but keep the
							-- annotation visible

							return Observable.just( false )

						end

					end

					return deleteLocalFileAnnotation( docStore, localAssetId, binaryType )
						:map( function()
							return false
						end )

				end )

		end )

end

--------------------------------------------------------------------------------

local function WFCryptoAsyncDigestFile( pathToFile )

	return WFCrypto.asyncDigestFile( pathToFile, 'SHA256' )

end

--------------------------------------------------------------------------------

local _overrideSha256DigestFuncForTest

--------------------------------------------------------------------------------

local function getSha256Digest( pathToFile )

	local digestFunc = _overrideSha256DigestFuncForTest or 
							WFCryptoAsyncDigestFile

	return digestFunc( pathToFile )

end

--------------------------------------------------------------------------------

local function checkExpectedPathForOriginal( ozUserId, docStore, cacheChecker, fileSpecObj, localAssetId )

	-- The following checks to see if the original already exists in the expected
	-- location but is untracked by our system. This can happen if the user
	-- manually copies the originals folder to a new machine.

	return docStore:docValue( localAssetId )
	
		:flatMapLatest( function( docValue )

			local captureDate = docValue.captureDate
			
			captureDate = type( captureDate ) == 'string' and 
							KSDate.isKSDate( captureDate ) and
							captureDate or 
							'0000-00-00T00:00:00.000' -- Unknown date
			
			local fileName = KSTableUtils_derefOpt( docValue, 'importSource', 'fileName' )
			
			if type( fileName ) ~= "string" then
				return Observable.just( false )
			end
			
			local sha256 = KSTableUtils_derefOpt( docValue, 'importSource', 'sha256' )
			
			local expectedPath = fileSpecObj:pathForOriginal( fileName, captureDate, true, true )

			local importedBy = KSTableUtils_derefOpt( docValue, 'importSource', 'importedBy' )
			local spaceId = KSTableUtils_derefOpt( docValue, '_spaceId' )

			-- Don't try to upload an original if it was imported by a different
			-- user as a proxy based photo from LrClassic.
			local wasImportedByThisUser = not spaceId or importedBy == ozUserId

			if not expectedPath
				or not sha256
				or not wasImportedByThisUser then
				
				return Observable.just( false )
				
			end

			logger:infof( "Checking for untracked negative for asset %d at path: %s", localAssetId, expectedPath )

			return cacheChecker:checkPath( expectedPath )
				:map( function( localPathExists )
					return expectedPath, localPathExists, sha256
				end )
				
		end )
		
		:flatMapLatest( function( pathToFile, localPathExists, sha256 )
		
			if not pathToFile 
				or not localPathExists 
				or not sha256 then
				
				return Observable.just( false )
				
			end
		
			logger:infof( "An untracked negative exists for asset %d at path: %s", localAssetId, pathToFile )
		
			return getSha256Digest( pathToFile, 'SHA256' )
				:map( function( digest )
					return pathToFile, digest == sha256 and sha256
				end )
		
		end )
		
		:flatMapLatest( function( pathToFile, maybeSha256 )
		
			if not pathToFile 
				or not maybeSha256 then
			
				logger:infof( "The untracked negative does not match for asset %d", localAssetId )
				return Observable.just( false )
				
			end
		
			logger:infof( "The untracked negative is a match for asset %d", localAssetId )
		
			local volumeGuid = WFFileUtils.getVolumeGuidForPath( pathToFile )
			
			local relativeFilePath = WFFileUtils.removeVolumeFromPath( pathToFile )
		
			if volumeGuid
				and type( volumeGuid ) == 'string'
				and relativeFilePath
				and type( relativeFilePath ) == 'string' then
		
				local binarySizes = {}
			
				binarySizes[ 'master' ] = {
				
					relativePath = relativeFilePath,
					volumeGuid = volumeGuid,
					sha256 = maybeSha256
					
				}
				
				local files = {}
				files[ 'original' ] = binarySizes

				return docStore:applyOverlay(
						{	
							_id = localAssetId,
							_localOnly = {  
								files = files
							}
						} )
						
					:map( function()
					
						return pathToFile
						
					end )
		
			end

			return Observable.just( false )

		end )
		
end

--------------------------------------------------------------------------------

local PixelProviderSharedTaskUtils_getOriginalDownloadInfoFromRenderInfo =
		assert( require "PixelProviderSharedTaskUtils".getOriginalDownloadInfoFromRenderInfo,
			"PixelProviderSharedTaskUtils.getOriginalDownloadInfoFromRenderInfo is missing!" )

--------------------------------------------------------------------------------

local PixelProviderSharedTaskUtils_renderInfoHasDownloadableProxy =
		assert( PixelProviderSharedTaskUtils.renderInfoHasDownloadableProxy,
				"missing PixelProviderSharedTaskUtils.renderInfoHasDownloadableProxy" )

--------------------------------------------------------------------------------

local function makeNoLocalNegativeHandler( scheduler, baseRenderInfoCache,
											pixStreamId, localAssetId, 
											isNonVideo )
	
	return function()

		local baseRenderInfo = baseRenderInfoCache:peekCacheItem( localAssetId )
		
		if type( baseRenderInfo ) ~= "table" then

			logger:warnf( "Can't check for downloadable local negative path %s.%s: no baseRenderInfo",
							pixStreamId, localAssetId )

			scheduler:_maybeSetNextState( pixStreamId, localAssetId, 'need_docstore_info' )

			return

		end
		
		local noLocalBinaryTransition =
				( PixelProviderSharedTaskUtils_renderInfoHasDownloadableProxy( 
							baseRenderInfo ) and
						getNeedDownloadTransition( isNonVideo, 'proxy' ) ) or
				( PixelProviderSharedTaskUtils_getOriginalDownloadInfoFromRenderInfo( 
							baseRenderInfo ) and
					getNeedDownloadTransition( isNonVideo, 'original' ) ) or
				'no_downloadable_uri'

		scheduler:_maybeSetNextState( pixStreamId, localAssetId, noLocalBinaryTransition )

	end

end

--------------------------------------------------------------------------------

local function isTrue( v )

	return v == true

end

--------------------------------------------------------------------------------

local PixelProviderSharedTaskUtils_calculateLocalProxyPath =
		assert( PixelProviderSharedTaskUtils.calculateLocalProxyPath,
				"missing PixelProviderSharedTaskUtils.calculateLocalProxyPath" )

--------------------------------------------------------------------------------

local PixelProviderSharedTaskUtils_calculateLocalOriginalPath =
		assert( PixelProviderSharedTaskUtils.calculateLocalOriginalPath,
				"missing PixelProviderSharedTaskUtils.calculateLocalOriginalPath" )

--------------------------------------------------------------------------------

local PixelProviderSharedTaskUtils_calculateLocalXmpSidecarPath =
		assert( PixelProviderSharedTaskUtils.calculateLocalXmpSidecarPath,
				"missing PixelProviderSharedTaskUtils.calculateLocalXmpSidecarPath" )

--------------------------------------------------------------------------------

local PixelProviderSharedTaskUtils_maybeGetBinarySubtypeForProxy =
		assert( PixelProviderSharedTaskUtils.maybeGetBinarySubtypeForProxy,
				"missing PixelProviderSharedTaskUtils.maybeGetBinarySubtypeForProxy" )

--------------------------------------------------------------------------------

local function handleLocalFileExistsImp( scheduler, localBinaryPathCache,
										pixStreamId, localAssetId,
										localNegativePath, fileType )

	localBinaryPathCache:pushCacheItem( 
		pixStreamId, localAssetId, 
		{ localNegativePath, fileType } )

	-- must call _maybeSetNextState instead of _applyTransitionToRequest because
	-- this is a thread callback and the request data may have changed
	scheduler
		:_maybeSetNextState( 
			pixStreamId, localAssetId, 'has_local' )

end

--------------------------------------------------------------------------------

local PixelProviderSharedTaskUtils_getValidOriginalDownloadInfoFromRenderInfo =
		assert( require "PixelProviderSharedTaskUtils".getValidOriginalDownloadInfoFromRenderInfo,
			"PixelProviderSharedTaskUtils.getValidOriginalDownloadInfoFromRenderInfo is missing!" )

--------------------------------------------------------------------------------

local PixelProviderSharedTaskUtils_isKnownInvalidSha256 =
		assert( require "PixelProviderSharedTaskUtils".isKnownInvalidSha256,
			"PixelProviderSharedTaskUtils.isKnownInvalidSha256 is missing!" )

--------------------------------------------------------------------------------

local function handleLocalOriginalFileExists( 
				scheduler, baseRenderInfoCache, localBinaryPathCache,
				pixStreamId, localAssetId, isNonVideo, localNegativePath, fileType,
				maybeLocalOriginalSha256 )
	
	-- fetch the baseRenderInfo again in case it changed while we were
	-- checking file existence on worker thread
	local baseRenderInfo = baseRenderInfoCache:peekCacheItem( localAssetId ) or {}

	local maybeOzValidOriginalLink = 
			PixelProviderSharedTaskUtils_getValidOriginalDownloadInfoFromRenderInfo( 
				baseRenderInfo )

	local maybeCurrLocalOriginalSha256 =
			KSTableUtils.derefOpt( baseRenderInfo, "localOriginalSha256" )

	-- 1) Is the local known to be invalid but there is a valid Oz link?
	if PixelProviderSharedTaskUtils_isKnownInvalidSha256( 
			maybeLocalOriginalSha256 ) and maybeOzValidOriginalLink then

		-- try a download even though the original exists
		scheduler
			:_maybeSetNextState( 
				pixStreamId, localAssetId, 
				getNeedDownloadTransition( isNonVideo, 'original' ) ) 

		return

	end

	-- 2) Do we know both the local sha256 and the current sha256 and that
	--    they mismatch? If so, is there a valid original link on Oz?
	if maybeLocalOriginalSha256 and maybeCurrLocalOriginalSha256 and
			maybeLocalOriginalSha256 ~= maybeCurrLocalOriginalSha256 and
			maybeOzValidOriginalLink then

		-- try a download even though the original exists
		scheduler
			:_maybeSetNextState( 
				pixStreamId, localAssetId, 
				getNeedDownloadTransition( isNonVideo, 'original' ) ) 

		return

	end

	-- 3) Otherwise, proceed as normal
	handleLocalFileExistsImp( 
		scheduler, localBinaryPathCache, pixStreamId, localAssetId,
		localNegativePath, fileType )

end

--------------------------------------------------------------------------------

local function handleLocalFileExists( scheduler, baseRenderInfoCache,
										localBinaryPathCache, pixStreamId,
										localAssetId, isNonVideo, localNegativePath,
										fileType, maybeLocalOriginalSha256 )

	if fileType == "original" then

		handleLocalOriginalFileExists( 
			scheduler, baseRenderInfoCache, localBinaryPathCache, pixStreamId,
			localAssetId, isNonVideo, localNegativePath, fileType,
			maybeLocalOriginalSha256 )

		return

	end

	handleLocalFileExistsImp( 
		scheduler, localBinaryPathCache, pixStreamId, localAssetId,
		localNegativePath, fileType )

end

--------------------------------------------------------------------------------

local function calculatedBinaryPathExistsStream( cacheChecker, localBinaryPath )

	return cacheChecker
			:checkPath( localBinaryPath )
			:map( 
				function( localPathExists, localPathIsOnOfflineVolume )

					if localPathExists then

						return localBinaryPath

					end

					if localPathIsOnOfflineVolume then

						return false, 'offline'

					end

					return false

				end )

end

--------------------------------------------------------------------------------
-- makeLocalBinaryExistsStreamImp

local function makeLocalBinaryExistsStreamImp( 
					ozUserId, docStore, fileChecker, fileSpecObj, localAssetId,
					initialMaybeDocValueLocalOnlyBinaryPath, binaryType,
					ozBinarySubtype )


	-- defines a stream that will handle all boolean permutations of
	--
	-- 		"has local binary file" and
	-- 		"docStore _localOnly files section exists"
	--
	-- return value is:
	--
	-- 	false|existingLocalBinaryPath, missingLocalPathReason
	--
	--
	-- As a side effect, this function also ensures the docValue._localOnly section
	-- is correct before returning 
	return docStore
			:docValue( localAssetId )
			:map(
				function( docValue )

					if docValue._deleted then
						return false
					end

					-- Calculate the calculatedLocalBinaryPath. We expect non-nil for the correct binaryType
					local maybeCalculatedLocalBinaryPath =
						( binaryType == 'proxy' and ozBinarySubtype and
							PixelProviderSharedTaskUtils_calculateLocalProxyPath( 
								fileSpecObj, docValue._id, ozBinarySubtype ) ) or
						( binaryType == 'original' and
							PixelProviderSharedTaskUtils_calculateLocalOriginalPath( 
								fileSpecObj, docValue ) ) or
						( binaryType == 'xmp_sidecar' and
							PixelProviderSharedTaskUtils_calculateLocalXmpSidecarPath( 
								fileSpecObj, docValue._id ) )

					-- maybeDocValueLocalOnlyBinaryPath is basically
					--
					-- derefOpt( docValue, "_localOnly", "files", binaryType, ozBinarySubtype, "relativePath )
					-- 
					-- i.e. it will be either nil or a path that the LrD app recorded
					-- when saving the file.
					local maybeDocValueLocalOnlyBinaryPath = 
						initialMaybeDocValueLocalOnlyBinaryPath or
						( binaryType == 'proxy' and ozBinarySubtype and
							WFCatFileSpec.buildProxyPathFromDocValue( 
								docValue, ozBinarySubtype ) ) or
						( binaryType == 'original' and
							WFCatFileSpec.buildOriginalPathFromDocValue( 
								docValue ) ) or
						( binaryType == 'xmp_sidecar' and
							WFCatFileSpec.getXmpSidecarPathFromDocValue( 
								docValue ) )

					return maybeCalculatedLocalBinaryPath, maybeDocValueLocalOnlyBinaryPath

				end )
			:flatMapLatest(
				function( maybeCalculatedLocalBinaryPath, maybeDocValueLocalOnlyBinaryPath )

					if not maybeCalculatedLocalBinaryPath and not maybeDocValueLocalOnlyBinaryPath then

						-- if neither paths exist, it's fatal. Upstream code is
						-- responsible for avoiding this state.
						logger:warnf( "Cannot check local binary path for type %s and subtype %s for localAssetId %s with docValue path %s",
										binaryType, ozBinarySubtype, localAssetId,
										initialMaybeDocValueLocalOnlyBinaryPath )

						return Rx.Observable.throw( 
									string.format( "Failed to get local binary path for type %s and subtype %s for localAssetId %s",
														binaryType, ozBinarySubtype, localAssetId ) )

					end

					local docValueLocalOnlyBinaryPathExistsStream =
							not maybeDocValueLocalOnlyBinaryPath and
								Rx.Observable.just ( false ) or
								-- check the docValue path and clear it if needed
								checkForBinaryAndMaybeDeleteAnnotation( 
										docStore, fileChecker, maybeDocValueLocalOnlyBinaryPath,
										localAssetId, binaryType )

					local localBinaryPathExistsStream =
							-- maybe follow up with a calculatedLocalBinaryPath look up
							docValueLocalOnlyBinaryPathExistsStream
								:take( 1 )
								:flatMap(
									function( existingDocValueLocalOnlyBinaryPath, missingDocValueLocalOnlyBinaryPathReason )

										if existingDocValueLocalOnlyBinaryPath then

											-- maybeDocValueLocalOnlyBinaryPath exists, return the path
											return Rx.Observable.just( existingDocValueLocalOnlyBinaryPath )

										end

										if missingDocValueLocalOnlyBinaryPathReason == 'offline' then

											-- skip checking the calculated path when docValue path
											-- is known to be offline. We do this so we can trigger 
											-- code to archive the local path
											return Rx.Observable.just( false, 'offline' )

										end

										if not maybeCalculatedLocalBinaryPath then

											logger:warnf( "Weird local binary state for %s.%s for assetId %s: has docValue path, but no calculatedPath",
															binaryType, ozBinarySubtype, localAssetId )

											-- this edge case might be impossible, but not worth throwing
											return Rx.Observable.just( false )

										end

										-- try checking the calculated path instead 
										return calculatedBinaryPathExistsStream( 
													fileChecker,
													maybeCalculatedLocalBinaryPath )
												:take( 1 )

									end )

					return localBinaryPathExistsStream 
							:flatMap(
								function( maybeLocalBinaryPath, missingLocalPathReason )

									if not maybeLocalBinaryPath then

										-- This covers the case where the local file doesn't exist
										--
										-- unless the local path is known to be offline volume, 
										-- the docValue no longer contains the path at this point

										return Rx.Observable.just( false, missingLocalPathReason )

									end

									local localBinaryPath = maybeLocalBinaryPath

									if binaryType == 'xmp_sidecar' then

										-- sidecars don't handle fixup
										return Rx.Observable.just( localBinaryPath )

									end

									if localBinaryPath == maybeDocValueLocalOnlyBinaryPath then

										-- This covers the case where the docValueLocalOnlyPath exists.
										--
										-- This should be very common for synced assets
										return Rx.Observable.just( localBinaryPath )

									end

									-- handle the final edge case which seems rare but is 
									-- occuring in the wild:
									--
									-- 		we have an existing local proxy or orig path but 
									-- 		somehow the docValueLocalOnlyPath
									-- 		is missing or mismatched
									--
									-- Must update the docStore here so docValue state contains the 
									-- existing local binary path.

									if binaryType == 'original' then

										-- Originals file names are made unique whenever the original
										-- is downloaded (or imported). Don't adopt the path without
										-- checking the sha256 hash of the local file against the doc.
										return checkExpectedPathForOriginal( ozUserId, docStore, fileChecker, fileSpecObj, localAssetId )

									end

									local localBinaryPathDocstoreUpdateCompletionStream =
										PixelProviderSharedTaskUtils.updateAssetDocValueLocalOnlyNegativePathResultStream( 
												docStore, localAssetId, binaryType, ozBinarySubtype,
												localBinaryPath )
											:catch(
												function( errStr )

													-- don't make docstore updates fatal here, just log
													-- log them
													logger:warnf( "failed to update docstore _localOnly proxy path for %s of size %s to path %s: errStr",
																		localAssetId, ozBinarySubtype, localBinaryPath, errStr )

													return Rx.Observable.empty()

												end )
											-- use takeLast( 0 ) to make the stream completion only
											:takeLast( 0 )

										-- This concat will only emit one value:
										--
										-- 		localBinaryPath
										--
										-- which is known to exist
										return Rx.Observable.concat(
												localBinaryPathDocstoreUpdateCompletionStream,
												Rx.Observable.just( localBinaryPath ) )

								end ) 

							end )

end

--------------------------------------------------------------------------------

local function makeLocalBinaryExistsStream( ozUserId, docStore, fileChecker, fileSpecObj,
												localAssetId, binaryType,
												ozBinarySubtype )

	return makeLocalBinaryExistsStreamImp( 
			ozUserId, docStore, fileChecker, fileSpecObj, localAssetId, nil,
			binaryType, ozBinarySubtype )

end

--------------------------------------------------------------------------------

local function makeLocalBinaryExistsStreamForPath( 
				ozUserId, docStore, fileChecker, fileSpecObj, localAssetId,
				docValueLocalOnlyBinaryPath, binaryType, 
				ozBinarySubtype )

	return makeLocalBinaryExistsStreamImp( 
			ozUserId, docStore, fileChecker, fileSpecObj, localAssetId,
			docValueLocalOnlyBinaryPath, binaryType, ozBinarySubtype )

end

--------------------------------------------------------------------------------

local function getOzBinaryProxySubtypeForSizeHint(
					baseRenderInfoCache, localAssetId, requestSizeHint )

	-- shared method for getting a valid ozBinarySubtype given the pixel
	-- provider request longSide field

	local maybeOzSubtype = maybeGetOzSubtypeFromPreferredSize( requestSizeHint )

	local ozBinarySubtype =
		PixelProviderSharedTaskUtils_maybeGetBinarySubtypeForProxy( 
			baseRenderInfoCache, localAssetId, maybeOzSubtype )

	return ozBinarySubtype

end

--------------------------------------------------------------------------------

local function checkForAnyValidLocalNegativePath( scheduler, ozUserId, docStore, fileChecker,
													fileSpecObj,
													localBinaryPathCache,
													baseRenderInfoCache,
													request )

	local pixStreamId, localAssetId, requestSizeHint = unpack( request )

	local maybeOzProxyBinarySubtype =
			getOzBinaryProxySubtypeForSizeHint( 
				baseRenderInfoCache, localAssetId, requestSizeHint )

	scheduler:_applyTransitionToRequest( request, 'start_non_rendition_check' )
	
	local baseRenderInfo = baseRenderInfoCache:peekCacheItem( localAssetId )
	
	if type( baseRenderInfo ) ~= "table" then

		logger:warnf( "Can't check for local negative path %s.%s: no baseRenderInfo",
						pixStreamId, localAssetId )

		return scheduler:_applyTransitionToRequest( request, 'need_docstore_info' )

	end

	local isNonVideo = not baseRenderInfo.isVideo or false
	
	local onNoLocalNegativeHandler = makeNoLocalNegativeHandler(
										scheduler, baseRenderInfoCache,
										pixStreamId, localAssetId, isNonVideo )

	local onErrorHandler = createErrorHandler( "checkForAnyValidLocalNegativePath", scheduler, pixStreamId, localAssetId )

	local localProxyPathCheckResultStream = 
		maybeOzProxyBinarySubtype and
			makeLocalBinaryExistsStream( 
				ozUserId, docStore, fileChecker, fileSpecObj, localAssetId, 'proxy',
				maybeOzProxyBinarySubtype ) or
			-- if there is no binary subtype for the proxy, it cannot be
			-- downloaded. local proxies are exclusively downloaded from
			-- the cloud. therefore, if there is no download link, there
			-- will not be a local proxy and we should return an observable
			-- that emits false immediately.
			Observable.just( false )

	local maybeLocalOriginalSha256 = baseRenderInfo.localOriginalSha256

	local localOriginalPathCheckResultStream = 
			baseRenderInfo.originalPath and
				makeLocalBinaryExistsStreamForPath( 
					ozUserId, docStore, fileChecker, fileSpecObj, localAssetId,
					baseRenderInfo.originalPath, 
					'original', 'master' ) or
				makeLocalBinaryExistsStream( 
					ozUserId, docStore, fileChecker, fileSpecObj, localAssetId,
					'original', 'master' )

	local checkExpectedPathForOriginalStream = checkExpectedPathForOriginal( ozUserId, docStore, fileChecker, fileSpecObj, localAssetId )
	
	-- Check if there's a local proxy.
	return localProxyPathCheckResultStream
			:flatMapLatest( 

				function( maybeLocalProxyPath )
					
					if not maybeLocalProxyPath then
						
						return localOriginalPathCheckResultStream
								:map( 
									function( maybeLocalOriginalPath )

										return nil, maybeLocalOriginalPath

									end )
					
					end

					return Observable.just( maybeLocalProxyPath )
					
				end )
				
			:flatMapLatest( function( maybeLocalProxyPath, maybeLocalOriginalPath )
			
				if not maybeLocalProxyPath and not maybeLocalOriginalPath then
				
					return checkExpectedPathForOriginalStream
						:map( function( maybeExpectedLocalOriginalPath )
							return maybeLocalProxyPath, maybeExpectedLocalOriginalPath
						end )
				
				end
			
				return Observable.just( maybeLocalProxyPath, maybeLocalOriginalPath )
			
			end )
				
			:map( 

				function( maybeLocalProxyPath, maybeLocalOriginalPath )

					if not scheduler:_hasPixstreamRequestForLocalAssetId(
											pixStreamId, localAssetId ) then

						return false

					end

					local localNegativePath = 
							maybeLocalProxyPath or maybeLocalOriginalPath 

					if not localNegativePath then

						return true

					end

					local localNegativeType = 
							maybeLocalProxyPath and 'proxy' or 'original'

					handleLocalFileExists( 
						scheduler, baseRenderInfoCache, localBinaryPathCache,
						pixStreamId, localAssetId, isNonVideo, localNegativePath,
						localNegativeType, maybeLocalOriginalSha256  )

					return false
				
				end )
			:filter( isTrue )
			:tap( onNoLocalNegativeHandler, onErrorHandler ) 
			:call( runUntilDone ) 
		
end

--------------------------------------------------------------------------------

local function checkForSpecificValidLocalPath( 
				scheduler, request, ozUserId, docStore, fileChecker, fileSpecObj,
				localBinaryPathCache, baseRenderInfoCache )

	-- requestSizeHint is only valid for some requests
	local pixStreamId, localAssetId, requestSizeHint, _, state = unpack( request )

	local binaryType =
			( ( state == LocalBinaryPathCheckStates.NeedCheckForLocalOriginal or
					state == LocalBinaryPathCheckStates.NeedCheckOriginalForBestPossibleLocalNegativePath or
					state == LocalBinaryPathCheckStates.NeedCheckOriginalOrFallbackProxyLocalNegative ) and 
						'original' ) or
			( ( state == LocalBinaryPathCheckStates.NeedCheckForLocalProxy or
					state == LocalBinaryPathCheckStates.NeedCheckLocalProxyForRender or
					state == LocalBinaryPathCheckStates.NeedCheckProxyForBestPossibleLocalNegativePath ) and 
						'proxy' ) or
			( ( state == LocalBinaryPathCheckStates.NeedCheckForLocalXmpSidecar ) and
						'xmp_sidecar' ) or
			nil
	
	if not binaryType then
	
		logger:errorf( "needsCheckForLocal state '%s' not recognized, will download original",
						state )

		scheduler:applyTransitionToRequest( request, 'invalid_check_specific_local_path_state' )

		return
	
	end

	scheduler:_applyTransitionToRequest( request, 'start_non_rendition_check' )
		
	local baseRenderInfo = baseRenderInfoCache:peekCacheItem( localAssetId )
	
	if type( baseRenderInfo ) ~= "table" then

		logger:warnf( "Can't check for local negative path %s.%s: no baseRenderInfo",
						pixStreamId, localAssetId )

		return scheduler:_applyTransitionToRequest( request, 'need_docstore_info' )

	end

	local isNonVideo = not baseRenderInfo.isVideo or false
		
	local maybeLocalPath 	
	local canDownloadBinary 
	local maybeLocalOriginalSha256
	local ozBinarySubtype
	
	if binaryType == 'xmp_sidecar' then

		-- Note: baseRenderInfo.xmpSidecarExists == nil means we don't know if there is a sidecar file or not yet

		maybeLocalPath = baseRenderInfo.xmpSidecarPath
		canDownloadBinary = ( baseRenderInfo.xmpSidecarExists or baseRenderInfo.xmpSidecarExists == nil ) and baseRenderInfo.ozXmpLink
		ozBinarySubtype = 'xmp_sidecar'

	elseif binaryType == 'original' then
		maybeLocalPath = baseRenderInfo.originalPath
		canDownloadBinary = PixelProviderSharedTaskUtils_getOriginalDownloadInfoFromRenderInfo( baseRenderInfo )
		maybeLocalOriginalSha256 = baseRenderInfo.localOriginalSha256
		ozBinarySubtype = 'master'

	elseif binaryType == 'proxy' then

		canDownloadBinary = PixelProviderSharedTaskUtils_renderInfoHasDownloadableProxy( baseRenderInfo )

		if baseRenderInfo.proxyPath and baseRenderInfo.proxyPathOzBinarySubtype == baseRenderInfo.ozProxySize then

			maybeLocalPath = baseRenderInfo.proxyPath
			ozBinarySubtype = baseRenderInfo.proxyPathOzBinarySubtype

		else

			ozBinarySubtype = 
				getOzBinaryProxySubtypeForSizeHint(
					baseRenderInfoCache, localAssetId, requestSizeHint )

		end

		if not maybeLocalPath and not ozBinarySubtype then

			-- if there is local path then we must download. But no
			-- ozBinarySubtype for a proxy means we can't download 
			--
			-- In practice this occurs when there is no local proxy and no
			-- proxy links on Oz, which in turn can happen when there is an
			-- invalid master on Oz.
			return scheduler:_applyTransitionToRequest( request, 'no_downloadable_uri' )

		end

	end

	local onNeedsDownloadHandler = createNeedsDownloadHandler( scheduler, canDownloadBinary, binaryType, 
																ozBinarySubtype, pixStreamId, localAssetId, 
																state, isNonVideo )
	
	local onErrorHandler = createErrorHandler( "checkForSpecificValidLocalPath", scheduler, pixStreamId, localAssetId )
	
	local checkExpectedPathForOriginalStream = checkExpectedPathForOriginal( ozUserId, docStore, fileChecker, fileSpecObj, localAssetId )
	
	local rawLocalBinaryPathExistsStream =
			maybeLocalPath and
				makeLocalBinaryExistsStreamForPath( ozUserId, docStore, fileChecker, fileSpecObj, localAssetId, maybeLocalPath, binaryType, ozBinarySubtype ) or
				makeLocalBinaryExistsStream( ozUserId, docStore, fileChecker, fileSpecObj, localAssetId, binaryType, ozBinarySubtype )

	local maybeLocalBinaryPathStream = 
			rawLocalBinaryPathExistsStream
				:flatMapLatest(
					function( maybeExistingLocalPath, missingLocalPathReason )

						if maybeExistingLocalPath then

							return Rx.Observable.just( maybeExistingLocalPath )

						end

						if missingLocalPathReason ~= 'offline' then

							return Rx.Observable.just( false )

						end

						-- at this point we know:
						--
						-- 	docValue has a local binary path
						-- 	the binary path is on an unmounted volume
						--
						-- rather that clear the docValue local binary path, we archive it


						if not ozBinarySubtype then

							logger:warnf( "Offline storage tracking not updated for asset %s binaryType %s due to missing binarySize",
											localAssetId, binaryType )
					
							return Rx.Observable.just( false )

						end

						-- Looking up a specific existing local binary path
						-- has failed. When path is known to be offline,
						-- update docValue to move curr local binary path
						-- info to "archive" field.
						--
			
						-- TODO 20230208 bkruse
						--
						-- I suspect this causes notification storms,
						-- especially with code that is checking the sync
						-- status of proxies. The cache checker could
						-- likely check the entire catalog in under a
						-- minute, meaning every single asset would be
						-- updated if previously docStore had offline
						-- volume proxy paths for all assets.
						--
						-- We'd be better off deferring this update until
						-- after we actually download and replace the
						-- offline local proxy path with the calculated
						-- local proxy path.
						return Rx.Observable.concat(
								NmbPathRestorer.archiveLocalFilePath( 
										docStore, localAssetId, binaryType,
										ozBinarySubtype )
									:takeLast( 0 ),
								Rx.Observable.just( false ) )

					end )

	return maybeLocalBinaryPathStream

		:flatMapLatest( function( maybeLocalBinaryPath )
			
			if not maybeLocalBinaryPath and binaryType == 'original' then
			
				return checkExpectedPathForOriginalStream
			
			end
		
			return Observable.just( maybeLocalBinaryPath )
		
		end )

		:flatMapLatest( function( maybeLocalBinaryPath )
		
			if maybeLocalBinaryPath then

				if not scheduler:_hasPixstreamRequestForLocalAssetId(
										pixStreamId, localAssetId ) then

					return Observable.empty()

				end

				handleLocalFileExists( 
					scheduler, baseRenderInfoCache, localBinaryPathCache,
					pixStreamId, localAssetId, isNonVideo, maybeLocalBinaryPath,
					binaryType, maybeLocalOriginalSha256 )

				return Observable.empty()

			end
	
			return Observable.just( true )
	
		end )

		:filter( function( maybeNeedsDownload )
			return maybeNeedsDownload
		end )
		
		:tap( onNeedsDownloadHandler, onErrorHandler )
			
		:call( runUntilDone )

end

--------------------------------------------------------------------------------

local _needLocalBinaryCheckStates = {
		LocalBinaryPathCheckStates.NeedCheckForLocalOriginal,
		LocalBinaryPathCheckStates.NeedCheckForLocalProxy,
		LocalBinaryPathCheckStates.NeedCheckForAnyLocalNegative,
		LocalBinaryPathCheckStates.NeedCheckAnyLocalNegativeForRender,
		LocalBinaryPathCheckStates.NeedCheckLocalProxyForRender,
		LocalBinaryPathCheckStates.NeedCheckOriginalForBestPossibleLocalNegativePath,
		LocalBinaryPathCheckStates.NeedCheckProxyForBestPossibleLocalNegativePath,
		LocalBinaryPathCheckStates.NeedCheckForLocalXmpSidecar,
		LocalBinaryPathCheckStates.NeedCheckOriginalOrFallbackProxyLocalNegative
		}

--------------------------------------------------------------------------------

local function processRequests( scheduler, taskState, localBinaryPathCache, baseRenderInfoCache )
	
	repeat
		
		local request =
			{ PixelPathProviderSchedulerPrivateMethods.getNextRequestWithState( 
					scheduler,  unpack( _needLocalBinaryCheckStates ) ) }
		
		local pixStreamId, _, _, _, state = unpack( request )

		if not pixStreamId then

			break

		end
		
		
		if state == LocalBinaryPathCheckStates.NeedCheckForAnyLocalNegative or
				state == LocalBinaryPathCheckStates.NeedCheckAnyLocalNegativeForRender then

			checkForAnyValidLocalNegativePath( 
				scheduler, taskState.ozUserId, taskState.docStore, taskState.fileChecker,
				taskState.fileSpecObj, localBinaryPathCache, baseRenderInfoCache, 
				request )
		else

			checkForSpecificValidLocalPath( 
				scheduler, request, taskState.ozUserId, taskState.docStore, taskState.fileChecker,
				taskState.fileSpecObj, localBinaryPathCache,
				baseRenderInfoCache )

		end
		
	until false
	
end

--------------------------------------------------------------------------------

local function setupTask( ozUserId, docStore, pixelsPath, fileSpecObj )

	return {

		ozUserId = ozUserId,

		docStore = docStore,
		
		fileChecker = WFCatPixelFileCacheCheck( pixelsPath ),
		
		fileSpecObj = fileSpecObj,
		
	}

end

--==============================================================================

return {

	processRequests = processRequests,

	setupTask = setupTask,
	
	checkExpectedPathForOriginal = checkExpectedPathForOriginal, -- Exposed for tests.

	checkForBinaryAndMaybeDeleteAnnotation = checkForBinaryAndMaybeDeleteAnnotation, -- Exposed for tests.

	privateUnitTestExports = {
		setSha256OverrideFunc = 
			function( f ) _overrideSha256DigestFuncForTest = f end
	}

}
