--[[----------------------------------------------------------------------------

NmbSimpleDocStoreUpdater.lua

--------------------------------------------------------------------------------

ADOBE CONFIDENTIAL
------------------
Copyright 2016 Adobe Systems Incorporated
All Rights Reserved.

NOTICE: All information contained herein is, and remains the property of
Adobe Systems Incorporated and its suppliers, if any. The intellectual and
technical concepts contained herein are proprietary to Adobe Systems
Incorporated and its suppliers and are protected by all applicable
intellectual property laws, including trade secret and copyright laws.
Dissemination of this information or reproduction of this material is
strictly forbidden unless prior written permission is obtained from
Adobe Systems Incorporated.

------------------------------------------------------------------------------]]

	-- RxLua
local Rx = require 'Rx'
local Observable = Rx.Observable

	-- WF.core
local KSDate = require 'KSDate'
local KSStringUtils = require 'KSStringUtils'
local KSTableUtils = require 'KSTableUtils'
local WFArguments = require 'WFArguments'
local WFAsyncFileUtils = require 'WFAsyncFileUtils'
local WFCloseSupport = require 'WFCloseSupport'
local WFCatIndexUtils = require 'WFCatIndexUtils'
local WFCatOzErrors = require 'WFCatOzErrors'
local WFDeviceInfo = require 'WFDeviceInfo'
local WFDiagnosticLog = require 'WFDiagnosticLog'
local WFFileUtils = require 'WFFileUtils'
local WFPlatformInfo = require 'WFPlatformInfo'
local WFProgressReporter = require 'WFProgressReporter'
local WFCrypto = require 'WFCrypto'
local WFLogger = require 'WFLogger'
local WFZString = require 'WFZString'

	-- WF.core.rxlua
local WFRxRunUntilDone = require 'WFRxRunUntilDone'

	-- WF.oz_client
local WFCatFileSpec = require 'WFCatFileSpec'

--------------------------------------------------------------------------------

local derefOpt = assert( KSTableUtils.derefOpt )
local check = assert( WFArguments.check )
local ZSTR = assert( WFZString.fromZString )
local runUntilDone = WFRxRunUntilDone

local kDelimiter = ","

local kSimpleUpdateFunctions = {}
local kObservableUpdateFunctions = {}

local kKeywordKey = 'keywords'
local kBinaryPathsKey = 'binaryPaths'
local kStorageExceededFixup = 'storageExceeded'
local kDeviceNameInvalidUTF8Fixup = 'invalidUtf8InDeviceName'
local kAlbumNameInvalidUTF8Fixup = 'invalidUtf8InAlbumName'
local kFalseInvalidOriginalFix = 'falseInvalidOriginalFix'
local kNoSha256Fix = 'noSha256Fix'
local kAssetNameInvalidUTF8Fixup = 'invalidUtf8InAssetName'
local kXmpDevelopSha256Fix = 'invalidXmpDevelopSha256'
local kNumFixesMade = 'numOverlaysApplied'
local kNegativeOriginalFileSize = 'negativeOriginalFileSize'
local kFalseInvalidLocalOriginalFix = 'falseInvalidLocalOriginalFix'

local logger = WFLogger( 'NmbSimpleDocStoreUpdater' )

--------------------------------------------------------------------------------

local function toBool( zeroOrOne )

	return zeroOrOne > 0

end

--------------------------------------------------------------------------------

local function docValueHasLocalOriginal( docValue )

	local localOriginalVolumeGuid = WFCatFileSpec.getOriginalVolumeGuidFromDocValue( docValue )
	local localOriginalRelativePath = WFCatFileSpec.getOriginalRelativePathFromDocValue( docValue )
	return localOriginalVolumeGuid and localOriginalRelativePath

end

--------------------------------------------------------------------------------

local function docValueHasFalsePositiveInvalidLocalOriginalError( docValue )

	local maybeLocalOriginalError = 
		KSTableUtils.derefOpt( docValue, '_localOnly', 'localOriginalError' ) 

	if maybeLocalOriginalError ~= 'InvalidLocalOriginal' then

		-- asset doesn't have InvalidLocalOriginal error, so it's not a false
		-- positive
		return false

	end

	-- at this point, the asset has an InvalidLocalOriginal error, so we need
	-- to check if it's a false positive

	if docValueHasLocalOriginal( docValue ) then

		-- if there is a local original path stored in the docValue, then we
		-- assume the InvalidLocalOriginal error is valid and thus also NOT a
		-- false positive
		return false
	
	end

	-- on the other hand, if there is no local original path stored in the
	-- docValue, then it doesn't make sense to have state InvalidLocalOriginal,
	-- so we assume it's a false positive
	return true

end

--------------------------------------------------------------------------------

local function simple_updates_table( )

	return {

		sql = [[CREATE TABLE simple_updates( updates, hasBlockingUpdate )]],

		relevantTypeSubtypeTuples = {
			{ '*', '*' } -- All types & subtypes
		},

		map = function( docValue, emit )
			
			local updates = {}
			
			local hasBlockingUpdate = 0
			
			local isValidAssetDocument = WFCatIndexUtils.isValidAssetDocument( docValue )
			
			if docValue.type == 'asset' and docValue._deleted ~= true then
				
				-- keywords used to be in payload.keywords, but were moved
				-- into xmp.dc.subject as part of NMB-1555. We need to 
				-- migrate any existing data from the old location to the new
				
				local maybeKeywords = docValue.keywords
				
				if ( type( maybeKeywords ) == 'table' ) and 
					next( maybeKeywords ) then
					
					table.insert( updates, kKeywordKey )
					
				end
				
				-- We used to store absolute original and proxy paths. We now store them as
				-- relative paths, along with volume GUIDs. We need to update any
				-- asset with an aboslute path to this new format.
				
				local maybeAbsoluteOriginalPath = WFCatFileSpec.getAbsoluteOriginalPathFromDocValue( docValue )
				local maybeAbsoluteProxyPath = WFCatFileSpec.getAbsoluteProxyPathFromDocValue( docValue )
				
				if maybeAbsoluteOriginalPath or maybeAbsoluteProxyPath then
				
					table.insert( updates, kBinaryPathsKey )
					hasBlockingUpdate = 1
					
				end
				
			end
			
			-- NMB-6449: document is completely empty other than _spm and _localOnly (and other _ fields) because we botched a StorageExceededError 
			-- from another client case.  If it's still in a StorageExceededError state because the user still doesn't have room, don't try to fix
			-- it up yet because we don't have anything to fix it up with.
			if docValue.type == nil and not docValue._deleted and not WFCatOzErrors.hasStorageExceededError( docValue )
			 	and not ( KSTableUtils.derefOpt( docValue, '_localOnly', 'localError', 'nmb6449fix' ) == 'failed' ) then
				table.insert( updates, kStorageExceededFixup )
			end
			
			-- NMB-7338: document has invalid UTF-8 string for either importedOnDevice or changedOnDevice and therefore
			-- won't sync or allow any other documents to sync.
			
			if isValidAssetDocument then
	
				local importedOnDevice = KSTableUtils.derefOpt( docValue, 'importSource', 'importedOnDevice' )
				local changedOnDevice = KSTableUtils.derefOpt( docValue, 'changedOnDevice' )
				
				if ( importedOnDevice and not KSStringUtils.validateUTF8( importedOnDevice ) ) or 
					( changedOnDevice and not KSStringUtils.validateUTF8( changedOnDevice ) ) then
				
					table.insert( updates, kDeviceNameInvalidUTF8Fixup )
				end
				
			end
			
			-- NMB-7339: For a while, album name truncation could cause invalid UTF-8 strings.
			
			if docValue and docValue.type == 'album' and not docValue._deleted then
	
				local albumName = docValue.name
				
				if ( albumName and not KSStringUtils.validateUTF8( albumName ) ) then
				
					table.insert( updates, kAlbumNameInvalidUTF8Fixup )
					
				end
				
			end
			
			-- WF-393: Fix up invalid UTF-8 preset names and other asset documents.
			
			if docValue and docValue.type == 'asset' and not docValue._deleted then
	
				local assetName = docValue.name
					-- Fix up the name fields for assets with invalid UTF8 names. Asset
					-- documents with a name field include camera profiles, lens profiles,
					-- presets, person, etc.
					
				if ( assetName and not KSStringUtils.validateUTF8( assetName ) ) then
				
					table.insert( updates, kAssetNameInvalidUTF8Fixup )
					
				end
				
			end
			
			-- NMB-7768: for a period of time, assets were being incorrectly marked as having invalid originals
			-- if an external volume containing the original was disconnected while an upload was in progress.
			
			local filePath = WFCatFileSpec.buildOriginalPathFromDocValue( docValue )
			local ozMasterUrl = WFCatFileSpec.getOriginalLinkFromDocValue( docValue )
			
			if isValidAssetDocument then
			
				local localOriginalError = KSTableUtils.derefOpt( docValue, '_localOnly', 'localOriginalError' )
				local hasLocalOriginalAnnotation = docValueHasLocalOriginal( docValue )
				
				if localOriginalError == 'InvalidOriginal' 
					or localOriginalError == 'UploadFailedOriginalMissing' then
				
					if hasLocalOriginalAnnotation and filePath then
				
						table.insert( updates, kFalseInvalidOriginalFix )
					
					elseif not hasLocalOriginalAnnotation and ozMasterUrl then
					
						table.insert( updates, kFalseInvalidOriginalFix )
						
					end
					
				end
				
			end
			
			-- NMB-10355: For a period of time we were failing to add SHA256 to assets
			-- that were first synced via Classic, and then imported into LrCC.
			-- Part 2 of the fix for NMB-10355 can be found within WFCatStaleBinaryRemover.lua
			
			if isValidAssetDocument then
			
				local localSha256 = WFCatFileSpec.getLocalOriginalSha256( docValue )
				
				if filePath and not localSha256 then
				
					local isMarkedForOffline = WFCatFileSpec.getMarkedForOfflineUseFromDocValue( docValue )

					-- If the no-sha256-original is not on Oz, or it's marked for offline use, then
					-- calculate the SHA256 and add it to the payload.
					
					if not ozMasterUrl or isMarkedForOffline then
				
						table.insert( updates, kNoSha256Fix )
						
					end
				
				end
				
			end

			-- NMB-18125: Fix up xmpCameraRaw sha256 for assets with develop settings > 4MB
			-- caused by an issue tha occurred in 2021
			-- see https://git.corp.adobe.com/lua/wf/pull/2857

			if isValidAssetDocument then

				-- Only recalculate develop sha if the userUpdated isn't set or it is < 2022 as the
				-- fix for this went in September 2020
				local developUserUpdated = derefOpt( docValue, 'develop', 'userUpdated' )
				if developUserUpdated and KSDate.differenceInSeconds( "2022-01-01T00:00:00.0Z", developUserUpdated ) <= 0 then

		            local kSize4MB = 4 * 1024 * 1024

					local xmpDevelopInfo = WFCatFileSpec.getXmpDevelopInfoFromDocValue( docValue ) or {}
					if xmpDevelopInfo
							and xmpDevelopInfo.path
							and xmpDevelopInfo.fileSize
							and xmpDevelopInfo.fileSize > kSize4MB then

						table.insert( updates, kXmpDevelopSha256Fix )

					end

				end

			end

			-- WF-1730: Fix up documents with negative file sizes.
			
			if isValidAssetDocument then
			
				local fileSize = WFCatFileSpec.getOriginalSizeFromDocValue( docValue )
				
				if fileSize and fileSize < 0 then
				
					table.insert( updates, kNegativeOriginalFileSize )
				
				end
			
			end

			-- NMB-25894: 
			--
			-- clear bogus InvalidLocalOriginal state
			--
			-- we were setting InvalidLocalOriginal on assets that didn't have
			-- local originals, we'll clear that error on all such assets
			--
			if isValidAssetDocument and 
					docValueHasFalsePositiveInvalidLocalOriginalError( docValue ) then

				table.insert( updates, kFalseInvalidLocalOriginalFix )

			end
			
			if next( updates ) then
				local updatesAsString = table.concat( updates, kDelimiter )
				emit( updatesAsString, hasBlockingUpdate )
			end
			
		end,

		hasOneRowPerDocument = true,
	}
	
end

--------------------------------------------------------------------------------

local function generateIndexSchema( _, indexStoreSchema )

	check( '_', 'WFRxIndexStoreSchema' )

	indexStoreSchema:addTable( simple_updates_table( ) )

end

--------------------------------------------------------------------------------

local function overlayToUpdateKeywords( docStore, docValue )
			
	local keywords = docValue.keywords
	
	local subject
	for keyword, _ in pairs( keywords ) do
		
		subject = subject or {}
		subject[ keyword ] = true
		
	end
	
	local keywordsOverlay = {
		_id = docValue._id,
		keywords = { _delete = true },
	}
	
	if subject then
		
		keywordsOverlay.xmp = {
				dc = {
					subject = subject
				}
			}
			
	end
				
	return keywordsOverlay
	
end

kSimpleUpdateFunctions[ kKeywordKey ] = overlayToUpdateKeywords

--------------------------------------------------------------------------------

local function overlayToUpdateBinaryPaths( docStore, docValue )

	local localOnly = {
		files = {}
	}

	local maybeAbsoluteOriginalPath = WFCatFileSpec.getAbsoluteOriginalPathFromDocValue( docValue )
	
	local maybeAbsoluteProxyPath, proxySize = WFCatFileSpec.getAbsoluteProxyPathFromDocValue( docValue )
	
	if maybeAbsoluteOriginalPath then
	
		local volumeGuid = WFFileUtils.getVolumeGuidForPath( maybeAbsoluteOriginalPath )
		local relativeFilePath = WFFileUtils.removeVolumeFromPath( maybeAbsoluteOriginalPath )
		
		if volumeGuid 
			and type( volumeGuid ) == 'string'
			and relativeFilePath
			and type( relativeFilePath ) == 'string' then
		
			localOnly.files[ 'original' ] = {

				[ 'master' ] = {
					relativePath = relativeFilePath,
					volumeGuid = volumeGuid,
					path = { _delete = true }
				}
				
			}
		
		else
		
			-- This should never happen.
		
			localOnly.files[ 'original' ] = {

				[ 'master' ] = {
					path = { _delete = true }
				}
				
			}
		
		end
		
	end
	
	if maybeAbsoluteProxyPath and proxySize then 
	
		local volumeGuid = WFFileUtils.getVolumeGuidForPath( maybeAbsoluteProxyPath )
		local relativeFilePath = WFFileUtils.removeVolumeFromPath( maybeAbsoluteProxyPath )
		
		if volumeGuid 
			and type( volumeGuid ) == 'string'
			and relativeFilePath
			and type( relativeFilePath ) == 'string' then
		
			localOnly.files[ 'proxy' ] = {
			
				[ proxySize ] = {
					relativePath = relativeFilePath,
					volumeGuid = volumeGuid,
					path = { _delete = true }
				}

			}
		
		else
		
			-- This should never happen.
		
			localOnly.files[ 'proxy' ] = {

				[ proxySize ] = {
					path = { _delete = true }
				}
				
			}
		
		end
	
	end
	
	local overlay = {
		_id = docValue._id,
		_localOnly = localOnly
	}
	
	return overlay
	
end

kSimpleUpdateFunctions[ kBinaryPathsKey ] = overlayToUpdateBinaryPaths

--------------------------------------------------------------------------------

local function overlayToFixNMB6449StorageIssue( docStore, docValue )
	-- The issue we have here is that we have a document in the doc store that
	-- has two separate revision trees with no revisions in common
	-- this means we have a baseless revision merge and we always choose the local
	-- revision. So what we need to do is to modify the local revision so that it
	-- looks correct, grab the last Oz revision and force it to be the one that's marked current
	
	docStore:docValue( docValue._id, 'oz' )
		:catch( Rx.Observable.just( nil ) )
		:flatMap( function( ozDocValue )

			if ozDocValue then
				
				local revisions = KSTableUtils.derefOpt( ozDocValue, '_serverProvidedMetadata', 'revision_ids' )
				revisions = revisions and KSTableUtils.shallowcopy( revisions ) or {}
				
				ozDocValue._changeCounter = nil
				ozDocValue._localId = nil
				ozDocValue._revId = nil
				revisions.force = true -- force this Oz revision to be accepted as current
				ozDocValue._revIds = revisions
				
				return docStore:setDocValue( ozDocValue )
				
			else
				return docStore:applyOverlay( {
					_id = docValue._id,
					_localOnly = {
						localError = {
							nmb6449fix = 'failed',
						}
					}
				} )
			end
		end )
		:call( runUntilDone )
	
	-- because we need to do a bit more complicated work than an overlay, do not return an overlay
	-- and, instead, let the Observable above do it's work.
	return nil
	
end

kSimpleUpdateFunctions[ kStorageExceededFixup ] = overlayToFixNMB6449StorageIssue

--------------------------------------------------------------------------------

local validatedDeviceName

local function getValidDeviceName()
	if validatedDeviceName then
		return validatedDeviceName
	end
	
	local deviceName = WFDeviceInfo.deviceName()
	if deviceName and KSStringUtils.validateUTF8( deviceName ) then
		validatedDeviceName = deviceName
	else
		-- we still can't get a valid device name, so just use the platform so that 
		-- we can sync and still identify the source to some extent
		validatedDeviceName = WFPlatformInfo.getOS()
	end
	
	return validatedDeviceName
end

--------------------------------------------------------------------------------

local function overlayToFixNmb7338InvalidDeviceName( docStore, docValue )
	
	local importedOnDevice = KSTableUtils.derefOpt( docValue, 'importSource', 'importedOnDevice' )
	local changedOnDevice = KSTableUtils.derefOpt( docValue, 'changedOnDevice' )
	
	local maybeOverlay = {}
	if importedOnDevice and not KSStringUtils.validateUTF8( importedOnDevice ) then
		maybeOverlay.importSource = {
			importedOnDevice = getValidDeviceName()
		}
	end
	
	if changedOnDevice and not KSStringUtils.validateUTF8( changedOnDevice ) then
		maybeOverlay.changedOnDevice = getValidDeviceName()
	end
	
	if next( maybeOverlay ) then 
		maybeOverlay._id = docValue._id
		return maybeOverlay
	else
		return nil
	end
end

kSimpleUpdateFunctions[ kDeviceNameInvalidUTF8Fixup ] = overlayToFixNmb7338InvalidDeviceName

--------------------------------------------------------------------------------

local function overlayToFixInvalidAlbumName( docStore, docValue )
	
	local OZ_MAX_ALBUM_NAME_CHARS = 250
	
	local albumName = docValue.name
	
	local maybeOverlay = {}
	
	if albumName and not KSStringUtils.validateUTF8( albumName ) then
	
		local characterCount = math.min( OZ_MAX_ALBUM_NAME_CHARS, string.len( albumName ) )
	
		while characterCount > 0 and not KSStringUtils.validateUTF8( albumName ) do
	
			albumName = KSStringUtils.truncateUTF8( albumName, characterCount )
		
			characterCount = characterCount - 1
		
		end
	
		if KSStringUtils.validateUTF8( albumName ) then
		
			maybeOverlay.name = albumName
		
		else
		
			maybeOverlay.name = ZSTR "$$$/NmbSimpleDocStoreUpdater/InvalidAlbumName=Invalid Album Name"
			
		end
	
	end
	
	
	if next( maybeOverlay ) then 
		maybeOverlay._id = docValue._id
		return maybeOverlay
	else
		return nil
	end
	
end

kSimpleUpdateFunctions[ kAlbumNameInvalidUTF8Fixup ] = overlayToFixInvalidAlbumName

--------------------------------------------------------------------------------

local function overlayToFixInvalidAssetName( docStore, docValue )
	
	local assetName = docValue.name
	
	local maybeOverlay = {}
	
	if assetName and not KSStringUtils.validateUTF8( assetName ) then
	
		maybeOverlay.name = ZSTR "$$$/NmbSimpleDocStoreUpdater/InvalidName=Invalid Name"
	
	end
	
	if next( maybeOverlay ) then 
		maybeOverlay._id = docValue._id
		return maybeOverlay
	else
		return nil
	end
	
end

kSimpleUpdateFunctions[ kAssetNameInvalidUTF8Fixup ] = overlayToFixInvalidAssetName

--------------------------------------------------------------------------------

local function overlayToFixFalseInvalidOriginal( docStore, docValue )
	
	local localOriginalError = KSTableUtils.derefOpt( docValue, '_localOnly', 'localOriginalError' )
	local hasLocalOriginalAnnotation = docValueHasLocalOriginal( docValue )
	local ozMasterUrl = WFCatFileSpec.getOriginalLinkFromDocValue( docValue )
	local filePath = WFCatFileSpec.buildOriginalPathFromDocValue( docValue )
	
	-- We can only fix the false invalid original issue if the volume containing
	-- the "invalid" original is online, or if there is no local original.
	
	local overlay = {
		_id = docValue._id,
		_localOnly = {}
	}
	
	if hasLocalOriginalAnnotation and filePath then
	
		local expectedSha256 = derefOpt( docValue, 'importSource', 'sha256' )
		
		return WFCrypto.asyncDigestFile( filePath, 'SHA256' )
		
			:map( function( actualSha256 )
		
				if expectedSha256 == actualSha256 then
				
					-- If the local original SHA256 matches the SHA256 calculated on
					-- import, then the error is a lie. Remove it.
				
					overlay._localOnly.localOriginalError = { _delete = true }
				
				else
				
					-- Otherwise, change the name of the error so that we don't keep
					-- processing it. 
					
					if localOriginalError == 'InvalidOriginal' then
					
						overlay._localOnly.localOriginalError = 'InvalidLocalOriginal'
					
					elseif localOriginalError == 'UploadFailedOriginalMissing' then
					
						overlay._localOnly.localOriginalError = 'UploadFailedMissingOriginal'
					
					end
				
				end
				
				return overlay
		
			end )
			
			:catch( function( ... )
			
				return Observable.just( nil )
			
			end )
			
	
	elseif not hasLocalOriginalAnnotation and ozMasterUrl then
	
		-- If there's no locally tracked file, then delete the error annotation if the asset
		-- is already on Oz.
	
		overlay._localOnly.localOriginalError = { _delete = true }
		return Observable.just( overlay )
		
	end
	
	return nil
	
end

kObservableUpdateFunctions[ kFalseInvalidOriginalFix ] = overlayToFixFalseInvalidOriginal

--------------------------------------------------------------------------------

local function overlayToFixNoSha256( docStore, docValue )
	
	local hasLocalOriginalAnnotation = docValueHasLocalOriginal( docValue )
	local filePath = WFCatFileSpec.buildOriginalPathFromDocValue( docValue )

	if hasLocalOriginalAnnotation and filePath then
	
		local overlay = {
			_id = docValue._id
		}
	
		return WFCrypto.asyncDigestFile( filePath, 'SHA256' )
		
			:map( function( sha256 )
		
				overlay._localOnly = {
					files = {
						original = {
							master = {
								sha256 = sha256
							}
						}
					}
				}

				return overlay
		
			end )

			:catch( function( ... )
			
				return Observable.just( nil )
			
			end )

	end
	
	return nil
	
end

kObservableUpdateFunctions[ kNoSha256Fix ] = overlayToFixNoSha256

--------------------------------------------------------------------------------

local _copyXmpDevelopObservables = {}

--------------------------------------------------------------------------------

local function overlayToFixXmpDevelopSha256Fix( docStore, docValue, fileSpec )

	-- Async digest had a bug when calculating xmp_develop sha hashes > 4MB.

	-- If the sha hash was incorrect this will move the file as well since
	-- it is named by it's sha hash.

	-- develop.xmpCameraRaw.sha256 is the only sha transmitted so it must
	-- have caused the issue on mobile.

	-- Refer to WFCatDevelopSettings.lua xmpCameraRawOverlay() for more info.

	local xmpDevelopInfo = WFCatFileSpec.getXmpDevelopInfoFromDocValue( docValue ) or {}

	local expectedSha256 = xmpDevelopInfo.sha256

	return WFCrypto.asyncDigestFile( xmpDevelopInfo.path, 'SHA256' )

		:flatMap( function( actualSha256 )

			if expectedSha256 ~= actualSha256 or derefOpt( docValue, 'develop', 'xmpCameraRaw', 'sha256' ) ~= actualSha256 then

				local newPath = fileSpec:pathForSettingsXmp( actualSha256 )

				-- The newPath is based on the sha256 so if two assets try to
				-- move to the same newPath, they will be moving identical
				-- contents to the same file path. Therefore we share the copy
				-- observable between them.
				_copyXmpDevelopObservables[ newPath ] = 
					_copyXmpDevelopObservables[ newPath ] or
					-- allow multiple assets to share the same copy observable
					{

						copyObservable =
							WFAsyncFileUtils.copyToUniquelyNamedFile( xmpDevelopInfo.path, newPath )
								:publish()
								:refCount(),

						assetIds = {}

					}

				_copyXmpDevelopObservables[ newPath ].assetIds[ docValue._id ] = true

				local copyObservable = _copyXmpDevelopObservables[ newPath ].copyObservable

				return copyObservable
					:map( function( copiedPathName )

						if _copyXmpDevelopObservables[ newPath ] then

							-- stop retaining observable when no longer shared
							_copyXmpDevelopObservables[ newPath ].assetIds[ docValue._id ] = nil

							if not next( _copyXmpDevelopObservables[ newPath ].assetIds ) then

								_copyXmpDevelopObservables[ newPath ] = nil

							end

						end

						if copiedPathName ~= newPath then
							-- File already there just delete the new path as xmp_develop files
							-- must have exact names based on hash.
							pcall( WFFileUtils.delete, copiedPathName )
						end

						-- Fix the sha hash as it was computed incorrectly
						xmpDevelopInfo.sha256 = actualSha256
						xmpDevelopInfo.path = newPath

						local overlay = {
							_id = docValue._id,
							develop = {
								xmpCameraRaw = {
									sha256 = actualSha256
								},
							},
							_localOnly = {
								files = {
									xmp_develop = xmpDevelopInfo
								}
							},
							sourceXmpDevelopLink = { _delete = true }
						}

						return overlay

					end )

			end

			return Observable.just( nil )

		end )

		:catch( function( ... )
			return Observable.just( nil )
		end )

end

kObservableUpdateFunctions[ kXmpDevelopSha256Fix ] = overlayToFixXmpDevelopSha256Fix

--------------------------------------------------------------------------------

local function overlayToFixNegativeOriginalFileSize( docStore, docValue )
	
	local hasLocalOriginalAnnotation = docValueHasLocalOriginal( docValue )
	local filePath = WFCatFileSpec.buildOriginalPathFromDocValue( docValue )
	
	if hasLocalOriginalAnnotation and filePath then
	
		local fileAttributes = WFFileUtils.fileAttributes( filePath ) or {}
		local localFileSize = fileAttributes.fileSize
	
		if localFileSize and localFileSize > 0 then
	
			local overlay = {
				_id = docValue._id,
				_localOnly = {
					files = {
						original = {
							master = {
								size = localFileSize
							}
						}
					}
				}
			}
		
			return Observable.just( overlay )
		
		end

	end
	
	return nil
	
end

--------------------------------------------------------------------------------

kObservableUpdateFunctions[ kNegativeOriginalFileSize ] = overlayToFixNegativeOriginalFileSize

--------------------------------------------------------------------------------

local function maybeOverlayToFixInvalidLocalOriginal( _, docValue )

	if not docValueHasFalsePositiveInvalidLocalOriginalError( docValue ) then

		return

	end

	return Rx.Observable.just( 
			{

				_id = docValue._id,

				_localOnly = {

					localOriginalError = { _delete = true },

				}

			} )

end

--------------------------------------------------------------------------------

kObservableUpdateFunctions[ kFalseInvalidLocalOriginalFix ] = 
	maybeOverlayToFixInvalidLocalOriginal

--------------------------------------------------------------------------------

local function getOverlayForUpdatesObservables( self, docValue, updates, arrayOfOverlayObservables )

	local docStore = self._docStore
	local fileSpec = self._fileSpec

	for _, update in ipairs( updates ) do

		local updateFn = kSimpleUpdateFunctions[ update ]
		local maybeUpdateOverlayObservable
		
		if updateFn then
		
			local maybeUpdateOverlay = updateFn( docStore, docValue, fileSpec )
			maybeUpdateOverlayObservable = maybeUpdateOverlay and Observable.just( maybeUpdateOverlay )
								
		else
		
			updateFn = kObservableUpdateFunctions[ update ]
			maybeUpdateOverlayObservable = updateFn( docStore, docValue, fileSpec )
			
		end

		if maybeUpdateOverlayObservable then

			local updateCount = self._updatesAppliedForDiagnostics[ update ] or 0
			self._updatesAppliedForDiagnostics[ update ] = updateCount + 1

			table.insert( arrayOfOverlayObservables, maybeUpdateOverlayObservable )

		end
		
	end
	
	return arrayOfOverlayObservables

end

--------------------------------------------------------------------------------

local function reportBlockingUpdatesFinished( self )

	if not self._blockingUpdatesProgress:isDone() then
		self._blockingUpdatesProgress:done()
	end
	
	if self._synchronizer then
		self._synchronizer:startSyncing()
	end
	
end

--------------------------------------------------------------------------------

local function reportBlockingUpdatesProgress( self, blockingUpdateCount )
	
	self._blockingUpdatesProgress:reportBulkSuccess( blockingUpdateCount, nil, true )
	
end

--------------------------------------------------------------------------------

local function reportAllUpdatesFinished( self )

	reportBlockingUpdatesFinished( self )

	if not self._allUpdatesProgress:isDone() then
		self._allUpdatesProgress:done()
	end
	
end

--------------------------------------------------------------------------------

local function reportAllUpdatesProgress( self, updateCount )
	
	self._allUpdatesProgress:reportBulkSuccess( updateCount, nil, true )
	
end

--------------------------------------------------------------------------------

local function initializeProgressReporters( self )

	return self._indexStore:queryOnce {
			sql = [[SELECT count( docId ), hasBlockingUpdate
					FROM simple_updates
					GROUP BY hasBlockingUpdate ]],
			quiesceUntilUpToDate = true,
		}
		
		:map( function( results )
			
			local numberOfBlockingUpdates = 0
			local numberOfUpdates = 0
			
			for _, result in ipairs( results ) do
			
				local count = result[ 1 ]
				local hasBlockingUpdate = toBool( result[ 2 ] )
			
				if hasBlockingUpdate then
				
					numberOfBlockingUpdates = count
				
				end
				
				numberOfUpdates = numberOfUpdates + count
				
			end
			
			self._allUpdatesProgress:setTotalMustDeliver( numberOfUpdates )
			self._blockingUpdatesProgress:setTotalMustDeliver( numberOfBlockingUpdates )
			
			if numberOfBlockingUpdates > 0 and self._synchronizer then
			
				self._synchronizer:stopSyncing()
					-- Don't sync while processing blocking updates.
			
			end
			
		end )
		
end 

--------------------------------------------------------------------------------

local function performUpdates( self )
	
	return self._indexStore:queryOnce {
			sql = [[SELECT docId, updates, hasBlockingUpdate 
					FROM simple_updates
					ORDER BY hasBlockingUpdate DESC]],
			quiesceUntilUpToDate = true,
		}

		:flatMap( function( results )
			-- break the results into chunks of 100 documents to process at a
			-- time so as not to ask for or update too many docValues at once
			local chunkedResults = {}
			local chunkNum = 1
			for i, result in ipairs( results ) do
				
				local chunk = chunkedResults[ chunkNum ] or {
					docIds = {},
					docIdsToUpdates = {},
					numberOfBlockingUpdates = 0 
				}
				chunkedResults[ chunkNum ] = chunk
				
				
				local fullDocId = result[ 1 ]
				local updatesAsString = result[ 2 ]
				local hasBlockingUpdate = toBool( result[ 3 ] )
				
				if hasBlockingUpdate then
					chunk.numberOfBlockingUpdates = chunk.numberOfBlockingUpdates + 1
				end
				
				chunk.docIds[ #chunk.docIds + 1 ] = fullDocId		
				if #chunk.docIds >= 100 then
					chunkNum = chunkNum + 1
				end
				
				chunk.docIdsToUpdates[ fullDocId ] = updatesAsString
			
			end
						
			return Observable.from( chunkedResults )
				:map( function( chunk )
					return chunk.docIds, chunk.docIdsToUpdates, chunk.numberOfBlockingUpdates
				end )
		end )
		:map( function( docIds, docIdsToUpdates, numberOfBlockingUpdates )

			return Rx.Observable.defer( function()
			
				return self._docStore:bulkDocValue( docIds, nil, "NMBSimpleDocStoreUpdater:performUpdates" )
				:flatMap( function( docValueArray )
					
					local arrayOfOverlayObservables = {}
					
					for _, docValue in ipairs( docValueArray or {} ) do
						
						local updatesAsString = docIdsToUpdates[ docValue._id ] 

						local updates = KSStringUtils.split( updatesAsString or "", kDelimiter )
						
						getOverlayForUpdatesObservables( self, docValue, updates, arrayOfOverlayObservables )

					end
					
					return Observable.from( arrayOfOverlayObservables )
						:flatMergeAll()
						:toArray()
				end )
				:flatMap( function( arrayOfOverlays )

					local update = kNumFixesMade
					local updateCount = self._updatesAppliedForDiagnostics[ update ] or 0
					self._updatesAppliedForDiagnostics[ update ] = updateCount + #arrayOfOverlays

					if not ( arrayOfOverlays and next( arrayOfOverlays ) ) then
						logger:warnf( "Failed to generate any update overlays for %s updates", #docIds )
						return Observable.just( false )
					end
					
					return self._docStore:bulkApplyOverlay( arrayOfOverlays, "NMBSimpleDocStoreUpdater" )

				end )
				:catch( function( errMessage )
					-- log error
					logger:errorf( "Error while processing updates (blocking %s, total %s) %s", numberOfBlockingUpdates, #docIds, errMessage )
					return Observable.just()
				end )
				:map( function()
					return #docIds, numberOfBlockingUpdates 
				end )

			end )

		end )

		:flatMerge( 1 )

end

--------------------------------------------------------------------------------

local nmbSimpleDocStoreUpdaterPrototype = {
	
	_docStore = false,
		-- (WFRxDocStore)
	
	_indexStore = false,
		-- (WFRxIndexStore)

	_fileSpec = false,
		-- (WFCatFileSpec)

	_synchronizer = false,
		-- (WFCatSynchronizer)
	
	_blockingUpdatesProgress = false,
		-- (WFProgressReporter) used for blocking progress updates
	
	_allUpdatesProgress = false,
		-- (WFProgressReporter) used for all progress updates	

	_updatesAppliedForDiagnostics = false,
		-- (array) counts of updates applied indexed by update function key (used by diagnostic log and unit tests)

	class = {
		generateIndexSchema = generateIndexSchema,
	},
	
	WFCloseSupport.addCloseSupportMethods,
	
}

--------------------------------------------------------------------------------

--[[!
	\brief Creates a NmbSimpleDocStoreUpdater which can be used to make simple updates to 
		documents in the doc store.  The simple updater can be used to make minor tweaks to 
		formats of documents in the doc store that can be performed while the app is otherwise
		running. It can also be used if Nimbus cannot be otherwise running while the document
		changes are being made by specifying whether or not the update is "blocking".
		
		Additionally, if the document changes are likely to cause significant work to be done by the app, 
		it may not be the most suitable approach.
		
		When adding update functions care should be taken that the updater will not cause an 
		undue impact to the performance of Nimbus during the update.  This may require that 
		future updaters modify the logic for performing updates to be less aggressive than the
		initial implementation for keywords.
		
	\param docStore (WFDocStore) doc store to update
	\param indexStore (WFIndexStore) index store for the doc store

]]--

function nmbSimpleDocStoreUpdaterPrototype:__init( docStore, indexStore, fileSpec )
	
	check( 'NmbSimpleDocStoreUpdater', 'WFRxDocStore', 'WFRxIndexStore', 'WFCatFileSpec' )
	
	self._docStore = docStore
	
	self._indexStore = indexStore
	
	self._fileSpec = fileSpec

	self._blockingUpdatesProgress = WFProgressReporter( "NmbSimpleDocStoreUpdater : Blocking Updates" )
	
	self._allUpdatesProgress = WFProgressReporter( "NmbSimpleDocStoreUpdater : All Updates" )

	self._updatesAppliedForDiagnostics = {}

	WFDiagnosticLog.addLogProvider( self )

end

--------------------------------------------------------------------------------

--[[!
	\brief Sets the synchronizer instance.
	
	\return (void)
--]]

function nmbSimpleDocStoreUpdaterPrototype:start()

	check( 'NmbSimpleDocStoreUpdater' )

	local observable = initializeProgressReporters( self )
	
		:flatMap( function()
		
			return performUpdates( self )		
		end )
		:share()
	
	observable:subscribe( 
			-- onNext
			function( updatesProcessed, blockingUpdatesProcessed )
			
				if blockingUpdatesProcessed == 0 then
					reportBlockingUpdatesFinished( self )
				else
					reportBlockingUpdatesProgress( self, blockingUpdatesProcessed )
				end
				
				if updatesProcessed == 0 then
					reportAllUpdatesFinished( self )
				else 
					reportAllUpdatesProgress( self, updatesProcessed )
				end
				
			end,
			-- onError
			nil,
			-- onComplete
			function()
				reportAllUpdatesFinished( self )
			end )
	
	return observable
end

--------------------------------------------------------------------------------

--[[!
	\brief Sets the synchronizer instance.
	
	\return (void)
--]]

function nmbSimpleDocStoreUpdaterPrototype:setSynchronizer( synchronizer )
	check( 'NmbSimpleDocStoreUpdater', 'WFCatSynchronizer' )
	self._synchronizer = synchronizer
	
end

--------------------------------------------------------------------------------

--[[!
	\brief Retrieve the progress stream that fires with blocking progress updates.

	\return (Rx.ReplaySubject< table >) Reports on the progress.
--]]

function nmbSimpleDocStoreUpdaterPrototype:blockingUpdatesProgress()

	check( 'NmbSimpleDocStoreUpdater' )
	
	return self._blockingUpdatesProgress:status()
	
end

--------------------------------------------------------------------------------

--[[!
	\brief Retrieve the progress stream that fires with the total progress updates.

	\return (Rx.ReplaySubject< table >) Reports on the progress.
--]]

function nmbSimpleDocStoreUpdaterPrototype:allUpdatesProgress()

	check( 'NmbSimpleDocStoreUpdater' )
	
	return self._allUpdatesProgress:status()
	
end

--------------------------------------------------------------------------------

--[[!
	\brief send diagnostic data for the diagnostic log.
--]]

function nmbSimpleDocStoreUpdaterPrototype:sendDiagnosticLogData()

	WFDiagnosticLog.addSectionToLog
		{

			id = string.format( 'NmbSimpleDocStoreUpdater' ),

			name = string.format( "NmbSimpleDocStoreUpdater" ),
			weight = 1620,
			parentSection = WFDiagnosticLog.Sections.DOC_STORE,
			body = self._updatesAppliedForDiagnostics

		}

end

--------------------------------------------------------------------------------

--[[!
	\brief Debug function used by unit test to validate proper execution.

	\return (array) counts of updates applied indexed by update function key.
--]]

function nmbSimpleDocStoreUpdaterPrototype:DEBUG_getUpdatesApplied()

	return self._updatesAppliedForDiagnostics

end

--------------------------------------------------------------------------------

return  Rx.Class 'NmbSimpleDocStoreUpdater'( nmbSimpleDocStoreUpdaterPrototype )
