--[[----------------------------------------------------------------------------

NmbSimpleDocStoreUpdaterTest.lua

--------------------------------------------------------------------------------

ADOBE CONFIDENTIAL
------------------
Copyright 2016 Adobe Systems Incorporated
All Rights Reserved.

NOTICE: All information contained herein is, and remains the property of
Adobe Systems Incorporated and its suppliers, if any. The intellectual and
technical concepts contained herein are proprietary to Adobe Systems
Incorporated and its suppliers and are protected by all applicable
intellectual property laws, including trade secret and copyright laws.
Dissemination of this information or reproduction of this material is
strictly forbidden unless prior written permission is obtained from
Adobe Systems Incorporated.

------------------------------------------------------------------------------]]

	-- RxLua
local Rx = require 'Rx'

local NmbSimpleDocStoreUpdater = require 'NmbSimpleDocStoreUpdater'
local KSIdGenerator = require 'KSIdGenerator'
local KSPathUtils = require 'KSPathUtils'
local WFFileUtils = require 'WFFileUtils'
local WFCatFileSpec = require 'WFCatFileSpec'
local WFCatAlbums = require 'WFCatAlbums'
local WFCrypto = require 'WFCrypto'
local KSData = require 'KSData'
local KSStringUtils = require 'KSStringUtils'
local KSTableUtils = require 'KSTableUtils'
local WFPlatformInfo = require 'WFPlatformInfo'

	-- WF.core.rxlua
local WFRxScheduler = require 'WFRxScheduler'

	-- WF.core.self_test
local WFOzTestUtils = require 'WFOzTestUtils'
local WFRxTestCase = require 'WFRxTestCase'
local WFCatMockCatalog = require 'WFCatMockCatalog'

local WFCatAssetTestUtils = require 'WFCatAssetTestUtils'

--------------------------------------------------------------------------------

local NmbSimpleDocStoreUpdaterTest = {}

--------------------------------------------------------------------------------

local function createCatalogForTest( rxTest )
	
	local wfCatalog = WFCatMockCatalog.createFakeWFCatalog( rxTest )
	
	return wfCatalog, wfCatalog:docStore(), wfCatalog:indexStore(), wfCatalog:fileSpec()
end

--------------------------------------------------------------------------------

local function addNFakeDocsWithOldKeywords( rxTest, docStore, n )
	
	local addAssetFn = WFOzTestUtils.v2.makeAdderOfAsset( rxTest, docStore, function( docValue, time ) 
	
		docValue.keywords = {
				keyword1 = time,
				keyword2 = time,
		}
		
		docValue._serverProvidedMetadata = {
			revision_ids = {
				KSIdGenerator.uuidInLowercaseHex()
			},
			links = {
				[ '/rels/master' ] = {
					href = 'path/to/original.jpg',
				}
			}
		}
		
		return docValue

	end )
	
	local assetIds = {}
	local time = require 'KSDate'.timeInIso8601FormatWithTimeZone()
	for i = 1, n, 1 do 
		local assetId = addAssetFn( time )
		table.insert( assetIds, assetId )
	end
	
	return assetIds
end

--------------------------------------------------------------------------------

function NmbSimpleDocStoreUpdaterTest.rxTestSimpleKeywordUpdates( rxTest )
	
	rxTest:setTimeLimit( 5 )
	
	local _, docStore, indexStore, fileSpec = createCatalogForTest( rxTest )
	
	local assetIds = addNFakeDocsWithOldKeywords( rxTest, docStore, 1 )
	
	local updater = NmbSimpleDocStoreUpdater( docStore, indexStore, fileSpec )
	updater:start()
	
	rxTest:finally( function()
		updater:close()
	end )
	
	local obs = rxTest:observe( updater:allUpdatesProgress() )
	obs:expectCompletedWithinTime( 3 )
	
	for _, assetId in ipairs( assetIds ) do
		
		rxTest:expectSingleValue( docStore:docValue( assetId )
									:map( function( docValue )
										return { oldKeywords = docValue.keywords or 'nil', -- should be nil
												 newKeywords = ( docValue.xmp and docValue.xmp.dc
																	and docValue.xmp.dc.subject )
												}
									end ), { oldKeywords = 'nil',
											 newKeywords = {
											 	keyword1 = true,
												keyword2 = true,
											 } } )
	
	end

end

--------------------------------------------------------------------------------

function NmbSimpleDocStoreUpdaterTest.rxTestLotsOfKeywordUpdates( rxTest )
	
	rxTest:setTimeLimit( 10 )
	
	local _, docStore, indexStore, fileSpec = createCatalogForTest( rxTest )
	
	local assetIds = addNFakeDocsWithOldKeywords( rxTest, docStore, 200 )
	
	local updater = NmbSimpleDocStoreUpdater( docStore, indexStore, fileSpec )
	updater:start()
	
	rxTest:finally( function()
		updater:close()
	end )
	
	local obs = rxTest:observe( updater:allUpdatesProgress() )
	obs:expectCompletedWithinTime( 5 )
	
	for _, assetId in ipairs( assetIds ) do
		
		rxTest:expectSingleValue( docStore:docValue( assetId )
									:map( function( docValue )
										return { oldKeywords = docValue.keywords or 'nil', -- should be nil
												 newKeywords = ( docValue.xmp and docValue.xmp.dc
																	and docValue.xmp.dc.subject )
												}
									end ), { oldKeywords = 'nil',
											 newKeywords = {
											 	keyword1 = true,
												keyword2 = true,
											 } } )
										
	end
	
end

--------------------------------------------------------------------------------

function NmbSimpleDocStoreUpdaterTest.rxTestFixingNmb6449Docs( rxTest )
	
	rxTest:setTimeLimit( 5 )
	
	local _, docStore, indexStore, fileSpec = createCatalogForTest( rxTest )
	
	-- apply an overlay for a nonexistent document with StorageExceededError which creates
	-- a completely empty doc
	local localRevisionId = KSIdGenerator.uuidInLowercaseHex()
	rxTest:expectSingleValue( docStore:applyOverlay( { 
		_id = 'nmb4669test',
		_newRevId = localRevisionId,
		_localOnly = {
			ozError = {
				subtype = 'StorageExceededError'
			}
		}
	} ), ':ignore:' )
	
	-- apply an update as a bulkDocValue with label 'oz' to mimic sync feed pull
	local ozRevisionId = KSIdGenerator.uuidInLowercaseHex()
	rxTest:expectSingleValue( docStore:bulkSetDocValue( {
		label = 'oz',
		{
			_id = 'nmb4669test',
			_revIds = { ozRevisionId },
			type = 'asset',
			subtype = 'image',
			_serverProvidedMetadata = {
				revision_ids = { ozRevisionId },
				links = {
					self = 'href'
				}
			},
			payload = {
				someData = 'someValue'
			}
		}
	} ), ':ignore:' )
	
	local conflictResolver = require 'WFCatConflictResolver'( docStore, indexStore )
	rxTest:addCloseTarget( conflictResolver )
	
	local resolvedDocValue = rxTest:observe( docStore:ongoingDocValue( 'nmb4669test' )
		:map( function( docValue ) 
			docValue._changeCounter = nil
			docValue._revId = nil
			return docValue
		end ) )
		
	resolvedDocValue:expectDataWithinTime( 1, 
		{
			_id = "nmb4669test",
			_localId = 1,
			_localOnly = {
				ozError = {
					subtype = "StorageExceededError",
				},
			},
			_serverProvidedMetadata = {
				links = {
					["self"] = "href",
				},
				revision_ids = {
					ozRevisionId,
				},
			},
			payload = {
				someData = "someValue",
			},
			type = 'asset',
			subtype = 'image',
		} )
	
	rxTest:expectSingleValue( docStore:docValue( 'nmb4669test', 'oz' )
		:map( function( docValue ) 
			docValue._changeCounter = nil
			return docValue
		end ),
		{
			_id = "nmb4669test",
			_labels = {
				oz = true,
			},
			_localId = 1,
			_localOnly = {
				ozError = {
					subtype = "StorageExceededError",
				},
			},
			_revId = ozRevisionId,
			_serverProvidedMetadata = {
				links = {
					["self"] = "href",
				},
				revision_ids = {
					ozRevisionId,
				},
			},
			payload = {
				someData = "someValue",
			},
			subtype = "image",
			type = "asset",
	} )
	
	local updater = NmbSimpleDocStoreUpdater( docStore, indexStore, fileSpec )
	
	-- we have storage available so lift the storage exceeded error
	rxTest:expectSingleValue( docStore:applyOverlay( { 
		_id = 'nmb4669test',
		_localOnly = {
			ozError = { _delete = true } 
		}
	} ), ':ignore:' )
	
	rxTest:expectOnlyCompleted( updater:start():takeLast(0) )
	rxTest:finally( function()
		updater:close()
	end )
	
	resolvedDocValue:expectDataWithinTime( 2,
		{
			_id = "nmb4669test",
			_localId = 1,
			_localOnly = {
			},
			_serverProvidedMetadata = {
				links = {
					["self"] = "href",
				},
				revision_ids = {
					ozRevisionId,
				},
			},
			payload = {
				someData = "someValue",
			},
			subtype = "image",
			type = "asset",
		} )
		
end

--------------------------------------------------------------------------------

function NmbSimpleDocStoreUpdaterTest.rxTestFixingNmb6449Docs_fail_noOzDoc( rxTest )

	rxTest:setTimeLimit( 5 )
	
	local _, docStore, indexStore, fileSpec = createCatalogForTest( rxTest )
	
	-- apply an overlay for a nonexistent document with StorageExceededError which creates
	-- a completely empty doc
	local localRevisionId = KSIdGenerator.uuidInLowercaseHex()
	rxTest:expectSingleValue( docStore:applyOverlay( { 
		_id = 'nmb4669test',
		_newRevId = localRevisionId,
		_localOnly = {
			ozError = {
				subtype = 'StorageExceededError'
			}
		},
		_serverProvidedMetadata = {
			links = {
				self = 'href'
			}
		}
	} ), ':ignore:' )
	
	local conflictResolver = require 'WFCatConflictResolver'( docStore, indexStore )
	rxTest:addCloseTarget( conflictResolver )
	
	local resolvedDocValue = rxTest:observe( docStore:ongoingDocValue( 'nmb4669test' )
		:map( function( docValue ) 
			docValue._changeCounter = nil
			return docValue
		end ) )
	
	resolvedDocValue:expectDataWithinTime( 1, 
		{
			_id = "nmb4669test",
			_revId = localRevisionId,
			_localId = 1,
			_localOnly = {
				ozError = {
					subtype = "StorageExceededError",
				},
			},
			_serverProvidedMetadata = {
				links = {
					["self"] = "href",
				},
			},
		} )
	
	local updater = NmbSimpleDocStoreUpdater( docStore, indexStore, fileSpec )
	
	-- we have storage available so lift the storage exceeded error
	-- this time, though, there is no ozDoc so we should ensure that 
	-- the document gets marked as failed so we don't try this fixup again
	rxTest:expectSingleValue( docStore:applyOverlay( { 
		_id = 'nmb4669test',
		_localOnly = {
			ozError = { _delete = true } 
		}
	} ), ':ignore:' )
	
	rxTest:expectOnlyCompleted( updater:start():takeLast(0) )
	rxTest:finally( function()
		updater:close()
	end )
	
	resolvedDocValue:expectDataWithinTime( 2,
		{
			_id = "nmb4669test",
			_localId = 1,
			_localOnly = {
				localError = {
					nmb6449fix = "failed",
				},
			},
			_revId = localRevisionId,
			_serverProvidedMetadata = {
				links = {
					["self"] = "href",
				},
			},
		} )
		
end

--------------------------------------------------------------------------------

local maybeUtf16Str = KSStringUtils.getUTF16String_DEBUG() or 
		table.concat( { string.char( 0xe7 ), string.char( 0xe3 ) } )

-- SPAM [jandrew 2018-08-28]: I couldn't figure out how to push a UTF16 string
-- from native code to Lua on Mac ( maybe a good thing?) so the following is 
-- Windows-only. I think that's okay though, considering all of the stuck syncs 
-- caused by this issue are on Windows machines.

if maybeUtf16Str then

function NmbSimpleDocStoreUpdaterTest.rxTestFixingInvalidAlbumName( rxTest )

	rxTest:setTimeLimit( 10 )
	
	local wfCatalog = WFCatMockCatalog.createFakeWFCatalog( rxTest )
	local docStore = wfCatalog:docStore()
	local indexStore = wfCatalog:indexStore()
	local fileSpec = wfCatalog:fileSpec()
	
	local albumId
	
	rxTest:expectOnlyCompleted(

		WFCatAlbums.createAlbum( wfCatalog, 'Foo' )
			:map( function( id )
			
				albumId = id
				return id
			
			end )
			:takeLast( 0 )
			
	)
	
	rxTest:waitFor( docStore:applyOverlay {
		_id = albumId,
		name = maybeUtf16Str
	} )
	
	local updater = NmbSimpleDocStoreUpdater( docStore, indexStore, fileSpec )
	updater:start()
	
	rxTest:finally( function()
		updater:close()
	end )
	
	local obs = rxTest:observe( updater:allUpdatesProgress() )
	obs:expectCompletedWithinTime( 10 )
	
	rxTest:expectSingleValue( docStore:docValue( albumId )
		:pluck( 'name' )
		:map( function( newAlbumName )
		
			return newAlbumName ~= maybeUtf16Str 
					and KSStringUtils.validateUTF8( newAlbumName )
		
		end ), true )

end

--------------------------------------------------------------------------------

local function addFakeDocsWithInvalidDeviceNames( rxTest, docStore )
	
	local addInvalidAssetFn = WFOzTestUtils.v2.makeAdderOfAsset( rxTest, docStore, 
		
			function( docValue ) 
		
				docValue._newRevId = KSIdGenerator.uuidInLowercaseHex()
			
				docValue.importSource = {
					importedOnDevice = maybeUtf16Str
				}

				docValue.changedOnDevice = maybeUtf16Str
			
				return docValue
				
			end )
	
	local assetIds = {}
	
	for i = 1, 3, 1 do 
		local assetId = addInvalidAssetFn()
		table.insert( assetIds, assetId )
	end
	
	return assetIds
	
end

--------------------------------------------------------------------------------

function NmbSimpleDocStoreUpdaterTest.rxTestNmb7338InvalidDeviceName( rxTest )

	rxTest:setTimeLimit( 10 )
	
	local wfCatalog = WFCatMockCatalog.createFakeWFCatalog( rxTest )
	local docStore = wfCatalog:docStore()
	local indexStore = wfCatalog:indexStore()
	local fileSpec = wfCatalog:fileSpec()
	
	local assetIds = addFakeDocsWithInvalidDeviceNames( rxTest, docStore )
	
	local updater = NmbSimpleDocStoreUpdater( docStore, indexStore, fileSpec )
	updater:start()
	
	rxTest:finally( function()
		updater:close()
	end )
	
	local obs = rxTest:observe( updater:allUpdatesProgress() )
	obs:expectCompletedWithinTime( 10 )
	
	for _, assetId in ipairs( assetIds ) do
		
		rxTest:expectSingleValue( docStore:docValue( assetId )
			:pluck( 'changedOnDevice' )
			:map( function( changedOnDevice )
		
				return changedOnDevice ~= maybeUtf16Str 
						and KSStringUtils.validateUTF8( changedOnDevice )
			
			end ), true )
			
		rxTest:expectSingleValue( docStore:docValue( assetId )
			:pluck( 'importSource', 'importedOnDevice' )
			:map( function( importedOnDevice )
		
				return importedOnDevice ~= maybeUtf16Str 
						and KSStringUtils.validateUTF8( importedOnDevice )
			
			end ), true )
	
	end

end

--------------------------------------------------------------------------------

local function addFakePresetWithInvalidName( rxTest, docStore )
	
	local presetDoc = {

		_id = KSIdGenerator.uuidInLowercaseHex(),
		
		type = 'asset',
		subtype = 'preset',
		name = maybeUtf16Str,
		group = 'foo',
		
		importSource = {
			fileSize = 100,
			sha256 = KSIdGenerator.uuidInLowercaseHex(),
			contentType = "application/rdf+xml",
		},
		
	}
	
	rxTest:expectSingleValue( docStore:setDocValue( presetDoc ), ':ignore:' )
	
	return presetDoc._id
	
end

--------------------------------------------------------------------------------

function NmbSimpleDocStoreUpdaterTest.rxTestWF393InvalidDeviceName( rxTest )

	rxTest:setTimeLimit( 10 )
	
	local wfCatalog = WFCatMockCatalog.createFakeWFCatalog( rxTest )
	local docStore = wfCatalog:docStore()
	local indexStore = wfCatalog:indexStore()
	local fileSpec = wfCatalog:fileSpec()
	
	local assetId = addFakePresetWithInvalidName( rxTest, docStore )
	
	rxTest:expectSingleValue( docStore:docValue( assetId )
		:pluck( 'name' )
		:map( function( assetName )
	
			return KSStringUtils.validateUTF8( assetName )
		
		end ), false )
	
	local updater = NmbSimpleDocStoreUpdater( docStore, indexStore, fileSpec )
	updater:start()
	
	rxTest:finally( function()
		updater:close()
	end )
	
	local obs = rxTest:observe( updater:allUpdatesProgress() )
	obs:expectCompletedWithinTime( 10 )
	
	rxTest:expectSingleValue( docStore:docValue( assetId )
		:pluck( 'name' )
		:map( function( assetName )
	
			return KSStringUtils.validateUTF8( assetName )
		
		end ), true )

end

end

--------------------------------------------------------------------------------

local basePath = KSPathUtils.child( KSPathUtils.getStandardFilePath( 'carouselDocuments' ), 'original.jpg' )
local volumeGuid = WFFileUtils.getVolumeGuidForPath( basePath )
local relativePath = WFFileUtils.removeVolumeFromPath( basePath )

local proxyBasePath = KSPathUtils.child( KSPathUtils.getStandardFilePath( 'carouselDocuments' ), 'proxy.jpg' )
local proxyVolumeGuid = WFFileUtils.getVolumeGuidForPath( proxyBasePath )
local proxyRelativePath = WFFileUtils.removeVolumeFromPath( proxyBasePath )

local mockImage = KSData.make( 'data' )
KSData.saveToFile( mockImage, basePath )

local expectedSha256 = WFCrypto.digestFile( basePath, 'SHA256' )

--------------------------------------------------------------------------------

local function addFakeDocsWithInvalidOriginals( rxTest, docStore, addInvalidAssetFn, 
				localError, validError, hasSimpleError, noLocalOriginalAnnotation,
				noOzLink )

	local importSourceSha256 = validError and 'IAMANINVALIDSHA' or expectedSha256
	
	if not addInvalidAssetFn then
	
		addInvalidAssetFn = WFOzTestUtils.v2.makeAdderOfAsset( rxTest, docStore, 
		
			function( docValue, sha256, localOriginalError, hasOutdatedKeywords, noOriginalAnnotation, noLink ) 
		
				docValue._newRevId = KSIdGenerator.uuidInLowercaseHex()
			
				docValue.importSource = {
					sha256 = sha256
				}
			
				if hasOutdatedKeywords then
			
					docValue.keywords = {
						keyword1 = 'foo',
						keyword2 = 'bar',
					}
						
				end
			
			
				docValue._localOnly = {
					localOriginalError = localOriginalError,
				}
			
				if not noOriginalAnnotation then
			
					docValue._localOnly.files = {
					
						original = {
							master = {
								relativePath = relativePath,
								volumeGuid = volumeGuid
							}
						}
						
					}
				
				end
			
				if not noLink then
	
					docValue._serverProvidedMetadata = {
						revision_ids = {
							KSIdGenerator.uuidInLowercaseHex()
						},
						links = {
							[ '/rels/master' ] = {
								href = 'path/to/original.jpg',
							}
						}
					}
				
				end
			
				return docValue
				
			end )
			
	end
	
	local assetIds = {}
	
	for i = 1, 3, 1 do 
		local assetId = addInvalidAssetFn( importSourceSha256, localError, hasSimpleError, noLocalOriginalAnnotation, noOzLink )
		table.insert( assetIds, assetId )
	end
	
	return assetIds, addInvalidAssetFn
	
end

--------------------------------------------------------------------------------

function NmbSimpleDocStoreUpdaterTest.rxTestFixingNmb7768Docs( rxTest )
	
	rxTest:setTimeLimit( 30 )
	
	local _, docStore, indexStore, fileSpec = createCatalogForTest( rxTest )
	
	local positiveIds, addInvalidAssetFn = addFakeDocsWithInvalidOriginals( rxTest, docStore, nil, 'InvalidOriginal', true )
	
	local falsePositiveIds = addFakeDocsWithInvalidOriginals( rxTest, docStore, addInvalidAssetFn, 'InvalidOriginal', false )
	
	local unrelatedIds = addFakeDocsWithInvalidOriginals( rxTest, docStore, addInvalidAssetFn, 'OtherTypeOfError', true )
	
	local unrelatedIds2 = addFakeDocsWithInvalidOriginals( rxTest, docStore, addInvalidAssetFn, 'OtherTypeOfError', false )
	
	local noOriginalAnnotation = addFakeDocsWithInvalidOriginals( rxTest, docStore, addInvalidAssetFn, 'InvalidOriginal', true, nil, true )
	
	local noOriginalAnnotationNoOzLink = addFakeDocsWithInvalidOriginals( rxTest, docStore, addInvalidAssetFn, 'InvalidOriginal', true, nil, true, true )
	
	local updater = NmbSimpleDocStoreUpdater( docStore, indexStore, fileSpec )
	updater:start()
	
	rxTest:finally( function()
		updater:close()
	end )
	
	local obs = rxTest:observe( updater:allUpdatesProgress() )
	obs:expectCompletedWithinTime( 10 )

	for _, assetId in ipairs( positiveIds ) do
		
		rxTest:expectSingleValue( docStore:docValue( assetId )
			:pluck( '_localOnly', 'localOriginalError' ),
			'InvalidLocalOriginal' )
	
	end
	
	for _, assetId in ipairs( falsePositiveIds ) do
		
		rxTest:expectSingleValue( docStore:docValue( assetId )
			:pluck( '_localOnly', 'localOriginalError' ),
			nil )
	
	end
	
	for _, assetId in ipairs( unrelatedIds ) do
		
		rxTest:expectSingleValue( docStore:docValue( assetId )
			:pluck( '_localOnly', 'localOriginalError' ),
			'OtherTypeOfError' )
	
	end
	
	for _, assetId in ipairs( unrelatedIds2 ) do
		
		rxTest:expectSingleValue( docStore:docValue( assetId )
			:pluck( '_localOnly', 'localOriginalError' ),
			'OtherTypeOfError' )
	
	end
	
	for _, assetId in ipairs( noOriginalAnnotation ) do
		
		rxTest:expectSingleValue( docStore:docValue( assetId )
			:pluck( '_localOnly', 'localOriginalError' ),
			nil )
	
	end
	
	for _, assetId in ipairs( noOriginalAnnotationNoOzLink ) do
		
		rxTest:expectSingleValue( docStore:docValue( assetId )
			:pluck( '_localOnly', 'localOriginalError' ),
			'InvalidOriginal' )
	
	end
	
end


--------------------------------------------------------------------------------

local function addFakeDocsWithNoLocalSha256( rxTest, docStore, addInvalidAssetFn, hasOzMasterUrl, isMarkedForOffline )

	if not addInvalidAssetFn then
	
		addInvalidAssetFn = WFOzTestUtils.v2.makeAdderOfAsset( rxTest, docStore, 
		
			function( docValue, hasOzMasterUrl, isMarkedForOffline ) 
		
				docValue._newRevId = KSIdGenerator.uuidInLowercaseHex()
		
				docValue._localOnly.files = {
				
					original = {
						master = {
							relativePath = relativePath,
							volumeGuid = volumeGuid
						}
					}
					
				}

				docValue._localOnly.isMarkedForOfflineUse = not not isMarkedForOffline

				if hasOzMasterUrl then
	
					docValue._serverProvidedMetadata = {
						revision_ids = {
							KSIdGenerator.uuidInLowercaseHex()
						},
						links = {
							[ '/rels/master' ] = {
								href = 'path/to/original.jpg',
							}
						}
					}
				
				end
			
				return docValue
				
			end )
			
	end
	
	local assetIds = {}
	
	for i = 1, 3, 1 do 
	
		local assetId = addInvalidAssetFn( hasOzMasterUrl, isMarkedForOffline )
		table.insert( assetIds, assetId )
		
	end
	
	return assetIds, addInvalidAssetFn
	
end

--------------------------------------------------------------------------------

function NmbSimpleDocStoreUpdaterTest.rxTestFixingNmb10355Docs( rxTest )
	
	rxTest:setTimeLimit( 30 )
	
	local _, docStore, indexStore, fileSpec = createCatalogForTest( rxTest )
	
	local shouldAddSHA256Ids, addInvalidAssetFn = addFakeDocsWithNoLocalSha256( rxTest, docStore, nil, false, true )
	
	local shouldAddSHA256Ids2 = addFakeDocsWithNoLocalSha256( rxTest, docStore, addInvalidAssetFn, false, false )
	
	local shouldAddSHA256Ids3 = addFakeDocsWithNoLocalSha256( rxTest, docStore, addInvalidAssetFn, true, true )
	
	local shouldNotAddSHA256Ids = addFakeDocsWithNoLocalSha256( rxTest, docStore, addInvalidAssetFn, true, false )

	local updater = NmbSimpleDocStoreUpdater( docStore, indexStore, fileSpec )
	updater:start()
	
	rxTest:finally( function()
		updater:close()
	end )
	
	local obs = rxTest:observe( updater:allUpdatesProgress() )
	obs:expectCompletedWithinTime( 10 )
	
	for _, assetId in ipairs( shouldAddSHA256Ids ) do
		
		rxTest:expectSingleValue( docStore:docValue( assetId )
			:pluck( '_localOnly', 'files', 'original', 'master', 'sha256' )
			:map( function( sha256 )
				return not not sha256
			end ),
			true )
	
	end
	
	for _, assetId in ipairs( shouldAddSHA256Ids2 ) do
		
		rxTest:expectSingleValue( docStore:docValue( assetId )
			:pluck( '_localOnly', 'files', 'original', 'master', 'sha256' )
			:map( function( sha256 )
				return not not sha256
			end ),
			true )
	end
	
	for _, assetId in ipairs( shouldAddSHA256Ids3 ) do
		
		rxTest:expectSingleValue( docStore:docValue( assetId )
			:pluck( '_localOnly', 'files', 'original', 'master', 'sha256' )
			:map( function( sha256 )
				return not not sha256
			end ),
			true )
	
	end
	
	for _, assetId in ipairs( shouldNotAddSHA256Ids ) do
		
		rxTest:expectSingleValue( docStore:docValue( assetId )
			:pluck( '_localOnly', 'files', 'original', 'master', 'sha256' )
			:map( function( sha256 )
				return not not sha256
			end ),
			false )
	
	end
	
end

--------------------------------------------------------------------------------

if WFPlatformInfo.isDesktop then

function NmbSimpleDocStoreUpdaterTest.rxTestFixingWF1730Docs( rxTest )
	
	rxTest:setTimeLimit( 30 )
	
	local wfCatalog, docStore, indexStore, fileSpec = createCatalogForTest( rxTest )
	
	local assetId = KSIdGenerator.uuidInLowercaseHex()
	
	local docValue = WFCatAssetTestUtils.createAssetDocument( rxTest, wfCatalog, assetId )
	KSTableUtils.setNested( docValue, -10000, '_localOnly', 'files', 'original', 'master', 'size' )
	
	rxTest:waitFor( docStore:setDocValue( docValue ) )
	
	local updater = NmbSimpleDocStoreUpdater( docStore, indexStore, fileSpec )
	updater:start()
	
	rxTest:finally( function()
		updater:close()
	end )
	
	local obs = rxTest:observe( updater:allUpdatesProgress() )
	obs:expectCompletedWithinTime( 10 )
	
	rxTest:expectSingleValue( docStore:docValue( assetId )
		:pluck( '_localOnly', 'files', 'original', 'master', 'size' ),
		13 )

end

end

--------------------------------------------------------------------------------

function NmbSimpleDocStoreUpdaterTest.rxTestSimpleAndObservableUpdates( rxTest )
	
	rxTest:setTimeLimit( 10 )
	
	local _, docStore, indexStore, fileSpec = createCatalogForTest( rxTest )
	
	local falsePositiveIds = addFakeDocsWithInvalidOriginals( rxTest, docStore, nil, 'InvalidOriginal', false, true )
	
	local updater = NmbSimpleDocStoreUpdater( docStore, indexStore, fileSpec )
	updater:start()
	
	rxTest:finally( function()
		updater:close()
	end )
	
	local obs = rxTest:observe( updater:allUpdatesProgress() )
	obs:expectCompletedWithinTime( 3 )
	
	for _, assetId in ipairs( falsePositiveIds ) do
		
		rxTest:expectSingleValue( docStore:docValue( assetId )
				:map( function( docValue )
					return {
						localOriginalError = docValue._localOnly.localOriginalError or 'nil',
						oldKeywords = docValue.keywords or 'nil', -- should be nil
						newKeywords = ( docValue.xmp and docValue.xmp.dc and docValue.xmp.dc.subject )
					}
				end ), 
				{
					localOriginalError = 'nil',
					oldKeywords = 'nil',
				 	newKeywords = {
						keyword1 = true,
						keyword2 = true,
				 	}
				} )
	
	end

end

--------------------------------------------------------------------------------

local function addNFakeDocsWithAbsolutePaths( rxTest, docStore, n )
	
	local addAssetFn = WFOzTestUtils.v2.makeAdderOfAsset( rxTest, docStore, function( docValue ) 
	
		docValue._localOnly = {
			files = {
				original = {
					master = {
						path = basePath
					}
				},
				proxy = {
					[ '2560' ] = {
						path = proxyBasePath
					}
				}
			}
		}
		
		return docValue
		
	end )
	
	local assetIds = {}
	
	for i = 1, n, 1 do 
		local assetId = addAssetFn()
		table.insert( assetIds, assetId )
	end
	
	return assetIds
	
end

--------------------------------------------------------------------------------

function NmbSimpleDocStoreUpdaterTest.rxTestAbsolutePathsUpdates( rxTest )
	
	rxTest:setTimeLimit( 10 )
	
	local _, docStore, indexStore, fileSpec = createCatalogForTest( rxTest )
	
	local assetIds = addNFakeDocsWithAbsolutePaths( rxTest, docStore, 1 )
	
	local updater = NmbSimpleDocStoreUpdater( docStore, indexStore, fileSpec )
	updater:start()
	
	rxTest:finally( function()
		updater:close()
	end )
	
	local obs = rxTest:observe( updater:allUpdatesProgress() )
	obs:expectCompletedWithinTime( 3 )
	
	for _, assetId in ipairs( assetIds ) do
		
		rxTest:expectSingleValue( docStore:docValue( assetId )
									:map( function( docValue )
										return { 
											WFCatFileSpec.getAbsoluteOriginalPathFromDocValue( docValue ),
											WFCatFileSpec.getOriginalVolumeGuidFromDocValue( docValue ),
											WFCatFileSpec.getOriginalRelativePathFromDocValue( docValue ),
											WFCatFileSpec.getAbsoluteProxyPathFromDocValue( docValue ),
											WFCatFileSpec.getProxyVolumeGuidFromDocValue( docValue ),
											WFCatFileSpec.getProxyRelativePathFromDocValue( docValue ),
										}
									end ), { 
										nil,
										volumeGuid, 
										relativePath,
										nil,
										proxyVolumeGuid,
										proxyRelativePath,
										'2560' } )
	
	end

end

--------------------------------------------------------------------------------

function NmbSimpleDocStoreUpdaterTest.rxTestLotsOfAbsolutePaths( rxTest )
	
	rxTest:setTimeLimit( 10 )
	
	local _, docStore, indexStore, fileSpec = createCatalogForTest( rxTest )
	
	local assetIds = addNFakeDocsWithAbsolutePaths( rxTest, docStore, 200 )
	
	local updater = NmbSimpleDocStoreUpdater( docStore, indexStore, fileSpec )
	updater:start()
	
	rxTest:finally( function()
		updater:close()
	end )
	
	local obs = rxTest:observe( updater:allUpdatesProgress() )
	obs:expectCompletedWithinTime( 5 )
	
	for _, assetId in ipairs( assetIds ) do
		
		rxTest:expectSingleValue( docStore:docValue( assetId )
									:map( function( docValue )
										return { 
											WFCatFileSpec.getAbsoluteOriginalPathFromDocValue( docValue ),
											WFCatFileSpec.getOriginalVolumeGuidFromDocValue( docValue ),
											WFCatFileSpec.getOriginalRelativePathFromDocValue( docValue ),
											WFCatFileSpec.getAbsoluteProxyPathFromDocValue( docValue ),
											WFCatFileSpec.getProxyVolumeGuidFromDocValue( docValue ),
											WFCatFileSpec.getProxyRelativePathFromDocValue( docValue ),
										}
									end ), { 
										nil,
										volumeGuid, 
										relativePath,
										nil,
										proxyVolumeGuid,
										proxyRelativePath,
										'2560' } )
	
	end
	
end

--------------------------------------------------------------------------------

local fnAddFakeDocsWithXmpDevelopSha256

local function addFakeDocsWithXmpDevelopSha256( rxTest, docStore, xmpDevelopFilePath, xmpDevelopSha256, xmpDevelopFileSize )

	fnAddFakeDocsWithXmpDevelopSha256 = fnAddFakeDocsWithXmpDevelopSha256 or WFOzTestUtils.v2.makeAdderOfAsset( rxTest, docStore,

			function( docValue, xmpDevelopFilePath, xmpDevelopSha256, xmpDevelopFileSize )

				docValue._newRevId = KSIdGenerator.uuidInLowercaseHex()

				docValue._localOnly.files = {

					original = {
						master = {
							relativePath = relativePath,
							volumeGuid = volumeGuid,
							sha256 = '0000', -- add a fake sha to prevent noSha256Fix from activating
						},
					},
					xmp_develop = {
						path = xmpDevelopFilePath,
						fileSize = xmpDevelopFileSize,
						sha256 = xmpDevelopSha256,
					},

				}

				docValue.develop = {
					xmpCameraRaw = {
						sha256 = xmpDevelopSha256
					},
				}

				return docValue

			end

	)

	local assetIds = {}

	for i = 1, 3, 1 do

		local assetId = fnAddFakeDocsWithXmpDevelopSha256( xmpDevelopFilePath, xmpDevelopSha256, xmpDevelopFileSize )
		table.insert( assetIds, assetId )

	end

	return assetIds

end

--------------------------------------------------------------------------------

local function generateXmpDevelopFile( rxTest, xmpDevelopSize, xmpDevelopFileName )

	local xmpDevelopFilePath = KSPathUtils.child( KSPathUtils.getStandardFilePath( 'carouselDocuments' ), xmpDevelopFileName )

	-- Delete any file that is already there, we will make it below.
	WFFileUtils.delete( xmpDevelopFilePath )

	local fileContents = string.rep( 'a', xmpDevelopSize )
	local fileData = KSData.make( fileContents )
	KSData.saveToFile( fileData, xmpDevelopFilePath )
	fileData:close()

	rxTest:finally( function()
		WFFileUtils.delete( xmpDevelopFilePath )
	end )

	local sha256 = WFCrypto.digestFile( xmpDevelopFilePath, 'SHA256' )

	return xmpDevelopFilePath, sha256

end

--------------------------------------------------------------------------------

local function changedAssetIdsFromCounter( docStore, changeCounter, includeContinueFrom )

	return Rx.Observable.timer( .1, 1, WFRxScheduler.sharedScheduler() )
		:flatMapLatest( function()
			return docStore:changedDocs { fromChangeCounter = changeCounter }
		end )
		:map( function( changedDocs )

			local changedAssetIds = {}

			for i, doc in ipairs( changedDocs ) do
				changedAssetIds[ i ] = doc._id
			end

			table.sort( changedAssetIds )

			if includeContinueFrom then
				changedAssetIds.continueFrom = changedDocs.continueFrom
			end

			return changedAssetIds

		end )

end

--------------------------------------------------------------------------------

function NmbSimpleDocStoreUpdaterTest.rxTestFixingInvalidXmpDevelopSha256_Nmb10185( rxTest )

	rxTest:setTimeLimit( 30 )

	local _, docStore, indexStore, fileSpec = createCatalogForTest( rxTest )

	local xmpDevelopFileSize_2MB = 2 * 1024 * 1024
	local xmpDevelopFileSize_3MB = 3 * 1024 * 1024
	local xmpDevelopFileSize_5MB = 5 * 1024 * 1024
	local xmpDevelopFileSize_6MB = 6 * 1024 * 1024
	local validXmpDevelopSha256_2MB_FilePath, validXmpDevelop_2MB_Sha256 = generateXmpDevelopFile( rxTest, xmpDevelopFileSize_2MB, 'XmpDevelop2MB' )
	local validXmpDevelopSha256_3MB_FilePath, validXmpDevelop_3MB_Sha256 = generateXmpDevelopFile( rxTest, xmpDevelopFileSize_3MB, 'XmpDevelop3MB' )
	local validXmpDevelopSha256_5MB_FilePath, validXmpDevelop_5MB_Sha256 = generateXmpDevelopFile( rxTest, xmpDevelopFileSize_5MB, 'XmpDevelop5MB' )
	local validXmpDevelopSha256_6MB_FilePath, validXmpDevelop_6MB_Sha256 = generateXmpDevelopFile( rxTest, xmpDevelopFileSize_6MB, 'XmpDevelop6MB' )

	local invalidXmpDevelop_5MB_Sha256 = "badbad90d179b2f3d734749996734efc5834f9045e3e91b55d9ec62889badbad"

	local assetIds2MB = addFakeDocsWithXmpDevelopSha256( rxTest, docStore, validXmpDevelopSha256_2MB_FilePath, validXmpDevelop_2MB_Sha256, xmpDevelopFileSize_2MB )

	local assetIds3MB = addFakeDocsWithXmpDevelopSha256( rxTest, docStore, validXmpDevelopSha256_3MB_FilePath, validXmpDevelop_3MB_Sha256, xmpDevelopFileSize_3MB )

	local assetIds5MB = addFakeDocsWithXmpDevelopSha256( rxTest, docStore, validXmpDevelopSha256_5MB_FilePath, validXmpDevelop_5MB_Sha256, xmpDevelopFileSize_5MB )

	local assetIds6MB = addFakeDocsWithXmpDevelopSha256( rxTest, docStore, validXmpDevelopSha256_6MB_FilePath, validXmpDevelop_6MB_Sha256, xmpDevelopFileSize_6MB )

	local assetIds5MBNeedFix = addFakeDocsWithXmpDevelopSha256( rxTest, docStore, validXmpDevelopSha256_5MB_FilePath, invalidXmpDevelop_5MB_Sha256, xmpDevelopFileSize_5MB )

	local changeCounterBeforeUpdate
	local changedAssetIdsObserver = rxTest:observe( changedAssetIdsFromCounter( docStore, 0, true )
		:map( function( changedAssetIds )
			changeCounterBeforeUpdate = changedAssetIds.continueFrom
			return #changedAssetIds
		end ) )
	changedAssetIdsObserver:expectDataWithinTime( 5, 15 )
	changedAssetIdsObserver:dispose()

	local updater = NmbSimpleDocStoreUpdater( docStore, indexStore, fileSpec )
	updater:start()

	rxTest:finally( function()
		updater:close()
	end )

	local obs = rxTest:observe( updater:allUpdatesProgress() )
	obs:expectCompletedWithinTime( 10 )

	-- Even though this test will only apply a fix to 3 of the documents it needs to process
	-- 6 of them in order to verify the sha hash is correct.
	rxTest:expectEqual( {
		invalidXmpDevelopSha256 = 9,
		numOverlaysApplied = 3,
	}, updater:DEBUG_getUpdatesApplied() )

	-- Verify only 3 documents were updated.
	local changedAssetIdsByUpdater = rxTest:observe( changedAssetIdsFromCounter( docStore, changeCounterBeforeUpdate ) )
	changedAssetIdsByUpdater:expectDataWithinTime( 10, assetIds5MBNeedFix )
	changedAssetIdsByUpdater:dispose()

	for _, assetId in ipairs( assetIds2MB ) do

		rxTest:expectSingleValue( docStore:docValue( assetId )
			:pluck( '_localOnly', 'files', 'xmp_develop', 'sha256' ),
			validXmpDevelop_2MB_Sha256 )

	end

	for _, assetId in ipairs( assetIds3MB ) do

		rxTest:expectSingleValue( docStore:docValue( assetId )
			:pluck( '_localOnly', 'files', 'xmp_develop', 'sha256' ),
			validXmpDevelop_3MB_Sha256 )

	end
	for _, assetId in ipairs( assetIds5MB ) do

		rxTest:expectSingleValue( docStore:docValue( assetId )
			:pluck( '_localOnly', 'files', 'xmp_develop', 'sha256' ),
			validXmpDevelop_5MB_Sha256 )

	end
	for _, assetId in ipairs( assetIds6MB ) do

		rxTest:expectSingleValue( docStore:docValue( assetId )
			:pluck( '_localOnly', 'files', 'xmp_develop', 'sha256' ),
			validXmpDevelop_6MB_Sha256 )

	end

	-- These are the assets that need the fix.
	for _, assetId in ipairs( assetIds5MBNeedFix ) do

		-- Verify the invalid sha hash has been updated in the files section.
		rxTest:expectSingleValue( docStore:docValue( assetId )
			:pluck( '_localOnly', 'files', 'xmp_develop', 'sha256' ),
			validXmpDevelop_5MB_Sha256 )

		-- Verify the invalid sha hash has been updated in the develop section.
		rxTest:expectSingleValue( docStore:docValue( assetId )
			:pluck( 'develop', 'xmpCameraRaw', 'sha256' ),
			validXmpDevelop_5MB_Sha256 )

		local expectedXmpDevelopPath = fileSpec:pathForSettingsXmp( validXmpDevelop_5MB_Sha256 )

		-- Verify the file path has been updated and the file has been moved
		-- to that location.
		rxTest:expectSingleValue( docStore:docValue( assetId )
			:pluck( '_localOnly', 'files', 'xmp_develop', 'path' ),
			expectedXmpDevelopPath )

		rxTest:expectEqual( 'file', WFFileUtils.exists( expectedXmpDevelopPath ) )

	end

end

--------------------------------------------------------------------------------

function NmbSimpleDocStoreUpdaterTest.rxTestBlockingProgressReporter( rxTest )
	
	rxTest:setTimeLimit( 10 )
	
	local _, docStore, indexStore, fileSpec = createCatalogForTest( rxTest )
	
	local docCount = 200
	
	addNFakeDocsWithAbsolutePaths( rxTest, docStore, docCount )
	
	local updater = NmbSimpleDocStoreUpdater( docStore, indexStore, fileSpec )
	updater:start()
	
	rxTest:finally( function()
		updater:close()
	end )
	
	local obs = rxTest:observe( updater:blockingUpdatesProgress() )

	obs:expectDataWithinTime( 4, {
		description = "NmbSimpleDocStoreUpdater : Blocking Updates",
		errorCount = 0,
		percentComplete = 0,
		processedCount = 0,
		totalCount = docCount,
	} )
	
	obs:expectDataWithinTime( 1, {
		description = "NmbSimpleDocStoreUpdater : Blocking Updates",
		errorCount = 0,
		percentComplete = 0.5,
		processedCount = 100,
		totalCount = docCount,
	} )
	
	obs:expectDataWithinTime( 1, {
		description = "NmbSimpleDocStoreUpdater : Blocking Updates",
		errorCount = 0,
		percentComplete = 1,
		processedCount = 200,
		totalCount = docCount,
	} )
	
	obs:expectCompleted()
	
end

--------------------------------------------------------------------------------

function NmbSimpleDocStoreUpdaterTest.rxTestAllProgressReporter( rxTest )
	
	rxTest:setTimeLimit( 10 )
	
	local _, docStore, indexStore, fileSpec = createCatalogForTest( rxTest )
	
	local docCount = 200
	
	addNFakeDocsWithOldKeywords( rxTest, docStore, docCount )
	
	local updater = NmbSimpleDocStoreUpdater( docStore, indexStore, fileSpec )
	
	rxTest:finally( function()
		updater:close()
	end )
	
	local obs = rxTest:observe( updater:allUpdatesProgress() )
	
	updater:start()
	obs:expectDataWithinTime( 4, {
		description = "NmbSimpleDocStoreUpdater : All Updates",
		errorCount = 0,
		percentComplete = 0,
		processedCount = 0,
		totalCount = docCount,
	} )
	
	obs:expectDataWithinTime( 1, {
		description = "NmbSimpleDocStoreUpdater : All Updates",
		errorCount = 0,
		percentComplete = 0.5,
		processedCount = 100,
		totalCount = docCount,
	} )
	
	obs:expectDataWithinTime( 1, {
		description = "NmbSimpleDocStoreUpdater : All Updates",
		errorCount = 0,
		percentComplete = 1,
		processedCount = 200,
		totalCount = docCount,
	} )
	
	obs:expectCompleted()
	
end

--------------------------------------------------------------------------------

local MockSynchronizerMethods = {
	_isSyncRunning = false,
}

--------------------------------------------------------------------------------

function MockSynchronizerMethods:__init()
	
	self._isSyncRunning = true
	
end

--------------------------------------------------------------------------------

function MockSynchronizerMethods:startSyncing()

	self._isSyncRunning = true
		
end

--------------------------------------------------------------------------------

function MockSynchronizerMethods:stopSyncing()
	self._isSyncRunning = false
end

--------------------------------------------------------------------------------

function MockSynchronizerMethods:isSyncRunningNow()
	
	return self._isSyncRunning

end

--------------------------------------------------------------------------------

function MockSynchronizerMethods:shouldSyncBeRunning()

	return true
	
end

--------------------------------------------------------------------------------

local MockSynchronizer = Rx.Class( 'WFCatSynchronizer', MockSynchronizerMethods )

--------------------------------------------------------------------------------

function NmbSimpleDocStoreUpdaterTest.rxTestSyncPauseAndResume( rxTest )
	
	rxTest:setTimeLimit( 10 )
	
	local _, docStore, indexStore, fileSpec = createCatalogForTest( rxTest )
	
	addNFakeDocsWithAbsolutePaths( rxTest, docStore, 200 )
	
	local updater = NmbSimpleDocStoreUpdater( docStore, indexStore, fileSpec )
	local synchronizer = MockSynchronizer()
	
	rxTest:expectEqual( synchronizer:isSyncRunningNow(), true )
	
	updater:setSynchronizer( synchronizer )
	
	updater:start()
		:tap( function()
		
			rxTest:expectEqual( synchronizer:isSyncRunningNow(), false )
		
		end )
	
	rxTest:finally( function()
		updater:close()
	end )
	
	local obs = rxTest:observe( updater:allUpdatesProgress() )
	obs:expectCompletedWithinTime( 5 )
	
	rxTest:expectEqual( synchronizer:isSyncRunningNow(), true )
	
end

--------------------------------------------------------------------------------

return WFRxTestCase.wrapSuite( NmbSimpleDocStoreUpdaterTest )
