--[[----------------------------------------------------------------------------

WFCatImportTest.lua

--------------------------------------------------------------------------------

ADOBE CONFIDENTIAL
------------------
Copyright 2015 Adobe Systems Incorporated
All Rights Reserved.

NOTICE: All information contained herein is, and remains the property of
Adobe Systems Incorporated and its suppliers, if any. The intellectual and
technical concepts contained herein are proprietary to Adobe Systems
Incorporated and its suppliers and are protected by all applicable
intellectual property laws, including trade secret and copyright laws.
Dissemination of this information or reproduction of this material is
strictly forbidden unless prior written permission is obtained from
Adobe Systems Incorporated.

----------------------------------------------------------------------------]]--

local WFPlatformInfo = require "WFPlatformInfo"

-- Disabled on iOS and Android because WFXmp is not present there

if WFPlatformInfo.isAndroid or WFPlatformInfo.isiOS then
       return {}
end

	-- RxLua
local Rx = require 'Rx'
local Observable = Rx.Observable

	-- WF.core.lua
local KSData = require 'KSData'

	-- WF.core.os
local KSDate = require 'KSDate'
local KSPathUtils = require 'KSPathUtils'
local KSStringUtils = require 'KSStringUtils'
local WFFileUtils = require 'WFFileUtils'
local WFFolderWalker = require 'WFFolderWalker'

	-- WF.core.rxlua
local WFRxScheduler = require 'WFRxScheduler'

	-- WF.core.self_test
local WFRxTestCase = require 'WFRxTestCase'
local WFTestFileUtils = require 'WFTestFileUtils'

	-- WF.core.utils
local KSIdGenerator = require 'KSIdGenerator'
local KSTableUtils = require 'KSTableUtils'

	-- WF.oz_client.managed_catalog
local WFCatDelete = require 'WFCatDelete'
local WFCatFileSpec = require 'WFCatFileSpec'
local WFCatImport = require 'WFCatImport'
local WFCatImportConstants = require 'WFCatImportConstants'
local WFCatImportDuplicateFiles = require 'WFCatImportDuplicateFiles'
local WFCatMockCatalog = require 'WFCatMockCatalog'
local WFCatAlbums = require 'WFCatAlbums'
local WFCatSearchQuery = require 'WFCatSearchQuery'
local WFCatSpecMock = require 'WFCatSpecMock'

	-- WF.oz_client.managed_catalog.metadata
local WFXMLNS = require 'WFXMLNS'
local WFXmp = require 'WFXmp'

local WFCrInfo = assert( require 'NmbAcr'.WFCrInfo )
local _, _, _, CRS_VERSION = WFCrInfo.getACRVersion()

--------------------------------------------------------------------------------

local createFakeWFCatalog = WFCatMockCatalog.createFakeWFCatalog

local kDefaultAlbumName = "My Album"

--============================================================================--

local WFCatImportTest = {}

--------------------------------------------------------------------------------

local testDataPath = KSPathUtils.child( KSPathUtils.getStandardFilePath( 'testdata' ),
										"import-samples" )

local testDataFolder1 = KSPathUtils.child( testDataPath, "folder-1" )

local bugFilesFolder = KSPathUtils.child( testDataPath, "import-bugfiles" )

local testDataFolder2 = KSPathUtils.child( testDataPath, "folder-2" )

--------------------------------------------------------------------------------

local function findOrCreateAllPhotosAlbum( wfCatalog )

	return wfCatalog:indexStore()
		:queryOnce {
			sql = [[SELECT docId FROM albums WHERE nameLC = ?]],
			KSStringUtils.lower( kDefaultAlbumName ),
			quiesceUntilUpToDate = true,
		}
		:map( function( rows )
			local albumId = KSTableUtils.derefOpt( rows, 1, 1 )
			if albumId then
				return Observable.just( albumId )
			end
			albumId = KSIdGenerator.uuidInLowercaseHex()
			local updateTimestamp = KSDate.timeInIso8601Format( nil, false )

			return Observable.just( albumId )
			:sample( wfCatalog:docStore():setDocValue {
				type = 'album',
				_id = albumId,
				name = kDefaultAlbumName,
				_localOnly = {
					updateTimeStamp = updateTimestamp,
				},
			}, WFRxScheduler.sharedScheduler() )
		end )
		:switch()

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestBasic( rxTest )
	
	rxTest:setTimeLimit( 5 )

	local fakeWFCatalog = createFakeWFCatalog( rxTest )
	local userId = fakeWFCatalog:ozUserId()

	local myAlbum
	rxTest:expectSingleValue(
		findOrCreateAllPhotosAlbum( fakeWFCatalog )
			:tap( function( albumId )
				myAlbum = albumId
			end ),
		":ignore:" )

	local docStore = fakeWFCatalog:docStore()

	local metadata = WFXmp.WFXmp()
	metadata:set( WFXMLNS.xap, "ModifyDate", "2019-12-11T10:09:08.765Z" )
	metadata:set( WFXMLNS.xap, "Rating", "4" )
	metadata:setStringList( WFXMLNS.dc, "subject", { "mary", "had", "a little lamb" } )
	metadata:set( WFXMLNS.exif, "GPSAltitude", "10000/2" )
	metadata:set( WFXMLNS.exif, "GPSAltitudeRef", "1" )
	metadata:set( WFXMLNS.exif, "GPSLongitude", "100,5.125S" )
	metadata:set( WFXMLNS.exif, "GPSLatitude", "20,1.5" )

	rxTest:expectOnlyCompleted( WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, {} ) )
		
	local testFilePath = KSPathUtils.child( testDataFolder1, "IMG_7763.jpg" )

	rxTest:expectSingleValue(
		WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, { {
			identifier = 10,
			xmp = tostring( metadata:serialize() ),
			filePath = testFilePath,
			width = 2048,
			height = 1024,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "Image.jpg",
			fileSize = 4096,
			hasEmbeddedDepthMap = true,
			location = {
				city = "Tuticorin",
				state = "Tamil Nadu",
				country = "India",
			},
			importTimestamp = "2016-03-07T05:20:34Z",
			importSource = {
				lrcatStoreProviderId = "11112222333344445555666677778888",
				lrcatImageId = "AAAABBBBCCCCDDDDEEEEFFFF00001111",
			},
		} }, nil, {
			albumId = myAlbum,
		} )
		:map( function( result )
			rxTest:expectEqual( result.filePath, testFilePath )
			rxTest:expectEqual( result.identifier, 10 )
			
			return docStore:docValue( result.assetId )
			:map( function( docValue )
				local result = {}
				for k, v in pairs( docValue ) do
					if k == '_localOnly' or not string.find( k, "^_" ) then
						result[ k ] = v
					end
				end
				
				if KSTableUtils.derefOpt( result, '_localOnly', 'files', 'original', 'master', 'relativePath' ) then
					result._localOnly.files.original.master.relativePath = "(path stripped)"
				end
				
				if KSTableUtils.derefOpt( result, '_localOnly', 'files', 'original', 'master', 'volumeGuid' ) then
					result._localOnly.files.original.master.volumeGuid = "(guid stripped)"
				end
				
				if KSTableUtils.derefOpt( result, '_localOnly', 'files', 'original', 'master', 'sha256' ) then
					result._localOnly.files.original.master.sha256 = "(sha256 stripped)"
				end
				
				if KSTableUtils.derefOpt( result, '_localOnly', 'files', 'original', 'master', 'size' ) then
					result._localOnly.files.original.master.size = "(size stripped)"
				end
				
				if KSTableUtils.derefOpt( result, '_localOnly', 'files', 'xmp_develop', 'path' ) then
					result._localOnly.files.xmp_develop.path = "(path stripped)"
				end

				if KSTableUtils.derefOpt( result, '_localOnly', 'cameraProfileInfo', 'xmpCameraRawProfileDigest' ) then
					result._localOnly.cameraProfileInfo.xmpCameraRawProfileDigest = '(xmpCameraRawProfileDigest)'
				end
				
				if result.userUpdated then
					result.userUpdated = "(valid ISO 8601 timestamp)"
				end
				
				if result.userCreated then
					result.userCreated = "(valid ISO 8601 timestamp)"
				end

				if KSTableUtils.derefOpt( result, 'develop', 'device') then
					rxTest:expectEqual(true, not not string.match(docValue.develop.device, "^(.*)({[A-Fa-f0-9]+})"))
					result.develop.device = "(device stripped)"
				end
				
				result.importSource.importedOnDevice = nil
				return result
			end )
		end )
		:switch(),
		{
			_localOnly = {
				cameraProfileInfo = {
					isStandard = false,
					profileDigest = "NA",
					xmpCameraRawProfileDigest = "(xmpCameraRawProfileDigest)",
				},
				files = {
					original = {
						master = {
							relativePath = "(path stripped)",
							volumeGuid = '(guid stripped)',
							sha256 = '(sha256 stripped)',
							size = '(size stripped)'
						},
					},
					xmp_develop = {
						fileSize = 23,
						path = "(path stripped)",
						sha256 = "13ada4e31740f4a6ef2f1dc05d959a0e078cb82106d2e1e555bb95cb1e3c346f",
					},
				},
				xmpSidecarExists = false,
			},
			captureDate = "2019-12-11T10:09:08.765Z",
			develop = {
				device = "(device stripped)",
				processingModel = 'lightroom',
				croppedHeight = 1024,
				croppedWidth = 2048,
				crsVersion = CRS_VERSION,
				userOrientation = 1,
				crsHDREditMode = false,
				xmpCameraRaw = {
					sha256 = "13ada4e31740f4a6ef2f1dc05d959a0e078cb82106d2e1e555bb95cb1e3c346f",
				},			},
			importSource = {
				fileName = "Image.jpg",
				fileSize = 4096,
				importedBy = userId,
				originalHeight = 1024,
				originalWidth = 2048,
				sha256 = "3bd8b936cc646bf1296e5ba4a76c410a27beb80809f525a8f5072f0115c54ccc",
				originalDigest = "00112233445566778899AABBCCDDEEFF",
				contentType = "image/jpeg",
				importTimestamp = "2016-03-07T05:20:34Z",
				lrcatStoreProviderId = "11112222333344445555666677778888",
				lrcatImageId = "AAAABBBBCCCCDDDDEEEEFFFF00001111",
				hasEmbeddedDepthMap = true,
			},
			location = {
				altitude = -5000,
				latitude = 20.025,
				longitude = -100.085417,
				city = "Tuticorin",
				state = "Tamil Nadu",
				country = "India",
			},
			ratings = {
				[ userId ] = {
					rating = 4,
				},
			},
			subtype = "image",
			type = "asset",
			userCreated = "(valid ISO 8601 timestamp)",
			userUpdated = "(valid ISO 8601 timestamp)",
			xmp = {
				dc = {
					subject = {
						["a little lamb"] = true,
						had = true,
						mary = true,
					}
				},
				xmp = {
					ModifyDate = "2019-12-11T10:09:08.765Z",
				},
			},
		}
	)
	
	-- Check if the original photo really got copied
	local originalPath = fakeWFCatalog:fileSpec():pathForOriginal( "Image.jpg", "2019-12-11T10:09:08.765" )

	rxTest:expectEqual( 'file', WFFileUtils.exists( originalPath ) )

	local testFileSize = KSTableUtils.derefOpt( WFFileUtils.fileAttributes( testFilePath ),
											'fileSize' )

	local importedfileSize = KSTableUtils.derefOpt( WFFileUtils.fileAttributes( originalPath ),
											'fileSize' )

	rxTest:expectEqual( testFileSize, importedfileSize )

	rxTest:expectEqual( true, testFilePath ~= originalPath )

end

--------------------------------------------------------------------------------

local function mapPluckFileName( docs )

	if type( docs ) ~= 'table' then return docs end

	for i, doc in ipairs( docs ) do
		docs[ i ] = KSTableUtils.derefOpt( doc, 'importSource', 'fileName' ) or "(no file name)"
	end

	docs.continueFrom = nil

	table.sort( docs )

	return docs

end

--------------------------------------------------------------------------------


local function getTimestampsMatchFn( rxTest )

	return function( docs )

		if type( docs ) ~= 'table' then return end

		local prevTimestamp = nil

		for _, doc in ipairs( docs ) do

			local timestamp = KSTableUtils.derefOpt( doc, 'importSource', 'importTimestamp' )

			if prevTimestamp then
				rxTest:expectEqual( prevTimestamp, timestamp )
			end

			prevTimestamp = timestamp

		end

	end

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestAddImagesFromArrayIgnoringDuplicates( rxTest )

	rxTest:setTimeLimit( 15 )

	local WFXmp = require 'WFXmp'

	local fakeWFCatalog = createFakeWFCatalog( rxTest )

	local metadata = WFXmp.WFXmp()
	metadata:set( WFXMLNS.xap, "ModifyDate", "2019-12-11T10:09:08.765Z" )
	metadata:set( WFXMLNS.xap, "Rating", "4" )
	metadata:setStringList( WFXMLNS.dc, "subject", { "mary", "had", "a little lamb" } )
	metadata:set( WFXMLNS.exif, "GPSAltitude", "10000/2" )
	metadata:set( WFXMLNS.exif, "GPSAltitudeRef", "1" )
	metadata:set( WFXMLNS.exif, "GPSLongitude", "100,5.125S" )
	metadata:set( WFXMLNS.exif, "GPSLatitude", "20,1.5" )

	local importArray = {
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7706.PNG" ),
			width = 640,
			height = 1136,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7706.PNG",
			fileSize = 9609,
		},
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7763.jpg" ),
			width = 3264,
			height = 2448,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7763.jpg",
			fileSize = 2072173,
		},
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7766.jpg" ),
			width = 1200,
			height = 1600,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7766.jpg",
			fileSize = 703991,
		},
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "IMG_9876.CR2" ),
			width = 2048,
			height = 3456,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_9876.CR2",
			fileSize = 26243176,
		},
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "NikonPhotoworldSafariMiami281.NEF" ),
			width = 3872,
			height = 2592,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "NikonPhotoworldSafariMiami281.NEF",
			fileSize = 9844596,
		},
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "ir-trees.dng" ),
			width = 500,
			height = 334,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "ir-trees.dng",
			fileSize = 165598,
		},
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "ir-trees.dng" ),
			width = 500,
			height = 334,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "ir-trees.dng",
			fileSize = 165598,
		},
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "ir-trees2.dng" ),
			width = 500,
			height = 334,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "ir-trees2.dng",
			fileSize = 165598,
		},
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "ir-trees2.dng" ),
			width = 500,
			height = 334,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "ir-trees2.dng",
			fileSize = 165598,
		}
	}

	local addImageStream = rxTest:observe( WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArray ) )
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectCompleted()

	rxTest:expectOnlyCompleted(
		WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArray )
			:filter( function( result )
				return not result.err
			end ) )

	rxTest:expectSingleValue(
		fakeWFCatalog:docStore():changedDocs { fromChangeCounter = 1 }
		:map( mapPluckFileName )
		:map( function( result )
			-- Since ir-trees.dng and ir-trees2.dng are copies of each other, we
			-- don't know which one will get imported first. Either is acceptible.
			
			if result[ 6 ] == 'ir-trees2.dng' then
				result[ 6 ] = 'ir-trees.dng'
			end
			
			return result
		end ),
		{
			"IMG_7706.PNG",
			"IMG_7763.jpg",
			"IMG_7766.jpg",
			"IMG_9876.CR2",
			"NikonPhotoworldSafariMiami281.NEF",
			"ir-trees.dng",
		} )

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestAddImagesFromArrayAddDuplicates( rxTest )

	rxTest:setTimeLimit( 15 )

	local WFXmp = require 'WFXmp'

	local fakeWFCatalog = createFakeWFCatalog( rxTest )
	local docStore = fakeWFCatalog:docStore()

	local metadata = WFXmp.WFXmp()
	metadata:set( WFXMLNS.xap, "ModifyDate", "2019-12-11T10:09:08.765Z" )
	metadata:set( WFXMLNS.xap, "Rating", "4" )
	metadata:setStringList( WFXMLNS.dc, "subject", { "mary", "had", "a little lamb" } )
	metadata:set( WFXMLNS.exif, "GPSAltitude", "10000/2" )
	metadata:set( WFXMLNS.exif, "GPSAltitudeRef", "1" )
	metadata:set( WFXMLNS.exif, "GPSLongitude", "100,5.125S" )
	metadata:set( WFXMLNS.exif, "GPSLatitude", "20,1.5" )

	local importArray = {
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7706.PNG" ),
			width = 640,
			height = 1136,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7706.PNG",
			fileSize = 9609,
		},
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7763.jpg" ),
			width = 3264,
			height = 2448,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7763.jpg",
			fileSize = 2072173,
		},
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7766.jpg" ),
			width = 1200,
			height = 1600,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7766.jpg",
			fileSize = 703991,
		},
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "IMG_9876.CR2" ),
			width = 2048,
			height = 3456,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_9876.CR2",
			fileSize = 26243176,
		},
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "NikonPhotoworldSafariMiami281.NEF" ),
			width = 3872,
			height = 2592,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "NikonPhotoworldSafariMiami281.NEF",
			fileSize = 9844596,
		},
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "ir-trees.dng" ),
			width = 500,
			height = 334,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "ir-trees.dng",
			fileSize = 165598,
		}
	}

	local addImageStream = rxTest:observe( WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArray ) )
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectCompleted()

	for i, fileInfo in ipairs( importArray ) do
		if i % 2 == 0 then
			fileInfo.importIfDuplicate = true
		end
	end
	
	rxTest:expectSingleValue(
		WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArray )
			:filter( function( result )
				return not result.err
			end )
			:toArray()
			:flatMapLatest( function( docArray )
			
				return Observable.from( docArray )
					:flatMap( function( result )
						return docStore:docValue( result.assetId )
					end )
					:toArray()
			end )
			:tap( function( docs )
				-- Verify that the originals have a unique master (fileName has '-2' appended)
				assert( type( docs ) == 'table' )
				for _, doc in ipairs( docs ) do
					local relativePath = KSTableUtils.derefOpt( doc, '_localOnly', 'files', 'original', 'master', 'relativePath' )
					assert( string.sub( relativePath, -6, -4 ) == '-2.', 'Not mapped to a unique original' )
				end
			end )
			:map( mapPluckFileName ),
		{
			"IMG_7763.jpg",
			"IMG_9876.CR2",
			"ir-trees.dng",
		}
	)

	rxTest:expectSingleValue(
		fakeWFCatalog:docStore():changedDocs { fromChangeCounter = 1 }
		:map( mapPluckFileName ),
		{
			"IMG_7706.PNG",
			"IMG_7763.jpg",
			"IMG_7763.jpg",
			"IMG_7766.jpg",
			"IMG_9876.CR2",
			"IMG_9876.CR2",
			"NikonPhotoworldSafariMiami281.NEF",
			"ir-trees.dng",
			"ir-trees.dng",
		} )

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestAddImagesFromArrayAddDuplicatesToAlbum( rxTest )

	-- Add an asset, then try to add a duplicate asset. But this time, try to add the duplicate
	-- to an album. The duplicate should not be imported, but the original assets should be added
	-- to the album

	rxTest:setTimeLimit( 15 )

	local WFXmp = require 'WFXmp'

	local fakeWFCatalog = createFakeWFCatalog( rxTest )

	local metadata = WFXmp.WFXmp()
	metadata:set( WFXMLNS.xap, "ModifyDate", "2019-12-11T10:09:08.765Z" )
	metadata:set( WFXMLNS.xap, "Rating", "4" )
	metadata:setStringList( WFXMLNS.dc, "subject", { "mary", "had", "a little lamb" } )
	metadata:set( WFXMLNS.exif, "GPSAltitude", "10000/2" )
	metadata:set( WFXMLNS.exif, "GPSAltitudeRef", "1" )
	metadata:set( WFXMLNS.exif, "GPSLongitude", "100,5.125S" )
	metadata:set( WFXMLNS.exif, "GPSLatitude", "20,1.5" )

	local importArray = {
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7706.PNG" ),
			width = 640,
			height = 1136,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7706.PNG",
			fileSize = 9609,
		},
	}
	
	local importedAssetId
	
	local addImageStream = rxTest:observe(
		WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArray )
			:tap( function( importedInfo )
				importedAssetId = importedInfo and importedInfo.localId
			end )
	)
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectCompleted()
	

	-- Create an album
	local albumId
	local albumName = "Album 1"

	rxTest:expectOnlyCompleted(

		WFCatAlbums.createAlbum( fakeWFCatalog, albumName )
			:tap( function( localAlbumId, globalAlbumId )
				albumId = localAlbumId
			end )
			:takeLast( 0 )
	)

	-- Add the image again - this is a duplicate and should be reported. But the original image should be added to the album
	
	local albumCountsStream = rxTest:observe( WFCatAlbums.getAlbumListForAssetIds( fakeWFCatalog, { importedAssetId } )
		:map( function( albums )
			return KSTableUtils.derefOpt( albums, 1, 'docId' )
		end )
	)
	
	local importOptions = {
		albumId = albumId,
	}

	local addDuplicateStream = rxTest:observe( 
		WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArray, nil, importOptions )
			:filter( function( result )
				return not result.err
			end )
	)
	addDuplicateStream:expectCompleted()

	albumCountsStream:expectDataWithinTime( 2, albumId )

end

--------------------------------------------------------------------------------

local function mapPluckAssetIds( docs )

	if type( docs ) ~= 'table' then return docs end

	for i, doc in ipairs( docs ) do
		docs[ i ] = doc._id
	end

	docs.continueFrom = nil

	table.sort( docs )

	return docs

end
--------------------------------------------------------------------------------

function WFCatImportTest.rxTestAddImagesFromArrayWithCancel( rxTest )

	rxTest:setTimeLimit( 15 )

	local WFXmp = require 'WFXmp'

	local fakeWFCatalog = createFakeWFCatalog( rxTest )

	local metadata = WFXmp.WFXmp()
	metadata:set( WFXMLNS.xap, "ModifyDate", "2019-12-11T10:09:08.765Z" )
	metadata:set( WFXMLNS.xap, "Rating", "4" )
	metadata:setStringList( WFXMLNS.dc, "subject", { "mary", "had", "a little lamb" } )
	metadata:set( WFXMLNS.exif, "GPSAltitude", "10000/2" )
	metadata:set( WFXMLNS.exif, "GPSAltitudeRef", "1" )
	metadata:set( WFXMLNS.exif, "GPSLongitude", "100,5.125S" )
	metadata:set( WFXMLNS.exif, "GPSLatitude", "20,1.5" )
	
	local importSidecarData = tostring( metadata )

	local importArray = {
		{
			xmp = tostring( metadata:serialize() ),
			additionalInfo = {
				XMPSideCarData = importSidecarData,
			},
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7706.PNG" ),
			width = 640,
			height = 1136,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7706.PNG",
			fileSize = 9609,
		},
		{
			xmp = tostring( metadata:serialize() ),
			additionalInfo = {
				XMPSideCarData = importSidecarData,
			},
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7763.jpg" ),
			width = 3264,
			height = 2448,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7763.jpg",
			fileSize = 2072173,
		},
		{
			xmp = tostring( metadata:serialize() ),
			additionalInfo = {
				XMPSideCarData = importSidecarData,
			},
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7766.jpg" ),
			width = 1200,
			height = 1600,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7766.jpg",
			fileSize = 703991,
		},
		{
			xmp = tostring( metadata:serialize() ),
			additionalInfo = {
				XMPSideCarData = importSidecarData,
			},
			filePath = KSPathUtils.child( testDataFolder1, "IMG_9876.CR2" ),
			width = 2048,
			height = 3456,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_9876.CR2",
			fileSize = 26243176,
		},
		{
			xmp = tostring( metadata:serialize() ),
			additionalInfo = {
				XMPSideCarData = importSidecarData,
			},
			filePath = KSPathUtils.child( testDataFolder1, "NikonPhotoworldSafariMiami281.NEF" ),
			width = 3872,
			height = 2592,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "NikonPhotoworldSafariMiami281.NEF",
			fileSize = 9844596,
		},
		{
			xmp = tostring( metadata:serialize() ),
			additionalInfo = {
				XMPSideCarData = importSidecarData,
			},
			filePath = KSPathUtils.child( testDataFolder1, "ir-trees.dng" ),
			width = 500,
			height = 334,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "ir-trees.dng",
			fileSize = 165598,
		}
	}

	local importResults = {}

	local addImageStream
	
	local importOptions = {
		batchSize = 2,
		numParallel = 2,
	}

	addImageStream = rxTest:observe( WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArray, nil, importOptions )
		:tap( function( importResult )
			table.insert( importResults, importResult )
			if #importResults == 2 then
				addImageStream:dispose()
			end
		end )
	)

	-- Only get one value as we disposed before the second one completed the tap.
	-- However in addition to the second asset more may have been in flight and will
	-- be imported.  (may get 2 or 4 assets for this case)

	addImageStream:expectData( ':ignore:' )

	-- Give the import some time to complete the ones started
	rxTest:sleep( 5 )
									
	local leftOverOriginalsStream = fakeWFCatalog:docStore():changedDocs { fromChangeCounter = 1 }
			:map( mapPluckFileName )
			:map( function( filesImported )

				-- There is typically 2 or 4 files completed at this point.

				local folderToScan = fakeWFCatalog:fileSpec():pathForOriginal(
					"Image.jpg", "2019-12-11T10:09:08.765" )

				folderToScan = KSPathUtils.parent( KSPathUtils.parent( KSPathUtils.parent( folderToScan ) ) )

				local filesInDir = {}

				for path in WFFolderWalker( folderToScan ) do
					filesInDir[ #filesInDir + 1 ] = path
				end

				return #filesImported == #filesInDir and #filesImported ~= 6

			end )
			
	local leftOverSidecarsStream = fakeWFCatalog:docStore():changedDocs { fromChangeCounter = 1 }
			:map( mapPluckAssetIds )
			:map( function( assetIds )

				-- There is typically 2 or 4 files completed at this point.

				local folderToScan = fakeWFCatalog:fileSpec():pathForXmpSidecar( assetIds[ 1 ] )

				folderToScan = KSPathUtils.parent( KSPathUtils.parent( folderToScan ) )

				local filesInDir = {}

				for path in WFFolderWalker( folderToScan ) do
					filesInDir[ #filesInDir + 1 ] = path
				end

				return #assetIds == #filesInDir and #assetIds ~= 6

			end )
	
	leftOverOriginalsStream = rxTest:observe( leftOverOriginalsStream )
	leftOverOriginalsStream:expectDataWithinTime( 30, true )

	leftOverSidecarsStream = rxTest:observe( leftOverSidecarsStream )
	leftOverSidecarsStream:expectDataWithinTime( 1, true )

end

--------------------------------------------------------------------------------

local function createDstPath( rxTest, testName )

	local dstPath = KSPathUtils.child( KSPathUtils.getStandardFilePath( 'carouselDocuments' ),
		testName .. '-' .. KSIdGenerator.uuidInLowercaseHex() )

	pcall( WFFileUtils.mkdir, dstPath )

	WFTestFileUtils.addTestFolder( dstPath )

	rxTest:finally( function()
		WFFileUtils.delete( dstPath )
	end )

	return dstPath

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestAddImagesWithDateChanged( rxTest )

	rxTest:setTimeLimit( 15 )

	local WFXmp = require 'WFXmp'

	local fakeWFCatalog = createFakeWFCatalog( rxTest )
	local docStore = fakeWFCatalog:docStore()
	local userId = fakeWFCatalog:ozUserId()

	local metadata = WFXmp.WFXmp()
	metadata:set( WFXMLNS.xap, "ModifyDate", "2019-12-11T10:09:08.765Z" )
	metadata:set( WFXMLNS.xap, "Rating", "4" )
	metadata:setStringList( WFXMLNS.dc, "subject", { "mary", "had", "a little lamb" } )
	metadata:set( WFXMLNS.exif, "GPSAltitude", "10000/2" )
	metadata:set( WFXMLNS.exif, "GPSAltitudeRef", "1" )
	metadata:set( WFXMLNS.exif, "GPSLongitude", "100,5.125S" )
	metadata:set( WFXMLNS.exif, "GPSLatitude", "20,1.5" )

	local now = KSDate.timeInIso8601Format()
	
	local importArray = {
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7706.PNG" ),
			width = 640,
			height = 1136,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7706.PNG",
			fileSize = 9609,
		},
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7763.jpg" ),
			width = 3264,
			height = 2448,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7763.jpg",
			fileSize = 2072173,
			dateChanged = now,
		},
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7766.jpg" ),
			width = 1200,
			height = 1600,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7766.jpg",
			fileSize = 703991,
			dateChanged = now,
			forceChangedDate = true,
		},
	}
	
	local expectedValues = {
		[ "IMG_7706.PNG" ] = {
			ratings = {
				[ userId ] = {
					rating = 4,
				}
			},
		},
		[ "IMG_7763.jpg" ] = {
			ratings = {
				[ userId ] = {
					rating = 4,
					date = now,
				}
			},
		},
		[ "IMG_7766.jpg" ] = {
			ratings = {
				[ userId ] = {
					rating = 4,
					date = now,
				}
			},
			reviews = {
				[ userId ] = {
					flag = 'unflagged',
					date = now,
				}
			},
		},
	}

	local addedDocObservable = WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArray )
		:pluck( 'assetId' )
		:flatMap( function( assetId )
			return docStore:docValue( assetId )
		end )
		:map( function( docValue )
			
			return {
				ratings = docValue.ratings,
				reviews = docValue.reviews,
				fileName = docValue.importSource.fileName,
			}
		end )
		:toArray()
		:map( function( arr )
			
			local ret =  {}
			
			for _, val in ipairs( arr ) do
				ret[ val.fileName ] = val
				val.fileName = nil
			end
			
			return ret
		end )
		
	rxTest:expectSingleValue( addedDocObservable, expectedValues )

end

--------------------------------------------------------------------------------

local function testAddImageArrayWithXmpSidecar( rxTest, passXmpAsData )

	rxTest:setTimeLimit( 15 )

	local WFXmp = require 'WFXmp'

	local fakeWFCatalog = createFakeWFCatalog( rxTest )

	local metadata = WFXmp.WFXmp()
	metadata:set( WFXMLNS.xap, "ModifyDate", "2019-12-11T10:09:08.765Z" )
	metadata:set( WFXMLNS.xap, "Rating", "4" )
	metadata:setStringList( WFXMLNS.dc, "subject", { "mary", "had", "a little lamb" } )
	metadata:set( WFXMLNS.exif, "GPSAltitude", "10000/2" )
	metadata:set( WFXMLNS.exif, "GPSAltitudeRef", "1" )
	metadata:set( WFXMLNS.exif, "GPSLongitude", "100,5.125S" )
	metadata:set( WFXMLNS.exif, "GPSLatitude", "20,1.5" )

	local testFilePath = KSPathUtils.child( testDataFolder1, "IMG_9876.CR2" )

	local importFolder = createDstPath( rxTest, 'rxTestAddImageMoveOriginal' )
	local importFilePath = KSPathUtils.child( importFolder, 'IMG_9876.CR2' )
	WFFileUtils.copy( testFilePath, importFilePath )

	-- Add and override some stuff in the sidecar file
	local sidecarDate = "2020-12-12T11:10:09.876Z"
	local sidecarRating = 1

	local sidecarMetadata = WFXmp.WFXmp()

	-- modified items
	sidecarMetadata:set( WFXMLNS.xap, "ModifyDate", sidecarDate )
	sidecarMetadata:set( WFXMLNS.xap, "Rating", sidecarRating )

	-- same items
	sidecarMetadata:setStringList( WFXMLNS.dc, "subject", { "mary", "had", "a little lamb" } )
	sidecarMetadata:set( WFXMLNS.exif, "GPSAltitude", "10000/2" )
	sidecarMetadata:set( WFXMLNS.exif, "GPSAltitudeRef", "1" )
	sidecarMetadata:set( WFXMLNS.exif, "GPSLongitude", "100,5.125S" )
	sidecarMetadata:set( WFXMLNS.exif, "GPSLatitude", "20,1.5" )

	-- added items
	sidecarMetadata:set( WFXMLNS.xap, "CreatorTool", "Adobe Nimbus Test" )
	sidecarMetadata:set( WFXMLNS.crs, "RawFileName", "IMG_9876.cr2" )

	local sidecarData = sidecarMetadata:serialize()

	local importSidecarPath = nil
	local importSidecarData = nil

	if passXmpAsData then
		importSidecarData = tostring( sidecarData )
	else
		importSidecarPath = KSPathUtils.child( importFolder, 'IMG_9876.XMP' )
		KSData.saveToFile( sidecarData, importSidecarPath )
		
		rxTest:finally( function()
			WFFileUtils.delete( importSidecarPath )
		end )
	end

	local xmpProvidedToOz = tostring( metadata:serialize() )
	local importArray = {
		{
			xmp = xmpProvidedToOz,
			additionalInfo = {
				XMPSideCarPath = importSidecarPath,
				XMPSideCarData = importSidecarData,
			},
			filePath = importFilePath,
			width = 2048,
			height = 3456,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_9876.CR2",
			fileSize = 26243176,
		},
	}

	local addImageStream = rxTest:observe( WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArray )
		:pluck( 'assetId' )
		:flatMap( function( assetId )
			return fakeWFCatalog:docStore():docValue( assetId )
			:map( function( docValue )

				local docSidecarFilePath = WFCatFileSpec.getXmpSidecarPathFromDocValue( docValue )

				rxTest:expectEqual( true, docSidecarFilePath ~= nil )

				rxTest:expectEqual( 3028, WFCatFileSpec.getXmpSidecarFileSizeFromDocValue( docValue ) )

				rxTest:expectEqual( 3028, WFFileUtils.fileAttributes( docSidecarFilePath ).fileSize )

				return true

			end )
		end )
	)

	addImageStream:expectData( true )

	addImageStream:expectCompleted()

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestAddImageArrayWithXmpSidecarPath( rxTest )
	testAddImageArrayWithXmpSidecar( rxTest, false )
end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestAddImageArrayWithXmpSidecarData( rxTest )
	testAddImageArrayWithXmpSidecar( rxTest, true )
end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestAddImagesToCatalog( rxTest )

	rxTest:setTimeLimit( 15 )

	local fakeWFCatalog = createFakeWFCatalog( rxTest )

	rxTest:expectOnlyCompleted(
		WFCatImport.addImagesToCatalogFromPath( fakeWFCatalog, testDataFolder1 ) )

	-- NOTE: WFCatImportFileProcessor does a much more detailed job of testing the
	-- content of the generated docs. Not duplicating that here.

	local checkTimestampsFn = getTimestampsMatchFn( rxTest )

	rxTest:expectSingleValue(

		fakeWFCatalog:docStore():changedDocs { fromChangeCounter = 1 }
		:tap( checkTimestampsFn )
		:map( mapPluckFileName ),

		{
			"IMG_7706.PNG",
			"IMG_7763.JPG",
			"IMG_7766.jpg",
			"IMG_7772.mp4",
			"IMG_9876.CR2",
			"NikonPhotoworldSafariMiami281.NEF",
			"ir-trees.dng",
		} )

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestAddImagesFromPathIgnoringDuplicates( rxTest )

	rxTest:setTimeLimit( 15 )

	local fakeWFCatalog = createFakeWFCatalog( rxTest )

	rxTest:expectOnlyCompleted(
		WFCatImport.addImagesToCatalogFromPath( fakeWFCatalog, testDataFolder1 ) )

	rxTest:expectOnlyCompleted(
		WFCatImport.addImagesToCatalogFromPath( fakeWFCatalog, testDataFolder1 ) )

	rxTest:expectSingleValue(

		fakeWFCatalog:docStore():changedDocs { fromChangeCounter = 1 }:map( mapPluckFileName ),
		{
			"IMG_7706.PNG",
			"IMG_7763.JPG",
			"IMG_7766.jpg",
			"IMG_7772.mp4",
			"IMG_9876.CR2",
			"NikonPhotoworldSafariMiami281.NEF",
			"ir-trees.dng",
		} )

end

--------------------------------------------------------------------------------

local function mapPluckTimestamps( docs )

	if type( docs ) ~= 'table' then return docs end

	local timestampCounter = 0
	local timestamps = {}
	
	for i, doc in ipairs( docs ) do

		local timestamp = KSTableUtils.derefOpt( doc, 'importSource', 'importTimestamp' )
		local timestampName = timestamps[ timestamp ]
		if timestampName == nil then
			timestampCounter = timestampCounter + 1
			timestampName = "timestamp" .. timestampCounter
			timestamps[ timestamp ] = timestampName
		end

		local fileName = KSTableUtils.derefOpt( doc, 'importSource', 'fileName' ) or "(no file name)"

		docs[ i ] = fileName .. " - " .. timestampName

	end

	docs.continueFrom = nil

	table.sort( docs )

	return docs

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestAddImagesFromPathTimestamp( rxTest )

	rxTest:setTimeLimit( 15 )

	local fakeWFCatalog = createFakeWFCatalog( rxTest )

	rxTest:expectOnlyCompleted(
		WFCatImport.addImagesToCatalogFromPath( fakeWFCatalog, KSPathUtils.child( testDataFolder1, "IMG_7766.jpg" ) ) )

	rxTest:sleep(1)
		-- timestamps need to be at least one second apart

	rxTest:expectOnlyCompleted(
		WFCatImport.addImagesToCatalogFromPath( fakeWFCatalog, testDataFolder1 ) )

	rxTest:expectSingleValue(

		fakeWFCatalog:docStore():changedDocs { fromChangeCounter = 1 }:map( mapPluckTimestamps ),
		{
			"IMG_7706.PNG - timestamp2",
			"IMG_7763.JPG - timestamp2",
			"IMG_7766.jpg - timestamp1",
			"IMG_7772.mp4 - timestamp2",
			"IMG_9876.CR2 - timestamp2",
			"NikonPhotoworldSafariMiami281.NEF - timestamp2",
			"ir-trees.dng - timestamp2",
		} )

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestAddImageArrayToAlbum( rxTest )

	rxTest:setTimeLimit( 10 )

	local fakeWFCatalog = createFakeWFCatalog( rxTest )
	local docStore = fakeWFCatalog:docStore()
	local indexStore = fakeWFCatalog:indexStore()
	local searchSchema = fakeWFCatalog:searchSchema()
		
	-- Create an album
	local albumId
	local albumName = "Album 1"

	rxTest:expectOnlyCompleted(

		WFCatAlbums.createAlbum( fakeWFCatalog, albumName )
			:tap( function( localAlbumId, globalAlbumId )
				albumId = localAlbumId
			end )
			:takeLast( 0 )
	)

	local testFilePath = KSPathUtils.child( testDataFolder1, "IMG_7763.jpg" )

	-- Add an image to the catalog and the newly created album
	local doc_localId
	rxTest:expectSingleValue(
		WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, { {
			filePath = testFilePath,
			width = 2048,
			height = 1024,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "Image.jpg",
			fileSize = 4096,
			importTimestamp = "2016-03-07T05:20:34Z",
		} }, nil, {
			albumId = albumId,
		} )
		:map( function( result )
			rxTest:expectEqual( result.filePath, testFilePath )
			
			return docStore:docValue( result.assetId )
			:map( function( docValue )
				local result = {}
				for k, v in pairs( docValue ) do
					if k == 'type' or k == 'subtype' then
						result[ k ] = v
					end
				end
				doc_localId = docValue._localId
				return result
			end )
		end )
		:switch(),
		{
			subtype = "image",
			type = "asset",
		}
	)
	
	-- Verify that the asset has been added to the album
	local function makeSearcher( rxTest, docStore, indexStore, searchSchema )

		return function( predicates )
			local terms = WFCatSearchQuery.createQueryParamsStream( searchSchema, {
				predicates = predicates
			} )

			return WFCatSearchQuery.resultSetQuery( 
				indexStore,
				[[SELECT docId FROM assets @JOIN ORDER BY height DESC]],
				terms,
				'assets.docId'
			)
		end

	end
	
	local runQueryOfTermsFromSavedSearch = makeSearcher( rxTest, docStore, indexStore, searchSchema )

	rxTest:observe( runQueryOfTermsFromSavedSearch {
		type = 'album',
		subtype = 'collection',
		album = albumId,
		operator = '==',
	} )
	:expectDataWithinTime( 5, { { doc_localId } } )

end

--------------------------------------------------------------------------------

local function testAddImagesFromPathDeleteAddAgain( rxTest )

	-- This verifies that duplicate detection does not prevent re-adding
	-- a recently deleted image.

	rxTest:setTimeLimit( 15 )

	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )

	local fakeWFCatalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, 'NmbCatalog' )

	local sha256ChangeCounts = rxTest:observe(
		WFCatImportDuplicateFiles._testOnly_getSha256ChangesFeed()
		:scan( function( numSha256, numDeleted, changeType, docId )
			if changeType == 'sha256' then
				numSha256 = numSha256 + 1
			elseif changeType == 'deleted' then
				numDeleted = numDeleted + 1
			end
			return numSha256, numDeleted
		end, 0, 0 )
	)

	rxTest:expectOnlyCompleted(
		WFCatImport.addImagesToCatalogFromPath( fakeWFCatalog, testDataFolder1 ) )

	rxTest:expectSingleValue(
		fakeWFCatalog:docStore():changedDocs { fromChangeCounter = 1 }:map( mapPluckFileName ),
		{
			"IMG_7706.PNG",
			"IMG_7763.JPG",
			"IMG_7766.jpg",
			"IMG_7772.mp4",
			"IMG_9876.CR2",
			"NikonPhotoworldSafariMiami281.NEF",
			"ir-trees.dng",
		} )

	rxTest:expectOnlyCompleted(
		fakeWFCatalog:docStore():changedDocs { fromChangeCounter = 1 }
			:map( mapPluckAssetIds )
			:flatMap( function( assetIds )
				return WFCatDelete.deleteAssets( fakeWFCatalog, assetIds )
			end )
		)

	local changeCounterAfterDelete

	rxTest:expectSingleValue(
		fakeWFCatalog:docStore():changedDocs { fromChangeCounter = 1 }
			:map( function( docs )
				changeCounterAfterDelete = docs.continueFrom
				return docs.continueFrom
			end ),
		':ignore:' )

	rxTest:expectOnlyCompleted(
		WFCatImport.addImagesToCatalogFromPath( fakeWFCatalog, testDataFolder1 ) )

	-- Make sure we see 7 assets were aded to duplicate detction and 7 deleted.
	sha256ChangeCounts:expectDataWithinTime( 5, 7, 7 ):dispose()

	rxTest:expectSingleValue(
		fakeWFCatalog:docStore():changedDocs { fromChangeCounter = changeCounterAfterDelete }:map( mapPluckFileName ),
		{
			"IMG_7706.PNG",
			"IMG_7763.JPG",
			"IMG_7766.jpg",
			"IMG_7772.mp4",
			"IMG_9876.CR2",
			"NikonPhotoworldSafariMiami281.NEF",
			"ir-trees.dng",
		} )

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTest_CT_AddImagesFromPathDeleteAddAgain( rxTest )

	testAddImagesFromPathDeleteAddAgain( rxTest )

end

--------------------------------------------------------------------------------

local function testAddImagesFromArrayDeleteAddAgain( rxTest )

	rxTest:setTimeLimit( 15 )

	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', 'd45516d8f82c4b2cbe78610ff9042d8a', rxTest )

	local fakeWFCatalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec, 'NmbCatalog' )

	local importArray = {
		{
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7706.PNG" ),
			width = 640,
			height = 1136,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7706.PNG",
			fileSize = 9609,
		},
		{
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7763.jpg" ),
			width = 3264,
			height = 2448,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7763.jpg",
			fileSize = 2072173,
		},
		{
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7766.jpg" ),
			width = 1200,
			height = 1600,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7766.jpg",
			fileSize = 703991,
		},
	}


	local sha256Changes = rxTest:observe(
		WFCatImportDuplicateFiles._testOnly_getSha256ChangesFeed()
		:map( function( changeType, docId )
			return { changeType, docId }
		end )
		:scan( function( accum, entry )
			table.insert( accum, entry )
			return accum
		end, {} )
	)

	local assetIdsAdded = {}
	rxTest:expectSingleValue( WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArray )
		:pluck( 'assetId' )
		:toArray()
		:map( function( assetIds )
			assetIdsAdded = assetIds
			return #assetIdsAdded
		end )
	, 3 )

	rxTest:expectOnlyCompleted( WFCatDelete.deleteAssets( fakeWFCatalog, assetIdsAdded ) )

	-- Make sure the trash assets aren't included in the sha256 changes feed
	-- used by the duplicate detection.
	sha256Changes:expectDataWithinTime( 5, {
		{ 'sha256', assetIdsAdded[ 1 ] },
		{ 'sha256', assetIdsAdded[ 2 ] },
		{ 'sha256', assetIdsAdded[ 3 ] },
		{ 'deleted', assetIdsAdded[ 1 ] },
		{ 'deleted', assetIdsAdded[ 2 ] },
		{ 'deleted', assetIdsAdded[ 3 ] },
	} )

	sha256Changes:dispose()

	-- Make sure we don't see any errors if we try to import again
	rxTest:expectSingleValue( WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArray )
		:filter( function( result )
			return result.err
		end )
		:toArray()
	, {} )


end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTest_CT_AddImagesFromArrayDeleteAddAgain( rxTest )

	testAddImagesFromArrayDeleteAddAgain( rxTest, true )

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestAddImagesToCatalogWithProgress( rxTest )

	rxTest:setTimeLimit( 15 )

	local fakeWFCatalog = createFakeWFCatalog( rxTest )

	local progress = rxTest:observe(
		WFCatImport.addImagesToCatalogFromPathWithProgress( fakeWFCatalog, nil, testDataFolder1 )
			:map( function( progressInfo )

				if KSTableUtils.derefOpt( progressInfo, 'extraInfo', 'filePath' ) then
					progressInfo.extraInfo.filePath = "<path stripped>/" .. KSPathUtils.leafName( progressInfo.extraInfo.filePath )
				end

				return progressInfo

			end )
	)

	progress:expectDataWithinTime( 10,
		{
			processedCount = 2,
			description = "Importing files",
			errorCount = 0,
			percentComplete = 2 / 8,
			totalCount = 8,
		} )

	progress:expectDataWithinTime( 10,
		{
			processedCount = 4,
			description = "Importing files",
			errorCount = 0,
			percentComplete = 4 / 8,
			totalCount = 8,
		} )

	progress:expectDataWithinTime( 10,
		{
			processedCount = 5,
			description = "Importing files",
			errorCount = 0,
			percentComplete = 5 / 8,
			totalCount = 8,
		} )

	progress:expectDataWithinTime( 10,
		{
			processedCount = 8,
			description = "Importing files",
			errorCount = 1,
			extraInfo = {
				failure = "errorDuplicate",
				filePath = "<path stripped>/ir-trees2.dng",
				message = WFCatImportConstants.kMatchesExistingFile,
			},
			percentComplete = 1,
			totalCount = 8,
		} )

	progress:expectCompleted()

	rxTest:expectSingleValue(

		fakeWFCatalog:docStore():changedDocs { fromChangeCounter = 1 }:map( mapPluckFileName ),

		{
			"IMG_7706.PNG",
			"IMG_7763.JPG",
			"IMG_7766.jpg",
			"IMG_7772.mp4",
			"IMG_9876.CR2",
			"NikonPhotoworldSafariMiami281.NEF",
			"ir-trees.dng",
		} )

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestAddEmptyFolderWithProgress( rxTest )

	rxTest:setTimeLimit( 15 )

	local fakeWFCatalog = createFakeWFCatalog( rxTest )

	local emptyFolderPath = KSPathUtils.child(
		KSPathUtils.getStandardFilePath( 'temp' ),
		'rxTestAddEmptyFolderWithProgress' )

	pcall ( WFFileUtils.mkdir, emptyFolderPath )

	rxTest:finally( function()
		WFFileUtils.delete( emptyFolderPath )
	end )

	local progress = rxTest:observe(
		WFCatImport.addImagesToCatalogFromPathWithProgress( fakeWFCatalog, nil, emptyFolderPath ) )

	progress:expectCompleted()

	rxTest:expectSingleValue(

		fakeWFCatalog:docStore():changedDocs { fromChangeCounter = 1 }
		:map( mapPluckFileName ),

		{} )

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestAddBugFilesToCatalog( rxTest )

	rxTest:setTimeLimit( 15 )

	local fakeWFCatalog = createFakeWFCatalog( rxTest )

	rxTest:expectOnlyCompleted(
		WFCatImport.addImagesToCatalogFromPath( fakeWFCatalog, bugFilesFolder ) )

	rxTest:expectSingleValue(

		fakeWFCatalog:docStore():changedDocs { fromChangeCounter = 1 }:map( mapPluckFileName ),

		{
			"IMG_0203.CR2",
		} )

end

--------------------------------------------------------------------------------

-- This test is disabled because it requires big test data to be present 
-- on desktop.This test must stay disabled always. You can enable 
-- this test to run locally but remember to tweak file path and expected
-- number of files etc.
		
function WFCatImportTest.DISABLED_rxTestAddImagesToCatalogFromPath_BigImport( rxTest )

	rxTest:setTimeLimit( 3500 )
	-- WF.core.os
	local KSDate = require 'KSDate'
	local fakeWFCatalog = createFakeWFCatalog( rxTest )
	
	-- from desktop
	local bigImportPath = KSPathUtils.child( KSPathUtils.getStandardFilePath( 'desktop' ), "bigImport" )
	
	local startTime = KSDate.timeNowForAnimation()
	rxTest:expectOnlyCompleted(
	WFCatImport.addImagesToCatalogFromPath( fakeWFCatalog, bigImportPath ) )
	
	local endTime = KSDate.timeNowForAnimation() - startTime
	print( "IMPORTTIME " .. endTime )
	
	local fromChangeCounter = 0
	local nFiles = 0
	while true do
	
		local changes
		
		rxTest:expectSingleValue(
		fakeWFCatalog:docStore():changedDocs { fromChangeCounter = fromChangeCounter }
		:tap( function( result )
			changes = result
		end ),
		':ignore:' )
		
		if #changes == 0 then
			break
		end
		fromChangeCounter = changes.continueFrom
		nFiles = nFiles + #changes
		print( "Number of Files " .. nFiles )
	
	end
	
	rxTest:expectEqual( nFiles, 9311 )

end

--------------------------------------------------------------------------------

local function hasWillNotUploadMasterStream()

	return WFCatImportDuplicateFiles._testOnly_getSha256ChangesFeed()
				:filter( 
					function( _, _, _, _, willNotUploadMaster )

						return willNotUploadMaster

					end )
				:take( 2 )
				:toArray() 

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestImportUpdatesWillNotUploadMaster( rxTest )

	rxTest:setTimeLimit( 15 )

	local fakeWFCatalog = createFakeWFCatalog( rxTest )

	local importArrayNoMaster = {
		{
			importIfDuplicate = true,
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7706.PNG" ),
			width = 640,
			height = 1136,
			incrementalWhiteBalance = true,
			isRawFile = false,
			monochrome = false,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7706.PNG",
			fileSize = 9609,
		},
		{
			importIfDuplicate = true,
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7706.PNG" ),
			width = 640,
			height = 1136,
			incrementalWhiteBalance = true,
			isRawFile = false,
			monochrome = false,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7706.PNG",
			fileSize = 9609,
		},
	}	

	-- Make sure the hashes are loaded
	rxTest:expectSingleValue( WFCatImportDuplicateFiles.findExistingFileBySha256( fakeWFCatalog:docStore(),
		fakeWFCatalog:indexStore(), "" ), nil, nil )

	-- Import two copies of the same asset and update both of them with
	-- importSource.willNotUploadMaster set to true, and remove the
	-- original file path after they have been imported.

	local watchHasWillNotUploadMasterStream =
			rxTest:observe( hasWillNotUploadMasterStream() )

	local expectedAssetIdsWithoutMaster = {}

	rxTest:expectSingleValue(
		WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArrayNoMaster )
		:pluck( 'assetId' )
		:flatMap( function( assetId )
			table.insert( expectedAssetIdsWithoutMaster, assetId )

			return fakeWFCatalog:docStore():applyOverlay {
				_id = assetId,
				importSource = {
					willNotUploadMaster = true,
				},
				_localOnly = {
					files = {
						original = { _delete = true }
					}
				},
			}

		end )
		:toArray(),
	':ignore:')

	table.sort( expectedAssetIdsWithoutMaster )

	watchHasWillNotUploadMasterStream:expectData( ':ignore:' )
	watchHasWillNotUploadMasterStream:expectCompleted()

	local continueFrom
	
	rxTest:expectSingleValue(
		fakeWFCatalog:docStore():changedDocs { fromChangeCounter = 1 }
			:tap( function( changedDocs )
				continueFrom = changedDocs.continueFrom
			end )
			:map( mapPluckAssetIds ),
		expectedAssetIdsWithoutMaster )

	-- now set up a new import array and import the same file

	local importArray = {
		{
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7706.PNG" ),
			width = 640,
			height = 1136,
			incrementalWhiteBalance = true,
			isRawFile = false,
			monochrome = false,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7706.PNG",
			fileSize = 9609,
		},
	}

	-- It should update both assets since they are a duplicates of the
	-- same file.

	rxTest:expectSingleValue(
		WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArray )
		:pluck( 'assetId' )
		:toArray()
		:map( function( docs )
			table.sort( docs )
			return docs
		end ),
	expectedAssetIdsWithoutMaster )

	rxTest:expectSingleValue(
		fakeWFCatalog:docStore():changedDocs { fromChangeCounter = continueFrom }
			:map( mapPluckAssetIds ),
	expectedAssetIdsWithoutMaster )

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestAddImageMoveOriginal( rxTest )

	rxTest:setTimeLimit( 15 )

	local fakeWFCatalog = createFakeWFCatalog( rxTest )
	
	local srcFile = KSPathUtils.child( testDataFolder1, "IMG_7763.jpg" )
	
	local backupPath = createDstPath( rxTest, 'rxTestAddImageMoveOriginal' )
	local backupFile = KSPathUtils.child( backupPath, 'copied_image.jpg' )
	WFFileUtils.copy( srcFile, backupFile )
	
	rxTest:expectSingleValue(
		WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, { {
			filePath = backupFile,
			width = 2048,
			height = 1024,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "Image.jpg",
			fileSize = 4096,
			importTimestamp = "2016-03-07T05:20:34Z",
			moveOrCopyOriginal = true,
		} } )
		:tap( function( result )
			rxTest:expectEqual( result.filePath, backupFile )
			rxTest:expectEqual( false, WFFileUtils.exists( backupFile ) )
			rxTest:expectEqual( result.assetId, result.assetId:match( '[0-9a-fA-F]+' ) )
		end ), ':ignore:' )

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestAddImageMoveOriginalFail( rxTest )

	rxTest:setTimeLimit( 15 )

	local fakeWFCatalog = createFakeWFCatalog( rxTest )
	
	local srcFile = KSPathUtils.child( testDataFolder1, "IMG_7763.jpg" )
	
	local backupPath = createDstPath( rxTest, 'rxTestAddImageMoveOriginal' )
	local backupFile = KSPathUtils.child( backupPath, 'copied_image.jpg' )
	WFFileUtils.copy( srcFile, backupFile )
	
	local fileData = assert(
			KSData.memoryMappedContentsOfFile( backupFile ), "Fail to read KSData" )
	
	rxTest:expectSingleValue(
		WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, { {
			filePath = backupFile,
			width = 2048,
			height = 1024,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "Image.jpg",
			fileSize = 4096,
			importTimestamp = "2016-03-07T05:20:34Z",
			moveOrCopyOriginal = true,
		} } )
		:tap( function( result )
		
			-- If a file descriptor is open by any process, move operation fails for 
			-- windows but succeeds for mac. Hence the difference in expectation for
			-- the two platforms. The test is not supposed to run for android platform.
			
			local backupFileState = false
			if WFPlatformInfo.isWindows then
				backupFileState = "file"
			end
			
			rxTest:expectEqual( result.filePath, backupFile )
			rxTest:expectEqual( backupFileState, WFFileUtils.exists( backupFile ) )
			rxTest:expectEqual( result.assetId, result.assetId:match( '[0-9a-fA-F]+' ) )
	
			fileData:close()
	
		end ), ':ignore:' )

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestAddImagesStreamToCatalog( rxTest )
	rxTest:setTimeLimit( 15 )
	
	local fakeWFCatalog = createFakeWFCatalog( rxTest )
	
	local sourceSubject, importResultObservable = WFCatImport.addImagesToCatalog( fakeWFCatalog )
	
	local WFXmp = require 'WFXmp'

	local metadata = WFXmp.WFXmp()
	metadata:set( WFXMLNS.xap, "ModifyDate", "2019-12-11T10:09:08.765Z" )
	metadata:set( WFXMLNS.xap, "Rating", "4" )
	metadata:setStringList( WFXMLNS.dc, "subject", { "mary", "had", "a little lamb" } )
	metadata:set( WFXMLNS.exif, "GPSAltitude", "10000/2" )
	metadata:set( WFXMLNS.exif, "GPSAltitudeRef", "1" )
	metadata:set( WFXMLNS.exif, "GPSLongitude", "100,5.125S" )
	metadata:set( WFXMLNS.exif, "GPSLatitude", "20,1.5" )

	local importArray = {
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7706.PNG" ),
			width = 640,
			height = 1136,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7706.PNG",
			fileSize = 9609,
		},
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7763.jpg" ),
			width = 3264,
			height = 2448,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7763.jpg",
			fileSize = 2072173,
		},
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7766.jpg" ),
			width = 1200,
			height = 1600,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7766.jpg",
			fileSize = 703991,
		},
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "IMG_9876.CR2" ),
			width = 2048,
			height = 3456,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_9876.CR2",
			fileSize = 26243176,
		},
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "NikonPhotoworldSafariMiami281.NEF" ),
			width = 3872,
			height = 2592,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "NikonPhotoworldSafariMiami281.NEF",
			fileSize = 9844596,
		},
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder1, "ir-trees.dng" ),
			width = 500,
			height = 334,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "ir-trees.dng",
			fileSize = 165598,
		},
	}
	
	-- We don't have control over the order of the results - so tick off from a pending list
	local pendingResults = {
		[ KSPathUtils.child( testDataFolder1, "IMG_7706.PNG" ) ] = true,
		[ KSPathUtils.child( testDataFolder1, "IMG_7763.jpg" ) ] = true,
		[ KSPathUtils.child( testDataFolder1, "IMG_7766.jpg" ) ] = true,
		[ KSPathUtils.child( testDataFolder1, "IMG_9876.CR2" ) ] = true,
		[ KSPathUtils.child( testDataFolder1, "NikonPhotoworldSafariMiami281.NEF" ) ] = true,
		[ KSPathUtils.child( testDataFolder1, "ir-trees.dng" ) ] = true,
	}

	local addImageStream = rxTest:observe( importResultObservable
		:tap( function( result )
			assert( result.assetId )
			pendingResults[ result.filePath ] = nil
		end ) )
	
	for _, importInfo in ipairs( importArray ) do
		sourceSubject:onNext( importInfo )
	end
	
	sourceSubject:onCompleted()

	addImageStream:expectData( ':ignore:' )
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectData( ':ignore:' )
	addImageStream:expectCompleted()
	
	rxTest:expectEqual( pendingResults, {} )
	
end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestImportMergingSyncedAssets( rxTest )

	rxTest:setTimeLimit( 15 )

	local fakeWFCatalog = createFakeWFCatalog( rxTest )
	local metadata = WFXmp.WFXmp()
	metadata:set( WFXMLNS.dc, 'title', 'TitleOld' )
	metadata:setStringList( WFXMLNS.dc, 'subject', { 'oldTag1', 'oldTag2' } )
	
	
	local importArrayNoMaster = {
		{
			importIfDuplicate = true,
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7706.PNG" ),
			width = 640,
			height = 1136,
			incrementalWhiteBalance = true,
			isRawFile = false,
			monochrome = false,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7706.PNG",
			fileSize = 9609,
			xmp = tostring( metadata:serialize() ),
		}
	}	

	-- Import an asset and update them with
	-- importSource.willNotUploadMaster set to true, and remove the
	-- original file path after they have been imported.

	local syncedAssetId

	rxTest:expectSingleValue(
		WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArrayNoMaster )
		:pluck( 'assetId' )
		:flatMap( function( assetId )
			syncedAssetId = assetId

			return fakeWFCatalog:docStore():applyOverlay {
				_id = assetId,
				importSource = {
					willNotUploadMaster = true,
				},
				_localOnly = {
					files = {
						original = { _delete = true }
					}
				},
			}

		end )
		:toArray(),
	':ignore:')

	local continueFrom
	
	rxTest:expectSingleValue(
		fakeWFCatalog:docStore():changedDocs { fromChangeCounter = 1 }
			:tap( function( changedDocs )
				continueFrom = changedDocs.continueFrom
			end )
			:map( mapPluckAssetIds ),
		{ syncedAssetId } )

	-- now set up a new import array and import the same file

	metadata = WFXmp.WFXmp()
	metadata:set( WFXMLNS.dc, 'title', 'TitleNew' )
	metadata:setStringList( WFXMLNS.dc, 'subject', { 'newTag1' } )
	
	local importArray = {
		{
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7706.PNG" ),
			width = 640,
			height = 1136,
			incrementalWhiteBalance = true,
			isRawFile = false,
			monochrome = false,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7706.PNG",
			fileSize = 9609,
			ozAssetId = syncedAssetId,
			xmp = tostring( metadata:serialize() ),
		},
	}

	-- It should update both assets since they are a duplicates of the
	-- same file.

	rxTest:expectSingleValue(
		WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArray )
			:pluck( 'assetId' ),
		syncedAssetId )
	
	local updatedDocs
	rxTest:expectSingleValue(
		fakeWFCatalog:docStore():changedDocs { fromChangeCounter = continueFrom }
			:map( function( docs )
				if type( docs ) ~= 'table' then return end

				updatedDocs = docs
				return docs
			end),
		':ignore:' )
	
	rxTest:expectEqual( 1, #updatedDocs )
	rxTest:expectEqual( "TitleNew", updatedDocs[ 1 ].xmp.dc.title )
	
	local keywordSet = updatedDocs[ 1 ].xmp.dc.subject
	table.sort( keywordSet )
	rxTest:expectEqual( { newTag1 = true, oldTag1 = true, oldTag2 = true }, keywordSet )

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestImportNotYetSyncedAssets( rxTest )

	rxTest:setTimeLimit( 15 )

	local fakeWFCatalog = createFakeWFCatalog( rxTest )
	
	local syncedAssetId = KSIdGenerator.uuidInLowercaseHex( )
	
	local importArrayNoMaster = {
		{
			importIfDuplicate = true,
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7706.PNG" ),
			width = 640,
			height = 1136,
			incrementalWhiteBalance = true,
			isRawFile = false,
			monochrome = false,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7706.PNG",
			fileSize = 9609,
			ozAssetId = syncedAssetId,
		}
	}	

	-- Import an asset after providing an oz-asset-id.
	-- Validate that the asset is getting created with
	-- the same id.

	rxTest:expectSingleValue(
		WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArrayNoMaster )
		:map( function( result )
			
			return result and result.assetId
		end ),
	syncedAssetId )

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestImportMatchingSyncedAssetWithNoMaster( rxTest )

	rxTest:setTimeLimit( 15 )

	local fakeWFCatalog = createFakeWFCatalog( rxTest )
	
	local importArrayNoMaster = {
		{
			importIfDuplicate = true,
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7706.PNG" ),
			width = 640,
			height = 1136,
			incrementalWhiteBalance = true,
			isRawFile = false,
			monochrome = false,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7706.PNG",
			fileSize = 9609,
			sha256 = "ABCD",
		}
	}	

	-- Import an asset and update them with
	-- importSource.willNotUploadMaster set to true, and remove the
	-- original file path after they have been imported.

	local syncedAssetId

	rxTest:expectSingleValue(
		WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArrayNoMaster )
		:pluck( 'assetId' )
		:flatMap( function( assetId )
			syncedAssetId = assetId

			return fakeWFCatalog:docStore():applyOverlay {
				_id = assetId,
				importSource = {
					willNotUploadMaster = true,
				},
				_localOnly = {
					files = {
						original = { _delete = true }
					}
				},
			}

		end )
		:toArray(),
	':ignore:')
	
	local continueFrom
	
	-- Verify that the information from the new import has overridden the one we got from LrClassic
	rxTest:expectSingleValue( fakeWFCatalog:docStore():docValue( syncedAssetId )
		:map( function( docValue )
			
			rxTest:expectEqual( docValue.importSource.sha256, 'ABCD' )
			rxTest:expectEqual( docValue.importSource.willNotUploadMaster, true )
			rxTest:expectEqual( docValue.importSource.fileSize, 9609 )
			
			return true
		end ), true )

	rxTest:expectSingleValue(
		fakeWFCatalog:docStore():changedDocs { fromChangeCounter = 1 }
			:tap( function( changedDocs )
				continueFrom = changedDocs.continueFrom
			end )
			:map( mapPluckAssetIds ),
		{ syncedAssetId } )
	-- now set up a new import array and import the same file
	
	local importArray = {
		{
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7706.PNG" ),
			width = 640,
			height = 1136,
			incrementalWhiteBalance = true,
			isRawFile = false,
			monochrome = false,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7706.PNG",
			fileSize = 10101010,
			ozAssetId = syncedAssetId,
		},
	}

	-- It should add the master file association

	rxTest:expectSingleValue(
		WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArray )
		:pluck( 'assetId' ),
	syncedAssetId )
	
	local updatedDocs
	rxTest:expectSingleValue(
		fakeWFCatalog:docStore():changedDocs { fromChangeCounter = continueFrom }
			:map( function( docs )
				if type( docs ) ~= 'table' then return end

				updatedDocs = docs
				return docs
			end),
		':ignore:' )
	
	rxTest:expectEqual( 1, #updatedDocs )
	rxTest:expectEqual( syncedAssetId, updatedDocs[ 1 ]._id )
	
	-- Check if willNotUploadMaster has been deleted
	rxTest:expectEqual( nil, updatedDocs[ 1 ].importSource.willNotUploadMaster )
	
	-- Check if original file exists
	rxTest:expectEqual( true, not not updatedDocs[ 1 ]._localOnly.files.original )
	
	
	-- Verify that the information from the new import has overridden the one we got from LrClassic
	rxTest:expectSingleValue( fakeWFCatalog:docStore():docValue( syncedAssetId )
		:map( function( docValue )
			
			rxTest:expectEqual( docValue.importSource.sha256, '246b14a2d88ef37934fe8cff242c788b49c9c1a3422cf157c829edef7b6179e5' )
			rxTest:expectEqual( docValue.importSource.willNotUploadMaster, nil )
			rxTest:expectEqual( docValue.importSource.fileSize, 10101010 )
			
			return true
		end ), true )

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestImportNonMatchingSyncedAsset( rxTest )

	rxTest:setTimeLimit( 15 )

	local fakeWFCatalog = createFakeWFCatalog( rxTest )
	
	local importArrayNoMaster = {
		{
			importIfDuplicate = true,
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7706.PNG" ),
			width = 640,
			height = 1136,
			incrementalWhiteBalance = true,
			isRawFile = false,
			monochrome = false,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7706.PNG",
			fileSize = 9609,
			sha256 = "ABCD",
		}
	}	

	-- Import an asset and update them with
	-- importSource.willNotUploadMaster set to true, and remove the
	-- original file path after they have been imported.

	local syncedAssetId

	rxTest:expectSingleValue(
		WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArrayNoMaster )
		:pluck( 'assetId' )
		:flatMap( function( assetId )
			syncedAssetId = assetId

			return fakeWFCatalog:docStore():applyOverlay {
				_id = assetId,
				importSource = {
					willNotUploadMaster = true,
				},
				_localOnly = {
					files = {
						original = { _delete = true }
					}
				},
			}

		end )
		:toArray(),
	':ignore:')
	
	local continueFrom
	
	rxTest:expectSingleValue(
		fakeWFCatalog:docStore():changedDocs { fromChangeCounter = 1 }
			:tap( function( changedDocs )
				continueFrom = changedDocs.continueFrom
			end )
			:map( mapPluckAssetIds ),
		{ syncedAssetId } )

	-- now set up a new import array and import the same file
	
	local importArray = {
		{
			filePath = KSPathUtils.child( testDataFolder1, "IMG_7706.PNG" ),
			width = 640,
			height = 1136,
			incrementalWhiteBalance = true,
			isRawFile = false,
			monochrome = false,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "IMG_7706.PNG",
			fileSize = 9609,
			ozAssetId = syncedAssetId,
		},
	}

	-- It should throw error for the file since there is a mismatch

	rxTest:expectSingleValue(
		WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArray )
		:pluck( 'assetId' ),
	syncedAssetId )

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestAddImageArrayToSharedWithYouAlbum( rxTest )

	rxTest:setTimeLimit( 10 )

	local wfCatalog = createFakeWFCatalog( rxTest )
	local docStore = wfCatalog:docStore()
	local indexStore = wfCatalog:indexStore()

	-- Simulate an album that has been shared with you
	local spaceId = KSIdGenerator.uuidInLowercaseHex()
	local albumId = KSIdGenerator.uuidInLowercaseHex()
	
	local albumDoc = {

		_id = albumId,
		_spaceId = spaceId,
		type = 'album',
		subtype = 'collection',
		name = 'foo',

	}
	
	rxTest:expectSingleValue( wfCatalog:docStore():setDocValue( albumDoc ), ':ignore:' )

	-- Add an asset to the shared-with-you album.
	
	local testFilePath = KSPathUtils.child( testDataFolder1, "IMG_7763.jpg" )
	
	local assetId
	rxTest:expectSingleValue(
		WFCatImport.addImageArrayToCatalog( wfCatalog, nil, { {
			filePath = testFilePath,
			width = 2048,
			height = 1024,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "Image.jpg",
			fileSize = 4096,
			importTimestamp = "2016-03-07T05:20:34Z",
		} }, nil, {
			albumId = albumId,
			redactionOptions = false,
		} )
		:flatMapLatest( function( result )
			rxTest:expectEqual( result.filePath, testFilePath )
			
			assetId = result.assetId
			
			return docStore:docValue( result.assetId )
				:map( function( docValue )
					return {
						type = docValue.type,
						subtype = docValue.subtype,
						_spaceId = docValue._spaceId,
					}
				end )
		end ),
		{
			subtype = "image",
			type = "asset",
			_spaceId = spaceId,
		}
	)
	
	-- Verify that the asset has been added to the album
	local albumAssetValue = indexStore:query( {
		sql = "SELECT docId FROM album_asset_v2 WHERE albumId = ?",
		albumId,
	} )
		:flatMapLatest( function( rows )
			if rows and rows[ 1 ] then
				local albumAssetId = rows[ 1 ][ 1 ]
				return docStore:docValue( albumAssetId )
					:map( function( docValue )
						return {
							type = docValue.type,
							album = docValue.album,
							asset = docValue.asset,
							_spaceId = docValue._spaceId,
						}
					end )
			end
			
			return Observable.empty()
		end )
		
		
	albumAssetValue = rxTest:observe( albumAssetValue )
	albumAssetValue:expectDataWithinTime( 10, {
		type = "album_asset",
		album = {
			id = albumId
		},
		asset = {
			id = assetId
		},
		_spaceId = spaceId,
	} )
end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestFailedOriginalHash( rxTest )

	-- The mocked WFCrContext fails for the file 'Screen Shot 2015-06-11 at 11.17.59.png'

	rxTest:setTimeLimit( 5 )

	local WFXmp = require 'WFXmp'

	local fakeWFCatalog = createFakeWFCatalog( rxTest )

	local metadata = WFXmp.WFXmp()
	metadata:set( WFXMLNS.xap, "ModifyDate", "2019-12-11T10:09:08.765Z" )
	metadata:set( WFXMLNS.xap, "Rating", "4" )
	metadata:setStringList( WFXMLNS.dc, "subject", { "mary", "had", "a little lamb" } )
	metadata:set( WFXMLNS.exif, "GPSAltitude", "10000/2" )
	metadata:set( WFXMLNS.exif, "GPSAltitudeRef", "1" )
	metadata:set( WFXMLNS.exif, "GPSLongitude", "100,5.125S" )
	metadata:set( WFXMLNS.exif, "GPSLatitude", "20,1.5" )

	local importArray = {
		{
			xmp = tostring( metadata:serialize() ),
			filePath = KSPathUtils.child( testDataFolder2, "Screen Shot 2015-06-11 at 11.17.59.png" ),
			width = 640,
			height = 1136,
			xmpCameraRaw = '<xmp:camera stuff .. />',
			fileName = "Screen Shot 2015-06-11 at 11.17.59.png",
			fileSize = 9609,
		}
	}

	local addImageStream = rxTest:observe( WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, importArray )
		:filter( function( importedInfo )
			return importedInfo.err
		end ) )
		
	-- If we filter for errors, we should not get anything
	addImageStream:expectCompleted()

end

--------------------------------------------------------------------------------

function WFCatImportTest.rxTestUpdateDuplicate( rxTest )

	rxTest:setTimeLimit( 10 )

	local WFXmp = require 'WFXmp'
	local WFCatVersions = require 'WFCatVersions'

	local fakeWFCatalog = createFakeWFCatalog( rxTest )

	local metadata = WFXmp.WFXmp()
	metadata:set( WFXMLNS.xap, "ModifyDate", "2019-12-11T10:09:08.765Z" )
	metadata:set( WFXMLNS.xap, "Rating", "4" )
	metadata:setStringList( WFXMLNS.dc, "subject", { "mary", "had", "a little lamb" } )
	metadata:set( WFXMLNS.exif, "GPSAltitude", "10000/2" )
	metadata:set( WFXMLNS.exif, "GPSAltitudeRef", "1" )
	metadata:set( WFXMLNS.exif, "GPSLongitude", "100,5.125S" )
	metadata:set( WFXMLNS.exif, "GPSLatitude", "20,1.5" )

	local importInfo = {
		xmp = tostring( metadata:serialize() ),
		filePath = KSPathUtils.child( testDataFolder1, "IMG_7763.JPG" ),
		width = 640,
		height = 1136,
		xmpCameraRaw = '<xmp:camera stuff .. />',
		fileName = "Screen Shot 2015-06-11 at 11.17.59.png",
		fileSize = 9609,
		location = {
			city = "Tuticorin",
			state = "Tamil Nadu",
			country = "India",
		},
	}
	
	local importedAssetId

	local addImageStream = rxTest:observe( WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, { importInfo } )
		:filter( function( importedInfo )
			importedAssetId = importedInfo.assetId
			return importedInfo.err
		end ) )
		
	-- If we filter for errors, we should not get anything
	addImageStream:expectCompleted()
	
	
	-- Keep a stream open observing the contents of the asset
	local assetContents = rxTest:observe( fakeWFCatalog:docStore():ongoingDocValue( importedAssetId )
		:map( function( docValue )
			return docValue.location.city, docValue.develop.xmpCameraRaw
		end ) )
		
	assetContents:expectData( "Tuticorin", 	{ sha256 = "13ada4e31740f4a6ef2f1dc05d959a0e078cb82106d2e1e555bb95cb1e3c346f" } )
	
	-- Also keep a stream observing the number of auto-versions of the asset
	local versionsCountStream = rxTest:observe( WFCatVersions.getAutoVersionsStreamForAsset( fakeWFCatalog, importedAssetId )
		:map( function( versions )
			return versions and #versions or 0
		end ) )
	versionsCountStream:expectData( 0 )
	
	
	-- Modify the develop settings and metadata. Import again by setting the dupe-check method to original digest.
	-- Import should fail
	
	importInfo.xmpCameraRaw = '<xmp:new camera stuff .. />'
	importInfo.location.city = "Kailasapuarm"
	
	local addDuplicateStream = rxTest:observe( WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, { importInfo }, nil, { useOriginalDigestToCheckDupes = true } )
		:map( function( importedInfo )
			return importedInfo.err
		end ) )
		
	addDuplicateStream:expectData( "matchesExistingFile" )
	addDuplicateStream:expectCompleted()
	
	-- Try importing again, but now, we tell it to update the duplicate.
	-- Import should succeed and the existing asset's develop settings should be updated
	local updateDevSettingsStream = rxTest:observe( WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, { importInfo }, nil, { useOriginalDigestToCheckDupes = true, updateDuplicates = true } )
		:filter( function( importedInfo )
			return importedInfo.err
		end ) )
		
	updateDevSettingsStream:expectCompleted()
	
	assetContents:expectData( "Tuticorin", { sha256 = "22f74408c82ed1b11072305a250e51ce089fc37aaa422d13113957d84053c877" } )

	-- Try importing again, but now, we tell it to update the duplicate.
	-- Import should succeed and the existing asset's develop settings should be updated
	local updateMetadataStream = rxTest:observe( WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, { importInfo }, nil, { useOriginalDigestToCheckDupes = true, updateDuplicates = true, updateMetadata = true } )
		:filter( function( importedInfo )
			return importedInfo.err
		end ) )
		
	updateMetadataStream:expectCompleted()
	
	assetContents:expectData( "Kailasapuarm", { sha256 = "22f74408c82ed1b11072305a250e51ce089fc37aaa422d13113957d84053c877" } )

	-- Make another change to the develop settings. Import it and create a version
	importInfo.xmpCameraRaw = '<xmp:another camera stuff .. />'
	local createVersionStream = rxTest:observe( WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, { importInfo }, nil, { useOriginalDigestToCheckDupes = true, updateDuplicates = true, updateMetadata = true, createVersionForUpdate = true } )
		:filter( function( importedInfo )
			return importedInfo.err
		end ) )
		
	createVersionStream:expectCompleted()
	
	assetContents:expectData( "Kailasapuarm", { sha256 = "4826659abc0a5decafacee100d5a7b739157df273a663e553007348115fc1c65" } )
	
	versionsCountStream:expectDataWithinTime( .9, 2 )
	
	
	-- Make another change to the develop settings (large string). Import it and create a version.
	importInfo.xmpCameraRaw = ''
	for i = 1, 1000 do
		importInfo.xmpCameraRaw = importInfo.xmpCameraRaw .. '<xmp:camera stuff .. />'
	end
	
	-- Sha256 of the above string
	local expectedSha256 = '68b55407aeefd5ccf85ad15081fda1d785f8b38acc56b8489ac6b4a8fbf803ca'
	
	local createLargeSettingsStream = rxTest:observe( WFCatImport.addImageArrayToCatalog( fakeWFCatalog, nil, { importInfo }, nil, { useOriginalDigestToCheckDupes = true, updateDuplicates = true, updateMetadata = true, createVersionForUpdate = true } )
		:filter( function( importedInfo )
			return importedInfo.err
		end ) )
		
	createLargeSettingsStream:expectCompleted()
	
	assetContents:expectData( "Kailasapuarm", { sha256 = expectedSha256 } )
	
	versionsCountStream:expectData( 3 )
	

end

--------------------------------------------------------------------------------

if WFPlatformInfo.isAndroid then
	WFCatImportTest.rxTestBasic = nil
	WFCatImportTest.rxTestAddImagesFromArrayIgnoringDuplicates = nil
	WFCatImportTest.rxTestAddImagesFromArrayAddDuplicates = nil
	WFCatImportTest.rxTestAddImageArrayToAlbum = nil
	WFCatImportTest.rxTestImportUpdatesWillNotUploadMaster = nil
	WFCatImportTest.rxTestAddImageMoveOriginal = nil
	WFCatImportTest.rxTestAddImageMoveOriginalFail = nil
	WFCatImportTest.rxTestAddImagesStreamToCatalog = nil

		-- TODO [scouten 2015-06-17]: This test is routinely failing on Android
		-- with timeout failures. Disable until we figure out why.
		--
		-- TODO [jandrew 2016-01-05]: This test is also disabled because it requires
		-- files that live in testdata/import-sources. We do not copy the contents
		-- of said folder to the Android emulator due to APK installation time
		-- constraints imposed by our build infrastructure. Until this is resolved,
		-- any test that references testdata in the import-sources folder must remain
		-- disabled on Android.
end

--------------------------------------------------------------------------------

-- TODO [kcarlson 2020-10-23]: These tests should be changed to use addImageArrayToCatalog as addImagesToCatalogFromPath no longer exists
if not WFPlatformInfo.hasACR or WFCrInfo.isMocked then
	WFCatImportTest.rxTestAddImagesToCatalog = nil
	WFCatImportTest.rxTestAddImagesToCatalogWithProgress = nil
	WFCatImportTest.rxTestAddBugFilesToCatalog = nil
	WFCatImportTest.rxTestAddImagesToCatalogFromPath_BigImport = nil
	WFCatImportTest.rxTestAddImagesFromPathIgnoringDuplicates = nil
	WFCatImportTest.rxTestAddEmptyFolderWithProgress = nil
	WFCatImportTest.rxTest_CT_AddImagesFromPathDeleteAddAgain = nil
	WFCatImportTest.rxTestAddImagesFromPathTimestamp = nil
end

--------------------------------------------------------------------------------

return WFRxTestCase.wrapSuite( 'WFCatImportTest', WFCatImportTest )
