--[[----------------------------------------------------------------------------

WFCatMetadataPullerTest.lua

--------------------------------------------------------------------------------

ADOBE CONFIDENTIAL
------------------
Copyright 2015 Adobe Systems Incorporated
All Rights Reserved.

NOTICE: All information contained herein is, and remain the property of
Adobe Systems Incorporated and its suppliers, if any. The intellectual and
technical concepts contained herein are proprietary to Adobe Systems
Incorporated and its suppliers and may be covered by U.S. and Foreign
Patents, patents in process, and are protected by trade secret or copyright
law.  Dissemination of this information or reproduction of this material is
strictly forbidden unless prior written permission is obtained from
Adobe Systems Incorporated.

------------------------------------------------------------------------------]]

	-- RxLua
local Rx = require 'Rx'

	-- WF.core.self_test
local WFRxTestCase = require 'WFRxTestCase'

	-- WF.core.utils
local KSIdGenerator = require 'KSIdGenerator'
local WFArguments = require 'WFArguments'

	-- WF.core.utils
local KSTableUtils = require 'KSTableUtils'

	-- WF.oz_client.managed_catalog
local WFCatMockCatalog = require 'WFCatMockCatalog'
local WFCatMetadataPuller = require 'WFCatMetadataPuller'
local WFCatSpecMock = require 'WFCatSpecMock'
local WFCatTestSession = require 'WFCatTestSession'
local WFCatSyncStatus = require 'WFCatSyncStatus'
local WFCatConflictResolver = require 'WFCatConflictResolver'

local WFRxIndexStore = require 'WFRxIndexStore'

local WFRxHttpClient = require 'WFRxHttpClient'

	-- WF.web_client.json
local KSJsonEncoder = require 'KSJsonEncoder'

--------------------------------------------------------------------------------

local check = assert( WFArguments.check )
local kUserId = '451502326e0c4ed9afe90c2ce310c4d4'
	-- Helpful bash one-liner. Put a new uuid on the clipboard (mac cmdline):
	-- uuidgen  | perl -p -e 's/-//g;s/[A-Z]/lc($&)/ge;' | pbcopy

--------------------------------------------------------------------------------

local function stripRevIds( changedDocsResult )

	for i, doc in ipairs( changedDocsResult ) do
		doc._revId = nil
	end

	return changedDocsResult

end

--------------------------------------------------------------------------------

local function stripWobblyBits( changedDocsResult )

	for i, doc in ipairs( changedDocsResult ) do
		doc._serverProvidedMetadata = nil
		-- Oz may create additional revisions of an asset automatically so the revId is wobbly in all cases
		doc._revId = nil
	end

	return changedDocsResult

end

--------------------------------------------------------------------------------

local function stripWobblyBitsPlusCounters( changedDocsResult )

	changedDocsResult = stripWobblyBits( changedDocsResult )

	for i, doc in ipairs( changedDocsResult ) do
		doc._changeCounter = nil
		doc._localId = nil
		doc.purgeDate = nil
		doc.userCreated = nil
		doc.userUpdated = nil

		if doc.subtype == 'deleted_image' then
			doc._id = nil
				-- We don't know the doc._id of deleted_images.
		end

	end

	return changedDocsResult

end

--------------------------------------------------------------------------------

local function sortByDocId( docs )
	table.sort( docs, function( a, b )
		local a_id = a._id or ""
		local b_id = b._id or ""
		return a_id < b_id
	end )
	return docs
end

--------------------------------------------------------------------------------

local catalogId = '98db98bfd554470dab2f862a4b853c40'

local catalogLink = "/v2/catalogs/" .. catalogId

local startingFeedLink = catalogLink .. "/updates"

local actualNextUpdateFeedLink = startingFeedLink .. '?updated_since=pull1'

--------------------------------------------------------------------------------

local spaceFeedId = KSIdGenerator.uuidInLowercaseHex()

local spaceLink = "/v2/spaces/" .. spaceFeedId

local startingSpaceFeedLink = spaceLink .. "/updates"

local actualNextUpdateSpaceFeedLink = startingSpaceFeedLink .. '?updated_since=pull1'

--------------------------------------------------------------------------------

local noopResponse =
	[[{
		"base": "https://photos-stage.adobe.io/v2/catalogs/98db98bfd554470dab2f862a4b853c40/",
		"updates": [],
		"links": {
			"next": {
				"href": "]] .. actualNextUpdateFeedLink .. [["
			}
		}
	}]]


local catalogPrefetchReqResp = {
		url = catalogLink,
		response = [[{
			"base": "https://photos-stage.adobe.io/v2/",
			"id": "]] .. catalogId .. [[",
			"type": "catalog"
			}]],
		headers = {
			status_code = '200',
		},
	}

local prefetchReqResp = {
			url = "/v2/catalogs/" .. catalogId .. "/albums?subtype=collection%3Bcollection_set",
		response = [[{
			"base": "https://photos-stage.adobe.io/v2/catalogs/]] .. catalogId .. [[/",
			"resources": []
			}]],
		headers = {
			status_code = '200',
		},
		options = {
			returnHttpErrorsAsData = true,
		}
	}

local function prefetchReqRespWithRetryDelay( delay )

	 return {
			url = "/v2/catalogs/" .. catalogId .. "/albums?subtype=collection%3Bcollection_set",
		response = [[{
			"base": "https://photos-stage.adobe.io/v2/catalogs/]] .. catalogId .. [[/",
			"resources": []
			}]],
		headers = {
			status_code = '200',
			[ 'Retry-After' ] = delay,
		},
		options = {
			returnHttpErrorsAsData = true,
		}
	}

end

local function getNoopCatalog( rxTest, requestCount )

 	requestCount = requestCount or 1

 	local headers = {
		status_code = '200',
	}

 	local expectedResponses = {
		prefetchReqResp,
		catalogPrefetchReqResp,
	}

 	for i = 1, requestCount do

		local url

		if i == 1 then
			url = startingFeedLink
		else
			url = actualNextUpdateFeedLink
		end

		expectedResponses[ #expectedResponses + 1 ] = {
			url = url,
			response = noopResponse,
			headers = headers,
			options = {
				bypassRateLimiter = true,
			}
		}
	end

 	local catalogSpec = WFCatSpecMock( catalogId, kUserId, rxTest, expectedResponses )

	return WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

 end

--------------------------------------------------------------------------------

local function getErrorCatalog( rxTest, overrideUpdateResponse, overrideNextLink )

 	local updates = overrideUpdateResponse or '[]'

 	local nextLink = overrideNextLink or actualNextUpdateFeedLink

 	local response =
		[[{
			"base": "https://photos-stage.adobe.io/v2/catalogs/98db98bfd554470dab2f862a4b853c40/",
			"updates": ]] .. updates .. [[,
			"links": {
				"next": {
					"href": "]] .. nextLink .. [["
				}
			}
		}]]

 	local headers = {
		status_code = '200',
	}

 	local expectedResponses = {
		prefetchReqResp,
		catalogPrefetchReqResp,
		{
			url = startingFeedLink,
			response = response,
			headers = headers,
			options = {
				bypassRateLimiter = true,
			}
		}
	}

 	local catalogSpec = WFCatSpecMock( catalogId, kUserId, rxTest, expectedResponses )

	return WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

 end

--------------------------------------------------------------------------------

local function verifyLogData( rxTest, actualLogData, expectedLogData, expectedUpdateFeedLink, maybeSpaceId )

	local nextUpdateFeedLink = actualLogData.nextUpdateFeedLink
	actualLogData.nextUpdateFeedLink = nil
	actualLogData.isOzCatalogDeleted = nil
	actualLogData.lastUpdateCycleWithData = nil
	actualLogData.isCurrentlySyncing = nil

	if maybeSpaceId then
		expectedUpdateFeedLink = expectedUpdateFeedLink or actualNextUpdateSpaceFeedLink
	else
		expectedUpdateFeedLink = expectedUpdateFeedLink or actualNextUpdateFeedLink
	end

	rxTest:expectEqual( expectedLogData, actualLogData )

	local nextUpdateFeedLinkDocId = 'Oz-nextUpdateFeedLink'

	if maybeSpaceId then
		nextUpdateFeedLinkDocId = string.format( "%s--%s", nextUpdateFeedLinkDocId, maybeSpaceId )
	end

	if expectedUpdateFeedLink == 'empty' then

		rxTest:expectSingleValue( nextUpdateFeedLink, {
			_empty = true,
			_id = nextUpdateFeedLinkDocId
		} )

	else

		rxTest:expectSingleValue( nextUpdateFeedLink, {
			link = expectedUpdateFeedLink,
			_id = nextUpdateFeedLinkDocId
		} )

	end


end

--============================================================================--

local WFCatMetadataPullerTest = {}

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.reportAllChanges( scanState, docValue )
	return scanState, docValue
end

--------------------------------------------------------------------------------

local options = {

	continuousTaskDelays = {
		progressiveBackoff = false,
		noWorkDelay = 0.5,
		maxNoWorkDelay = 1,
		errorDelay = 1,
		maxDelay = 1,
	},

	isCatalogPuller = true

}
--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestGetUpdates_realOz_testNightly( rxTest )

	rxTest:setTimeLimit( 120 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog

	rxTest:expectSingleValue(

		catalogTestSession:catalog()
			:tap( function( _catalog )
				check( 'WFCatalog' )
				catalog = _catalog
			end ),

		':ignore:' )

	local initialSyncObs = rxTest:observe( WFCatSyncStatus.isDoingInitialSync( catalog ) )

	initialSyncObs:expectDataWithinTime( 0.3, true )

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )

	local syncDiagnosticsObserver = rxTest:observe(
		puller:syncDiagnostics( 0.1, 1 )
			:map( function( diagnostics )
				diagnostics.recentlySyncedDocIds = nil
				return diagnostics
			end ) )

	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	syncDiagnosticsObserver:expectData(
		{
			documentsSyncedByType = {},
			isCurrentlySyncing = false,
			summaryStr = "Test Puller synced down 0 updates, 0 deletions from Oz.",
			totalDocumentsSynced = 0,
		} )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 0 }
			:map( stripWobblyBitsPlusCounters ),

		{
			{
				_id = catalog:ozCatalogId(),
				_labels = {
					oz = true,
				},
				name = "WFCatTestSession",
				subtype = "lightroom",
				type = "catalog",
			},
			continueFrom = 2,
		} )

	-- Very simple test: We push one document of each type to Oz.

	local catalogSpec

	rxTest:expectSingleValue(

		catalogTestSession:catalogSpec()
			:tap( function( _catalogSpec )
				check( 'WFCatSpec' )
				catalogSpec = _catalogSpec
			end ),

		':ignore:' )

	-- asset document ...

	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local assetDocRevId = KSIdGenerator.uuidInLowercaseHex()

	local assetDocPutUrl = string.format( '%s/assets/%s/revisions/%s',
										  catalogSpec:catalogHref(),
										  assetId, assetDocRevId )

	local assetDocContent = KSJsonEncoder.encodeOzUploadJson {
		user_id = catalogSpec:wfOzUser():userId(),
		payload = {
			captureDate = "2004-11-02T11:10:09",
			importSource = {
--				fileCreationDate = "2004-11-02T08:37:24",
					-- TODO [scouten 2016-01-29]: Withdrawn for now as Oz rejects
					-- this field. May come back depending on outcome of
					-- https://git.corp.adobe.com/oz/data-model/pull/156.
				importTimestamp = '2016-01-29T15:53:04Z',
				importedBy = catalogSpec:ozUserId(),
				fileName = "fakeFile01.jpg",
				originalHeight = 16,
				originalWidth = 16,
			},
		},
		type = 'asset',
		subtype = 'image',
	}

	local assetDocPutArgs = {
		requestHeaders = {
			{ header = 'Content-Type', value = 'application/json' },
			{ header = 'Accept', value = 'application/json' },
		},
	}

	rxTest:expectOnlyCompleted(
		catalogSpec:put( assetDocPutUrl, assetDocContent, assetDocPutArgs )
			:filter( function() return false end ) )

	-- album doc ...

	local albumId = KSIdGenerator.uuidInLowercaseHex()

	local albumDocPutUrl = string.format( '%s/albums/%s',
										  catalogSpec:catalogHref(),
										  albumId )

	local albumDocContent = KSJsonEncoder.encodeOzUploadJson {
		subtype = 'collection',
		payload = {
			name = "WFCatMetadataPullerTest album",

		},
			-- FIXME [scouten 2016-01-26]: Oz API documentation says there should
			-- be a top-level payload with name inside it. But current implementation
			-- of server doesn't accept that.
	}

	local albumDocPutArgs = {
		requestHeaders = {
			{ header = 'Content-Type', value = 'application/json' },
			{ header = 'Accept', value = 'application/json' },
		},
	}

	rxTest:expectOnlyCompleted(
		catalogSpec:put( albumDocPutUrl, albumDocContent, albumDocPutArgs )
			:filter( function() return false end ) )

	-- album_asset doc ...

	local albumAssetId = KSIdGenerator.uuidInLowercaseHex()

	local albumAssetDocPutUrl = string.format( '%s/albums/%s/assets',
											  catalogSpec:catalogHref(),
											  albumId )

	local albumAssetDocContent = KSJsonEncoder.encodeOzUploadJson {
		resources = {
			{
				id = assetId,
				album_asset_id = albumAssetId,
			},
		},
	}

	local albumAssetDocPutArgs = {
		requestHeaders = {
			{ header = 'Content-Type', value = 'application/json' },
			{ header = 'Accept', value = 'application/json' },
		},
	}

	rxTest:expectOnlyCompleted(
		catalogSpec:put( albumAssetDocPutUrl, albumAssetDocContent, albumAssetDocPutArgs )
			:filter( function() return false end ) )

	puller:getUpdatesContinuously()
		-- Now make sure puller grabs these documents from Oz.

	local expectedDocUpdates = {
			{
				_id = catalog:ozCatalogId(),
				_labels = {
					oz = true,
				},
				name = "WFCatTestSession",
				subtype = "lightroom",
				type = "catalog",
			},

			{
				_id = assetId,
				_labels = {
					oz = true,
				},
				captureDate = "2004-11-02T11:10:09",
				importSource = {
--					fileCreationDate = "2004-11-02T08:37:24",
						-- TODO [scouten 2016-01-29]: Withdrawn for now as Oz rejects
						-- this field. May come back depending on outcome of
						-- https://git.corp.adobe.com/oz/data-model/pull/156.
					importTimestamp = '2016-01-29T15:53:04Z',
					importedBy = catalogSpec:ozUserId(),
					fileName = "fakeFile01.jpg",
					originalHeight = 16,
					originalWidth = 16,
				},
				subtype = "image",
				type = "asset",
			},
			{
				_id = albumId,
				_labels = {
					oz = true,
				},
				name = "WFCatMetadataPullerTest album",
				subtype = "collection",
				type = "album",
			},
			{
				_id = albumAssetId,
				_labels = {
					oz = true,
				},
				album = {
					id = albumId,
				},
				asset = {
					id = assetId,
				},
				type = "album_asset",
			},
	}

	local resultDocUpdates = {}

	local changedDocsObserver = rxTest:observe(
		catalog:docStore():scannedFilteredMappedChangesFeedSince( "WFCatMetadataPullerTest" , "reportAllChanges", {}, 0 )
			:map( function( doc )
				resultDocUpdates[ #resultDocUpdates + 1 ] = KSTableUtils.shallowcopy( doc )
				return sortByDocId( resultDocUpdates )
			end )
			:map( stripWobblyBitsPlusCounters ) )

	changedDocsObserver:expectDataWithinTime( 15, sortByDocId( expectedDocUpdates ) )

	changedDocsObserver:dispose()

	syncDiagnosticsObserver:expectDataWithinTime( 5, {
		documentsSyncedByType = {
			update = {
				album = {
					collection = 1,
				},
				album_asset = {
					NULL = 1,
				},
				asset = {
					image = 1,
				},
				total = 3,
			},
		},
		isCurrentlySyncing = false,
		summaryStr = "Test Puller synced down 3 updates, 0 deletions from Oz.",
		totalDocumentsSynced = 3,
	} )

	-- Validate that all href's in the links section are valid

	rxTest:expectSingleValue(

		catalog:docStore():docValue( assetId )
			:map( function( docValue )

				local links = KSTableUtils.derefOpt( docValue, '_serverProvidedMetadata', 'links' ) or {}

				local allLinksValid = true

				for k, v in pairs( links ) do

					if k == '/rels/main_create/parts' or
						k == '/rels/aux_create/parts' or
						k == '/rels/versions/xmp_develop_create/parts' or
						k == '/rels/xmp_develop_create/parts' then

						-- used for direct upload, this href is a different structure than the other links in this payload
						if v.href and not v.href:match( '^http?s' ) then
							allLinksValid = false
						end

					else

						if v.href and not v.href:match( '^/' ) then
							allLinksValid = false
						end

					end

				end

				return allLinksValid

			end ), true )


	-- Now edit each of these documents and make sure those edits come through.

	local assetDocRev2Id = KSIdGenerator.uuidInLowercaseHex()

	local assetDocPutUrl2 = string.format( '%s/assets/%s/revisions/%s/children/%s',
										   catalogSpec:catalogHref(),
										   assetId,
										   assetDocRevId,
										   assetDocRev2Id )

	local assetDocContent2 = KSJsonEncoder.encodeOzUploadJson {
		user_id = catalogSpec:wfOzUser():userId(),
		payload = {
			xmp = {
				dc = {
					description = "mumble mumble mumble",
				},
			},
		},
	}

	local assetDocPutArgs2 = {
		requestHeaders = {
			{ header = 'Content-Type', value = 'application/json' },
			{ header = 'Accept', value = 'application/json' },
		},
	}

	rxTest:expectOnlyCompleted(
		catalogSpec:put( assetDocPutUrl2, assetDocContent2, assetDocPutArgs2 )
			:filter( function() return false end ) )

	-- album doc ...

	local albumDocUpdateContent = KSJsonEncoder.encodeOzUploadJson {
		payload = {
			name = "UPDATED WFCatMetadataPullerTest album",
		},
	}

	local albumDocPostArgs = {
		requestHeaders = {
			{ header = 'Content-Type', value = 'application/json' },
			{ header = 'Accept', value = 'application/json' },
		},
		requestMethod = 'POST',
	}

	rxTest:expectOnlyCompleted(
		catalogSpec:put( albumDocPutUrl, albumDocUpdateContent, albumDocPostArgs )
			:filter( function() return false end ) )

	-- album_asset doc ...

	local albumAssetDocUpdateUrl = string.format( '%s/albums/%s/assets/%s',
												  catalogSpec:catalogHref(),
												  albumId,
												  assetId )

	local albumAssetDocUpdateContent = KSJsonEncoder.encodeOzUploadJson {
		payload = {
			order = "X",
		}
	}

	local albumAssetDocUpdateArgs = {
		requestHeaders = {
			{ header = 'Content-Type', value = 'application/json' },
			{ header = 'Accept', value = 'application/json' },
		},
		requestMethod = 'POST',
	}

	rxTest:expectOnlyCompleted(
		catalogSpec:put( albumAssetDocUpdateUrl, albumAssetDocUpdateContent, albumAssetDocUpdateArgs )
			:filter( function() return false end ) )

	local resultEditDocUpdates = {}

	local expectedEditDocUpdates =
		{
			{
				_id = assetId,
				_labels = {
					oz = true,
				},
				captureDate = "2004-11-02T11:10:09",
				importSource = {
						-- fileCreationDate = "2004-11-02T08:37:24",
						-- TODO [scouten 2016-01-29]: Withdrawn for now as Oz rejects
						-- this field. May come back depending on outcome of
						-- https://git.corp.adobe.com/oz/data-model/pull/156.
					importTimestamp = '2016-01-29T15:53:04Z',
					importedBy = catalogSpec:ozUserId(),
					fileName = "fakeFile01.jpg",
					originalHeight = 16,
					originalWidth = 16,
				},
				subtype = "image",
				type = "asset",
				xmp = {
					dc = {
						description = "mumble mumble mumble",
					},
				},
			},
			{
				_id = albumId,
				_labels = {
					oz = true,
				},
				name = "UPDATED WFCatMetadataPullerTest album",
				subtype = "collection",
				type = "album",
			},
			{
				_id = albumAssetId,
				_labels = {
					oz = true,
				},
				album = {
					id = albumId,
				},
				asset = {
					id = assetId,
				},
				order = "X",
				type = "album_asset",
			},

		}

	local changedEditDocsObserver = rxTest:observe(
		catalog:docStore():scannedFilteredMappedChangesFeedSince( "WFCatMetadataPullerTest" , "reportAllChanges", {}, 5 )
			:map( function( doc )
				resultEditDocUpdates[ #resultEditDocUpdates + 1 ] = KSTableUtils.shallowcopy( doc )
				return sortByDocId( resultEditDocUpdates )
			end )
			:map( stripWobblyBitsPlusCounters ) )

	changedEditDocsObserver:expectDataWithinTime( 15, sortByDocId( expectedEditDocUpdates ) )

	changedEditDocsObserver:dispose()

	syncDiagnosticsObserver:expectDataWithinTime( 5, {
		documentsSyncedByType = {
			update = {
				album = {
					collection = 2,
				},
				album_asset = {
					NULL = 2,
				},
				asset = {
					image = 2,
				},
				total = 6,
			},
		},
		isCurrentlySyncing = false,
		summaryStr = "Test Puller synced down 6 updates, 0 deletions from Oz.",
		totalDocumentsSynced = 6,
	} )

	-- Now delete the album_asset document and make sure that comes through.
	-- Do this separately from the other docs because album_asset deletion
	-- *could* be implied from the other results.

	local albumAssetDocDeleteUrl = string.format( '%s/albums/%s/assets/%s',
												  catalogSpec:catalogHref(),
												  albumId,
												  assetId )

	local albumAssetDocDeleteArgs = {
		requestMethod = 'DELETE',
	}

	rxTest:expectOnlyCompleted(
		catalogSpec:get( albumAssetDocDeleteUrl, albumAssetDocDeleteArgs )
			:filter( function() return false end ) )

	local resultDeleteDocUpdates = {}

	local expectedDeleteDocUpdates =
		{
			{
				_deleted = true,
				_id = albumAssetId,
				_labels = {
					oz = true,
				},
				type = 'album_asset',
			},

		}

	local changedDeleteDocsObserver = rxTest:observe(
		catalog:docStore():scannedFilteredMappedChangesFeedSince( "WFCatMetadataPullerTest" , "reportAllChanges", {}, 8 )
			:map( function( doc )
				resultDeleteDocUpdates[ #resultDeleteDocUpdates + 1 ] = KSTableUtils.shallowcopy( doc )
				return sortByDocId( resultDeleteDocUpdates )
			end )
			:map( stripWobblyBitsPlusCounters ) )

	changedDeleteDocsObserver:expectDataWithinTime( 40, sortByDocId( expectedDeleteDocUpdates ) )

	changedDeleteDocsObserver:dispose()

	-- Now delete the other docs and make sure that comes through.

	local assetDocDeleteUrl = string.format( '%s/assets/%s', catalogSpec:catalogHref(), assetId )

	local assetDocDeleteArgs = {
		requestHeaders = {
			{ header = 'Content-Type', value = 'application/json' },
			{ header = 'Accept', value = 'application/json' },
		},
		requestMethod = 'DELETE',
	}

	rxTest:expectOnlyCompleted(
		catalogSpec:get( assetDocDeleteUrl, assetDocDeleteArgs )
			:filter( function() return false end ) )

	local albumDocDeleteUrl = string.format( '%s/albums/%s', catalogSpec:catalogHref(), albumId )

	local albumDocDeleteArgs = {
		requestHeaders = {
			{ header = 'Content-Type', value = 'application/json' },
			{ header = 'Accept', value = 'application/json' },
		},
		requestMethod = 'DELETE',
	}

	rxTest:expectOnlyCompleted(
		catalogSpec:get( albumDocDeleteUrl, albumDocDeleteArgs )
			:filter( function() return false end ) )

	resultDeleteDocUpdates = {}

	expectedDeleteDocUpdates =
		{
			{
				_deleted = true,
				_id = assetId,
				_labels = {
					oz = true,
				},
				type = "asset",
			},
			{
				_deleted = true,
				_id = albumId,
				_labels = {
					oz = true,
				},
				type = "album",
			},
			{
				_labels = {
					oz = true,
				},
				captureDate = "2004-11-02T11:10:09",
				importSource = {
					importTimestamp = '2016-01-29T15:53:04Z',
					importedBy = catalogSpec:ozUserId(),
					fileName = "fakeFile01.jpg",
					originalHeight = 16,
					originalWidth = 16,
				},
				original = {
					id = assetId,
					userCreated = "2016-01-29T15:53:04.000000Z",
				},
				subtype = "deleted_image",
				type = "asset",
				xmp = {
					dc = {
						description = "mumble mumble mumble",
					},
				},
			},
		}

	changedDeleteDocsObserver = rxTest:observe(
		catalog:docStore():scannedFilteredMappedChangesFeedSince( "WFCatMetadataPullerTest" , "reportAllChanges", {}, 9 )
			:map( function( doc )
				resultDeleteDocUpdates[ #resultDeleteDocUpdates + 1 ] = KSTableUtils.shallowcopy( doc )
				return sortByDocId( resultDeleteDocUpdates )
			end )
			:map( stripWobblyBitsPlusCounters ) )

	changedDeleteDocsObserver:expectDataWithinTime( 40, sortByDocId( expectedDeleteDocUpdates ) )

	changedDeleteDocsObserver:dispose()

	initialSyncObs:expectDataWithinTime( 0.5, false )

	syncDiagnosticsObserver:expectDataWithinTime( 5, {
		documentsSyncedByType = {
			delete = {
				album = {
					NULL = 1,
				},
				album_asset = {
					NULL = 1,
				},
				asset = {
					NULL = 1,
				},
				total = 3,
			},
			update = {
				album = {
					collection = 2,
				},
				album_asset = {
					NULL = 2,
				},
				asset = {
					deleted_image = 1,
					image = 2,
				},
				total = 7,
			},
		},
		isCurrentlySyncing = false,
		summaryStr = "Test Puller synced down 7 updates, 3 deletions from Oz.",
		totalDocumentsSynced = 10,
	} )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestGetUpdates_VideoAsset_InlineDevelopSettingSync_testNightly( rxTest )

	rxTest:setTimeLimit( 120 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog, catalogSpec, catalogId, docStore

	rxTest:waitFor(

		catalogTestSession:catalog()
			:map ( function( _catalog )
				check( 'WFCatalog' )
				catalog = _catalog
				catalogSpec = _catalog:ozCatalogSpec()
				catalogId = _catalog:ozCatalogSpec():catalogId()
				docStore = catalog:docStore()
				return _catalog
			end ) )

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )

	local syncDiagnosticsObserver = rxTest:observe(
	puller:syncDiagnostics( 0.1, 1 )
		:map( function( diagnostics )
			diagnostics.recentlySyncedDocIds = nil
			return diagnostics
		end ) )

	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	-- Crate a video asset document with inLine develop settings

	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local assetDocRevId = KSIdGenerator.uuidInLowercaseHex()

	local assetDocPutUrl = string.format( '%s/assets/%s/revisions/%s',
									  catalogSpec:catalogHref(),
										  assetId, assetDocRevId )
	-- Reuse inline develop settings
	local develop = {
		version = 1,
		processVersion = 1,
		croppedHeight = 1000,
		croppedWidth = 1000,
		adjustments = {
			userUpdated = "2020-09-15T12:18:38.23Z",
			adjustmentsFraction = 1000,
			exposure2012 = 0,
			contrast2012 = 0,
			highlights2012 = 0,
			shadows2012 = 0,
			whites2012 = 0,
			blacks2012 = 0,
			incrementalTemperature = 0,
			incrementalTint = 0,
			vibrance = 0,
			saturation = 0,
			hueAdjustmentRed = 0,
			hueAdjustmentOrange = 0,
			hueAdjustmentYellow = 0,
			hueAdjustmentGreen = 0,
			hueAdjustmentAqua = 0,
			hueAdjustmentBlue = 0,
			hueAdjustmentPurple = 0,
			hueAdjustmentMagenta = 0,
			saturationAdjustmentRed = 0,
			saturationAdjustmentOrange = 0,
			saturationAdjustmentYellow = 0,
			saturationAdjustmentGreen = 0,
			saturationAdjustmentAqua = 0,
			saturationAdjustmentBlue = 0,
			saturationAdjustmentPurple = 0,
			saturationAdjustmentMagenta = 0,
			luminanceAdjustmentRed = 0,
			luminanceAdjustmentOrange = 0,
			luminanceAdjustmentYellow = 0,
			luminanceAdjustmentGreen = 0,
			luminanceAdjustmentAqua = 0,
			luminanceAdjustmentBlue = 0,
			luminanceAdjustmentPurple = 0,
			luminanceAdjustmentMagenta = 0,
			splitToningHighlightHue = 0,
			splitToningHighlightSaturation = 0,
			splitToningBalance = 0,
			splitToningShadowHue = 0,
			splitToningShadowSaturation = 0,
			colorGradeMidtoneHue = 0,
			colorGradeMidtoneSat = 0,
			colorGradeShadowLum = 0,
			colorGradeMidtoneLum = 0,
			colorGradeHighlightLum = 0,
			colorGradeBlending = 0,
			colorGradeGlobalHue = 0,
			colorGradeGlobalSat = 0,
			colorGradeGlobalLum = 0,
			postCropVignetteAmount = 0,
			postCropVignetteRoundness = 0,
			postCropVignetteMidpoint = 50000,
			postCropVignetteFeather = 0,
			postCropVignetteHighlightContrast = 50000,
			grainAmount = 0,
			grainSize = 25000,
			grainFrequency = 50000,
			lutAmount = 0,
			profile = "neutral",
			geometric = {
				cropZoom = 100000,
				cropOffsetX = 0,
				cropOffsetY = 0,
				cropRectTop = 0,
				cropRectLeft = 0,
				cropRectBottom = 1000,
				cropRectRight = 1000,
				cropRotation = 0,
				cropFraction = 100000,
				orientation = "normal"
			}
		},
		temporal = {
			userUpdated = "2020-07-25T00:00:42.12Z",
			temporalFraction = 100000000,
			inPoint = 0,
			duration = 10000000
		}
	}

	local assetDocContent = KSJsonEncoder.encodeOzUploadJson {
		user_id = catalogSpec:wfOzUser():userId(),
		payload = {
			captureDate = "2004-11-02T11:10:09",
			importSource = {
				importTimestamp = '2016-01-29T15:53:04Z',
				importedBy = catalogSpec:ozUserId(),
				fileName = "IMG_7772.mp4",
				originalHeight = 480,
				originalWidth = 270,
			},
			develop = develop,
			video = {
				frameRate = { 93817 , 3215 },
				duration = { 416, 125 }
			}
		},
		type = 'asset',
		subtype = 'video',
	}

	local assetDocPutArgs = {
		requestHeaders = {
			{ header = 'Content-Type', value = 'application/json' },
			{ header = 'Accept', value = 'application/json' },
		},
	}

	rxTest:expectOnlyCompleted(
		catalogSpec:put( assetDocPutUrl, assetDocContent, assetDocPutArgs )
			:filter( function() return false end ) )

	puller:getUpdatesContinuously()
	-- Now make sure puller grabs these documents from Oz.

	local expectedDocUpdates = {
		{
			_id = catalog:ozCatalogId(),
			_labels = {
				oz = true,
			},
			name = "WFCatTestSession",
			subtype = "lightroom",
			type = "catalog",
		},

		{
			_id = assetId,
			_labels = {
				oz = true,
			},
			captureDate = "2004-11-02T11:10:09",
			importSource = {
				importTimestamp = '2016-01-29T15:53:04Z',
				importedBy = catalogSpec:ozUserId(),
				fileName = "IMG_7772.mp4",
				originalHeight = 480,
				originalWidth = 270,
			},
			video = {
				frameRate = { 93817 , 3215 },
				duration = { 416, 125 }
			},
			develop = develop,
			subtype = "video",
			type = "asset",
		},
	}

	local resultDocUpdates = {}

	local changedDocsObserver = rxTest:observe(
		catalog:docStore():scannedFilteredMappedChangesFeedSince( "WFCatMetadataPullerTest" , "reportAllChanges", {}, 0 )
			:map( function( doc )
				resultDocUpdates[ #resultDocUpdates + 1 ] = KSTableUtils.shallowcopy( doc )
				return sortByDocId( resultDocUpdates )
			end )
			:map( stripWobblyBitsPlusCounters ) )

	changedDocsObserver:expectDataWithinTime( 15, sortByDocId( expectedDocUpdates ) )

	changedDocsObserver:dispose()

	syncDiagnosticsObserver:expectDataWithinTime( 5, {
		documentsSyncedByType = {
			update = {
				asset = {
					video = 1,
				},
				total = 1,
			},
		},
		isCurrentlySyncing = false,
		summaryStr = "Test Puller synced down 1 updates, 0 deletions from Oz.",
		totalDocumentsSynced = 1,
	} )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestGetUpdates_retryAfter_testNightly( rxTest )

	rxTest:setTimeLimit( 120 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog, catalogSpec

	rxTest:expectSingleValue(

		catalogTestSession:catalog()
			:tap( function( _catalog )
				check( 'WFCatalog' )
				catalog = _catalog
				catalogSpec = _catalog:ozCatalogSpec()
			end ),

		':ignore:' )


	local retryAfterSecondsOnError = 10
	local retryAfterSecondsOnSuccess = 15

	local sawRetryAfterOnError = nil
	local sawRetryAfterOnSuccess = nil

	local inject429 = false
	local injectRetryAfter = false

	local catalogSpec_get = catalogSpec.get
	rawset( catalogSpec, 'get', function( self, url, options )

		local modifiedOptions = options and KSTableUtils.deepCopy( options ) or { }

		if inject429 then

			inject429 = false

			modifiedOptions.requestHeaders = modifiedOptions.requestHeaders or {}

			table.insert( modifiedOptions.requestHeaders, {
				header = 'X-Return-429-Status',
				value = 1,
			} )

			table.insert( modifiedOptions.requestHeaders, {
				header = 'X-Return-Retry-After',
				value = retryAfterSecondsOnError,
			} )

		end

		if injectRetryAfter then

			injectRetryAfter = false

			modifiedOptions.requestHeaders = modifiedOptions.requestHeaders or {}

			table.insert( modifiedOptions.requestHeaders, {
				header = 'X-Return-Retry-After',
				value = retryAfterSecondsOnSuccess,
			} )

		end

		return catalogSpec_get( self, url, modifiedOptions )
			:tap( function( responseData, responseHeader )
				if responseHeader and responseHeader[ 'Retry-After' ] then
					sawRetryAfterOnSuccess = tonumber( responseHeader[ 'Retry-After' ] )
				end
			end, function( responseData, responseHeader )
				if responseHeader and responseHeader[ 'Retry-After' ] then
					sawRetryAfterOnError = tonumber( responseHeader[ 'Retry-After' ] )
				end
			end )

	end )


	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	-- Prime the updates feed pump.

	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 0 }
			:map( stripWobblyBits ),

		{
			{
				_changeCounter = 1,
				_id = catalog:ozCatalogId(),
				_labels = {
					oz = true,
				},
				_localId = 1,
				name = "WFCatTestSession",
				subtype = "lightroom",
				type = "catalog",
			},
			continueFrom = 2,
		} )


	-- asset document ...

	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local assetDocRevId = KSIdGenerator.uuidInLowercaseHex()

	local assetDocPutUrl = string.format( '%s/assets/%s/revisions/%s',
										  catalogSpec:catalogHref(),
										  assetId, assetDocRevId )

	local assetDocContent = KSJsonEncoder.encodeOzUploadJson {
		user_id = catalogSpec:wfOzUser():userId(),
		payload = {
			captureDate = "2004-11-02T11:10:09",
			importSource = {
				importTimestamp = '2016-01-29T15:53:04Z',
				importedBy = catalogSpec:ozUserId(),
				fileName = "fakeFile01.jpg",
				originalHeight = 16,
				originalWidth = 16,
			},
		},
		type = 'asset',
		subtype = 'image',
	}

	local assetDocPutArgs = {
		requestHeaders = {
			{ header = 'Content-Type', value = 'application/json' },
			{ header = 'Accept', value = 'application/json' },
		},
	}

	rxTest:expectOnlyCompleted(
		catalogSpec:put( assetDocPutUrl, assetDocContent, assetDocPutArgs )
			:filter( function() return false end ) )


	-- Wait while Oz twiddles its bits.

	rxTest:sleep( 6 )

	inject429 = true

	-- This should take longer than the two retry delays added when retrying, also
	-- note that this is actually looking for an error.
	rxTest:expectSingleValueWithinTimeLimits( retryAfterSecondsOnError * 2, 60,
		puller:getUpdatesOnce()
		:catch( function( responseData, responseHeader )
			return Rx.Observable.just {
				[ 'Retry-After' ] = responseHeader[ 'Retry-After' ],
				retryCount = responseHeader.retryCount,
				status_code = responseHeader.status_code,
			}
		end ),

		{
			retryCount = 3,
			status_code = '429',
			[ 'Retry-After' ] = tostring( retryAfterSecondsOnError ),
		} )

	rxTest:expectEqual( retryAfterSecondsOnError, sawRetryAfterOnError )
	rxTest:expectEqual( nil, sawRetryAfterOnSuccess )

	sawRetryAfterOnError = nil

	-- Try again, this time it should be successful.  The response will cause the puller
	-- to delay the next pull but this call should complete quickly.
	injectRetryAfter = true
	rxTest:expectOnlyCompletedWithinTimeLimits( 0, retryAfterSecondsOnError - 1, puller:getUpdatesOnce() )

	rxTest:expectEqual( nil, sawRetryAfterOnError )
	rxTest:expectEqual( retryAfterSecondsOnSuccess, sawRetryAfterOnSuccess )

	sawRetryAfterOnSuccess = nil

	-- Nothing to get but this should wait the delay before completing.
	rxTest:expectOnlyCompletedWithinTimeLimits( retryAfterSecondsOnError, 60, puller:getUpdatesOnce() )

	rxTest:expectEqual( nil, sawRetryAfterOnError )
	rxTest:expectEqual( nil, sawRetryAfterOnSuccess )

	-- Nothing to get but now there is no Retry-After delay.
	rxTest:expectOnlyCompletedWithinTimeLimits( 0, retryAfterSecondsOnError - 1, puller:getUpdatesOnce() )

	rxTest:expectEqual( nil, sawRetryAfterOnError )
	rxTest:expectEqual( nil, sawRetryAfterOnSuccess )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 0 }
			:map( stripWobblyBits ),

		{
			{
				_changeCounter = 1,
				_id = catalog:ozCatalogId(),
				_labels = {
					oz = true,
				},
				_localId = 1,
				name = "WFCatTestSession",
				subtype = "lightroom",
				type = "catalog",
			},

			{
				_changeCounter = 2,
				_id = assetId,
				_labels = {
					oz = true,
				},
				_localId = 2,
				captureDate = "2004-11-02T11:10:09",
				importSource = {
					importTimestamp = '2016-01-29T15:53:04Z',
					importedBy = catalogSpec:ozUserId(),
					fileName = "fakeFile01.jpg",
					originalHeight = 16,
					originalWidth = 16,
				},
				subtype = "image",
				type = "asset",
			},
			continueFrom = 3,
		} )

	rxTest:expectSingleValue(

		catalog:docStore():docValue( assetId ):map( function( docValue )
			return KSTableUtils.derefOpt( docValue, '_serverProvidedMetadata', 'links' ) ~= nil
		end ),

		true )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestGetUpdates_retryAfterContinuous_testNightly( rxTest )

-- The purpose of this test is to make sure the WFCatMetadataPusher recovers from 429 errors
-- when run in continous mode.  So it doesn't check for any delays, delays are checked
-- in a different test.

	rxTest:setTimeLimit( 120 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog, catalogSpec

	rxTest:expectSingleValue(

		catalogTestSession:catalog()
			:tap( function( _catalog )
				check( 'WFCatalog' )
				catalog = _catalog
				catalogSpec = _catalog:ozCatalogSpec()
			end ),

		':ignore:' )

	local inject429 = false
	local injectRetryAfter = false

	local retryAfterSecondsOnError = 1
	local retryAfterSecondsOnSuccess = 1

	local sawRetryAfterOnError = nil
	local sawRetryAfterOnSuccess = nil

	local catalogSpec_get = catalogSpec.get
	rawset( catalogSpec, 'get', function( self, url, options )

		local modifiedOptions = options and KSTableUtils.deepCopy( options ) or { }

		if inject429 then

			inject429 = false
			injectRetryAfter = true

			modifiedOptions.requestHeaders = modifiedOptions.requestHeaders or {}

			table.insert( modifiedOptions.requestHeaders, {
				header = 'X-Return-429-Status',
				value = 1,
			} )

			table.insert( modifiedOptions.requestHeaders, {
				header = 'X-Return-Retry-After',
				value = retryAfterSecondsOnError,
			} )

		elseif injectRetryAfter then

			injectRetryAfter = false

			modifiedOptions.requestHeaders = modifiedOptions.requestHeaders or {}

			table.insert( modifiedOptions.requestHeaders, {
				header = 'X-Return-Retry-After',
				value = retryAfterSecondsOnSuccess,
			} )

		end

		return catalogSpec_get( self, url, modifiedOptions )
			:tap( function( responseData, responseHeader )
				if responseHeader and responseHeader[ 'Retry-After' ] then
					sawRetryAfterOnSuccess = tonumber( responseHeader[ 'Retry-After' ] )
				end
			end, function( responseData, responseHeader )
				if responseHeader and responseHeader[ 'Retry-After' ] then
					sawRetryAfterOnError = tonumber( responseHeader[ 'Retry-After' ] )
				end
			end )

	end )

	local options = {
		continuousTaskDelays = {
			progressiveBackoff = false,
			noWorkDelay = 1,
			maxNoWorkDelay = 1,
			errorDelay = 1,
			maxDelay = 1,
		},
		isCatalogPuller = true
	}

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	-- Prime the updates feed pump.

	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 0 }
			:map( stripWobblyBits ),

		{
			{
				_changeCounter = 1,
				_id = catalog:ozCatalogId(),
				_labels = {
					oz = true,
				},
				_localId = 1,
				name = "WFCatTestSession",
				subtype = "lightroom",
				type = "catalog",
			},
			continueFrom = 2,
		} )

	puller:getUpdatesContinuously()

	-- asset document ...

	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local assetDocRevId = KSIdGenerator.uuidInLowercaseHex()

	local assetDocPutUrl = string.format( '%s/assets/%s/revisions/%s',
										  catalogSpec:catalogHref(),
										  assetId, assetDocRevId )

	local assetDocContent = KSJsonEncoder.encodeOzUploadJson {
		user_id = catalogSpec:wfOzUser():userId(),
		payload = {
			captureDate = "2004-11-02T11:10:09",
			importSource = {
				importTimestamp = '2016-01-29T15:53:04Z',
				importedBy = catalogSpec:ozUserId(),
				fileName = "fakeFile01.jpg",
				originalHeight = 16,
				originalWidth = 16,
			},
		},
		type = 'asset',
		subtype = 'image',
	}

	local assetDocPutArgs = {
		requestHeaders = {
			{ header = 'Content-Type', value = 'application/json' },
			{ header = 'Accept', value = 'application/json' },
		},
	}

	rxTest:expectOnlyCompleted(
		catalogSpec:put( assetDocPutUrl, assetDocContent, assetDocPutArgs )
			:filter( function() return false end ) )

	local docUpdates = {}

	local expectedDocUpdates =
		{
			{
				_id = assetId,
				_labels = {
					oz = true,
				},
				captureDate = "2004-11-02T11:10:09",
				importSource = {
					importTimestamp = '2016-01-29T15:53:04Z',
					importedBy = catalogSpec:ozUserId(),
					fileName = "fakeFile01.jpg",
					originalHeight = 16,
					originalWidth = 16,
				},
				type = 'asset',
				subtype = 'image',
			}
		}

	local changedDocsObserver = rxTest:observe(
		catalog:docStore():scannedFilteredMappedChangesFeedSince( "WFCatMetadataPullerTest" , "reportAllChanges", {}, 2 )
			:map( function( doc )
				docUpdates[ #docUpdates + 1 ] = KSTableUtils.shallowcopy( doc )
				return sortByDocId( docUpdates )
			end )
			:map( stripWobblyBitsPlusCounters ) )

	changedDocsObserver:expectDataWithinTime( 40, sortByDocId( expectedDocUpdates ) )

	changedDocsObserver:dispose()

	inject429 = true

	rxTest:finally( function()
		puller:stopContinuousUpdates()
	end )

	rxTest:observe( catalog:docStore():ongoingDocValue( assetId ):map( function( docValue )
			return KSTableUtils.derefOpt( docValue, '_serverProvidedMetadata', 'links' ) ~= nil
		end ) )
		:expectDataWithinTime( 60, true )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestGetUpdatesOnce_mockEmpty( rxTest )

	rxTest:setTimeLimit( 10 )

	local catalog = getNoopCatalog( rxTest )
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 0 }
			:map( stripWobblyBits ),
		{
			{
				_changeCounter = 1,
				_id = catalogId,
				_labels = {
					oz = true,
				},
				_localId = 1,
				type = "catalog",
			},
			continueFrom = 2,
		} )

	local expectedLogData = {
		documentsReceivedByType = {},
		successfulUpdateCycles = 1,
		totalDocumentsReceived = 0,
	}

	verifyLogData( rxTest, puller:getDiagnosticLogData(), expectedLogData )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestGetUpdatesOnce_updateDoc_preservesLocalOnly( rxTest )

	rxTest:setTimeLimit( 10 )

	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', kUserId, rxTest,

		{
			prefetchReqResp,
			catalogPrefetchReqResp,
			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates",

					-- somewhat minimized version of payload

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "8462f5ba3b39fd050d3cd6b7352ba561",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"revision_ids": [
								"a1e880f718a348b89663cf38f9f29d23",
								"cd1ab30cf8464ef2a9e736b50c0ba33d",
								"96f33a7215e04c9aa49346bbf8aae921",
								"813fdbda1b244c2d9f41008ba63eedba"
							],
							"payload": {
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12",
								"xmp": {
									"dc": {
										"rights": "test copyright string"
									}
								},
								"location": {
									"longitude": -93.600107,
									"latitude": 45.951733,
									"altitude": 377.9
								}
							}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1",

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "8462f5ba3b39fd050d3cd6b7352ba561",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"revision_ids": [
								"a1e880f718a348b89663cf38f9f29d23",
								"cd1ab30cf8464ef2a9e736b50c0ba33d",
								"96f33a7215e04c9aa49346bbf8aae921",
								"813fdbda1b244c2d9f41008ba63eedba",
								"3a4a43dc2a9a4f1897c27f7a26e2e068"
							],
							"payload": {
								"xmp": {
									"dc": {
										"rights": "something changed"
									}
								},
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12"
							}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2",

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

		} )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 2 },

		{
			{
				_changeCounter = 2,
				_id = "8462f5ba3b39fd050d3cd6b7352ba561",
				_labels = {
					oz = true,
				},
				_localId = 2,
				_revId = "813fdbda1b244c2d9f41008ba63eedba",
				_serverProvidedMetadata = {
					base = "https://photos.adobe.io",
					created = "2015-05-27T00:34:56.669987Z",
					revision_ids = {
						"a1e880f718a348b89663cf38f9f29d23",
						"cd1ab30cf8464ef2a9e736b50c0ba33d",
						"96f33a7215e04c9aa49346bbf8aae921",
						"813fdbda1b244c2d9f41008ba63eedba",
					},
					updated = "2015-05-27T00:38:29.160369Z",
				},
				captureDate = "2015-05-17T15:50:12",
				importSource = {
					importTimestamp = "2015-05-26T17:34:23-07:00",
					originalHeight = 1600,
					originalWidth = 1068,
				},
				location = {
					altitude = 377.9,
					latitude = 45.951733,
					longitude = -93.600107,
				},
				subtype = "image",
				type = "asset",
				xmp = {
					dc = {
						rights = "test copyright string",
					},
				},
			},
			continueFrom = 3,
			} )

	-- Now add a _localOnly annotation to this document. It should be preserved
	-- when we pull changes again.

	rxTest:expectSingleValue(

		catalog:docStore():applyOverlay {
			_id = "8462f5ba3b39fd050d3cd6b7352ba561",
			_localOnly = {
				someImportantStuff = "that should survive",
			},
		},

		':ignore:' )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 3 }
			:map( stripRevIds ),

		{
			{
				_changeCounter = 3,
				_id = "8462f5ba3b39fd050d3cd6b7352ba561",
				_labels = {
					oz = true,
				},
				_localId = 2,
				_localOnly = {
					someImportantStuff = "that should survive",
				},
				_serverProvidedMetadata = {
					base = "https://photos.adobe.io",
					created = "2015-05-27T00:34:56.669987Z",
					revision_ids = {
						"a1e880f718a348b89663cf38f9f29d23",
						"cd1ab30cf8464ef2a9e736b50c0ba33d",
						"96f33a7215e04c9aa49346bbf8aae921",
						"813fdbda1b244c2d9f41008ba63eedba",
					},
					updated = "2015-05-27T00:38:29.160369Z",
				},
				captureDate = "2015-05-17T15:50:12",
				importSource = {
					importTimestamp = "2015-05-26T17:34:23-07:00",
					originalHeight = 1600,
					originalWidth = 1068,
				},
				location = {
					altitude = 377.9,
					latitude = 45.951733,
					longitude = -93.600107,
				},
				subtype = "image",
				type = "asset",
				xmp = {
					dc = {
						rights = "test copyright string",
					},
				},
			},
			continueFrom = 4,
			} )

	-- Pull changes again from mock Oz. Should preserve the local changes
	-- and merge server-originated changes.

	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 4 }
			:map( stripRevIds ),

		{
			{
				_changeCounter = 4,
				_id = "8462f5ba3b39fd050d3cd6b7352ba561",
				_labels = {
					oz = true,
				},
				_localId = 2,
				_localOnly = {
					someImportantStuff = "that should survive",
				},
				_serverProvidedMetadata = {
					base = "https://photos.adobe.io",
					created = "2015-05-27T00:34:56.669987Z",
					revision_ids = {
						"a1e880f718a348b89663cf38f9f29d23",
						"cd1ab30cf8464ef2a9e736b50c0ba33d",
						"96f33a7215e04c9aa49346bbf8aae921",
						"813fdbda1b244c2d9f41008ba63eedba",
						"3a4a43dc2a9a4f1897c27f7a26e2e068",
					},
					updated = "2015-05-27T00:38:29.160369Z",
				},
				captureDate = "2015-05-17T15:50:12",
				importSource = {
					importTimestamp = "2015-05-26T17:34:23-07:00",
					originalHeight = 1600,
					originalWidth = 1068,
				},
				subtype = "image",
				type = "asset",
				xmp = {
					dc = {
						rights = "something changed",
					},
				},
			},
			continueFrom = 5,
		} )

	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 5 }
			:map( stripRevIds ),

		{
			continueFrom = 5,
		} )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestGetUpdatesOnce_deleteDoc_preservesType( rxTest )

	rxTest:setTimeLimit( 10 )

	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', kUserId, rxTest,

		{
			prefetchReqResp,
			catalogPrefetchReqResp,
			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates",

					-- somewhat minimized version of payload

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "8462f5ba3b39fd050d3cd6b7352ba561",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"revision_ids": [
								"a1e880f718a348b89663cf38f9f29d23",
								"cd1ab30cf8464ef2a9e736b50c0ba33d",
								"96f33a7215e04c9aa49346bbf8aae921",
								"813fdbda1b244c2d9f41008ba63eedba"
							],
							"payload": {
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12",
								"xmp": {
									"dc": {
										"rights": "test copyright string"
									}
								},
								"location": {
									"longitude": -93.600107,
									"latitude": 45.951733,
									"altitude": 377.9
								}
							}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1",

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "8462f5ba3b39fd050d3cd6b7352ba561",
							"deleted": true,
							"deleted_by": "2a6f036cb28b46cb96121612cb9c27ec",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image"
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

		} )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 2 },

		{
			{
				_changeCounter = 2,
				_id = "8462f5ba3b39fd050d3cd6b7352ba561",
				_labels = {
					oz = true,
				},
				_localId = 2,
				_revId = "813fdbda1b244c2d9f41008ba63eedba",
				_serverProvidedMetadata = {
					base = "https://photos.adobe.io",
					created = "2015-05-27T00:34:56.669987Z",
					revision_ids = {
						"a1e880f718a348b89663cf38f9f29d23",
						"cd1ab30cf8464ef2a9e736b50c0ba33d",
						"96f33a7215e04c9aa49346bbf8aae921",
						"813fdbda1b244c2d9f41008ba63eedba",
					},
					updated = "2015-05-27T00:38:29.160369Z",
				},
				captureDate = "2015-05-17T15:50:12",
				importSource = {
					importTimestamp = "2015-05-26T17:34:23-07:00",
					originalHeight = 1600,
					originalWidth = 1068,
				},
				location = {
					altitude = 377.9,
					latitude = 45.951733,
					longitude = -93.600107,
				},
				subtype = "image",
				type = "asset",
				xmp = {
					dc = {
						rights = "test copyright string",
					},
				},
			},
			continueFrom = 3,
			} )


	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	rxTest:expectSingleValue(

		catalog:docStore():deletedDocsSinceNamedChangeCounter( 'PullerTest' )
			:take( 1 )
			:map( function( docValues )

				local docValue = docValues[ 1 ]
				docValue._revId = nil
				return docValue

			end ),

		{
			_changeCounter = 3,
			_id = "8462f5ba3b39fd050d3cd6b7352ba561",
			_deleted = true,
			_localId = 2,
			_labels = {
				oz = true,
			},
			_serverProvidedMetadata = {
				base = "https://photos.adobe.io",
				deleted_by = "2a6f036cb28b46cb96121612cb9c27ec",
				updated = "2015-05-27T00:38:29.160369Z",
			},
			subtype = "image",
			type = "asset",
		} )

	print('foo')

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestGetUpdatesOnce_updateDocNoRevIds( rxTest )

	rxTest:setTimeLimit( 30 )

	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', kUserId, rxTest,

		{
			prefetchReqResp,
			catalogPrefetchReqResp,
			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates",

					-- somewhat minimized version of payload

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "8462f5ba3b39fd050d3cd6b7352ba561",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"payload": {
								"xmp": {
									"dc": {
										"description": "Rum River State Forest, near Onamia, Minnesota",
										"rights": "test copyright string"
									}
								},
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12",
								"location": {
									"longitude": -93.600107,
									"latitude": 45.951733,
									"altitude": 377.9
								}
							}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1",

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "8462f5ba3b39fd050d3cd6b7352ba561",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"payload": {
								"xmp": {
									"dc": {
										"rights": "something changed",
										"description": "Rum River State Forest, near Onamia, Minnesota"
									}
								},
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12"
							}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2",

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

		} )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 2 }
			:map( stripRevIds ),

		{
			{
				_changeCounter = 2,
				_id = "8462f5ba3b39fd050d3cd6b7352ba561",
				_labels = {
					oz = true,
				},
				_localId = 2,
				_serverProvidedMetadata = {
					base = "https://photos.adobe.io",
					created = "2015-05-27T00:34:56.669987Z",
					updated = "2015-05-27T00:38:29.160369Z",
				},
				captureDate = "2015-05-17T15:50:12",
				importSource = {
					importTimestamp = "2015-05-26T17:34:23-07:00",
					originalHeight = 1600,
					originalWidth = 1068,
				},
				location = {
					altitude = 377.9,
					latitude = 45.951733,
					longitude = -93.600107,
				},
				subtype = "image",
				type = "asset",
				xmp = {
					dc = {
						rights = "test copyright string",
						description = "Rum River State Forest, near Onamia, Minnesota",
					},
				},
			},
			continueFrom = 3,
			} )

	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 3 }
			:map( stripRevIds ),

		{
			{
				_changeCounter = 3,
				_conflict = true,
				_id = "8462f5ba3b39fd050d3cd6b7352ba561",
				_labels = {
					oz = true,
				},
				_localId = 2,
				_serverProvidedMetadata = {
					base = "https://photos.adobe.io",
					created = "2015-05-27T00:34:56.669987Z",
					updated = "2015-05-27T00:38:29.160369Z",
				},
				captureDate = "2015-05-17T15:50:12",
				importSource = {
					importTimestamp = "2015-05-26T17:34:23-07:00",
					originalHeight = 1600,
					originalWidth = 1068,
				},
				subtype = "image",
				type = "asset",
				xmp = {
					dc = {
						description = "Rum River State Forest, near Onamia, Minnesota",
						rights = "something changed",
					},
				},
			},
			continueFrom = 4,
		} )

	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 4 }
			:map( stripRevIds ),

		{
			continueFrom = 4,
		} )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestGetUpdatesOnce_updateDocHandles429AndRetryDelay( rxTest )

	rxTest:setTimeLimit( 30 )

	local kRetryDelay = .5 -- Amount of time used in Retry-After calls.
	local kNoDelay = .2 -- An amount of time any of the requests can easily be completed in if there are no Retry-Afters requested.

	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', kUserId, rxTest,

		{
			-- This should end up slowing down the next request by 1 second.
			-- Because the metadata puller imposes the delay.

			-- The code further down will verify the delay occurred.
--1
			prefetchReqRespWithRetryDelay( kRetryDelay ),

--2
			-- Catalog prefetch
			catalogPrefetchReqResp,

			-- Add a 429 error on the updates, but don't expect this to increase the time
			-- as the mock http client doesn't retry or delay. Just make sure it keeps pulling.
--3
			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates",
				response = nil,
				fail = {
					{},
					{
						status_code = '429',
						['Retry-After' ] = 1, -- no delay expected, WFRxHttpClient would impose it
					},
				},

				options = {
					bypassRateLimiter = true,
				}
			},

--4

-- Unfortunately we are re-requesting the albums if we error on the first /updates call and don't get
-- Oz-nextUpdateFeedLink link set.

			prefetchReqResp,

--5
			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates",

					-- somewhat minimized version of payload

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "8462f5ba3b39fd050d3cd6b7352ba561",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"payload": {
								"xmp": {
									"dc": {
										"description": "Rum River State Forest, near Onamia, Minnesota",
										"rights": "test copyright string"
									}
								},
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12",
								"location": {
									"longitude": -93.600107,
									"latitude": 45.951733,
									"altitude": 377.9
								}
							}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

--6
			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1",

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "8462f5ba3b39fd050d3cd6b7352ba561",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"payload": {
								"xmp": {
									"dc": {
										"rights": "something changed",
										"description": "Rum River State Forest, near Onamia, Minnesota"
									}
								},
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12"
							}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
						}
					}
				}]],

				headers = {
					status_code = '200',
					[ 'Retry-After' ] = kRetryDelay,
				},

				options = {
					bypassRateLimiter = true,
				}

			},

--7
			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2",

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2",

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

		} )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )

	rxTest:addCloseTarget( puller )


	rxTest:expectOnlyError( puller:getUpdatesOnce(), {}, {
		["Retry-After"] = 1,
		status_code = "429",
	} )

	rxTest:expectEqual( 3, catalogSpec._httpMock:requestCount() )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 0 }
			:map( stripWobblyBits ),

		{
			{
				_changeCounter = 1,
				_id = catalogId,
				_labels = {
					oz = true,
				},
				_localId = 1,
				type = "catalog",
			},
			continueFrom = 2,
		} )


	rxTest:expectOnlyCompletedWithinTimeLimits( 0, kNoDelay, puller:getUpdatesOnce() )

	rxTest:expectEqual( 5, catalogSpec._httpMock:requestCount() )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 0 }
			:map( stripWobblyBits ),

		{
			{
				_changeCounter = 1,
				_id = catalogId,
				_labels = {
					oz = true,
				},
				_localId = 1,
				type = "catalog",
			},
			{
				_changeCounter = 2,
				_id = "8462f5ba3b39fd050d3cd6b7352ba561",
				_labels = {
					oz = true,
				},
				_localId = 2,
				captureDate = "2015-05-17T15:50:12",
				importSource = {
					importTimestamp = "2015-05-26T17:34:23-07:00",
					originalHeight = 1600,
					originalWidth = 1068,
				},
				location = {
					altitude = 377.9,
					latitude = 45.951733,
					longitude = -93.600107,
				},
				subtype = "image",
				type = "asset",
				xmp = {
					dc = {
						rights = "test copyright string",
						description = "Rum River State Forest, near Onamia, Minnesota",
					},
				},
			},
			continueFrom = 3,
			} )


	rxTest:expectOnlyCompletedWithinTimeLimits( 0, kNoDelay, puller:getUpdatesOnce() )

	rxTest:expectEqual( 6, catalogSpec._httpMock:requestCount() )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 3 }
			:map( stripRevIds ),

		{
			{
				_changeCounter = 3,
				_conflict = true,
				_labels = {
					oz = true,
				},
				_id = "8462f5ba3b39fd050d3cd6b7352ba561",
				_localId = 2,
				_serverProvidedMetadata = {
					base = "https://photos.adobe.io",
					created = "2015-05-27T00:34:56.669987Z",
					updated = "2015-05-27T00:38:29.160369Z",
				},
				captureDate = "2015-05-17T15:50:12",
				importSource = {
					importTimestamp = "2015-05-26T17:34:23-07:00",
					originalHeight = 1600,
					originalWidth = 1068,
				},
				subtype = "image",
				type = "asset",
				xmp = {
					dc = {
						description = "Rum River State Forest, near Onamia, Minnesota",
						rights = "something changed",
					},
				},
			},
			continueFrom = 4,
		} )


	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )
	rxTest:expectEqual( true, catalogSpec._httpMock:timeBetweenRequests() >= kRetryDelay )

	rxTest:expectEqual( 7, catalogSpec._httpMock:requestCount() )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 4 }
			:map( stripRevIds ),

		{
			continueFrom = 4,
		} )


	rxTest:expectOnlyCompletedWithinTimeLimits( 0, kNoDelay, puller:getUpdatesOnce() )

	rxTest:expectEqual( 8, catalogSpec._httpMock:requestCount() )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 4 }
			:map( stripRevIds ),

		{
			continueFrom = 4,
		} )


end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestGetLatestUpdates_noop( rxTest )

	local catalog = getNoopCatalog( rxTest )
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	rxTest:expectOnlyCompleted( puller:getLatestUpdates() )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 0 }
			:map( stripWobblyBits ),
		{
			{
				_changeCounter = 1,
				_id = catalogId,
				_labels = {
					oz = true,
				},
				_localId = 1,
				type = "catalog",
			},
			continueFrom = 2,
		} )

	local expectedLogData = {
		documentsReceivedByType = {},
		successfulUpdateCycles = 1,
		totalDocumentsReceived = 0,
	}

	verifyLogData( rxTest, puller:getDiagnosticLogData(), expectedLogData )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestGetLatestUpdates_noopTwice( rxTest )

	local catalog = getNoopCatalog( rxTest, 2 )
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	rxTest:expectOnlyCompleted( puller:getLatestUpdates() )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 0 }
			:map( stripWobblyBits ),
		{
			{
				_changeCounter = 1,
				_id = catalogId,
				_labels = {
					oz = true,
				},
				_localId = 1,
				type = "catalog",
			},
			continueFrom = 2,
		} )

	rxTest:expectOnlyCompleted( puller:getLatestUpdates() )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 2 }
			:map( stripWobblyBits ),
		{
			continueFrom = 2,
		} )

	local expectedLogData = {
		documentsReceivedByType = {},
		successfulUpdateCycles = 2,
		totalDocumentsReceived = 0,
	}

	verifyLogData( rxTest, puller:getDiagnosticLogData(), expectedLogData )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestGetLatestUpdates_overlappingRequestsCoalesced( rxTest )

	local catalog = getNoopCatalog( rxTest, 1 )
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	local update1 = puller:getLatestUpdates()
	local update2 = puller:getLatestUpdates()

	rxTest:expectEqual( update1, update2 )

	rxTest:expectOnlyCompleted( update1 )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 0 }
			:map( stripWobblyBits ),
		{
			{
				_changeCounter = 1,
				_id = catalogId,
				_labels = {
					oz = true,
				},
				_localId = 1,
				type = "catalog",
			},
			continueFrom = 2,
		} )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 2 }
			:map( stripWobblyBits ),
		{
			continueFrom = 2,
		} )

	local expectedLogData = {
		documentsReceivedByType = {},
		successfulUpdateCycles = 1,
		totalDocumentsReceived = 0,
	}

	verifyLogData( rxTest, puller:getDiagnosticLogData(), expectedLogData )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestGetLatestUpdates_gripeResponseNotTable( rxTest )

	local overrideUpdateResponse = '42'

	local catalog = getErrorCatalog( rxTest, overrideUpdateResponse )
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	rxTest:expectOnlyError( puller:getLatestUpdates(), ":ignore:" )

	local expectedLogData = {
		documentsReceivedByType = {},
		successfulUpdateCycles = 0,
		totalDocumentsReceived = 0,
	}

	verifyLogData( rxTest, puller:getDiagnosticLogData(), expectedLogData, 'empty' )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestGetLatestUpdates_gripeResponseMissingNextLink( rxTest )

	local overrideNextLink = ''

	local catalog = getErrorCatalog( rxTest, nil, overrideNextLink )
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	rxTest:expectOnlyCompleted( puller:getLatestUpdates() )

	local expectedLogData = {
		documentsReceivedByType = {},
		successfulUpdateCycles = 1,
		totalDocumentsReceived = 0,
	}

	-- The next Feed Link should not have changed, since it was missing in the response.

	verifyLogData( rxTest, puller:getDiagnosticLogData(), expectedLogData, 'empty' )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestGetLatestUpdates_gripeResponseBadJSON( rxTest )

	local overrideUpdateResponse = [[{
		"updates": [
			{
				"id = "8462f5ba3b39\fd050d3cd6b7\352ba561",
			}
		]
	}]]

	local catalog = getErrorCatalog( rxTest, overrideUpdateResponse )
	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	rxTest:expectOnlyError( puller:getLatestUpdates(), ':ignore:' )

	local expectedLogData = {
		documentsReceivedByType = {},
		successfulUpdateCycles = 0,
		totalDocumentsReceived = 0,
	}

	verifyLogData( rxTest, puller:getDiagnosticLogData(), expectedLogData, 'empty' )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestGetLatestUpdates_multipleUpdates( rxTest )

	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', kUserId, rxTest,

		{
			prefetchReqResp,
			catalogPrefetchReqResp,
			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates",

				-- somewhat minimized version of payload

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "8462f5ba3b39fd050d3cd6b7352ba561",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"revision_ids": [
								"a1e880f718a348b89663cf38f9f29d23",
								"cd1ab30cf8464ef2a9e736b50c0ba33d",
								"96f33a7215e04c9aa49346bbf8aae921",
								"813fdbda1b244c2d9f41008ba63eedba"
							],
							"payload": {
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12",
								"xmp": {
									"dc": {
										"rights": "test copyright string"
									}
								},
								"location": {
									"longitude": -93.600107,
									"latitude": 45.951733,
									"altitude": 377.9
								}
							}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1",

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "8462f5ba3b39fd050d3cd6b7352ba561",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"revision_ids": [
								"a1e880f718a348b89663cf38f9f29d23",
								"cd1ab30cf8464ef2a9e736b50c0ba33d",
								"96f33a7215e04c9aa49346bbf8aae921",
								"813fdbda1b244c2d9f41008ba63eedba",
								"3a4a43dc2a9a4f1897c27f7a26e2e068"
							],
							"payload": {
								"xmp": {
									"dc": {
										"rights": "something changed"
									}
								},
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12"
							}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},
			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2",

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

		} )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	-- SPAM [matthewj 2016-10-19]: Reaching in to tweak this value directly (rather than by calling
	-- startContinuousTask) is a bit hacky but necessary for the tests with the current impl of
	-- _syncIsStopped
	puller._syncIsStopped = false

	rxTest:expectOnlyCompleted( puller:getLatestUpdates() )

	local expectedLogData = {
		documentsReceivedByType = {
			update = {
				asset = {
					image = 2,
				},
				total = 2,
			},
		},
		successfulUpdateCycles = 3,
		totalDocumentsReceived = 2,
	}

	local expectedLink = '/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2'

	verifyLogData( rxTest, puller:getDiagnosticLogData(), expectedLogData, expectedLink )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestGetLatestUpdates_assetTypeHandler( rxTest )

	rxTest:setTimeLimit( 120 )
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', kUserId, rxTest,

		{
			prefetchReqResp,
			catalogPrefetchReqResp,
			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates",

				-- somewhat minimized version of payload

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "1462f5ba3b39fd050d3cd6b7352ba561",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"revision_ids": [
								"a1e880f718a348b89663cf38f9f29d23",
								"cd1ab30cf8464ef2a9e736b50c0ba33d",
								"96f33a7215e04c9aa49346bbf8aae921",
								"813fdbda1b244c2d9f41008ba63eedba"
							],
							"links": {
								"/rels/proxy_type/2560": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/proxy"
								},
								"/rels/master": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/master"
								}
							},
							"payload": {
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12",
								"xmp": {
									"dc": {
										"rights": "test copyright string"
									}
								},
								"location": {
									"longitude": -93.600107,
									"latitude": 45.951733,
									"altitude": 377.9
								}
							}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1",

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "8462f5ba3b39fd050d3cd6b7352ba561",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"revision_ids": [
								"a1e880f718a348b89663cf38f9f29d23",
								"cd1ab30cf8464ef2a9e736b50c0ba33d",
								"96f33a7215e04c9aa49346bbf8aae921",
								"813fdbda1b244c2d9f41008ba63eedba",
								"3a4a43dc2a9a4f1897c27f7a26e2e068"
							],
							"links": {
								"/rels/master": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/master"
								}
							},
							"payload": {
								"xmp": {
									"dc": {
										"rights": "something changed"
									}
								},
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12"
							}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},
			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2",

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

		} )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	-- SPAM [matthewj 2016-10-19]: Reaching in to tweak this value directly (rather than by calling
	-- startContinuousTask) is a bit hacky but necessary for the tests with the current impl of
	-- _syncIsStopped
	puller._syncIsStopped = false

	local addOpinionOfPie = function( metadataPuller, docValue )

		docValue._serverProvidedMetadata = {
			opinionOnPie = "Pretty good."
		}

	end

	puller:addAssetTypeHandler( 'asset', addOpinionOfPie )

	rxTest:expectOnlyCompleted( puller:getLatestUpdates() )

	local expectedLogData = {
		documentsReceivedByType = {
			update = {
				asset = {
					image = 2,
				},
				total = 2,
			},
		},
		successfulUpdateCycles = 3,
		totalDocumentsReceived = 2,
	}

	local expectedLink = '/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2'

	verifyLogData( rxTest, puller:getDiagnosticLogData(), expectedLogData, expectedLink )

	rxTest:expectSingleValue( catalog:docStore()
		:docValue( '8462f5ba3b39fd050d3cd6b7352ba561' )
		:pluck( '_serverProvidedMetadata', 'opinionOnPie' ), "Pretty good." )

	rxTest:expectSingleValue( catalog:docStore()
		:docValue( '1462f5ba3b39fd050d3cd6b7352ba561' )
		:pluck( '_serverProvidedMetadata', 'opinionOnPie' ), "Pretty good." )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestDocTypeHandlerFailure( rxTest )

	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', kUserId, rxTest,

		{
			prefetchReqResp,
			catalogPrefetchReqResp,
			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates",

				-- somewhat minimized version of payload

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "1462f5ba3b39fd050d3cd6b7352ba561",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"revision_ids": [
								"a1e880f718a348b89663cf38f9f29d23",
								"cd1ab30cf8464ef2a9e736b50c0ba33d",
								"96f33a7215e04c9aa49346bbf8aae921",
								"813fdbda1b244c2d9f41008ba63eedba"
							],
							"links": {
								"/rels/proxy_type/2560": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/proxy"
								},
								"/rels/master": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/master"
								}
							},
							"payload": {
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12",
								"xmp": {
									"dc": {
										"rights": "test copyright string"
									}
								},
								"location": {
									"longitude": -93.600107,
									"latitude": 45.951733,
									"altitude": 377.9
								}
							}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},
		} )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	local addOpinionOfPie = function( metadataPuller, docValue )

		docValue._localOnly = {
			opinionOnPie = "Pretty good."
		}

	end

	puller:addAssetTypeHandler( 'asset', addOpinionOfPie )

	rxTest:expectOnlyError( puller:getLatestUpdates(),
		'_localOnly is not permitted here, as it would overwrite existing data' )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestGetLatestUpdates_multipleUpdatesWithError( rxTest )

	rxTest:setTimeLimit( 30 )

	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', kUserId, rxTest,
		{
			prefetchReqResp,
			catalogPrefetchReqResp,
			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates",
				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "8462f5ba3b39fd050d3cd6b7352ba561",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"revision_ids": [
								"a1e880f718a348b89663cf38f9f29d23",
								"cd1ab30cf8464ef2a9e736b50c0ba33d",
								"96f33a7215e04c9aa49346bbf8aae921",
								"813fdbda1b244c2d9f41008ba63eedba"
							],
							"payload": {
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12",
								"xmp": {
									"dc": {
										"rights": "test copyright string"
									}
								},
								"location": {
									"longitude": -93.600107,
									"latitude": 45.951733,
									"altitude": 377.9
								}
							}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},
			{
				-- Erroneous response.
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1",
				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [ {} ],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
						}
					}
				}]],
				headers = {
					status_code = '200',
				},
				options = {
					bypassRateLimiter = true,
				}
			},

		} )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	-- SPAM [matthewj 2016-10-19]: Reaching in to tweak this value directly (rather than by calling
	-- startContinuousTask) is a bit hacky but necessary for the tests with the current impl of
	-- _syncIsStopped
	puller._syncIsStopped = false

	rxTest:expectOnlyError( puller:getLatestUpdates(), ':ignore:' )

	local expectedLogData = {
		documentsReceivedByType = {
			update = {
				asset = {
					image = 1,
				},
				total = 1,
			},
		},
		successfulUpdateCycles = 1,
		totalDocumentsReceived = 2,
	}

	local expectedFeedLink = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1"

	verifyLogData( rxTest, puller:getDiagnosticLogData(), expectedLogData, expectedFeedLink )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestOzCatalogDeleted( rxTest )

	rxTest:setTimeLimit( 10 )

	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', kUserId, rxTest,

		{
			prefetchReqResp,
			catalogPrefetchReqResp,
			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates",

				fail = {
					[[ {
						"errors": {
							"catalog": [
								"does not exist"
							]
						}
					} ]],
					{
						status_code = '404',
					}
				},

				options = {
					bypassRateLimiter = true,
				}

			},

		} )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	rxTest:expectOnlyError( puller:getUpdatesOnce(),
		[[ {
						"errors": {
							"catalog": [
								"does not exist"
							]
						}
					} ]], { status_code = '404' } )

	local obs = rxTest:observe( puller:isOzCatalogDeleted() )

	obs:expectDataWithinTime( 1, true )

end

--------------------------------------------------------------------------------

-- @mark RevisionCompressed handling

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestCompressionListHandler_withLabelUpdate( rxTest )


	-- revisions used in this test
		-- "a1e880f718a348b89663cf38f9f29d23" revision in original doc from Oz
		-- "cd1ab30cf8464ef2a9e736b50c0ba33d" revision in original doc from Oz  compressed away
		-- "96f33a7215e04c9aa49346bbf8aae921" revision in original doc from Oz, compressed away
		-- "813fdbda1b244c2d9f41008ba63eedba" revision in original doc from Oz, compressed away
		-- "3f936b6412914c938af12340d3f6977c" local edit, synced to Oz but not received back, compressed away
		-- "ae4849b69c6f4b8a8d2c1ef84df2b651" local edit, never synced
		-- "9c8f4534178546df94aa0cbaa9f73657" remote edit, never send because it got superceded
		-- "a24c9b924a284a09af67221ff00bd96b" remote edit
		-- "ignore me" new revision created by the conflict resolver, which we will ignore


	rxTest:setTimeLimit( 120 )
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', kUserId, rxTest,

		{
			prefetchReqResp,
			catalogPrefetchReqResp,
			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates",

				-- somewhat minimized version of payload

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "8462f5ba3b39fd050d3cd6b7352ba561",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"revision_ids": [
								"a1e880f718a348b89663cf38f9f29d23",
								"cd1ab30cf8464ef2a9e736b50c0ba33d",
								"96f33a7215e04c9aa49346bbf8aae921",
								"813fdbda1b244c2d9f41008ba63eedba"
							],
							"links": {
								"/rels/proxy_type/2560": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/proxy"
								},
								"/rels/master": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/master"
								}
							},
							"payload": {
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12",
								"xmp": {
									"dc": {
										"rights": "original copyright string"
									}
								},
								"location": {
									"longitude": -93.600107,
									"latitude": 45.951733,
									"altitude": 377.9
								}
							}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1",

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
						    "id":"e702f961573a033e0d264fdc22b83d12",
    						"updated":"2017-04-18T18:23:12.859814Z",
    						"type":"warning",
    						"subtype":"RevisionCompressed",
    						"payload": {
        						"original_revisions": [
									"a1e880f718a348b89663cf38f9f29d23",
									"cd1ab30cf8464ef2a9e736b50c0ba33d",
									"96f33a7215e04c9aa49346bbf8aae921",
									"813fdbda1b244c2d9f41008ba63eedba",
									"3f936b6412914c938af12340d3f6977c",
									"9c8f4534178546df94aa0cbaa9f73657"
        						],
        						"new_revisions": [
									"a1e880f718a348b89663cf38f9f29d23",
									"9c8f4534178546df94aa0cbaa9f73657"
        						]
    						},
    						"source": {
        						"id":"8462f5ba3b39fd050d3cd6b7352ba561",
        						"type":"asset"
    						}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2",

				-- somewhat minimized version of payload

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "8462f5ba3b39fd050d3cd6b7352ba561",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"revision_ids": [
								"a1e880f718a348b89663cf38f9f29d23",
								"9c8f4534178546df94aa0cbaa9f73657",
								"a24c9b924a284a09af67221ff00bd96b"
							],
							"links": {
								"/rels/proxy_type/2560": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/proxy"
								},
								"/rels/master": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/master"
								}
							},
							"payload": {
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12",
								"xmp": {
									"dc": {
										"rights": "change the copyright string again, this should be the final one after merge"
									}
								},
								"location": {
									"longitude": -93.600107,
									"latitude": 45.951733,
									"altitude": 377.9
								}
							}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull3"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull3",

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull3"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

		} )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	-- SPAM [matthewj 2016-10-19]: Reaching in to tweak this value directly (rather than by calling
	-- startContinuousTask) is a bit hacky but necessary for the tests with the current impl of
	-- _syncIsStopped
	puller._syncIsStopped = false

	local indexStore = WFRxIndexStore( catalog:docStore(), 'WFCatConflictResolverTestIndexDefinitions.lua' )
	rxTest:addCloseTarget( indexStore )


	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	-- create two new local revisions, the first of which is mentioned in the compression doc

	rxTest:expectSingleValue(

		catalog:docStore():setDocValue {
				_id = "8462f5ba3b39fd050d3cd6b7352ba561",
				_revId = "813fdbda1b244c2d9f41008ba63eedba",
				_newRevId = "3f936b6412914c938af12340d3f6977c",
				captureDate = "2015-05-17T15:50:12",
				importSource = {
					importTimestamp = "2015-05-26T17:34:23-07:00",
					originalHeight = 1600,
					originalWidth = 1068,
				},
				location = {
					altitude = 377.9,
					latitude = 45.951733,
					longitude = -93.600107,
				},
				subtype = "image",
				type = "asset",
				xmp = {
					dc = {
						rights = "this should not be the copyright string after merge",
					},
				},
			},

	':ignore:' )

	rxTest:expectSingleValue(

		catalog:docStore():setDocValue {
				_id = "8462f5ba3b39fd050d3cd6b7352ba561",
				_revId = "3f936b6412914c938af12340d3f6977c",
				_newRevId = "ae4849b69c6f4b8a8d2c1ef84df2b651",
				captureDate = "2015-05-17T15:50:12",
				importSource = {
					importTimestamp = "2015-05-26T17:34:23-07:00",
					originalHeight = 1600,
					originalWidth = 1068,
				},
				location = {
					altitude = 377.9,
					latitude = 45.951733,
					longitude = -93.600107,
				},
				subtype = "image",
				type = "asset",
				xmp = {
					dc = {
						rights = "this should not be the copyright string after merge",
						title = "Cows. I want cows."
					},

				},
			},

	':ignore:' )

	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	rxTest:expectSingleValue(

		catalog:docStore():getRevisionForLabel('8462f5ba3b39fd050d3cd6b7352ba561', 'oz'),

		'3f936b6412914c938af12340d3f6977c' )


	rxTest:expectOnlyCompleted( puller:getLatestUpdates() )

	local expectedLogData = {
		documentsReceivedByType = {
			update = {
				asset = {
					image = 2,
				},
				total = 3,
				warning = {
					RevisionCompressed = 1,
				},
			},
		},
		successfulUpdateCycles = 4,
		totalDocumentsReceived = 3,
	}

	local expectedLink = '/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull3'

	verifyLogData( rxTest, puller:getDiagnosticLogData(), expectedLogData, expectedLink )

	-- Now attach conflict resolver.

	local conflictResolver = WFCatConflictResolver( catalog:docStore(), indexStore )
	rxTest:addCloseTarget( conflictResolver )

	-- Wait a few seconds while the conflict handler does its thing.
	local docValue = rxTest:observe(
		catalog:docStore():ongoingDocValue( '8462f5ba3b39fd050d3cd6b7352ba561' )
			:map( function( value )
				value._revId = "ignore me"
				value._changeCounter = "ignore me"
				return value
			end ) )

	docValue:expectDataWithinTime( 2,
			{
				_id = "8462f5ba3b39fd050d3cd6b7352ba561",
				_localId = 2,
				_revId = "ignore me",
				_changeCounter = "ignore me",
				_serverProvidedMetadata = {
					base = "https://photos.adobe.io",
					created = "2015-05-27T00:34:56.669987Z",
					links = {
						["/rels/master"] = {
							href = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/master",
						},
						["/rels/proxy_type/2560"] = {
							href = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/proxy",
						},
					},
					revision_ids = {
						"a1e880f718a348b89663cf38f9f29d23",
						"9c8f4534178546df94aa0cbaa9f73657",
						"a24c9b924a284a09af67221ff00bd96b",
					},
					updated = "2015-05-27T00:38:29.160369Z",
				},
				captureDate = "2015-05-17T15:50:12",
				importSource = {
					importTimestamp = "2015-05-26T17:34:23-07:00",
					originalHeight = 1600,
					originalWidth = 1068,
				},
				location = {
					altitude = 377.9,
					latitude = 45.951733,
					longitude = -93.600107,
				},
				subtype = "image",
				type = "asset",
				xmp = {
					dc = {
						rights = "change the copyright string again, this should be the final one after merge",
						title = "Cows. I want cows."
					},

				},
		} )


end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestCompressionListHandler_LabelUpdateFailure( rxTest )

	-- Receive revision compress document for an asset id which is not in the docstore.
	-- The error from WFRxDocStore::updateLabelFromRevisionList should be ignored and not
	-- propagated to the MetadataPuller.

	rxTest:setTimeLimit( 120 )
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', kUserId, rxTest,

		{
			prefetchReqResp,
			catalogPrefetchReqResp,
			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates",

				-- somewhat minimized version of payload

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "8462f5ba3b39fd050d3cd6b7352ba561",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"revision_ids": [
								"a1e880f718a348b89663cf38f9f29d23",
								"cd1ab30cf8464ef2a9e736b50c0ba33d",
								"96f33a7215e04c9aa49346bbf8aae921",
								"813fdbda1b244c2d9f41008ba63eedba"
							],
							"links": {
								"/rels/proxy_type/2560": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/proxy"
								},
								"/rels/master": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/master"
								}
							},
							"payload": {
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12",
								"xmp": {
									"dc": {
										"rights": "original copyright string"
									}
								},
								"location": {
									"longitude": -93.600107,
									"latitude": 45.951733,
									"altitude": 377.9
								}
							}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1",

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
						    "id":"e702f961573a033e0d264fdc22b83d12",
    						"updated":"2017-04-18T18:23:12.859814Z",
    						"type":"warning",
    						"subtype":"RevisionCompressed",
    						"payload": {
        						"original_revisions": [
									"a1e880f718a348b89663cf38f9f29d23",
									"cd1ab30cf8464ef2a9e736b50c0ba33d",
									"96f33a7215e04c9aa49346bbf8aae921",
									"813fdbda1b244c2d9f41008ba63eedba",
									"3f936b6412914c938af12340d3f6977c",
									"9c8f4534178546df94aa0cbaa9f73657"
        						],
        						"new_revisions": [
									"a1e880f718a348b89663cf38f9f29d23",
									"9c8f4534178546df94aa0cbaa9f73657"
        						]
    						},
    						"source": {
        						"id":"deadbeefdeadbeefdeadbeefdeadbeef",
        						"type":"asset"
    						}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2",

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

		} )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	puller._syncIsStopped = false

	rxTest:expectOnlyCompleted( puller:getLatestUpdates() )

	local expectedLogData = {
		documentsReceivedByType = {
			update = {
				asset = {
					image = 1,
				},
				total = 2,
				warning = {
					RevisionCompressed = 1,
				},
			},
		},
		successfulUpdateCycles = 3,
		totalDocumentsReceived = 2,
	}

	local expectedLink = '/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2'

	verifyLogData( rxTest, puller:getDiagnosticLogData(), expectedLogData, expectedLink )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestCompressionListHandler_noLabelUpdate( rxTest )


	-- revisions used in this test
		-- "a1e880f718a348b89663cf38f9f29d23" revision in original doc from Oz - doc 'a'
		-- "cd1ab30cf8464ef2a9e736b50c0ba33d" revision in second doc from Oz - doc 'b', compressed away
		-- "3f936b6412914c938af12340d3f6977c" local edit, doc 'c'
		-- "9c8f4534178546df94aa0cbaa9f73657" remote edit, doc 'x', conflicts with 'c'
		-- "ignore me" new revision created by the conflict resolver, which we will ignore


	rxTest:setTimeLimit( 120 )
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', kUserId, rxTest,

		{
			prefetchReqResp,
			catalogPrefetchReqResp,
			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates",

				-- somewhat minimized version of payload

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "8462f5ba3b39fd050d3cd6b7352ba561",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"revision_ids": [
								"a1e880f718a348b89663cf38f9f29d23"
							],
							"links": {
								"/rels/proxy_type/2560": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/proxy"
								},
								"/rels/master": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/master"
								}
							},
							"payload": {
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12",
								"xmp": {
									"dc": {
										"rights": "original copyright string"
									}
								},
								"location": {
									"longitude": -93.600107,
									"latitude": 45.951733,
									"altitude": 377.9
								}
							}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1",

				-- somewhat minimized version of payload

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "8462f5ba3b39fd050d3cd6b7352ba561",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"revision_ids": [
								"a1e880f718a348b89663cf38f9f29d23",
								"cd1ab30cf8464ef2a9e736b50c0ba33d"
							],
							"links": {
								"/rels/proxy_type/2560": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/proxy"
								},
								"/rels/master": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/master"
								}
							},
							"payload": {
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12",
								"xmp": {
									"dc": {
										"rights": "version b copyright string"
									}
								},
								"location": {
									"longitude": -93.600107,
									"latitude": 45.951733,
									"altitude": 377.9
								}
							}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2",

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
						    "id":"e702f961573a033e0d264fdc22b83d12",
    						"updated":"2017-04-18T18:23:12.859814Z",
    						"type":"warning",
    						"subtype":"RevisionCompressed",
    						"payload": {
        						"original_revisions": [
									"a1e880f718a348b89663cf38f9f29d23",
									"cd1ab30cf8464ef2a9e736b50c0ba33d",
									"9c8f4534178546df94aa0cbaa9f73657"
        						],
        						"new_revisions": [
									"a1e880f718a348b89663cf38f9f29d23",
									"9c8f4534178546df94aa0cbaa9f73657"
        						]
    						},
    						"source": {
        						"id":"8462f5ba3b39fd050d3cd6b7352ba561",
        						"type":"asset"
    						}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull3"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull3",

				-- somewhat minimized version of payload

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "8462f5ba3b39fd050d3cd6b7352ba561",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"revision_ids": [
								"a1e880f718a348b89663cf38f9f29d23",
								"9c8f4534178546df94aa0cbaa9f73657"
							],
							"links": {
								"/rels/proxy_type/2560": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/proxy"
								},
								"/rels/master": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/master"
								}
							},
							"payload": {
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12",
								"xmp": {
									"dc": {
										"rights": "version b copyright string",
										"title":  "version x title"
									}
								},
								"location": {
									"longitude": -93.600107,
									"latitude": 45.951733,
									"altitude": 377.9
								}
							}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull4"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull4",

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull4"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

		} )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	-- SPAM [matthewj 2016-10-19]: Reaching in to tweak this value directly (rather than by calling
	-- startContinuousTask) is a bit hacky but necessary for the tests with the current impl of
	-- _syncIsStopped
	puller._syncIsStopped = false

	local indexStore = WFRxIndexStore( catalog:docStore(), 'WFCatConflictResolverTestIndexDefinitions.lua' )
	rxTest:addCloseTarget( indexStore )


	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() ) -- fetch doc 'a'

	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() ) -- fetch doc 'b'

	-- create a new local revision 'c' on top of 'b' which is not synced

	rxTest:expectSingleValue(

		catalog:docStore():setDocValue {
				_id = "8462f5ba3b39fd050d3cd6b7352ba561",
				_revId = "cd1ab30cf8464ef2a9e736b50c0ba33d",
				_newRevId = "3f936b6412914c938af12340d3f6977c",
				captureDate = "2015-05-17T15:50:12",
				importSource = {
					importTimestamp = "2015-05-26T17:34:23-07:00",
					originalHeight = 1600,
					originalWidth = 1068,
				},
				location = {
					altitude = 377.9,
					latitude = 45.951733,
					longitude = -93.600107,
				},
				subtype = "image",
				type = "asset",
				xmp = {
					dc = {
						caption = "version c caption",
						rights = "version b copyright string",
					},
				},
			},

	':ignore:' )


	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() ) -- fetch RevisionCompressed doc and doc 'x'


	rxTest:expectOnlyCompleted( puller:getLatestUpdates() )

	local expectedLogData = {
		documentsReceivedByType = {
			update = {
				asset = {
					image = 3,
				},
				total = 4,
				warning = {
					RevisionCompressed = 1,
				},
			},
		},
		successfulUpdateCycles = 5,
		totalDocumentsReceived = 4,
	}

	local expectedLink = '/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull4'

	verifyLogData( rxTest, puller:getDiagnosticLogData(), expectedLogData, expectedLink )

	-- Now attach conflict resolver.

	local conflictResolver = WFCatConflictResolver( catalog:docStore(), indexStore )
	rxTest:addCloseTarget( conflictResolver )

	-- Wait a few seconds while the conflict handler does its thing.
	local docValue = rxTest:observe(
		catalog:docStore():ongoingDocValue( '8462f5ba3b39fd050d3cd6b7352ba561' )
			:map( function( value )
				value._revId = "ignore me"
				value._changeCounter = "ignore me"
				return value
			end ) )

	docValue:expectDataWithinTime( 2,
			{
				_id = "8462f5ba3b39fd050d3cd6b7352ba561",
				_localId = 2,
				_revId = "ignore me",
				_changeCounter = "ignore me",
				_serverProvidedMetadata = {
					base = "https://photos.adobe.io",
					created = "2015-05-27T00:34:56.669987Z",
					links = {
						["/rels/master"] = {
							href = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/master",
						},
						["/rels/proxy_type/2560"] = {
							href = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/proxy",
						},
					},
					revision_ids = {
						"a1e880f718a348b89663cf38f9f29d23",
						"9c8f4534178546df94aa0cbaa9f73657",
					},
					updated = "2015-05-27T00:38:29.160369Z",
				},
				captureDate = "2015-05-17T15:50:12",
				importSource = {
					importTimestamp = "2015-05-26T17:34:23-07:00",
					originalHeight = 1600,
					originalWidth = 1068,
				},
				location = {
					altitude = 377.9,
					latitude = 45.951733,
					longitude = -93.600107,
				},
				subtype = "image",
				type = "asset",
				xmp = {
					dc = {
						rights = "version b copyright string",
						title = "version x title",
						caption = "version c caption",
					},

				},
		} )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestCompressionListHandler_multipleCompresssionLists( rxTest )

	rxTest:setTimeLimit( 120 )
	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', kUserId, rxTest,

		{
			prefetchReqResp,
			catalogPrefetchReqResp,
			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates",

				-- somewhat minimized version of payload

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
							"id": "b8942049096c42a9927d11d63fb4f32f",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"revision_ids": [
								"eca9d5b4729d482d8a461ff9e52e83c0",
								"97e6984fedd24b25ad2757ea6f3abe2f",
								"bff56cf0ef48453fb6672a73fd461dbb"
							],
							"links": {
								"/rels/proxy_type/2560": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/proxy"
								},
								"/rels/master": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/master"
								}
							},
							"payload": {
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12",
								"xmp": {
									"dc": {
										"rights": "original copyright string"
									}
								},
								"location": {
									"longitude": -93.600107,
									"latitude": 45.951733,
									"altitude": 377.9
								}
							}
						},

						{
							"id":"aa800c0c4b1b44a3b26657bc3a298319",
							"created": "2015-05-27T00:34:56.669987Z",
							"updated": "2015-05-27T00:38:29.160369Z",
							"type": "asset",
							"subtype": "image",
							"revision_ids": [
								"fcb4df54f56a4469a92450c56118b2aa",
								"bebccda8a5d14d80b1b081e01e62f9c1",
								"7069fb71e390496f993ab9d687839da3"
							],
							"links": {
								"/rels/proxy_type/2560": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/proxy"
								},
								"/rels/master": {
									"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/123/master"
								}
							},
							"payload": {
								"importSource": {
									"importTimestamp": "2015-05-26T17:34:23-07:00",
									"originalHeight": 1600,
									"originalWidth": 1068
								},
								"captureDate": "2015-05-17T15:50:12",
								"xmp": {
									"dc": {
										"rights": "original copyright string"
									}
								},
								"location": {
									"longitude": -93.600107,
									"latitude": 45.951733,
									"altitude": 377.9
								}
							}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull1",

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [
						{
						    "id":"45d80eef70434bf39763ede19663425e",
    						"updated":"2017-04-18T18:23:12.859814Z",
    						"type":"warning",
    						"subtype":"RevisionCompressed",
    						"payload": {
        						"original_revisions": [
									"eca9d5b4729d482d8a461ff9e52e83c0",
									"97e6984fedd24b25ad2757ea6f3abe2f",
									"bff56cf0ef48453fb6672a73fd461dbb",
									"d757085698fa49b6bbb009fabeb1a1dc",
									"6ccd10c9f00f40c5bcc25100426256c0"
        						],
        						"new_revisions": [
									"eca9d5b4729d482d8a461ff9e52e83c0",
									"6ccd10c9f00f40c5bcc25100426256c0"
        						]
    						},
    						"source": {
        						"id":"b8942049096c42a9927d11d63fb4f32f",
        						"type":"asset"
    						}
						},

						{
						    "id":"9be93094ecd1406faed0a02d4c2995a1",
    						"updated":"2017-04-18T18:23:12.859814Z",
    						"type":"warning",
    						"subtype":"RevisionCompressed",
    						"payload": {
        						"original_revisions": [
									"fcb4df54f56a4469a92450c56118b2aa",
									"bebccda8a5d14d80b1b081e01e62f9c1",
									"7069fb71e390496f993ab9d687839da3",
									"73505a2ada9e48fc9d787cd9a80d9207",
									"f26714f0d0094c528979200cb2687d2f"
        						],
        						"new_revisions": [
									"fcb4df54f56a4469a92450c56118b2aa",
									"f26714f0d0094c528979200cb2687d2f"
        						]
    						},
    						"source": {
        						"id":"aa800c0c4b1b44a3b26657bc3a298319",
        						"type":"asset"
    						}
						}
					],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2",

				response = [[{
					"base": "https://photos.adobe.io",
					"updates": [],
					"links": {
						"next": {
							"href": "/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2"
						}
					}
				}]],

				headers = {
					status_code = '200',
				},

				options = {
					bypassRateLimiter = true,
				}

			},

		} )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	-- SPAM [matthewj 2016-10-19]: Reaching in to tweak this value directly (rather than by calling
	-- startContinuousTask) is a bit hacky but necessary for the tests with the current impl of
	-- _syncIsStopped
	puller._syncIsStopped = false

	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() ) -- fetch two docs

	-- create a new local revision on top of both documents

	rxTest:expectSingleValue(

		catalog:docStore():setDocValue {
				_id = "b8942049096c42a9927d11d63fb4f32f",
				_revId = "bff56cf0ef48453fb6672a73fd461dbb",
				_newRevId = "d757085698fa49b6bbb009fabeb1a1dc",
				captureDate = "2015-05-17T15:50:12",
				importSource = {
					importTimestamp = "2015-05-26T17:34:23-07:00",
					originalHeight = 1600,
					originalWidth = 1068,
				},
				location = {
					altitude = 377.9,
					latitude = 45.951733,
					longitude = -93.600107,
				},
				subtype = "image",
				type = "asset",
				xmp = {
					dc = {
						rights = "local change",
					},
				},
			},

	':ignore:' )

	rxTest:expectSingleValue(

		catalog:docStore():setDocValue {
				_id = "aa800c0c4b1b44a3b26657bc3a298319",
				_revId = "7069fb71e390496f993ab9d687839da3",
				_newRevId = "73505a2ada9e48fc9d787cd9a80d9207",
				captureDate = "2015-05-17T15:50:12",
				importSource = {
					importTimestamp = "2015-05-26T17:34:23-07:00",
					originalHeight = 1600,
					originalWidth = 1068,
				},
				location = {
					altitude = 377.9,
					latitude = 45.951733,
					longitude = -93.600107,
				},
				subtype = "image",
				type = "asset",
				xmp = {
					dc = {
						rights = "local change",
					},
				},
			},

	':ignore:' )



	rxTest:expectOnlyCompleted( puller:getLatestUpdates() ) -- fetch RevisionCompressed docs

	local expectedLogData = {
		documentsReceivedByType = {
			update = {
				asset = {
					image = 2,
				},
				total = 4,
				warning = {
					RevisionCompressed = 2,
				},
			},
		},
		successfulUpdateCycles = 3,
		totalDocumentsReceived = 4,
	}

	local expectedLink = '/v2/catalogs/98db98bfd554470dab2f862a4b853c40/updates?updated_since=pull2'

	verifyLogData( rxTest, puller:getDiagnosticLogData(), expectedLogData, expectedLink )

	-- check labeled Revisions

	rxTest:expectSingleValue(

		catalog:docStore():getRevisionForLabel('b8942049096c42a9927d11d63fb4f32f', 'oz'),

		'd757085698fa49b6bbb009fabeb1a1dc' )


	rxTest:expectSingleValue(

		catalog:docStore():getRevisionForLabel('aa800c0c4b1b44a3b26657bc3a298319', 'oz'),

		'73505a2ada9e48fc9d787cd9a80d9207' )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.reportAllChanges( scanState, docValue )
	return scanState, docValue
end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestCompressionListHandler_testNightly( rxTest )

	rxTest:setTimeLimit( 120 )

	local catalogTestSession = WFCatTestSession( rxTest )

	local catalog

	rxTest:expectSingleValue(

		catalogTestSession:catalog()
			:tap( function( _catalog )
				check( 'WFCatalog' )
				catalog = _catalog
			end ),

		':ignore:' )

	-- start the index store
	local indexStore = WFRxIndexStore( catalog:docStore(), 'WFCatConflictResolverTestIndexDefinitions.lua' )
	rxTest:addCloseTarget( indexStore )

	local initialSyncObs = rxTest:observe( WFCatSyncStatus.isDoingInitialSync( catalog ) )

	initialSyncObs:expectDataWithinTime( 0.3, true )

	local options = {
		continuousTaskDelays = {
			progressiveBackoff = false,
			noWorkDelay = 1,
			maxNoWorkDelay = 1,
			errorDelay = 1,
			maxDelay = 1,
		},
		isCatalogPuller = true
	}

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	rxTest:expectOnlyCompleted( puller:getUpdatesOnce() )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 0 }
			:map( stripWobblyBits ),
		{
			{
				_changeCounter = 1,
				_id = catalog:ozCatalogId(),
				_labels = {
					oz = true,
				},
				_localId = 1,
				name = "WFCatTestSession",
				subtype = "lightroom",
				type = "catalog",
			},
			continueFrom = 2,
		} )

	puller:getUpdatesContinuously()

	-- Push one document to Oz.

	local catalogSpec

	rxTest:expectSingleValue(

		catalogTestSession:catalogSpec()
			:tap( function( _catalogSpec )
				check( 'WFCatSpec' )
				catalogSpec = _catalogSpec
			end ),

		':ignore:' )

	-- asset document ...

	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local assetDocRevId = KSIdGenerator.uuidInLowercaseHex()

	local assetDocPutUrl = string.format( '%s/assets/%s/revisions/%s',
										  catalogSpec:catalogHref(),
										  assetId, assetDocRevId )

	local assetDocContent = KSJsonEncoder.encodeOzUploadJson {
		user_id = catalogSpec:wfOzUser():userId(),
		payload = {
			captureDate = "2004-11-02T11:10:09",
			importSource = {
				importTimestamp = '2016-01-29T15:53:04Z',
				importedBy = catalogSpec:ozUserId(),
				fileName = "fakeFile01.jpg",
				originalHeight = 16,
				originalWidth = 16,
			},
		},
		type = 'asset',
		subtype = 'image',
	}

	local assetDocPutArgs = {
		requestHeaders = {
			{ header = 'Content-Type', value = 'application/json' },
			{ header = 'Accept', value = 'application/json' },
		},
	}

	rxTest:expectOnlyCompleted(
		catalogSpec:put( assetDocPutUrl, assetDocContent, assetDocPutArgs )
			:filter( function() return false end ) )

	puller:getUpdatesContinuously()

	-- Now make sure puller grabbed these documents from Oz.

	local numDocUpdates = 0
	local expectedDocUpdates = {
			{
				_changeCounter = 1,
				_id = catalog:ozCatalogId(),
				_labels = {
					oz = true,
				},
				_localId = 1,
				name = "WFCatTestSession",
				subtype = "lightroom",
				type = "catalog",
			},
			{
				_changeCounter = 2,
				_id = assetId,
				_labels = {
					oz = true,
				},
				_localId = 2,
				captureDate = "2004-11-02T11:10:09",
				importSource = {
					importTimestamp = '2016-01-29T15:53:04Z',
					importedBy = catalogSpec:ozUserId(),
					fileName = "fakeFile01.jpg",
					originalHeight = 16,
					originalWidth = 16,
				},
				subtype = "image",
				type = "asset",
			},
		}

	local resultDocUpdates = {}

	local changedDocsObserver = rxTest:observe(
		catalog:docStore():scannedFilteredMappedChangesFeedSince( "WFCatMetadataPullerTest" , "reportAllChanges", {}, 0 )
			:map( function( doc )
				numDocUpdates = numDocUpdates + 1
				resultDocUpdates[ numDocUpdates ] = KSTableUtils.deepCopy( doc )
				return numDocUpdates
			end )
	)
	-- expect 2 updates: catalog doc and asset doc
	changedDocsObserver:expectDataWithinTime( 15, 2 )
	rxTest:expectEqual( expectedDocUpdates, stripWobblyBits( resultDocUpdates ) )
	changedDocsObserver:dispose()

	-- Now edit the document several times and make sure those edits come through.

	local assetDocRev2Id = KSIdGenerator.uuidInLowercaseHex()

	local assetDocPutUrl2 = string.format( '%s/assets/%s/revisions/%s/children/%s',
										   catalogSpec:catalogHref(),
										   assetId,
										   assetDocRevId,
										   assetDocRev2Id )

	local assetDocContent2 = KSJsonEncoder.encodeOzUploadJson {
		user_id = catalogSpec:wfOzUser():userId(),
		payload = {
			xmp = {
				dc = {
					description = "mumble mumble mumble",
				},
			},
		},
	}

	rxTest:expectOnlyCompleted(
		catalogSpec:put( assetDocPutUrl2, assetDocContent2, assetDocPutArgs )
			:filter( function() return false end ) )

	local assetDocRev3Id = KSIdGenerator.uuidInLowercaseHex()

	local assetDocPutUrl3 = string.format( '%s/assets/%s/revisions/%s/children/%s',
										   catalogSpec:catalogHref(),
										   assetId,
										   assetDocRev2Id,
										   assetDocRev3Id )

	local assetDocContent3 = KSJsonEncoder.encodeOzUploadJson {
		user_id = catalogSpec:wfOzUser():userId(),
		payload = {
			xmp = {
				dc = {
					description = "description version 3",
				},
			},
		},
	}

	rxTest:expectOnlyCompleted(
		catalogSpec:put( assetDocPutUrl3, assetDocContent3, assetDocPutArgs )
			:filter( function() return false end ) )


	local assetDocRev4Id = KSIdGenerator.uuidInLowercaseHex()

	local assetDocPutUrl4 = string.format( '%s/assets/%s/revisions/%s/children/%s',
										   catalogSpec:catalogHref(),
										   assetId,
										   assetDocRev3Id,
										   assetDocRev4Id )

	local assetDocContent4 = KSJsonEncoder.encodeOzUploadJson {
		user_id = catalogSpec:wfOzUser():userId(),
		payload = {
			xmp = {
				dc = {
					description = "description version 4",
				},
			},
		},
	}

	rxTest:expectOnlyCompleted(
		catalogSpec:put( assetDocPutUrl4, assetDocContent4, assetDocPutArgs )
			:filter( function() return false end ) )

	puller:getUpdatesContinuously()

	-- Now make sure puller grabbed these documents from Oz.

	numDocUpdates = 0
	expectedDocUpdates = {
			{
				_changeCounter = 3,
				_id = assetId,
				_labels = {
					oz = true,
				},
				_localId = 2,
				captureDate = "2004-11-02T11:10:09",
				importSource = {
					importTimestamp = '2016-01-29T15:53:04Z',
					importedBy = catalogSpec:ozUserId(),
					fileName = "fakeFile01.jpg",
					originalHeight = 16,
					originalWidth = 16,
				},
				subtype = "image",
				type = "asset",
				xmp = {
					dc = {
						description = "description version 4",
					},
				},
			},
	}

	resultDocUpdates = {}

	changedDocsObserver = rxTest:observe(
		catalog:docStore():scannedFilteredMappedChangesFeedSince( "WFCatMetadataPullerTest" , "reportAllChanges", {}, 3 )
			:map( function( doc )
				numDocUpdates = numDocUpdates + 1
				resultDocUpdates[ numDocUpdates ] = KSTableUtils.deepCopy( doc )
				return numDocUpdates
			end )
	)
	-- expect 1 update: asset doc with description changed to "description version 4".
	changedDocsObserver:expectDataWithinTime( 15, 1 )
	rxTest:expectEqual( expectedDocUpdates, stripWobblyBits( resultDocUpdates ) )
	changedDocsObserver:dispose()


	-- Find out which revision the latest Oz change has

	local taggedRevId

	rxTest:expectSingleValue(

			catalog:docStore():getRevisionForLabel( assetId, 'oz' )
			:tap( function( revId )
				taggedRevId = revId
			end ),

		':ignore:' )

	-- Now make a local change to the document on top the taggedRevId

	rxTest:expectSingleValue(

		catalog:docStore():setDocValue {
				_id = assetId,
				_revId = taggedRevId,
				_newRevId = "9b16e1f0739d4f0e8f880563ef2ba071",
				captureDate = "2004-11-02T11:10:09",
				importSource = {
					importTimestamp = '2016-01-29T15:53:04Z',
					importedBy = catalogSpec:ozUserId(),
					fileName = "fakeFile01.jpg",
					originalHeight = 16,
					originalWidth = 16,
				},
				subtype = "image",
				type = "asset",
				xmp = {
					dc = {
						description = "description version 4",
						title = "locally set title",
					},
				},
			},

	':ignore:' )

	-- make one more change remote

	local assetDocRev5Id = KSIdGenerator.uuidInLowercaseHex()

	local assetDocPutUrl5 = string.format( '%s/assets/%s/revisions/%s/children/%s',
										   catalogSpec:catalogHref(),
										   assetId,
										   assetDocRev4Id,
										   assetDocRev5Id )

	local assetDocContent5 = KSJsonEncoder.encodeOzUploadJson {
		user_id = catalogSpec:wfOzUser():userId(),
		payload = {
			xmp = {
				dc = {
					description = "description version 5",
				},
			},
		},
	}

	rxTest:expectOnlyCompleted(
		catalogSpec:put( assetDocPutUrl5, assetDocContent5, assetDocPutArgs )
			:filter( function() return false end ) )

	-- Force a RevisionCompressed doc for this asset

	local accessToken = catalogTestSession._userTestSession._accessToken

	local postParams = {
		requestMethod = 'POST',
		requestHeaders = {
			{ header = 'Authorization', value = 'Bearer ' .. accessToken },
			{ header = 'Content-Type', value = 'application/json' },
		}
	}

	local compressUrl = string.format( '%s/assets/%s/compress',
										catalogSpec:catalogHref(), assetId )

	local compressFullUrl = 'https://admin.adbephotos-stage.com' .. compressUrl

	rxTest:expectOnlyCompleted(
		WFRxHttpClient.put(compressFullUrl, '{}', postParams )
		:filter( function() return false end ) )

	puller:getUpdatesContinuously()

	numDocUpdates = 0
	resultDocUpdates = {}

	changedDocsObserver = rxTest:observe(
		catalog:docStore():scannedFilteredMappedChangesFeedSince( "WFCatMetadataPullerTest" , "reportAllChanges", {}, 4 )
			:map( function( doc )
				numDocUpdates = numDocUpdates + 1
				resultDocUpdates[ numDocUpdates ] = KSTableUtils.deepCopy( doc )
				return numDocUpdates
			end )
	)

	-- expect 3 updates: doc, compressed revision list, doc with conflict
	changedDocsObserver:expectDataWithinTime( 40, 3 )
	changedDocsObserver:dispose()

	-- Now attach conflict resolver.

	local conflictResolver = WFCatConflictResolver( catalog:docStore(), indexStore )
	rxTest:addCloseTarget( conflictResolver )

	-- Wait a few seconds while the conflict handler does its thing.
	local docValue = rxTest:observe(
		catalog:docStore():ongoingDocValue( assetId )
			:map( function( value )
				value._revId = "ignore me"
				value._changeCounter = "ignore me"
				local revision_ids = KSTableUtils.derefOpt(value, '_serverProvidedMetadata', 'revision_ids' )
				if revision_ids and #revision_ids == 2 then
					value._compressed = true
				end
				value._serverProvidedMetadata = "ignore me"
				return value
			end ) )

	docValue:expectDataWithinTime( 2,
			{
				_id = assetId,
				_localId = 2,
				_revId = "ignore me",
				_changeCounter = "ignore me",
				_compressed = true, -- did compression kick in
				_serverProvidedMetadata = "ignore me",
				captureDate = "2004-11-02T11:10:09",
				importSource = {
					importTimestamp = '2016-01-29T15:53:04Z',
					importedBy = catalogSpec:ozUserId(),
					fileName = "fakeFile01.jpg",
					originalHeight = 16,
					originalWidth = 16,
				},
				subtype = "image",
				type = "asset",
				xmp = {
					dc = {
						description = "description version 5",
						title = "locally set title",
					},
				},
			} )


	initialSyncObs:expectDataWithinTime( 0.5, false )

end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestOzCatalogPrefetchErrorCatalogDeleted( rxTest )

	rxTest:setTimeLimit( 10 )

	local catalogSpec = WFCatSpecMock( '98db98bfd554470dab2f862a4b853c40', kUserId, rxTest,

		{
			prefetchReqResp,

			{
				url = "/v2/catalogs/98db98bfd554470dab2f862a4b853c40",

				response = [[{
					"base": "https://photos-stage.adobe.io/v2/",
					"id": "]] .. catalogId .. [[",
					"type": "catalog"
					}]],

				fail = {
					[[ {
						"errors": {
							"catalog": [
								"does not exist"
							]
						}
					} ]],
					{
						status_code = '404',
					}
				},

			},

		} )

	local catalog = WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

	local puller = WFCatMetadataPuller( catalog, "Test Puller", options )
	rxTest:addCloseTarget( puller )

	rxTest:expectOnlyError( puller:getUpdatesOnce(),
		[[ {
						"errors": {
							"catalog": [
								"does not exist"
							]
						}
					} ]], { status_code = '404' } )

	local obs = rxTest:observe( puller:isOzCatalogDeleted() )

	obs:expectDataWithinTime( 1, true )

end

--------------------------------------------------------------------------------

local function getNoopCatalogForSpacePuller( rxTest, requestCount )

 	requestCount = requestCount or 1

 	local headers = {
		status_code = '200',
	}

 	local expectedResponses = {}

 	local noopSpaceResponse =
			[[{
				"base": "https://photos-stage.adobe.io/v2/spaces/]] .. spaceFeedId .. [[ /",
				"updates": [],
				"links": {
					"next": {
						"href": "]] .. actualNextUpdateSpaceFeedLink .. [["
					}
				}
			}]]

 	for i = 1, requestCount do

		local url

		if i == 1 then
			url = startingSpaceFeedLink
		else
			url = actualNextUpdateSpaceFeedLink
		end

		expectedResponses[ #expectedResponses + 1 ] = {
			url = url,
			response = noopSpaceResponse,
			headers = headers,
			options = {
				bypassRateLimiter = true,
			}
		}

	end

 	local catalogSpec = WFCatSpecMock( catalogId, kUserId, rxTest, expectedResponses )

	return WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

 end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestCreateSpacePuller_noop( rxTest )

	local catalog = getNoopCatalogForSpacePuller( rxTest )

	local spaceUpdatesUrl = catalog:ozCatalogSpec():spaceUpdatesUrlForSpaceId( spaceFeedId )

	local pullerOptions = KSTableUtils.deepCopy( options )
	pullerOptions.spaceId = spaceFeedId
	pullerOptions.updatesFeedUrl = spaceUpdatesUrl
	pullerOptions.isCatalogPuller = false

	local puller = WFCatMetadataPuller( catalog, spaceFeedId, pullerOptions )
	rxTest:addCloseTarget( puller )

	rxTest:expectOnlyCompleted( puller:getLatestUpdatesIfNeeded() )

	rxTest:expectSingleValue(

		catalog:docStore():changedDocs { fromChangeCounter = 0 }
			:map( stripWobblyBits ),
		{
			continueFrom = 1,
		} )

	local expectedLogData = {
		documentsReceivedByType = {},
		successfulUpdateCycles = 1,
		totalDocumentsReceived = 0,
	}

	verifyLogData( rxTest, puller:getDiagnosticLogData(), expectedLogData, nil, spaceFeedId )

end

--------------------------------------------------------------------------------

local function getSimpleCatalogForSpacePuller( rxTest )

 	local expectedResponses = {
		{

			url = "/v2/spaces/" .. spaceFeedId .. "/updates",

			-- somewhat minimized version of payload

			response = string.format( [[{
				"base": "https://photos.adobe.io",
				"updates": [
					{
						"id": "8462f5ba3b39fd050d3cd6b7352ba561",
						"created": "2015-05-27T00:34:56.669987Z",
						"updated": "2015-05-27T00:38:29.160369Z",
						"type": "asset",
						"subtype": "image",
						"revision_ids": [
							"a1e880f718a348b89663cf38f9f29d23",
							"cd1ab30cf8464ef2a9e736b50c0ba33d",
							"96f33a7215e04c9aa49346bbf8aae921",
							"813fdbda1b244c2d9f41008ba63eedba"
						],
						"payload": {
							"importSource": {
								"importTimestamp": "2015-05-26T17:34:23-07:00",
								"originalHeight": 1600,
								"originalWidth": 1068
							},
							"captureDate": "2015-05-17T15:50:12",
							"xmp": {
								"dc": {
									"rights": "test copyright string"
								}
							},
							"location": {
								"longitude": -93.600107,
								"latitude": 45.951733,
								"altitude": 377.9
							}
						}
					}
				],
				"links": {
					"next": {
						"href": "/v2/spaces/%s/updates?updated_since=pull1"
					}
				}
			}]], spaceFeedId ),

			headers = {
				status_code = '200',
			},

			options = {
				bypassRateLimiter = true,
			}

		},

		{
			url = "/v2/spaces/" .. spaceFeedId .. "/updates?updated_since=pull1",

			response = string.format( [[{
				"base": "https://photos.adobe.io",
				"updates": [
					{
						"id": "8462f5ba3b39fd050d3cd6b7352ba561",
						"created": "2015-05-27T00:34:56.669987Z",
						"updated": "2015-05-27T00:38:29.160369Z",
						"type": "asset",
						"subtype": "image",
						"revision_ids": [
							"a1e880f718a348b89663cf38f9f29d23",
							"cd1ab30cf8464ef2a9e736b50c0ba33d",
							"96f33a7215e04c9aa49346bbf8aae921",
							"813fdbda1b244c2d9f41008ba63eedba",
							"3a4a43dc2a9a4f1897c27f7a26e2e068"
						],
						"payload": {
							"xmp": {
								"dc": {
									"rights": "something changed"
								}
							},
							"importSource": {
								"importTimestamp": "2015-05-26T17:34:23-07:00",
								"originalHeight": 1600,
								"originalWidth": 1068
							},
							"captureDate": "2015-05-17T15:50:12"
						}
					}
				],
				"links": {
					"next": {
						"href": "/v2/spaces/%s/updates?updated_since=pull2"
					}
				}
			}]], spaceFeedId ),

			headers = {
				status_code = '200',
			},

			options = {
				bypassRateLimiter = true,
			}

		},
		{
			url = "/v2/spaces/" .. spaceFeedId .. "/updates?updated_since=pull2",

			response = string.format( [[{
				"base": "https://photos.adobe.io",
				"updates": [],
				"links": {
					"next": {
						"href": "/v2/spaces/%s/updates?updated_since=pull2"
					}
				}
			}]], spaceFeedId ),

			headers = {
				status_code = '200',
			},

			options = {
				bypassRateLimiter = true,
			}

		}

 	}

 	local catalogSpec = WFCatSpecMock( catalogId, kUserId, rxTest, expectedResponses )

	return WFCatMockCatalog.createWFCatalogForTestWithSpec( rxTest, catalogSpec )

 end

--------------------------------------------------------------------------------

function WFCatMetadataPullerTest.rxTestCreateSpacePuller_BasicUpdates( rxTest )

	local catalog = getSimpleCatalogForSpacePuller( rxTest )

	local spaceUpdatesUrl = catalog:ozCatalogSpec():spaceUpdatesUrlForSpaceId( spaceFeedId )

	local pullerOptions = KSTableUtils.deepCopy( options )
	pullerOptions.spaceId = spaceFeedId
	pullerOptions.updatesFeedUrl = spaceUpdatesUrl
	pullerOptions.isCatalogPuller = false

	local puller = WFCatMetadataPuller( catalog, spaceFeedId, pullerOptions )
	rxTest:addCloseTarget( puller )

	puller._syncIsStopped = false

	rxTest:expectOnlyCompleted( puller:getLatestUpdatesIfNeeded() )

	local expectedLogData = {
		documentsReceivedByType = {
			update = {
				asset = {
					image = 2,
				},
				total = 2,
			},
		},
		successfulUpdateCycles = 3,
		totalDocumentsReceived = 2,
	}

	local expectedLink = '/v2/spaces/' .. spaceFeedId .. '/updates?updated_since=pull2'

	verifyLogData( rxTest, puller:getDiagnosticLogData(), expectedLogData, expectedLink, spaceFeedId )

end

--------------------------------------------------------------------------------

-- TODO [scouten 2016-01-26]: Album cover.

-- TODO [scouten 2015-07-21]: Merging with locally-originated changes.

-- TODO [scouten 2015-07-21]: Push some documents via actual upload API and verify.

-- TODO [scouten 2015-08-05]: Cascading delete of related documents.

--------------------------------------------------------------------------------

return WFRxTestCase.wrapSuite( 'WFCatMetadataPullerTest', WFCatMetadataPullerTest )
