--[[----------------------------------------------------------------------------

WFRxDocStoreTest.lua

--------------------------------------------------------------------------------

ADOBE CONFIDENTIAL
------------------
Copyright 2015 Adobe Systems Incorporated
All Rights Reserved.

NOTICE: All information contained herein is, and remain the property of
Adobe Systems Incorporated and its suppliers, if any. The intellectual and
technical concepts contained herein are proprietary to Adobe Systems
Incorporated and its suppliers and may be covered by U.S. and Foreign
Patents, patents in process, and are protected by trade secret or copyright
law.  Dissemination of this information or reproduction of this material is
strictly forbidden unless prior written permission is obtained from
Adobe Systems Incorporated.

------------------------------------------------------------------------------]]

	-- RxLua
local Rx = require 'Rx'
local Observable = Rx.Observable

	-- WF.core.self_test
local WFRxTestCase = require 'WFRxTestCase'

	-- WF.core.utils
local KSIdGenerator = require 'KSIdGenerator'
local KSPathUtils = require 'KSPathUtils'
local WFCloseSupport = require 'WFCloseSupport'
local WFFileUtils = require 'WFFileUtils'

	-- WF.sqlite.database
local WFSQLiteDatabase = require "WFSQLiteDatabase"
local WFSQLiteDatabaseSchema = require "WFSQLiteDatabaseSchema"

	-- WF.sqlite.doc_store
local WFRxDocStore = require 'WFRxDocStore'
local WFRxDocStoreMessageProcessor = require 'WFRxDocStoreMessageProcessor'
local WFRxDocStoreTestFactory = require 'WFRxDocStoreTestFactory'

local WFCapabilities = require "WFCapabilities"

--------------------------------------------------------------------------------

local TEST_WAIT_TIME = 6

--============================================================================--

local WFRxDocStoreTest = WFRxDocStoreTestFactory( 'WFRxDocStore', { testNonTransitableData = false } )
	-- Most of this test suite is shared with WFRxDocStoreImpTest,
	-- but we add some additional tests here for methods that aren't present
	-- in the imp.

--------------------------------------------------------------------------------

WFRxDocStoreTest.doTestCreateDocStoreAtPath( 'uuid', KSIdGenerator.uuidInLowercaseHex() )

--------------------------------------------------------------------------------

local function deleteDB( path )

	WFFileUtils.delete( path )
	WFFileUtils.delete( path .. '-shm' )
	WFFileUtils.delete( path .. '-wal' )

end

--------------------------------------------------------------------------------

-- @mark logging

--------------------------------------------------------------------------------

local function docStoreSanityCheck( rxTest, docStore )

	-- Rough sanity check on doc store when running in specific modes.

	rxTest:setTimeLimit( TEST_WAIT_TIME )

	-- Create first revision of empty document.

	local firstRevId

	rxTest:expectSingleValue(

		docStore:setDocValue {
			_id = 'testDocId',
			test = 42,
		}
		:tap( function( rev )
			firstRevId = rev
		end ),
		
		':ignore:' )

	-- Replace that with second revision and read back.

	local secondRevId

	rxTest:expectSingleValue(

		docStore:setDocValue {
			_id = 'testDocId',
			_revId = firstRevId,
			test = 45,
		}
		:tap( function( rev )
			secondRevId = rev
		end ),
		
		':ignore:' )

	rxTest:expectSingleValue(

		docStore:docValue( 'testDocId' ),

		{
			_id = 'testDocId',
			_localId = 1,
			_revId = secondRevId,
			_changeCounter = 2,
			test = 45,
		} )

end

--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestRecordTransactionSources( rxTest )

	-- This test is mostly here to ensure that the doc store doesn't blow up
	-- when running in this mode. Not currently validating its output.

	local docStore = WFRxDocStoreTest.setUpDocStore( rxTest )

	docStore:recordTransactionSources()

	docStoreSanityCheck( rxTest, docStore )

end

--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestRecordApiPerformance( rxTest )

	-- This test is mostly here to ensure that the doc store doesn't blow up
	-- when running in this mode. Not currently validating its output.

	local docStore = WFRxDocStoreTest.setUpDocStore( rxTest )

	docStore:recordApiPerformance()

	docStoreSanityCheck( rxTest, docStore )

	docStore:sendDiagnosticLogData()

end

--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestCaptureTracebackForFuncs( rxTest )

	-- This test is mostly here to ensure that the doc store doesn't blow up
	-- when running in this mode. Not currently validating its output.

	local docStore = WFRxDocStoreTest.setUpDocStore( rxTest )

	docStore:captureTracebackForFuncs { setDocValue = true }

	docStoreSanityCheck( rxTest, docStore )

	rxTest:expectOnlyError(

		Observable.pcall( function()
			return docStore:setDocValue {} -- intentional failure -- no ID
		end ),

		':ignore:' )

end

--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestSetLogLevel( rxTest )

	-- This test is mostly here to ensure that the doc store doesn't blow up
	-- when running in this mode. Not currently validating its output.

	local docStore = WFRxDocStoreTest.setUpDocStore( rxTest )

	docStore:setLogLevel( 'debug' )

	docStoreSanityCheck( rxTest, docStore )

end

--------------------------------------------------------------------------------

-- @mark full document values

--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestDocValueCheckType( rxTest )

	local docStore = WFRxDocStoreTest.setUpDocStore( rxTest )
	
	rxTest:setTimeLimit( TEST_WAIT_TIME )
	
	-- Set doc value and capture rev ID.

	local firstRevId

	rxTest:expectSingleValue(

		docStore:setDocValue {
			_id = 'testDocId',
			test = 42,
			type = 'whatever',
		}
		:tap( function( rev )
			firstRevId = rev
		end ),

		':ignore:' )

	rxTest:expectSingleValue(

		docStore:docValueCheckType( 'whatever', 'testDocId' ),

		{
			_id = 'testDocId',
			_localId = 1,
			_revId = firstRevId,
			_changeCounter = 1,
			test = 42,
			type = 'whatever',
		} )

	rxTest:expectOnlyError(
		docStore:docValueCheckType( 'not_whatever', 'testDocId' ),
		':ignore:' )

	-- verify that the type is inserted into the docs table
	local docsOfType = docStore:getDocsOfType( { whatever =  { "NULL" } } )
		:map( function( docIds )
			return docIds
		end )

	local obs = rxTest:observe( docsOfType )

	obs:expectData( {
		'testDocId',
	})

end

--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestSetDocValue_recordError( rxTest )

	-- Make sure we fail gracefully even while recording error conditions.
	-- (Basically a lot of test code to eke out just one more line of code
	-- coverage. Yeah, I'm picky like that.)

	local docStore = WFRxDocStoreTest.setUpDocStore( rxTest )

	rxTest:setTimeLimit( TEST_WAIT_TIME )

	docStore:recordApiPerformance()

	-- Repeat the same error >10 times.
	-- It's an intentional failure: we're omitting the _id field.

	for _ = 1, 11 do

		rxTest:expectOnlyError( docStore:setDocValue {}, ':ignore:' )

	end

end

--------------------------------------------------------------------------------

-- @mark partial document values

--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestOngoingFieldValue( rxTest )

	local docStore = WFRxDocStoreTest.setUpDocStore( rxTest )

	rxTest:setTimeLimit( TEST_WAIT_TIME )

	local ongoingFieldValue = docStore:ongoingFieldValue( 'testDocId', 'rating' )

	local ofv = rxTest:observe( ongoingFieldValue )

	ofv:expectData( nil )

	local firstRevId

	rxTest:expectSingleValue(
	
		docStore:setDocValue {
			_id = 'testDocId',
			rating = 2,
			title = "My Nifty Photo",
		}
		:tap( function( rev )
			firstRevId = rev
		end ),
		
		':ignore:' )

	ofv:expectData( 2 )

	rxTest:expectSingleValue(
		
		docStore:setDocValue {
			_id = 'testDocId',
			_revId = firstRevId,
			rating = 0,
			title = "My Nifty Photo",
		},
		
		':ignore:' )
	
	ofv:expectData( 0 )

end

--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestOngoingFieldValue_nilByDefault( rxTest )

	local docStore = WFRxDocStoreTest.setUpDocStore( rxTest )

	rxTest:setTimeLimit( TEST_WAIT_TIME )

	local ongoingFieldValue = docStore:ongoingFieldValue( 'testDocId', 'rating' )

	local ofv = rxTest:observe( ongoingFieldValue )

	ofv:expectData( nil )

	local firstRevId

	rxTest:expectSingleValue(

		docStore:setDocValue {
			_id = 'testDocId',
			rating = 2,
			title = "My Nifty Photo",
		}
		:tap( function( rev )
			firstRevId = rev
		end ),
		
		':ignore:' )

	ofv:expectData( 2 )

	rxTest:expectSingleValue(

		docStore:setDocValue {
			_id = 'testDocId',
			_revId = firstRevId,
			-- rating = nil,
			title = "My Nifty Photo",
		},
		
		':ignore:' )

	ofv:expectData( nil )

end

--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestOngoingFieldValue_nested( rxTest )

	local docStore = WFRxDocStoreTest.setUpDocStore( rxTest )

	rxTest:setTimeLimit( TEST_WAIT_TIME )

	local ongoingFieldValue = docStore:ongoingFieldValue( 'testDocId', 'foo/1/rating' )

	local ofv = rxTest:observe( ongoingFieldValue )

	ofv:expectData( nil )

	local firstRevId

	rxTest:expectSingleValue(

		docStore:setDocValue {
			_id = 'testDocId',
			foo = {
				{
					rating = 2
				}
			},
			title = "My Nifty Photo",
		}
		:tap( function( rev )
			firstRevId = rev
		end ),
		
		':ignore:' )

	ofv:expectData( 2 )

	local secondRevId

	rxTest:expectSingleValue(

		docStore:setDocValue {
			_id = 'testDocId',
			_revId = firstRevId,
			-- foo = nil,
			title = "My Nifty Photo",
		}
		:tap( function( rev )
			secondRevId = rev
		end ),
		
		':ignore:' )

	ofv:expectData( nil )

	rxTest:expectSingleValue(

		docStore:setDocValue {
			_id = 'testDocId',
			_localId = 1,
			_revId = secondRevId,
			foo = {
				{
					rating = 4
				}
			},
			title = "My Nifty Photo",
		},
		
		':ignore:' )

	ofv:expectData( 4 )

end

--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestOngoingFieldValue_errorNoPath( rxTest )

	local docStore = WFRxDocStoreTest.setUpDocStore( rxTest )

	rxTest:setTimeLimit( TEST_WAIT_TIME )

	rxTest:expectOnlyError( 
		
		Observable.pcall( function()

			return docStore:ongoingFieldValue( 'testDocId', '/' )
		
		end ),
		
		':ignore:' )
		
end

--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestHalt( rxTest )

	local docStore = WFRxDocStoreTest.setUpDocStore( rxTest )
	
	rxTest:expectOnlyCompleted( docStore:halt() )
	
	rxTest:expectOnlyError( 
		
		Observable.pcall( function()

			return docStore:ongoingFieldValue( 'testDocId', '/' )
		
		end ),
		
		':ignore:' )
		
end

--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestCloseDbEarly( rxTest )

	-- capture a separate docstore thread handle that will close the db 
	-- early
	local extraDocStore = nil

	local hookId =
		WFRxDocStoreMessageProcessor.onBootstrapWFRxDocStoreImp( 
			function( path )

				extraDocStore = WFRxDocStore( path )

			end )

	local docStore = WFRxDocStore( ":memory:" )

	WFRxDocStoreMessageProcessor.removeBootstrapWFRxDocStoreImpHook( 
		hookId )
	
	rxTest:expectNotEqual( extraDocStore, nil )

	-- check that capturing extraDocStore doesn't break the normal
	-- doc store
	rxTest:expectSingleValue(

		docStore:setDocValue {
			_id = 'doc1',
			wut = 'wut',
		},

		':ignore:' )

	-- close the db early

	rxTest:expectOnlyCompleted( extraDocStore:closeDb() )

	-- close the extra docstore instance
	extraDocStore:close()

	-- verify we fail in the orig docstore
	rxTest:expectOnlyError(

		docStore:setDocValue {
			_id = 'doc3',
			wut = 'wot',
		},

		':ignore:' )

	docStore:close()
	
		
end

--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestCloseDbLate( rxTest )

	-- capture a separate docstore thread handle that will close the db 
	-- early
	local extraDocStore = nil
	local extraDocStore2 = nil

	local hookId =
		WFRxDocStoreMessageProcessor.onBootstrapWFRxDocStoreImp( 
			function( path )

				extraDocStore = WFRxDocStore( path )

				extraDocStore2 = WFRxDocStore( path )

			end )

	local docStore = WFRxDocStore( ":memory:" )

	WFRxDocStoreMessageProcessor.removeBootstrapWFRxDocStoreImpHook( 
		hookId )
	
	rxTest:expectNotEqual( extraDocStore, nil )
	
	rxTest:expectNotEqual( extraDocStore2, nil )

	-- close the docstore
	docStore:close()

	-- check that the thread is still kicking
	rxTest:expectSingleValue(

		extraDocStore
			:setDocValue(
				{
					_id = 'doc1',
					wut = 'wut',
				} ),

		':ignore:' )

	-- check that the thread is still kicking
	rxTest:expectSingleValue(

		extraDocStore2
			:setDocValue(
				{
					_id = 'doc2',
					wut = 'wat',
				} ),

		':ignore:' )
	
	-- close the db 

	rxTest:expectOnlyCompleted( extraDocStore:closeDb() )

	-- close the extra docstore instance
	extraDocStore:close()

	-- verify that we can't add to the closed db
	rxTest:expectOnlyError(

		extraDocStore2
			:setDocValue(
				{
					_id = 'doc3',
					wut = 'wot',
				} ),

		':ignore:' )

	-- check a second close doesn't screw up stuff.

	rxTest:expectOnlyCompleted( extraDocStore2:closeDb() )

	-- close the extra docstore instance
	extraDocStore2:close()

		
end

--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestCompactProgress( rxTest )

	rxTest:setTimeLimit( TEST_WAIT_TIME )

	-- This test needs to exercise some code that can't run on in-memory
	-- databases.

	WFCapabilities.setCapability( 'WFRxDocStore.disableRevisionCleanupForCompact', 'true' )

	rxTest:finally( function()
		WFCapabilities.setCapability( 'WFRxDocStore.WFRxDocStore.disableRevisionCleanupForCompact', nil )
	end )

	local path = KSPathUtils.child( KSPathUtils.getStandardFilePath 'temp',
									'RxDocStoreForCompactTest' )

	deleteDB( path )

	local docStore = WFRxDocStoreTest.setUpDocStore( rxTest, WFRxDocStore( path ) )

	-- Create first revision of empty document.

	local rev1Id = WFRxDocStoreTest.setDocValueAndValidateRevId( rxTest, docStore,
		{
			_id = 'testDoc',
			test = 42,
		} )

	-- Replace that with second (deleted) revision.

	local rev2Id = WFRxDocStoreTest.setDocValueAndValidateRevId( rxTest, docStore,
		{
			_id = 'testDoc',
			_revId = rev1Id,
			_deleted = true,
		} )

	-- Replace that with third revision.

	local rev3Id = WFRxDocStoreTest.setDocValueAndValidateRevId( rxTest, docStore,
		{
			_id = 'testDoc',
			_revId = rev2Id,
			_labels = {
				saveMe = true,
			},
			because = "NoSQL",
		} )

	-- Now read back all three revisions.

	rxTest:expectSingleValue(

		docStore:docValue( 'testDoc', rev1Id ),
		{
			_id = 'testDoc',
			_localId = 1,
			_revId = rev1Id,
			_changeCounter = 1,
			test = 42,
		} )

	rxTest:expectSingleValue(

		docStore:docValue( 'testDoc', rev2Id ),
		{
			_id = 'testDoc',
			_localId = 1,
			_revId = rev2Id,
			_changeCounter = 2,
			_deleted = true,
		} )

	local expectedValueRev3 = {
		-- This one gets reused later in the test.
		_id = 'testDoc',
		_localId = 1,
		_revId = rev3Id,
		_changeCounter = 3,
		_labels = {
			saveMe = true,
		},
		because = "NoSQL",
	}

	rxTest:expectSingleValue(

		docStore:docValue( 'testDoc', rev3Id ),
		expectedValueRev3 )

	-- Read back document history before compaction.

	rxTest:expectSingleValue(

		docStore:docRevisionHistory( 'testDoc' ),
		{
			{
				revId = rev3Id,
				status = 'available',
			},
			{
				revId = rev2Id,
				status = 'deleted',
			},
			{
				revId = rev1Id,
				status = 'available',
			},
		} )

	-- Compact the DB.

	local obs = rxTest:observe( docStore:compact() )

	obs:expectData(	
		{
			percentComplete = 0,
			revisionsRemoved = 0
		} )

	obs:expectData(	
		{
			percentComplete = 100,
			revisionsRemoved = 2
		} )
		
	obs:expectCompleted()
	
	-- Now read back all three revisions. Only the most recent should succeed.

	rxTest:expectOnlyError(
		Observable.pcall( function()
			return docStore:docValue( 'testDoc', rev1Id )
		end ),
		':ignore:' )

	rxTest:expectOnlyError(
		Observable.pcall( function()
			return docStore:docValue( 'testDoc', rev2Id )
		end ),
		':ignore:' )

	rxTest:expectSingleValue(
		docStore:docValue( 'testDoc', rev3Id ),
		expectedValueRev3 )

	-- Read back history. Should only show the remaining (labelled) rev.

	rxTest:expectSingleValue(

		docStore:docRevisionHistory( 'testDoc' ),

		{
			{
				revId = rev3Id,
				status = 'available',
			}
		} )

	rxTest:expectOnlyCompleted( docStore:haltAndDelete() )
		
end

--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestCompactVacuumProgress( rxTest )

	rxTest:setTimeLimit( 240 )
	
	-- This test needs to exercise some code that can't run on in-memory
	-- databases.

	WFCapabilities.setCapability( 'WFRxDocStore.disableRevisionCleanupForCompact', 'true' )

	rxTest:finally( function()
		WFCapabilities.setCapability( 'WFRxDocStore.disableRevisionCleanupForCompact', nil )
	end )

	local path = KSPathUtils.child( KSPathUtils.getStandardFilePath 'temp',
									'rxTestCompactVacuumProgress' )

	deleteDB( path )

	local docStore = WFRxDocStoreTest.setUpDocStore( rxTest, WFRxDocStore( path ) )

	-- Create first revision of empty document.

	local rev1Id = WFRxDocStoreTest.setDocValueAndValidateRevId( rxTest, docStore,
		{
			_id = 'testDoc',
			test = 42,
		} )

	-- Replace that with second (deleted) revision.

	local rev2Id = WFRxDocStoreTest.setDocValueAndValidateRevId( rxTest, docStore,
		{
			_id = 'testDoc',
			_revId = rev1Id,
			_deleted = true,
		} )

	-- Replace that with third revision.

	local rev3Id = WFRxDocStoreTest.setDocValueAndValidateRevId( rxTest, docStore,
		{
			_id = 'testDoc',
			_revId = rev2Id,
			_labels = {
				saveMe = true,
			},
			because = "NoSQL",
		} )

	-- Compact the DB and perform a vacuum

	local obs = rxTest:observe( docStore:compact( true ) )

	obs:expectData(	
		{
			percentComplete = 0,
			revisionsRemoved = 0
		} )

	obs:expectData(	
		{
			percentComplete = 50,
			revisionsRemoved = 2
		} )

	obs:expectDataWithinTime( TEST_WAIT_TIME / 2,
			{
			percentComplete = 100,
			revisionsRemoved = 2
		} )
	
	obs:expectCompleted()
	
	-- Now read back all three revisions. Only the most recent should succeed.

	rxTest:expectOnlyError(
		Observable.pcall( function()
			return docStore:docValue( 'testDoc', rev1Id )
		end ),
		':ignore:' )

	rxTest:expectOnlyError(
		Observable.pcall( function()
			return docStore:docValue( 'testDoc', rev2Id )
		end ),
		':ignore:' )

	rxTest:expectSingleValue(
		docStore:docValue( 'testDoc', rev3Id ),
		{
			_id = 'testDoc',
			_localId = 1,
			_revId = rev3Id,
			_changeCounter = 3,
			_labels = {
				saveMe = true,
			},
			because = "NoSQL",
	} )

	rxTest:expectOnlyCompleted( docStore:haltAndDelete() )
		
end

--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestCheckVacuumSpace( rxTest )

	-- for meaningful code coverage, use a database on disk

	local path = KSPathUtils.child( KSPathUtils.getStandardFilePath 'temp',
									'rxTestCheckVacuumSpace' )

	deleteDB( path )

	local docStore = WFRxDocStoreTest.setUpDocStore( rxTest, WFRxDocStore( path ) )

	-- create a revision (otherwise, the docStore will take no space on disk)
	WFRxDocStoreTest.setDocValueAndValidateRevId( rxTest, docStore,
		{
			_id = 'testDoc',
			test = 42,
		} )

	rxTest:expectEqual( true, docStore:enoughSpaceForVacuum() )

	rxTest:expectOnlyCompleted( docStore:haltAndDelete() )

end

--------------------------------------------------------------------------------

local function stripLocalIdRevIdAndChangeCounter( docValue )
	docValue._localId = nil
	docValue._revId = nil
	docValue._changeCounter = nil
	return docValue
end 
	
--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestApplyOverlayWithNewId( rxTest )
	
	local docStore = WFRxDocStoreTest.setUpDocStore( rxTest )
	
	rxTest:expectSingleValue( docStore:setDocValue( {
				_id = 'testDoc',
				test = 42,
			} ), ':ignore:' )
	
	---- applyOverlay ----
	
	-- applyOverlay _newId on non-delete, throw error
	rxTest:expectOnlyError( docStore:applyOverlay( { _id = 'testDoc', _newId = 'testDoc2' } ), 
			"value contains _newId value: testDoc2, but not _deleted (must be true)" )
	
	-- applyOverlay _newId on delete, should succeed and create testDoc2 with testDoc contents
	rxTest:expectSingleValue( docStore:applyOverlay( { _id = 'testDoc', _deleted = true, _newId = 'testDoc2' } ), 
		':ignore:' ) -- error message is not constant due to being wrapped in transaction, use :ignore:
	
	-- get the doc with a _newId value, it should be deleted
	rxTest:expectSingleValue( docStore:docValue( 'testDoc' ):map( stripLocalIdRevIdAndChangeCounter ),
		{ _id = 'testDoc', _deleted = true, _empty = true, _newId = 'testDoc2' } )
	
	-- get the _newId doc value, it should have the contents of the original doc
	rxTest:expectSingleValue( docStore:docValue( 'testDoc2' ):map( stripLocalIdRevIdAndChangeCounter ),
		{ _id = 'testDoc2', test = 42 } )
	
	-- applyOverlay _newId + _deleted on already deleted doc
	rxTest:expectOnlyError( docStore:applyOverlay( { _id = 'testDoc', _deleted = true, _newId = 'testDoc5' } ), 
		':ignore:' ) -- error message is not constant due to being wrapped in transaction, use :ignore:
		
	-- applyOverlay with _newId + _deleted while changing type/subtype on the new doc
	rxTest:expectSingleValue( docStore:applyOverlay( {_id = 'testDoc2', _deleted = true, _newId = 'testDoc3', 
			_newIdOverlay = { type = 'asset', subtype = 'deleted_image', test = 45 } } ),
		':ignore:' )
		
	rxTest:expectSingleValue( docStore:docValue( 'testDoc3' ):map( stripLocalIdRevIdAndChangeCounter ),
		{ _id = 'testDoc3', type = 'asset', subtype = 'deleted_image', test = 45 } )

	-- verify that the type which is recorded in the docs table is "deleted_image"
	
	local docsOfType = docStore:getDocsOfType( { asset =  { "deleted_image" } } )
		:map( function( docIds )
			return docIds
		end )

	local obs = rxTest:observe( docsOfType )

	obs:expectData( {
		'testDoc3',
	})
	
end

--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestBulkApplyOverlayWithNewId( rxTest )
	
	local docStore = WFRxDocStoreTest.setUpDocStore( rxTest )
	
	rxTest:expectSingleValue( docStore:setDocValue( {
				_id = 'testDoc',
				test = 42,
			} ), ':ignore:' )
	
	-- bulkApplyOverlay _newId on non-delete, throw error
	rxTest:expectOnlyError( docStore:bulkApplyOverlay( { { _id = 'testDoc', _newId = 'testDoc2' } } ), 
			':ignore:' )
	
	-- bulkApplyOverlay _newId on delete, should succeed and create testDoc2 with testDoc contents
	rxTest:expectSingleValue( docStore:bulkApplyOverlay( { { _id = 'testDoc', _deleted = true, _newId = 'testDoc2' } } ), 
		':ignore:' ) -- error message is not constant due to being wrapped in transaction, use :ignore:
	
	-- get the doc with a _newId value, it should be deleted
	rxTest:expectSingleValue( docStore:docValue( 'testDoc' ):map( stripLocalIdRevIdAndChangeCounter ),
		{ _id = 'testDoc', _deleted = true, _empty = true, _newId = 'testDoc2' } )
	
	-- get the _newId doc value, it should have the contents of the original doc
	rxTest:expectSingleValue( docStore:docValue( 'testDoc2' ):map( stripLocalIdRevIdAndChangeCounter ),
		{ _id = 'testDoc2', test = 42 } )
	
	-- bulkApplyOverlay _newId + _delete on already deleted doc
	rxTest:expectOnlyError( docStore:bulkApplyOverlay( { { _id = 'testDoc', _deleted = true, _newId = 'testDoc5' } } ), 
		':ignore:' ) -- error message is not constant due to being wrapped in transaction, use :ignore:
		
	-- bulkApplyOverlay with _newId + _deleted while changing type/subtype on the new doc
	rxTest:expectSingleValue( docStore:bulkApplyOverlay( { {_id = 'testDoc2', _deleted = true, _newId = 'testDoc3', _newIdOverlay = { type = 'asset', subtype = 'deleted_image', test = 45 } } } ),
		':ignore:' )
		
	rxTest:expectSingleValue( docStore:docValue( 'testDoc3' ):map( stripLocalIdRevIdAndChangeCounter ),
		{ _id = 'testDoc3', type = 'asset', subtype = 'deleted_image', test = 45 } )

	-- verify that the type which is recorded in table is "deleted_image"
	local docsOfType = docStore:getDocsOfType( { asset =  { "deleted_image" } } )
		:map( function( docIds )
			return docIds
		end )

	local obs = rxTest:observe( docsOfType )

	obs:expectData( {
		'testDoc3',
	})

end

--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestSetDocValueWithNewId( rxTest )
	
	-- setting a docValue containing a _newId is not supported so all of these should
	-- produce an error. Using applyOverlay is the only supported way to set a _newId
	local docStore = WFRxDocStoreTest.setUpDocStore( rxTest )
	
	---- setDocValue ----
	
	-- setDocValue should not allow _newId with or without the _deleted flag set to true
	rxTest:expectOnlyError( docStore:setDocValue( { _id = 'testDoc3', _newId = 'testDoc4' } ),
		"value contains _newId value: testDoc4, but not _deleted (must be true)" )
	
	rxTest:expectOnlyError( docStore:setDocValue( { _id = 'testDoc3', _newId = 'testDoc4', _deleted = true } ),
		':ignore:' ) -- error message is not constant due to being wrapped in transaction, use :ignore:
	
	---- bulkSetDocValue ----
	
	rxTest:expectOnlyError( docStore:bulkSetDocValue( { { _id = 'testDoc3', _newId = 'testDoc4' } } ),
		':ignore:' ) -- error message is not constant due to being wrapped in transaction, use :ignore:
	
	rxTest:expectOnlyError( docStore:bulkSetDocValue( { { _id = 'testDoc3', _newId = 'testDoc4', _deleted = true } } ),
		':ignore:' ) -- error message is not constant due to being wrapped in transaction, use :ignore:

	
end

--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestApplyOverlayToDeletedWithNewId( rxTest )
	
	local docStore = WFRxDocStoreTest.setUpDocStore( rxTest )
	
	-- apply overlay to deleted with _newId, second asset gets update
	rxTest:expectSingleValue( docStore:setDocValue( { _id = 'testDoc', test = 42 } ), ':ignore:' )
	rxTest:expectSingleValue( docStore:applyOverlay( { _id = 'testDoc', _deleted = true, _newId = 'testDoc2' } ), ':ignore:' )
	
	rxTest:expectSingleValue( docStore:docValue( 'testDoc2' ):map( stripLocalIdRevIdAndChangeCounter ),
		{ _id = 'testDoc2', test = 42 } )
	
	rxTest:expectSingleValue( docStore:applyOverlay( { _id = 'testDoc', test = 45 } ),
		':ignore:' )
	
	rxTest:expectSingleValue( docStore:docValue( 'testDoc2' ):map( stripLocalIdRevIdAndChangeCounter ), 
		{ _id = 'testDoc2', test = 45 } )
		
	-- apply overlay, but don't follow the _newId if the overlay is a delete, so testDoc2 should not end up deleted
	rxTest:expectSingleValue( docStore:applyOverlay( { _id = 'testDoc', _deleted = true } ),
		':ignore:' )
	
	rxTest:expectSingleValue( docStore:docValue( 'testDoc2' ):map( stripLocalIdRevIdAndChangeCounter ), 
		{ _id = 'testDoc2', test = 45 } )
	
	-- apply overlay, should follow _newId multiple times
	rxTest:expectSingleValue( docStore:applyOverlay( { _id = 'testDoc2', _deleted = true, _newId = 'testDoc3' } ), 
		':ignore:' )
	
	rxTest:expectSingleValue( docStore:docValue( 'testDoc3' ):map( stripLocalIdRevIdAndChangeCounter ),
		{ _id = 'testDoc3', test = 45 } )
	
	rxTest:expectSingleValue( docStore:applyOverlay( { _id = 'testDoc', test = 48 } ),
		':ignore:' )
	
	rxTest:expectSingleValue( docStore:docValue( 'testDoc3' ):map( stripLocalIdRevIdAndChangeCounter ),
		{ _id = 'testDoc3', test = 48 } )
	
end

--------------------------------------------------------------------------------

function WFRxDocStoreTest.rxTestDocStoreInitializationWithProgress( rxTest )

	rxTest:setTimeLimit( 5 )

	local docStorePath = KSPathUtils.getStandardFilePath( "temp" )
	docStorePath = KSPathUtils.child( docStorePath, 'rxTestDocStoreInitializationWithProgress' )
	docStorePath = WFFileUtils.ensureFileNameIsUnique( docStorePath )
	local docStore = WFRxDocStore( docStorePath )

	-- prepare docStore
	local docValues = {}

	for i = 1, 2000 do
		docValues[ i ] = {
			_id = KSIdGenerator.uuidInLowercaseHex(),
			test = 42
		}

	end

	rxTest:expectSingleValue( docStore:bulkSetDocValue( docValues ),
		':ignore:' )

	rxTest:expectOnlyCompleted( docStore:halt() )

	-- brute force deletion of indexes
	local db = WFSQLiteDatabase( docStorePath, WFSQLiteDatabaseSchema( "1.2" ) )
	db:execOnce( [[DROP INDEX docs_mostRecentRevSequence]] )
	db:execOnce( [[DROP INDEX docs_winningRevSequence]] )
	db:close()
	db = nil

	-- reopen docStore, this time with missing indexes
	local docStoreOptArgs = {
		initProgressRequested = true,
	}
	docStore = WFRxDocStore( docStorePath, docStoreOptArgs )
	local progressStream = docStore:getInitializationProgress()
			:take( 3 )
			:toArray()

	local obs = rxTest:observe( progressStream )

	obs:expectData( {
			{
				description = "docStore initialization",
				errorCount = 0,
				percentComplete = 0,
				processedCount = 0,
				totalCount = 1000,
			},
			{
				description = "docStore initialization",
				errorCount = 0,
				percentComplete = 0.01,
				processedCount = 10,
				totalCount = 1000,
			},
			{
				description = "docStore initialization",
				errorCount = 0,
				percentComplete = 0.02,
				processedCount = 20,
				totalCount = 1000,
			} } )

	docStore:haltAndDelete()

end

-----------------------------------------------------------------

function WFRxDocStoreTest.rxTestOnSchemaConflictHooks( rxTest )
	
	-- make sure that if we get a schema conflict and there are conflict hooks installed
	-- that we will properly call the hooks. Note that we also open a doc store successfully
	-- on this thread in this test too because of WF-1065 and the previous implementation of the
	-- hooks not firing on a conflict if another docStore was previously opened successfully in
	-- that thread.
	rxTest:setTimeLimit( 5 )
	
	local docStore, docStore2
	
	local docStorePath = KSPathUtils.getStandardFilePath( "temp" )
	docStorePath = KSPathUtils.child( docStorePath, 'rxTestOnSchemaConflictHooks' )
	docStorePath = WFFileUtils.ensureFileNameIsUnique( docStorePath ) -- will have schema conflict
	
	docStore = WFRxDocStore( docStorePath )
	
	local docStorePath2 = WFFileUtils.ensureFileNameIsUnique( docStorePath ) -- will not have conflict
	
	local function closeDocStore( docStore )
		if docStore and not WFCloseSupport.hasBeenOrWillBeClosed( docStore ) then
			return docStore:haltAndDelete():catch(function() return Observable.just() end )
		end
	end
	
	rxTest:finally( function() 
		return Observable.combineLatest( closeDocStore( docStore ), closeDocStore( docStore2 ) )
			:finally( function()
				deleteDB( docStorePath )
				deleteDB( docStorePath2 )
			end )
	end )

	rxTest:expectOnlyCompleted( docStore:halt() )

	-- brute force some changes that will cause a conflict
	local db = WFSQLiteDatabase( docStorePath, WFSQLiteDatabaseSchema( "1.2" ) )
	db:execOnce( [[ALTER TABLE docs ADD COLUMN conflictTest]] )
	db:close()
	db = nil
	
	local counter = 0
	local counterObservable = Rx.ReplaySubject( 1 )
	local counterObserver = rxTest:observe( counterObservable )
	
	local hookId
	hookId = WFRxDocStoreMessageProcessor.onSchemaConflictWFRxDocStoreImp( function( obs ) 
		if hookId then
			WFRxDocStoreMessageProcessor.removeSchemaConflictWFRxDocStoreImpHook( hookId )
		end
		obs:onNext( counter + 1 )
	end, counterObservable )
	
	docStore2 = WFRxDocStore( docStorePath2 )
	
	docStore = WFRxDocStore( docStorePath )
	
	counterObserver:expectDataWithinTime( 2, 1 )
	
end

--==============================================================================

return WFRxTestCase.wrapSuite( WFRxDocStoreTest )
