--[[----------------------------------------------------------------------------

WFRxIndexStoreSchema.lua

--------------------------------------------------------------------------------

ADOBE CONFIDENTIAL
------------------
Copyright 2011 Adobe Systems Incorporated
All Rights Reserved.

NOTICE: All information contained herein is, and remain the property of
Adobe Systems Incorporated and its suppliers, if any. The intellectual and
technical concepts contained herein are proprietary to Adobe Systems
Incorporated and its suppliers and may be covered by U.S. and Foreign
Patents, patents in process, and are protected by trade secret or copyright
law.  Dissemination of this information or reproduction of this material is
strictly forbidden unless prior written permission is obtained from
Adobe Systems Incorporated.

----------------------------------------------------------------------------]]--

	-- RxLua
local Rx = require 'Rx'

	-- WF.core.os
local KSPathUtils = require 'KSPathUtils'
local WFFileUtils = require 'WFFileUtils'

	-- WF.core.utils
local KSTableUtils = require 'KSTableUtils'
local WFArguments = require 'WFArguments'
local WFCrypto = require 'WFCrypto'
local WFConfiguration = require 'WFConfiguration'
local WFLogger = require 'WFLogger'

	-- WF.sqlite.database
local WFSQLiteDatabase = require 'WFSQLiteDatabase'
local WFSQLiteDatabaseSchema = require 'WFSQLiteDatabaseSchema'
local WFSQLiteUtils = require 'WFSQLiteUtils'

--------------------------------------------------------------------------------

local KSTableUtils_setNested = assert( KSTableUtils.setNested )
	
--------------------------------------------------------------------------------

local check = assert( WFArguments.check, "Cannot find WFArguments.check" )

local md5 = assert( WFCrypto.MD5, "Cannot find MD5 digest" )

local logger = WFLogger( 'WFRxIndexStoreSchema' )

-- in FTS, all ASCII characters with codepoint  value less than 128 and not alpha numeric 
-- are used as separator and hence will not be present in the searchable string.
-- hHowerver, we can specify which all characters need not be considered as separators and which all
-- additional characters need to be considered as token.
-- what all characters are to be treated as part of token , not as separator is to specified as
-- tokenchars as below.

-- in example below (! * - ) are added as additional token chars, unless specified would have 
-- been treated as as separators.
-- tokenize=unicode61 \"tokenchars=!*-\"

-- "unicode61" is the tokenizer used here , which add some standard unicode separators also
-- as separators and as per the documentation , the default tokenizer ( "simple" ) does not have
-- an assured way to accept the tokenchars
-- https://jira.corp.adobe.com/browse/NMB-1280 related to the analysis of the bug
-- I am adding the support to make some commonly used special characters part of the token.
-- What all we need to allow as token characters and what all we need to use as the separators 
-- have to discussed and decided as part of the above mentioned bug resolution.

local _specialCharsTobeTreatedPartOfToken = ".-!|()&=\\/'"

--------------------------------------------------------------------------------

local kAllDocuments = "all"
local kCatalogDocuments = "catalog"
local kSpaceFeedDocuments = "spaceFeed"

--------------------------------------------------------------------------------

local kAllDocTypesAndSubtypes = {
	[ "*" ] = { "*" }
}

--------------------------------------------------------------------------------

local function isAllDocTypesAndSubtypes( typeSubtypeMap ) 

	return KSTableUtils.deepEquals( typeSubtypeMap, kAllDocTypesAndSubtypes )
	
end

--------------------------------------------------------------------------------

local function isDocIdPrimaryKey( createTableSql ) 

	-- simple check to determine if the docId is set as a primary key
	return ( not not createTableSql:find( "docId INTEGER PRIMARY KEY" ) or
			not not createTableSql:find( "docId UNIQUE" ) or 
			not not createTableSql:find( "docId PRIMARY KEY" ) )
	
end

--------------------------------------------------------------------------------

-- Note: this list contains far more docTypes than what is actually ever indexed in the indexStore. Still these are
-- the docTypes which we could encounter in an actual catalog and we might want to query for them one day.

local validDocTypesAndSubtypes = {
		catalog = { "lightroom", "X-lightroom", "profile", "pivot", "temp" }, --  X-lightroom is a documented doc type

		space = { "shared", "adhoc", "gallery", "review" },

		asset = { "image", "video", "creation", "creation_segment", "camera_profile",
			"lens_profile", "layout_segment", "deleted_image", "deleted_video",
			"search", "preset", "profile", "task", "person", "pivot", "stack", "model" }, -- model as subtype is tentative, not yet finalized

		[ "error" ] = { "Undefined", "UnknownError", "ResourceNotFoundError", "ResourceExistsError", "RevisionExistsError",
			"NotHeadRevisionPreconditionError", "InputValidationError", "PayloadValidationError",
			"PreconditionFailed", "InlineXmpDevelopSettingTooBigError", "MultipartCompleteError",
			"ParentSubtypeNotAllowed", "ResourceSubTypeNotAllowed", "AddAssetToAlbumError", "JoinIdAlreadyInUseError",
			"ResourceLockedError", "DocumentTooBigError", "PendingAccessRequestError", "NoAccessRequestError",
			"DuplicateAccessRequestError", "InvalidAccessRequestError", "InsufficientPermissionError", "SpaceUserExistsError",
			"AssetGoneError", "AssetNotRestored", "PurgeError", "RestoreAssetError", "RedisServerError", "IMSAdminProfileRequestError",
			"PipelineError", "StorageExceededError", "IneligibleForEnterpriseError", "UserRemovedError", "TrialOrSubscriptionExpiredError",
			"EntitlementExpiredError", "AuthorizationError", "PermissionError", "TooManyResourcesError", "TooManyAssetsError",
			"TooManyRevisionsError", "SharedCloudError" },

		warning = { "RevisionCompressed" },

		space_album = {},

		album = { "adhoc", "cluster", "collection", "collection_set", "export", "project", "project_set", "topic", "smart" },

		album_asset = {},

		stack_asset = {},

		account = {},

		comment = { "user" },

		favorite = { "user" },

		feedbackuser = {},

		spaceinvite = { "request", "single_use" },

		space_user = {},

		version = { "edit", "version" }, -- subtype version: undocumented, but found in one catalog, early version implementation?

		watermark = { "text" },
		
		-- client specific docTypes
		[ "gen-album" ] = {}, -- LriOS generic album type

		profile = {}, -- Found in old QE LrDesktop catalog. Subtype "camera" is written in _localOnly. Probably early camera profile implementation or bug.

}

-------------------------------------------------------------------------------

local function isInList( t, list )
	
	for _, v in ipairs( list ) do
		if v == t then 
			return true
		end
	end

	return false

end

-------------------------------------------------------------------------------
	
local function validateAndAddTypes( types, typeSubtypeMap )

	for _, typeAndSubtype in ipairs( types ) do

		if type( typeAndSubtype ) ~= 'table' or #typeAndSubtype ~= 2 then
			error( string.format( "WFRxIndexStoreSchemaVersionList.addTypes(): invalid types entry in schema version list '%s'", typeAndSubtype) )
		end

		local docType, docSubtype = unpack( typeAndSubtype )

		if docType == "*" then
			
			-- A "*" docType indicates that all types/subtypes are relevant.
			
			typeSubtypeMap = KSTableUtils.shallowcopy( kAllDocTypesAndSubtypes )
			
			break

		elseif validDocTypesAndSubtypes[ docType ] == nil then
		
			error( string.format( "WFRxIndexStoreSchemaVersionList.addTypes(): invalid doc type '%s'", docType) )

		else
			
			local validSubtypes = validDocTypesAndSubtypes[ docType ]

			if docSubtype == "NULL" and #validSubtypes > 0 then
				error( string.format( "WFRxIndexStoreSchemaVersionList.addTypes(): docs of type '%s' require a subtype", docType) )
			end
			
			if docSubtype == "*" and #validSubtypes == 0 then
				error( string.format( "WFRxIndexStoreSchemaVersionList.addTypes(): docs of type '%s' have no subtypes", docType) )
			end

			if docSubtype ~= "NULL" and docSubtype ~= "*" and not isInList( docSubtype, validSubtypes ) then
				error( string.format( "WFRxIndexStoreSchemaVersionList.addTypes(): invalid subtype '%s' to doc type: '%s'", 
					docSubtype, docType) )
			end

			if docSubtype == "*" then
				typeSubtypeMap[ docType ] = { "*" } -- it's OK to overwrite an already existing subtype list
			elseif docSubtype == "NULL" then
				typeSubtypeMap[ docType ] = { "NULL" } -- it's OK to overwrite an already existing subtype list
			else
				local alreadyAddedSubtypes = typeSubtypeMap[ docType ] or {}
				if not isInList( "*", alreadyAddedSubtypes ) and not isInList( docSubtype, alreadyAddedSubtypes )  then
					alreadyAddedSubtypes[ #alreadyAddedSubtypes + 1 ] = docSubtype
					typeSubtypeMap[ docType ] = alreadyAddedSubtypes
				end
			end
		
		end
		
	end

	return typeSubtypeMap

end

--============================================================================--
--[[!
	\class WFRxIndexStoreSchema

	\brief Represents a database schema used by an <b>index definition file</b>
		for an index store.

	An <b>index definition file</b> is application-specific code that describes
	what data should be extracted (indexed) from a document store
	(\c WFRxDocStore) to support the queries it will need to perform.

	You can start a new index store at the end of the index definition file
	using <code>WFRxIndexStoreImp.start( schema, ... )</code>.

	Create tables and indexes to support the queries you will be performing
	against this data using the \c WFRxIndexStore interface.
	Use <code>:addTable</code> to add any number of tables, and
	<code>:addIndex</code> to add any number of indexes.
	Be aware of performance hits for more tables and indexes.

	A typical index definition file looks like this:

\code
local WFRxIndexStoreSchema = require 'WFRxIndexStoreSchema'
local indexStoreSchema = WFRxIndexStoreSchema()

indexStoreSchema:addTable {

	sql = "CREATE TABLE coreInfo( aspectRatio, displayDate )",
	map = function( docValue, emit )

		-- WARNING: This version is grossly simplified and omits error checking.
		-- See OzCatalogIndexDefinitions.lua for a better example.

		if docValue._docType == 'asset' then
			emit( docValue.width / docValue.height,
				  docValue.captureDate:sub( 1, 10 ) )
		end
	end,
}

indexStoreSchema:addIndex
	"CREATE INDEX coreInfo_displayDate ON coreInfo( displayDate )"

require 'WFRxIndexStoreImp'.start( schema, ... )
\endcode

]]--

-- @mark WFRxIndexStoreSchema class

local function mapFunctionHash( mapFn )
	local chunkid = debug.getinfo(mapFn).source
	if chunkid and chunkid ~= "=?" and debug.applychunkid then
		-- This is wrong on so many levels, but allows source or stripped/compiled Lua
		-- schema mappers to produce the same (fragile) hash so switching between them
		-- does not trigger full index store rebuilds.
		--
		-- try to pick the last path component of the chunk's name to match the string
		-- used by luac to generate the hash
		chunkid = chunkid:match("[^/\\]*$") or chunkid
		-- if we loaded this chunk from source, reload it and stamp it with the chunk id
		-- it would get in a release build so we get the same hash in debug or release
		--
		-- note this is done on a copy of the function and not the original because
		-- stripping the debug info and stamping a chunk id into the chunk will make
		-- debugging within that map function misbehave
		mapFn = loadstring(string.dump(mapFn, true), "=?")
		debug.applychunkid(mapFn, chunkid)
	end

	return md5.digest( string.dump( mapFn ) .. '+2012-06-25' )
end


local WFRxIndexStoreSchema = {

	_schema = false,
		-- (WFSQLiteDatabaseSchema)

	_tableInfo = false,
		-- (table)
			-- key (string): table name as declared in SQL
			-- value (table): args to :addTable method, plus several other
				-- bits of information we gather/generate along the way
	_manualTableInfo = false,
		-- (table)
			-- key (string): manual table name as declared in SQL (manual table: no map funtion)
			-- value (table): args to :addTable method, plus several other
				-- bits of information we gather/generate along the way

	_lcTableNames = false,
		-- (table)
			-- key (string): table name as declared in SQL (normalized to LC)
			-- value: true

	_niceToHaveIndices = false,
		-- (table)
			-- key (string) : table name as declared in SQL or fields
			-- value (table): a set of columns that are nice to have (often the
			-- empty set but never nil)

	_tablesThatHaveOneRowPerDocument = false,
		-- (table)
			-- key (string) : table name as declared in SQL or fields
			-- value (boolean): true if this table allows only one row per doc

	_tablesThatUseLocalIds = false,
		-- (table)
			-- key (string) : table name as declared in SQL or fields
			-- value (boolean): true if this table uses local id

	_fullTextSearchTables = false,
		-- (table)
			-- key (string) : table name as declared in SQL or fields
			-- value (boolean): true if this table is a fts4 virtual table
			
	_tablesThatShouldIncludeDocumentsWithoutSpaceIds = false,
		-- (table)
			-- key (string) : table name as declared in SQL or fields
			-- value (boolean): true if this table should process catalog-based documents.
			
	_tablesThatShouldIncludeDocumentsWithSpaceIds = false,
		-- (table)
			-- key (string) : table name as declared in SQL or fields
			-- value (boolean): true if this table should process space-based documents.
			
	_tablesThatNeedAggressiveUpdates = false,
		-- (table)
			-- key (string) : table name as declared in SQL or fields
			-- value (boolean): true if this table is a requested to be kept updated always

	_dbInitRoutines = false,
		-- (array) list of functions that do something with the db after being created.
		
	_tablesResetAtInit = false,
		-- (table) for diagnostic log purposes
			-- key (string) : table name of table that was reset on launch due to changes
			-- value (string): 'schema', 'map', 'force', 'versionList', 'new', 'other' (reason for reset)
				-- 'schema': Resetting because the table's schema has changed.
				-- 'map': Resetting because the map function has changed.
				-- 'force': Resetting by force.
				-- 'versionList': Resetting via the schema-version-list.
				-- 'new': Resetting because the table is new.
				-- 'other': Unknown cause.

	_typeSubtypeTuplesByTableName = false,
		-- (table)
			-- key (string): A table name as declared in SQL or fields
			-- value (table): A table of type/subtype tuples.
				
	_useSchemaVersionList = false,
		 -- (boolean) True if the schema relies on an explicit Version List for reindexing.

	class = {
	
		mapFunctionHashForUnitTest = function( _, fn )
			return mapFunctionHash( fn )
		end,
		
		sourceAllDocuments = kAllDocuments,
		sourceCatalogDocuments = kCatalogDocuments,
		sourceSpaceFeedDocuments = kSpaceFeedDocuments
		
	}

}

--------------------------------------------------------------------------------
--[[!
	\brief Creates a new index-store schema instance.
	
	\param optSQLitePageSize (optional number) Optional SQLite Page Size
	\param useSchemaVersionList (optional boolean) Defaults to true. If false,
		the associated index store will use hash-based forced reindexing.
		
	\return (<code>WFRxIndexStoreSchema</code>)
]]--

function WFRxIndexStoreSchema:__init( optSQLitePageSize, useSchemaVersionList )

	check( 'WFRxIndexStoreSchema', '?number', '?boolean' )

	self._schema = WFSQLiteDatabaseSchema( '1.0' )
		:useWAL()
		:allowMultipleConnections()
		:setSynchronous( 1 )
			-- TODO [scouten 2014-02-08]: Consider using synchronous = 0 here?
			-- Everything in this DB is rebuildable and it would be a big perf boost.
			-- Concern is we might not detect corruption at the right time and
			-- index store might fail in unexpected ways. Suggestion: Switch sync
			-- to 0 early in 5.5 branch and observe.

	local pageSize = optSQLitePageSize or WFConfiguration.IndexStore.SQLITE_PAGE_SIZE

	if pageSize then
	
		self._schema = self._schema
			:setPageSize( pageSize )
	
	end

	-- Create a table to track status of indexes.

	self._schema:addTable [[CREATE TABLE indexStatus(
					indexTableName UNIQUE,
					needsUpdateFrom,
					sqlSchemaHash,
					mapFunctionHash)]]
		:onConflict( 'replace' )

		-- indexTableName = name of table created via :addTable below
		-- needsUpdateFrom = doc change counter from WFRxDocStore
			-- when this table was last known to be up to date
		-- sqlSchemaHash = MD5 hash of the SQL schema string used to
			-- create this table
		-- mapFunctionHash = MD5 hash of the map function used to
			-- extract data into this table

		-- Note that there is a special entry titled "__invalidate"
		-- which is used to keep the docChangeCounter value for when
		-- invalidation noticees were sent to all other index tables.

	-- Create a table to store index-store local data.

	self._schema:addTable [[CREATE TABLE localState( name UNIQUE NOT NULL, value )]]
	self._schema:addTable [[CREATE TEMPORARY TABLE TEMP_ids( docId )]]

	self._tableInfo = {}
	self._manualTableInfo = {}
	self._lcTableNames = {}
	self._niceToHaveIndices = {}
	self._tablesThatHaveOneRowPerDocument = {}
	self._tablesThatUseLocalIds = {}
	self._dbInitRoutines = {}
	self._fullTextSearchTables = {}
	self._tablesThatNeedAggressiveUpdates = {}
	self._tablesThatShouldIncludeDocumentsWithoutSpaceIds = {}
	self._tablesThatShouldIncludeDocumentsWithSpaceIds = {}
	self._typeSubtypeTuplesByTableName = {}
	
	self._useSchemaVersionList = useSchemaVersionList ~= false
	
	self._tablesResetAtInit = {}
		
	-- Rani : why are are creating this here ?
	self:addTable( {
		sql = [[CREATE TEMPORARY TABLE TEMP_search( docId, sortOrder )]],
		isManualTable = true
	} )
	
	self:addIndex( [[CREATE INDEX TEMP_search_docId ON TEMP_search (docId)]] )
	self:addIndex( [[CREATE INDEX TEMP_search_sortOrder ON TEMP_search (sortOrder)]] )
	
end

--------------------------------------------------------------------------------
--[[!
	\brief Adds an index table to this schema.

	\param args (table) description of the index table
		- \c sql: (string) a <code>CREATE TABLE</code> SQL statement
		- \c map: (function) a function taking <code>( docValue, emit )</code>
			that maps a single document value to zero or more rows in this table.
			Rows are created by calling the \c emit function with parameters in
			the sequence specified in the \c sql statement. This field is not required
			for manual tables.
		- \c relevantTypeSubtypeTuples (table) A complete list of document type/subtype
			tuples relevant to the index store map function. Only documents matching
			these tuples will be indexed for this table. This field is not required
			for manual tables.
		- \c doNotReindexForChangedMapFunction (optional boolean) do not reindex, even if 
			the map function has changed.
		- \c isManualTable (optional boolean)
		- \c useLocalId (optional boolean)
		- \c hasOneRowPerDocument (optional boolean) 
		- \c includeDocumentsFromSource (optional, string): A string indicating
			which documents to index: "all", "catalog" or "spaceFeed". The
			default is "all".
		
	The index store implementation records the values of both
	\c sql and \c map. (Actually, it stores an MD5 hash of both.) If either
	\c sql or \c map is changed, the existing contents of the index is
	discarded and the table's contents is regenerated automatically.

	These table names are reserved for use by \c WFRxIndexStore:

		- \c indexStatus
		- \c localState
		- Any table name with the prefix \c WFIndexStore_
		- Any table name with the prefix \c WFRxIndexStore_

	Remember that SQL table names are case-insensitive, so any case variation
	of those names are also reserved.
]]--

function WFRxIndexStoreSchema:addTable( args )

	check( 'WFRxIndexStoreSchema', 'table' )

	if type( args.sql ) ~= 'string' then
		if type( args.fields ) ~= 'table' or type( args.tableName ) ~= 'string' then
			error( "WFRxIndexStoreSchema:addTable: args.sql must be a string or ( args.fields must be an array and args.tableName must be a string )", 2 )
		end
	end

	local isFts = args.isFullTextSearch
	if not args.fields then
		assert( not isFts, 'Full Text Search must have fields and tableName' )
		if not string.lower( args.sql ):match( "^%s*create%s+table%s+" ) and not string.lower( args.sql ):match( "^%s*create%s+temporary%s+table%s+" ) then
			error( "WFRxIndexStoreSchema:addTable: args.sql must be a CREATE (TEMPORARY) TABLE statement", 2 )
		end
	end
	
	local isManualTable = args.isManualTable
	
	if not isManualTable then
	
		if type( args.relevantTypeSubtypeTuples ) ~= 'table' then
			error( string.format( "WFRxIndexStoreSchema:addTable: relevantTypeSubtypeTuples is required, and must be a table of type/subtype tuples: %s", args.sql ), 2 )
		else
			validateAndAddTypes( args.relevantTypeSubtypeTuples, {} )
		end

	end
	
	-- Default to NOT updating tables lazily as it can create large pauses, previously only mobile was behaving this way. (see commit f5b1dbbeaa78705f977bbe215c0ecbbe57ba3703).
	if args.updateLazily ~= true and not isManualTable then
		args.updateLazily = false
	end
	
	if not isManualTable and type( args.map ) ~= 'function' then
		error( "WFRxIndexStoreSchema:addTable: args.map must be a function", 2 )
	end

	local hasOneRowPerDocument = args.hasOneRowPerDocument
	local useLocalId = args.useLocalId

	local docIdDeclaration = "docId,"
	if hasOneRowPerDocument then
		if useLocalId then
			docIdDeclaration = "docId INTEGER PRIMARY KEY,"
		elseif isFts then
			docIdDeclaration = nil
				-- use fts implicit docid column
		else
			docIdDeclaration = "docId UNIQUE,"
		end
	elseif isFts then
		error( "Full Text Search tables must be marked as hasOneRowPerDocument" )
	end

	-- Insert standard fields.

	local sql
	local fields = args.fields
	local tableName = args.tableName
	local nColumns

	if isManualTable then
	
		sql = args.sql
		local columns = sql:match( "%((.*)%)" )
		
		local _, nCommas = columns:replace( ',', ',' )
		
		nColumns = nCommas + 1
		
	else
	
		if not fields then
		
			sql = args.sql:replace( "(", "( " .. docIdDeclaration, 1 )

			local columns = sql:match( "%((.*)%)" )
			local _, nCommas = columns:replace( ',', ',' )
	
			nColumns = nCommas
				-- +1 because there is one more column than comma,
				-- but -1 because we don't count our docId column here
		else
			if isFts then
				
				sql = string.format( "CREATE VIRTUAL TABLE %s USING fts4( %s, tokenize=unicode61 \"tokenchars=%s\"  )",
					tableName, table.concat( fields, ", " ), _specialCharsTobeTreatedPartOfToken )
						-- WE could specify content="" to reduce size overhead, but doing so
						-- causes SQLite to generate the error "Error: SQL logic error or missing database"
						-- for a variety of common queries!
				nColumns = #fields
			else
				sql = string.format( "CREATE TABLE %s ( %s%s )",
					tableName, docIdDeclaration, table.concat( fields, ", " ) )
				nColumns = #fields
			end
		end
	
	end
		
	args.nColumns = nColumns

	if not tableName then
		tableName = WFSQLiteUtils.tableNameFromCreateTableStatement( sql )
	end

	if not tableName then
		error( "WFRxIndexStoreSchema:addTable: could not find table name in SQL " .. sql, 2 )
	end

	local updateLazily = args.updateLazily  -- dont wait for the table to be used. Do indexing ASAP
	self._fullTextSearchTables[ tableName ] = isFts
	
	if updateLazily == false then
		self._tablesThatNeedAggressiveUpdates[ tableName ] = updateLazily
	end

	local niceToHaveIndices = {}

	if args.niceToHave then

		assert( fields, "niceToHave fields must be declared via fields and tableName" )

		local columnNameIndices = {}

		for index, field in ipairs( fields ) do
			columnNameIndices[ field ] = index
		end

		for _, field in ipairs( args.niceToHave ) do
			niceToHaveIndices[ columnNameIndices[ field ] ] = true
		end

	end

	local lcTableName = string.lower( tableName )

	if lcTableName:match( "^wfindexstore_" )
		or lcTableName:match( "^wfrxindexstore_" )
		or lcTableName == 'indexstatus'
		or lcTableName == 'localstate'
	then
		error( string.format( "WFRxIndexStoreSchema:addTable: illegal table name: %s",
							  tableName ), 2 )
	end

	if self._lcTableNames[ lcTableName ] then
		error(
			string.format( "WFRxIndexStoreSchema:addTable: table named %q already exists", tableName ),
			2 )
	end

	args.sqlSchemaHash = md5.digest( sql )
	if args.map then
		args.mapFunctionHash = mapFunctionHash( args.map )
		
		-- [scouten 2012-06-25]: The string concat will force all index tables in
		-- all catalogs to rebuild because they might be missing data. (PR #3221655)

		-- [scouten 2015-06-02]: Since we're sharing this code with WFIndexStore,
		-- we need to leave this in place. :-/
	end
	
	if args.docValueFilter then
		args.docValueFilterHash = mapFunctionHash( args.docValueFilter )
	end
	
	args.indexInfo = {}
	
	if isManualTable then
		self._manualTableInfo[ tableName ] = args
	else
		self._tableInfo[ tableName ] = args
	end

	self._niceToHaveIndices[ tableName ] = niceToHaveIndices
	self._tablesThatHaveOneRowPerDocument[ tableName ] = hasOneRowPerDocument
	self._tablesThatUseLocalIds[ tableName ] = useLocalId
	self._lcTableNames[ lcTableName ] = true
	self._typeSubtypeTuplesByTableName[ tableName ] = args.relevantTypeSubtypeTuples
	
	local includeDocumentsFromSource = args.includeDocumentsFromSource or kAllDocuments
	
	if includeDocumentsFromSource ~= kCatalogDocuments and
		includeDocumentsFromSource ~= kSpaceFeedDocuments and
		includeDocumentsFromSource ~= kAllDocuments then
		
		-- default to kCatalogDocuments if the supplied source is invalid.
		
		includeDocumentsFromSource = kCatalogDocuments
		
	end
	
	local includeCatalogDocuments = 
			includeDocumentsFromSource == kCatalogDocuments or
			includeDocumentsFromSource == kAllDocuments
	
	local includeSpaceFeedDocuments =
			includeDocumentsFromSource == kSpaceFeedDocuments or
			includeDocumentsFromSource == kAllDocuments
		
	self._tablesThatShouldIncludeDocumentsWithoutSpaceIds[ tableName ] = includeCatalogDocuments
	
	self._tablesThatShouldIncludeDocumentsWithSpaceIds[ tableName ] = includeSpaceFeedDocuments
	
	if not isFts and not isManualTable and not isDocIdPrimaryKey( sql ) then
		self:addIndex( string.format(
			[[CREATE INDEX %s_docId ON %s( docId )]], tableName, tableName ) )
	end
	
	return self._schema:addTable( sql ):onConflict( 'replace' )
	
end

--------------------------------------------------------------------------------

function WFRxIndexStoreSchema:dropTable( tableName )

	check( 'WFRxIndexStoreSchema', 'string' )
	
	return self._schema:dropTable( tableName )
	
end

--------------------------------------------------------------------------------
--[[!
	\brief Adds an index to the schema.

	\param sql (string) A <code>CREATE INDEX</code> SQL statement.
		This index must refer to one of the tables created with
		<code>:addTable</code>.
]]--

function WFRxIndexStoreSchema:addIndex( sql )

	check( 'WFRxIndexStoreSchema', 'string' )
	
	local _, _, _, tableName = sql:find( "([%a_][%w_]*)%s+[Oo][Nn]%s+([%a_][%w_]*)" )

	if not tableName or #tableName < 1 then
		error( "WFRxIndexStoreSchema:addIndex: could not find table name in " .. sql, 5 )
	end
	
	local indexEntry = self._schema:addIndex( sql )
	
	if self._tableInfo[ tableName ] then
		self._tableInfo[ tableName ].indexInfo[ indexEntry._indexName ] = indexEntry._sql
	elseif self._manualTableInfo[ tableName ] then
		self._manualTableInfo[ tableName ].indexInfo[ indexEntry._indexName ] = indexEntry._sql
	else
		error( "WFRxIndexStoreSchema:addIndex: table in index statement not declared " .. tableName, 5 )
	end

	return indexEntry

end

--------------------------------------------------------------------------------
--[[!
	\brief Adds an init function.  This function is called whenever a db
		is created using this schema.

	\param fn (function) the function to run.  It will be given a single
		argument, the WFSQLiteDatbase.
]]--

function WFRxIndexStoreSchema:addDatabaseInitRoutine( fn )
	self._dbInitRoutines[ #self._dbInitRoutines + 1 ] = fn
end

--------------------------------------------------------------------------------
--[[!
	\brief Opens an existing database or creates a new one with this index
		store schema.

	\param path (string) The path at which to find or create the database.

	\return (\c WFSQLiteDatabase) The opened database object.
]]--

function WFRxIndexStoreSchema:openOrCreateDatabaseAtPath( path )

	check( 'WFRxIndexStoreSchema', 'string' )

	if #path > 0 and not path:match( '^:' ) 
		and not path:match( 'mode=memory' ) then
		WFFileUtils.mkdir( KSPathUtils.parent( path ) )
	end

	local db = WFSQLiteDatabase( path, self._schema )

	for _, initRoutine in ipairs( self._dbInitRoutines ) do
		initRoutine( db )
	end

	return db

end

--------------------------------------------------------------------------------
--[[!
	\brief Retrieves the set of table names declared in this schema.

	\return (table)
		- key (string): A table name as declared in SQL
		- value (boolean): \c true
]]--

function WFRxIndexStoreSchema:tableNames()

	check( 'WFRxIndexStoreSchema' )

	local result = {}

	for k in pairs( self._tableInfo ) do
		result[ k ] = true
	end

	return result

end

--------------------------------------------------------------------------------
--[[!
	\brief Retrieves the set of temporary table names declared in this schema.

	\return (table)
		- key (string): A temporary table name as declared in SQL
		- value (boolean): \c true
]]--

function WFRxIndexStoreSchema:manualTableNames()

	check( 'WFRxIndexStoreSchema' )

	local result = {}

	for k in pairs( self._manualTableInfo ) do
		result[ k ] = true
	end

	return result

end

--------------------------------------------------------------------------------
--[[!
	\brief Retrieves the set of SQL indices declared for this schema.

	\return (table)
		- key (string): A table name as declared in SQL
		- value (table): \c a list of SQL index names and their corresponding SQL strings
]]--

function WFRxIndexStoreSchema:indexInfos()

	check( 'WFRxIndexStoreSchema' )

	local result = {}
	
	for k, v in pairs( self._tableInfo ) do
		if v.indexInfo then
			result[ k ] = v.indexInfo
		end
	end

	return result

end

--------------------------------------------------------------------------------
--[[!
	\brief Retrieves an array of niceToHave column indices for each table
		declared in this schema.

	\return (table)
		- key (string): A table name as declared in SQL or via tableName
		- value (set): a table mapping updated column index to true if
		that column is niceToHave
]]--

function WFRxIndexStoreSchema:niceToHaveIndices()

	check( 'WFRxIndexStoreSchema' )

	return self._niceToHaveIndices

end

--------------------------------------------------------------------------------
--[[!
	\brief Retrieves the set of tables that indicate they have at most
		one row per document.

	\return (table)
		- key (string): A table name as declared in SQL or via tableName
		- value (boolean): true if this table only has at most one row per document
]]--

function WFRxIndexStoreSchema:tablesThatHaveOneRowPerDocument()

	check( 'WFRxIndexStoreSchema' )

	return self._tablesThatHaveOneRowPerDocument

end

--------------------------------------------------------------------------------
--[[!
	\brief Retrieves the set of tables that indicate they care about catalog
		based documents.

	\return (table)
		- key (string): A table name as declared in SQL or via tableName
		- value (boolean): true if this table should process catalog documents.
]]--

function WFRxIndexStoreSchema:tablesThatShouldIncludeDocumentsWithoutSpaceIds()

	check( 'WFRxIndexStoreSchema' )

	return self._tablesThatShouldIncludeDocumentsWithoutSpaceIds

end

--------------------------------------------------------------------------------
--[[!
	\brief Retrieves the set of tables that indicate they care about space-feed
		based documents.

	\return (table)
		- key (string): A table name as declared in SQL or via tableName
		- value (boolean): true if this table should process space-feed documents.
]]--

function WFRxIndexStoreSchema:tablesThatShouldIncludeDocumentsWithSpaceIds()

	check( 'WFRxIndexStoreSchema' )

	return self._tablesThatShouldIncludeDocumentsWithSpaceIds

end

--------------------------------------------------------------------------------
--[[!
	\brief Retrieves the set of tables that indicate they use document local ids
		instead of GUID-based docId.

	\return (table)
		- key (string): A table name as declared in SQL or via tableName
		- value (boolean): true if this table uses local ids
]]--

function WFRxIndexStoreSchema:tablesThatUseLocalIds()

	check( 'WFRxIndexStoreSchema' )

	return self._tablesThatUseLocalIds

end

--------------------------------------------------------------------------------
--[[!
	\brief Retrieves the set of tables that are full-text search virtual tables
		using the SQLite fts4 algorithm.

	\return (table)
		- key (string): A table name as declared in SQL or via tableName
		- value (boolean): true if this table uses local ids
]]--

function WFRxIndexStoreSchema:tablesThatUseFTS4()

	check( 'WFRxIndexStoreSchema' )

	return self._fullTextSearchTables

end


--------------------------------------------------------------------------------
--[[!
	\brief Retrieves the set of tables that needs to be always updated , even when 
	it was not used.
]]--

function WFRxIndexStoreSchema:tablesThatNeedAggressiveUpdates()

	check( 'WFRxIndexStoreSchema' )

	return self._tablesThatNeedAggressiveUpdates

end

--------------------------------------------------------------------------------
--[[!
	\brief Retrieves the set of tables that were reset at index store initialization.
	This will only be populated if resetTablesIfNecessary has been called already on the 
	index store schema.
]]--

function WFRxIndexStoreSchema:tablesResetAtInitialization()
	
	check( 'WFRxIndexStoreSchema' )
	
	return self._tablesResetAtInit
	
end

--------------------------------------------------------------------------------
--[[!
	\brief Retrieves the map function for each table declared in this schema.

	\return (table)
		- key (string): A table name as declared in SQL
		- value (function): The map function for this table
]]--

function WFRxIndexStoreSchema:mapFunctions()

	check( 'WFRxIndexStoreSchema' )

	local result = {}

	for k, tableInfo in pairs( self._tableInfo ) do
		result[ k ] = tableInfo.map
	end

	return result

end

--------------------------------------------------------------------------------
--[[!
	\brief Retrieves the filter function for each table declared in this schema.

	\return (table)
		- key (string): A table name as declared in SQL
		- value (function): The filter function for this table
]]--

function WFRxIndexStoreSchema:docValueFilters()

	check( 'WFRxIndexStoreSchema' )

	local result = {}
	
	local filterNothing = function() return true end
	
	for k, tableInfo in pairs( self._tableInfo ) do
		result[ k ] = tableInfo.docValueFilter or filterNothing
	end

	return result

end

--------------------------------------------------------------------------------

--[[!
	\brief Retrieves the relevant type/subtype map for each table declared in this schema.

	\return (table) Nested tables containing the relevant type/subtypes for each table in this schema
		-- key: (string) A table name as declared in SQL
		-- value: (table)
			-- key: (string) relevant type
			-- value: (table)
				-- key (string) relevant subtype
				-- value: (boolean) true
]]--

function WFRxIndexStoreSchema:relevantTypeAndSubtypesByTable()

	check( 'WFRxIndexStoreSchema' )

	local result = {}
	
	for tableName in pairs( self._tableInfo ) do
	
		local typeSubtypeTuples = self._typeSubtypeTuplesByTableName[ tableName ]
	
		for _, typeSubtypeTuple in pairs( typeSubtypeTuples or {} ) do
		
			local docType = typeSubtypeTuple[ 1 ]
			local docSubtype = typeSubtypeTuple[ 2 ]
			
			KSTableUtils_setNested( result, true, tableName, docType, docSubtype )
		
		end
		
	end
	
	return result

end

--------------------------------------------------------------------------------
--[[!
	\brief Retrieves the number of user-defined columns for each table declared
		in this schema.

	\return (table)
		- key (string): A table name as declared in SQL
		- value (number): The number of user-defined columns in this table
]]--

function WFRxIndexStoreSchema:nColumnsPerTable()

	check( 'WFRxIndexStoreSchema' )

	local result = {}

	for k, tableInfo in pairs( self._tableInfo ) do
		result[ k ] = assert( tableInfo.nColumns )
	end

	return result

end

--------------------------------------------------------------------------------
--[[!
	\brief Retrieves the number of columns for each manual table declared
		in this schema.

	\return (table)
		- key (string): A table name as declared in SQL
		- value (number): The number of columns in this table
]]--

function WFRxIndexStoreSchema:nColumnsPerManualTable()

	check( 'WFRxIndexStoreSchema' )

	local result = {}

	for k, tableInfo in pairs( self._manualTableInfo ) do
		result[ k ] = assert( tableInfo.nColumns )
	end

	return result

end

--------------------------------------------------------------------------------
--[[!
	\brief Removes data from any tables where schema or map function have changed
		since the previous app session.

	\param db (\c WFSQLiteDatabase) The database to be updated.
	\param forceRefresh (optional, boolean) When \c true, reset all existing tables.

]]--

function WFRxIndexStoreSchema:resetTablesIfNecessary( db, forceRefresh )

	check( 'WFRxIndexStoreSchema', 'WFSQLiteDatabase', '?boolean' )

	-- Find out the status of existing tables in the database.

	local indexStatusRows = db:gatherArrayOfArray
		[[SELECT indexTableName, sqlSchemaHash, mapFunctionHash, needsUpdateFrom
			FROM indexStatus]]

	local indexStatusByTable = {}

	for i = 1, #indexStatusRows do

		local row = indexStatusRows[ i ]

		indexStatusByTable[ row[ 1 ] ] = {
			sqlSchemaHash = row[ 2 ],
			mapFunctionHash = row[ 3 ],
			needsUpdateFrom = row[ 4 ],
		}

	end

	-- Reset any tables where we have a potential mismatch.

	db:withTransactionDo( function()

		for tableName, tableInfo in pairs( self._tableInfo ) do

			local indexStatus = indexStatusByTable[ tableName ]

			local clearTable

			if indexStatus then

				if indexStatus.sqlSchemaHash ~= tableInfo.sqlSchemaHash
					or ( indexStatus.mapFunctionHash ~= tableInfo.mapFunctionHash
						and not tableInfo.doNotReindexForChangedMapFunction ) 
					or forceRefresh
				then

					db:execOnce(
						[[DELETE FROM indexStatus WHERE indexTableName == ?]],
								tableName )

					clearTable = true

					if indexStatus.sqlSchemaHash ~= tableInfo.sqlSchemaHash then	
						self._tablesResetAtInit[ tableName ] = 'schema'
					elseif indexStatus.mapFunctionHash ~= tableInfo.mapFunctionHash then
						self._tablesResetAtInit[ tableName ] = 'map'
					elseif forceRefresh then
						self._tablesResetAtInit[ tableName ] = 'force'
					else
						self._tablesResetAtInit[ tableName ] = 'other'
					end

				end

			else

				clearTable = true
					
				self._tablesResetAtInit[ tableName ] = 'new'
	
			end

			if clearTable then

				db:execOnce( [[INSERT INTO indexStatus VALUES( ?, ?, ?, ? )]],
							 tableName, 1, tableInfo.sqlSchemaHash,
							 tableInfo.mapFunctionHash )

				db:execOnce( string.format( [[DELETE FROM %s]], tableName ) )

				if indexStatus then
					indexStatus.needsUpdateFrom = 0
				end

			end

			tableInfo.lastUpdated = indexStatus and ( indexStatus.needsUpdateFrom - 1 ) or 0

		end

	end )

end

--------------------------------------------------------------------------------
--[[!
	\brief Resets the indexStatus entry for all tables that needs to be reindexed,
		and clears the contents of the tables that need to be reindexed.
		The needsUpdateFrom column within the indexStatus table is used in 
		version-schema-based reindexing as a way to measure reindexing progress,
		and as a way to resume reindexing if it's interrupted.
		
	\param db (\c WFSQLiteDatabase) The database to be updated.
	\param toBeReindexedTables (table) A table of index store tables that need
		reindexing according to the schema version list.

]]--

function WFRxIndexStoreSchema:resetTablesBySchemaVersionList( db, toBeReindexedTables )

	check( 'WFRxIndexStoreSchema', 'WFSQLiteDatabase', 'table' )

	-- Find out the status of existing tables in the database.

	local indexStatusRows = db:gatherArrayOfArray
		[[SELECT indexTableName, sqlSchemaHash, mapFunctionHash, needsUpdateFrom
			FROM indexStatus]]

	local indexStatusByTable = {}

	for i = 1, #indexStatusRows do

		local row = indexStatusRows[ i ]

		indexStatusByTable[ row[ 1 ] ] = {
			sqlSchemaHash = row[ 2 ],
			mapFunctionHash = row[ 3 ],
			needsUpdateFrom = row[ 4 ],
		}

	end

	db:withTransactionDo( function()

		local tablesAddedToIndexStatus = {}

		for _, tableName in ipairs( toBeReindexedTables ) do

			local indexStatus = indexStatusByTable[ tableName ]

			if indexStatus then

				db:execOnce(
					[[DELETE FROM indexStatus WHERE indexTableName == ?]],
							tableName )

			end

			self._tablesResetAtInit[ tableName ] = 'versionList'

			local tableInfo = self._tableInfo[ tableName ]

			db:execOnce( [[INSERT INTO indexStatus VALUES( ?, ?, ?, ? )]],
						 tableName, 1, tableInfo.sqlSchemaHash,
						 tableInfo.mapFunctionHash )

			tablesAddedToIndexStatus[ tableName ] = true

			db:execOnce( string.format( [[DELETE FROM %s]], tableName ) )

			if indexStatus then
				indexStatus.needsUpdateFrom = 0
			end

			tableInfo.lastUpdated = indexStatus and ( indexStatus.needsUpdateFrom - 1 ) or 0

		end

		for tableName, tableInfo in pairs( self._tableInfo ) do

			-- Iterate through all of the tables in the index store schema;
			-- if there are any new tables in the schema that are not tracked
			-- in the indexStatus table, then insert entries for those
			-- new tables. If there are changes to existing table schemas,
			-- insert entries as well.

			if not tablesAddedToIndexStatus[ tableName ] then

				local indexStatus = indexStatusByTable[ tableName ]

				local clearTable = false

				if not indexStatus then

					clearTable = true

					self._tablesResetAtInit[ tableName ] = 'new'
						-- This causes any new table to be force-reindexed on startup.
						-- This may not be ideal during feature development, but it's
						-- necessary for users upgrading from old releases of Lightroom. Older
						-- releases will not contain some of the more recent index store tables
						-- which were introduced prior to the 2021 MAX release (and therefore
						-- absent from the schema version list).

				elseif indexStatus.sqlSchemaHash ~= tableInfo.sqlSchemaHash then

					clearTable = true

					logger:warnf( "Index Store table %s needs to be reindexed due to a SQL Schema Change, but is not present in WFRxIndexStoreSchemaVersionList", tableName )

					db:execOnce(
						[[DELETE FROM indexStatus WHERE indexTableName == ?]],
								tableName )

					self._tablesResetAtInit[ tableName ] = 'schema'
						-- This causes any table with a SQLite schema change to be force-reindexed
						-- on startup. We don't currently support ALTER TABLE mechanics on index
						-- store tables, so any change to the schema results in the entire table
						-- being dropped; hence why reindexing is necessary.

				end

				if clearTable then

					db:execOnce( [[INSERT INTO indexStatus VALUES( ?, ?, ?, ? )]],
									 tableName, 1, tableInfo.sqlSchemaHash,
									 tableInfo.mapFunctionHash )

					db:execOnce( string.format( [[DELETE FROM %s]], tableName ) )

				end

			end

		end

	end )

end

--------------------------------------------------------------------------------

--[[!
	\brief Returns a boolean indicating if the explicit schema version list defined
		within WFRxIndexStoreSchemaVersionList should be used for forced reindexing.
		
	\return (<code>boolean</code>)
]]--

function WFRxIndexStoreSchema:useSchemaVersionList()
	return self._useSchemaVersionList
end

--------------------------------------------------------------------------------

--[[!
	\brief For the given list of tables, retrieves all relevant doc types/subtypes.
		This function is only used by unit tests.
		
	\return (<code>Observable< table ></code>) A map with doc types as 
		keys and a list of subtypes as values.
]]--

function WFRxIndexStoreSchema:getTypeSubtypeMapForTables( tableNames )

	local aggregatedTypeSubtypeMap = {}

	for _, tableName in ipairs( tableNames ) do
	
		local typeSubtypeTuples = self._typeSubtypeTuplesByTableName[ tableName ]
		
		if type( typeSubtypeTuples ) == 'table' then
			aggregatedTypeSubtypeMap = validateAndAddTypes( typeSubtypeTuples, aggregatedTypeSubtypeMap )
		end
		
		if isAllDocTypesAndSubtypes( aggregatedTypeSubtypeMap ) then
			break
		end
		
	end

	return aggregatedTypeSubtypeMap

end

--------------------------------------------------------------------------------

function WFRxIndexStoreSchema:schema()
	return self._schema
end

--------------------------------------------------------------------------------

return Rx.Class 'WFRxIndexStoreSchema'( WFRxIndexStoreSchema )
