--[[----------------------------------------------------------------------------

ADOBE CONFIDENTIAL
------------------
Copyright 2022 Adobe Inc.
All Rights Reserved.

NOTICE: All information contained herein is, and remain the property of
Adobe Systems Incorporated and its suppliers, if any. The intellectual and
technical concepts contained herein are proprietary to Adobe Systems
Incorporated and its suppliers and may be covered by U.S. and Foreign
Patents, patents in process, and are protected by trade secret or copyright
law.  Dissemination of this information or reproduction of this material is
strictly forbidden unless prior written permission is obtained from
Adobe Systems Incorporated.

----------------------------------------------------------------------------]]--

--[[

This file contains the migrator for Android that mainly moves the renditions from the previewCache
to the file system.

It is based upon the LriMigrateCacheModel that was used on iOS for the BlobProvider migration.

The most important changes since then are:
- cloud  trash
- versions

The subtypes 'deleted_image'/'deleted_video' are contained in the cacheReferences table,
so they'll be covered by the migration.

Versions will be migrated when they are available, through the reference from the asset itself.

]]--

	-- Rx
local Rx = require "Rx"

	-- Utilities
local KSPathUtils = require "KSPathUtils"
local WFFileUtils = require "WFFileUtils"
local KSData = require "KSData"
local KSTableUtils = require "KSTableUtils"
local KSDataCache = require "KSDataCache"

local WFLogger = require "WFLogger"

	-- WF.oz_client.managed_catalog
local WFCatUser = require 'WFCatUser'
local WFCatSpec = require 'WFCatSpec'
local WFCatalog = require 'WFCatalog'
local WFCrypto = require "WFCrypto"
local WFPlatformInfo = require "WFPlatformInfo"
local WFProgressReporter = require "WFProgressReporter"

local WFAnalytics = require "WFAnalytics"

	-- WF.oz_client.sync_agent
local OzFilePaths = require "OzFilePaths"
local OzCatalogUploadStatus = require "OzCatalogUploadStatus"

local LraMigrateCacheModel = {}

local WFCatFileConductorConstants = require "WFCatFileConductorConstants"

local derefOpt = assert( KSTableUtils.derefOpt )

-- TODO [immisch 2022-03-22]: turn off logging before production
local logger = WFLogger "LraMigrateCacheModel"
	:setLevel 'info'

--------------------------------------------------------------------------------

-- create a table from renditionName to the layout table in WFCatFileConductorConstants

local renditionNameToLayout = {}

for linkKey, layoutInfo in pairs( WFCatFileConductorConstants.layoutConventions ) do

	if type(layoutInfo ) == 'table' then
		-- put the linkKey into the layout info too
		layoutInfo.linkKey = linkKey
		renditionNameToLayout[ layoutInfo.shortName ] = layoutInfo
	end

end

--------------------------------------------------------------------------------

-- page through cacheReferences.

local function enumerateCacheReferences( indexStore )

	local sql = [[SELECT DISTINCT docId FROM cacheReferences
			WHERE docId > ?
			ORDER BY docId ASC LIMIT 50 ]]

    local lastDocIdSubject = Rx.ReplaySubject(1)
    lastDocIdSubject:onNext( '' )

    return lastDocIdSubject:flatMap(function( lastDocId )

        return indexStore:queryOnce { sql = sql, quiesceUntilUpToDate = true, singleColumnQuery = true,	lastDocId }
			:tap( function( rows )

				if not next( rows) then
					lastDocIdSubject:onCompleted()
				else

					local lastDocId = rows[ #rows ]

					lastDocIdSubject:onNext( lastDocId )
				end

			end )

    end )

end

--------------------------------------------------------------------------------

local function writeCacheRendition( strippedDoc, renditionName, data, wfCatalog )

	if not strippedDoc then
		return nil
	end

    local docId = strippedDoc._id
    local layoutInfo = renditionNameToLayout[ renditionName ]

	if not layoutInfo or not docId then
		logger:errorf( "%s: no layoutInfo found for %s", renditionName, docId or "unknown")

		WFAnalytics.trackEvent {
			['event.appWorkflow'] = 'APP_UPGRADE',
			['event.type'] = 'error',
			['event.subtype'] = 'migrate-rendition',
			['event.docId'] = strippedDoc._id,
			['event.error_desc'] = string.format( 'no layoutInfo found for %s', renditionName ),
		}

		return nil

	end

    local contentId = layoutInfo:contentIdFn( strippedDoc )
    local relPath = layoutInfo:relPathFn( strippedDoc, contentId )

	local absPath = KSPathUtils.child( WFCatFileConductorConstants.layoutConventions.basePathFn( wfCatalog ), relPath )

	WFFileUtils.mkdir( KSPathUtils.parent( absPath ) )

	if not pcall( KSData.saveToFile, data, absPath ) then

		WFAnalytics.trackEvent {
			['event.appWorkflow'] = 'APP_UPGRADE',
			['event.type'] = 'error',
			['event.subtype'] = 'migrate-rendition',
			['event.docId'] = strippedDoc._id,
			['event.error_desc'] = string.format( 'could not write file %s', absPath ),
		}

		return nil
	end

	return relPath, contentId

end

--------------------------------------------------------------------------------

local function migrateProxyOrMaster( strippedDoc, renditionName, filePath, catalog )

    local docId = strippedDoc._id
    local layoutInfo = renditionNameToLayout[ renditionName ]

	if not layoutInfo or not docId then
		logger:errorf( "%s: no layoutInfo found for %s", renditionName, docId or "unknown")
		return nil
	end

    local contentId = layoutInfo:contentIdFn( strippedDoc )
    local relPath = layoutInfo:relPathFn( strippedDoc, contentId )

	local absPath = KSPathUtils.child( WFCatFileConductorConstants.layoutConventions.basePathFn( catalog ), relPath )

	filePath = OzCatalogUploadStatus.checkAndCorrectLocalPath( filePath )

	local attributes = WFFileUtils.fileAttributes( filePath )

	if not attributes or not next( attributes  ) then

		WFAnalytics.trackEvent {
			['event.appWorkflow'] = 'APP_UPGRADE',
			['event.type'] = 'error',
			['event.subtype'] = 'migrate-original',
			['event.docId'] = strippedDoc._id,
			['event.error_desc'] = string.format( '%s not found', filePath ),
		}

		return {}
	end

	if renditionName == 'latestOriginal' then
		contentId = WFCrypto.digestFile( filePath, 'SHA256' )
	end

	local fileName = KSPathUtils.leafName( filePath )

	local ext = fileName:match( "%.[^%.]+$" )
	local baseName = fileName:sub( 1, #fileName - #ext )

	WFFileUtils.mkdir( absPath )

	if renditionName == 'proxy' then

		-- drop _proxy
		local proxyExt = baseName:sub(-#'_proxy')
		if proxyExt == '_proxy' then
			baseName = baseName:sub( 1, #baseName - #'_proxy' )
		end

		absPath = KSPathUtils.child( absPath, baseName .. '_' .. docId .. '_proxy.dng' )
	else
		absPath = KSPathUtils.child( absPath, baseName .. '_' .. docId .. ext )
	end

	if absPath ~= filePath then
		WFFileUtils.move( filePath, absPath )
	end

	relPath = OzFilePaths.makeRelativeFrom( absPath, 'Originals' )

	return {
		relativePath = relPath,
		fileSize = attributes.fileSize,
		sha256 = contentId
	}

end

--------------------------------------------------------------------------------

local function migrateXmpDevelop( strippedDoc, filePath, catalogPath )

    local docId = strippedDoc._id
    local layoutInfo = renditionNameToLayout[ 'xmp_develop' ]

	if not layoutInfo or not docId then
		logger:errorf( "xmp_develop: no contentId found for %s", docId or "unknown")
		return nil
	end

    local contentId = layoutInfo:contentIdFn( strippedDoc )
    local relPath = layoutInfo:relPathFn( strippedDoc, contentId )

    local absPath = KSPathUtils.child( catalogPath, relPath )

    if absPath ~= filePath then

        WFFileUtils.mkdir( KSPathUtils.parent( absPath ) )

        if not pcall( WFFileUtils.move, filePath, absPath ) then
			WFAnalytics.trackEvent {
				['event.appWorkflow'] = 'APP_UPGRADE',
				['event.type'] = 'error',
				['event.subtype'] = 'migrate-xmp-develop',
				['event.docId'] = strippedDoc._id,
				['event.error_desc'] = string.format( 'could not move %s', relPath ),
			}
		end
    end

    local xmpAttributes = WFFileUtils.fileAttributes( filePath )

    return {
		path = { _delete = true },
        relativePath = relPath,
        sha256 = contentId,
        fileSize = xmpAttributes.fileSize
    }

end

--------------------------------------------------------------------------------

local referenceNames = { 'thumbnail2x', 'preview', 'proxy', 'original', 'latestOriginal', 'xmp_develop' }

local function migrateDocument( strippedDoc, wfCatalog, catalogPath, documentsDir  )

	local overlay = {
		_localOnly = {}
	}

	for _, renditionName in ipairs( referenceNames ) do

		local renditionInfo = {}

		if renditionName == 'xmp_develop' then
			local filePath = derefOpt( strippedDoc, '_localOnly', 'files', 'xmp_develop', 'path' )

			if filePath then
				filePath = KSPathUtils.child( documentsDir, filePath )
				if WFFileUtils.exists( filePath ) then
					renditionInfo = migrateXmpDevelop( strippedDoc, filePath, catalogPath )
				end
			end
		else

			local localInfo = derefOpt( strippedDoc, '_localOnly', renditionName )
			if localInfo then

				if renditionName == 'thumbnail2x' or renditionName == 'preview' then

					renditionInfo = {
						relativePath = localInfo.path,
						fileSize = localInfo.fileSize,
						contentId = localInfo.contentId,
					}

				elseif renditionName == 'proxy' or renditionName == 'original' or renditionName == 'latestOriginal' then

					local filePath

					if renditionName == 'original' then
						filePath = derefOpt( strippedDoc, '_localOnly', 'originalFilePath' )
					else
						filePath = localInfo.filePath
					end

					if filePath then

						renditionInfo = migrateProxyOrMaster( strippedDoc, renditionName, filePath, wfCatalog )
					end
				end
			end
		end

		local layoutInfo = renditionNameToLayout[ renditionName ]

		if renditionInfo and next( renditionInfo ) then
			KSTableUtils.setNested( overlay, renditionInfo, unpack( layoutInfo.localOnly ) )
		end

	end

	overlay._localOnly.originalFilePath = { _delete = true }
	overlay._localOnly.original = { _delete = true }
	overlay._localOnly.latestOriginal = { _delete = true }
	overlay._localOnly.proxy = { _delete = true }
	overlay._localOnly.preview = { _delete = true }
	overlay._localOnly.thumbnail2x= { _delete = true }

	overlay._id = strippedDoc._id

	return overlay

end

--------------------------------------------------------------------------------

function LraMigrateCacheModel.makeModel(catalogPath, catalogId, userId)

	local userInfoCachePath = KSPathUtils.child( KSPathUtils.getStandardFilePath 'caches', 'Adobe' )
	WFFileUtils.mkdir( userInfoCachePath )

	-- TODO [immisch 2022-03-15]: verify the SD card case
    local documentsDir = KSPathUtils.getStandardFilePath 'carouselDocuments'

	local renditionDir = KSPathUtils.child( catalogPath, "renditions" )

	local cachePath = KSPathUtils.child( renditionDir, "previewCache" )

	local user = WFCatUser( 'https://photos.adobe.io', userId, userInfoCachePath )
	local catSpec = WFCatSpec( user, catalogId )
	local catalogOptions = {
		indexSpecScript = 'LraMigrateCacheIndexDefinitions.lua'
	}

	local catalog = WFCatalog( catSpec, KSPathUtils.parent( catalogPath ), catalogOptions )

	local progressReporter = WFProgressReporter( 'migrate cacheReferences' )

	local dataCache

	if WFFileUtils.exists( cachePath ) then
		dataCache = KSDataCache( cachePath )
	end

	-- We need all fields that are required for the contentId
	local fieldSpec = {
		_id = true,
		userUpdated = true,
		captureDate = true,
		develop = {
			xmpCameraRaw = true,
			userOrientation = true
		},
		importSource = {
			sha256 = true
		},
		_localOnly = true
	}

	local renditionNames = { 'thumbnail2x', 'preview' }

	-- For the unit test(s), return the ids of the migrated documents
	local TEST_migratedDocs
	if WFPlatformInfo.isUnitTest() then
		TEST_migratedDocs = {}
	end

	local count = catalog:indexStore():queryOnce {
			sql = 'SELECT COUNT (DISTINCT docId) FROM cacheReferences',
			quiesceUntilUpToDate = true,
			singleColumnQuery = true,
		}
		:map( function( rows )
			local nDocuments = rows[ 1 ]

			progressReporter:setTotal( nDocuments )
			progressReporter:setProcessedCount( 0 )

			return nDocuments
		end )
		:shareReplay( 1 )

	local migratePreviews = count
		:flatMap( function( totalCount )

			local processedCount = 0

			return enumerateCacheReferences( catalog:indexStore() )
				:takeWhile( function( rows )
					return next( rows )
				end )
				:flatMap( function( rows )

					if WFPlatformInfo.isUnitTest() then
						for _, docId in ipairs( rows ) do
							TEST_migratedDocs[ #TEST_migratedDocs + 1 ] = docId
						end
					end

					-- rows is already a list of docIds
					return catalog:docStore():bulkDocValue( rows, fieldSpec )
						:flatMap( function( strippedDocs )

							return Rx.Observable.from( strippedDocs )
								:flatMap( function( strippedDoc )

									-- collect data in the cache
									local namesAndKeys = {}

									for _, renditionName in ipairs( renditionNames ) do

										local renditionInfo = derefOpt( strippedDoc, '_localOnly', renditionName )
										if renditionInfo and renditionInfo.md5 and dataCache then
											namesAndKeys[ #namesAndKeys + 1 ] = {
												renditionName, renditionInfo.md5
											}
										end
									end

									if #namesAndKeys > 0 then

										return Rx.Observable.from( namesAndKeys )
											:flatMap( function( nameAndKey )

												local renditionName, key = unpack( nameAndKey )

												return dataCache:get( key )
													:catch( function( errMsg )

														WFAnalytics.trackEvent {
															['event.appWorkflow'] = 'APP_UPGRADE',
															['event.type'] = 'error',
															['content.id'] = strippedDoc._id,
															['content.type'] = renditionName,
															['event.subtype'] = 'migratecache-cachelookup',
															['event.error_desc'] = errMsg,
														}

														return Rx.Observable.just( strippedDoc )

													end )
													:map( function( data )

														strippedDoc._localOnly[ renditionName ].fileSize = KSData.length( data )

														local path, contentId = writeCacheRendition( strippedDoc, renditionName, data, catalog )

														-- side effect - update strippedDoc with the old layout
														strippedDoc._localOnly[ renditionName ].path = path
	                                                    strippedDoc._localOnly[ renditionName ].contentId = contentId

														return strippedDoc
													end )

											end )
											:bufferWithCount( #namesAndKeys )
											:map( function()
												return strippedDoc
											end )

									else
										return Rx.Observable.just( strippedDoc )
									end

								end )

							:bufferWithCount( #strippedDocs )
						end )


				end )
				:flatMap( function( strippedDocs )

					local overlays = {}

					for _, strippedDoc in ipairs( strippedDocs ) do
						local overlay = migrateDocument( strippedDoc, catalog, catalogPath, documentsDir )
						overlays[ #overlays + 1 ] = overlay
					end

					if not next( overlays ) then
						return Rx.Observable.just( {} )
					end

					return catalog:docStore():bulkApplyOverlay( overlays )

				end )
				:map( function( newRevIds )

					-- update progress
					processedCount = processedCount + #newRevIds

					-- omit last event, it will be sent after the error handler
					if processedCount < totalCount then
						progressReporter:setProcessedCount( processedCount )
					end

					return #newRevIds
				end )
				:sum() -- sums until completed
				:defaultIfEmpty(0)
				:catch( function( errorMsg )

					WFAnalytics.trackEvent {
						['event.appWorkflow'] = 'APP_UPGRADE',
						['event.type'] = 'error',
						['event.subtype'] = 'migratecache',
						['event.error_desc'] = errorMsg,
					}

					return Rx.Observable.just( processedCount )
				end )
				:flatMap( function( count )

					progressReporter:setProcessedCount( processedCount )

					progressReporter:done()

					logger:infof( "%d assets migrated", count )

					if dataCache then
						return catalog:docStore():halt()
							:concat( catalog:indexStore():halt() )
							:concat( dataCache:haltAndDelete()
								:ignoreElements() )
							:concat( Rx.Observable.just( count ) )
					else
						return catalog:docStore():halt()
							:concat( catalog:indexStore():halt() )
							:concat( Rx.Observable.just( count ) )
					end

				end )
				:map( function( count )

					if dataCache then
						dataCache:close()
					end

					catalog:close()
					user:close()

					-- nuke the preview cache
					WFFileUtils.deleteDirectory( renditionDir )

					-- nuke the old Originals
					WFFileUtils.deleteDirectory( KSPathUtils.child( documentsDir, 'Originals' ) )

					progressReporter:done()

					if WFPlatformInfo.isUnitTest() then
						return {
							result = count,
							TEST_migratedDocs = TEST_migratedDocs
						}
					else
						return { result = count }
					end

				end )

		end )

	return {
		value = progressReporter:status()
			:map( function( progress )
				return {
					progress = progress
				}
			end )
			:merge( migratePreviews )
	}

end

--------------------------------------------------------------------------------

return LraMigrateCacheModel
