--[[----------------------------------------------------------------------------

ADOBE CONFIDENTIAL
------------------
Copyright 2018 Adobe Systems Incorporated
All Rights Reserved.

NOTICE: All information contained herein is, and remain the property of
Adobe Systems Incorporated and its suppliers, if any. The intellectual and
technical concepts contained herein are proprietary to Adobe Systems
Incorporated and its suppliers and may be covered by U.S. and Foreign
Patents, patents in process, and are protected by trade secret or copyright
law.  Dissemination of this information or reproduction of this material is
strictly forbidden unless prior written permission is obtained from
Adobe Systems Incorporated.

----------------------------------------------------------------------------]]--

	-- Rx
local Rx = require "Rx"

	-- Utilities
local KSPathUtils = require "KSPathUtils"
local WFFileUtils = require "WFFileUtils"
local KSDate = require "KSDate"
local KSData = require "KSData"
local KSTableUtils = require "KSTableUtils"
local KSDataCache = require "KSDataCache"

local WFLogger = require "WFLogger"

	-- WF.oz_client.managed_catalog
local WFCatUser = require 'WFCatUser'
local WFCatSpec = require 'WFCatSpec'
local WFCatalog = require 'WFCatalog'
local WFCrypto = require "WFCrypto"
local WFCatFileConductorConstants = require 'WFCatFileConductorConstants'

local WFAnalytics = require "WFAnalytics"

	-- WF.oz_client.sync_agent
local OzCatalogUrls = require 'OzCatalogUrls'
local OzFilePaths = require "OzFilePaths"
local OzCatalogUploadStatus = require "OzCatalogUploadStatus"

local LriMigrateCacheModel = {}

local derefOpt = assert( KSTableUtils.derefOpt )

local logger = WFLogger "LrmMigrateThumbsModel"
	:setLevel 'info'


--------------------------------------------------------------------------------

-- page through cacheReferences.


local function enumerateCacheReferences( indexStore )

	local sql = [[SELECT DISTINCT docId FROM cacheReferences
			WHERE docId > ?
			ORDER BY docId ASC LIMIT 50 ]]

    local lastDocIdSubject = Rx.ReplaySubject(1)
    lastDocIdSubject:onNext( '' )

    return lastDocIdSubject:flatMap(function( lastDocId, lastRenditionName )

        return indexStore:queryOnce { sql = sql, lastDocId, lastRenditionName }
			:tap( function( rows )

				if not next( rows) then
					lastDocIdSubject:onCompleted()
				else

					local lastDocId = rows[ #rows ][ 1 ]

					lastDocIdSubject:onNext( lastDocId )
				end

			end )

    end )

end

--------------------------------------------------------------------------------

local function writeCacheRendition( strippedDoc, renditionName, data, catalogId )

	if not strippedDoc then
		return nil
	end

	local contentId = WFCatFileConductorConstants.renditionContentId( strippedDoc )
	local docId = strippedDoc._id

	if not contentId or not docId then
		logger:errorf( "thumbnail: no contentId found for %s", docId or "unknown")
		return nil
	end

	local path = OzFilePaths.getRelativeBlobPath( docId, catalogId )

	if renditionName == 'thumbnail2x' then
		path = KSPathUtils.child( path, 'th_' .. contentId .. '.jpg' )
	elseif renditionName == 'preview' then
		path = KSPathUtils.child( path, 'pv_' .. contentId .. '.jpg' )
	else
		logger:errorf( "can not save rendition for %s", path, docId )
		return nil
	end

	local absPath = KSPathUtils.child( OzFilePaths.getCatalogBasePath( catalogId ), path )

	if not pcall( KSData.saveToFile, data, absPath ) then
		logger:errorf( "could not save %s for %s", renditionName, docId )
	end

	return path, contentId

end

--------------------------------------------------------------------------------

local function migrateProxyOrMaster( strippedDoc, renditionName, filePath, catalogId )

	local contentId = derefOpt(strippedDoc, 'importSource', 'sha256' )
	local captureDate = strippedDoc.captureDate or KSDate.timeInIso8601Format()
	local docId = strippedDoc._id

	filePath = OzCatalogUploadStatus.checkAndCorrectLocalPath( filePath )

	local attributes = WFFileUtils.fileAttributes( filePath )

	if attributes and next( attributes ) then

		if renditionName == 'latestOriginal' then
			contentId = WFCrypto.digestFile( filePath, 'SHA256' )
		end

		local fileName = KSPathUtils.leafName( filePath )
		local originalsPath = KSPathUtils.child( OzFilePaths.getCatalogBasePath( catalogId ), 'Originals' )
		local destPath = OzFilePaths.originalsFolderForDate( originalsPath, captureDate )

		local ext = fileName:match( "%.[^%.]+$" )
		local baseName = fileName:sub( 1, #fileName - #ext )

		WFFileUtils.mkdir( destPath )

		if renditionName == 'proxy' then

			-- drop _proxy
			local proxyExt = baseName:sub(-#'_proxy')
			if proxyExt == '_proxy' then
				baseName = baseName:sub( 1, #baseName - #'_proxy' )
			end

			destPath = KSPathUtils.child( destPath, baseName .. '_' .. docId .. '_proxy.dng' )
		else
			destPath = KSPathUtils.child( destPath, baseName .. '_' .. docId .. ext )
		end

		if destPath ~= filePath then
		 	WFFileUtils.move( filePath, destPath )
		end

		destPath = OzFilePaths.makeRelativeFrom( destPath, 'Originals' )

		return {
			path = destPath,
			fileSize = attributes.fileSize,
			contentId = contentId
		}
	end

	return {}

end

--------------------------------------------------------------------------------

local function migrateXmpDevelop( strippedDoc, filePath, catalogId, catalogPath )

	local contentId = WFCrypto.digestFile( filePath, 'SHA256' )

	local dest = OzFilePaths.getRelativeBlobPath( strippedDoc._id, catalogId )
	local absDest = KSPathUtils.child( catalogPath, dest )

	local destName = string.format( 'xmp_%s.rdf', contentId )
	absDest = KSPathUtils.child( absDest, destName )
	dest = KSPathUtils.child( dest, destName )

	local xmpAttributes = WFFileUtils.fileAttributes( filePath )

	pcall( WFFileUtils.deleteFile, absDest )

	if pcall( WFFileUtils.copy, filePath, absDest ) then

		return {
			path = dest,
			contentId = contentId,
			fileSize = xmpAttributes.fileSize
		}

	end

	return nil

end
--------------------------------------------------------------------------------

local referenceNames = { 'thumbnail2x', 'preview', 'proxy', 'original', 'latestOriginal', 'xmp_develop' }

local function migrateDocument( strippedDoc, catalogId, catalogPath, documentsDir  )

	local overlay = {}
	local localOnly = overlay._localOnly or {}

	for _, renditionName in ipairs( referenceNames ) do

		local renditionInfo = {}

		if renditionName == 'xmp_develop' then
			local filePath = derefOpt( strippedDoc, '_localOnly', 'files','xmp_develop', 'path' )

			if filePath then
				filePath = KSPathUtils.child( documentsDir, filePath )
				if WFFileUtils.exists( filePath ) then
					renditionInfo = migrateXmpDevelop( strippedDoc, filePath, catalogId, catalogPath )
				end
			end
		else

			local localInfo = derefOpt( strippedDoc, '_localOnly', renditionName )
			if localInfo then

				if renditionName == 'thumbnail2x' or renditionName == 'preview' then

					renditionInfo = {
						path = localInfo.path,
						fileSize = localInfo.fileSize,
						contentId = localInfo.contentId,
						referenceHeight = localInfo.referenceHeight,
						referenceWidth = localInfo.referenceWidth
					}

				elseif renditionName == 'proxy' or renditionName == 'original' or renditionName == 'latestOriginal' then

					local filePath

					if renditionName == 'original' then
						filePath = derefOpt( strippedDoc, '_localOnly', 'originalFilePath' )
					else
						filePath = localInfo.filePath
					end

					if filePath then
						renditionInfo = migrateProxyOrMaster( strippedDoc, renditionName, filePath, catalogId )
					end

					-- rewrite rendition names and original upload fixups
					if renditionName == 'original' then

						renditionInfo.ozUrl = derefOpt( strippedDoc, '_localOnly', 'original', 'ozUrl' )
						renditionInfo.uploaded = derefOpt( strippedDoc, '_localOnly', 'original', 'uploaded' )
						renditionInfo.chunkedUpload = derefOpt( strippedDoc, '_localOnly', 'original', 'chunkedUpload' )
						renditionInfo.chunkedUploadRestartCount = derefOpt( strippedDoc, '_localOnly', 'original', 'chunkedUploadRestartCount' )

						renditionName = 'master'
					elseif renditionName == 'latestOriginal' then
						renditionName = 'latestMaster'
					end
				end
			end
		end

		if renditionInfo and next( renditionInfo ) then
			KSTableUtils.setNested( localOnly, renditionInfo, 'files', renditionName )
		end

	end

	localOnly.originalFilePath = { _delete = true }
	localOnly.original = { _delete = true }
	localOnly.latestOriginal = { _delete = true }
	localOnly.proxy = { _delete = true }
	localOnly.preview = { _delete = true }
	localOnly.thumbnail2x= { _delete = true }

	overlay._localOnly = localOnly
	overlay._id = strippedDoc._id

	return overlay

end

--------------------------------------------------------------------------------

function LriMigrateCacheModel.makeModel(rootCatalogPath, catalogId, userId)

	local userInfoCachePath = KSPathUtils.child( KSPathUtils.getStandardFilePath 'caches', 'Adobe' )
	WFFileUtils.mkdir( userInfoCachePath )
	userInfoCachePath = KSPathUtils.child( userInfoCachePath, 'user' )
	WFFileUtils.mkdir( userInfoCachePath )

	local catalogPath = OzFilePaths.getCatalogBasePath( catalogId )

	local ozCatDir = KSPathUtils.child( catalogPath, "Catalog.ozcat" )

	local cachePath = KSPathUtils.child( ozCatDir, "previewCache" )

	local serverBaseUrl = OzCatalogUrls.baseServerUrl
	local user = WFCatUser( serverBaseUrl, userId, userInfoCachePath )
	local catSpec = WFCatSpec( user, catalogId )
	local catalogOptions = {
		indexSpecScript = 'LriMigrateCacheIndexDefinitions.lua'
	}

	local documentsDir = KSPathUtils.getStandardFilePath 'carouselDocuments'

	local catalog = WFCatalog( catSpec, rootCatalogPath, catalogOptions )

	local dataCache

	if WFFileUtils.exists( cachePath ) then
		dataCache = KSDataCache( cachePath )
	end

	-- We need all fields that are required for the contentId
	local fieldSpec = {
		_id = true,
		userUpdated = true,
		captureDate = true,
		develop = {
			xmpCameraRaw = true,
			userOrientation = true
		},
		importSource = {
			sha256 = true
		},
		_localOnly = true
	}

	local renditionNames = { 'thumbnail2x', 'preview' }

	local progress = Rx.ReplaySubject( 1 )

	local count = catalog:indexStore():queryOnce { sql = 'SELECT COUNT (DISTINCT docId) FROM cacheReferences' }
		:map( function( rows )
			local nDocuments = rows[ 1 ][ 1 ]

			progress:onNext( { progress = { processedCount = 0, totalCount = nDocuments } } )

			return nDocuments
		end )
		:shareReplay( 1 )

	local migratePreviews = count
		:flatMap( function( totalCount )

			local processedCount = 0

			return enumerateCacheReferences( catalog:indexStore() )
				:takeWhile( function( rows )
					return next( rows )
				end )
				:flatMap( function( rows )

					local docIds = {}

					for _, row in ipairs( rows ) do
						docIds[ #docIds + 1 ] = row[ 1 ]
					end

					return catalog:docStore():bulkDocValue( docIds, fieldSpec )
						:flatMap( function( strippedDocs )

							return Rx.Observable.from( strippedDocs )
								:flatMap( function( strippedDoc )

									-- collect data in the cache
									local namesAndKeys = {}

									for _, renditionName in ipairs( renditionNames ) do

										local renditionInfo = derefOpt( strippedDoc, '_localOnly', renditionName )
										if renditionInfo and renditionInfo.md5 and dataCache then
											namesAndKeys[ #namesAndKeys + 1 ] = {
												renditionName, renditionInfo.md5
											}
										end
									end

									if #namesAndKeys > 0 then

										return Rx.Observable.from( namesAndKeys )
											:flatMap( function( nameAndKey )

												local renditionName, key = unpack( nameAndKey )

												return dataCache:get( key )
													:catch( function( errMsg )

														WFAnalytics.trackEvent {
															['event.appWorkflow'] = 'APP_UPGRADE',
															['event.type'] = 'error',
															['content.id'] = strippedDoc._id,
															['content.type'] = renditionName,
															['event.subtype'] = 'migratecache-cachelookup',
															['event.error_desc'] = errMsg,
														}

														return Rx.Observable.just( strippedDoc )

													end )
													:map( function( data )

														strippedDoc._localOnly[ renditionName ].fileSize = KSData.length( data )

														local path, contentId = writeCacheRendition( strippedDoc, renditionName, data, catalogId )

														-- side effect - update strippedDoc

														strippedDoc._localOnly[ renditionName ].path = path
														strippedDoc._localOnly[ renditionName ].contentId = contentId

														return strippedDoc
													end )

											end )
											:bufferWithCount( #namesAndKeys )
											:map( function()
												return strippedDoc
											end )

									else
										return Rx.Observable.just( strippedDoc )
									end

								end )

							:bufferWithCount( #strippedDocs )
						end )


				end )
				:flatMap( function( strippedDocs )

					local overlays = {}

					for _, strippedDoc in ipairs( strippedDocs ) do
						local overlay = migrateDocument( strippedDoc, catalogId, catalogPath, documentsDir )
						overlays[ #overlays + 1 ] = overlay
					end

					if not next( overlays ) then
						return Rx.Observable.just( {} )
					end

					return catalog:docStore():bulkApplyOverlay( overlays )

				end )
				:map( function( newRevIds )

					-- update progress
					processedCount = processedCount + #newRevIds
					progress:onNext( { progress = { processedCount = processedCount, totalCount = totalCount } } )

					return #newRevIds
				end )
				:sum() -- sums until completed
				:defaultIfEmpty(0)
				:catch( function( errorMsg )

					WFAnalytics.trackEvent {
						['event.appWorkflow'] = 'APP_UPGRADE',
						['event.type'] = 'error',
						['event.subtype'] = 'migratecache',
						['event.error_desc'] = errorMsg,
					}

					return Rx.Observable.just( processedCount )
				end )
				:flatMap( function( count )

					progress:onNext( { progress = { processedCount = totalCount, totalCount = totalCount } } )

					progress:onCompleted()

					logger:infof( "%d assets migrated", count )

					if dataCache then
						return catalog:docStore():halt()
							:concat( catalog:indexStore():halt() )
							:concat( dataCache:haltAndDelete()
								:ignoreElements() )
							:concat( Rx.Observable.just( count ) )
					else
						return catalog:docStore():halt()
							:concat( catalog:indexStore():halt() )
							:concat( Rx.Observable.just( count ) )
					end

				end )
				:map( function( count )

					catalog:close()
					user:close()

					-- nuke the preview cache
					WFFileUtils.deleteDirectory( ozCatDir )

					-- nuke the old xmp_develop folder
					WFFileUtils.deleteDirectory( KSPathUtils.child( catalogPath, 'xmp_develop' ) )

					-- nuke the old Originals
					WFFileUtils.deleteDirectory( KSPathUtils.child( documentsDir, 'Originals' ) )

					progress:onCompleted()

					return { result = count }

				end )

		end )

	return {
		value = progress
			:merge( migratePreviews )
	}

end

--------------------------------------------------------------------------------

return LriMigrateCacheModel
