--[[----------------------------------------------------------------------------

WFCatDirectChunkedUploaderTest.lua

--------------------------------------------------------------------------------

ADOBE CONFIDENTIAL
------------------
Copyright 2021 Adobe Systems Incorporated
All Rights Reserved.

NOTICE: All information contained herein is, and remain the property of
Adobe Systems Incorporated and its suppliers, if any. The intellectual and
technical concepts contained herein are proprietary to Adobe Systems
Incorporated and its suppliers and may be covered by U.S. and Foreign
Patents, patents in process, and are protected by trade secret or copyright
law.  Dissemination of this information or reproduction of this material is
strictly forbidden unless prior written permission is obtained from
Adobe Systems Incorporated.

------------------------------------------------------------------------------]]

	-- WF.core.self_test
local WFRxTestCase = require 'WFRxTestCase'
local WFTestFileUtils = require 'WFTestFileUtils'

	-- WF.core.utils
local KSIdGenerator = require 'KSIdGenerator'
local KSTableUtils = require 'KSTableUtils'
local WFFileUtils = require 'WFFileUtils'
local WFCatAssetTestUtils = require 'WFCatAssetTestUtils'
local WFCrypto = require 'WFCrypto'

	-- WF.oz_client.managed_catalog
local WFCatOzTestSession = require 'WFCatOzTestSession'
local WFCatSpecMock = require 'WFCatSpecMock'
local WFCatDirectChunkedUploader = require 'WFCatDirectChunkedUploader'

	-- WF.web.utils
local WFURI = require 'WFURI'

--------------------------------------------------------------------------------

local derefOpt = KSTableUtils.derefOpt

--------------------------------------------------------------------------------

local OZ_CHUNK_FILE_SIZE_RECOMMENDED = 5 * 1024 * 1024

local userId = KSIdGenerator.uuidInLowercaseHex()
local catalogId = '98db98bfd554470dab2f862a4b853c40'

--============================================================================--

local WFCatDirectChunkedUploaderTest = {}

--------------------------------------------------------------------------------

local function createCatalogSpecForTest( rxTest, assetDoc, additionalResponses, optArgs )

	optArgs = optArgs or {}

	local oneChunk = optArgs.oneChunk
	local isBigFile2GB = optArgs.isBigFile2GB
	local previousPartialComplete = optArgs.previousPartialComplete
	local previousFullComplete = optArgs.previousFullComplete
	
	local expectedResponses = {}

	-- insert mainPartsRequest expected response
	
	local revisionIds = derefOpt( assetDoc, '_serverProvidedMetadata', 'revision_ids' )
	local latestRevisionId = revisionIds and revisionIds[ #revisionIds ]
	
	local mainPartsUrl = derefOpt( assetDoc, '_serverProvidedMetadata', 'links', '/rels/main_create/parts', 'href' )
	mainPartsUrl = mainPartsUrl:replace( '{revision_id}', latestRevisionId )

	local fileSize = derefOpt( assetDoc, 'importSource', 'fileSize' )

	local numParts = math.ceil( fileSize / OZ_CHUNK_FILE_SIZE_RECOMMENDED )
	
	if oneChunk then
		rxTest:expectEqual( ( numParts == 1 ), oneChunk )
	elseif not isBigFile2GB then
		rxTest:expectEqual( ( numParts == 6 ), not oneChunk )
	end
	
	local contentType = derefOpt( assetDoc, 'importSource', 'contentType' )
	
	mainPartsUrl = mainPartsUrl:replace( '{upload_status}', 'pending' )
	mainPartsUrl = mainPartsUrl:replace( '{file_size}', tostring( fileSize ) )
	mainPartsUrl = mainPartsUrl:replace( '{part_size}', tostring( OZ_CHUNK_FILE_SIZE_RECOMMENDED ) )
	mainPartsUrl = mainPartsUrl:replace( '{content_type}', WFURI.escape( contentType, 'http_query_key' ) )
	
	expectedResponses[ #expectedResponses + 1 ] = 
		{
		
			url = mainPartsUrl,
			
			response = [[{
			
				"base": "https://photos-stage.adobe.io/v2/",
				"part_size": 5242880,
				"file_size": <file_size>,
				"num_parts": <num_parts>,
				"expires": "2100-07-25T22:27:41.466149Z",
				"parts": [
					<parts>
				],
				"links": {
					"rels/complete_upload": {
						"href": "main/parts?file_size={file_size}"
					},
					"rels/abort_upload": {
						"href": "main/parts"
					}
				}
			}]],
			
			responseFilter = function( response )

				local parts = ""
				local numPendingParts = 0
				
				if not previousFullComplete then
				
					for i = 1, numParts do
					
						if oneChunk or not previousPartialComplete or i > 2 then
					
							if numPendingParts >= 1 then
								parts = parts .. ','
							end
							
							numPendingParts = numPendingParts + 1
							
							local partNum = i
							local byteStart = OZ_CHUNK_FILE_SIZE_RECOMMENDED * ( i - 1 )
							local byteEnd = OZ_CHUNK_FILE_SIZE_RECOMMENDED * i - 1
							
							local partStr = string.format( [[
									{
										"part_num": %d,
							 			"href": "xyz.s3-us-west-2.amazonaws.com/%d",
										"href_expires": "2100-07-25T22:27:41.466149Z",
										"upload_status": "pending",
										"byte_start": %d,
										"byte_end": %d
									}
								]], partNum, partNum, byteStart, byteEnd )
						
							parts = parts .. partStr
					
						end
						
					end
				
				end
				
				response = response:gsub( '<parts>', parts )
				response = response:gsub( '<file_size>', tostring( fileSize ) )
				response = response:gsub( '<num_parts>', tostring( numParts )  )
				
				return response

			end,
			
			headers = {
				status_code = '200',
			},
			
			options = {
				requestHeaders = {
					{
						header = 'X-Generate-Renditions',
						value = 'all'
					},
					{
						header = "Content-Type",
						value = contentType, 
					},
					{
						header = 'Accept',
						value = 'application/json'
					}
				},
				notifyWhenHttpRequestIsInProgress = true,
				requestCost = 1,
			}
		}
			
	-- insert each part expected response
	
	local function addOneDirectChunkExpectedResponse( partNum )
	
		expectedResponses[ #expectedResponses + 1 ] = 
		{
		
			url = string.format( "xyz.s3-us-west-2.amazonaws.com/%d", partNum ),
			
			data = string.format( "data" ),
			
			dataFilter = function( data ) return "data" end,
			
			headers = {
				status_code = '200',
			},
		}
	
	end
	
	if not previousFullComplete then
		
		for i = 1, numParts do
			if oneChunk or not previousPartialComplete or i > 2 then
				addOneDirectChunkExpectedResponse( i )
			end
		end
	
	end
	
	-- insert complete url expected response
	
	expectedResponses[ #expectedResponses + 1 ] = 
		{
		
			url = string.format( "https://photos-stage.adobe.io/v2/main/parts?file_size={file_size}", "" ),
			
			headers = {
				status_code = '202',
			},
			
			data = "",
			
			options = {
				requestMethod = "POST",
				
				requestHeaders = {
					{
						header = 'X-Generate-Renditions',
						value = 'all'
					},
					{
						header = 'Content-Type',
						value = contentType
					},
					{
						header = 'Accept',
						value = 'application/json'
					}
				},
				requestCost = 1,
				bypassRateLimiter = true
			}
		}

	-- Add extra expected responses
	
	expectedResponses = KSTableUtils.appendArray( expectedResponses, additionalResponses )

	-- create catalogSpec
	
	local catalogSpec = WFCatSpecMock( catalogId, userId, rxTest, expectedResponses )

	return catalogSpec

end

--------------------------------------------------------------------------------

-- From WFCatContentRangeUploaderTest

local function SeekWriteBigFile( f, offset )

	local maxInt32 = 2^31

	f:seek( 'set', math.min( offset, maxInt32-1 ) )
	f:write( 'A' )
	local remainingOffset = offset - maxInt32

	while remainingOffset > 0 do
		f:seek( 'end', math.min( remainingOffset, maxInt32-1 ) )
		f:write( 'A' )
		remainingOffset = remainingOffset - maxInt32
	end

end

--------------------------------------------------------------------------------

local function makeFileOfSize2GB( originFilePath, fileName )

	local MB = 1024*1024
	local fileSize = 2*1024*MB + 100

	local f = io.open( originFilePath, "w" )
	if f then
		SeekWriteBigFile( f, fileSize - 1 )
		f:close()
	end

end

--------------------------------------------------------------------------------

local function getFileInfo( rxTest, options )

	local oneChunk = options.oneChunk

	local tempTestFolder = WFTestFileUtils.createTestFolder( rxTest )
	-- 'IMG_9876.CR2' is 26MB
	local fileName = ( oneChunk ) and 'IMG_7763.JPG' or 'IMG_9876.CR2'

	local originalFilePath = WFCatAssetTestUtils.getTestDataOriginalFilePath( tempTestFolder, fileName )

	local isBigFile2GB = options.isBigFile2GB

	local sha256

	if isBigFile2GB then
		makeFileOfSize2GB( originalFilePath, fileName )
		sha256 = KSIdGenerator.uuidInLowercaseHex()
			-- FIXME [jandrew 2021-10-27]: We use a fake SHA256 for the 2GB
			-- file, because `WFCrypto.digestFile` times out. This may be
			-- the result of a bug in the WFCrypto.digestFile implementation.
	else
		sha256 = WFCrypto.digestFile( originalFilePath, 'SHA256' )
	end
	
	return fileName, originalFilePath, sha256
	
end

--------------------------------------------------------------------------------

local function createAssetDoc( rxTest, options )

	local assetId = KSIdGenerator.uuidInLowercaseHex()
	local revisionId = KSIdGenerator.uuidInLowercaseHex()
	
	local _, originalFilePath, sha256 = getFileInfo( rxTest, options )
	
	local assetDocValue = WFCatAssetTestUtils.createBaseAssetDocument( catalogId, assetId, revisionId, sha256, originalFilePath, userId )
	
	assetDocValue = WFCatAssetTestUtils.addOriginalPathToDocument( assetDocValue, sha256, originalFilePath )

	return assetDocValue, assetId, revisionId, originalFilePath
	
end

--------------------------------------------------------------------------------

local function createAssetDocWithOzMetadataUpdate( rxTest, options )

	local assetDocValue, assetId, revisionId, originalFilePath = createAssetDoc( rxTest, options )
	
	WFCatAssetTestUtils.addBasicOzMetadataUpdateForDirectUpload( assetDocValue, catalogId, assetId, revisionId )
	
	return assetDocValue, assetId, revisionId, originalFilePath

end 

--==============================================================================
-- @mark Test Happy path. Fresh, partial or fully uploaded scenarios
--==============================================================================

local function testMockedDirectUpload( rxTest, options )

	options = options or {}

	if options.isBigFile2GB then
		rxTest:setTimeLimit( 60 )
	else
		rxTest:setTimeLimit( 10 )
	end
	
	local assetDoc, assetId, revisionId, originalFilePath = createAssetDocWithOzMetadataUpdate( rxTest, options )
	
	-- setup catSpec with expected direct upload responses
	
	local catalogSpec = createCatalogSpecForTest( rxTest, assetDoc, {}, options )
	
	-- do directChunkedUpload
	
	local directChunkedUploader = WFCatDirectChunkedUploader( catalogSpec )
	
	local content_type = derefOpt( assetDoc, 'importSource', 'contentType' )
	
	local chunkedUploadOptions = {
		requestHeaders = {	
			{
				header = 'X-Generate-Renditions',
				value = 'all'
			},
			{
				header = 'Content-Type',
				value = content_type
			},
			{
				header = 'Accept',
				value = 'application/json'
			}
		},
		notifyWhenHttpRequestIsInProgress = true,
		requestCost = 2,
	}
	
	local mainPartsUrl = '/v2/catalogs/' .. catalogId .. '/assets/' .. assetId .. '/revisions/' .. revisionId .. '/main/parts?access=put&upload_status={upload_status}&file_size={file_size}&part_size={part_size}&content_type={content_type}'
	
	local gotHere = false

	local mockUploadHasStartedHandler = function()
		gotHere = true
	end

	rxTest:waitFor( directChunkedUploader:doDirectChunkedUpload( mainPartsUrl, originalFilePath, chunkedUploadOptions, mockUploadHasStartedHandler ) )

	rxTest:expectEqual( true, gotHere )

end

--------------------------------------------------------------------------------

function WFCatDirectChunkedUploaderTest.rxTestMockedDirectUploadOneChunk( rxTest )

	testMockedDirectUpload( rxTest, { oneChunk = true } )

end

--------------------------------------------------------------------------------

function WFCatDirectChunkedUploaderTest.rxTestMockedDirectUploadMultipleChunks( rxTest )

	testMockedDirectUpload( rxTest, { } )

end

--------------------------------------------------------------------------------

function WFCatDirectChunkedUploaderTest.rxTestMockedDirectUploadMultipleChunks_BigFile2GB( rxTest )

	testMockedDirectUpload( rxTest, { isBigFile2GB = true } )

end

--------------------------------------------------------------------------------

function WFCatDirectChunkedUploaderTest.rxTestMockedDirectUploadMultipleChunks_previousPartialComplete( rxTest )

	testMockedDirectUpload( rxTest, { previousPartialComplete = true } )

end

--------------------------------------------------------------------------------

function WFCatDirectChunkedUploaderTest.rxTestMockedDirectUploadOneChunk_previousFullComplete( rxTest )

	testMockedDirectUpload( rxTest, { oneChunk = true, previousFullComplete = true } )

end

--------------------------------------------------------------------------------

function WFCatDirectChunkedUploaderTest.rxTestMockedDirectUploadMultipleChunks_previousFullComplete( rxTest )

	testMockedDirectUpload( rxTest, { previousFullComplete = true } )

end

--------------------------------------------------------------------------------

local function manualDirectChunkedUpload( rxTest, catalogSpec, directChunkedUploader, mainPartsRequestUrl, filePath, chunkedUploadOptions, options )

	local oneChunk = options.oneChunk
	local previousPartialComplete = options.previousPartialComplete
	local previousFullComplete = options.previousFullComplete
	
	local fileSize = WFFileUtils.fileAttributes( filePath ).fileSize or 0
	
	if previousPartialComplete or previousFullComplete then
		
		local mainPartsResponseData
		
		rxTest:waitFor( 
			directChunkedUploader.private._getMainParts( directChunkedUploader, mainPartsRequestUrl, fileSize, chunkedUploadOptions )
				:map( function( responseData )
						mainPartsResponseData = responseData
	 			end ) )
		
		local parts = derefOpt( mainPartsResponseData, 'parts' )
		local numParts = derefOpt( mainPartsResponseData, 'num_parts' )
		
		rxTest:expectEqual( numParts, math.ceil( fileSize / OZ_CHUNK_FILE_SIZE_RECOMMENDED ) )
		rxTest:expectEqual( type( parts ), 'table' )
		rxTest:expectEqual( #parts, numParts )
		
		
		for i, part in ipairs( parts ) do
		
			if previousFullComplete or ( not oneChunk and previousPartialComplete and i <= 2 ) then
			
				local putChunkUrl = derefOpt( part, 'href' )
 				local startOffset = derefOpt( part, 'byte_start' )
 				local endOffset = derefOpt( part, 'byte_end' )
				
				local args = {}
			
				rxTest:waitFor( directChunkedUploader.private._putDirectChunk( directChunkedUploader, putChunkUrl,  filePath, args, startOffset, endOffset, fileSize ) )
			
			end
		
		end
		
		-- Check if pending parts number is correct
		
		local expectedRemainingPendingPartsNum = ( previousFullComplete and 0 ) 
														or ( oneChunk and 1 ) 
														or ( numParts - 2 )
		
		rxTest:expectSingleValue( 
			directChunkedUploader.private._getMainParts( directChunkedUploader, mainPartsRequestUrl, fileSize, chunkedUploadOptions )
				:map( function( pendingPartsData )
					local parts = derefOpt( pendingPartsData, 'parts' )
					
					if parts and type( parts ) == 'table' then
						return #parts == expectedRemainingPendingPartsNum
					end
					
					return false
				end ),
			true )
	
	end

end

--------------------------------------------------------------------------------

local function abortUploadedChunks( rxTest, catalogSpec, abortUrl, directChunkedUploader, mainPartsRequestUrl, filePath, chunkedUploadOptions )

	local fileSize = WFFileUtils.fileAttributes( filePath ).fileSize or 0

	local abortArgs = {
		requestHeaders = {
			{ header = 'Content-Type', value = 'application/json' }
		},
		requestMethod = 'DELETE',
		bypassRateLimiter = true
	}

	rxTest:waitFor( catalogSpec:get( abortUrl, abortArgs ) )
	
	-- Check if all chunks are pending again
		
	local expectedRemainingPendingPartsNum = math.ceil( fileSize / OZ_CHUNK_FILE_SIZE_RECOMMENDED )
	
	rxTest:expectSingleValue( 
		directChunkedUploader.private._getMainParts( directChunkedUploader, mainPartsRequestUrl, fileSize, chunkedUploadOptions )
			:map( function( pendingPartsData )
				local parts = derefOpt( pendingPartsData, 'parts' )
				
				if parts and type( parts ) == 'table' then
					return #parts == expectedRemainingPendingPartsNum
				end
				
				return false
			end ),
		true )

end

--------------------------------------------------------------------------------

local function testOzDirectUpload( rxTest, options )

	options = options or {}

	rxTest:setTimeLimit( 300 )
	rxTest:setRetryLimit( 3, 30 )
	
	local withProgress = options.withProgress
	
	-- setup testSession
	
	local testSession, catalog, catalogSpec, pusher, puller, _, _ = WFCatOzTestSession.setupOzTest( rxTest, nil, nil, nil, 'allow_direct_upload' )
	assert( testSession )

	local catalogId = catalog:ozCatalogId()
	
	-- Add asset and sync with Oz
	
	local assetDocValue, assetId, _, originalFilePath = createAssetDoc( rxTest, options )
	
	rxTest:waitFor( catalog:docStore():setDocValue( assetDocValue ) )
	
	puller:getUpdatesContinuously()
	
	local sendNUpdates = rxTest:observe( pusher:_DEBUG_sendNUpdates( 1 ) )
	sendNUpdates:expectCompletedWithinTime( 30 )
	sendNUpdates:dispose()
	
	local latestRevisionId
	
	local mainPartsTemplateUrlFromOz
	
	local observeDocOnOzWithMainPartsUrl = rxTest:observe( catalog:docStore():ongoingDocValue( assetId )
			:map( function( docValue )
				mainPartsTemplateUrlFromOz = derefOpt( docValue, '_serverProvidedMetadata', 'links', '/rels/main_create/parts', 'href' )
				
				local revisionIds = derefOpt( docValue, '_serverProvidedMetadata', 'revision_ids' )
				local latestRevisionIdInDoc = revisionIds and revisionIds[ #revisionIds ]
				
				latestRevisionId = latestRevisionIdInDoc
				
				return mainPartsTemplateUrlFromOz ~= nil and latestRevisionIdInDoc ~= nil
			end ) )
			
	observeDocOnOzWithMainPartsUrl:expectDataWithinTime( 30, true )
	observeDocOnOzWithMainPartsUrl:dispose()
	
	puller:stopContinuousUpdates()
	
	local stageBaseUrl = "https://storage.adbephotos-stage.com"
	local mainPartsUrl = string.format( '%s/v2/catalogs/%s/assets/%s/revisions/{revision_id}/main/parts', stageBaseUrl, catalogId, assetId )
	local mainPartsTemplateUrl = mainPartsUrl .. '?access=put&upload_status={upload_status}&file_size={file_size}&part_size={part_size}&content_type={content_type}'
	rxTest:expectEqual( mainPartsTemplateUrl, mainPartsTemplateUrlFromOz )
	
	local mainPartsRequestUrl = mainPartsTemplateUrlFromOz:replace( '{revision_id}', latestRevisionId )
	
	-- setup for previously parital or fully complete
	
	local directChunkedUploader = WFCatDirectChunkedUploader( catalogSpec )
	
	local contentType = derefOpt( assetDocValue, 'importSource', 'contentType' )
	
	local chunkedUploadOptions = {
		requestHeaders = {
			{
				header = 'X-Generate-Renditions',
				value = 'all'
			},
			{
				header = "Content-Type",
				value = contentType, 
			},
			{
				header = 'Accept',
				value = 'application/json'
			}
		},
	}
	
	manualDirectChunkedUpload( rxTest, catalogSpec, directChunkedUploader, mainPartsRequestUrl, originalFilePath, chunkedUploadOptions, options )
	
	-- mock expiry by abort uploaded chunks
	
	if options.mockExpiry then
		local abortUrl = mainPartsUrl:replace( '{revision_id}', latestRevisionId )
		abortUploadedChunks( rxTest, catalogSpec, abortUrl, directChunkedUploader, mainPartsRequestUrl, originalFilePath, chunkedUploadOptions )
	end
	
	-- do directChunkedUpload
	
	local gotProgress = false
	local requestInProgress = false

	local uploadHasStartedHandler
	
	if withProgress then
		chunkedUploadOptions.includeProgress = true
		chunkedUploadOptions.notifyWhenHttpRequestIsInProgress = true
		
		uploadHasStartedHandler = function()
			if not requestInProgress then
				requestInProgress = true
			end
		end
	end
	
	
	
	rxTest:waitFor( directChunkedUploader:doDirectChunkedUpload( mainPartsRequestUrl, originalFilePath, chunkedUploadOptions, uploadHasStartedHandler )
	
		:tap( function( _, _, progress )
		
			if progress then
				gotProgress = true
			end
			
		end ) )
	
	if withProgress then
		rxTest:expectEqual( gotProgress, true )
		rxTest:expectEqual( requestInProgress, true )
	end
	
	-- Check if binary is on Oz
	
	puller:getUpdatesContinuously()
	
	local observeBinaryOnOz = rxTest:observe( catalog:docStore():ongoingDocValue( assetId )
			:map( function( docValue )
				local isMasterOnOz = derefOpt( docValue, '_serverProvidedMetadata', 'links', '/rels/master', 'href' ) ~= nil
				
				return isMasterOnOz
			end ) )
			
	observeBinaryOnOz:expectDataWithinTime( 120, true )
	observeBinaryOnOz:dispose()
	
	puller:stopContinuousUpdates()

end


--------------------------------------------------------------------------------

function WFCatDirectChunkedUploaderTest.rxTestOzDirectUploadOneChunk_testNightly( rxTest )

	testOzDirectUpload( rxTest, { oneChunk = true } )

end

--------------------------------------------------------------------------------

function WFCatDirectChunkedUploaderTest.rxTestOzDirectUploadMultipleChunks_testNightly( rxTest )

	testOzDirectUpload( rxTest, {} )

end

--------------------------------------------------------------------------------

function WFCatDirectChunkedUploaderTest.rxTestOzDirectUploadMultipleChunks_previousPartialComplete_testNightly( rxTest )

	testOzDirectUpload( rxTest, { previousPartialComplete = true } )

end

--------------------------------------------------------------------------------

function WFCatDirectChunkedUploaderTest.rxTestOzDirectUploadOneChunk_previousFullComplete_testNightly( rxTest )

	testOzDirectUpload( rxTest, { oneChunk = true, previousFullComplete = true } )

end

--------------------------------------------------------------------------------

function WFCatDirectChunkedUploaderTest.rxTestOzDirectUploadMultipleChunks_previousFullComplete_testNightly( rxTest )

	testOzDirectUpload( rxTest, { previousFullComplete = true } )

end

--------------------------------------------------------------------------------

function WFCatDirectChunkedUploaderTest.rxTestOzDirectUploadOneChunk_withProgress_testNightly( rxTest )

	testOzDirectUpload( rxTest, { oneChunk = true, withProgress = true } )

end

--------------------------------------------------------------------------------

function WFCatDirectChunkedUploaderTest.rxTestOzDirectUploadChunksExpiry_oneChunk_testNightly( rxTest )

	testOzDirectUpload( rxTest, { oneChunk = true, previousFullComplete = true, mockExpiry = true } )

end

--------------------------------------------------------------------------------

function WFCatDirectChunkedUploaderTest.rxTestOzDirectUploadChunksExpiry_multipleChunks_testNightly( rxTest )

	testOzDirectUpload( rxTest, { previousPartialComplete = true, mockExpiry = true } )

end

--------------------------------------------------------------------------------

-- SPAM [jandrew 2021-09-01]: WFCatDirectChunkedUploader progress event support is
-- covered more by WFCatBinaryUploaderTest.rxTestUploadAdd_WithProgress_testNightly

--------------------------------------------------------------------------------

return WFRxTestCase.wrapSuite( WFCatDirectChunkedUploaderTest )
