--[[----------------------------------------------------------------------------

KSStringUtils.lua

--------------------------------------------------------------------------------

ADOBE CONFIDENTIAL
------------------
Copyright 2011 Adobe Systems Incorporated
All Rights Reserved.

NOTICE: All information contained herein is, and remain the property of
Adobe Systems Incorporated and its suppliers, if any. The intellectual and
technical concepts contained herein are proprietary to Adobe Systems
Incorporated and its suppliers and may be covered by U.S. and Foreign
Patents, patents in process, and are protected by trade secret or copyright
law.  Dissemination of this information or reproduction of this material is
strictly forbidden unless prior written permission is obtained from
Adobe Systems Incorporated.

------------------------------------------------------------------------------]]

local KSStringUtilsImp = require 'KSStringUtilsImp'

	-- WF.core.lua
local KSData = require 'KSData'
local WFRatio = require 'WFRatio'

	-- WF.core.string
local WFZString = require 'WFZString'

	-- WF.core.utils
local WFArguments = require 'WFArguments'
local WFPlatformInfo = require 'WFPlatformInfo'

local bit = require 'bit'

--------------------------------------------------------------------------------

local ZSTR = assert( WFZString.fromZString )

local string_byte = assert( string.byte )
local string_len = assert( string.len )
local string_find = assert( string.find )
local string_gfind = assert( string.gfind )
local string_sub = assert( string.sub )
local math_floor = math.floor
local math_pow = math.pow
local math_huge = math.huge
local log10 = math.log10
local abs = math.abs
local table_insert = table.insert
local table_concat = table.concat
local byte = string.byte
local string_char = assert( string.char )
local string_reverse = assert( string.reverse )

--------------------------------------------------------------------------------

local kXorKey = 0x39

--============================================================================--

-- @mark KSStringUtils namespace

--[[!
	\class KSStringUtils
	\brief This namespace provides a cross-platform string-manipulation utility.
--]]


--============================================================================--

local KSStringUtils = {}

--------------------------------------------------------------------------------

-- GRUNGE [mhamburg 2011-03-02]: This could probably use string.gmatch and
-- simplify the logic a bit but that would mess with the split pattern.

--[[!
	 Separates a string into table entries using a given separator character.

	 \param str The string.
	 \param inSplitPattern	A string containing the separator character(s).
	 \param outResults An optional table in which to return the result.

	 \return The output table with the new entries appended,
	 	or a new table containing the substrings.
--]]

function KSStringUtils.split( --[[string]] str, --[[string]] inSplitPattern, --[[table]] outResults )
	if not outResults then
		outResults = { }
	end
	local theStart = 1
	local theSplitStart, theSplitEnd = string_find( str, inSplitPattern, theStart )
	while theSplitStart do
		outResults[ #outResults + 1 ] = string_sub( str, theStart, theSplitStart - 1 )
		theStart = theSplitEnd + 1
		theSplitStart, theSplitEnd = string_find( str, inSplitPattern, theStart )
	end
	outResults[ #outResults + 1 ] = string_sub( str, theStart )
	return outResults
end

--------------------------------------------------------------------------------
-- This function does an Unicode string compare, not a localized string compare.
--[[!
	 Compares two strings (using a Unicode, not localized, comparison).
	 
		When doing string comparison in Lua, numbers in the string are
	 	detected and compared numerically, so that n2 comes before n10,
	 	which is not the case with a strict character comparison.
	 	For example, this order is observed:
	 
	 	<pre>
	 	A
	 	N
	 	n
	 	N1
	 	n1
	 	N2
	 	n2
	 	N10
	 	n10
	 	&Ntilde;
	 	&Ntilde;1
	 	&Ntilde;2
	 	&Ntilde;10
	 	Z
	 	</pre>

	 \param string1 The first string.
	 \param string2 The second string.
	 \return  0 if the strings are equal,
	 	-1 if string1 < string2,
	 	1 if string1 > string2.
--]]

function KSStringUtils.localizedStringCompare( --[[string]] string1, --[[string]] string2 )
	return KSStringUtilsImp.i18n_collate( string1, string2 )
end

--------------------------------------------------------------------------------
--[[!
	 Compares two strings using a combination of localized, case insensitive,
	 and numeric sort.

	 \param string1 The first string.
	 \param string2 The second string.
	 \return  0 if the strings are equal,
	 	-1 if string1 < string2,
	 	1 if string1 > string2.
--]]

function KSStringUtils.localizedStandardStringCompare( --[[string]] string1, --[[string]] string2 )
	return KSStringUtilsImp.i18n_localized_standard_compare( string1, string2 )
end

--------------------------------------------------------------------------------

--[[!
	 Converts a string to all lowercase characters.

	 \param string  The string.
 	 \return A new string.
--]]

function KSStringUtils.lower( --[[string]] string )
	return KSStringUtilsImp.lower( string )
end

--------------------------------------------------------------------------------

--[[!
	 Converts a string to all uppercase characters.

	 \param string  The string.
 	 \return A new string.
--]]
function KSStringUtils.upper( --[[string]] string )
	return KSStringUtilsImp.upper( string )
end

--------------------------------------------------------------------------------

--[[!
	 Removes whitespace (space and tab characters) from
	 the front and back ends of a string.
	
	\deprecated Use KSStringUtils.trimWhitespace for better locale-sensitive removal
	of whitespace characters.
	 \param string The string.
 	 \return A new string.
--]]

function KSStringUtils.trim( --[[string]] string )
	return KSStringUtilsImp.trim( string )
end

--------------------------------------------------------------------------------

--[[!
	Encode a KSData as Base64.
	
	\param data input KSData
	\return A new string.
--]]

function KSStringUtils.encodeBase64( --[[KSData]] data )
	return KSStringUtilsImp.encodeBase64( data )
end

--------------------------------------------------------------------------------

--[[!
	Decode a string containing Base64 data into KSData.

	\param string input
	\return A new KSData
--]]

function KSStringUtils.decodeBase64( --[[string]] string )
	return KSStringUtilsImp.decodeBase64( string, 'blob' )
end

--------------------------------------------------------------------------------

--[[!
	Decode a string containing Base64 data into another string.

	\param string input
	\return A new string
--]]

function KSStringUtils.decodeBase64AsString( --[[string]] string )
	return KSStringUtilsImp.decodeBase64( string )
end

--------------------------------------------------------------------------------

--[[!
	Encode a number as a localized string.

	\param number (number) the number to encode
	\param precision (optional number, default 0) the number of digits to include
		after the decimal point.
	\param dropTrailingZeroes (optional boolean, default false) whether to strip
		any tailing zeroes after the decimal point

	\return A new string representing this number
--]]

function KSStringUtils.numberToString( number, precision, dropTrailingZeroes )
	WFArguments.check( 'number', '?number', '?boolean', '?boolean' )
	
	precision = precision or 0
	if dropTrailingZeroes == nil then
		dropTrailingZeroes = false
	end
	
	if number ~= number then
		number = 0
	end
	
	local multiplier = math_pow( 10, precision )
	
	number = math_floor( 0.5 + number * multiplier ) / multiplier

	-- This prevents the appearance of a -0
	if number == 0 then
		number = 0
	end
	
	local formatTemplate
	local lt = 0
	
	if dropTrailingZeroes then
		formatTemplate = "%%0.%dg"
		lt = ( number < 1 ) and 0 or log10( abs( number )  )
		if lt < 0 then
			lt = 0
		end
		precision = precision + math_floor( lt ) + 1
	else
		formatTemplate = "%%0.%df"
	end
	
	local format = string.format( formatTemplate, precision )
	local numberString = string.format( format, number )
	
	
	-- replace decimal with locale specific decimal
	local decimal = ZSTR "$$$/KSStringUtils/numberFormatting/decimalNumberSeparator=."
	if decimal ~= '.' then
		numberString = numberString:replace( '.', decimal )
	end
	
	return numberString
end

local numberToString = KSStringUtils.numberToString

--------------------------------------------------------------------------------

--[[!
	 Converts a string that contains HTML code to one that
	 contains escaped characters for the "<", ">", and '&' character.

	 \param s The string.
 	 \return A new string.
--]]

function KSStringUtils.escapeHtml( --[[string]] s )

	-- FIXME [scouten 2012-05-30]: Probably not entirely correct, but good
	-- enough to get us started for diagnostic log ...

	s = tostring( s ) -- just in case we got passed a number or something else

	local result = s:replace( '&', '&amp;' )
	result = result:replace( '<', '&lt;' )
	result = result:replace( '>', '&gt;' )
	result = result:replace( '"', '&quot;' )

	return result

end

--------------------------------------------------------------------------------

--[[!
	 Extract the type+subtype string from a mime-type string.

	 \param contentType (string) The complete mime-type, including optional parameters.

 	 \return Just the type+subtype
--]]

function KSStringUtils.getBaseContentType( contentType )

	WFArguments.check( "string" )

	local strippedOfProperties = contentType:gsub( ";.*", "" )
	return strippedOfProperties

end

--------------------------------------------------------------------------------

--[[!
	Converts a string or KSData into hexadecimal.
	
	\param input( string or KSData ) string or KSData to be converted to hex.
	\return str( string ) converted to hexadecimal.

--]]

function KSStringUtils.convertToHex( --[[ string|KSData ]] input )

	assert( KSData.isData( input ) or type( input ) == 'string', "KSStringUtils:convertToHex input should be a KSData or string value" )

	if KSData.isData( input ) then
		input = tostring( input )
	end

	return string.gsub( input,"(.)",
			function( ch )
				return string.format("%02x",string.byte( ch ) )
			end )

end

--------------------------------------------------------------------------------

-- pclarke 2019-09-26: macOS and Windows use different conversion factors for
-- KB, MB, etc.  Windows uses kibibytes (base 2) while macOS uses "consumer kilobytes"
-- (base 10)
-- KSStringUtils.platformByteString() below converts according to the current platform
-- so that Lr's filesizes correspond to what the user sees in Finder and Windows Explorer
-- NOTE: not every file-size-ish number should be converted according to the platform;
-- for example, Adobe uses kibibytes to describe the user's cloud storage capacity

local KiB = 1024
local normalByteNames = {
    { 1, KiB, "$$$/AgStringUtils/byteString/bytes=^1 bytes" },
    { KiB, (KiB * KiB), "$$$/AgStringUtils/byteString/kilobytes=^1 KB" },
    { (KiB * KiB), (KiB * KiB * KiB), "$$$/AgStringUtils/byteString/megabytes=^1 MB" },
    { (KiB * KiB * KiB), (KiB * KiB * KiB * KiB), "$$$/AgStringUtils/byteString/gigabytes=^1 GB" },
	{ (KiB * KiB * KiB * KiB), (KiB * KiB * KiB * KiB * KiB), "$$$/AgStringUtils/byteString/terabytes=^1 TB" },
    { (KiB * KiB * KiB * KiB * KiB), (KiB * KiB * KiB * KiB * KiB * KiB), "$$$/AgStringUtils/byteString/petabytes=^1 PB" },
}

local KB = 1000
local consumerByteNames = {
    { 1, KB, "$$$/AgStringUtils/byteString/bytes=^1 bytes" },
    { KB, (KB * KB), "$$$/AgStringUtils/byteString/kilobytes=^1 KB" },
    { (KB * KB), (KB * KB * KB), "$$$/AgStringUtils/byteString/megabytes=^1 MB" },
    { (KB * KB * KB), (KB * KB * KB * KB), "$$$/AgStringUtils/byteString/gigabytes=^1 GB" },
    { (KB * KB * KB * KB), (KB * KB * KB * KB * KB), "$$$/AgStringUtils/byteString/terabytes=^1 TB" },
	{ (KB * KB * KB * KB * KB), (KB * KB * KB * KB * KB * KB), "$$$/AgStringUtils/byteString/petabytes=^1 PB" },
}

--[[!
	Converts an integer number into a nicely formatted number of bytes:
		* 1-1024 becomes "N bytes"
		* 1024-1024*1024 becomes "N KB"
		*  - 1024 ^ 3 becomes "N MB"
		*  - 1024 ^ 4 becomes "N GB"
		*  - 1024 ^ 5 becomes "N TB"
		*  - 1024 ^ 6 becomes "N PB"
	By default the number displayed will always be between 1 and 1024, scaled to
	the unit that puts it into this range, unless the value is >= 1024 PB.

	The above holds true for 1000 when converting to consumer byte sizes.

	If x is a string then this function returns that string.

	Otherwise, if it is not an integer then this function returns an empty string.
	(For our purposes, infinity is not an integer, nor is not-a-number.)

	\param byteNames (table) mapping of byte count to string (one of normalByteNames or consumerByteNames)
	\param x (integer) the number of bytes
	\param optPrecision (optional, non-negative integer) How many decimal places to display
	when using a suffix other than "byte" or "bytes".  (Defaults to 2.)
	\param optMaximum (optional) if specified, the numeric value will always be
		between 0 and that value (exclusive), scaled to the smallest unit that
		puts it into this range. For example, if a maximum of 1000 is given
		then "1004 MB" would become "0.9 GB"

	\return str( string ) converted to hexadecimal.

--]]

local function byteStringImp( byteNames, x, optPrecision, optMaximum )

	if not optPrecision then
		optPrecision = 2
	end

	if type( x ) == 'string' then
		return x
	end

	if type( x ) ~= 'number' then
		return ""
	end

	if x ~= x or x <= -math_huge or x >= math_huge then
		-- for not-a-number, non-integer or infinite values
		return ""
	end

	if x ~= math_floor( x ) then
		-- Round up.
		x = math_floor( x + 1 )
	end

	local formatted
	if x == 1 then
		formatted = ZSTR "$$$/AgStringUtils/byteString/byte=1 byte"
	else
		local isNegative = x < 0
		if isNegative then
			x = abs( x )
		end

		local mult = 10 ^ optPrecision
		local function round( x )
			x = math_floor( x * mult + 0.5 )
			x = x / mult
			return x
		end

		local scaling
		local formatString

		for _, scale in ipairs( byteNames ) do

			scaling = scale[ 1 ]
			formatString = scale[ 3 ]
			
			if x < scale[ 2 ] then
				if ( not optMaximum or x / scaling < optMaximum ) then
					break
				end

			end
		end

		local value = round( x / scaling )
		if value % 1 == 0 then
			formatted = ZSTR( formatString, value )
		else
			formatted = ZSTR( formatString, numberToString( value, optPrecision, nil ) )
		end

		if isNegative then
			formatted = ZSTR( "$$$/AgStringUtils/byteString/negativeBytes=-^1", formatted )
		end
	end

	return formatted

end

--[[!
	Converts an integer number into a nicely formatted number of bytes using a conversion that
	matches user visible byte counts on the current platform.  Windows uses kibibytes (base 2)
	while macOS uses "consumer kilobytes" (base 10).  This ensures the file sizes the user sees
	in Lr match what they see in Finder or Windows Explorer.

	See function byteStringImp() for a technical description.

	If x is a string then this function returns that string.

	Otherwise, if it is not an integer then this function returns an empty string.
	(For our purposes, infinity is not an integer, nor is not-a-number.)

	\param x (integer) the number of bytes
	\param optPrecision (optional, non-negative integer) How many decimal places to display
	when using a suffix other than "byte" or "bytes".  (Defaults to 2.)
	\param optMaximum (optional) if specified, the numeric value will always be
		between 0 and that value (exclusive), scaled to the smallest unit that
		puts it into this range. For example, if a maximum of 1000 is given
		then "1004 MB" would become "0.9 GB"

	\return str( string ) converted to hexadecimal.

--]]

function KSStringUtils.platformByteString( x, optPrecision, optMaximum )

	if WFPlatformInfo.isCocoa then
		-- Don't show fractional kilobytes
		if type( x ) == 'number' and x < (KB * KB) then
			optPrecision = 0
		end
		return byteStringImp( consumerByteNames, x, optPrecision, optMaximum )
	else
		-- Don't show fractional kibibytes
		if type( x ) == 'number' and x < (KiB * KiB) then
			optPrecision = 0
		end
		return byteStringImp( normalByteNames, x, optPrecision, optMaximum )
	end
end

--[[!
	Converts an integer number into a nicely formatted number of bytes assuming 1024
	bytes in a kilobyte.

	See function byteStringImp() for a technical description.

	If x is a string then this function returns that string.

	Otherwise, if it is not an integer then this function returns an empty string.
	(For our purposes, infinity is not an integer, nor is not-a-number.)

	\param x (integer) the number of bytes
	\param optPrecision (optional, non-negative integer) How many decimal places to display
	when using a suffix other than "byte" or "bytes".  (Defaults to 2.)
	\param optMaximum (optional) if specified, the numeric value will always be
		between 0 and that value (exclusive), scaled to the smallest unit that
		puts it into this range. For example, if a maximum of 1000 is given
		then "1004 MB" would become "0.9 GB"

	\return str( string ) converted to hexadecimal.

--]]

function KSStringUtils.byteString( x, optPrecision, optMaximum )
	return byteStringImp( normalByteNames, x, optPrecision, optMaximum )
end

--------------------------------------------------------------------------------


--[[!
	Converts an integer number into a nicely formatted number of bytes.
	
	This function is similar to KSStringUtils.byteString, but correctly formats
	the string based on the OS interpretation of 1GB. This should only be used
	when displaying byte strings related to disk storage.
	
	macOS and iOS reports storage capacity using the decimal system (base 10), 
	which calculates 1GB as 1 billion bytes. Windows uses the binary system (base 2), 
	which calculates 1GB as 1,073,741,824 bytes. See the following:
		https://support.apple.com/en-us/HT201402
		
	\param value (integer) the number of bytes
	\param optPrecision (optional, non-negative integer) How many decimal places to display
	when using a suffix other than "byte" or "bytes".  (Defaults to 2.)

	\return str( string ) A user-friendly string.
--]]

function KSStringUtils.storageByteString( value, optPrecision )

	WFArguments.check( "number", "?number" )

	optPrecision = optPrecision or 2

	if value ~= value or value <= -math_huge or value >= math_huge then
		-- infinite values
		return ""
	end

	local isNegative = false
	
	if value < 0 then
		isNegative = true
		value = math.abs( value )
	end
	
	local formattedString
	
	if value == 1 then
	
		formattedString = ZSTR "$$$/KSStringUtils/byteString/byte=1 byte"
		
	else

		local bytesInKb = 1024
		
		if WFPlatformInfo.isCocoa then
			bytesInKb = 1000
		end

		local tokens = {
			"$$$/KSStringUtils/byteString/bytes=^1 bytes",
			"$$$/KSStringUtils/byteString/kilobytes=^1 KB",
			"$$$/KSStringUtils/byteString/megabytes=^1 MB",
			"$$$/KSStringUtils/byteString/gigabytes=^1 GB",
			"$$$/KSStringUtils/byteString/terabytes=^1 TB",
			"$$$/KSStringUtils/byteString/petabytes=^1 PB",
			"$$$/KSStringUtils/byteString/exabytes=^1 EB",
			"$$$/KSStringUtils/byteString/zettabytes=^1 ZB",
			"$$$/KSStringUtils/byteString/yottabytes=^1 YB",
		}

		local multiplyFactor = 1
		local convertedValue = value

		while convertedValue >= bytesInKb do
		
			convertedValue = convertedValue / bytesInKb
			multiplyFactor = multiplyFactor + 1
		
		end

		local formatString = tokens[ multiplyFactor ] or "^1"

		if convertedValue % 1 == 0 then
			formattedString = ZSTR( formatString, convertedValue )
		else
			formattedString = ZSTR( formatString, numberToString( convertedValue, optPrecision ) )
		end

	end
	
	if isNegative then
		formattedString = ZSTR( "$$$/KSStringUtils/byteString/negativeBytes=-^1", formattedString )
	end
	
	return formattedString

end

--------------------------------------------------------------------------------
--[[
	\brief
	Create a function that will process UTF8 bytes and escape multi
	byte characters using your provided escape function.

	This is the minimal implementation of a routine that would consume the
	stateMachine function.  multiByteFunc is some function that will convert
	a integer value for a unicode character into a string representation.

	This function handles lower ascii characters itself (so you couldn't use
	this to create a UTF-8 to UTF-16 translator).

	local result = {}
	local charFun = function( strChr )
		table_insert( result, strChr )
	end
	local process = KSStringUtils.makeUTF8Processor( charFun, multiByteFunc )

	for x in string_gfind( str, '.' ) do

		process( x )
 
	end

	result = table_concat( result, "" )

	\param processChar (function) a function receiving two arguments for each
	UTF-8 code point: The converted multichar or the low ascii char as a lua String,
	and the unicode codepoint as a number.  Store this in an upvalue or process
	it however you wish -- the processor will not remember these values.

	\param multiByteFunc (function) a function to convert a unicode code point
	into a string representation.  This function is called every time a multibyte
	char is encountered.  It is passed as parameters the unicode codepoint as a
	number.  The return value of this function must be a string, and that is used
	as the converted multichar that is passed to the processChar fn.

	\return processor (function) a processor.  Repeatedly call this processor
	with each byte (as a single char lua string, not a number) of the string to
	be processed.  Note: Processors should not be reused, because they hold internal
	state information about what the current UTF-8 char that is being processed.
	Instead, simply make a new processor for each string.
]]--
function KSStringUtils.makeUTF8Processor( processChar, multiByteFunc )

	local mode, temp = 1, 0

	return function( strChr )

		local b = string_byte( strChr )

		if mode == 1 then	-- first byte of a UTF-8 character sequence

			--if b < 32 then
			--	temp = string_format( "&#x%4X;", b )
			-- else ...
			if b <= 127 then
				processChar( strChr, b )
			elseif b >= 192 and b <= 223 then
				mode = 2
				temp = ( b - 192 ) * 64
			elseif b >= 224 and b <= 239 then
				mode = 3
				temp = ( b - 224 ) * 4096
			else
				-- skip corrupt byte
			end

		elseif mode == 2 then	-- second byte of a UTF-8 character sequence

			if b >= 128 then
				temp = temp + ( b - 128 )
			else
				-- skip corrupt byte
			end
			mode = 1
			processChar( multiByteFunc( temp ), temp )
			temp = 0

		elseif mode == 3 then	-- third byte of a UTF-8 character sequence

			if b >= 128 then
				temp = temp + ( b - 128 ) * 64
			else
				-- skip corrupt byte
			end
			mode = 2

		end

	end

end

--------------------------------------------------------------------------------

local makeUTF8Processor = KSStringUtils.makeUTF8Processor

--------------------------------------------------------------------------------

local find = string.find

function KSStringUtils.validateUTF8( str )

	-- This implementation comes from:
	-- https://github.com/kikito/utf8_validator.lua/blob/master/utf8_validator.lua

	--[[
		MIT LICENSE
		Copyright (c) 2013 Enrique Garcia Cota
		Permission is hereby granted, free of charge, to any person obtaining a
		copy of this software and associated documentation files (the
		"Software"), to deal in the Software without restriction, including
		without limitation the rights to use, copy, modify, merge, publish,
		distribute, sublicense, and/or sell copies of the Software, and to
		permit persons to whom the Software is furnished to do so, subject to
		the following conditions:
		The above copyright notice and this permission notice shall be included
		in all copies or substantial portions of the Software.
		THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
		OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
		MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
		IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
		CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
		TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
		SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
	]]--

	local i, len = 1, #str
	
	while i <= len do
	
		if i == find( str, "[%z\1-\127]", i ) then 
		
			i = i + 1
			
		elseif i == find( str, "[\194-\223][\128-\191]", i ) then
		
			i = i + 2
			
		elseif i == find( str, "\224[\160-\191][\128-\191]", i )
			or i == find( str, "[\225-\236][\128-\191][\128-\191]", i )
			or i == find( str, "\237[\128-\159][\128-\191]", i )
			or i == find( str, "[\238-\239][\128-\191][\128-\191]", i ) then
		
			i = i + 3
			
		elseif i == find( str, "\240[\144-\191][\128-\191][\128-\191]", i )
			or i == find( str, "[\241-\243][\128-\191][\128-\191][\128-\191]", i )
			or i == find( str, "\244[\128-\143][\128-\191][\128-\191]", i ) then 
			
			i = i + 4
			
		else
		
			return false
			
		end
		
	end

	return true

end

--------------------------------------------------------------------------------

function KSStringUtils.getUTF16String_DEBUG()

	if KSStringUtilsImp.getUTF16String_DEBUG then
	
		return KSStringUtilsImp.getUTF16String_DEBUG()
	
	end

end

--------------------------------------------------------------------------------

--[[
	Copied from native fn WFStringUtils_isWhitespace
	Any changes here should also be made in the native fn.
	
	http://www.unicode.org/Public/UNIDATA/PropList.txt lists the whitespace characters as:
	0009..000D    ; White_Space # Cc   [5] <control-0009>..<control-000D>
	0020          ; White_Space # Zs       SPACE
	0085          ; White_Space # Cc       <control-0085>
	00A0          ; White_Space # Zs       NO-BREAK SPACE
	1680          ; White_Space # Zs       OGHAM SPACE MARK
	180E          ; White_Space # Zs       MONGOLIAN VOWEL SEPARATOR
	2000..200A    ; White_Space # Zs  [11] EN QUAD..HAIR SPACE
	2028          ; White_Space # Zl       LINE SEPARATOR
	2029          ; White_Space # Zp       PARAGRAPH SEPARATOR
	202F          ; White_Space # Zs       NARROW NO-BREAK SPACE
	205F          ; White_Space # Zs       MEDIUM MATHEMATICAL SPACE
	3000          ; White_Space # Zs       IDEOGRAPHIC SPACE
	# Total code points: 26
--]]
local function isWhitespace( c )

	if c >= 0x0009 and c <= 0x000D then
		return true
	end

	if c >= 0x2000 and c <= 0x200A then
		return true
	end

	if c == 0x0020 or c == 0x0085 or c == 0x00A0 or c == 0x1680
		or c == 0x180E or c == 0x2028 or c == 0x2029 or c == 0x202F
		or c == 0x205F or c == 0x3000 then
		return true
	end

	return false

end

--------------------------------------------------------------------------------

--[[
	 Removes whitespace (space and tab characters) from
	 the front and back ends of a string.
	
	\deprecated Use KSStringUtils.trimWhitespace for better locale-sensitive removal
	of whitespace characters.
	 \param string The string.
 	 \return A new string.
--]]
function KSStringUtils.trimWhitespace( str )
	local start = 0
	local ending = #str
	local n
	local hasNotSeenNonSpace = true
	local processUTF8Byte = makeUTF8Processor(
		function( strChr, unicode )
			local w = isWhitespace( unicode )
			if w then
				if hasNotSeenNonSpace then
					start = n
				end
			else
				ending = n
				hasNotSeenNonSpace = false
			end
		end,
		function(...)
			return nil
		end )

	n = 0
	for x in string_gfind( str, '.' ) do
		n = n + 1
		processUTF8Byte( x )
	end
	return string_sub( str, start + 1, ending )
end

--------------------------------------------------------------------------------

local siPrefixes = {
--	{ 24, ZSTR "###/SystemInternational/Unit/yotta=yotta", "###/SystemInternational/Unit/Prefix/Y=^1 Y^2" },
--	{ 21, ZSTR "###/SystemInternational/Unit/zetta=zetta", "###/SystemInternational/Unit/Prefix/Z=^1 Z^2" },
	{ 18, ZSTR "$$$/SystemInternational/Unit/exa=exa", "$$$/SystemInternational/Unit/Prefix/E=^1 E^2" },
	{ 15, ZSTR "$$$/SystemInternational/Unit/peta=peta", "$$$/SystemInternational/Unit/Prefix/P=^1 P^2" },
	{ 12, ZSTR "$$$/SystemInternational/Unit/tera=tera", "$$$/SystemInternational/Unit/Prefix/T=^1 T^2" },
	{ 9, ZSTR "$$$/SystemInternational/Unit/giga=giga", "$$$/SystemInternational/Unit/Prefix/G=^1 G^2" },
	{ 6, ZSTR "$$$/SystemInternational/Unit/mega=mega", "$$$/SystemInternational/Unit/Prefix/M=^1 M^2" },
	{ 3, ZSTR "$$$/SystemInternational/Unit/kilo=kilo", "$$$/SystemInternational/Unit/Prefix/k=^1 k^2" },
--	{ 2, ZSTR "###/SystemInternational/Unit/hecto=hecto", "###/SystemInternational/Unit/Prefix/h=^1 h^2" },
--	{ 1, ZSTR "###/SystemInternational/Unit/deka=deka", "###/SystemInternational/Unit/Prefix/da=^1 da^2" },
	{ 0, ZSTR "$$$/SystemInternational/Unit/unity=", "$$$/SystemInternational/Unit/Prefix/unity=^1 ^2" },
--	{ -1, ZSTR "###/SystemInternational/Unit/deci=deci", "###/SystemInternational/Unit/Prefix/d=^1 d^2" },
--	{ -2, ZSTR "###/SystemInternational/Unit/centi=centi", "###/SystemInternational/Unit/Prefix/c=^1 c^2" },
	{ -3, ZSTR "$$$/SystemInternational/Unit/milli=milli", "$$$/SystemInternational/Unit/Prefix/m=^1 m^2" },
	{ -6, ZSTR "$$$/SystemInternational/Unit/micro=micro", "$$$/SystemInternational/Unit/Prefix/micro=^1 ^U+00B5^2" },
	{ -9, ZSTR "$$$/SystemInternational/Unit/nano=nano", "$$$/SystemInternational/Unit/Prefix/n=^1 n^2" },
	{ -12, ZSTR "$$$/SystemInternational/Unit/pico=pico", "$$$/SystemInternational/Unit/Prefix/p=^1 p^2" },
--	{ -15, ZSTR "###/SystemInternational/Unit/femto=femto", "###/SystemInternational/Unit/Prefix/f=^1 f^2" },
--	{ -18, ZSTR "###/SystemInternational/Unit/atto=atto", "###/SystemInternational/Unit/Prefix/a=^1 a^2" },
--	{ -21, ZSTR "###/SystemInternational/Unit/zepto=zepto", "###/SystemInternational/Unit/Prefix/z=^1 z^2" },
--	{ -24, ZSTR "###/SystemInternational/Unit/yocto=yocto", "###/SystemInternational/Unit/Prefix/y=^1 y^2" },
}

--[[ \brief Convert a number into a string representation that's written
	using the "System Internationale" (SI -- a.k.a metric) prefix.  For example,
	* 10 ^ -15 prefix = atto
	* 10 ^ -3 = milli
	* 10 ^ 3 = kilo
	* etc..

	\param x (number) the raw numeric value
	\param baseUnit (localized unit abbreviation) the unit.
	\param optPrecision (integer, default 2) the number of decimal places to show
	
	\return (string) localized string representing that value and unit with an appropriate prefix.
--]]
function KSStringUtils.systemInternationalString( x, baseUnit, optPrecision )
	
	if x == 0 then
		return ZSTR( "$$$/SystemInternational/Unit/Prefix/unity=^1 ^2", x, baseUnit )
	end
	if not optPrecision then
		optPrecision = 2
	end
	local mult = 10 ^ optPrecision
	local function round( x )
		x = math_floor( x * mult )
		x = x / mult
		return x
	end
	
	if type( x ) ~= 'number' then
		return x
	end
	local formatted
	
	local isNegative = x < 0
	if isNegative then
		x = 0 - x
	end
	
	local scaling
	local formatString

	for _, scale in ipairs( siPrefixes ) do
		local factor = 10 ^ scale[ 1 ]
		scaling = factor
		formatString = scale[ 3 ]
		if x >= factor then
			break
		end
	end

	local value = round( x / scaling )
	if value % 1 == 0 then
		formatted = ZSTR( formatString, value, baseUnit )
	else
		formatted = ZSTR( formatString, numberToString( value, optPrecision ), baseUnit )
	end
	
	if isNegative then
		formatted = ZSTR( "$$$/AgStringUtils/byteString/negativeBytes=-^1", formatted )
	end

	return formatted
	
end

--------------------------------------------------------------------------------

local function reduceFraction( num, den )
	-- ( num den -- integer newNum newDen )
		
	if (num == 0) or (den == 0) then
		return 0, 0, 0, false
	end

	-- The math doesn't work cleanly for negative fractions, so treat all
	-- numbers as positive, and return a status flag if the result is to be
	-- presented as a negative number.

	local resultIsNegative = num * den < 0
	num = math.abs( num )
	den = math.abs( den )

	local ratio = WFRatio.reduce { num, den }
	
	num, den = ratio[ 1 ], ratio[ 2 ]
	
	local wholeNumber = math.floor(num / den )
	num = num - ( wholeNumber * den )

	return wholeNumber, num, den, resultIsNegative

end

--------------------------------------------------------------------------------

local function formatFractionASCII( wholeNumber, numerator, denominator, isNegative )

	-- 1 / 2 --> "1/2"

	if wholeNumber == 0 then

		if isNegative then
			numerator = -numerator
		end

		return ZSTR( "$$$/AgStringUtils/FormatFraction/FractionTemplateASCII=^1/^2",
			tostring( numerator ), tostring( denominator ) )

	end

	if isNegative then
		wholeNumber = -wholeNumber
	end

	-- 2 / 1 --> "2"
	if numerator == 0 then
		return tostring( wholeNumber )
	end

	-- 13 / 10 --> "1 3/10"
	return ZSTR( "$$$/AgStringUtils/FormatFraction/IntegerAndFractionTemplateASCII=^1 ^2/^3",
		tostring( wholeNumber ), tostring( numerator ), tostring( denominator ) )

end

--------------------------------------------------------------------------------

local superscriptDigits = {
	["0"] = ZSTR "$$$/private/SuperscriptDigit0=^U+2070",
	["1"] = ZSTR "$$$/private/SuperscriptDigit1=^U+00B9",
	["2"] = ZSTR "$$$/private/SuperscriptDigit2=^U+00B2",
	["3"] = ZSTR "$$$/private/SuperscriptDigit3=^U+00B3",
	["4"] = ZSTR "$$$/private/SuperscriptDigit4=^U+2074",
	["5"] = ZSTR "$$$/private/SuperscriptDigit5=^U+2075",
	["6"] = ZSTR "$$$/private/SuperscriptDigit6=^U+2076",
	["7"] = ZSTR "$$$/private/SuperscriptDigit7=^U+2077",
	["8"] = ZSTR "$$$/private/SuperscriptDigit8=^U+2078",
	["9"] = ZSTR "$$$/private/SuperscriptDigit9=^U+2079",
}

--------------------------------------------------------------------------------

local subscriptDigits = {
	["0"] = ZSTR "$$$/private/SubscriptDigit0=^U+2080",
	["1"] = ZSTR "$$$/private/SubscriptDigit1=^U+2081",
	["2"] = ZSTR "$$$/private/SubscriptDigit2=^U+2082",
	["3"] = ZSTR "$$$/private/SubscriptDigit3=^U+2083",
	["4"] = ZSTR "$$$/private/SubscriptDigit4=^U+2084",
	["5"] = ZSTR "$$$/private/SubscriptDigit5=^U+2085",
	["6"] = ZSTR "$$$/private/SubscriptDigit6=^U+2086",
	["7"] = ZSTR "$$$/private/SubscriptDigit7=^U+2087",
	["8"] = ZSTR "$$$/private/SubscriptDigit8=^U+2088",
	["9"] = ZSTR "$$$/private/SubscriptDigit9=^U+2089",
}

--------------------------------------------------------------------------------

local unitReciprocalFractionGlyphs = {
	[2] = ZSTR "$$$/private/UnicodeGlyphOneHalf=^U+00BD",
	[3] = ZSTR "$$$/private/UnicodeGlyphOneThird=^U+2153",
	[4] = ZSTR "$$$/private/UnicodeGlyphOneFourth=^U+00BC",
	[5] = ZSTR "$$$/private/UnicodeGlyphOneFifth=^U+2155",
	[6] = ZSTR "$$$/private/UnicodeGlyphOneSixth=^U+2159",
	[8] = ZSTR "$$$/private/UnicodeGlyphOneEighth=^U+215B",
}

--------------------------------------------------------------------------------

local function translateDigits( aNumber, digitTable )

	local s = tostring( aNumber )
	local t = { }

	for i = 1, string.len( s ) do
		local aChar = string.sub( s, i, i )
		table_insert( t, digitTable[ aChar ] )
	end

	return table_concat( t )

end

--------------------------------------------------------------------------------

local function makeSpecialUnicodeFractionGlyph( numerator, denominator )

	if numerator == 1 then
		return unitReciprocalFractionGlyphs[ denominator ]
	end

end

--------------------------------------------------------------------------------

function KSStringUtils.formattedFraction( num, den, restrictToASCII )

	if ( type( num ) ~= "number" ) or ( type( den ) ~= "number" ) or ( den == 0 ) then
		return "?"
	end

	local wholeNumber, numerator, denominator, isNegative = reduceFraction( num, den )

	if restrictToASCII then
		return formatFractionASCII( wholeNumber, numerator, denominator, isNegative )
	end

	-- Not restricted to ASCII, so use spiffy Unicode tricks like superscript chars,
	-- subscript chars and even special unicode fraction glyphs. [tgogolin]

	local superscriptNumerator = translateDigits( numerator, superscriptDigits )
	local subscriptDenominator = translateDigits( denominator, subscriptDigits )

	local superSubFractionString = makeSpecialUnicodeFractionGlyph(  numerator, denominator )

	-- Maybe there is no special glyph for this numerator and denominator,
	-- so use superscript and subscript characters.

	if superSubFractionString == nil then
		superSubFractionString = ZSTR ("$$$/AgStringUtils/FormatFraction/FractionTemplate=^1^U+2044^2",
				superscriptNumerator, subscriptDenominator )
	end

	local function negated( str )
		return ZSTR( "$$$/AgStringUtils/FormatNumber/DeltaNegative/UnicodeFigureDash=^U+2012 ^1",
						str )
			-- U+2012 is "Figure Dash"
	end

	if wholeNumber == 0 then
		if isNegative then
			return negated( superSubFractionString )
		else
			return superSubFractionString
		end
	end

	local str

	if numerator == 0 then
		str = tostring( wholeNumber )
	else
		str = ZSTR( "$$$/AgStringUtils/FormatFraction/IntegerAndFractionTemplate=^1 ^2",
				tostring( wholeNumber ), superSubFractionString )
	end

	if isNegative then
		return negated( str )
	else
		return str
	end

end

--------------------------------------------------------------------------------
--[[
	Returns a valid UTF8 string truncated string that is guaranteed to be less than
	or equal to maxBytes in length
]]--

function KSStringUtils.truncateUTF8( str, maxBytes )
	local ret = str

	if string.len( str ) > maxBytes then
		-- Make sure the last byte in the sequence is either 7 bit ascii or the last
		-- byte in a multibyte character. Test for last byte of mb char by seing if next
		-- byte is 7 bit ascii or start of new mb char
		local i = maxBytes
		while i > 0 and string_byte( str, i ) > 127
				and ( string_byte( str, i + 1 ) > 127
				and string_byte( str, i + 1 ) < 192 ) do
			i = i - 1
		end
		ret = i > 0 and string.sub( ret, 1, i ) or ""
	end

	return ret
end

--------------------------------------------------------------------------------

-- SPAM [jandrew 2020-03-27]: The following code was adapated from from an MIT-
-- licensed Lua Utility: https://github.com/Stepets/utf8.lua

-- :::START::: Stepets/utf8.lua

--[[
Copyright (c) 2006-2007, Kyle Smith
All rights reserved.
Contributors:
	Alimov Stepan
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
    * Redistributions of source code must retain the above copyright notice,
      this list of conditions and the following disclaimer.
    * Redistributions in binary form must reproduce the above copyright
      notice, this list of conditions and the following disclaimer in the
      documentation and/or other materials provided with the distribution.
    * Neither the name of the author nor the names of its contributors may be
      used to endorse or promote products derived from this software without
      specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
]]--

-- returns the number of bytes used by the UTF-8 character at byte i in s
-- also doubles as a UTF-8 character validator

local function utf8charbytes (s, i)
	-- argument defaults
	i = i or 1

	-- argument checking
	if type(s) ~= "string" then
		error("bad argument #1 to 'utf8charbytes' (string expected, got ".. type(s).. ")")
	end
	if type(i) ~= "number" then
		error("bad argument #2 to 'utf8charbytes' (number expected, got ".. type(i).. ")")
	end

	local c = byte(s, i)

	-- determine bytes needed for character, based on RFC 3629
	-- validate byte 1
	if c > 0 and c <= 127 then
		-- UTF8-1
		return 1

	elseif c >= 194 and c <= 223 then
		-- UTF8-2
		local c2 = byte(s, i + 1)

		if not c2 then
			error("UTF-8 string terminated early")
		end

		-- validate byte 2
		if c2 < 128 or c2 > 191 then
			error("Invalid UTF-8 character")
		end

		return 2

	elseif c >= 224 and c <= 239 then
		-- UTF8-3
		local c2 = byte(s, i + 1)
		local c3 = byte(s, i + 2)

		if not c2 or not c3 then
			error("UTF-8 string terminated early")
		end

		-- validate byte 2
		if c == 224 and (c2 < 160 or c2 > 191) then
			error("Invalid UTF-8 character")
		elseif c == 237 and (c2 < 128 or c2 > 159) then
			error("Invalid UTF-8 character")
		elseif c2 < 128 or c2 > 191 then
			error("Invalid UTF-8 character")
		end

		-- validate byte 3
		if c3 < 128 or c3 > 191 then
			error("Invalid UTF-8 character")
		end

		return 3

	elseif c >= 240 and c <= 244 then
		-- UTF8-4
		local c2 = byte(s, i + 1)
		local c3 = byte(s, i + 2)
		local c4 = byte(s, i + 3)

		if not c2 or not c3 or not c4 then
			error("UTF-8 string terminated early")
		end

		-- validate byte 2
		if c == 240 and (c2 < 144 or c2 > 191) then
			error("Invalid UTF-8 character")
		elseif c == 244 and (c2 < 128 or c2 > 143) then
			error("Invalid UTF-8 character")
		elseif c2 < 128 or c2 > 191 then
			error("Invalid UTF-8 character")
		end

		-- validate byte 3
		if c3 < 128 or c3 > 191 then
			error("Invalid UTF-8 character")
		end

		-- validate byte 4
		if c4 < 128 or c4 > 191 then
			error("Invalid UTF-8 character")
		end

		return 4

	else
		error("Invalid UTF-8 character")
	end
end

--------------------------------------------------------------------------------

--[[
	Returns the number of characters in a UTF-8 string.
]]--

function KSStringUtils.lengthOfUTF8String( str )

	WFArguments.check( 'string' )

	local pos = 1
	local bytes = string_len( str )
	local length = 0

	while pos <= bytes do
	
		length = length + 1
		pos = pos + utf8charbytes( str, pos )
		
	end

	return length
	
end

--------------------------------------------------------------------------------

--[[
	Functions identically to string.sub except that i and j are UTF-8 characters
	instead of bytes.
]]--

function KSStringUtils.UTF8Substring( str, i, j )

	WFArguments.check( 'string', 'number', '?number' )

	j = j or -1

	local pos = 1
	local bytes = string_len( str )
	local length = 0

	-- only set l if i or j is negative
	
	local l = ( i >= 0 and j >= 0 ) or KSStringUtils.lengthOfUTF8String( str )
	local startChar = ( i >= 0 ) and i or l + i + 1
	local endChar   = ( j >= 0 ) and j or l + j + 1

	-- can't have start before end!
	
	if startChar > endChar then
		return ""
	end

	-- byte offsets to pass to string.sub
	local startByte, endByte = 1, bytes

	while pos <= bytes do

		length = length + 1

		if length == startChar then
			startByte = pos
		end

		pos = pos + utf8charbytes( str, pos )

		if length == endChar then
			endByte = pos - 1
			break
		end
		
	end

	if startChar > length then
		startByte = bytes + 1
	end
	
	if endChar < 1 then 
		endByte = 0         
	end

	return string_sub( str, startByte, endByte )
	
end

-- :::END::: Stepets/utf8.lua

--------------------------------------------------------------------------------

--[[!
	Obfuscates a string using a multi-step process for secure storage.

	\param text (string) The string to obfuscate.
	\return (string) The obfuscated string.
--]]

function KSStringUtils.obfuscate( text )

	-- 1. Reverse the string

	local step1 = string_reverse( text )

	-- 2. XOR with a key and encode to base64

	local step2 = {}
	local buffer = {}

	for i = 1, #step1 do
		buffer[ i ] = string_char( bit.bxor( string_byte( step1, i ), kXorKey ) )
	end

	local data = KSData.make( table_concat( buffer ) )
	step2 = KSStringUtils.encodeBase64( data )

	-- 3. Swap bytes

	local step3 = {}

	local len = #step2

	for i = 1, len, 2 do

		local b1 = string_byte( step2, i )
		local b2 = i + 1 <= len and string_byte( step2, i + 1 ) or nil

		if b2 then
			step3[ i ] = string_char( b2 )
			step3[ i + 1 ] = string_char( b1 )
		else
			step3[ i ] = string_char( b1 )
		end

	end

	return table_concat( step3 )

end

--------------------------------------------------------------------------------

--[[!
	Deobfuscates a string that was previously obfuscated using KSStringUtils.obfuscate.

	\param text (string) The obfuscated string.
	\return (string) The original string.
--]]

function KSStringUtils.deobfuscate( text )

	-- Reverse the steps

	local step3 = {}
	local len = #text

	for i = 1, len, 2 do

		local b1 = string_byte( text, i )
		local b2 = i + 1 <= len and string_byte( text, i + 1 ) or nil

		if b2 then
			step3[ i ] = string_char( b2 )
			step3[ i + 1 ] = string_char( b1 )
		else
			step3[ i ] = string_char( b1 )
		end

	end

	step3 = table_concat( step3 )

	-- Decode from base64

	local data = KSStringUtils.decodeBase64AsString( step3 )
	local buffer = {}

	for i = 1, #data do
		local xorValue = bit.bxor( string_byte( data, i ), kXorKey )
		table.insert( buffer, string_char( xorValue ) )
	end

	local step2 = table_concat( buffer )

	return string_reverse( step2 )

end

--==============================================================================

return KSStringUtils
