--!strict

--[[
    Contains the validation handlers for each of the config values. Validation handlers
    should validate the config value provided via these three means:

    1. Command Line Arguments
    2. Config File
    3. Command Prompt

    If the config value is valid, it should convert it to the correct type (+ default value if applicable).
]]

-- Native Libraries
local process = require('@lune/process')
local task = require('@lune/task')

-- External Libraries
local llc_tasks = require('../../libs/llc_tasks/tasks')

-- Local Libraries
local Constants = require('../shared/constants')
local FileIO = require('../util/fileio')
local Formulas = require('../util/formulas')
local OpenCloudClient = require('../util/open-cloud-client')
local RetryUtil = require('../util/retry-util')
local Types = require('../shared/types')

-- Constants
local API_KEY_VALIDATION_MAP_NAME = "_RBX_batch-process-api-key-validation"
local VALIDATION_KEY = "TEST"
local MEMORY_STORES_EXPIRATION_MINIMUM = 600
local MEMORY_STORES_EXPIRATION_MAXIMUM = 3888000
local MAX_ITEMS_PER_LIST_REQUEST = 500
local MAX_ITEMS_PER_LIST_REQUEST_EXCLUDE_DELETED_KEYS = 100

--[[
    Validates the universeId config
    @param configValue string - The config value
    @return (boolean, string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function universeIdValidator(configValue: string): (boolean, string)
    -- Try to convert the config value to a number
    local tryGetNumber = tonumber(configValue)
    if not tryGetNumber then
        return false, "'universeId' must be a number"
    end

    -- Check if the number is a valid integer
    if math.floor(tryGetNumber) ~= tryGetNumber then
        return false, "'universeId' must be an integer"
    end

    return true, tostring(tryGetNumber)
end

--[[
    Validates the placeId config
    @param configValue string - The config value
    @return (boolean, string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function placeIdValidator(configValue: string): (boolean, string)
    -- Try to convert the config value to a number
    local tryGetNumber = tonumber(configValue)
    if not tryGetNumber then
        return false, "'placeId' must be a number"
    end

    -- Check if the number is a valid integer
    if math.floor(tryGetNumber) ~= tryGetNumber then
        return false, "'placeId' must be an integer"
    end

    return true, tostring(tryGetNumber)
end

--[[
    Validates the callback config
    @param configValue string - The config value
    @return (boolean, string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function callbackValidator(configValue: string): (boolean, string)
    -- Check if the config value is an empty string
    local configString = tostring(configValue)
    if configString == "" then
        return false, "'callback' can't be an empty string"
    end

    -- Check if the file exists
    local file = FileIO.ReadFile(configString)
    if not file then
        return false, "'callback' must be a valid filepath"
    end

    return true, configString
end

--[[
    Validates the numProcessingInstances config
    @param configValue string - The config value
    @return (boolean, number | string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function numProcessingInstancesValidator(configValue: string): (boolean, number | string)
    -- Try to convert the config value to a number
    local tryGetNumber = tonumber(configValue)
    if not tryGetNumber then
        return false, "'numProcessingInstances' must be an integer"
    end

    -- Check if the number is a positive value
    if tryGetNumber < 1 then
        return false, "'numProcessingInstances' must be a positive number"
    end

    -- Check if the number is a valid integer
    if math.floor(tryGetNumber) ~= tryGetNumber then
        return false, "'numProcessingInstances' must be an integer"
    end

    return true, tryGetNumber
end

--[[
    Validates the outputDirectory config
    @param configValue string - The config value
    @return (boolean, string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function outputDirectoryValidator(configValue: string): (boolean, string)
    -- Default to the output directory if the config value provided is an empty string
    if configValue == "" then
        return true, "output"
    end
    return true, configValue
end

--[[
    Validates the dataStoreName config
    @param configValue string - The config value
    @return (boolean, string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function dataStoreNameValidator(configValue: string): (boolean, string)
    -- Check if the config value is an empty string
    if configValue == "" then
        return false, "'dataStoreName' must be a non-empty string"
    end
    return true, configValue
end

--[[
    Validates the allScopes config
    @param configValue string - The config value
    @return (boolean, boolean | string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function allScopesValidator(configValue: string): (boolean, boolean | string)
    -- Check if the config value is a boolean (config file or command line)
    if configValue == "true" or configValue == "1" then
        return true, true
    elseif configValue == "false" or configValue == "0" then
        return true, false
    end
    return false, "'allScopes' must be a value of 1 (true) or 0 (false)"
end

--[[
    Validates the dataStoreScope config
    @param configValue string - The config value
    @return (boolean, string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function dataStoreScopeValidator(configValue: string): (boolean, string)
    -- Default to the global scope if the config value provided is an empty string
    if configValue == "" then
        return true, "global"
    end
    return true, configValue
end

--[[
    Validates the keyPrefix config
    @param configValue string - The config value
    @return (boolean, string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function keyPrefixValidator(configValue: string): (boolean, string)
    return true, configValue
end

--[[
    Validates the dataStorePrefix config
    @param configValue string - The config value
    @return (boolean, string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function dataStorePrefixValidator(configValue: string): (boolean, string)
    local configString: string = tostring(configValue)
    return true, configString
end

--[[
    Validates the excludeDeletedKeys config
    @param configValue string - The config value
    @return (boolean, boolean | string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function excludeDeletedKeysValidator(configValue: string): (boolean, boolean | string)
    -- Check if the config value is a boolean (config file or command line)
    if configValue == "true" or configValue == "1" then
        return true, true
    elseif configValue == "false" or configValue == "0" then
        return true, false
    end
    return false, "'excludeDeletedKeys' must be a value of 1 (true) or 0 (false)"
end

--[[
    Validates the processItemRateLimit config
    @param configValue string - The config value
    @return (boolean, number | string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function processItemRateLimitValidator(configValue: string): (boolean, number | string)
    -- Try to convert the config value to a number
    local tryGetNumber = tonumber(configValue)
    if not tryGetNumber then
        return false, "processItemRateLimit must be a number"
    end

    -- Check if the number is a positive value
    if tryGetNumber <= 0 then
        return false, "processItemRateLimit must be a positive number"
    end

    return true, tryGetNumber
end

--[[
    Validates the numRetries config
    @param configValue string - The config value
    @return (boolean, number | string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function numRetriesValidator(configValue: string): (boolean, number | string)
    -- Try to convert the config value to a number
    local tryGetNumber = tonumber(configValue)
    if not tryGetNumber then
        return false, "'numRetries' must be a number"
    end

    -- Check if the number is a valid integer
    if math.floor(tryGetNumber) ~= tryGetNumber then
        return false, "'numRetries' must be an integer"
    end

    -- Check if the number is a non-negative value
    if tryGetNumber < 0 then
        return false, "'numRetries' must be a non-negative number"
    end

    return true, tryGetNumber
end

--[[
    Validates the retryTimeoutBase config
    @param configValue string - The config value
    @return (boolean, number | string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function retryTimeoutBaseValidator(configValue: string): (boolean, number | string)
    -- Try to convert the config value to a number
    local tryGetNumber = tonumber(configValue)
    if not tryGetNumber then
        return false, "'retryTimeoutBase' must be a number"
    end

    -- Check if the number is a non-negative value
    if tryGetNumber < 0 then
        return false, "'retryTimeoutBase' must be a non-negative number"
    end

    return true, tryGetNumber
end

--[[
    Validates the retryExponentialBackoff config
    @param configValue string - The config value
    @return (boolean, number | string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function retryExponentialBackoffValidator(configValue: string): (boolean, number | string)
    -- Try to convert the config value to a number
    local tryGetNumber = tonumber(configValue)
    if not tryGetNumber then
        return false, "'retryExponentialBackoff' must be a number"
    end

    -- Check if the number is a positive value
    if tryGetNumber <= 0 then
        return false, "'retryExponentialBackoff' must be a positive number"
    end

    return true, tryGetNumber
end

--[[
    Validates the maxTotalFailedItems config
    @param configValue string - The config value
    @return (boolean, number | string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function maxTotalFailedItemsValidator(configValue: string): (boolean, number | string)
    -- Try to convert the config value to a number
    local tryGetNumber = tonumber(configValue)
    if not tryGetNumber then
        return false, "'maxTotalFailedItems' must be a number"
    end

    -- Check if the number is a valid integer
    if math.floor(tryGetNumber) ~= tryGetNumber then
        return false, "'maxTotalFailedItems' must be an integer"
    end

    -- Check if the number is a positive value
    if tryGetNumber <= 0 then
        return false, "'maxTotalFailedItems' must be a positive number"
    end

    return true, tryGetNumber
end

--[[
    Validates the maxItemsPerJob config
    @param configValue string - The config value
    @return (boolean, number | string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function maxItemsPerJobValidator(configValue: string): (boolean, number | string)
    -- Try to convert the config value to a number
    local tryGetNumber = tonumber(configValue)
    if not tryGetNumber then
        return false, "'maxItemsPerJob' must be a number"
    end

    -- Check if the number is a valid integer
    if math.floor(tryGetNumber) ~= tryGetNumber then
        return false, "'maxItemsPerJob' must be an integer"
    end

    -- Check if the number is a positive value
    if tryGetNumber <= 0 then
        return false, "'maxItemsPerJob' must be a positive number"
    end

    -- Check if the number is less than the maximum number of items that can be listed
    if tryGetNumber > MAX_ITEMS_PER_LIST_REQUEST then
        return false, "'maxItemsPerJob' must be less than or equal to " .. MAX_ITEMS_PER_LIST_REQUEST
    end

    return true, tryGetNumber
end

--[[
    Validates the jobQueueMaxSize config
    @param configValue string - The config value
    @return (boolean, number | string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function jobQueueMaxSizeValidator(configValue: string): (boolean, number | string)
    -- Try to convert the config value to a number
    local tryGetNumber = tonumber(configValue)
    if not tryGetNumber then
        return false, "'jobQueueMaxSize' must be a number"
    end

    -- Check if the number is a valid integer
    if math.floor(tryGetNumber) ~= tryGetNumber then
        return false, "'jobQueueMaxSize' must be an integer"
    end

    -- Check if the number is a positive value
    if tryGetNumber <= 0 then
        return false, "'jobQueueMaxSize' must be a positive number"
    end

    return true, tryGetNumber
end

--[[
    Validates the errorLogMaxLength config
    @param configValue string - The config value
    @return (boolean, number | string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function errorLogMaxLengthValidator(configValue: string): (boolean, number | string)
    -- Try to convert the config value to a number
    local tryGetNumber = tonumber(configValue)
    if not tryGetNumber then
        return false, "'errorLogMaxLength' must be a number"
    end

    -- Check if the number is a valid integer
    if math.floor(tryGetNumber) ~= tryGetNumber then
        return false, "'errorLogMaxLength' must be an integer"
    end

    -- Check if the number is a non-negative value
    if tryGetNumber < 0 then
        return false, "'errorLogMaxLength' must be a non-negative number"
    end

    return true, tryGetNumber
end

--[[
    Validates the progressRefreshTimeout config
    @param configValue string - The config value
    @return (boolean, number | string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function progressRefreshTimeoutValidator(configValue: string): (boolean, number | string)
    -- Try to convert the config value to a number
    local tryGetNumber = tonumber(configValue)
    if not tryGetNumber then
        return false, "'progressRefreshTimeout' must be a number"
    end

    -- Check if the number is a positive value
    if tryGetNumber <= 0 then
        return false, "'progressRefreshTimeout' must be a positive number"
    end

    return true, tryGetNumber
end

--[[
    Validates the memoryStoresExpiration config
    @param configValue string - The config value
    @return (boolean, number | string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function memoryStoresExpirationValidator(configValue: string): (boolean, number | string)
    -- Try to convert the config value to a number
    local tryGetNumber = tonumber(configValue)
    if not tryGetNumber then
        return false, "'memoryStoresExpiration' must be a number"
    end

    -- Check if the number is within the bounds
    if tryGetNumber < MEMORY_STORES_EXPIRATION_MINIMUM or tryGetNumber > MEMORY_STORES_EXPIRATION_MAXIMUM then
        return false, "'memoryStoresExpiration' must be between " .. MEMORY_STORES_EXPIRATION_MINIMUM .. " and " .. MEMORY_STORES_EXPIRATION_MAXIMUM .. " seconds"
    end

    return true, tryGetNumber
end

--[[
    Validates the memoryStoresStorageLimit config
    @param configValue string - The config value
    @return (boolean, number | string) - A success boolean, and the validated config value or an error message if the config value is invalid
--]]
local function memoryStoresStorageLimitValidator(configValue: string): (boolean, number | string)
    -- Try to convert the config value to a number
    local tryGetNumber = tonumber(configValue)

    -- Check if the number is valid
    if not tryGetNumber then
        return false, "'memoryStoresStorageLimit' must be a number"
    end

    return true, tryGetNumber
end

--[[
    Validates that the failed items config is large enough to resume
    @param configValue number - The config value
    @param currentFailedItems number - The current number of failed items
    @return (boolean, string?) - A success boolean, and an error message if the config value is invalid
--]]
local function validateEnoughFailedItems(configValue: number, currentFailedItems: number): (boolean, string?)
    if configValue <= currentFailedItems then
        return false, "The 'maxTotalFailedItems' config is not large enough to resume the process. Please set the value of 'maxTotalFailedItems' to at least " .. (currentFailedItems + 1) .. " in your config file or using the --max-total-failed-items option."
    end
    return true
end

--[[
    Validates the memoryStoresStorageLimit config
    @param configs Types.ConfigValues - The configurations
    @return (boolean, string?) - A success boolean, and an error message if the config value is invalid
--]]
local function validateMemoryStoresStorageLimitBounds(configs: Types.ConfigValues): (boolean, string?)
    local memoryStoresStorageLimit = tonumber(configs.memoryStoresStorageLimit)

    -- Get the min value for the config
    local minStorage = Formulas.CalculateMinStorage(configs)
    local maxStorage = Formulas.CalculateMaxStorage(configs)

    -- Check if the number is less than the min value
    if memoryStoresStorageLimit < minStorage and memoryStoresStorageLimit ~= -1 then
        return false, "'memoryStoresStorageLimit' must be at least " .. minStorage .. " kb in order to run."
    end

    -- Get the max value for the config
    if memoryStoresStorageLimit == -1 or memoryStoresStorageLimit > maxStorage then
        configs.memoryStoresStorageLimit = maxStorage
    end

    return true
end

--[[
    Validates the maxItemsPerJob config with excluded deleted keys
    @param configs Types.ConfigValues - The configurations
    @return (boolean, string?) - A success boolean, and an error message if the config value is invalid
--]]
local function validateMaxItemsPerJobWithExcludedDeletedKeys(configs: Types.ConfigValues): (boolean, string?)
    -- If the excludeDeletedKeys config is false, then we don't need to validate the maxItemsPerJob config
    if not configs.excludeDeletedKeys then
        return true
    end

    -- Validate that the maxItemsPerJob is an allowable value for excluding deleted keys
    local maxItemsPerJob = tonumber(configs.maxItemsPerJob)
    if maxItemsPerJob > MAX_ITEMS_PER_LIST_REQUEST_EXCLUDE_DELETED_KEYS then
        return false, "'maxItemsPerJob' must be less than or equal to " .. MAX_ITEMS_PER_LIST_REQUEST_EXCLUDE_DELETED_KEYS .. " when 'excludeDeletedKeys' is true."
    end

    return true
end

--[[
    Validates the API Key Permissions
    @param openCloudClient OpenCloudClient.OpenCloudClient - The open cloud client
    @param apiKey string - The API Key
    @return (boolean, string?) - A success boolean, and an error message if the config value is invalid
--]]
local function validateAPIKey(openCloudClient: OpenCloudClient.OpenCloudClient, apiKey: string): (boolean, string?)
    -- Validate Memory Stores Set
    local setSessionStatus = openCloudClient.createSortedMapEntry(API_KEY_VALIDATION_MAP_NAME, VALIDATION_KEY, VALIDATION_KEY, nil, 5)
    if setSessionStatus.status == 403 then
        return false, "The API Key is invalid. Please verify the following:\n1. Universe Id\n2. API Key\n3. API Key Permission for memory-store.sorted-map write"
    end

    -- Validate Memory Stores Get
    local getSessionStatus = openCloudClient.getSortedMapEntry(API_KEY_VALIDATION_MAP_NAME, VALIDATION_KEY)
    if getSessionStatus.status == 403 then
        return false, "The API Key is invalid. Please verify the following:\n1. Universe Id\n2. API Key\n3. API Key Permission for memory-store.sorted-map read"
    end

    -- Validate Data Stores List
    local listDataStoresStatus = openCloudClient.listDataStoreEntries(API_KEY_VALIDATION_MAP_NAME, "global")
    if listDataStoresStatus.status == 403 then
        return false, "The API Key is invalid. Please verify the following:\n1. Universe Id\n2. API Key\n3. API Key Permission for universe-datastores.objects:list"
    end

    -- Validate Data Stores Get
    local getDataStoreEntryStatus = openCloudClient.getDataStoreEntry(API_KEY_VALIDATION_MAP_NAME, "global", VALIDATION_KEY)
    if getDataStoreEntryStatus.status == 403 then
        return false, "The API Key is invalid. Please verify the following:\n1. Universe Id\n2. API Key\n3. API Key Permission for universe-datastores.objects:read"
    end

    -- Validate Data Stores Set
    local setDataStoreEntryStatus = openCloudClient.createDataStoreEntry(API_KEY_VALIDATION_MAP_NAME, "global", VALIDATION_KEY, VALIDATION_KEY)
    if setDataStoreEntryStatus.status == 403 then
        return false, "The API Key is invalid. Please verify the following:\n1. Universe Id\n2. API Key\n3. API Key Permissions for universe-datastores.objects:create and universe-datastores.control:create"
    end

    -- Validate Ordered Data Stores Get
    local getOrderedDataStoreRangeStatus = openCloudClient.getOrderedDataStoreRange(VALIDATION_KEY, "global", true, 1)
    if getOrderedDataStoreRangeStatus.status == 403 then
        return false, "The API Key is invalid. Please verify the following:\n1. Universe Id\n2. API Key\n3. API Key Permission for universe.ordered-data-store.scope.entry:read"
    end

    return true
end

--[[
    Validates the callback using a Session Task
    @param openCloudClient OpenCloudClient.OpenCloudClient - The open cloud client
    @param callback string - The callback function filepath
    @param universeId number | string - The universe ID
    @param placeId number | string - The place ID
    @return (boolean, string?) - A success boolean, and an error message if the config value is invalid
--]]
local function validateCallbackWithSessionTask(openCloudClient: OpenCloudClient.OpenCloudClient, callback: string, universeId: number | string, placeId: number | string): (boolean, string?)
    local file = FileIO.ReadFile(callback)
    if not file then
        return false, "'callback' must be a valid filepath"
    end

     -- Create the callback function wrapper
    local callbackWrapper = "local processItemBuilder = function()\n" .. file :: string .. "\nend\n"
    local validationScript = callbackWrapper .. [[
        return function()
            local processItem = processItemBuilder()
            return 0
        end
    ]]

    -- Check that the Session Task doesn't fail
    local validateCallbackSession = llc_tasks.create(validationScript, {
        api_key = process.env['API_KEY'],
        universe_id = universeId,
        place_id = placeId
    })
    local validateCallbackSuccess: boolean, validateCallbackResult: any = RetryUtil.RetryAsync(function()
        local validateCallbackRes, validateCallbackError = validateCallbackSession:start()
        if validateCallbackError then
            error("")
        end
        return validateCallbackRes
    end, Constants.SessionTaskRetryConfig.MaxAttempts, Constants.SessionTaskRetryConfig.TimeoutBase, Constants.SessionTaskRetryConfig.TimeoutExponentialBackoff)
    if not validateCallbackSuccess then
        return false, "The callback could not be validated with a Session Task. Please verify the following:\n1. Universe ID\n2. Place ID\n3. API Key\n4. API Key Permission for luau-execution-sessions write\n5. There are available Session Tasks"
    end

    local sessionId = validateCallbackResult.task_create_res.path

    -- Check if the callback succeeded after a short delay
    repeat
        task.wait(1)
        local getSessionStatus = openCloudClient.getLuauExecutionSessionStatus(sessionId)
        if getSessionStatus.body and getSessionStatus.body.state == "FAILED" then
            return false, "The callback is invalid. Please ensure the callback is a valid Luau script and try again."
        end
    until getSessionStatus.status == 200 and getSessionStatus.body.state == "COMPLETE"

    return true
end

--[[
    The validation handlers for each of the config values.
]]
local Validators: { [string]: (any) -> (boolean, string | number | boolean) } = {
    ["universeId"] = universeIdValidator,
    ["placeId"] = placeIdValidator,
    ["callback"] = callbackValidator,
    ["numProcessingInstances"] = numProcessingInstancesValidator,
    ["outputDirectory"] = outputDirectoryValidator,
    ["dataStoreName"] = dataStoreNameValidator,
    ["allScopes"] = allScopesValidator,
    ["dataStoreScope"] = dataStoreScopeValidator,
    ["keyPrefix"] = keyPrefixValidator,
    ["dataStorePrefix"] = dataStorePrefixValidator,
    ["excludeDeletedKeys"] = excludeDeletedKeysValidator,
    ["processItemRateLimit"] = processItemRateLimitValidator,
    ["numRetries"] = numRetriesValidator,
    ["retryTimeoutBase"] = retryTimeoutBaseValidator,
    ["retryExponentialBackoff"] = retryExponentialBackoffValidator,
    ["maxTotalFailedItems"] = maxTotalFailedItemsValidator,
    ["maxItemsPerJob"] = maxItemsPerJobValidator,
    ["jobQueueMaxSize"] = jobQueueMaxSizeValidator,
    ["errorLogMaxLength"] = errorLogMaxLengthValidator,
    ["progressRefreshTimeout"] = progressRefreshTimeoutValidator,
    ["memoryStoresExpiration"] = memoryStoresExpirationValidator,
    ["memoryStoresStorageLimit"] = memoryStoresStorageLimitValidator
}

return {
    Validators = Validators,
    ValidateAPIKey = validateAPIKey,
    ValidateCallbackWithSessionTask = validateCallbackWithSessionTask,
    ValidateEnoughFailedItems = validateEnoughFailedItems,
    ValidateMaxItemsPerJobWithExcludedDeletedKeys = validateMaxItemsPerJobWithExcludedDeletedKeys,
    ValidateMemoryStoresStorageLimitBounds = validateMemoryStoresStorageLimitBounds
}


--!strict

--[[
    Main entry point for the batch processor CLI.
    This module handles command line argument parsing, configuration loading,
    and execution of batch processing commands.
]]

-- Native Libraries
local process = require('@lune/process')

-- Local Libraries
local CLI = require('./cli/core')
local CLIO = require('./util/clio')
local ConfigLoader = require('./config/loader')
local Constants = require('./shared/constants')
local Cleanup = require('./commands/cleanup')
local List = require('./commands/list')
local ProcessDataStores = require('./commands/process-data-stores')
local ProcessKeys = require('./commands/process-keys')
local Resume = require('./commands/resume')
local Types = require('./shared/types')

-- Constants
local MAX_PROCESS_NAME_LENGTH = 18

--[[
    Main entry point for the batch processor.
]]
return function(): ()
    -- Parse CLI arguments
    local parsedInput = CLI.Parse()
    local command: Types.CommandType = parsedInput.command
    local processName: string = parsedInput.values.processName
    local options: Types.ConfigValues = parsedInput.values
    local flags: Types.ConfigValues = parsedInput.flags
    local configToOptionFlagMap: { [string]: string } = parsedInput.configToOptionFlagMap
    local inverseFlagMap: { [string]: string } = parsedInput.inverseFlagMap

    -- Validate an API Key is set
    if not process.env['API_KEY'] then
        CLIO.Error('API_KEY is not set. Set the API_KEY environment variable to and then try again.')
    end

    -- Validate the process name length (if applicable)
    if processName and string.len(processName) > MAX_PROCESS_NAME_LENGTH then
        CLIO.Error("Process name can be at most " .. MAX_PROCESS_NAME_LENGTH .. " characters")
    end

    -- Load and validate configurations
    local configs: Types.ConfigValues = ConfigLoader.LoadConfigs(command, options, flags, configToOptionFlagMap, inverseFlagMap)
    ConfigLoader.ValidateConfigs(configs)
    local finalConfigs = ConfigLoader.PromptForMissingConfigs(command, configs)

    -- Execute the appropriate command based on the command type
    if command == Constants.Commands.ProcessKeys then
        ProcessKeys.Execute(processName, finalConfigs :: Types.ProcessKeysConfigs)
    elseif command == Constants.Commands.ProcessDataStores then
        ProcessDataStores.Execute(processName, finalConfigs :: Types.ProcessDataStoresConfigs)
    elseif command == Constants.Commands.Resume then
        Resume.Execute(processName, finalConfigs :: Types.ResumeConfigs)
    elseif command == Constants.Commands.Cleanup then
        Cleanup.Execute(processName, finalConfigs :: Types.CleanupConfigs)
    elseif command == Constants.Commands.List then
        List.Execute(finalConfigs :: Types.ListConfigs)
    end
end


--!strict

--[[
    Constants used throughout the codebase.
]]

-- Local Libraries
local Types = require('./types')

-- Constants
local ActiveStage1TasksFile = "stage1_session_paths_active.output" -- The output file that contains the active stage 1 session paths
local ActiveStage2TasksFile = "stage2_session_paths_active.output" -- The output file that contains the active stage 2 session paths
local BatchProcessFile = "process.output" -- The output file that contains the batch process table
local BatchProcessFailedItemsOrderedDataStorePrefix = "_RBX_batch-process-failed-items-" -- The prefix for the failed items ordered data store
local BatchProcessFailedItemLogsMapPrefix = "_RBX_batch-process-failed-item-logs-" -- The prefix for the failed item logs map
local BatchProcessGuidToSessionPathsMapName = "_RBX_batch-process-guid-to-session-paths" -- The name of the map that contains the guid-session path mapping
local BatchProcessMapName = "_RBX_batch-processes" -- The name of the map that contains the batch processes
local BatchProcessTaskStatusMapName = "_RBX_batch-process-task-status" -- The name of the map that contains the Process Task status
local CLILogFile = "cli.output" -- The output file that contains the CLI logs
local CompletedStage1TasksFile = "stage1_session_paths_complete.output" -- The output file that contains the completed stage 1 session paths
local CompletedStage2TasksFile = "stage2_session_paths_complete.output" -- The output file that contains the completed stage 2 session paths
local FailedItemsFile = "failed_items.output" -- The output file that contains the failed items
local GuidToSessionPathsMapExpiration = 60 -- 1 minute expiration for the guid to session paths map
local PlaceHolder = "\x01" -- A placeholder value for a config that signifies that the config was not provided via the config file or command line arguments
local SessionLogsDirectory = "session_logs" -- The directory that contains the session logs
local TableColumnWidth = 23 -- The number of spaces in each column of the table

--[[
    Command type mappings.
    Maps command names to their string identifiers.
]]
local Commands: { [string]: Types.CommandType } = {
    ["ProcessDataStores"] = "process-data-stores",
    ["ProcessKeys"] = "process-keys",
    ["Resume"] = "resume",
    ["Cleanup"] = "cleanup",
    ["List"] = "list"
}

--[[
    Scan type mappings.
    Maps scan type names to their string identifiers.
]]
local ScanTypes: { [string]: Types.ScanType } = {
    ["DataStores"] = "data-stores",
    ["Keys"] = "keys"
}

--[[
    Stage type mappings.
    Maps stage type names to their string identifiers.
]]
local StageTypes: { [string]: Types.StageType } = {
    ["Stage1"] = "stage1",
    ["Stage2"] = "stage2"
}

--[[
    Retry configuration for session tasks.
]]
local SessionTaskRetryConfig: { [string]: number } = {
    ["MaxAttempts"] = 3,
    ["TimeoutBase"] = 0.5,
    ["TimeoutExponentialBackoff"] = 2
}

--[[
    List of configuration options.
    Defines all command line options for the application.
]]
local ConfigurationOptions: { [string]: Types.ConfigOption } = {
    -- Execution options
    callback = {
        option = "callback",
        shortOption = "-C",
        description = "Filepath to the callback function to process items",
        prompt = "Enter a filepath for 'callback': "
    },
    numProcessingInstances = {
        option = "num-processing-instances",
        shortOption = "-i",
        description = "Number of stage 2 processing instances to use",
        prompt = "Enter a value for 'numProcessingInstances': "
    },
    outputDirectory = {
        option = "output-directory",
        shortOption = "-o",
        description = "Directory for output files",
        prompt = "Enter a directory filepath for 'outputDirectory' (default: 'output/'): "
    },
    placeId = {
        option = "place-id",
        shortOption = "-p",
        description = "The place ID",
        prompt = "Enter a value for 'placeId': "
    },
    universeId = {
        option = "universe-id",
        shortOption = "-u",
        description = "The universe ID",
        prompt = "Enter a value for 'universeId': "
    },

    -- Processing options
    errorLogMaxLength = {
        option = "error-log-max-length",
        description = "Maximum length of error logs",
        default = 50
    },
    jobQueueMaxSize = {
        option = "job-queue-max-size",
        description = "Maximum size of the job queue",
        default = 20
    },
    maxItemsPerJob = {
        option = "max-items-per-job",
        description = "Maximum items to process in one job",
        default = 50
    },
    maxTotalFailedItems = {
        option = "max-total-failed-items",
        description = "Maximum number of failed items before failing the entire batch process",
        default = 100
    },
    memoryStoresExpiration = {
        option = "memory-stores-expiration",
        description = "Expiration time for memory stores (in seconds)",
        default = 3888000
    },
    memoryStoresStorageLimit = {
        option = "memory-stores-storage-limit",
        description = "Storage limit for memory stores (in kilobytes) (-1 for no limit)",
        default = -1
    },
    numRetries = {
        option = "num-retries",
        description = "Number of times to retry processing an item (choose 0 for no retries)",
        default = 4
    },
    processItemRateLimit = {
        option = "process-item-rate-limit",
        description = "Rate limit for processing items on a single instance (per second)",
        default = 50
    },
    progressRefreshTimeout = {
        option = "progress-refresh-timeout",
        description = "Timeout for refreshing progress (in seconds) for stage 1",
        default = 5
    },
    retryExponentialBackoff = {
        option = "retry-exponential-backoff",
        description = "Exponential backoff for retries (choose 1 for no backoff)",
        default = 2
    },
    retryTimeoutBase = {
        option = "retry-timeout-base",
        description = "Base timeout for retries (in seconds) (choose 0 for no timeout)",
        default = 0.5
    },

    -- Scan keys options
    dataStoreName = {
        option = "data-store-name",
        shortOption = "-d",
        description = "Name of the DataStore",
        prompt = "Enter a value for 'dataStoreName': "
    },
    dataStoreScope = {
        option = "data-store-scope",
        shortOption = "-s",
        description = "Scope of the DataStore",
        prompt = "Enter a value for 'dataStoreScope' (default: 'global'): "
    },
    keyPrefix = {
        option = "key-prefix",
        shortOption = "-P",
        description = "Prefix of the keys to scan",
        prompt = "Enter a value for 'keyPrefix' (default: ''): "
    },

    -- Scan DataStores options
    dataStorePrefix = {
        option = "data-store-prefix",
        shortOption = "-P",
        description = "Prefix of the DataStores to scan",
        prompt = "Enter a value for 'dataStorePrefix' (default: ''): "
    }
}

--[[
    List of configuration flags.
    Defines all command line flags for the application.
]]
local ConfigurationFlags: { [string]: Types.ConfigFlag } = {
    -- Scan keys flags
    excludeDeletedKeys = {
        flag = "exclude-deleted-keys",
        flagInverse = "include-deleted-keys",
        description = "Exclude deleted keys from scan",
        descriptionInverse = "Include deleted keys in scan",
        prompt = "Enter a value for 'excludeDeletedKeys' (0: false, 1: true): "
    },
    allScopes = {
        flag = "all-scopes",
        description = "Scan all scopes",
        override = "dataStoreScope"
    }
}

--[[
    List of commands and their associated configs.
]]
local CommandDetails: { [string]: { [string]: any } } = {
    [Commands.ProcessKeys] = {
        description = "Start a batch process that processes over key names inside of a single data store.",
        positionalArgs = {
            { name = "processName", description = "The name of the process to run" }
        },
        configurations = {
            "universeId",
            "placeId",
            "callback",
            "outputDirectory",
            "dataStoreName",
            "allScopes",
            "dataStoreScope",
            "keyPrefix",
            "excludeDeletedKeys",
            "numProcessingInstances",
            "processItemRateLimit",
            "numRetries",
            "retryTimeoutBase",
            "retryExponentialBackoff",
            "maxTotalFailedItems",
            "maxItemsPerJob",
            "jobQueueMaxSize",
            "errorLogMaxLength",
            "progressRefreshTimeout",
            "memoryStoresExpiration",
            "memoryStoresStorageLimit"
        }
    },
    [Commands.ProcessDataStores] = {
        description = "Start a batch process that processes over data store names.",
        positionalArgs = {
            { name = "processName", description = "The name of the process to run" }
        },
        configurations = {
            "universeId",
            "placeId",
            "callback",
            "outputDirectory",
            "dataStorePrefix",
            "numProcessingInstances",
            "processItemRateLimit",
            "numRetries",
            "retryTimeoutBase",
            "retryExponentialBackoff",
            "maxTotalFailedItems",
            "maxItemsPerJob",
            "jobQueueMaxSize",
            "errorLogMaxLength",
            "progressRefreshTimeout",
            "memoryStoresExpiration",
            "memoryStoresStorageLimit"
        }
    },
    [Commands.Resume] = {
        description = "Resume a previously running batch process.",
        positionalArgs = {
            { name = "processName", description = "The name of the process to resume" }
        },
        configurations = {
            "universeId",
            "outputDirectory",
            "numProcessingInstances",
            "processItemRateLimit",
            "numRetries",
            "retryTimeoutBase",
            "retryExponentialBackoff",
            "maxTotalFailedItems",
            "jobQueueMaxSize",
            "errorLogMaxLength",
            "progressRefreshTimeout",
            "memoryStoresExpiration",
            "memoryStoresStorageLimit"
        }
    },
    [Commands.Cleanup] = {
        description = "Cleanup an old batch process.",
        positionalArgs = {
            { name = "processName", description = "The name of the process to cleanup" }
        },
        configurations = {
            "universeId"
        }
    },
    [Commands.List] = {
        description = "List all batch processes.",
        configurations = {
            "universeId"
        }
    }
}

return {
    ActiveStage1TasksFile = ActiveStage1TasksFile,
    ActiveStage2TasksFile = ActiveStage2TasksFile,
    BatchProcessFile = BatchProcessFile,
    BatchProcessFailedItemLogsMapPrefix = BatchProcessFailedItemLogsMapPrefix,
    BatchProcessFailedItemsOrderedDataStorePrefix = BatchProcessFailedItemsOrderedDataStorePrefix,
    BatchProcessGuidToSessionPathsMapName = BatchProcessGuidToSessionPathsMapName,
    BatchProcessMapName = BatchProcessMapName,
    BatchProcessTaskStatusMapName = BatchProcessTaskStatusMapName,
    CLILogFile = CLILogFile,
    CommandDetails = CommandDetails,
    Commands = Commands,
    CompletedStage1TasksFile = CompletedStage1TasksFile,
    CompletedStage2TasksFile = CompletedStage2TasksFile,
    ConfigurationFlags = ConfigurationFlags,
    ConfigurationOptions = ConfigurationOptions,
    FailedItemsFile = FailedItemsFile,
    GuidToSessionPathsMapExpiration = GuidToSessionPathsMapExpiration,
    PlaceHolder = PlaceHolder,
    SessionLogsDirectory = SessionLogsDirectory,
    SessionTaskRetryConfig = SessionTaskRetryConfig,
    ScanTypes = ScanTypes,
    StageTypes = StageTypes,
    TableColumnWidth = TableColumnWidth
}


--!strict

--[[
    File I/O utility functions.
]]

-- Native Libraries
local fs = require('@lune/fs')
local serde = require('@lune/serde')

-- Local Libraries
local Constants = require('../shared/constants')

-- File IO
local FileIO = {}

--[[
    Gets the directory from a filepath.
    @param filepath string - The full path to the file
    @return string - The directory of the file
]]
local function getDirectoryFromPath(filepath: string): string
    -- Replace backslashes with forward slashes for consistency
    filepath = string.gsub(filepath, "\\", "/")

    -- Find the last slash
    local lastSlash = string.find(filepath, "/[^/]*$")
    if not lastSlash then
        return ""
    end

    -- Return everything before the last slash
    return string.sub(filepath, 1, lastSlash - 1)
end

--[[
    Reads a file and returns the contents.
    @param filePath string - The path to the file to read
    @return string? - The contents of the file, or nil if the file does not exist
]]
FileIO.ReadFile = function(filePath: string): string?
    -- Read the file
    local success: boolean, file: string = pcall(fs.readFile, filePath)
    if not success then
        return nil
    end

    return file
end

--[[
    Reads a JSON file and returns the contents.
    @param filePath string - The path to the file to read
    @return {[string]: any}? - The contents of the file as a table, or nil if the file does not exist
]]
FileIO.ReadJsonFile = function(filePath: string): { [string]: any }?
    local file = FileIO.ReadFile(filePath)
    if not file then
        return nil
    end

    -- Try to decode the file
    local success, result = pcall(serde.decode, 'json', file)
    if not success then
        return nil
    end

    -- Return the decoded file as a table
    return result
end

--[[
    Reads a file by line.
    @param filepath string - The path to the file to read
    @return {string}? - A table of lines from the file or nil if the file does not exist
]]
FileIO.ReadFileByLine = function(filepath: string): { string }?
    local file = FileIO.ReadFile(filepath)
    if not file then
        return nil
    end

    -- Add each line to the table
    local lines = {}
    for line in file:gmatch("([^\n]*)\n?") do
        table.insert(lines, line)
    end

    return lines
end

--[[
    Writes a file.
    @param filepath string - The full path to the file
    @param contents string - The contents of the file
]]
FileIO.WriteFile = function(filepath: string, contents: string): ()
    local dir = getDirectoryFromPath(filepath)
    local success: boolean = pcall(fs.writeDir, dir)
    if not success then
        return
    end
    success = pcall(fs.writeFile, filepath, contents)
    if not success then
        return
    end
end

--[[
    Appends a line to a file.
    @param filepath string - The full path to the file
    @param line string - The line to append to the file
]]
FileIO.AppendLineToFile = function(filepath: string, line: string): ()
    local file = FileIO.ReadFile(filepath)
    if not file then
        return
    end

    -- Append the line to the file
    FileIO.WriteFile(filepath, file .. line .. "\n")
end

--[[
    Clears a file.
    @param filepath string - The full path to the file
]]
FileIO.ClearFile = function(filepath: string): ()
    local file = FileIO.ReadFile(filepath)
    if not file then
        return
    end

    -- Clear the file
    FileIO.WriteFile(filepath, "")
end

--[[
    Appends a line to the CLI log file.
    @param processName string - The name of the process
    @param outputDir string - The output directory
    @param line string - The line to append to the CLI log file
]]
FileIO.AppendToCLILogFile = function(processName: string, outputDir: string, line: string): ()
    FileIO.AppendLineToFile(outputDir .. "/" .. processName .. "/" .. Constants.CLILogFile, line)
end

--[[
    Removes the output files for a process.
    @param processName string - The name of the process
    @param outputDirectory string - The output directory
]]
FileIO.RemoveOutputFiles = function(processName: string, outputDirectory: string): ()
    pcall(function()
        fs.removeFile(outputDirectory .. "/" .. processName .. "/" .. Constants.CLILogFile)
    end)
    pcall(function()
        fs.removeFile(outputDirectory .. "/" .. processName .. "/" .. Constants.FailedItemsFile)
    end)
    pcall(function()
        fs.removeFile(outputDirectory .. "/" .. processName .. "/" .. Constants.BatchProcessFile)
    end)
    pcall(function()
        fs.removeFile(outputDirectory .. "/" .. processName .. "/" .. Constants.ActiveStage1TasksFile)
    end)
    pcall(function()
        fs.removeFile(outputDirectory .. "/" .. processName .. "/" .. Constants.ActiveStage2TasksFile)
    end)
    pcall(function()
        fs.removeFile(outputDirectory .. "/" .. processName .. "/" .. Constants.CompletedStage1TasksFile)
    end)
    pcall(function()
        fs.removeFile(outputDirectory .. "/" .. processName .. "/" .. Constants.CompletedStage2TasksFile)
    end)
end

--[[
    Creates the output files for a process.
    @param processName string - The name of the process
    @param outputDirectory string - The output directory
]]
FileIO.CreateOutputFiles = function(processName: string, outputDirectory: string): ()
    FileIO.WriteFile(outputDirectory .. "/" .. processName .. "/" .. Constants.CLILogFile, "")
    FileIO.WriteFile(outputDirectory .. "/" .. processName .. "/" .. Constants.FailedItemsFile, "")
    FileIO.WriteFile(outputDirectory .. "/" .. processName .. "/" .. Constants.BatchProcessFile, "")
    FileIO.WriteFile(outputDirectory .. "/" .. processName .. "/" .. Constants.ActiveStage1TasksFile, "")
    FileIO.WriteFile(outputDirectory .. "/" .. processName .. "/" .. Constants.ActiveStage2TasksFile, "")
    FileIO.WriteFile(outputDirectory .. "/" .. processName .. "/" .. Constants.CompletedStage1TasksFile, "")
    FileIO.WriteFile(outputDirectory .. "/" .. processName .. "/" .. Constants.CompletedStage2TasksFile, "")
end

return FileIO


--!strict

--[[
    This file contains formulas for calculating the maximum and minimum memory stores storage and access that a batch process can use.
]]

-- Local Libraries
local Types = require('../shared/types')

-- Constants
local BUFFER_BYTES = 1024 -- Extra 1kb buffer
local BATCH_PROCESS_OVERHEAD_BYTES = 1125 -- Length of the batch process entry
local FAILED_ITEM_LOG_OVERHEAD_BYTES = 209 -- Length of the failed item log without the truncated error log
local TASK_STATUS_OVERHEAD_BYTES = 229 -- Worst case size of the task status entry without the session paths
local SESSION_PATH_BYTES = 161 -- Length of the session path, accounting for quotes
local STORAGE_POOL_OVERHEAD_BYTES = 45 -- Length of the storage bytes entry
local JOB_OVERHEAD_BYTES = 97 -- Length of the job entry
local WORST_CASE_ITEM_LENGTH_BYTES = 53 -- Worst case length of an item including quotes and comma
local WORST_CASE_ITEM_PAGE_BASE_BYTES = 225 -- Worst case length of an item page including quotes and comma
local SESSION_TASK_RATE_LIMIT = 40 -- The number of session tasks that can be spun up in a minute
local PAGES_SORTED_MAP_MAX_SIZE = 200 -- The maximum size of the pages sorted map
local BYTES_PER_KILOBYTE = 1024 -- The number of bytes in a kilobyte
local ACCESS_OVERHEAD = 17 -- The overhead of the access logic in RUs
local LISTS_PER_MINUTE = 50 -- The number of lists that can be performed in a minute
local CREATE_JOBS_RU = 8 -- The RU cost of creating jobs
local PROCESS_PAGES_RU = 5 -- The RU cost of processing pages
local LOOP_UPDATE_RU = 6 -- The RU cost of updating the loop
local FAILED_ITEM_LOG_RECONCILIATION_RU = 3 -- The RU cost of reconciling the failed item logs map
local STAGE_2_POLLING_RU = 30 -- The RU cost of polling for stage 2
local COMPLETE_PAGE_RU = 4 -- The RU cost of completing a page and reserving quota for failed items

-- Formulas
local Formulas = {}

--[[
    Calculates the maximum memory storesstorage that a batch process can use.
    @param configs Types.ConfigValues - The configurations to use
    @return number - The maximum storage that a batch process can use
]]
Formulas.CalculateMaxStorage = function(configs: Types.ConfigValues): number
    local maxStorageBytes = BUFFER_BYTES
        + BATCH_PROCESS_OVERHEAD_BYTES + STORAGE_POOL_OVERHEAD_BYTES + TASK_STATUS_OVERHEAD_BYTES -- overhead contribution
        + configs.jobQueueMaxSize * (WORST_CASE_ITEM_LENGTH_BYTES * configs.maxItemsPerJob + JOB_OVERHEAD_BYTES) -- job queue variable contribution
        + math.min(2 * configs.jobQueueMaxSize, PAGES_SORTED_MAP_MAX_SIZE) * (WORST_CASE_ITEM_LENGTH_BYTES * configs.maxItemsPerJob + WORST_CASE_ITEM_PAGE_BASE_BYTES)  -- item page variable contribution
        + configs.maxTotalFailedItems * (FAILED_ITEM_LOG_OVERHEAD_BYTES + configs.errorLogMaxLength)  -- failed items variable contribution
        + configs.numProcessingInstances * (SESSION_PATH_BYTES + 1) -- guid to session path variable contribution
        + math.min(configs.numProcessingInstances, SESSION_TASK_RATE_LIMIT) * SESSION_PATH_BYTES -- task status variable contribution
    local maxStorage = math.ceil(maxStorageBytes / BYTES_PER_KILOBYTE)
    return maxStorage
end

--[[
    Calculates the minimum memory stores storage that a batch process can use. The minimum is defined to be the amount of usage assuming
    that the batch process can hold at least 2 jobs and 2 item pages. This is needed to ensure the batch process does not freeze indefinitely.
    @param configs Types.ConfigValues - The configurations to use
    @return number - The minimum storage for a batch process to run
]]
Formulas.CalculateMinStorage = function(configs: Types.ConfigValues): number
    local minStorageBytes = BUFFER_BYTES
        + BATCH_PROCESS_OVERHEAD_BYTES + STORAGE_POOL_OVERHEAD_BYTES + TASK_STATUS_OVERHEAD_BYTES -- overhead contribution
        + 2 * (WORST_CASE_ITEM_LENGTH_BYTES * configs.maxItemsPerJob + JOB_OVERHEAD_BYTES) -- job queue variable contribution for 2 jobs
        + 2 * (WORST_CASE_ITEM_LENGTH_BYTES * configs.maxItemsPerJob + WORST_CASE_ITEM_PAGE_BASE_BYTES) -- item page variable contribution for 2 pages
        + configs.numProcessingInstances * (SESSION_PATH_BYTES + 1) -- guid to session path variable contribution
        + math.min(configs.numProcessingInstances, SESSION_TASK_RATE_LIMIT) * SESSION_PATH_BYTES -- task status variable contribution
    local minStorage = math.ceil(minStorageBytes / BYTES_PER_KILOBYTE)
    return minStorage
end

--[[
    Calculates the maximum access that a batch process can use.
    @param configs Types.ConfigValues - The configurations to use
    @return number - The maximum access that a batch process can use
]]
Formulas.CalculateMaxAccess = function(configs: Types.ConfigValues): number
    local loopsPerMinute = 60 / configs.progressRefreshTimeout
    local maxPageMapSize = math.min(2 * configs.jobQueueMaxSize, PAGES_SORTED_MAP_MAX_SIZE)
    local maxAccess = ACCESS_OVERHEAD
        + maxPageMapSize -- Reconciliation
        + CREATE_JOBS_RU * LISTS_PER_MINUTE -- Creating pages and jobs
        + maxPageMapSize * loopsPerMinute -- Listing pages
        + PROCESS_PAGES_RU * maxPageMapSize -- Processing pages
        + LOOP_UPDATE_RU * loopsPerMinute -- Loop update logic
        + FAILED_ITEM_LOG_RECONCILIATION_RU * loopsPerMinute -- Failed item log reconciliation logic
        + math.max(STAGE_2_POLLING_RU * configs.numProcessingInstances, LISTS_PER_MINUTE) -- Stage 2 polling
        + COMPLETE_PAGE_RU * LISTS_PER_MINUTE -- Completing pages
        + math.min(LISTS_PER_MINUTE * configs.maxItemsPerJob, configs.maxTotalFailedItems) -- Reserving quota and writing failed item logs
        + configs.maxTotalFailedItems -- Reading failed item logs on CLI
    return maxAccess
end

return Formulas


--!strict

--[[
    Generates random GUIDs (UUID v4) using basic Lua functions.
]]

-- Constants for GUID generation
local HEX_CHARS = "0123456789abcdef"
local GUID_TEMPLATE = "xxxxxxxx-xxxx-4xxx-yxxx-xxxxxxxxxxxx"

-- GuidUtil
local GuidUtil = {}

--[[
    Generates a random hex character
    @return string - A random hex character
]]
local function randomHexChar(): string
    local index = math.random(1, #HEX_CHARS)
    return string.sub(HEX_CHARS, index, index)
end

--[[
    Generates a random GUID (UUID v4).
    This implementation follows the UUID v4 specification:
    - 8 random hex digits
    - 4 random hex digits
    - 4 random hex digits (first digit is 4)
    - 4 random hex digits (first digit is 8, 9, A, or B)
    - 12 random hex digits
    @return string - A random GUID
]]
GuidUtil.GenerateGuid = function(): string
    local guid = GUID_TEMPLATE

    -- Replace each 'x' with a random hex digit
    guid = string.gsub(guid, "x", function()
        return randomHexChar()
    end)

    -- Replace 'y' with 8, 9, A, or B
    guid = string.gsub(guid, "y", function()
        local chars = "89ab"
        local index = math.random(1, #chars)
        return string.sub(chars, index, index)
    end)

    return guid
end

return GuidUtil


--!strict

--[[
    Main utility functions for managing batch processes.
]]

-- Native Libraries
local fs = require('@lune/fs')
local serde = require('@lune/serde')
local task = require('@lune/task')

-- Local Libraries
local CLIO = require('../util/clio')
local Constants = require('../shared/constants')
local FileIO = require('../util/fileio')
local OpenCloudClient = require('../util/open-cloud-client')
local TableUtil = require('../util/table-util')
local TimeUtil = require('../util/time-util')
local Types = require('../shared/types')

-- Constants
local READ_TASK_STATUS_INTERVAL = 5

-- Global variables
local nextTimestamp: number? = nil

--[[
    Writes the logs of a completed session to the output file.
    @param processName string - The name of the process
    @param sessionPath string - The path of the session task
    @param stageType Types.StageType - The type of stage that the session task is in
    @param configs Types.AnyConfigs - The configurations for the process
    @param openCloudClient OpenCloudClient.OpenCloudClient - The open cloud client
]]
local function writeCompletedSessionLogs(
    processName: string,
    sessionPath: string,
    sessionStatusBody: any,
    stageType: Types.StageType,
    configs: Types.AnyConfigs,
    openCloudClient: OpenCloudClient.OpenCloudClient
): ()
    -- Parse out the session id
    local sessionId = string.match(sessionPath, "([^/]+)$")
    if not sessionId then
        CLIO.Error("Failed to parse session ID from session path... Something went wrong.")
    end

    -- Get the logs
    local logs = openCloudClient.getLuauExecutionSessionLogs(sessionPath)
    if logs.status ~= 200 then
        FileIO.AppendToCLILogFile(processName, configs.outputDirectory, "Warning: Failed to get session logs. Reason: " .. OpenCloudClient.GetBodyAsString(logs))
        return
    end

    local logsOutput = table.concat(logs.body.luauExecutionSessionTaskLogs[1].messages, "\n")
    if sessionStatusBody.state == "COMPLETE" then
        logsOutput = logsOutput .. "\nSession Task Completed Successfully"
    elseif sessionStatusBody.state == "FAILED" and sessionStatusBody.error then
        logsOutput = logsOutput .. "\nSession Task Failed with error: " .. sessionStatusBody.error.message
    end

    -- Write the logs to the output file
    FileIO.WriteFile(
        configs.outputDirectory .. "/" ..
        processName .. "/" ..
        Constants.SessionLogsDirectory .. "/" ..
        stageType .. "/" ..
        sessionId :: string .. ".output",
        logsOutput
    )
end

-- ProcessUtil
local ProcessUtil = {}

--[[
    Expires Stage 1 Session Tasks that may have died.
    @param processName string - The name of the process
    @param configs Types.AnyConfigs - The configurations for the process
    @param openCloudClient OpenCloudClient.OpenCloudClient - The open cloud client
]]
ProcessUtil.ExpireStage1SessionTasks = function(
    processName: string,
    configs: Types.AnyConfigs,
    openCloudClient: OpenCloudClient.OpenCloudClient
): ()
    local processTaskStatus = openCloudClient.getSortedMapEntry(
        Constants.BatchProcessTaskStatusMapName,
        processName
    )
    if processTaskStatus.status ~= 200 then
        FileIO.AppendToCLILogFile(processName, configs.outputDirectory, "Warning: Failed to get process task status. Reason: " .. OpenCloudClient.GetBodyAsString(processTaskStatus))
        return
    end

    local status: Types.ProcessTaskStatus = processTaskStatus.body.value

    -- Remove completed stage 1 tasks
    local activeStage1 = {}
    for _, stage1SessionPath in ipairs(status.stage1SessionPaths) do
        local getStage1Status = openCloudClient.getLuauExecutionSessionStatus(stage1SessionPath)
        if getStage1Status.status == 404 then
            continue
        elseif getStage1Status.status ~= 200 then
            FileIO.AppendToCLILogFile(processName, configs.outputDirectory, "Warning: Failed to get stage 1 status for " .. stage1SessionPath .. ". Reason: " .. OpenCloudClient.GetBodyAsString(getStage1Status))
            return
        end
        if getStage1Status.body.state == "PROCESSING" then
            table.insert(activeStage1, stage1SessionPath)
        else
            FileIO.AppendLineToFile(
                configs.outputDirectory .. "/" .. processName .. "/" .. Constants.CompletedStage1TasksFile,
                stage1SessionPath
            )
            writeCompletedSessionLogs(processName, stage1SessionPath, getStage1Status.body, Constants.StageTypes.Stage1, configs, openCloudClient)
        end
    end

    -- Update the Session Task status
    local newStatus = {
        numInstances = status.numInstances,
        stage1SessionPaths = activeStage1,
        stage2SessionPaths = status.stage2SessionPaths,
    }
    openCloudClient.updateSortedMapEntry(
        Constants.BatchProcessTaskStatusMapName,
        processName,
        newStatus,
        nil,
        configs.memoryStoresExpiration
    )

    -- Update the active stage 1 and 2 task files
    FileIO.WriteFile(
        configs.outputDirectory .. "/" .. processName .. "/" .. Constants.ActiveStage1TasksFile,
        table.concat(activeStage1, "\n") .. (#activeStage1 > 0 and "\n" or "")
    )
end

--[[
    Expires Stage 2 Session Tasks that may have died.
    @param processName string - The name of the process
    @param configs Types.AnyConfigs - The configurations for the process
    @param openCloudClient OpenCloudClient.OpenCloudClient - The open cloud client
]]
ProcessUtil.ExpireStage2SessionTasks = function(
    processName: string,
    configs: Types.AnyConfigs,
    openCloudClient: OpenCloudClient.OpenCloudClient
): ()
    local processTaskStatus = openCloudClient.getSortedMapEntry(
        Constants.BatchProcessTaskStatusMapName,
        processName
    )
    if processTaskStatus.status ~= 200 then
        FileIO.AppendToCLILogFile(processName, configs.outputDirectory, "Warning: Failed to get process task status. Reason: " .. OpenCloudClient.GetBodyAsString(processTaskStatus))
        return
    end

    local status: Types.ProcessTaskStatus = processTaskStatus.body.value

    -- Remove completed stage 2 tasks
    local activeStage2 = {}
    for _, stage2SessionPath in ipairs(status.stage2SessionPaths) do
        local getStage2Status = openCloudClient.getLuauExecutionSessionStatus(stage2SessionPath)
        if getStage2Status.status == 404 then
            continue
        elseif getStage2Status.status ~= 200 then
            FileIO.AppendToCLILogFile(processName, configs.outputDirectory, "Warning: Failed to get stage 2 status for " .. stage2SessionPath .. ". Reason: " .. OpenCloudClient.GetBodyAsString(getStage2Status))
            return
        end
        if getStage2Status.body.state == "PROCESSING" then
            table.insert(activeStage2, stage2SessionPath)
        else
            FileIO.AppendLineToFile(
                configs.outputDirectory .. "/" .. processName .. "/" .. Constants.CompletedStage2TasksFile,
                stage2SessionPath
            )
            writeCompletedSessionLogs(processName, stage2SessionPath, getStage2Status.body, Constants.StageTypes.Stage2, configs, openCloudClient)
        end
    end

    -- Update the Session Task status
    local newStatus = {
        numInstances = status.numInstances,
        stage1SessionPaths = status.stage1SessionPaths,
        stage2SessionPaths = activeStage2,
    }
    openCloudClient.updateSortedMapEntry(
        Constants.BatchProcessTaskStatusMapName,
        processName,
        newStatus,
        nil,
        configs.memoryStoresExpiration
    )

    -- Update the active stage 2 task files
    FileIO.WriteFile(
        configs.outputDirectory .. "/" .. processName .. "/" .. Constants.ActiveStage2TasksFile,
        table.concat(activeStage2, "\n") .. (#activeStage2 > 0 and "\n" or "")
    )
end

--[[
    Writes failed items to the output file.
    @param processName string - The name of the process
    @param outputDir string - The output directory
    @param openCloudClient OpenCloudClient.OpenCloudClient - The open cloud client
]]
ProcessUtil.WriteFailedItems = function(
    processName: string,
    outputDir: string,
    openCloudClient: OpenCloudClient.OpenCloudClient
) : ()
    -- Get the failed items ordered data store and failed item logs map
    local failedItemsOrderedDataStoreName = Constants.BatchProcessFailedItemsOrderedDataStorePrefix .. processName
    local failedItemLogsMapName = Constants.BatchProcessFailedItemLogsMapPrefix .. processName

    -- Loop over all of the failed items
    repeat
        local page = openCloudClient.getOrderedDataStoreRange(failedItemsOrderedDataStoreName, "global", true, 100, nil, nextTimestamp, nil)
        if page.status ~= 200 then
            FileIO.AppendToCLILogFile(processName, outputDir, "Warning: Failed to get failed items. Reason: " .. OpenCloudClient.GetBodyAsString(page))
            return
        end
        if page.body.orderedDataStoreEntries == nil then
            break
        end
        for _, failedItem in ipairs(page.body.orderedDataStoreEntries) do
            -- Get the failed item logs
            local failedItemLogs = openCloudClient.getSortedMapEntry(failedItemLogsMapName, failedItem.id)
            if failedItemLogs.status ~= 200 and failedItemLogs.status ~= 404 then
                FileIO.AppendToCLILogFile(processName, outputDir, "Warning: Failed to get failed item logs. Reason: " .. OpenCloudClient.GetBodyAsString(failedItemLogs))
                return
            end

            -- If the failed item logs are not found, write just the failed item to the file
            if failedItemLogs.status == 404 then
                local failedItemLine = failedItem.id .. "|Could not load the failed item's error log and session path. Please re-run the callback on the item to diagnose."
                FileIO.AppendLineToFile(outputDir .. '/' .. processName .. '/' .. Constants.FailedItemsFile, failedItemLine)
                continue
            end

            -- Otherwise, write the failed item and logs to the file
            local failedItemData: Types.FailedItem = failedItemLogs.body.value
            local failedItemLine = string.format(
                "%s|%s|%s|%s",
                failedItem.id,
                TimeUtil.FormatTimestamp(failedItemData.time),
                failedItemData.error,
                failedItemData.sessionPath
            )
            FileIO.AppendLineToFile(outputDir .. '/' .. processName .. '/' .. Constants.FailedItemsFile, failedItemLine)

            local deleteFailedItemLog = openCloudClient.deleteSortedMapEntry(failedItemLogsMapName, failedItem.id)
            if deleteFailedItemLog.status ~= 200 then
                FileIO.AppendToCLILogFile(processName, outputDir, "Warning: Failed to delete failed item log for " .. failedItem.id .. ". Reason: " .. OpenCloudClient.GetBodyAsString(deleteFailedItemLog))
            end
        end
        nextTimestamp = page.body.orderedDataStoreEntries[#page.body.orderedDataStoreEntries].value + 1
    until page.body.nextPageToken == nil
end

--[[
    Writes the batch process to the output file.
    @param processName string - The name of the process
    @param openCloudClient OpenCloudClient.OpenCloudClient - The open cloud client
    @param outputDir string - The output directory
]]
ProcessUtil.WriteBatchProcess = function(
    processName: string,
    openCloudClient: OpenCloudClient.OpenCloudClient,
    outputDir: string
): ()
    local batchProcess = openCloudClient.getSortedMapEntry(
        Constants.BatchProcessMapName,
        processName
    )
    if batchProcess.status ~= 200 then
        FileIO.AppendToCLILogFile(processName, outputDir, "Warning: Failed to get batch process information. Reason: " .. OpenCloudClient.GetBodyAsString(batchProcess))
        return
    end
    local values = batchProcess.body.value
    local batchProcessHeader = TableUtil.FormatHeader(
        'Process Name',
        'Create Time',
        'Last Update Time',
        'Status',
        'Scanned',
        'Processed',
        'Failed'
    )
    local batchProcessSeparator = TableUtil.FormatSeparator(7)
    local batchProcessLine = TableUtil.FormatLine(
        processName,
        TimeUtil.FormatTimestamp(values.startTime),
        TimeUtil.FormatTimestamp(values.updateTime),
        values.status,
        values.scannedItems,
        values.processedItems,
        values.failedItems
    )
    local batchProcessBody = string.format(
        "%s\n%s\n%s\n\nConfigurations:\n---------------\n%s",
        batchProcessHeader,
        batchProcessSeparator,
        batchProcessLine,
        serde.encode("json", batchProcess.body.value.configs, true)
    )
    FileIO.WriteFile(outputDir .. '/' .. processName .. '/' .. Constants.BatchProcessFile, batchProcessBody)
end

--[[
    Sends a stop signal to the batch processing session tasks.
    @param processName string - The name of the process
    @param openCloudClient OpenCloudClient.OpenCloudClient - The open cloud client
    @param configs Types.AnyConfigs - The configurations for the process
]]
ProcessUtil.StopSessionTasks = function(processName: string, openCloudClient: OpenCloudClient.OpenCloudClient, configs: Types.AnyConfigs): ()
    CLIO.Print("Stopping all session tasks (this may take up to 5 minutes)...")

    local process = openCloudClient.getSortedMapEntry(
        Constants.BatchProcessMapName,
        processName
    )
    if process.status == 404 then
        return
    elseif process.status ~= 200 then
        CLIO.Error("There was an error checking the process info. Please try again later. Reason: " .. OpenCloudClient.GetBodyAsString(process))
    end
    local processInfo: Types.BatchProcess = process.body.value

    -- Only stop if the process is in progress
    if processInfo.status ~= "InProgress" then
        return
    end

    -- Send the stop signal to the process
    processInfo.status = "Stopped"
    local updateResult = openCloudClient.updateSortedMapEntry(
        Constants.BatchProcessMapName,
        processName,
        processInfo,
        nil,
        configs.memoryStoresExpiration
    )
    if updateResult.status ~= 200 then
        CLIO.Error("There was an error stopping the process. Please try again later. Reason: " .. OpenCloudClient.GetBodyAsString(updateResult))
    end
end

--[[
    Waits for all session tasks for a batch processing operation to complete.
    @param processName string - The name of the process
    @param openCloudClient OpenCloudClient.OpenCloudClient - The open cloud client
]]
ProcessUtil.WaitForSessionsToComplete = function(processName: string, openCloudClient: OpenCloudClient.OpenCloudClient): ()
    -- Get the process task status
    local processTaskStatus = openCloudClient.getSortedMapEntry(
        Constants.BatchProcessTaskStatusMapName,
        processName
    )
    if processTaskStatus.status == 404 then
        return
    elseif processTaskStatus.status ~= 200 then
        CLIO.Error("There was an error checking the process task status. Please try again later. Reason: " .. OpenCloudClient.GetBodyAsString(processTaskStatus))
    end
    local status: Types.ProcessTaskStatus = processTaskStatus.body.value

    local completedSessions = {
        stage1 = {},
        stage2 = {}
    }

    while true do
        local allSessionsCompleted = true
        for _, stage1SessionPath in ipairs(status.stage1SessionPaths) do
            if completedSessions.stage1[stage1SessionPath] then
                continue
            end

            -- Check the status of the stage 1 session task
            local getStage1Status = openCloudClient.getLuauExecutionSessionStatus(stage1SessionPath)
            if getStage1Status.status ~= 200 and getStage1Status.status ~= 404 then
                CLIO.Error("There was an error checking the status of session task with path " .. stage1SessionPath .. ". Please try again later. Reason: " .. OpenCloudClient.GetBodyAsString(getStage1Status))
            end
            if getStage1Status.body and getStage1Status.body.state == "PROCESSING" then
                allSessionsCompleted = false
            else
                table.insert(completedSessions.stage1, stage1SessionPath)
            end
        end

        -- Check all stage 2 tasks
        for _, stage2SessionPath in ipairs(status.stage2SessionPaths) do
            if completedSessions.stage2[stage2SessionPath] then
                continue
            end

            -- Check the status of the stage 2 session task
            local getStage2Status = openCloudClient.getLuauExecutionSessionStatus(stage2SessionPath)
            if getStage2Status.status ~= 200 and getStage2Status.status ~= 404 then
                CLIO.Error("There was an error checking the status of session task with path " .. stage2SessionPath .. ". Please try again later. Reason: " .. OpenCloudClient.GetBodyAsString(getStage2Status))
            end
            if getStage2Status.body and getStage2Status.body.state == "PROCESSING" then
                allSessionsCompleted = false
            else
                table.insert(completedSessions.stage2, stage2SessionPath)
            end
        end

        -- If all sessions are completed, break the loop
        if allSessionsCompleted then
            break
        end

        task.wait(READ_TASK_STATUS_INTERVAL)
    end
end

--[[
    Exits the batch process and finalizes the output.
    @param processName string - The name of the process
    @param configs Types.AnyConfigs - The configurations for the process
    @param openCloudClient OpenCloudClient.OpenCloudClient - The open cloud client
]]
ProcessUtil.FinalizeOnExit = function(processName: string, configs: Types.AnyConfigs, openCloudClient: OpenCloudClient.OpenCloudClient): ()
    ProcessUtil.StopSessionTasks(processName, openCloudClient, configs)
    ProcessUtil.WaitForSessionsToComplete(processName, openCloudClient)

    CLIO.Print("Writing output files...")
    ProcessUtil.WriteFailedItems(processName, configs.outputDirectory, openCloudClient)
    ProcessUtil.WriteBatchProcess(processName, openCloudClient, configs.outputDirectory)
    ProcessUtil.ExpireStage1SessionTasks(processName, configs, openCloudClient)
    ProcessUtil.ExpireStage2SessionTasks(processName, configs, openCloudClient)
    pcall(function()
        fs.removeFile(configs.outputDirectory .. "/" .. processName .. "/" .. Constants.ActiveStage1TasksFile)
    end)
    pcall(function()
        fs.removeFile(configs.outputDirectory .. "/" .. processName .. "/" .. Constants.ActiveStage2TasksFile)
    end)
    CLIO.Print("Done!")
end

return ProcessUtil


--!strict

--[[
    This module handles the creation and management of stage 1 tasks for batch processing.
]]

-- Native Libraries
local process = require('@lune/process')

-- External Libraries
local llc_tasks = require('../../libs/llc_tasks/tasks')

-- Local Libraries
local CLIO = require('./clio')
local Constants = require('../shared/constants')
local FileIO = require('./fileio')
local GuidUtil = require('./guid-util')
local OpenCloudClient = require('./open-cloud-client')
local ProcessUtil = require('./process-util')
local RetryUtil = require('./retry-util')
local Types = require('../shared/types')

--[[
    Stage1Util
]]
local Stage1Util = {}

--[[
    Gets the stage 1 script content.
    @param scanType Types.ScanType - The type of scan to perform
    @return string - The contents of the stage 1 script file
]]
local function getStage1Script(scanType: Types.ScanType): string
    -- Read the correct stage 1 script
    local script: string?
    if scanType == Constants.ScanTypes.Keys then
        script = FileIO.ReadFile('src/scripts/stage1-scan-keys.luau')
    elseif scanType == Constants.ScanTypes.DataStores then
        script = FileIO.ReadFile('src/scripts/stage1-scan-data-stores.luau')
    end
    if not script then
        CLIO.Error('Failed to read stage 1 script. Please try again.')
    end

    return script :: string
end

--[[
    Initializes the process task status.
    @param processName string - The name of the batch process
    @param processTaskStatus Types.ProcessTaskStatus - The process task status
    @param configs Types.AnyConfigs - The configuration settings for the process
    @param openCloudClient OpenCloudClient.OpenCloudClient - The client for interacting with Open Cloud
]]
Stage1Util.InitializeProcessTaskStatus = function(
    processName: string,
    processTaskStatus: Types.ProcessTaskStatus,
    configs: Types.AnyConfigs,
    openCloudClient: OpenCloudClient.OpenCloudClient
): ()
    local createProcessTaskStatusRes = openCloudClient.createSortedMapEntry(
        Constants.BatchProcessTaskStatusMapName,
        processName,
        processTaskStatus,
        nil,
        configs.memoryStoresExpiration
    )
    if createProcessTaskStatusRes.status == 409 then
        -- If the process task status already exists, update it instead
        local updateProcessTaskStatusRes = openCloudClient.updateSortedMapEntry(
            Constants.BatchProcessTaskStatusMapName,
            processName,
            processTaskStatus,
            nil,
            configs.memoryStoresExpiration
        )
        if updateProcessTaskStatusRes.status ~= 200 then
            CLIO.Error("Failed to initialize process task status. Reason: " .. OpenCloudClient.GetBodyAsString(updateProcessTaskStatusRes))
        end
    elseif createProcessTaskStatusRes.status ~= 200 then
        CLIO.Error("Failed to initialize process task status. Reason: " .. OpenCloudClient.GetBodyAsString(createProcessTaskStatusRes))
    end
end

--[[
    Creates a Session Task for stage 1 processing.
    @param scanType Types.ScanType - The type of scan to perform
    @param processName string - The name of the batch process
    @param previousProcessTaskStatus Types.ProcessTaskStatus - The previous process task status
    @param configs Types.AnyConfigs - The configuration settings for the process
    @param openCloudClient OpenCloudClient.OpenCloudClient - The client for interacting with Open Cloud
]]
Stage1Util.CreateSessionTask = function(
    scanType: Types.ScanType,
    processName: string,
    previousProcessTaskStatus: Types.ProcessTaskStatus,
    configs: Types.AnyConfigs,
    openCloudClient: OpenCloudClient.OpenCloudClient
): ()
    -- Load the stage 1 script that contains the scanning logic
    local stage1Script = getStage1Script(scanType)

    -- Create a new task with the script and basic configuration
    local stage1Task = llc_tasks.create(
        stage1Script,
        {
            api_key = process.env['API_KEY'],
            universe_id = configs.universeId,
            place_id = configs.placeId,
        }
    )

    -- Generate a unique identifier for this task
    local guid = GuidUtil.GenerateGuid()

    -- Start the task with the provided configurations
    local success, stage1Res, stage1Error = RetryUtil.RetryAsync(function()
            return stage1Task:start(guid, processName, configs)
        end,
        Constants.SessionTaskRetryConfig.MaxAttempts,
        Constants.SessionTaskRetryConfig.TimeoutBase,
        Constants.SessionTaskRetryConfig.TimeoutExponentialBackoff
    )
    if not success or stage1Error then
        FileIO.AppendToCLILogFile(processName, configs.outputDirectory, "Warning: Failed to start stage 1 task.")
        return
    end

    -- Update the Session Task status map
    local updateProcessTaskStatusRes = openCloudClient.updateSortedMapEntry(
        Constants.BatchProcessTaskStatusMapName,
        processName,
        {
            numInstances = previousProcessTaskStatus.numInstances,
            stage1SessionPaths = { stage1Res.task_create_res.path },
            stage2SessionPaths = previousProcessTaskStatus.stage2SessionPaths,
        },
        nil,
        configs.memoryStoresExpiration
    )
    if updateProcessTaskStatusRes.status == 500 or updateProcessTaskStatusRes.status == 429 then
        FileIO.AppendToCLILogFile(processName, configs.outputDirectory,
            "Error: There was a critical Open Cloud error. Please try again later. Reason: " .. OpenCloudClient.GetBodyAsString(updateProcessTaskStatusRes)
        )
        CLIO.Error("There was a critical Open Cloud error. Please try again later. Reason: " .. OpenCloudClient.GetBodyAsString(updateProcessTaskStatusRes), function()
            ProcessUtil.FinalizeOnExit(processName, configs, openCloudClient)
        end)
    end
    if updateProcessTaskStatusRes.status ~= 200 then
        FileIO.AppendToCLILogFile(processName, configs.outputDirectory,
            "Warning: Failed to update process task status map for " .. processName .. ". Reason: " .. OpenCloudClient.GetBodyAsString(updateProcessTaskStatusRes)
        )
        return
    end

    -- Create a sorted map entry to track this task in Open Cloud
    local createSortedMapEntryRes = openCloudClient.createSortedMapEntry(
        Constants.BatchProcessGuidToSessionPathsMapName,
        guid,
        stage1Res.task_create_res.path,
        nil,
        Constants.GuidToSessionPathsMapExpiration
    )
    if createSortedMapEntryRes.status == 500 or createSortedMapEntryRes.status == 429 then
        FileIO.AppendToCLILogFile(processName, configs.outputDirectory,
            "Error: There was a critical Open Cloud error. Please try again later. Reason: " .. OpenCloudClient.GetBodyAsString(createSortedMapEntryRes)
        )
        CLIO.Error("There was a critical Open Cloud error. Please try again later. Reason: " .. OpenCloudClient.GetBodyAsString(createSortedMapEntryRes), function()
            ProcessUtil.FinalizeOnExit(processName, configs, openCloudClient)
        end)
    end
    if createSortedMapEntryRes.status ~= 200 then
        FileIO.AppendToCLILogFile(processName, configs.outputDirectory,
            "Warning: Failed to create sorted map entry mapping the guid to the stage 1 session path for " .. stage1Res.task_create_res.path ..
            ". Reason: " .. OpenCloudClient.GetBodyAsString(createSortedMapEntryRes)
        )
        return
    end

    -- Record the task in the active tasks file for monitoring
    FileIO.AppendLineToFile(
        configs.outputDirectory .. "/" .. processName .. "/" .. Constants.ActiveStage1TasksFile,
        tostring(stage1Res.task_create_res.path)
    )
end

return Stage1Util


--!strict

--[[
    This module handles the creation and management of stage 2 tasks for batch processing.
]]

-- Native Libraries
local process = require('@lune/process')

-- External Libraries
local llc_tasks = require('../../libs/llc_tasks/tasks')

-- Local Libraries
local CLIO = require('./clio')
local Constants = require('../shared/constants')
local FileIO = require('./fileio')
local GuidUtil = require('./guid-util')
local OpenCloudClient = require('./open-cloud-client')
local ProcessUtil = require('./process-util')
local RetryUtil = require('./retry-util')
local Types = require('../shared/types')

-- Global Variables
local stage2Callback: string? = nil

--[[
    Gets the stage 2 script content and injects the callback function.
    @param callback string - The filepath to the Luau script that will be executed for each item
    @return string - The complete stage 2 script with injected callback
]]
local function getStage2Script(callback: string): string
    -- Read the callback file that contains the user's processing logic
    if stage2Callback == nil then
        local file = FileIO.ReadFile(callback)
        if not file then
            CLIO.Error('Failed to read callback file. Please try again.')
        end
        stage2Callback = file
    end

    -- Create the callback function wrapper
    local callbackWrapper = "local processItemBuilder = function()\n" .. stage2Callback :: string .. "\nend\n"

    -- Read the stage 2 template file that contains the processing framework
    local script = FileIO.ReadFile('src/scripts/stage2.luau')
    if not script then
        CLIO.Error('Failed to read stage 2 script. Please try again.')
    end

    -- Combine the callback wrapper with the stage 2 template
    local combinedScript = callbackWrapper .. "\n" .. script :: string
    return combinedScript
end

--[[
    Stage2Util
]]
local Stage2Util = {}

--[[
    Creates a Session Task for stage 2 processing.
    @param processName string - The name of the batch process
    @param previousProcessTaskStatus Types.ProcessTaskStatus - The previous process task status
    @param configs Types.AnyConfigs - The configuration settings for the process
    @param openCloudClient OpenCloudClient.OpenCloudClient - The client for interacting with Open Cloud
]]
Stage2Util.CreateSessionTask = function(
    processName: string,
    previousProcessTaskStatus: Types.ProcessTaskStatus,
    configs: Types.AnyConfigs,
    openCloudClient: OpenCloudClient.OpenCloudClient
): ()
    -- Load and prepare the stage 2 script with the user's callback
    local stage2Script = getStage2Script(configs.callback)

    -- Create a new task with the script and basic configuration
    local stage2Task = llc_tasks.create(
        stage2Script,
        {
            api_key = process.env['API_KEY'],
            universe_id = configs.universeId,
            place_id = configs.placeId,
        }
    )

    -- Generate a unique identifier for this task
    local guid = GuidUtil.GenerateGuid()

    -- Start the task with the provided configurations
    local success, stage2Res, stage2Error = RetryUtil.RetryAsync(function()
            return stage2Task:start(guid, processName, configs)
        end,
        Constants.SessionTaskRetryConfig.MaxAttempts,
        Constants.SessionTaskRetryConfig.TimeoutBase,
        Constants.SessionTaskRetryConfig.TimeoutExponentialBackoff
    )
    if not success or stage2Error then
        FileIO.AppendToCLILogFile(processName, configs.outputDirectory, "Warning: Failed to start stage 2 task.")
        return
    end

    -- Update the Session Task status map
    local newStage2SessionPaths = previousProcessTaskStatus.stage2SessionPaths
    table.insert(newStage2SessionPaths, stage2Res.task_create_res.path)
    local updateProcessTaskStatusRes = openCloudClient.updateSortedMapEntry(
        Constants.BatchProcessTaskStatusMapName,
        processName,
        {
            numInstances = previousProcessTaskStatus.numInstances,
            stage1SessionPaths = previousProcessTaskStatus.stage1SessionPaths,
            stage2SessionPaths = newStage2SessionPaths,
        },
        nil,
        configs.memoryStoresExpiration
    )
    if updateProcessTaskStatusRes.status == 500 or updateProcessTaskStatusRes.status == 429 then
        FileIO.AppendToCLILogFile(processName, configs.outputDirectory,
            "Error: There was a critical Open Cloud error. Please try again later. Reason: " .. OpenCloudClient.GetBodyAsString(updateProcessTaskStatusRes)
        )
        CLIO.Error("There was a critical Open Cloud error. Please try again later. Reason: " .. OpenCloudClient.GetBodyAsString(updateProcessTaskStatusRes), function()
            ProcessUtil.FinalizeOnExit(processName, configs, openCloudClient)
        end)
    end
    if updateProcessTaskStatusRes.status ~= 200 then
        FileIO.AppendToCLILogFile(processName, configs.outputDirectory,
            "Warning: Failed to update process task status map for " .. processName .. ". Reason: " .. OpenCloudClient.GetBodyAsString(updateProcessTaskStatusRes)
        )
        return
    end

    -- Create a sorted map entry to track this task in Open Cloud
    local createSortedMapEntryRes = openCloudClient.createSortedMapEntry(
        Constants.BatchProcessGuidToSessionPathsMapName,
        guid,
        stage2Res.task_create_res.path,
        nil,
        Constants.GuidToSessionPathsMapExpiration
    )
    if createSortedMapEntryRes.status == 500 or createSortedMapEntryRes.status == 429 then
        FileIO.AppendToCLILogFile(processName, configs.outputDirectory,
            "Error: There was a critical Open Cloud error. Please try again later. Reason: " .. OpenCloudClient.GetBodyAsString(createSortedMapEntryRes)
        )
        CLIO.Error("There was a critical Open Cloud error. Please try again later. Reason: " .. OpenCloudClient.GetBodyAsString(createSortedMapEntryRes), function()
            ProcessUtil.FinalizeOnExit(processName, configs, openCloudClient)
        end)
    end
    if createSortedMapEntryRes.status ~= 200 then
        FileIO.AppendToCLILogFile(processName, configs.outputDirectory,
            "Warning: Failed to create sorted map entry mapping the guid to the stage 2 session path for " .. stage2Res.task_create_res.path ..
            ". Reason: " .. OpenCloudClient.GetBodyAsString(createSortedMapEntryRes)
        )
        return
    end

    -- Record the task in the active tasks file for monitoring
    FileIO.AppendLineToFile(
        configs.outputDirectory .. "/" .. processName .. "/" .. Constants.ActiveStage2TasksFile,
        tostring(stage2Res.task_create_res.path)
    )
end

return Stage2Util


--!strict

--[[
    Formats arguments into fixed-width blocks with separators.
]]

-- Local Libraries
local Constants = require('../shared/constants')

-- TableUtil
local TableUtil = {}

--[[
    Formats a value to fit within a block of specified width.
    If the value is too long, it will be truncated with "...".
    If the value is too short, it will be padded with spaces.
    Each block starts with a space.
    @param value any - The value to format
    @return string - The formatted value
]]
local function formatBlock(value: any): string
    local str = tostring(value)
    local result: string

    if #str > Constants.TableColumnWidth - 1 then
        result = " " .. string.sub(str, 1, Constants.TableColumnWidth - 4) .. "..."
    else
        -- Calculate padding needed to reach exactly width characters total
        local padding = Constants.TableColumnWidth - 1 - #str
        result = " " .. str .. string.rep(" ", padding)
    end

    return result
end

--[[
    Formats a line of values into fixed-width blocks separated by "|".
    @param ...any - The values to format
    @return string - The formatted line
]]
TableUtil.FormatLine = function(...: any): string
    local args = {...}
    local blocks = {}

    for _, value in ipairs(args) do
        table.insert(blocks, formatBlock(value))
    end

    return table.concat(blocks, "|")
end

--[[
    Formats a header line with the names of the columns.
    @param ...string - The column names
    @return string - The formatted header line
]]
TableUtil.FormatHeader = function(...: string): string
    return TableUtil.FormatLine(...)
end

--[[
    Formats a separator line with "-" characters and "+" at intersections.
    @param count number - The number of columns
    @return string - The formatted separator line
]]
TableUtil.FormatSeparator = function(count: number): string
    local blocks = {}
    for _ = 1, count do
        table.insert(blocks, string.rep("-", Constants.TableColumnWidth))
    end
    return table.concat(blocks, "+")
end

return TableUtil


--!strict

--[[
    Time utility for handling Unix timestamps and datetime formatting.
]]

-- Native Libraries
local datetime = require("@lune/datetime")

-- TimeUtil
local TimeUtil = {}

--[[
    Converts a Unix timestamp in milliseconds to a DateTime string.
    @param timestamp number - Unix timestamp in milliseconds
    @return string - The converted DateTime string
]]
TimeUtil.FormatTimestamp = function(timestamp: number): string
    -- Convert milliseconds to seconds for DateTime.fromUnix
    local dt = datetime.fromUnixTimestamp(timestamp)
    return dt:formatLocalTime()
end

return TimeUtil


local ProcessService = game:GetService("ProcessService")
local Root = script.Parent.LuauPolyfillTestModel

local Packages = Root.Packages
-- Load JestRoblox source into Packages folder so it's next to Roact as expected
local TestEZ = require(Packages._Workspace:FindFirstChild("TestEZ", true))

-- Run all tests, collect results, and report to stdout.
local result = TestEZ.TestBootstrap:run({ Packages._Workspace }, TestEZ.Reporters.TextReporterQuiet)

if result.failureCount == 0 and #result.errors == 0 then
	ProcessService:ExitAsync(0)
else
	ProcessService:ExitAsync(1)
end


--!strict
local Packages = script.Parent.Parent
local Number = require(Packages.Number)

-- https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Boolean
return function(val: any): boolean
	return not not val and val ~= 0 and val ~= "" and not Number.isNaN(val)
end


--!strict
local __DEV__ = _G.__DEV__
local Array = script.Parent
local Packages = Array.Parent.Parent
local isArray = require(Array.isArray)
local types = require(Packages.ES7Types)
type Array<T> = types.Array<T>

local RECEIVED_OBJECT_ERROR = "Array.concat(...) only works with array-like tables but "
	.. "it received an object-like table.\nYou can avoid this error by wrapping the "
	.. "object-like table into an array. Example: `concat({1, 2}, {a = true})` should "
	.. "be `concat({1, 2}, { {a = true} }`"

-- capture two separate generic arguments so that the type error in abuse cases is actionable, but needs CLI-49876 to avoid a false negative
local function concat<T, S>(source: Array<T> | T, ...: Array<S> | S): Array<T> & Array<S>
	local array
	local elementCount = 0

	if isArray(source) then
		array = table.clone(source :: Array<T>)
		elementCount = #(source :: Array<T>)
	else
		elementCount += 1
		array = {}
		array[elementCount] = source :: T
	end

	for i = 1, select("#", ...) do
		local value = select(i, ...)
		local valueType = typeof(value)
		if value == nil then
			-- do not insert nil
		elseif valueType == "table" then
			-- deviation: assume that table is an array, to avoid the expensive
			-- `isArray` check. In DEV mode, we throw if it is given an object-like
			-- table.
			if __DEV__ then
				if not isArray(value) then
					error(RECEIVED_OBJECT_ERROR)
				end
			end
			for k = 1, #value do
				elementCount += 1
				array[elementCount] = value[k]
			end
		else
			elementCount += 1
			array[elementCount] = value
		end
	end

	return (array :: any) :: Array<T> & Array<S>
end

return concat


--!strict
local __DEV__ = _G.__DEV__
local Array = script.Parent
local Packages = Array.Parent.Parent
local types = require(Packages.ES7Types)
type Array<T> = types.Array<T>
type Object = types.Object
type callbackFn<T> = (element: T, index: number, array: Array<T>) -> boolean
type callbackFnWithThisArg<T, U> = (self: U, element: T, index: number, array: Array<T>) -> boolean

-- Implements Javascript's `Array.prototype.every` as defined below
-- https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/every
return function<T, U>(t: Array<T>, callback: callbackFn<T> | callbackFnWithThisArg<T, U>, thisArg: U?): boolean
	if __DEV__ then
		if typeof(t) ~= "table" then
			error(string.format("Array.every called on %s", typeof(t)))
		end
		if typeof(callback) ~= "function" then
			error("callback is not a function")
		end
	end

	local len = #t
	local k = 1

	while k <= len do
		local kValue = t[k]
		local testResult

		if kValue ~= nil then
			if thisArg ~= nil then
				testResult = (callback :: callbackFnWithThisArg<T, U>)(thisArg, kValue, k, t)
			else
				testResult = (callback :: callbackFn<T>)(kValue, k, t)
			end

			if not testResult then
				return false
			end
		end
		k += 1
	end
	return true
end


--!strict
local __DEV__ = _G.__DEV__
local Array = script.Parent
local Packages = Array.Parent.Parent
local types = require(Packages.ES7Types)
type Array<T> = types.Array<T>
type Object = types.Object
type callbackFn<T> = (element: T, index: number, array: Array<T>) -> boolean
type callbackFnWithThisArg<T, U> = (thisArg: U, element: T, index: number, array: Array<T>) -> boolean

-- Implements Javascript's `Array.prototype.filter` as defined below
-- https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/filter
return function<T, U>(t: Array<T>, callback: callbackFn<T> | callbackFnWithThisArg<T, U>, thisArg: U?): Array<T>
	if __DEV__ then
		if typeof(t) ~= "table" then
			error(string.format("Array.filter called on %s", typeof(t)))
		end
		if typeof(callback) ~= "function" then
			error("callback is not a function")
		end
	end

	local len = #t
	local res = {}
	local index = 1

	if thisArg == nil then
		for i = 1, len do
			local kValue = t[i]
			if kValue ~= nil then
				if (callback :: callbackFn<T>)(kValue, i, t) then
					res[index] = kValue
					index += 1
				end
			end
		end
	else
		for i = 1, len do
			local kValue = t[i]
			if kValue ~= nil then
				if (callback :: callbackFnWithThisArg<T, U>)(thisArg, kValue, i, t) then
					res[index] = kValue
					index += 1
				end
			end
		end
	end

	return res
end


--!strict
local Array = script.Parent
local Packages = Array.Parent.Parent
local types = require(Packages.ES7Types)
type Array<T> = types.Array<T>
type PredicateFunction<T> = (value: T, index: number, array: Array<T>) -> boolean

return function<T>(array: Array<T>, predicate: PredicateFunction<T>): T | nil
	for i = 1, #array do
		local element = array[i]
		if predicate(element, i, array) then
			return element
		end
	end
	return nil
end


--!strict
local Array = script.Parent
local Packages = Array.Parent.Parent
local types = require(Packages.ES7Types)
type Array<T> = types.Array<T>
type PredicateFunction<T> = (T, number, Array<T>) -> boolean

return function<T>(array: Array<T>, predicate: PredicateFunction<T>): number
	for i = 1, #array do
		local element = array[i]
		if predicate(element, i, array) then
			return i
		end
	end
	return -1
end


local __DEV__ = _G.__DEV__
local Array = script.Parent
local Packages = Array.Parent.Parent
local isArray = require(Array.isArray)
local types = require(Packages.ES7Types)
type Array<T> = types.Array<T>
local function flat<T>(array: Array<T>, depth_: number?): Array<T>
	if __DEV__ then
		if typeof(array) ~= "table" then
			error(string.format("Array.flat called on %s", typeof(array)))
		end
		if depth_ ~= nil and typeof(depth_) ~= "number" then
			error("depth is not a number or nil")
		end
	end
	local depth = depth_ or 1
	local newArray = {}

	for _, v in array do
		if isArray(v) then
			local vArray = (v :: any) :: Array<T>
			local innerArrFlat: Array<T> = if depth > 1 then flat(vArray, depth - 1) else vArray
			for _, innerValue in innerArrFlat do
				table.insert(newArray, innerValue)
			end
		else
			table.insert(newArray, v)
		end
	end

	return newArray
end

return flat


local __DEV__ = _G.__DEV__
local Array = script.Parent
local Packages = Array.Parent.Parent
local flat = require(Array.flat)
local map = require(Array.map)
local types = require(Packages.ES7Types)
type Array<T> = types.Array<T>
type callbackFn<T, U> = (element: T, index: number, array: Array<T>) -> U
type callbackFnWithThisArg<T, U, V> = (thisArg: V, element: T, index: number, array: Array<T>) -> U

local function flatMap<T, U, V>(
	array: Array<T>,
	callback: callbackFn<T, U> | callbackFnWithThisArg<T, U, V>,
	thisArg: V?
): Array<U>
	if __DEV__ then
		if typeof(array) ~= "table" then
			error(string.format("Array.flatMap called on %s", typeof(array)))
		end
		if typeof(callback) ~= "function" then
			error("callback is not a function")
		end
	end
	return flat(map(array, callback, thisArg))
end

return flatMap


--!strict
local __DEV__ = _G.__DEV__
local Array = script.Parent
local Packages = Array.Parent.Parent
local types = require(Packages.ES7Types)
type Array<T> = types.Array<T>
type Object = types.Object
type callbackFn<T> = (element: T, index: number, array: Array<T>) -> ()
type callbackFnWithThisArg<T, U> = (thisArg: U, element: T, index: number, array: Array<T>) -> ()

-- Implements Javascript's `Array.prototype.forEach` as defined below
-- https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/forEach
return function<T, U>(t: Array<T>, callback: callbackFn<T> | callbackFnWithThisArg<T, U>, thisArg: U?): ()
	if __DEV__ then
		if typeof(t) ~= "table" then
			error(string.format("Array.forEach called on %s", typeof(t)))
		end
		if typeof(callback) ~= "function" then
			error("callback is not a function")
		end
	end

	local len = #t
	local k = 1

	while k <= len do
		local kValue = t[k]

		if thisArg ~= nil then
			(callback :: callbackFnWithThisArg<T, U>)(thisArg, kValue, k, t)
		else
			(callback :: callbackFn<T>)(kValue, k, t)
		end

		if #t < len then
			-- don't iterate on removed items, don't iterate more than original length
			len = #t
		end
		k += 1
	end
end