<?php declare(strict_types=1);
/**
 * hcl-catalog-generate.php
 *
 * CLI script (run by cron every 4 hours) that queries the HarpGamer IPS
 * Downloads API and builds catalog.json for consumption by HC Launcher clients.
 *
 * Usage:
 *   php hcl-catalog-generate.php [/path/to/hcl-config.php]
 *
 * If the config path is omitted it defaults to hcl-config.php in the same
 * directory as this script.
 *
 * Exit codes:
 *   0 – success (catalog written)
 *   1 – fatal error (previous catalog left intact, error logged)
 */

// ---------------------------------------------------------------------------
// Bootstrap
// ---------------------------------------------------------------------------

if (php_sapi_name() !== 'cli') {
    http_response_code(403);
    exit('CLI only');
}

// Configuration is loaded in the Main section at the bottom of this file.  A
// test harness may define HCL_GENERATOR_LIB_ONLY before including this file to
// get the functions without running the generator.

// ---------------------------------------------------------------------------
// Options and counters
// ---------------------------------------------------------------------------

/**
 * Process-wide generator options.  Main sets them once from the config; the
 * defaults below apply when a key is absent (and in the test harness).
 *
 * @param  array<string,mixed>|null $set  Values to merge in, or null to read.
 * @return array{
 *   user_agent: string,
 *   inspect_artifact_types: string[],
 *   inspect_max_bytes: int,
 *   meta_retry_seconds: int,
 *   max_lookups_per_run: int,
 *   lookup_delay_ms: int,
 *   max_version_hashes_per_run: int,
 *   companions: array<int, list<array{0:int,1:string}>>
 * }
 */
function hclOptions(?array $set = null): array
{
    static $opts = [
        // IPS refuses downloads to the old 'HCLauncher-CatalogGenerator/1.0'
        // string with error EX1406 (an HTML 500 page), while near-identical
        // strings work.  That page was being recorded as every file's size.
        'user_agent'             => 'HCLauncher-CatalogBuilder/1.1',
        // Categories of these types have each file's archive inspected to
        // find its real artifact type.
        'inspect_artifact_types' => ['Tool'],
        // Archives larger than this are not downloaded for inspection.
        'inspect_max_bytes'      => 50 * 1024 * 1024,
        // A failed lookup becomes due again after this long.  Shorter than the
        // 4-hour cron interval, so every run retries within its budget.
        'meta_retry_seconds'     => 10800,
        // Live lookups (size/filename, or an archive inspection) per run.  The
        // first deploy fired 740 in 2.5 minutes from the server's own IP and
        // only the first ~45 succeeded; a budget spreads the backlog over
        // several runs instead of tripping whatever limit that was.
        'max_lookups_per_run'    => 100,
        // Pause before each live lookup after the first, in milliseconds.
        'lookup_delay_ms'        => 2000,
        // Previous-version files hashed per run.  Each is hashed once, ever
        // (cached by record), but the first run meets the whole history at
        // once; past the cap a version is listed without a hash until a later
        // run gets to it.
        'max_version_hashes_per_run' => 500,
        // Files an entry needs from another entry, which no metadata records:
        // IPS file ID => list of [companion file ID, file it supplies].
        // A database without its own commondb.rsr normally uses the stock one
        // in the game directory and needs nothing.  Only entries that name a
        // specific picture file are listed: "picturefile for HCxxtoxxdb"
        // (328) names René Haar's 65to80, 80to95 and 95toxx databases, and the
        // hc-timelines-db collection (605) asks for it ("add the mediafile").
        'companions' => [
            605 => [[328, 'commondb.rsr']], // hc-timelines-db collection
            257 => [[328, 'commondb.rsr']], // HC_65to80_DB
            276 => [[328, 'commondb.rsr']], // HC_80to95_DB
            597 => [[328, 'commondb.rsr']], // HC_95to10_DB
        ],
    ];
    if ($set !== null) {
        $opts = array_merge($opts, $set);
    }
    return $opts;
}

/**
 * Increment and/or read a named run counter, reported in the summary log line.
 *
 * @param  string|null $key  Counter to increment; null to read all.
 * @param  int         $by
 * @return array<string,int> All counters.
 */
function hclCount(?string $key = null, int $by = 1): array
{
    static $counts = [];
    if ($key !== null) {
        $counts[$key] = ($counts[$key] ?? 0) + $by;
    }
    return $counts;
}

/**
 * Tally a lookup failure reason, and/or read the tally.
 *
 * @param  string|null $reason  e.g. "download: HTTP 500 text/html EX1406"; null to read.
 * @return array<string,int>    Reason => count, most frequent first.
 */
function hclFailureReasons(?string $reason = null): array
{
    static $reasons = [];
    if ($reason !== null) {
        $reasons[$reason] = ($reasons[$reason] ?? 0) + 1;
    }
    arsort($reasons);
    return $reasons;
}

/**
 * Wait lookup_delay_ms before every live lookup except the first of the run,
 * so a run's requests are spread out rather than fired in a burst.
 */
function hclPace(): void
{
    static $first = true;
    if ($first) {
        $first = false;
        return;
    }
    $ms = hclOptions()['lookup_delay_ms'];
    if ($ms > 0) {
        usleep($ms * 1000);
    }
}

// ---------------------------------------------------------------------------
// Logging helper
// ---------------------------------------------------------------------------

/**
 * Append a timestamped line to the generate log.
 *
 * @param string $message  Line text (no trailing newline needed).
 * @param string $logFile  Path to the log file.
 */
function logLine(string $message, string $logFile): void
{
    $line = date('Y-m-d\TH:i:s\Z') . "\t" . $message . "\n";
    file_put_contents($logFile, $line, FILE_APPEND | LOCK_EX);
}

// ---------------------------------------------------------------------------
// HTTP helpers (curl)
//
// Every request is pinned to HTTP/1.1.  Over HTTP/2, harpgamer.com answers a
// Range request with a malformed DATA frame, which libcurl reports as a
// protocol error even though the status is 206 — so the metadata lookup fails
// whenever the local libcurl negotiates HTTP/2.
// ---------------------------------------------------------------------------

/**
 * Perform a GET request and return the decoded JSON body.
 *
 * @param  string               $url      Full URL.
 * @param  array<string,string> $headers  Extra request headers.
 * @return array<mixed>|null    Decoded JSON array, or null on failure.
 */
function httpGetJson(string $url, array $headers = []): ?array
{
    $ch = curl_init($url);
    curl_setopt_array($ch, [
        CURLOPT_HTTP_VERSION   => CURL_HTTP_VERSION_1_1,
        CURLOPT_RETURNTRANSFER => true,
        CURLOPT_FOLLOWLOCATION => true,
        CURLOPT_TIMEOUT        => 30,
        CURLOPT_HTTPHEADER     => buildCurlHeaders($headers),
        CURLOPT_USERAGENT      => hclOptions()['user_agent'],
    ]);
    $body = curl_exec($ch);
    $httpCode = curl_getinfo($ch, CURLINFO_HTTP_CODE);
    $curlError = curl_error($ch);

    if ($curlError !== '' || $httpCode < 200 || $httpCode >= 300) {
        return null;
    }

    $decoded = json_decode((string)$body, true);
    return is_array($decoded) ? $decoded : null;
}

/**
 * Perform a GET request for a raw HTML page and return the body string.
 *
 * @param  string               $url
 * @param  array<string,string> $headers
 * @param  string[]             &$responseCookies  Populated with Set-Cookie values.
 * @param  string|null          &$failure          On failure, why (e.g. "HTTP 403 EX2S100/1").
 * @return string|null
 */
function httpGetHtml(string $url, array $headers = [], array &$responseCookies = [], ?string &$failure = null): ?string
{
    $ch = curl_init($url);
    curl_setopt_array($ch, [
        CURLOPT_HTTP_VERSION   => CURL_HTTP_VERSION_1_1,
        CURLOPT_RETURNTRANSFER => true,
        CURLOPT_FOLLOWLOCATION => true,
        CURLOPT_TIMEOUT        => 30,
        CURLOPT_HEADER         => true,
        CURLOPT_HTTPHEADER     => buildCurlHeaders($headers),
        CURLOPT_USERAGENT      => hclOptions()['user_agent'],
    ]);
    $raw = curl_exec($ch);
    $httpCode    = curl_getinfo($ch, CURLINFO_HTTP_CODE);
    $headerSize  = curl_getinfo($ch, CURLINFO_HEADER_SIZE);
    $curlError   = curl_error($ch);

    $rawStr = (string)$raw;
    $rawHeaders = substr($rawStr, 0, $headerSize);
    $body       = substr($rawStr, $headerSize);

    if ($curlError !== '' || $raw === false) {
        $failure = "curl: {$curlError}";
        return null;
    }
    if ($httpCode < 200 || $httpCode >= 300) {
        $failure = describeResponseFailure($rawHeaders, $body);
        return null;
    }

    // Collect Set-Cookie headers.
    foreach (explode("\r\n", $rawHeaders) as $line) {
        if (stripos($line, 'set-cookie:') === 0) {
            $responseCookies[] = trim(substr($line, strlen('set-cookie:')));
        }
    }

    return $body;
}

/**
 * Issue a Range: bytes=0-0 GET to retrieve filename and total file size from
 * Content-Disposition / Content-Range headers without downloading the full file.
 *
 * @param  string               $url
 * @param  array<string,string> $headers
 * @param  string|null          &$failure  On failure, why (e.g. "HTTP 500 text/html EX1406").
 * @return array{filename:string,size:int}|null
 */
function httpGetFileMeta(string $url, array $headers = [], ?string &$failure = null): ?array
{
    $headerText = '';
    $errorBody  = '';
    $isFile     = null; // decided when the body starts

    $ch = curl_init($url);
    curl_setopt_array($ch, [
        CURLOPT_HTTP_VERSION   => CURL_HTTP_VERSION_1_1,
        CURLOPT_FOLLOWLOCATION => true,
        CURLOPT_TIMEOUT        => 30,
        CURLOPT_RANGE          => '0-0',
        CURLOPT_HTTPHEADER     => buildCurlHeaders($headers),
        CURLOPT_USERAGENT      => hclOptions()['user_agent'],
        CURLOPT_HEADERFUNCTION => static function ($ch, string $line) use (&$headerText): int {
            $headerText .= $line;
            return strlen($line);
        },
        // For the file itself only the headers are needed, so stop as soon as
        // the body starts.  This also sidesteps a server quirk: harpgamer.com's
        // 206 response declares the whole file's Content-Length while sending
        // one byte, so letting curl read the body ends in a "bytes missing"
        // error.  And if a server ignores Range, this avoids downloading the
        // whole file.  An error page, though, is worth keeping (up to 64 KB):
        // it carries the IPS error code that says what went wrong.
        CURLOPT_WRITEFUNCTION  => static function ($ch, string $data) use (&$headerText, &$errorBody, &$isFile): int {
            if ($isFile === null) {
                $isFile = parseFileMetaHeaders($headerText) !== null;
            }
            if ($isFile || strlen($errorBody) >= 65536) {
                return 0;
            }
            $errorBody .= $data;
            return strlen($data);
        },
    ]);
    curl_exec($ch);
    $errno    = curl_errno($ch);
    $curlText = curl_error($ch);

    // Aborting the body is a write error; a short body is a partial-file error.
    // Either is expected here — the headers have already arrived.
    if ($errno !== 0 && $errno !== CURLE_WRITE_ERROR && $errno !== CURLE_PARTIAL_FILE) {
        $failure = "curl: {$curlText}";
        return null;
    }

    $meta = parseFileMetaHeaders($headerText);
    if ($meta === null) {
        $failure = describeResponseFailure($headerText, $errorBody);
    }
    return $meta;
}

/**
 * Summarise why a response was not what was wanted, for the log.
 *
 * @param  string $rawHeaders  Header text, possibly several redirect blocks.
 * @param  string $body        Whatever part of the body was kept (may be empty).
 * @return string  e.g. "HTTP 500 text/html EX1406", "HTTP 429", "HTTP 206 no filename/size".
 */
function describeResponseFailure(string $rawHeaders, string $body): string
{
    $status = 0;
    $type   = '';
    foreach (preg_split("/\r?\n/", $rawHeaders) ?: [] as $line) {
        if (preg_match('#^HTTP/\S+\s+(\d{3})#', $line, $m)) {
            // A new response block: forget the previous hop's headers.
            $status = (int)$m[1];
            $type   = '';
        } elseif (stripos($line, 'content-type:') === 0) {
            $type = strtolower(trim(explode(';', substr($line, 13))[0]));
        }
    }

    $parts = [$status > 0 ? "HTTP {$status}" : 'no response'];
    if ($type !== '') {
        $parts[] = $type;
    }
    // IPS error pages say "Error code: EX1406" (or "2D161/4" and the like).
    $text = strip_tags($body);
    if (preg_match('/Error code:\s*([0-9A-Z]+(?:\/[0-9]+)?)/', $text, $m)) {
        $parts[] = $m[1];
    }
    if (($status === 200 || $status === 206) && !str_contains($type, 'html')) {
        $parts[] = 'no filename/size';
    }
    return implode(' ', $parts);
}

/**
 * Extract filename and total size from the response headers of a download
 * request, rejecting anything that is not actually the file.
 *
 * With redirects followed, curl reports every hop's headers; only the last
 * block describes the response that was received.  That response must be a
 * 200 or 206 and must not be HTML — an IPS error page is a 200/500 text/html
 * response whose Content-Length is the page's size, and accepting it is what
 * once gave every file in the catalog a size of about 65 KB and no name.
 *
 * @param  string $rawHeaders  Raw header text, possibly several blocks.
 * @return array{filename:string,size:int}|null  null when the response is not a file.
 */
function parseFileMetaHeaders(string $rawHeaders): ?array
{
    $blocks = preg_split("/\r?\n\r?\n/", trim($rawHeaders)) ?: [];
    $last   = '';
    foreach ($blocks as $block) {
        if (preg_match('#^HTTP/\S+\s+\d{3}#', ltrim($block))) {
            $last = $block;
        }
    }
    if ($last === '') {
        return null;
    }

    $lines = preg_split("/\r?\n/", ltrim($last)) ?: [];
    if (!preg_match('#^HTTP/\S+\s+(\d{3})#', $lines[0] ?? '', $m)) {
        return null;
    }
    $status = (int)$m[1];
    if ($status !== 200 && $status !== 206) {
        return null;
    }

    $filename    = '';
    $filenameStd = '';
    $size        = 0;
    $length      = 0;

    foreach ($lines as $line) {
        $lower = strtolower($line);

        if (str_starts_with($lower, 'content-type:')
            && str_contains($lower, 'text/html')) {
            return null;
        }

        // Content-Disposition: attachment; filename="GIUK.zip"; filename*=UTF-8''GIUK.zip
        if (str_starts_with($lower, 'content-disposition:')) {
            if (preg_match("/filename\\*\\s*=\\s*[^']*'[^']*'([^;\\s]+)/i", $line, $m)) {
                $filenameStd = rawurldecode($m[1]);
            }
            if (preg_match('/(?:^|[;\s])filename\s*=\s*("([^"]*)"|[^;\s]+)/i', $line, $m)) {
                $filename = ($m[2] ?? '') !== '' ? $m[2] : trim($m[1], '"');
            }
        }

        // Content-Range: bytes 0-0/123456  (total size after the slash)
        if (str_starts_with($lower, 'content-range:')
            && preg_match('#/(\d+)#', $line, $m)) {
            $size = (int)$m[1];
        }

        if (str_starts_with($lower, 'content-length:')) {
            $length = (int)trim(substr($line, strlen('content-length:')));
        }
    }

    // Content-Length is the whole file only when the server ignored the Range.
    if ($size === 0 && $status === 200) {
        $size = $length;
    }

    if ($filenameStd !== '') {
        $filename = $filenameStd;
    }
    $filename = basename(str_replace('\\', '/', $filename));

    if ($filename === '' && $size === 0) {
        return null;
    }

    return ['filename' => $filename, 'size' => $size];
}

/**
 * Stream a download to a local file, refusing HTML responses and anything
 * larger than the given limit.
 *
 * @param  string               $url
 * @param  array<string,string> $headers
 * @param  string               $destPath  Written only on success; removed on failure.
 * @param  int                  $maxBytes
 * @return bool
 */
function httpDownloadToFile(string $url, array $headers, string $destPath, int $maxBytes): bool
{
    $fp = fopen($destPath, 'wb');
    if ($fp === false) {
        return false;
    }

    $written = 0;

    $ch = curl_init($url);
    curl_setopt_array($ch, [
        CURLOPT_HTTP_VERSION   => CURL_HTTP_VERSION_1_1,
        CURLOPT_FOLLOWLOCATION => true,
        CURLOPT_TIMEOUT        => 300,
        CURLOPT_HTTPHEADER     => buildCurlHeaders($headers),
        CURLOPT_USERAGENT      => hclOptions()['user_agent'],
        CURLOPT_MAXFILESIZE    => $maxBytes,
        // MAXFILESIZE only applies when the server declares a length, so the
        // writer enforces the limit too.  (A write callback rather than a
        // progress callback: CURLOPT_XFERINFOFUNCTION needs PHP 8.2.)
        CURLOPT_WRITEFUNCTION  => static function ($ch, string $data) use ($fp, &$written, $maxBytes): int {
            $written += strlen($data);
            if ($written > $maxBytes) {
                return 0; // abort
            }
            return fwrite($fp, $data) === strlen($data) ? strlen($data) : 0;
        },
    ]);
    $ok          = curl_exec($ch) !== false;
    $httpCode    = (int)curl_getinfo($ch, CURLINFO_HTTP_CODE);
    $contentType = strtolower((string)curl_getinfo($ch, CURLINFO_CONTENT_TYPE));
    fclose($fp);

    if (!$ok || $httpCode !== 200 || str_contains($contentType, 'text/html')) {
        @unlink($destPath);
        return false;
    }
    return true;
}

/**
 * Convert an associative header array to the format curl expects.
 *
 * @param  array<string,string> $headers
 * @return string[]
 */
function buildCurlHeaders(array $headers): array
{
    $out = [];
    foreach ($headers as $name => $value) {
        $out[] = "{$name}: {$value}";
    }
    return $out;
}

/**
 * Extract the IPS csrfKey value from a page's HTML.
 *
 * IPS embeds it in a <meta> tag:
 *   <meta name="csrf-token" content="abc123">
 * or in JavaScript: csrfKey: "abc123"
 *
 * @param  string $html
 * @return string  Empty string if not found.
 */
function extractCsrfKey(string $html): string
{
    // Try <meta name="csrf-token" content="...">
    if (preg_match('/<meta[^>]+name=["\']csrf-token["\'][^>]+content=["\']([^"\']+)["\']/', $html, $m)) {
        return $m[1];
    }
    // Try JS variable: csrfKey: "..."
    if (preg_match('/csrfKey["\']?\s*:\s*["\']([^"\']+)["\']/', $html, $m)) {
        return $m[1];
    }
    return '';
}

/**
 * Build a Cookie header string from an array of Set-Cookie values,
 * keeping only the name=value pair from each.
 *
 * @param  string[] $setCookies
 * @return string   e.g. "session_id=abc; other=xyz"
 */
function buildCookieHeader(array $setCookies): string
{
    $pairs = [];
    foreach ($setCookies as $raw) {
        // Each Set-Cookie value: "name=value; Path=/; ..."
        $pair = strtok($raw, ';');
        if ($pair !== false) {
            $pairs[] = trim($pair);
        }
    }
    return implode('; ', $pairs);
}

// ---------------------------------------------------------------------------
// Category helpers
// ---------------------------------------------------------------------------

/**
 * Fetch the full IPS category list and return it as a flat array keyed by ID.
 *
 * @param  string            $apiBase
 * @param  string            $apiKey
 * @return array<int,array<string,mixed>>|null  null on API failure.
 */
function fetchAllCategories(string $apiBase, string $apiKey): ?array
{
    $url  = "{$apiBase}/downloads/categories?key=" . urlencode($apiKey);
    $data = httpGetJson($url);
    if ($data === null) {
        return null;
    }

    // IPS returns { "results": [...], "totalResults": N }
    $results = $data['results'] ?? $data;
    if (!is_array($results)) {
        return null;
    }

    $map = [];
    foreach ($results as $cat) {
        if (isset($cat['id'])) {
            $map[(int)$cat['id']] = $cat;
        }
    }
    return $map;
}

/**
 * Determine whether a category is within the HC subtree.
 *
 * @param  int                           $categoryId
 * @param  int                           $hcParentId
 * @param  array<int,array<string,mixed>> $allCategories
 * @return bool
 */
function isHCCategory(int $categoryId, int $hcParentId, array $allCategories): bool
{
    $visited = [];
    $id = $categoryId;
    while ($id !== 0) {
        if ($id === $hcParentId) {
            return true;
        }
        if (isset($visited[$id])) {
            break; // cycle guard
        }
        $visited[$id] = true;
        $parentId = (int)(($allCategories[$id] ?? [])['parentId'] ?? 0);
        if ($parentId === $id) {
            break; // self-parent guard
        }
        $id = $parentId;
    }
    return false;
}

/**
 * Determine the artifact type for a category.
 *
 * @param  int                           $categoryId
 * @param  array<int,string>             $artifactMap
 * @param  array<int,array<string,mixed>> $allCategories
 * @return string
 */
function categoryArtifactType(int $categoryId, array $artifactMap, array $allCategories): string
{
    // Direct mapping.
    if (isset($artifactMap[$categoryId])) {
        return $artifactMap[$categoryId];
    }

    // Inherit from parent (handles scenario sub-categories, etc.).
    $parentId = (int)(($allCategories[$categoryId] ?? [])['parentId'] ?? 0);
    if ($parentId !== 0 && $parentId !== $categoryId && isset($artifactMap[$parentId])) {
        return $artifactMap[$parentId];
    }

    return 'Unknown';
}

// ---------------------------------------------------------------------------
// File metadata (size, filename, refined artifact type) with local cache
// ---------------------------------------------------------------------------

/**
 * Cache entry format version.  Entries written by an older format are treated
 * as missing.  Version 1 entries hold the sizes of IPS error pages rather than
 * of the files, so they must never be served again.
 */
const HCL_META_CACHE_VERSION = 2;

/**
 * Return cached file metadata if it is current.
 *
 * An entry is current when it has the current format version, its 'updated'
 * timestamp matches the file's, and — if the lookup failed — the retry
 * interval has not yet elapsed.
 *
 * @param  int    $fileId
 * @param  string $updated    ISO date string from the API.
 * @param  string $cacheDir
 * @param  int    $now        Current Unix time (a parameter so tests can control it).
 * @return array{filename:string,size:int,ok:bool,artifactType:?string}|null
 */
function getCachedFileMeta(int $fileId, string $updated, string $cacheDir, int $now): ?array
{
    $path = "{$cacheDir}/{$fileId}.json";
    if (!is_file($path)) {
        return null;
    }
    $cached = json_decode(file_get_contents($path) ?: '', true);
    if (!is_array($cached)) {
        return null;
    }
    if ((int)($cached['v'] ?? 1) !== HCL_META_CACHE_VERSION) {
        return null; // older format
    }
    if (($cached['updated'] ?? '') !== $updated) {
        return null; // file changed since
    }

    $ok = (bool)($cached['ok'] ?? false);
    if (!$ok && ($now - (int)($cached['checked'] ?? 0)) >= hclOptions()['meta_retry_seconds']) {
        return null; // failed last time; worth another try
    }

    $type = $cached['artifactType'] ?? null;
    return [
        'filename'     => (string)($cached['filename'] ?? ''),
        'size'         => (int)($cached['size'] ?? 0),
        'ok'           => $ok,
        'artifactType' => $type === null ? null : (string)$type,
        'reason'       => isset($cached['reason']) ? (string)$cached['reason'] : null,
        'record_id'    => isset($cached['record_id']) ? (int)$cached['record_id'] : null,
        'sha256'       => isset($cached['sha256']) ? (string)$cached['sha256'] : null,
    ];
}

/**
 * What a file needs from this run, judged from its cache entry alone.
 *
 * @param  int    $fileId
 * @param  string $updated
 * @param  string $cacheDir
 * @param  int    $now
 * @param  bool   $inspect  True when the file's category is one whose archives are inspected.
 * @return string  'fresh'           — nothing to do;
 *                 'need_inspect'    — metadata is good but the archive hasn't been inspected;
 *                 'need_meta_new'   — never looked up in the current format (or the file changed);
 *                 'need_meta_retry' — a previous lookup failed and is due again.
 */
function cacheState(int $fileId, string $updated, string $cacheDir, int $now, bool $inspect): string
{
    $meta = getCachedFileMeta($fileId, $updated, $cacheDir, $now);
    if ($meta !== null) {
        return ($inspect && $meta['ok'] && $meta['artifactType'] === null) ? 'need_inspect' : 'fresh';
    }

    $path   = "{$cacheDir}/{$fileId}.json";
    $cached = is_file($path) ? json_decode(file_get_contents($path) ?: '', true) : null;
    $sameVersion = is_array($cached)
        && (int)($cached['v'] ?? 1) === HCL_META_CACHE_VERSION
        && ($cached['updated'] ?? '') === $updated;
    return $sameVersion ? 'need_meta_retry' : 'need_meta_new';
}

/**
 * Persist file metadata to the per-file cache.
 *
 * @param  int    $fileId
 * @param  string $updated
 * @param  array{filename:string,size:int,ok:bool,artifactType:?string,reason?:?string,
 *               record_id?:?int,sha256?:?string} $meta
 * @param  string $cacheDir
 * @param  int    $now
 */
function saveFileMeta(int $fileId, string $updated, array $meta, string $cacheDir, int $now): void
{
    $path = "{$cacheDir}/{$fileId}.json";
    $data = json_encode([
        'v'            => HCL_META_CACHE_VERSION,
        'updated'      => $updated,
        'checked'      => $now,
        'ok'           => $meta['ok'],
        'filename'     => $meta['filename'],
        'size'         => $meta['size'],
        // null = not inspected; '' = inspected, nothing more specific found.
        'artifactType' => $meta['artifactType'],
        // Why the last lookup failed, for diagnosis; null on success.
        'reason'       => $meta['reason'] ?? null,
        // IPS upload record the entry describes, when it came from the
        // database; a different record means a new file, so hash and
        // inspection are redone.
        'record_id'    => $meta['record_id'] ?? null,
        // SHA-256 of the stored file, when it could be read from disk.
        'sha256'       => $meta['sha256'] ?? null,
    ]);
    file_put_contents($path, $data, LOCK_EX);
}

/**
 * Open a guest IPS session on a file page and build the matching download
 * request.
 *
 * The csrfKey is bound to the session cookie, so the two must travel together.
 *
 * @param  string $pageUrl  The IPS file page URL (e.g. .../files/file/123-name/).
 * @return array{url:string,headers:array<string,string>}|null
 */
function ipsDownloadRequest(string $pageUrl, ?string &$failure = null): ?array
{
    $cookies = [];
    $why  = null;
    $html = httpGetHtml($pageUrl, [], $cookies, $why);
    if ($html === null) {
        $failure = "page: {$why}";
        return null;
    }

    $csrfKey = extractCsrfKey($html);
    if ($csrfKey === '') {
        $failure = 'page: no csrfKey';
        return null;
    }

    $headers = [];
    $cookieHeader = buildCookieHeader($cookies);
    if ($cookieHeader !== '') {
        $headers['Cookie'] = $cookieHeader;
    }

    return [
        'url'     => rtrim($pageUrl, '/') . '/?do=download&csrfKey=' . urlencode($csrfKey),
        'headers' => $headers,
    ];
}

/**
 * Fetch file size and filename from the download URL via a Range request.
 *
 * @param  string $pageUrl  The IPS file page URL.
 * @return array{filename:string,size:int,ok:bool,artifactType:?string,reason:?string}
 *         'ok' is false on any failure (non-fatal), with 'reason' saying which
 *         step failed and how, e.g. "download: HTTP 500 text/html EX1406".
 */
function fetchFileMeta(string $pageUrl): array
{
    $why     = null;
    $request = ipsDownloadRequest($pageUrl, $why);
    if ($request === null) {
        return ['filename' => '', 'size' => 0, 'ok' => false, 'artifactType' => null, 'reason' => $why];
    }

    $meta = httpGetFileMeta($request['url'], $request['headers'], $why);
    if ($meta === null) {
        return ['filename' => '', 'size' => 0, 'ok' => false, 'artifactType' => null, 'reason' => "download: {$why}"];
    }

    return ['filename' => $meta['filename'], 'size' => $meta['size'], 'ok' => true, 'artifactType' => null, 'reason' => null];
}

// ---------------------------------------------------------------------------
// Artifact-type inspection
// ---------------------------------------------------------------------------

/**
 * Classify an artifact from the names of the files it contains.
 *
 * Rules, first match wins:
 *   any .exe          → Tool       (an application, even if it ships DLLs)
 *   any .dll          → ExportDLL
 *   any .tif / .tiff  → Map        (HC uses TIFF only for BSBuilder maps;
 *                                   map sets also carry a .bmp thumbnail)
 *   any .wav          → Sound
 *   any .bmp          → Graphic
 *   otherwise         → ''         (nothing more specific than the category)
 *
 * Directory entries and macOS resource-fork debris (__MACOSX/, ._*) are ignored.
 *
 * @param  string[] $names  Entry names (or a single bare filename).
 * @return string  Artifact type, or '' for no refinement.
 */
function classifyArchiveEntries(array $names): string
{
    $exts = [];
    foreach ($names as $name) {
        $name = str_replace('\\', '/', (string)$name);
        if ($name === '' || str_ends_with($name, '/')) {
            continue;
        }
        $base = basename($name);
        if (str_starts_with($name, '__MACOSX/') || str_starts_with($base, '._')) {
            continue;
        }
        $exts[strtolower(pathinfo($base, PATHINFO_EXTENSION))] = true;
    }

    $rules = [
        'exe'  => 'Tool',
        'dll'  => 'ExportDLL',
        'tif'  => 'Map',
        'tiff' => 'Map',
        'wav'  => 'Sound',
        'bmp'  => 'Graphic',
    ];
    foreach ($rules as $ext => $type) {
        if (isset($exts[$ext])) {
            return $type;
        }
    }
    return '';
}

/**
 * List the entry names in a ZIP file.
 *
 * @param  string $path
 * @return string[]|null  null when the file is not a readable ZIP or the zip
 *                        extension is unavailable.
 */
function listZipEntries(string $path): ?array
{
    if (!class_exists('ZipArchive')) {
        return null;
    }
    $zip = new ZipArchive();
    if ($zip->open($path, ZipArchive::RDONLY) !== true) {
        return null;
    }
    $names = [];
    for ($i = 0; $i < $zip->numFiles; $i++) {
        $name = $zip->getNameIndex($i);
        if ($name !== false) {
            $names[] = $name;
        }
    }
    $zip->close();
    return $names;
}

/**
 * Work out a file's real artifact type by looking inside its archive.
 *
 * @param  string $pageUrl
 * @param  array{filename:string,size:int} $meta  Result of a successful fetchFileMeta().
 * @return string|null  The type ('' when nothing more specific was found), or
 *                      null on a transient failure worth retrying next run.
 */
function inspectArtifactType(string $pageUrl, array $meta): ?string
{
    $filename = $meta['filename'];

    // Not an archive: the file's own name is all there is to go on.
    if (!str_ends_with(strtolower($filename), '.zip')) {
        return classifyArchiveEntries([$filename]);
    }

    $maxBytes = hclOptions()['inspect_max_bytes'];
    if ($meta['size'] <= 0 || $meta['size'] > $maxBytes) {
        hclCount('inspect_skipped_size');
        return '';
    }

    if (!class_exists('ZipArchive')) {
        hclCount('inspect_no_zip_extension');
        return null;
    }

    $why     = null;
    $request = ipsDownloadRequest($pageUrl, $why);
    if ($request === null) {
        hclCount('inspect_failed');
        hclFailureReasons("inspect {$why}");
        return null;
    }

    $tmp = tempnam(sys_get_temp_dir(), 'hcl');
    if ($tmp === false) {
        return null;
    }
    try {
        if (!httpDownloadToFile($request['url'], $request['headers'], $tmp, $maxBytes)) {
            hclCount('inspect_failed');
            hclFailureReasons('inspect download: failed');
            return null;
        }
        $names = listZipEntries($tmp);
        if ($names === null) {
            return ''; // named .zip but not one: nothing to learn, don't retry
        }
        hclCount('inspected');
        return classifyArchiveEntries($names);
    } finally {
        @unlink($tmp);
    }
}

// ---------------------------------------------------------------------------
// IPS database source
//
// The generator runs on the same host as IPS, so a read-only database user
// can supply what the guest download flow otherwise has to scrape: each
// file's name and size from ibf_downloads_files_records, with no HTTP request,
// no guest throttling and nothing added to the public download counts.  IPS
// keeps Downloads files on local disk (the FileSystem storage method), so
// hashes and archive listings can be read straight from the stored files too.
//
// Only files the REST API listed are looked up, so visibility and approval
// still come from IPS's own public view; the database only fills in details.
// Record semantics follow IPS's own queries (applications/downloads/sources/
// File/File.php): record_type 'upload' = a file ('link' = external URL,
// 'ssupload'/'sslink' = screenshots); record_backup = 1 marks a superseded
// version's file.
// ---------------------------------------------------------------------------

/**
 * Open a read-only connection to the IPS database.
 *
 * @param  array{host?:string,name?:string,user?:string,pass?:string} $db  The config's ips_db block.
 * @param  string|null &$failure  Why no connection was made; never contains the password.
 * @return (callable(string, array): array)|null  A query function taking SQL with ? placeholders
 *         and its parameters and returning the rows as associative arrays; null when unavailable.
 */
function ipsDbOpen(array $db, ?string &$failure = null): ?callable
{
    if (!class_exists('mysqli')) {
        $failure = 'mysqli extension not available';
        return null;
    }
    foreach (['host', 'name', 'user', 'pass'] as $key) {
        if ((string)($db[$key] ?? '') === '' || $db[$key] === 'CHANGE_ME') {
            $failure = "ips_db.{$key} is not set";
            return null;
        }
    }

    try {
        mysqli_report(MYSQLI_REPORT_ERROR | MYSQLI_REPORT_STRICT); // errors as exceptions
        $conn = new mysqli((string)$db['host'], (string)$db['user'], (string)$db['pass'], (string)$db['name']);
        $conn->set_charset('utf8mb4');
    } catch (Throwable $e) {
        $failure = 'connect: ' . $e->getMessage(); // mysqli never includes the password
        return null;
    }

    return static function (string $sql, array $params = []) use ($conn): array {
        $stmt = $conn->prepare($sql);
        if ($params) {
            $types = '';
            foreach ($params as $p) {
                $types .= is_int($p) ? 'i' : 's';
            }
            $stmt->bind_param($types, ...$params);
        }
        $stmt->execute();
        $result = $stmt->get_result();
        $rows   = $result ? $result->fetch_all(MYSQLI_ASSOC) : [];
        $stmt->close();
        return $rows;
    };
}

/**
 * Find the directory IPS stores Downloads files in, from its own settings.
 *
 * @param  callable $q        Query function from ipsDbOpen().
 * @param  string   $prefix   IPS table prefix.
 * @param  string   $ipsRoot  IPS install directory, substituted for {root}.
 * @param  string|null &$failure  Why no directory could be determined.
 * @return string|null  Absolute directory without a trailing slash, or null.
 */
function ipsDownloadsStorageDir(callable $q, string $prefix, string $ipsRoot, ?string &$failure = null): ?string
{
    $rows = $q("SELECT conf_value FROM {$prefix}core_sys_conf_settings WHERE conf_key = ?", ['upload_settings']);
    $settings = json_decode((string)($rows[0]['conf_value'] ?? ''), true);
    $storageId = is_array($settings) ? ($settings['filestorage__downloads_Files'] ?? null) : null;
    if ($storageId === null) {
        $failure = 'upload_settings has no filestorage__downloads_Files entry';
        return null;
    }

    $rows = $q("SELECT method, configuration FROM {$prefix}core_file_storage WHERE id = ?", [(int)$storageId]);
    if (!$rows) {
        $failure = "file storage configuration {$storageId} not found";
        return null;
    }
    if ((string)$rows[0]['method'] !== 'FileSystem') {
        $failure = "Downloads storage method is {$rows[0]['method']}, not FileSystem";
        return null;
    }

    $conf = json_decode((string)$rows[0]['configuration'], true);
    $dir  = is_array($conf) ? (string)($conf['dir'] ?? '') : '';
    if ($dir === '') {
        $failure = "file storage configuration {$storageId} has no dir";
        return null;
    }
    return rtrim(str_replace('{root}', rtrim($ipsRoot, '/\\'), $dir), '/\\');
}

/**
 * Fetch the current upload records for the given files.
 *
 * @param  callable $q        Query function from ipsDbOpen().
 * @param  string   $prefix   IPS table prefix.
 * @param  int[]    $fileIds  IPS file IDs, as listed by the REST API.
 * @return array<int, list<array<string,mixed>>>  File ID => its records, oldest first.
 *         Files with no current upload (external links, or none at all) are absent.
 */
function ipsFetchUploadRecords(callable $q, string $prefix, array $fileIds): array
{
    $byFile = [];
    $ids    = array_values(array_unique(array_map('intval', $fileIds)));
    foreach (array_chunk($ids, 200) as $chunk) {
        $in   = implode(',', array_fill(0, count($chunk), '?'));
        $rows = $q("SELECT record_id, record_file_id, record_realname, record_size, record_location, record_default"
                 . " FROM {$prefix}downloads_files_records"
                 . " WHERE record_type = 'upload' AND record_backup = 0 AND record_hidden = 0"
                 . " AND record_file_id IN ({$in})"
                 . " ORDER BY record_file_id, record_id", $chunk);
        foreach ($rows as $row) {
            $byFile[(int)$row['record_file_id']][] = $row;
        }
    }
    return $byFile;
}

/**
 * Pick the record that represents a file in the catalog, which has room for
 * one filename per entry.
 *
 * @param  list<array<string,mixed>> $records  A file's current upload records, oldest first.
 * @return array<string,mixed>|null  The record flagged as default, else the first .zip,
 *         else the oldest; null when there are none.
 */
function choosePrimaryRecord(array $records): ?array
{
    if (!$records) {
        return null;
    }
    foreach ($records as $r) {
        if ((int)($r['record_default'] ?? 0) === 1) {
            return $r;
        }
    }
    foreach ($records as $r) {
        if (str_ends_with(strtolower((string)$r['record_realname']), '.zip')) {
            return $r;
        }
    }
    return $records[0];
}

/**
 * Resolve a record's stored file on disk, refusing anything outside the
 * storage directory.
 *
 * @param  string $storageDir  From ipsDownloadsStorageDir() (or the ips_downloads_dir override).
 * @param  string $location    record_location, e.g. "monthly_2023_03/Foo_zip.<hash>".
 * @param  string|null &$why   Why no path was returned, e.g. "missing on disk".
 * @return string|null  Readable absolute path, or null.
 */
function resolveStoredFile(string $storageDir, string $location, ?string &$why = null): ?string
{
    if ($storageDir === '') {
        $why = 'storage directory unknown';
        return null;
    }
    if ($location === '') {
        $why = 'record has no location';
        return null;
    }
    $base = realpath($storageDir);
    if ($base === false) {
        $why = 'storage directory missing';
        return null;
    }
    $candidate = $storageDir . '/' . ltrim($location, '/\\');
    $real = realpath($candidate);
    if ($real === false) {
        $why = 'missing on disk';
        return null;
    }
    if (!str_starts_with($real, $base . DIRECTORY_SEPARATOR)) {
        $why = 'outside the storage directory';
        return null;
    }
    if (!is_file($real)) {
        $why = 'not a file';
        return null;
    }
    if (!is_readable($real)) {
        $why = 'not readable';
        return null;
    }
    return $real;
}

/**
 * Collect notes of a given kind for the log (e.g. which stored files could
 * not be read), and/or read them back.
 *
 * @param  string      $kind
 * @param  string|null $note  Note to add; null to read.
 * @return string[]    All notes of that kind, in order added.
 */
function hclNote(string $kind, ?string $note = null): array
{
    static $notes = [];
    if ($note !== null) {
        $notes[$kind][] = $note;
    }
    return $notes[$kind] ?? [];
}

/**
 * Fetch a summary of every record, of any type or state, for files that
 * have no current upload - so the log can say why they have no file.
 *
 * @param  callable $q
 * @param  string   $prefix
 * @param  int[]    $fileIds
 * @return array<int, list<array{record_type:string,record_backup:int,record_hidden:int}>>
 */
function ipsFetchRecordSummary(callable $q, string $prefix, array $fileIds): array
{
    $byFile = [];
    $ids    = array_values(array_unique(array_map('intval', $fileIds)));
    foreach (array_chunk($ids, 200) as $chunk) {
        $in   = implode(',', array_fill(0, count($chunk), '?'));
        $rows = $q("SELECT record_file_id, record_type, record_backup, record_hidden"
                 . " FROM {$prefix}downloads_files_records WHERE record_file_id IN ({$in})", $chunk);
        foreach ($rows as $row) {
            $byFile[(int)$row['record_file_id']][] = [
                'record_type'   => (string)$row['record_type'],
                'record_backup' => (int)$row['record_backup'],
                'record_hidden' => (int)$row['record_hidden'],
            ];
        }
    }
    return $byFile;
}

/**
 * Say why a listed file has no current upload record.
 *
 * @param  list<array{record_type:string,record_backup:int,record_hidden:int}> $records
 *         All its records (ipsFetchRecordSummary()).
 * @return string  'external_link'     - its current download is a link to another site;
 *                 'no_current_upload' - files or links exist, but only superseded or pending ones;
 *                 'no_records'        - no file records at all (screenshots at most).
 */
function classifyNoUploadRecords(array $records): string
{
    foreach ($records as $r) {
        if ($r['record_type'] === 'link' && $r['record_backup'] === 0 && $r['record_hidden'] === 0) {
            return 'external_link';
        }
    }
    foreach ($records as $r) {
        if ($r['record_type'] === 'upload' || $r['record_type'] === 'link') {
            return 'no_current_upload';
        }
    }
    return 'no_records';
}

/**
 * Fetch the previous versions IPS keeps for the given files, with each
 * version's upload records.
 *
 * Follows IPS's own view: hidden versions (b_hidden = 1) are left out, as
 * guests cannot see them.  A version's files are the records listed in its
 * b_records; they carry record_backup = 1 once superseded.
 *
 * @param  callable $q        Query function from ipsDbOpen().
 * @param  string   $prefix   IPS table prefix.
 * @param  int[]    $fileIds  IPS file IDs, as listed by the REST API.
 * @return array<int, list<array{b_id:int,version:string,changelog:string,updated:int,backup:int,
 *         records:list<array<string,mixed>>}>>  File ID => its previous versions, newest first.
 *         Files without any are absent; a version whose files are all gone has no records.
 */
function ipsFetchVersions(callable $q, string $prefix, array $fileIds): array
{
    $byFile    = [];
    $recordIds = [];
    $ids       = array_values(array_unique(array_map('intval', $fileIds)));
    foreach (array_chunk($ids, 200) as $chunk) {
        $in   = implode(',', array_fill(0, count($chunk), '?'));
        $rows = $q("SELECT b_id, b_fileid, b_version, b_changelog, b_updated, b_backup, b_records"
                 . " FROM {$prefix}downloads_filebackup"
                 . " WHERE b_hidden = 0 AND b_fileid IN ({$in})"
                 . " ORDER BY b_fileid, b_backup DESC, b_id DESC", $chunk);
        foreach ($rows as $row) {
            $ownIds = array_values(array_filter(array_map('intval', explode(',', (string)$row['b_records']))));
            array_push($recordIds, ...$ownIds);
            $byFile[(int)$row['b_fileid']][] = [
                'b_id'      => (int)$row['b_id'],
                'version'   => trim((string)$row['b_version']),
                'changelog' => (string)$row['b_changelog'],
                'updated'   => (int)$row['b_updated'],
                'backup'    => (int)$row['b_backup'],
                'recordIds' => $ownIds,
            ];
        }
    }

    // The versions' upload records, in one pass.
    $records = [];
    foreach (array_chunk(array_values(array_unique($recordIds)), 200) as $chunk) {
        $in   = implode(',', array_fill(0, count($chunk), '?'));
        $rows = $q("SELECT record_id, record_file_id, record_realname, record_size, record_location,"
                 . " record_default, record_time"
                 . " FROM {$prefix}downloads_files_records"
                 . " WHERE record_type = 'upload' AND record_hidden = 0 AND record_id IN ({$in})", $chunk);
        foreach ($rows as $row) {
            $records[(int)$row['record_id']] = $row;
        }
    }

    foreach ($byFile as $fileId => &$versions) {
        foreach ($versions as &$v) {
            $own = [];
            foreach ($v['recordIds'] as $rid) {
                // A record belongs to its own file only; b_records is trusted no further.
                if (isset($records[$rid]) && (int)$records[$rid]['record_file_id'] === $fileId) {
                    $own[] = $records[$rid];
                }
            }
            usort($own, fn($a, $b) => (int)$a['record_id'] <=> (int)$b['record_id']);
            $v['records'] = $own;
            unset($v['recordIds']);
        }
        unset($v);
    }
    unset($versions);
    return $byFile;
}

/**
 * A short, human label for one version of a file.
 *
 * Authors rarely fill in IPS's version field, but HC files usually carry
 * their version in the filename (HCDB2-170611.zip), so that stands in.
 *
 * @param  string $version   IPS's version field; may be ''.
 * @param  string $fileName  The version's main file, e.g. "HCDB2-170611.zip"; may be ''.
 * @return string  The filename without its extension when the version field is empty or
 *         the filename already contains it ("161204" in "HCDB2-161204.zip" names the same
 *         build, less completely); otherwise the version field; '' when neither is known.
 */
function versionLabel(string $version, string $fileName): string
{
    $version = trim($version);
    $dot     = strrpos($fileName, '.');
    $stem    = trim($dot === false || $dot === 0 ? $fileName : substr($fileName, 0, $dot));
    if ($version === '' || ($stem !== '' && stripos($stem, $version) !== false)) {
        return $stem !== '' ? $stem : $version;
    }
    return $version;
}

/**
 * SHA-256 of a stored record, cached by record ID.  A record's file never
 * changes once written (a new version gets new records), so each is hashed
 * once, ever.
 *
 * @param  string $cacheDir
 * @param  int    $recordId
 * @param  string $path       Readable stored file (resolveStoredFile()).
 * @param  string $location   record_location; a cached hash is only reused for the same one.
 * @return string  Hex digest, or '' when the file could not be hashed or this
 *         run's max_version_hashes_per_run is spent (counted as version_hash_deferred).
 */
function recordSha256(string $cacheDir, int $recordId, string $path, string $location): string
{
    $dir  = $cacheDir . '/records';
    $file = "{$dir}/{$recordId}.json";
    if (is_file($file)) {
        $entry = json_decode((string)file_get_contents($file), true);
        if (is_array($entry) && ($entry['location'] ?? null) === $location
            && is_string($entry['sha256'] ?? null) && strlen($entry['sha256']) === 64) {
            return $entry['sha256'];
        }
    }
    if ((hclCount()['version_sha256_computed'] ?? 0) >= hclOptions()['max_version_hashes_per_run']) {
        hclCount('version_hash_deferred');
        return '';
    }
    $hash = hash_file('sha256', $path);
    if ($hash === false) {
        return '';
    }
    hclCount('version_sha256_computed');
    if (!is_dir($dir)) {
        @mkdir($dir, 0755, true);
    }
    @file_put_contents($file, json_encode(['location' => $location, 'sha256' => $hash]), LOCK_EX);
    return $hash;
}

/**
 * Plain-text form of an IPS changelog (stored as HTML), shortened for the catalog.
 *
 * @param  string $html
 * @param  int    $max  Maximum length in characters.
 * @return string
 */
function plainChangelog(string $html, int $max = 1000): string
{
    $text = html_entity_decode(strip_tags(preg_replace('/<(br|\/p|\/li)\b[^>]*>/i', "\n", $html) ?? ''),
                               ENT_QUOTES | ENT_HTML5, 'UTF-8');
    $text = trim(preg_replace("/[ \t]+/", ' ', preg_replace("/\s*\n\s*/", "\n", $text) ?? '') ?? '');
    // Character-safe without mbstring, which not every PHP build has.
    if (preg_match_all('/./su', $text) > $max && preg_match('/^.{0,' . ($max - 1) . '}/su', $text, $m)) {
        $text = rtrim($m[0]) . '…';
    }
    return $text;
}

/**
 * Catalog objects for a file's previous versions.
 *
 * @param  int    $fileId
 * @param  string $title       For log notes.
 * @param  string $pageUrl     The entry's page.
 * @param  list<array<string,mixed>> $versions  ipsFetchVersions()'s list for this file, newest first.
 * @param  string $storageDir  Where IPS stores Downloads files; '' when unknown (then no hashes).
 * @param  string $cacheDir
 * @param  string $currentLabel The current version's label: an older version with the same
 *         label is left out, as is any older one repeating a newer one's label.
 * @return list<array{version:string,label:string,date:string,changelog:string,fileName:string,
 *         size:int,sha256:string,downloadUrl:string}>  Newest first.  Versions with no file
 *         left to download are omitted.
 */
function buildVersionObjects(int $fileId, string $title, string $pageUrl, array $versions,
                             string $storageDir, string $cacheDir, string $currentLabel = ''): array
{
    $out  = [];
    // An older upload under the same label is a respin or a correction of the
    // same build: HCDB2 has two "HCDB2-170714"s four days apart, and a first
    // "HCDB2-161204" that held the previous HCDB-151001 by mistake.  Clients
    // key cache and install folders on the label, so only the newest is
    // offered.  Compared case-insensitively.
    $seen = [];
    if ($currentLabel !== '') {
        $seen[strtolower($currentLabel)] = true;
    }
    foreach ($versions as $v) {
        $rec = choosePrimaryRecord($v['records']);
        if ($rec === null) {
            hclCount('versions_without_file');
            continue;
        }
        $name   = (string)$rec['record_realname'];
        $label  = versionLabel($v['version'], $name);
        if ($label !== '' && isset($seen[strtolower($label)])) {
            hclCount('versions_superseded_same_label');
            continue;
        }
        $seen[strtolower($label)] = true;
        $sha256 = '';
        $why    = null;
        $path   = resolveStoredFile($storageDir, (string)$rec['record_location'], $why);
        if ($path !== null) {
            $sha256 = recordSha256($cacheDir, (int)$rec['record_id'], $path, (string)$rec['record_location']);
        } else {
            hclCount('version_file_unreadable');
            hclNote('version_file_unreadable', sprintf('file %d "%s" version %d (%s): %s',
                $fileId, $title, $v['b_id'], (string)$rec['record_location'], $why));
        }
        // When this version was published: IPS copies the entry's last-updated
        // time into the backup.  b_backup is when it was *replaced*.
        $when = $v['updated'] ?: (int)($rec['record_time'] ?? 0) ?: $v['backup'];
        $out[] = [
            'version'     => $v['version'],
            'label'       => $label,
            'date'        => $when > 0 ? gmdate('Y-m-d', $when) : '',
            'changelog'   => plainChangelog($v['changelog']),
            'fileName'    => $name,
            'size'        => (int)$rec['record_size'],
            'sha256'      => $sha256,
            // IPS serves an old version's files with version=<b_id>; r picks
            // the record within it, as for the current version.
            'downloadUrl' => rtrim($pageUrl, '/') . '/?do=download&version=' . $v['b_id']
                           . '&r=' . (int)$rec['record_id'],
        ];
        hclCount('versions_listed');
    }
    return $out;
}

/**
 * The catalog's record of files an entry needs from other entries
 * (hclOptions()['companions']).
 *
 * @param  int    $fileId
 * @param  string $sourceId  e.g. "harpgamer".
 * @return list<array{id:string,provides:string}>  e.g. [['id' => 'harpgamer:328',
 *         'provides' => 'commondb.rsr']]; [] when none.  Malformed config rows are skipped.
 */
function companionObjects(int $fileId, string $sourceId): array
{
    $out = [];
    foreach ((array)(hclOptions()['companions'][$fileId] ?? []) as $row) {
        $id       = (int)(is_array($row) ? ($row[0] ?? 0) : 0);
        $provides = trim((string)(is_array($row) ? ($row[1] ?? '') : ''));
        // A bare file name only: clients place it by this name.
        if ($id <= 0 || $provides === '' || preg_match('#[/\\\\]|^\.\.?$#', $provides)) {
            continue;
        }
        $out[] = ['id' => "{$sourceId}:{$id}", 'provides' => $provides];
    }
    return $out;
}

/**
 * Collect every file ID in a collected tree.
 *
 * @param  array $tree  Output of collectCategoryTree().
 * @return int[]
 */
function collectFileIds(array $tree): array
{
    $ids = [];
    foreach ($tree as $cat) {
        foreach ($cat['rawFiles'] as $raw) {
            $ids[] = (int)($raw['id'] ?? 0);
        }
        array_push($ids, ...collectFileIds($cat['subcategories']));
    }
    return $ids;
}

// ---------------------------------------------------------------------------
// Custom field parsing
// ---------------------------------------------------------------------------

/**
 * Parse an IPS Downloads file's custom fields into a dependencies record.
 *
 * Field mapping:
 *   Field 1  ("DB Used")        → platformDB
 *   Field 2  ("Authors")        → authorOverride (returned separately)
 *   Fields 3–8, 11–15           → BattleSet-<region> fields; first non-null/empty
 *                                  gives battleSet (region name) + battleSetVariant (value)
 *   Field 9  ("Battleset")      → battleSet fallback
 *   Field 10 ("Game Version")   → gameVersion
 *
 * @param  array<mixed> $customFields  The 'customFields' array from the IPS API file object.
 * @return array{
 *   platformDB: string,
 *   gameVersion: string,
 *   battleSet: string,
 *   battleSetVariant: string,
 *   authorOverride: string
 * }
 */
function parseCustomFields(array $customFields): array
{
    $result = [
        'platformDB'       => '',
        'gameVersion'      => '',
        'battleSet'        => '',
        'battleSetVariant' => '',
        'authorOverride'   => '',
    ];

    // Build a quick lookup: field_id => ['name' => ..., 'value' => ...]
    // IPS returns customFields as an array of objects with 'id', 'name', 'value'.
    $byId = [];
    foreach ($customFields as $field) {
        if (isset($field['id'])) {
            $byId[(int)$field['id']] = $field;
        }
    }

    // Field 1: DB Used → platformDB
    if (!empty($byId[1]['value'])) {
        $result['platformDB'] = (string)$byId[1]['value'];
    }

    // Field 2: Authors → authorOverride
    if (!empty($byId[2]['value'])) {
        $result['authorOverride'] = (string)$byId[2]['value'];
    }

    // Fields 3–8, 11–15: BattleSet-<region> fields (first non-empty wins).
    $battleSetFieldIds = [3, 4, 5, 6, 7, 8, 11, 12, 13, 14, 15];
    foreach ($battleSetFieldIds as $fid) {
        if (empty($byId[$fid]['value'])) {
            continue;
        }
        $fieldName = (string)($byId[$fid]['name'] ?? '');
        $fieldValue = (string)$byId[$fid]['value'];

        // Extract region from field name, e.g. "Battleset-GIUK" → "GIUK".
        if (preg_match('/battleset[-_\s]+(.+)/i', $fieldName, $m)) {
            $result['battleSet'] = trim($m[1]);
        }
        $result['battleSetVariant'] = $fieldValue;
        break;
    }

    // Field 9: Battleset (fallback if no BattleSet-* field matched).
    if ($result['battleSet'] === '' && !empty($byId[9]['value'])) {
        $result['battleSet'] = (string)$byId[9]['value'];
    }

    // Field 10: Game Version → gameVersion
    if (!empty($byId[10]['value'])) {
        $result['gameVersion'] = (string)$byId[10]['value'];
    }

    return $result;
}

// ---------------------------------------------------------------------------
// File list fetching
// ---------------------------------------------------------------------------

/**
 * Fetch all files in a category by paginating through the IPS API.
 *
 * @param  int    $categoryId
 * @param  string $apiBase
 * @param  string $apiKey
 * @return array<int,array<string,mixed>>  Keyed by file ID.  Null on fatal API failure.
 */
function fetchCategoryFiles(int $categoryId, string $apiBase, string $apiKey): ?array
{
    $perPage = 25;
    $page    = 1;
    $files   = [];

    do {
        $url = "{$apiBase}/downloads/files"
            . '?key='        . urlencode($apiKey)
            . '&categories=' . $categoryId
            . '&perPage='    . $perPage
            . '&page='       . $page;

        $data = httpGetJson($url);
        if ($data === null) {
            return null; // propagate as fatal
        }

        $results      = $data['results']      ?? [];
        $totalResults = (int)($data['totalResults'] ?? 0);

        foreach ($results as $file) {
            if (isset($file['id'])) {
                $files[(int)$file['id']] = $file;
            }
        }

        $page++;
        $fetched = count($files);
    } while ($fetched < $totalResults && count($results) >= $perPage);

    return $files;
}

// ---------------------------------------------------------------------------
// File object builder
// ---------------------------------------------------------------------------

/**
 * Convert a raw IPS file array to the catalog file object schema.
 *
 * @param  array<string,mixed>  $raw           Raw IPS API file record.
 * @param  string               $sourceId      e.g. "harpgamer".
 * @param  string               $cacheDir      Cache directory path.
 * @param  string               $categoryType  Artifact type of the enclosing category.
 * @param  bool                 $mayLookUp     True when planLookups() gave this file a
 *         slot in this run's budget.  Without one, a file whose cache entry
 *         isn't fresh is emitted without size/filename (or refined type) and
 *         nothing is fetched or written; a later run picks it up.  Ignored
 *         when $dbRecord is given: the database path needs no budget.
 * @param  array<string,mixed>|null $dbRecord  The file's primary upload record from the
 *         IPS database (choosePrimaryRecord()), or null to use the web flow.
 * @param  string               $storageDir    Where IPS stores Downloads files; '' when unknown
 *         (then no hash or local inspection, but name and size still come from $dbRecord).
 * @param  string|null          $dbNoUpload    Set when the database was consulted and the file
 *         has no current upload: classifyNoUploadRecords()'s verdict.  Such files get no web
 *         lookup - the web flow would only find a web page or an error - and an external
 *         link is flagged 'externalLink' so clients open it rather than download it.
 * @param  list<array<string,mixed>> $versions  The file's previous versions from
 *         ipsFetchVersions(), newest first; [] when none or no database.
 * @return array<string,mixed>  Catalog file object.  Carries its own
 *         'artifactType' only when inspection found one more specific than
 *         the category's; otherwise clients inherit the category's type.
 *         'versionLabel' names the current version (see versionLabel()), and
 *         'versions' lists earlier ones that can still be downloaded.
 */
function buildFileObject(array $raw, string $sourceId, string $cacheDir, string $categoryType,
                         bool $mayLookUp = true, ?array $dbRecord = null, string $storageDir = '',
                         ?string $dbNoUpload = null, array $versions = []): array
{
    $fileId    = (int)($raw['id']       ?? 0);
    $title     = (string)($raw['title'] ?? '');
    $version   = (string)($raw['version'] ?? '');
    $pageUrl   = (string)($raw['url']     ?? '');
    $downloads = (int)($raw['downloads'] ?? 0);
    $views     = (int)($raw['views']     ?? 0);
    $rating    = (float)($raw['rating']  ?? 0);

    // Dates: prefer 'updated', fall back to 'date'.  Normalise to YYYY-MM-DD.
    $updatedRaw = (string)($raw['updated'] ?? $raw['date'] ?? '');
    $updated    = substr($updatedRaw, 0, 10); // take date portion of ISO string

    // Description: strip HTML tags and normalise whitespace.
    $descRaw = (string)($raw['description'] ?? '');
    $desc    = trim(preg_replace('/\s+/', ' ', strip_tags($descRaw)) ?? '');

    // Author: from author.name, potentially overridden by custom field 2.
    $author = (string)(($raw['author'] ?? [])['name'] ?? '');

    // Screenshot URL.
    $screenshotUrl = (string)(($raw['primaryScreenshot'] ?? [])['url'] ?? '');

    // Custom fields → dependencies + possible author override.
    $customFields = (array)($raw['customFields'] ?? []);
    $deps         = parseCustomFields($customFields);

    if ($deps['authorOverride'] !== '' && $deps['authorOverride'] !== $author) {
        $author = $deps['authorOverride'];
    }

    // Size, filename, hash and refined type.  The IPS database answers the
    // first two for free; the stored file on disk gives the other two, once
    // per file version.  Without a database record, fall back to the guest
    // download flow, which is budgeted.
    $now     = time();
    $dirty   = false;
    $inspect = in_array($categoryType, hclOptions()['inspect_artifact_types'], true);
    $meta    = getCachedFileMeta($fileId, $updatedRaw, $cacheDir, $now);

    if ($dbRecord !== null) {
        $recordId = (int)$dbRecord['record_id'];
        if ($meta === null || !$meta['ok'] || $meta['record_id'] !== $recordId) {
            // New or replaced file: start afresh so hash and type are redone.
            $meta  = ['filename' => '', 'size' => 0, 'ok' => true, 'artifactType' => null,
                      'reason' => null, 'record_id' => $recordId, 'sha256' => null];
            $dirty = true;
        }
        // The database is authoritative for name and size, and asking is free.
        $dbName = (string)$dbRecord['record_realname'];
        $dbSize = (int)$dbRecord['record_size'];
        if ($meta['filename'] !== $dbName || $meta['size'] !== $dbSize) {
            $meta['filename'] = $dbName;
            $meta['size']     = $dbSize;
            $dirty = true;
        }
        hclCount('db_resolved');

        $why  = null;
        $path = resolveStoredFile($storageDir, (string)$dbRecord['record_location'], $why);
        if ($path === null) {
            hclCount('stored_file_unreadable');
            hclNote('stored_file_unreadable', sprintf('file %d "%s" (%s): %s',
                $fileId, $title, (string)$dbRecord['record_location'], $why));
        } else {
            if ($meta['sha256'] === null) {
                $hash = hash_file('sha256', $path);
                if ($hash !== false) {
                    $meta['sha256'] = $hash;
                    $dirty = true;
                    hclCount('sha256_computed');
                }
            }
            if ($inspect && $meta['artifactType'] === null) {
                $isZip = str_ends_with(strtolower($dbName), '.zip');
                $names = $isZip ? listZipEntries($path) : [$dbName];
                if ($names !== null) {
                    $meta['artifactType'] = classifyArchiveEntries($names);
                    $dirty = true;
                    hclCount('inspected_local');
                } elseif (!class_exists('ZipArchive')) {
                    hclCount('inspect_no_zip_extension'); // leave it for a run that can
                } else {
                    $meta['artifactType'] = ''; // named .zip but isn't one: nothing to learn
                    $dirty = true;
                }
            }
        }
    } elseif ($dbNoUpload !== null) {
        // The database says there is no file to describe; the web flow would
        // only find a web page (for a link) or an error.  Nothing to fetch.
        $meta = ['filename' => '', 'size' => 0, 'ok' => false, 'artifactType' => null, 'reason' => null];
        hclCount("db_{$dbNoUpload}");
    } elseif ($meta === null) {
        if ($mayLookUp) {
            hclPace();
            $meta  = fetchFileMeta($pageUrl);
            $dirty = true;
            hclCount($meta['ok'] ? 'meta_fetched' : 'meta_failed');
            if (!$meta['ok']) {
                hclFailureReasons((string)$meta['reason']);
            }
        } else {
            // Over this run's budget: emit without size/filename for now.
            $meta = ['filename' => '', 'size' => 0, 'ok' => false, 'artifactType' => null, 'reason' => null];
            hclCount('deferred');
        }
    }

    // Web fallback only: look inside the archive when the category is too
    // coarse to classify its files.  Done once per file version; the result
    // rides in the cache.
    if ($dbRecord === null && $inspect && $meta['ok'] && $meta['artifactType'] === null) {
        if ($mayLookUp) {
            hclPace();
            $type = inspectArtifactType($pageUrl, $meta);
            if ($type !== null) {
                $meta['artifactType'] = $type;
                $dirty = true;
            }
        } else {
            hclCount('deferred_inspect');
        }
    }

    // Failures are cached too, but only for meta_retry_seconds.
    if ($dirty) {
        saveFileMeta($fileId, $updatedRaw, $meta, $cacheDir, $now);
    }

    $refinedType = '';
    if ($inspect && ($meta['artifactType'] ?? '') !== '' && $meta['artifactType'] !== $categoryType) {
        $refinedType = $meta['artifactType'];
        hclCount('refined');
    }

    $file = [
        'id'          => "{$sourceId}:{$fileId}",
        'title'       => $title,
        'version'     => $version,
        'author'      => $author,
        'size'        => $meta['size'],
        'downloads'   => $downloads,
        'updated'     => $updated,
        'description' => $desc,
        'pageUrl'     => $pageUrl,
        // For database-resolved files, a link to exactly the record that was
        // described and hashed (IPS keys an entry's files by record_id).
        // Without it a multi-file entry answers with an HTML file chooser.
        // Clients add their session's csrfKey; '' = build from pageUrl.
        'downloadUrl' => $dbRecord !== null
            ? rtrim($pageUrl, '/') . '/?do=download&r=' . (int)$dbRecord['record_id']
            : '',
        'fileName'    => $meta['filename'],
        // Hashed from the stored file when it could be read from disk; the
        // launcher verifies downloads against it.  '' when unknown.
        'sha256'      => (string)($meta['sha256'] ?? ''),
        'screenshotUrl' => $screenshotUrl,
        'rating'      => $rating,
        'dependencies' => [
            'platformDB'       => $deps['platformDB'],
            'gameVersion'      => $deps['gameVersion'],
            'battleSet'        => $deps['battleSet'],
            'battleSetVariant' => $deps['battleSetVariant'],
        ],
    ];

    if ($refinedType !== '') {
        $file['artifactType'] = $refinedType;
    }
    if ($dbNoUpload === 'external_link') {
        $file['externalLink'] = true;
    }

    $label = versionLabel($version, (string)$meta['filename']);
    if ($label !== '') {
        $file['versionLabel'] = $label;
    }
    $previous = buildVersionObjects($fileId, $title, $pageUrl, $versions, $storageDir, $cacheDir, $label);
    if ($previous) {
        $file['versions'] = $previous;
    }

    $companions = companionObjects($fileId, $sourceId);
    if ($companions) {
        $file['companions'] = $companions;
    }

    return $file;
}

// ---------------------------------------------------------------------------
// Category tree: collect, plan, render
//
// The tree is built in three passes so that the lookup budget can be spent
// by priority across the whole catalog rather than first-come in category
// order (which would starve Tools/Mods/Docs, the last category and the one
// whose files need inspecting).
// ---------------------------------------------------------------------------

/**
 * Pass 1: fetch every HC category's raw file records from the IPS API.
 *
 * @param  int                            $parentId       Parent whose children to collect.
 * @param  int                            $hcParentId     Root HC category ID (not included in output).
 * @param  array<int,array<string,mixed>> $allCategories  All IPS categories keyed by ID.
 * @param  array<int,string>              $artifactMap
 * @param  string                         $apiBase
 * @param  string                         $apiKey
 * @param  string[]                       &$errors        Accumulated non-fatal error messages.
 * @return array<int,array{catId:int,name:string,parentId:int,artifactType:string,
 *                         rawFiles:array<int,array<string,mixed>>,subcategories:array}>
 */
function collectCategoryTree(
    int    $parentId,
    int    $hcParentId,
    array  $allCategories,
    array  $artifactMap,
    string $apiBase,
    string $apiKey,
    array  &$errors
): array {
    $tree = [];

    foreach ($allCategories as $catId => $cat) {
        $catParentId = (int)($cat['parentId'] ?? 0);
        if ($catParentId !== $parentId) {
            continue;
        }

        $catName  = (string)($cat['name'] ?? "Category {$catId}");
        $rawFiles = fetchCategoryFiles($catId, $apiBase, $apiKey);
        if ($rawFiles === null) {
            $errors[] = "Failed to fetch files for category {$catId} ({$catName})";
            $rawFiles = [];
        }

        $tree[] = [
            'catId'         => (int)$catId,
            'name'          => $catName,
            'parentId'      => $catParentId,
            'artifactType'  => categoryArtifactType($catId, $artifactMap, $allCategories),
            'rawFiles'      => $rawFiles,
            'subcategories' => collectCategoryTree($catId, $hcParentId, $allCategories,
                                                   $artifactMap, $apiBase, $apiKey, $errors),
        ];
    }

    return $tree;
}

/**
 * Pass 2: decide which files get a live lookup this run.
 *
 * Every file whose cache entry isn't fresh competes for $budget slots, in
 * this order (catalog order within each):
 *   1. files in inspected categories (inspect_artifact_types) — the launcher's
 *      per-file types depend on them;
 *   2. files never looked up in the current cache format, or changed since;
 *   3. files whose last lookup failed and is due for a retry.
 * Files the IPS database accounts for take no part: with an upload record
 * they need no web request, and without one the web flow has nothing to find.
 * Reads the cache only; makes no network requests.
 *
 * @param  array  $tree      Output of collectCategoryTree().
 * @param  string $cacheDir
 * @param  int    $now
 * @param  int    $budget    Maximum files to look up.
 * @param  array<int,mixed> $dbRecords  File ID => truthy for every file the database
 *                           accounts for; those files are skipped.
 * @return array<int,true>   File IDs allowed a live lookup.
 */
function planLookups(array $tree, string $cacheDir, int $now, int $budget, array $dbRecords = []): array
{
    $inspectTypes = hclOptions()['inspect_artifact_types'];
    $buckets = [0 => [], 1 => [], 2 => []];

    $walk = function (array $cats) use (&$walk, &$buckets, $cacheDir, $now, $inspectTypes, $dbRecords): void {
        foreach ($cats as $cat) {
            $inspect = in_array($cat['artifactType'], $inspectTypes, true);
            foreach ($cat['rawFiles'] as $raw) {
                $fileId  = (int)($raw['id'] ?? 0);
                if (!empty($dbRecords[$fileId])) {
                    continue; // the database accounts for it; no web request needed
                }
                $updated = (string)($raw['updated'] ?? $raw['date'] ?? '');
                $state   = cacheState($fileId, $updated, $cacheDir, $now, $inspect);
                if ($state === 'fresh') {
                    continue;
                }
                $priority = $inspect ? 0 : ($state === 'need_meta_retry' ? 2 : 1);
                $buckets[$priority][] = $fileId;
            }
            $walk($cat['subcategories']);
        }
    };
    $walk($tree);

    $queue   = array_merge($buckets[0], $buckets[1], $buckets[2]);
    $allowed = array_slice($queue, 0, max(0, $budget));

    hclCount('lookups_planned', count($allowed));
    hclCount('backlog_after_run', count($queue) - count($allowed));
    return array_fill_keys($allowed, true);
}

/**
 * Pass 3: build the catalog's category objects, looking files up as planned.
 *
 * @param  array            $tree         Output of collectCategoryTree().
 * @param  int              $hcParentId   Root HC category ID (children of it get a null parentId).
 * @param  string           $sourceId
 * @param  string           $cacheDir
 * @param  array<int,true>  $allowed      Output of planLookups().
 * @param  int              &$totalFiles  Running total for logging.
 * @param  array<int,mixed> $dbRecords    File ID => upload records from ipsFetchUploadRecords().
 * @param  string           $storageDir   Where IPS stores Downloads files; '' when unknown.
 * @param  array<int,string> $dbNoUpload  File ID => classifyNoUploadRecords() verdict, for
 *                                        files the database has no current upload for.
 * @param  array<int,list<array<string,mixed>>> $dbVersions  File ID => previous versions
 *                                        from ipsFetchVersions().
 * @return array<int,array<string,mixed>> Category objects with nested 'subcategories' and 'files'.
 */
function renderCategoryTree(array $tree, int $hcParentId, string $sourceId, string $cacheDir,
                            array $allowed, int &$totalFiles,
                            array $dbRecords = [], string $storageDir = '', array $dbNoUpload = [],
                            array $dbVersions = []): array
{
    $out = [];
    foreach ($tree as $cat) {
        $files = [];
        foreach ($cat['rawFiles'] as $raw) {
            $fileId    = (int)($raw['id'] ?? 0);
            $records   = $dbRecords[$fileId] ?? [];
            if (count($records) > 1) {
                // The catalog lists one file per entry; the rest go unlisted for now.
                hclCount('multi_file_entries');
            }
            $files[] = buildFileObject($raw, $sourceId, $cacheDir, $cat['artifactType'],
                                       isset($allowed[$fileId]), choosePrimaryRecord($records), $storageDir,
                                       $dbNoUpload[$fileId] ?? null, $dbVersions[$fileId] ?? []);
            $totalFiles++;
        }

        $out[] = [
            'id'            => "{$sourceId}:{$cat['catId']}",
            'name'          => $cat['name'],
            'parentId'      => ($cat['parentId'] === $hcParentId) ? null : "{$sourceId}:{$cat['parentId']}",
            'artifactType'  => $cat['artifactType'],
            'files'         => $files,
            'subcategories' => renderCategoryTree($cat['subcategories'], $hcParentId, $sourceId,
                                                  $cacheDir, $allowed, $totalFiles,
                                                  $dbRecords, $storageDir, $dbNoUpload, $dbVersions),
        ];
    }
    return $out;
}

// ---------------------------------------------------------------------------
// Main
// ---------------------------------------------------------------------------

if (defined('HCL_GENERATOR_LIB_ONLY')) {
    return; // included by a test harness for its functions only
}

$startTime  = microtime(true);
$configPath = $argv[1] ?? (__DIR__ . '/hcl-config.php');

if (!is_file($configPath)) {
    fwrite(STDERR, "Config file not found: {$configPath}\n");
    exit(1);
}

/** @var array<string,mixed> $cfg */
$cfg = require $configPath;

$apiKey       = (string)($cfg['api_key']       ?? '');
$apiBase      = rtrim((string)($cfg['api_base'] ?? ''), '/');
$outputDir    = rtrim((string)($cfg['output_dir'] ?? __DIR__), '/\\');
$cacheDir     = rtrim((string)($cfg['cache_dir']  ?? $outputDir . '/cache'), '/\\');
$sourceId     = (string)($cfg['source_id']      ?? 'harpgamer');
$sourceName   = (string)($cfg['source_name']    ?? '');
$sourceUrl    = (string)($cfg['source_url']     ?? '');
$sourceContact= (string)($cfg['source_contact'] ?? '');
$hcParentId   = (int)($cfg['hc_parent_category_id'] ?? 1);
/** @var array<int,string> $artifactMap */
$artifactMap  = (array)($cfg['category_artifact_map'] ?? []);

// Optional tuning keys; anything absent keeps the defaults in hclOptions().
hclOptions(array_intersect_key($cfg, array_flip([
    'user_agent', 'inspect_artifact_types', 'inspect_max_bytes', 'meta_retry_seconds',
    'max_lookups_per_run', 'lookup_delay_ms', 'max_version_hashes_per_run', 'companions',
])));

$logFile      = $outputDir . '/hcl-catalog-generate.log';
$catalogTmp   = $outputDir . '/catalog.json.tmp';
$catalogFinal = $outputDir . '/catalog.json';

if (!is_dir($cacheDir)) {
    mkdir($cacheDir, 0755, true);
}

logLine("START generate", $logFile);

if (!class_exists('ZipArchive')) {
    logLine("WARN: zip extension not available - archive inspection disabled", $logFile);
}

// 1. Fetch all IPS categories.
$allCategories = fetchAllCategories($apiBase, $apiKey);
if ($allCategories === null) {
    $msg = "FATAL: Failed to fetch categories from API";
    logLine($msg, $logFile);
    fwrite(STDERR, $msg . "\n");
    exit(1);
}

// 2. Filter to HC subtree only.
$hcCategories = [];
foreach ($allCategories as $catId => $cat) {
    if (isHCCategory($catId, $hcParentId, $allCategories)) {
        $hcCategories[$catId] = $cat;
    }
}

if (empty($hcCategories)) {
    $msg = "FATAL: No HC categories found under parent ID {$hcParentId}";
    logLine($msg, $logFile);
    fwrite(STDERR, $msg . "\n");
    exit(1);
}

// 3. Build the category tree: collect from the API, plan this run's lookups
//    against the budget, then render (looking files up as planned).
$totalFiles = 0;
$errors     = [];

$collected = collectCategoryTree(
    $hcParentId,   // direct children of the HC root
    $hcParentId,
    $hcCategories,
    $artifactMap,
    $apiBase,
    $apiKey,
    $errors
);
// Optional: the IPS database supplies names, sizes and stored-file locations.
// Any problem here only means falling back to the guest download flow.
$dbRecords  = [];
$dbNoUpload = [];   // file ID => why it has no current upload
$dbKnown    = [];   // file ID => true for every file the database accounts for
$dbVersions = [];   // file ID => previous versions, newest first
$storageDir = '';
if (is_array($cfg['ips_db'] ?? null)) {
    $prefix = (string)($cfg['ips_db']['prefix'] ?? '');
    $why    = null;
    $query  = preg_match('/^[A-Za-z0-9_]*$/', $prefix) ? ipsDbOpen($cfg['ips_db'], $why) : null;
    if ($query === null) {
        $why = $why ?? 'ips_db.prefix contains invalid characters';
        logLine("WARN: IPS database not used ({$why}); using the web flow", $logFile);
        hclCount('db_unavailable');
    } else {
        try {
            $listedIds = collectFileIds($collected);
            $dbRecords = ipsFetchUploadRecords($query, $prefix, $listedIds);

            // Listed files without a current upload: say why, and keep them
            // away from the web flow, which could only find a page or an error.
            $noUploadIds = array_values(array_diff($listedIds, array_keys($dbRecords)));
            $summary     = ipsFetchRecordSummary($query, $prefix, $noUploadIds);
            foreach ($noUploadIds as $id) {
                $dbNoUpload[$id] = classifyNoUploadRecords($summary[$id] ?? []);
            }
            $dbKnown = array_fill_keys($listedIds, true);

            // Earlier versions IPS still offers, e.g. every HCDB2 build that
            // scenarios name as their platform database.
            $dbVersions = ipsFetchVersions($query, $prefix, $listedIds);

            // Where the files live: an explicit override, else IPS's own
            // storage settings ({root} = the IPS install directory).
            $storageDir = rtrim((string)($cfg['ips_downloads_dir'] ?? ''), '/\\');
            if ($storageDir === '') {
                $ipsRoot    = (string)($cfg['ips_root'] ?? (dirname(__DIR__) . '/public_html/harpforum'));
                $storageDir = ipsDownloadsStorageDir($query, $prefix, $ipsRoot, $why) ?? '';
            }
            $kinds = array_count_values($dbNoUpload);
            logLine(sprintf("IPS database: %d of %d files have upload records; without: %d external link, %d no current upload, %d no records; storage %s%s",
                count($dbRecords), count($listedIds),
                $kinds['external_link'] ?? 0, $kinds['no_current_upload'] ?? 0, $kinds['no_records'] ?? 0,
                $storageDir === '' ? "unknown ({$why})" : $storageDir,
                ($storageDir !== '' && !is_readable($storageDir)) ? ' (NOT READABLE)' : ''), $logFile);
            logLine(sprintf('IPS database: %d previous version(s) across %d file(s)',
                array_sum(array_map('count', $dbVersions)), count($dbVersions)), $logFile);
        } catch (Throwable $e) {
            logLine('WARN: IPS database query failed (' . $e->getMessage() . '); using the web flow', $logFile);
            hclCount('db_unavailable');
            $dbRecords  = [];
            $dbNoUpload = [];
            $dbKnown    = [];
            $dbVersions = [];
            $storageDir = '';
        }
    }
}

$allowed      = planLookups($collected, $cacheDir, time(), hclOptions()['max_lookups_per_run'], $dbKnown);
$categoryTree = renderCategoryTree($collected, $hcParentId, $sourceId, $cacheDir, $allowed, $totalFiles,
                                   $dbRecords, $storageDir, $dbNoUpload, $dbVersions);

// Name any stored files that couldn't be read, so they can be fixed in IPS.
foreach (['stored_file_unreadable' => 'stored file(s)', 'version_file_unreadable' => 'previous-version file(s)'] as $kind => $what) {
    $unreadable = hclNote($kind);
    if ($unreadable) {
        logLine(sprintf('WARN: %d %s unreadable: %s%s', count($unreadable), $what,
            implode('; ', array_slice($unreadable, 0, 10)),
            count($unreadable) > 10 ? '; ...' : ''), $logFile);
    }
}

// 4. Assemble the catalog document.
$catalog = [
    'schemaVersion' => '1.0',
    'generated'     => gmdate('Y-m-d\TH:i:s\Z'),
    'source'        => [
        'id'             => $sourceId,
        'name'           => $sourceName,
        'url'            => $sourceUrl,
        'contact'        => $sourceContact,
        'downloadMethod' => 'ips-csrf',
    ],
    'categories' => $categoryTree,
];

// 5. Atomic write: tmp → rename.
$json = json_encode($catalog, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
if ($json === false) {
    $msg = "FATAL: json_encode failed: " . json_last_error_msg();
    logLine($msg, $logFile);
    fwrite(STDERR, $msg . "\n");
    exit(1);
}

if (file_put_contents($catalogTmp, $json, LOCK_EX) === false) {
    $msg = "FATAL: Could not write {$catalogTmp}";
    logLine($msg, $logFile);
    fwrite(STDERR, $msg . "\n");
    exit(1);
}

if (!rename($catalogTmp, $catalogFinal)) {
    $msg = "FATAL: Could not rename {$catalogTmp} to {$catalogFinal}";
    logLine($msg, $logFile);
    fwrite(STDERR, $msg . "\n");
    exit(1);
}

// 6. Summary log line.
$duration = round(microtime(true) - $startTime, 2);
$errorSummary = empty($errors) ? 'no errors' : count($errors) . ' error(s): ' . implode('; ', $errors);
$counters = '';
foreach (hclCount() as $name => $n) {
    $counters .= "\t{$name}={$n}";
}
// The most frequent failure reasons, so the log says what refused the lookups.
$topReasons = array_slice(hclFailureReasons(), 0, 5, true);
if ($topReasons) {
    $counters .= "\tfailures=" . implode('; ', array_map(
        fn($reason, $n) => "{$n}x {$reason}", array_keys($topReasons), $topReasons));
}
logLine(
    "END generate\tfiles={$totalFiles}{$counters}\tduration={$duration}s\t{$errorSummary}",
    $logFile
);

if (!empty($errors)) {
    fwrite(STDERR, implode("\n", $errors) . "\n");
    exit(1);
}

exit(0);