<?php
header('Content-Type: application/json');
header('X-Content-Type-Options: nosniff');
header('Cache-Control: no-store');
date_default_timezone_set('UTC');

ini_set('display_errors', 0);
ini_set('display_startup_errors', 0);
error_reporting(E_ALL);
set_time_limit(180);

register_shutdown_function(function () {
    $e = error_get_last();
    if ($e !== null) {
        error_log("FATAL: {$e['message']} @ {$e['file']}:{$e['line']}");
    }
});



// ---------------- Query Params ----------------
$folder   = $_GET['folder'] ?? null;
$startStr = $_GET['start']  ?? null;  // YYYY-MM-DD
$endStr   = $_GET['end']    ?? null;  // YYYY-MM-DD
$debugOn  = isset($_GET['debug']) && $_GET['debug'] === '1';
$tzParam  = $_GET['tz'] ?? null;
$country  = $_GET['country'] ?? null;

if (!$folder || !$startStr || !$endStr) {
    echo json_encode([
        "error" => "Missing params. Example: ?folder=username&start=2025-09-19&end=2025-09-23&country=Nigeria&debug=1"
    ]);
    exit;
}

function valid_date($s) {
    $d = DateTime::createFromFormat('Y-m-d', $s);
    return $d && $d->format('Y-m-d') === $s;
}
if (!valid_date($startStr) || !valid_date($endStr) || $endStr < $startStr) {
    echo json_encode(["error" => "Provide valid start/end (YYYY-MM-DD) and ensure end >= start."]);
    exit;
}

$DEBUG = [];
function dbg($m) {
    global $debugOn, $DEBUG;
    if ($debugOn) $DEBUG[] = '['.date('H:i:s')."] $m";
}

// ---------------- TZ ----------------
$countryToTimezone = [
    'Nigeria' => 'Africa/Lagos',
    'Romania' => 'Europe/Bucharest',
    'USA'     => 'America/New_York',
    'UK'      => 'Europe/London',
    'India'   => 'Asia/Kolkata',
    'Thailand'=> 'Asia/Bangkok',
    'Egypt'   => 'Africa/Cairo',
    'UAE'     => 'Asia/Dubai',
];
if ($tzParam) {
    try { $TARGET_TZ = new DateTimeZone($tzParam); }
    catch (Exception $e) { echo json_encode(["error"=>"Invalid tz: $tzParam"]); exit; }
    $tzLabel = $tzParam;
} elseif ($country) {
    if (!isset($countryToTimezone[$country])) {
        echo json_encode(["error"=>"Unknown country: $country"]); exit;
    }
    $tzLabel = $countryToTimezone[$country];
    $TARGET_TZ = new DateTimeZone($tzLabel);
} else {
    $tzLabel = 'Africa/Lagos';
    $TARGET_TZ = new DateTimeZone($tzLabel);
}

// ---------------- Paths ----------------
$rootDir     = __DIR__;
$userDir     = $rootDir . '/' . basename($folder);
$dataFile    = $userDir . '/data.txt';

// ---------------- Bootstrap user folder ----------------
if (!is_dir($userDir)) {
    if (!mkdir($userDir, 0777, true)) {
        echo json_encode(["error" => "Failed to create folder: $folder"]); 
        exit;
    }
    file_put_contents($dataFile, json_encode([], JSON_PRETTY_PRINT));
}

// ---------------- Load global config.php ----------------
require __DIR__ . '/config.php';

$url = "https://www.threads.net/graphql/query";


// --- SessionID helpers (add once) ---
function is_session_invalid($json) {
    if (!is_array($json)) return false;

    // Case A: your API nests it under raw_response.errors[0].message
    $msg = $json['raw_response']['errors'][0]['message'] ?? null;
    if (is_string($msg) && stripos($msg, 'execution error') !== false) {
        return true;
    }

    // Case B: GraphQL-style top-level errors
    $msg2 = $json['errors'][0]['message'] ?? null;
    if (is_string($msg2) && stripos($msg2, 'execution error') !== false) {
        return true;
    }

    return false;
}

function exit_session_invalid() {
    // Optional: log
    if (function_exists('dbg')) dbg("Threads returned 'execution error' → SessionID invalid.");

    // Return a clean, short JSON and stop.
    echo json_encode([
        "status"  => "session_invalid",
        "message" => "SessionID invalid. Please re-login.",
        "posts"   => [],
        "debug"   => $GLOBALS['DEBUG'] ?? []
    ], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE);
    exit;
}


// ---------------- I/O helpers ----------------
function sort_desc(&$arr) {
    usort($arr, function($a,$b){
        $d = strcmp($b['date'],$a['date']);
        if ($d !== 0) return $d;
        return strcmp($b['time'],$a['time']);
    });
}
function dedupe_posts($arr) {
    $seen = [];
    $out  = [];
    foreach ($arr as $p) {
        if (!isset($p['id'])) continue;
        if (!isset($seen[$p['id']])) {
            $seen[$p['id']] = true;
            $out[] = $p;
        }
    }
    return $out;
}
function reindex(&$arr) {
    sort_desc($arr);
    $n=1;
    foreach ($arr as &$p) $p['nr'] = $n++;
}
function normalize_post($post, DateTimeZone $TARGET_TZ) {
    $id = $post['id'] ?? null;
    $ts = (int)($post['taken_at'] ?? 0);
    if (!$id || !$ts) return null;

    $cap = $post['caption']['text'] ?? '';
    $dt  = new DateTime("@$ts", new DateTimeZone('UTC')); // raw UTC
    $dt->setTimezone($TARGET_TZ);                          // convert to locked tz

    return [
        "id"      => $id,
        "nr"      => 0,
        "caption" => $cap,
        "date"    => $dt->format('Y-m-d'),
        "time"    => $dt->format('H:i:s'),
    ];
}

function posts_from_edges($edges, DateTimeZone $TARGET_TZ) {
    $out = [];
    foreach ($edges as $edge) {
        $items = $edge['node']['thread_items'] ?? [];
        foreach ($items as $it) {
            $post = $it['post'] ?? [];

            $pinned = $post['text_post_app_info']['pinned_post_info']['is_pinned_to_profile'] ?? false;
            if ($pinned) {
                dbg("Skipping pinned post with id: " . ($post['id'] ?? 'unknown'));
                continue;
            }

            $np = normalize_post($post, $TARGET_TZ);
            if ($np) $out[] = $np;
        }
    }
    return $out;
}

function fetch_page($url, $headers, $after) {
    $body = buildBody($after);
    $ch = curl_init($url);
    curl_setopt_array($ch, [
        CURLOPT_RETURNTRANSFER => true,
        CURLOPT_POST           => true,
        CURLOPT_HTTPHEADER     => $headers,
        CURLOPT_POSTFIELDS     => $body,
        CURLOPT_TIMEOUT        => 30,
    ]);
    $result = curl_exec($ch);
    $err    = curl_error($ch);
    curl_close($ch);
    if ($err) return ['error'=>$err];
    $json = json_decode($result, true);
    if (!is_array($json)) return ['error'=>'Invalid JSON from Threads'];
    return $json;
}


/**
 * FRONT SYNC (newest side):
 * Fetch up to $roundTarget posts from the newest end (after=null) until:
 *  - we encounter $firstKnownId (the current newest in data.txt) → merge only posts *before* that match, then stop
 *  - or we hit $roundTarget without seeing a match → merge those $roundTarget as new and stop
 *
 * Returns number of new posts merged at the front.
 */
function front_sync_newest(&$data, $firstKnownId, $url, $headers, DateTimeZone $TARGET_TZ, $roundTarget = 50) {
    $allNewPosts = []; // Collect all posts before match
    $after = null;     // Start from newest
    dbg("Front-sync: start (roundTarget=$roundTarget) looking for known top id: $firstKnownId");

    while (true) {
        $json = fetch_page($url, $headers, $after);
        if (is_session_invalid($json)) {
    exit_session_invalid();
}
        if (isset($json['error'])) {
            dbg("Front-sync fetch error: " . $json['error']);
            break;
        }

        $edges = $json['data']['mediaData']['edges'] ?? [];
        $pi    = $json['data']['mediaData']['page_info'] ?? [];
        $has   = $pi['has_next_page'] ?? false;
        $endc  = $pi['end_cursor'] ?? null;

        if (empty($edges)) {
            dbg("Front-sync: no edges → stop.");
            break;
        }

        $batch = posts_from_edges($edges, $TARGET_TZ);
        if (empty($batch)) {
            dbg("Front-sync: normalized batch empty → stop.");
            break;
        }

        // Walk through this batch
        foreach ($batch as $np) {
            if (($np['id'] ?? '') === $firstKnownId) {
                dbg("Front-sync: FOUND known id $firstKnownId → stopping merge here.");
                // Merge collected posts into the main data
                if (!empty($allNewPosts)) {
                    $data = dedupe_posts(array_merge($allNewPosts, $data));
                    reindex($data);
                    dbg("Front-sync: merged " . count($allNewPosts) . " new posts before known id.");
                    return count($allNewPosts);
                } else {
                    dbg("Front-sync: found known id immediately, no new posts to merge.");
                    return 0;
                }
            }
            $allNewPosts[] = $np;
        }

        if (!$has || !$endc) {
            dbg("Front-sync: no more pages from server, matched id not found.");
            // Merge everything collected so far
            if (!empty($allNewPosts)) {
                $data = dedupe_posts(array_merge($allNewPosts, $data));
                reindex($data);
                return count($allNewPosts);
            }
            break;
        }

        $after = $endc; // continue scraping older posts
    }

    return 0;
}



function filter_range($posts, $start, $end) {
    $out = [];
    $startTs = strtotime($start);
    $endTs   = strtotime($end);

    foreach ($posts as $p) {
        $postTs = strtotime($p['date'] ?? '');

        // ✅ If start and end are the same → match EXACT date only
        if ($start === $end) {
            if ($p['date'] === $start) {
                $out[] = $p;
            }
        } 
        // ✅ Otherwise → use full date range
        else {
            if ($postTs >= $startTs && $postTs <= $endTs) {
                $out[] = $p;
            }
        }
    }
    return $out;
}

/**
 * Filter posts by date range and convert their timestamps to target timezone
 */

function daily_counts($posts, $start, $end) {
    $c = [];

    // First count posts normally
    foreach ($posts as $p) {
        $d = $p['date'] ?? null;
        if ($d) $c[$d] = ($c[$d] ?? 0) + 1;
    }

    // Ensure all days in range are included, even with 0 posts
    for ($ts = strtotime($start); $ts <= strtotime($end); $ts += 86400) {
        $day = date('Y-m-d', $ts);
        if (!isset($c[$day])) {
            $c[$day] = 0;
        }
    }

    ksort($c);
    return $c;
}

function filter_only($posts, $start, $end) {
    return filter_range($posts, $start, $end);
}
// ---------------- Validation helpers ----------------
/**
 * Validate days for the given range.
 * Rule: day is validated if:
 *  - we have >= 1 post on that day, AND
 *  - we have >= 5 posts strictly older than that day
 * OR if $timelineEnded === true, any day that has at least one post is validated.
 * Returns [map, allValidated]
 */
function compute_validation_map($posts, $start, $end, $timelineEnded) {
    // Build counts and quick lookups
    $byDate = [];
    foreach ($posts as $p) {
        $d = $p['date'] ?? null;
        if ($d) $byDate[$d] = ($byDate[$d] ?? 0) + 1;
    }

    $dates = [];
    for ($t = strtotime($start); $t <= strtotime($end); $t += 86400) {
        $dates[] = date('Y-m-d', $t);
    }

    // Precompute "older than day" counts (strictly older)
    $allDatesSeen = array_keys($byDate);
    sort($allDatesSeen); // oldest -> newest
    $olderPrefix = []; // $olderPrefix[$day] = total posts with date < $day

    $totalSeenByDay = [];
    foreach ($byDate as $d => $n) $totalSeenByDay[$d] = $n;

    // Compute cumulative totals by walking oldest -> newest
    $cum = 0;
    foreach ($allDatesSeen as $d) {
        $olderPrefix[$d] = $cum;
        $cum += $byDate[$d];
    }

    $map = [];
    $allOk = true;
    foreach ($dates as $day) {
        $hasDay = isset($byDate[$day]) && $byDate[$day] > 0;
        $older  = 0;
        // For a day not seen at all, older doesn't matter
        if ($hasDay) {
            // Count all posts strictly older than $day
            foreach ($allDatesSeen as $d) {
                if ($d < $day) $older += $byDate[$d];
                else break;
            }
        }
        $validated = false;
        if ($hasDay) {
            if ($older >= 5) $validated = true;
            elseif ($timelineEnded) $validated = true; // end of timeline -> accept trailing days
        }

        $map[$day] = $validated ? 'validated' : 'invalidated';
        if (!$validated) $allOk = false;
    }

    // Mark the newest day in range as 'current' if present in posts
$allDatesSeen = array_unique(array_column($posts, 'date'));
rsort($allDatesSeen); // newest first
$absoluteNewest = $allDatesSeen[0] ?? null;

if ($absoluteNewest) {
    $map[$absoluteNewest] = 'current';
}

    return [$map, $allOk];
}

function detect_missing_days_from_map($map) {
    $missing = [];
    foreach ($map as $day => $status) {
        if ($status !== 'validated' && $status !== 'current') $missing[] = $day;
    }
    return $missing;
}

// ---------------- Load existing data/state ----------------
// ---------------- Load existing data/state ----------------
$data  = [];
$savedValMap = [];
$lockedCountry = null;
$lockedTzLabel = null;

if (file_exists($dataFile)) {
    $raw = @file_get_contents($dataFile);
    $arr = json_decode($raw, true);

    if (is_array($arr)) {
        // New structure with meta
        if (isset($arr['posts']) && is_array($arr['posts'])) {
            $data = $arr['posts'];
        } elseif (isset($arr[0]) && is_array($arr[0])) {
            // Old structure compatibility (plain array of posts)
            $data = $arr;
        }

        if (isset($arr['validation_map']) && is_array($arr['validation_map'])) {
            $savedValMap = $arr['validation_map'];
        }

        // Folder lock (if previously saved)
        if (!empty($arr['country']))  $lockedCountry = $arr['country'];
        if (!empty($arr['timezone'])) $lockedTzLabel = $arr['timezone'];
    }
}

// ---------------- Enforce or establish country/timezone lock ----------------
if ($lockedCountry && $lockedTzLabel) {
    // Folder already locked → refuse mismatches
    if ($country && strcasecmp($country, $lockedCountry) !== 0) {
        echo json_encode([
            "status"  => "error",
            "message" => "Folder '$folder' is locked to country '$lockedCountry' ($lockedTzLabel). Refusing request for '$country'."
        ], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE);
        exit;
    }
    if ($tzParam && $tzLabel !== $lockedTzLabel) {
        echo json_encode([
            "status"  => "error",
            "message" => "Folder '$folder' is locked to timezone '$lockedTzLabel'. Refusing request for '$tzLabel'."
        ], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE);
        exit;
    }

    // Force usage of locked tz for this run
    $tzLabel   = $lockedTzLabel;
    $TARGET_TZ = new DateTimeZone($tzLabel);
} else {
    // Not locked yet → require a country to initialize the lock
    if (!$country) {
        echo json_encode([
            "status"  => "error",
            "message" => "Folder '$folder' is not initialized. Call this endpoint with &country=... to lock it.",
            "examples" => [
                "?folder=$folder&start=$startStr&end=$endStr&country=Nigeria",
                "?folder=$folder&start=$startStr&end=$endStr&country=Romania"
            ]
        ], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE);
        exit;
    }
    // The lock will be saved on the first successful write
    $lockedCountry = $country;
    $lockedTzLabel = $tzLabel;
}


// Clean memory data
$data = dedupe_posts($data);
reindex($data);


// ---------------- Verify end date has started in target timezone ----------------
$nowInTarget = new DateTime('now', $TARGET_TZ);
$currentDateInTarget = $nowInTarget->format('Y-m-d');

if ($endStr > $currentDateInTarget) {
    // The requested end date is in the future for the selected country
    echo json_encode([
        "status" => "error",
            "country_locked" => $lockedCountry, // ADD
    "timezone_used"  => $tzLabel,       // ADD
        "message" => "The end date ($endStr) has not started yet in $country timezone ($tzLabel). Current date there is $currentDateInTarget.",
    ], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE);
    exit;
}

dbg("Verified: End date $endStr is valid for timezone $tzLabel (current date there: $currentDateInTarget).");



// Detect if the requested END date is newer than the current date from data.txt
$currentFromFile = null;
foreach ($savedValMap as $d => $s) {
    if ($s === 'current') {
        $currentFromFile = $d;
        break;
    }
}

$handleFutureDays = false;
if ($currentFromFile && $endStr > $currentFromFile) {
    $handleFutureDays = true;
    dbg("End date ($endStr) is newer than current date in data.txt ($currentFromFile). Will handle future days first.");
}
$frontSyncHandledFutureDays = false; // ✅ ADD THIS LINE

// ---------------- Initial validation on cached data ----------------
list($valMapInitial, $allOkInitial) = compute_validation_map($data, $startStr, $endStr, false);
$missingInitial = detect_missing_days_from_map($valMapInitial);

// Always check current day first if present in savedValMap
$savedCurrentDay = null;
foreach ($savedValMap as $d => $s) {
    if ($s === 'current') {
        $savedCurrentDay = $d;
        break;
    }
}

$needFrontSync = $savedCurrentDay && ($savedCurrentDay >= $startStr && $savedCurrentDay <= $endStr);

// Skip front sync only if data.txt is completely empty
if (empty($data)) {
    dbg("Data file is empty → first scrape, no front sync needed.");
    $needFrontSync = false;
}

// ✅ ALWAYS run newest-side sync first if current day is in range
if ($needFrontSync) {
    $firstKnownId = $data[0]['id'] ?? null;
    if ($firstKnownId) {
        dbg("Newest-side sync: current day in-range ($savedCurrentDay). Checking for posts newer than top id $firstKnownId.");
        $addedFront = front_sync_newest($data, $firstKnownId, $url, $headers, $TARGET_TZ, 50);

        if ($addedFront > 0) {
            dbg("Front sync added $addedFront posts. Checking for new current day...");

            // Step 1: Identify newest date after merge
            $allDates = array_unique(array_column($data, 'date'));
            rsort($allDates); // newest first
            $newestDate = $allDates[0] ?? null;

            if ($newestDate && $newestDate > $currentFromFile) {
                dbg("Newer day detected: $newestDate (old current was $currentFromFile)");

                // Step 2: Mark new current and validate only days between old current and new current
                foreach ($savedValMap as $day => $status) {
                    if ($day === $newestDate) {
                        $savedValMap[$day] = 'current';
                    } elseif ($day === $currentFromFile) {
                        $savedValMap[$day] = 'validated';
                    } elseif ($day < $newestDate && $day > $currentFromFile) {
                        $savedValMap[$day] = 'validated';
                    }
                }

                // Step 3: Add missing new days not present in map
                $cursor = $currentFromFile;
                while ($cursor < $newestDate) {
                    $cursor = date('Y-m-d', strtotime($cursor . ' +1 day'));
                    if (!isset($savedValMap[$cursor])) {
                        $savedValMap[$cursor] = ($cursor === $newestDate) ? 'current' : 'validated';
                    }
                }

                // Sort newest → oldest
                uksort($savedValMap, function($a, $b) {
                    return strcmp($b, $a);
                });
                   $frontSyncHandledFutureDays = true; 
                   list($valMapAfterFront, $allOkAfterFront) = compute_validation_map($data, $startStr, $endStr, false);
$missingInitial = detect_missing_days_from_map($valMapAfterFront);
dbg("Recomputed missing days after front sync: " . json_encode($missingInitial));
                dbg("Validation map updated after new posts: " . json_encode($savedValMap));
                
                // ✅ Recompute validation after front sync
list($valMapAfterFront, $allOkAfterFront) = compute_validation_map($data, $startStr, $endStr, false);

// If everything is validated now, stop and return JSON
if ($allOkAfterFront) {
    dbg("All requested days validated after front sync → no further scraping required.");

    // Sort newest → oldest
    uksort($valMapAfterFront, function($a, $b) {
        return strcmp($b, $a);
    });

    // Merge with savedValMap
    $mergedMap = $savedValMap;
    foreach ($valMapAfterFront as $day => $status) {
        $mergedMap[$day] = $status;
    }
    uksort($mergedMap, function($a, $b) { return strcmp($b, $a); });

    // Save to data.txt
$savePayload = [
    "country"        => $lockedCountry,
    "timezone"       => $lockedTzLabel,
    "validation_map" => $mergedMap,
    "posts"          => $data
];
    file_put_contents($dataFile, json_encode($savePayload, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE));

    // Prepare filtered output
    $out = filter_only($savePayload['posts'], $startStr, $endStr);

    echo json_encode([
        "status"          => "cached",
        "folder"          => $folder,
            "country_locked"  => $lockedCountry,  // ADD
        "timezone_used"   => $tzLabel,
        "message"         => "All requested days validated after front sync. No older scraping needed.",
        "validation_map"  => $valMapAfterFront,
        "round_target"    => 50,
        "total_captions"  => count($out),
        "total_data_txt"  => count($data),
        "daily_counts"    => daily_counts($out, $startStr, $endStr),
        "posts"           => $out,
        "saved_to"        => $dataFile,
        "debug"           => $DEBUG
    ], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE);
    exit;
}



            } else {
                dbg("No newer day found. Current day ($currentFromFile) remains unchanged.");
            }
        } else {
            dbg("Newest-side sync: no new posts found.");
        }
    }
}


// Handle case where requested END date is newer than the current date in data.txt
if ($handleFutureDays && !$frontSyncHandledFutureDays) {
    dbg("Handling future days after front sync...");

    // Get the newest date now present in the merged posts
    $allDates = array_unique(array_column($data, 'date'));
    rsort($allDates); // newest first
    $newCurrentDay = $allDates[0] ?? null;

    if ($newCurrentDay && $newCurrentDay > $currentFromFile) {
        dbg("New current day detected: $newCurrentDay (old was $currentFromFile)");

        // Update validation map
        foreach ($savedValMap as $day => $status) {
            if ($day === $newCurrentDay) {
                $savedValMap[$day] = 'current';
            } elseif ($day === $currentFromFile) {
                $savedValMap[$day] = 'validated';
            } elseif ($day > $currentFromFile && $day < $newCurrentDay) {
                $savedValMap[$day] = 'validated'; // intermediate days
            }
        }

        // Add missing new days if they aren't in the map yet
        $dayCursor = $currentFromFile;
        while ($dayCursor < $newCurrentDay) {
            $dayCursor = date('Y-m-d', strtotime($dayCursor . ' +1 day'));
            if (!isset($savedValMap[$dayCursor])) {
                $savedValMap[$dayCursor] = ($dayCursor === $newCurrentDay) ? 'current' : 'validated';
            }
        }

        // Sort newest → oldest
        uksort($savedValMap, function($a, $b) {
            return strcmp($b, $a);
        });
    }
}


// If after (possible) front-sync we still have nothing missing, we can return without older scraping
if (empty($missingInitial)) {
    dbg("Range validated (after newest-side sync if needed). No older scraping required.");
    // Prepare output and SAVE like usual so data.txt & validation_map are refreshed

    // Build final validation map (with absolute newest day marked 'current')
    list($finalMap, $finalAllOk) = compute_validation_map($data, $startStr, $endStr, false);
    // Sort newest → oldest for saving
    uksort($finalMap, function($a, $b){ return strcmp($b, $a); });

    // Save canonical data with validation_map on top
// ---------------- Merge validation_map before saving ----------------
$mergedMap = $savedValMap;

// Update existing days or add new ones
foreach ($finalMap as $day => $status) {
    $mergedMap[$day] = $status; // overwrite status if day already exists
}

// Sort newest → oldest for readability
uksort($mergedMap, function($a, $b) {
    return strcmp($b, $a);
});

// Save merged validation_map + posts
$savePayload = [
    "country"        => $lockedCountry,
    "timezone"       => $lockedTzLabel,
    "validation_map" => $mergedMap,
    "posts"          => $data
];
file_put_contents($dataFile, json_encode($savePayload, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE));


    // Prepare filtered output in the user TZ
$out = filter_only($savePayload['posts'], $startStr, $endStr);

    echo json_encode([
        "status"          => "cached",
        "folder"          => $folder,
            "country_locked"  => $lockedCountry,  // ADD
        "timezone_used"   => $tzLabel,
        "message"         => "Range validated. Newest-side sync applied (if needed).",
        "validation_map"  => $finalMap,
        "round_target"    => 50,
        "total_captions"  => count($out),
        "total_data_txt"  => count($data),
        "daily_counts"    => daily_counts($out, $startStr, $endStr),
        "posts"           => $out,
        "saved_to"        => $dataFile,
        "debug"           => $DEBUG
    ], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE);
    exit;
}

$firstUnvalidatedDay = null;
foreach ($valMapInitial as $day => $status) {
    if ($status !== 'validated' && $status !== 'current') {
        $firstUnvalidatedDay = $day;
        break;
    }
}

if ($firstUnvalidatedDay) {
    dbg("First unvalidated day found: $firstUnvalidatedDay");
}

// ---------------- Specific-day handling ----------------
if ($startStr === $endStr && $currentFromFile) {
    if ($endStr > $currentFromFile) {
        // Case B: requested day is newer than current in data.txt
        dbg("Specific-day newer than current ($endStr > $currentFromFile). Running front sync from newest → first known id.");

        $firstKnownId = $data[0]['id'] ?? null;
        if ($firstKnownId) {
            $added = front_sync_newest($data, $firstKnownId, $url, $headers, $TARGET_TZ, 50);
            dbg("Specific-day front sync merged $added new posts.");

            // Step 1: find absolute newest date after merge
            $allDates = array_unique(array_column($data, 'date'));
            rsort($allDates);
            $newestDate = $allDates[0] ?? null;

            if ($newestDate) {
    // Step 2: rebuild validation_map globally
    $mergedMap = $savedValMap;

    // Demote the old current if it still exists
    if (isset($mergedMap[$currentFromFile]) && $currentFromFile !== $newestDate) {
        $mergedMap[$currentFromFile] = 'validated';
    }

    // Walk from old current → new current
    $cursor = $currentFromFile;
    while ($cursor < $newestDate) {
        $cursor = date('Y-m-d', strtotime($cursor . ' +1 day'));
        $mergedMap[$cursor] = ($cursor === $newestDate) ? 'current' : 'validated';
    }

    // Ensure all posts’ days exist in the map
    foreach ($allDates as $d) {
        if (!isset($mergedMap[$d])) {
            $mergedMap[$d] = ($d === $newestDate) ? 'current' : 'validated';
        }
    }

    // Sort newest → oldest
    uksort($mergedMap, function($a, $b) {
        return strcmp($b, $a);
    });

    // Step 3: Save everything
    $savePayload = [
        "country"        => $lockedCountry,
        "timezone"       => $lockedTzLabel,
        "validation_map" => $mergedMap,
        "posts"          => $data
    ];
    file_put_contents($dataFile, json_encode($savePayload, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE));


                // Step 4: Filter only the requested day for output
                $out = filter_only($savePayload['posts'], $startStr, $endStr);

                echo json_encode([
                    "status"          => $allOkAfterFront ? "cached" : "partial",
                    "folder"          => $folder,
                    "country_locked"  => $lockedCountry,
                    "timezone_used"   => $tzLabel,
                    "message"         => "Specific-day validated after front sync. Validation map updated through $newestDate.",
                    "validation_map"  => $mergedMap,
                    "total_captions"  => count($out),
                    "total_data_txt"  => count($data),
                    "daily_counts"    => daily_counts($out, $startStr, $endStr),
                    "posts"           => $out,
                    "saved_to"        => $dataFile,
                    "debug"           => $DEBUG
                ], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE);
                exit;
            }
        }
    }
    // else → requested day <= currentFromFile → let normal pagination logic handle it
}



// ---------------- Scrape rounds (50 per round) ----------------
$ROUND_TARGET = 50;     // exactly what you asked
$timelineEnded = false; // set true when no more posts are available
$after = null;

if (!empty($data) && $firstUnvalidatedDay) {
    for ($i = count($data) - 1; $i >= 0; $i--) {
        $post = $data[$i];
        // Allow tokens from the same day OR earlier
        if ($post['date'] <= $firstUnvalidatedDay && isset($post['pagination']) && $post['pagination']) {
            $after = $post['pagination'];
            dbg("Found pagination token to resume from day $firstUnvalidatedDay or earlier → $after");
            break;
        }
    }
}

// Extra safety: if still no pagination token found, fallback to last post's token
if ($after === null && !empty($data)) {
    $lastPost = end($data);
    if (!empty($lastPost['pagination'])) {
        $after = $lastPost['pagination'];
        dbg("Fallback: Using last post's pagination token → $after");
    }
}


// If no pagination token found, stop early BUT not on very first scrape
if ($after === null && !empty($data)) {
    dbg("Pagination token is null → no older posts available, stopping before scrape.");
    echo json_encode([
        "status" => "done_at_end",
            "folder" => $folder,
    "country_locked" => $lockedCountry, // ADD
    "timezone_used" => $tzLabel,       // ADD
        "message" => "No more older posts to scrape. Timeline fully scanned.",
        "validation_map" => $valMapInitial,
        "posts" => filter_only($data, $startStr, $endStr),
        "debug" => $DEBUG
    ], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE);
    exit;
}


$round = 0;

dbg("Begin scraping rounds. Need days: " . implode(', ', $missingInitial));




while (true) {
    $round++;
    $addedThisRound = 0;
    $buffer = [];
    $pageCount = 0;
    dbg("Round #$round: target $ROUND_TARGET posts.");

    // --- INNER LOOP: Collect posts until 50 or no more pages ---
    while ($addedThisRound < $ROUND_TARGET) {
        $pageCount++;
        $json = fetch_page($url, $headers, $after);
        if (is_session_invalid($json)) {
    exit_session_invalid();
}

        if (isset($json['error'])) {
            dbg("Fetch error: ".$json['error']);
            $timelineEnded = true;
            break;
        }

        $edges = $json['data']['mediaData']['edges'] ?? [];
        $pi    = $json['data']['mediaData']['page_info'] ?? [];
        $has   = $pi['has_next_page'] ?? false;
        $endc  = $pi['end_cursor'] ?? null;

        if (empty($edges)) {
            dbg("No edges on this page. Assuming end of timeline.");
            $timelineEnded = true;
            break;
        }

        $batch = posts_from_edges($edges, $TARGET_TZ);
        if (empty($batch)) {
            dbg("Normalized batch empty. Assuming end.");
            $timelineEnded = true;
            break;
        }

        // Add batch to buffer
$buffer = array_merge($buffer, $batch);
$addedThisRound = count($buffer);
dbg("  Page $pageCount: collected $addedThisRound / $ROUND_TARGET this round.");

// --- Attach pagination token to LAST post in buffer
if (!empty($buffer)) {
    $lastIndex = count($buffer) - 1;
    $buffer[$lastIndex]['pagination'] = $endc ?? null;
    dbg($endc
        ? "Attached pagination token to last post of buffer: $endc"
        : "No pagination token returned → last post marked with pagination=null");
}

// --- Merge immediately and validate mid-round ---
$data = dedupe_posts(array_merge($data, $buffer));
reindex($data);

// If last post has pagination=null, timeline ended
$lastPost = end($data);
if (!isset($lastPost['pagination']) || $lastPost['pagination'] === null) {
    dbg("Pagination is null → validating all posts from last scrape automatically.");
    $timelineEnded = true;
}

// Mid-round validation
list($valMap, $allOk) = compute_validation_map($data, $startStr, $endStr, $timelineEnded);
dbg("Mid-round validation after page $pageCount: " . json_encode($valMap));

if ($allOk) {
    dbg("All requested days validated mid-round → stopping early.");
    break 2;
} elseif (!in_array($firstUnvalidatedDay, array_keys($valMap))) {
    dbg("Unvalidated day '$firstUnvalidatedDay' now validated → stopping scrape.");
    break 2;
}

        // Attach pagination token to last post

        if (!$has || !$endc) {
            dbg("  No more pages from server.");
            $timelineEnded = true;
            break;
        }

        $after = $endc;
    }

    // --- MERGE BUFFER INTO DATA ---
    if (!empty($buffer)) {
        dbg("Merging ".count($buffer)." new posts from round #$round.");
        $data = dedupe_posts(array_merge($data, $buffer));
        reindex($data);

        // If last post has pagination = null, force validation
        $lastPost = end($data);
        if (!isset($lastPost['pagination']) || $lastPost['pagination'] === null) {
            dbg("Pagination is null → validating all posts from last scrape automatically.");
            $timelineEnded = true;
        }
    } else {
        dbg("Round #$round merged 0 posts.");
    }

    // --- VALIDATE UPDATED DATA ---
    list($valMap, $allOk) = compute_validation_map($data, $startStr, $endStr, $timelineEnded);
    dbg("Validation after round #$round: " . json_encode($valMap));

    if ($allOk) {
        dbg("All requested days validated. Stopping.");
        break;
    }
    if ($timelineEnded) {
        dbg("End of timeline reached. Stopping.");
        break;
    }
}



// ---------------- Final validation map ----------------
list($finalMap, $finalAllOk) = compute_validation_map($data, $startStr, $endStr, $timelineEnded);

// Sort validation_map newest → oldest
uksort($finalMap, function($a, $b) {
    return strcmp($b, $a);
});

// --- Ensure every day between start and end exists in validation_map ---
// Get current date in target timezone
$todayInTarget = (new DateTime('now', $TARGET_TZ))->format('Y-m-d');

for ($ts = strtotime($startStr); $ts <= strtotime($endStr); $ts += 86400) {
    $day = date('Y-m-d', $ts);

    // If this day is missing completely
    if (!isset($finalMap[$day])) {
        // If it's the very last day (today), mark as "current"
        if ($day === $todayInTarget || $day === $endStr) {
            $finalMap[$day] = 'current';
        } else {
            // Otherwise, mark as validated
            $finalMap[$day] = 'validated';
        }
    }
}

// Sort newest → oldest again after adding missing days
uksort($finalMap, function($a, $b) {
    return strcmp($b, $a);
});


// ---------------- Save canonical data ----------------
// ---------------- Merge validation_map before saving ----------------
$mergedMap = $savedValMap;

// Update existing days or add new ones
foreach ($finalMap as $day => $status) {
    $mergedMap[$day] = $status; // overwrite status if day already exists
}

// Sort newest → oldest for readability
uksort($mergedMap, function($a, $b) {
    return strcmp($b, $a);
});

// Save merged validation_map + posts
$savePayload = [
    "country"        => $lockedCountry,
    "timezone"       => $lockedTzLabel,
    "validation_map" => $mergedMap,
    "posts"          => $data
];
file_put_contents($dataFile, json_encode($savePayload, JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE));


// ---------------- Prepare output ----------------
$out = filter_only($savePayload['posts'], $startStr, $endStr);


// ---------------- Final response ----------------
echo json_encode([
    "status"          => $finalAllOk ? "scraped" : ($timelineEnded ? "done_at_end" : "partial"),
    "folder"          => $folder,
    "country_locked"  => $lockedCountry,   // <-- ADD THIS
    "timezone_used"   => $tzLabel,         // already exists, keep it
    "message"         => $finalAllOk
        ? "All requested days have been validated."
        : ($timelineEnded
            ? "Stopped at end of timeline; remaining days (if any) validated by end-of-timeline rule."
            : "Stopped early (limits/time). Some days may still be invalidated."),
    "validation_map"  => $finalMap,
    "round_target"    => 50,
    "total_captions"  => count($out),
    "total_data_txt"  => count($data),
    "daily_counts"    => daily_counts($out, $startStr, $endStr),
    "posts"           => $out,
    "saved_to"        => $dataFile,
    "debug"           => $DEBUG
], JSON_PRETTY_PRINT | JSON_UNESCAPED_UNICODE);
