Mettre des URL en file et les traiter

// Queue and Process — Use work queues for batch tasks
//
// Queues let you line up items for processing, with built-in
// retry logic and dead-letter handling. Great for crawling
// lists of pages or processing data in order.
// Requires an open workspace.

const queueName = "pages_to_visit";

// Enqueue several items
const urls = [
    "https://example.com/page1",
    "https://example.com/page2",
    "https://example.com/page3"
];

for (const url of urls) {
    g.queue.enqueue(queueName, {
        url: url,
        addedAt: new Date().toISOString()
    });
}

g.log("Enqueued " + urls.length + " items");

// Check queue size
const count = g.queue.count(queueName);
g.log("Queue has " + count + " item(s)");

// Dequeue and process each item
let processed = 0;
while (true) {
    const item = g.queue.dequeue(queueName);
    if (!item) break;

    g.log("Processing: " + item.data.url);

    // Do your work here (navigate, scrape, etc.)
    // ...

    // Commit to remove from queue, or abort to retry later
    g.queue.commit(item.id);
    processed++;
}

g.log("Processed " + processed + " item(s)");