"use node";
// For more information, see https://crawlee.dev/
import { Configuration, PlaywrightCrawler, ProxyConfiguration } from "crawlee";
import { router } from "./routes.js";
import { action } from "./_generated/server.js";

export const runCrawlee = action({
    args: {},
    handler: async () => {
        const startUrls = ["https://crawlee.dev/"];

        const crawler = new PlaywrightCrawler(
            {
                // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
                requestHandler: router,
                // Comment this option to scrape the full website.
                maxRequestsPerCrawl: 20,
            },
            new Configuration({ persistStorage: false })
        );

        await crawler.run(startUrls);

        return await crawler.getData();
    },
});

"use node";
// For more information, see https://crawlee.dev/
import { Configuration, PlaywrightCrawler, ProxyConfiguration } from "crawlee";
import { router } from "./routes.js";
import { action } from "./_generated/server.js";

export const runCrawlee = action({
    args: {},
    handler: async () => {
        const startUrls = ["https://crawlee.dev/"];

        const crawler = new PlaywrightCrawler(
            {
                // proxyConfiguration: new ProxyConfiguration({ proxyUrls: ['...'] }),
                requestHandler: router,
                // Comment this option to scrape the full website.
                maxRequestsPerCrawl: 20,
            },
            new Configuration({ persistStorage: false })
        );

        await crawler.run(startUrls);

        return await crawler.getData();
    },
});

"use node";
import { createPlaywrightRouter } from "crawlee";

export const router = createPlaywrightRouter();

router.addDefaultHandler(async ({ enqueueLinks, log }) => {
    log.info(`enqueueing new URLs`);
    await enqueueLinks({
        globs: ["https://crawlee.dev/**"],
        label: "detail",
    });
});

router.addHandler("detail", async ({ request, page, log, pushData }) => {
    const title = await page.title();
    log.info(`${title}`, { url: request.loadedUrl });

    await pushData({
        url: request.loadedUrl,
        title,
    });
});

"use node";
import { createPlaywrightRouter } from "crawlee";

export const router = createPlaywrightRouter();

router.addDefaultHandler(async ({ enqueueLinks, log }) => {
    log.info(`enqueueing new URLs`);
    await enqueueLinks({
        globs: ["https://crawlee.dev/**"],
        label: "detail",
    });
});

router.addHandler("detail", async ({ request, page, log, pushData }) => {
    const title = await page.title();
    log.info(`${title}`, { url: request.loadedUrl });

    await pushData({
        url: request.loadedUrl,
        title,
    });
});