Node.js Puppeteer代理采集

export PROXY_USER="替换为控制台代理用户名"
export PROXY_PASS="替换为控制台代理密码"
# 隧道代理固定网关：proxy.123proxy.cn:36923

$env:PROXY_USER="替换为控制台代理用户名"
$env:PROXY_PASS="替换为控制台代理密码"
# 隧道代理固定网关：proxy.123proxy.cn:36923

// requirements:
//   npm init -y
//   npm install puppeteer
// run:
//   node scrape-scroll.js
const fs = require("node:fs/promises");
const puppeteer = require("puppeteer");

const PROXY_HOST = "proxy.123proxy.cn";
const PROXY_PORT = 36923;
const PROXY_USER = process.env.PROXY_USER;
const PROXY_PASS = process.env.PROXY_PASS;
const TARGET_URL = "https://quotes.toscrape.com/scroll";

if (!PROXY_USER || !PROXY_PASS) {
  throw new Error("set PROXY_USER and PROXY_PASS first");
}

async function main() {
  const browser = await puppeteer.launch({
    headless: true,
    args: [
      "--proxy-server=http://" + PROXY_HOST + ":" + PROXY_PORT,
    ],
  });
  const page = await browser.newPage();
  await page.authenticate({
    username: PROXY_USER,
    password: PROXY_PASS,
  });
  await page.setUserAgent("123Proxy-Developer-Example/1.0");
  page.setDefaultNavigationTimeout(45_000);

  await page.goto(TARGET_URL, { waitUntil: "networkidle2" });
  await page.waitForSelector("div.quote");

  let previousCount = 0;
  let stableRounds = 0;
  for (let round = 0; round < 30 && stableRounds < 3; round += 1) {
    const count = await page.$$eval(
      "div.quote",
      (nodes) => nodes.length,
    );
    stableRounds = count === previousCount ? stableRounds + 1 : 0;
    previousCount = count;
    await page.evaluate(() => {
      window.scrollTo(0, document.body.scrollHeight);
    });
    await new Promise((resolve) => setTimeout(resolve, 900));
  }

  const records = await page.$$eval("div.quote", (nodes) =>
    nodes.map((node) => ({
      text: node.querySelector("span.text")?.textContent.trim(),
      author: node.querySelector("small.author")?.textContent.trim(),
      tags: [...node.querySelectorAll("a.tag")]
        .map((tag) => tag.textContent.trim()),
      source_url: location.href,
    }))
  );

  const uniqueRecords = [
    ...new Map(
      records.map((record) => [
        record.author + "\u0000" + record.text,
        record,
      ]),
    ).values(),
  ];
  await fs.writeFile(
    "quotes-puppeteer.json",
    JSON.stringify(uniqueRecords, null, 2),
    "utf8",
  );
  console.log(
    "saved " + uniqueRecords.length + " records to quotes-puppeteer.json",
  );
  await browser.close();
}

main().catch((error) => {
  console.error(error);
  process.exitCode = 1;
});