Python Requests代理爬虫

export PROXY_USER="替换为控制台代理用户名"
export PROXY_PASS="替换为控制台代理密码"
# 隧道代理固定网关：proxy.123proxy.cn:36923

$env:PROXY_USER="替换为控制台代理用户名"
$env:PROXY_PASS="替换为控制台代理密码"
# 隧道代理固定网关：proxy.123proxy.cn:36923

# requirements:
#   python -m pip install requests beautifulsoup4
import json
import os
import time
from urllib.parse import quote, urljoin

import requests
from bs4 import BeautifulSoup
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry

PROXY_HOST = "proxy.123proxy.cn"
PROXY_PORT = 36923
PROXY_USER = os.environ["PROXY_USER"]
PROXY_PASS = os.environ["PROXY_PASS"]
START_URL = "https://quotes.toscrape.com/"
MAX_PAGES = 3

proxy_user = quote(PROXY_USER, safe="")
proxy_pass = quote(PROXY_PASS, safe="")
proxy_url = (
    f"http://{proxy_user}:{proxy_pass}@{PROXY_HOST}:{PROXY_PORT}"
)
proxies = {"http": proxy_url, "https": proxy_url}

retry = Retry(
    total=3,
    backoff_factor=1,
    status_forcelist=(429, 500, 502, 503, 504),
    allowed_methods=("GET",),
)
session = requests.Session()
session.headers.update({
    "User-Agent": "123Proxy-Developer-Example/1.0",
    "Accept-Language": "en-US,en;q=0.8",
})
session.mount("https://", HTTPAdapter(max_retries=retry))

records = []
url = START_URL
for page_number in range(1, MAX_PAGES + 1):
    response = session.get(
        url,
        proxies=proxies,
        timeout=(15, 30),
    )
    response.raise_for_status()
    soup = BeautifulSoup(response.text, "html.parser")

    for quote_node in soup.select("div.quote"):
        records.append({
            "text": quote_node.select_one("span.text").get_text(strip=True),
            "author": quote_node.select_one("small.author").get_text(strip=True),
            "tags": [
                node.get_text(strip=True)
                for node in quote_node.select("a.tag")
            ],
            "source_url": url,
        })

    next_link = soup.select_one("li.next > a")
    if not next_link:
        break
    url = urljoin(url, next_link["href"])
    time.sleep(0.5)

with open("quotes.jsonl", "w", encoding="utf-8") as output:
    for record in records:
        output.write(json.dumps(record, ensure_ascii=False) + "\n")

print(f"saved {len(records)} records to quotes.jsonl")