huyue
497fae99c6
FastAPI + Playwright(headed Chrome/xvfb 过 Cloudflare)抓内容详情页元数据; 可选把 m3u8 视频转 mp4 + 封面上传对象存储(配置由调用方下发,本服务不持密钥); 走主站 lobster 执行器契约(invoke/poll),回传 ResourceIngestItem 形式的 resources。 当前内置 xchina.co 的 scraper(core/scraper.py)。
59 lines
2.1 KiB
Python
59 lines
2.1 KiB
Python
"""Playwright 浏览器上下文 —— 无头、持久化资料目录。
|
|
|
|
持久目录(BROWSER_PROFILE_DIR,默认 ./browser_profile)用于复用登录态 / Cookie /
|
|
过验证后的状态。无头服务器上若 xchina 弹 Cloudflare,没有人工过验证的退路 ——
|
|
运维需一次性在桌面浏览器里登录并把 profile 目录拷进该路径(容器里挂卷)。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from contextlib import asynccontextmanager
|
|
from pathlib import Path
|
|
|
|
from playwright.async_api import async_playwright
|
|
|
|
DEFAULT_USER_AGENT = (
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
|
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0 Safari/537.36"
|
|
)
|
|
|
|
PROFILE_DIR = Path(os.environ.get("BROWSER_PROFILE_DIR", Path(__file__).resolve().parent.parent / "browser_profile"))
|
|
# 用真实 Google Chrome 通道(BROWSER_CHANNEL=chrome) + 有头(HEADLESS=0,容器里靠 xvfb)
|
|
# 对绕 Cloudflare 的 JS 质询效果最好;留空则用 Playwright 自带 Chromium
|
|
BROWSER_CHANNEL = os.environ.get("BROWSER_CHANNEL", "").strip() or None
|
|
|
|
_LAUNCH_ARGS = [
|
|
"--disable-blink-features=AutomationControlled",
|
|
"--disable-dev-shm-usage",
|
|
"--no-first-run",
|
|
"--no-default-browser-check",
|
|
"--no-sandbox",
|
|
]
|
|
|
|
|
|
@asynccontextmanager
|
|
async def browser_context(headless: bool = True):
|
|
"""打开一个持久化 BrowserContext,用完自动关闭。"""
|
|
PROFILE_DIR.mkdir(parents=True, exist_ok=True)
|
|
pw = await async_playwright().start()
|
|
context = None
|
|
try:
|
|
kwargs = dict(
|
|
headless=headless,
|
|
user_agent=DEFAULT_USER_AGENT,
|
|
viewport={"width": 1366, "height": 900},
|
|
locale="zh-CN",
|
|
timezone_id="Asia/Shanghai",
|
|
accept_downloads=False,
|
|
args=_LAUNCH_ARGS,
|
|
)
|
|
if BROWSER_CHANNEL:
|
|
kwargs["channel"] = BROWSER_CHANNEL
|
|
context = await pw.chromium.launch_persistent_context(str(PROFILE_DIR), **kwargs)
|
|
context.set_default_timeout(20000)
|
|
yield context
|
|
finally:
|
|
if context is not None:
|
|
await context.close()
|
|
await pw.stop()
|