huyue
497fae99c6
FastAPI + Playwright(headed Chrome/xvfb 过 Cloudflare)抓内容详情页元数据; 可选把 m3u8 视频转 mp4 + 封面上传对象存储(配置由调用方下发,本服务不持密钥); 走主站 lobster 执行器契约(invoke/poll),回传 ResourceIngestItem 形式的 resources。 当前内置 xchina.co 的 scraper(core/scraper.py)。
223 lines
8.6 KiB
Python
223 lines
8.6 KiB
Python
"""内存级作业注册表 + 后台执行。
|
|
|
|
- 单并发:同一时刻只跑一个浏览器作业(xchina 怕风控);用 asyncio.Lock 串行化。
|
|
- 作业状态只存进程内存,重启即丢 —— 主站靠轮询 GET /tasks/{id} 拿结果,够用。
|
|
- 流程:① headed Chrome/xvfb 抓元数据 ② 若给了完整 MinIO 配置则逐条下载 m3u8→mp4 + 封面,
|
|
上传 MinIO,把 video_url/cover_image 改成 MinIO 地址;失败的条目降级保留原 URL。
|
|
- 临时下载目录用完即删,不长期落盘。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import time
|
|
import uuid
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any, Optional
|
|
|
|
from core.browser import browser_context
|
|
from core.media import (
|
|
MinioConfig,
|
|
download_cover,
|
|
download_video,
|
|
guess_content_type,
|
|
upload_to_minio,
|
|
)
|
|
from core.scraper import ScrapeCancelled, XChinaScraper
|
|
|
|
logger = logging.getLogger("harvester.jobs")
|
|
|
|
_RUN_LOCK = asyncio.Lock()
|
|
|
|
TMP_ROOT = Path(os.environ.get("TMP_DIR", "/tmp")) / "content-harvester"
|
|
CONCURRENT_FRAGMENTS = int(os.environ.get("CONCURRENT_FRAGMENTS", "8"))
|
|
MAX_VIDEO_MB = int(os.environ.get("MAX_VIDEO_MB", "0")) # 0 = 不限
|
|
|
|
|
|
@dataclass
|
|
class Job:
|
|
id: str
|
|
series_url: str
|
|
source_name: str
|
|
start_page: int
|
|
end_page: int
|
|
minio_cfg: MinioConfig = field(default_factory=MinioConfig)
|
|
status: str = "running" # running | succeeded | failed
|
|
output: Optional[dict[str, Any]] = None
|
|
error: Optional[str] = None
|
|
created_at: float = field(default_factory=time.time)
|
|
finished_at: Optional[float] = None
|
|
log_lines: list[str] = field(default_factory=list)
|
|
_cancelled: bool = False
|
|
_task: Optional[asyncio.Task] = None
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"task_id": self.id,
|
|
"status": self.status,
|
|
"output": self.output,
|
|
"error": self.error,
|
|
"created_at": self.created_at,
|
|
"finished_at": self.finished_at,
|
|
"log_tail": self.log_lines[-80:],
|
|
}
|
|
|
|
|
|
_JOBS: dict[str, Job] = {}
|
|
|
|
|
|
def get_job(job_id: str) -> Optional[Job]:
|
|
return _JOBS.get(job_id)
|
|
|
|
|
|
def _log(job: Job, msg: str) -> None:
|
|
# 注意:不要把 secret_key 等敏感值传进来
|
|
line = f"{time.strftime('%H:%M:%S')} {msg}"
|
|
job.log_lines.append(line)
|
|
if len(job.log_lines) > 800:
|
|
del job.log_lines[: len(job.log_lines) - 800]
|
|
logger.info("[job %s] %s", job.id, msg)
|
|
|
|
|
|
def _bytes_h(n: int) -> str:
|
|
f = float(n)
|
|
for unit in ("B", "KB", "MB", "GB"):
|
|
if f < 1024 or unit == "GB":
|
|
return f"{f:.1f}{unit}"
|
|
f /= 1024
|
|
return f"{f:.1f}GB"
|
|
|
|
|
|
async def _process_uploads(job: Job, result, tmp_dir: Path) -> None:
|
|
"""对 result.resources 逐条:下载视频+封面 → 上传 MinIO → 回写 video_url/cover_image。失败降级。"""
|
|
cfg = job.minio_cfg
|
|
total = len(result.resources)
|
|
for idx, item in enumerate(result.resources, 1):
|
|
if job._cancelled:
|
|
raise ScrapeCancelled()
|
|
vid = item.video_id or f"item{idx}"
|
|
item_dir = tmp_dir / vid
|
|
try:
|
|
if not item.video_url:
|
|
item.extra["upload_skipped_reason"] = "no video url"
|
|
_log(job, f"[{idx}/{total}] {item.title}: 无视频地址,跳过上传")
|
|
continue
|
|
_log(job, f"[{idx}/{total}] {item.title}: 下载视频 …")
|
|
mp4_path = await asyncio.to_thread(
|
|
download_video, item.video_url, item.source_url, item_dir, vid,
|
|
CONCURRENT_FRAGMENTS, MAX_VIDEO_MB,
|
|
)
|
|
size = mp4_path.stat().st_size
|
|
video_key = f"xchina/{vid}{mp4_path.suffix.lower() or '.mp4'}"
|
|
_log(job, f"[{idx}/{total}] {item.title}: 视频 {_bytes_h(size)},上传 MinIO {video_key} …")
|
|
video_public = await asyncio.to_thread(
|
|
upload_to_minio, mp4_path, video_key, "video/mp4", cfg,
|
|
)
|
|
item.extra["original_m3u8"] = item.video_url
|
|
item.extra["minio_object_video"] = video_key
|
|
item.extra["video_bytes"] = size
|
|
item.video_url = video_public
|
|
|
|
# 封面(失败不致命)
|
|
if item.cover_image:
|
|
cover_path = await asyncio.to_thread(
|
|
download_cover, item.cover_image, item.source_url, item_dir, vid,
|
|
)
|
|
if cover_path is not None:
|
|
cover_key = f"xchina/{vid}{cover_path.suffix.lower() or '.jpg'}"
|
|
try:
|
|
cover_public = await asyncio.to_thread(
|
|
upload_to_minio, cover_path, cover_key, guess_content_type(cover_path, "image/jpeg"), cfg,
|
|
)
|
|
item.extra["original_cover"] = item.cover_image
|
|
item.extra["minio_object_cover"] = cover_key
|
|
item.cover_image = cover_public
|
|
except Exception as exc: # noqa: BLE001
|
|
_log(job, f"[{idx}/{total}] {item.title}: 封面上传失败({exc}),保留原图床地址")
|
|
_log(job, f"[{idx}/{total}] {item.title}: OK -> {item.video_url}")
|
|
except ScrapeCancelled:
|
|
raise
|
|
except Exception as exc: # noqa: BLE001
|
|
reason = f"{type(exc).__name__}: {exc}"
|
|
item.extra["upload_skipped_reason"] = reason
|
|
_log(job, f"[{idx}/{total}] {item.title}: 下载/上传失败,降级保留原 m3u8 地址 -> {reason}")
|
|
finally:
|
|
shutil.rmtree(item_dir, ignore_errors=True)
|
|
|
|
|
|
async def _run(job: Job, headless: bool) -> None:
|
|
tmp_dir = TMP_ROOT / job.id
|
|
try:
|
|
async with _RUN_LOCK:
|
|
_log(job, f"开始采集: {job.series_url} (page {job.start_page}-{job.end_page})")
|
|
if job.minio_cfg.is_complete:
|
|
_log(job, f"启用 MinIO 上传: {job.minio_cfg.safe_repr()}")
|
|
else:
|
|
_log(job, "未提供完整 MinIO 配置,本次不上传(video_url 保留原始 m3u8 地址)")
|
|
|
|
async with browser_context(headless=headless) as ctx:
|
|
scraper = XChinaScraper(
|
|
ctx,
|
|
is_cancelled=lambda: job._cancelled,
|
|
log=lambda m: _log(job, m),
|
|
)
|
|
result = await scraper.run(job.series_url, job.start_page, job.end_page)
|
|
# 浏览器已关闭;开始下载+上传(若配置完整)
|
|
if job.minio_cfg.is_complete:
|
|
await _process_uploads(job, result, tmp_dir)
|
|
else:
|
|
for item in result.resources:
|
|
item.extra.setdefault("upload_skipped_reason", "minio config absent")
|
|
|
|
# 统一覆盖 source_name(invoke 里指定的 platform_name)
|
|
for item in result.resources:
|
|
item.source_name = job.source_name
|
|
output = result.to_output()
|
|
job.output = output
|
|
if not output["resources"] and output["item_errors"]:
|
|
job.status = "failed"
|
|
job.error = "no resources scraped; " + "; ".join(
|
|
e.get("reason", "") for e in output["item_errors"][:3]
|
|
)
|
|
else:
|
|
job.status = "succeeded"
|
|
uploaded = sum(1 for r in output["resources"] if r.get("extra_data", {}).get("minio_object_video"))
|
|
_log(job, f"完成: {len(output['resources'])} 条(其中 {uploaded} 条已上传 MinIO), 页数 {output['pages_scanned']}, 抓取失败 {len(output['item_errors'])}")
|
|
except ScrapeCancelled:
|
|
job.status = "failed"
|
|
job.error = "cancelled"
|
|
_log(job, "已取消")
|
|
except Exception as exc: # noqa: BLE001
|
|
job.status = "failed"
|
|
job.error = f"{type(exc).__name__}: {exc}"
|
|
_log(job, f"失败: {job.error}")
|
|
finally:
|
|
shutil.rmtree(tmp_dir, ignore_errors=True)
|
|
job.finished_at = time.time()
|
|
|
|
|
|
def submit(series_url: str, source_name: str, start_page: int, end_page: int,
|
|
headless: bool = True, minio_cfg: Optional[MinioConfig] = None) -> Job:
|
|
job = Job(
|
|
id=uuid.uuid4().hex,
|
|
series_url=series_url,
|
|
source_name=source_name or "xchina",
|
|
start_page=max(1, start_page),
|
|
end_page=max(start_page, end_page),
|
|
minio_cfg=minio_cfg or MinioConfig(),
|
|
)
|
|
_JOBS[job.id] = job
|
|
job._task = asyncio.create_task(_run(job, headless))
|
|
return job
|
|
|
|
|
|
def cancel(job_id: str) -> bool:
|
|
job = _JOBS.get(job_id)
|
|
if not job or job.status != "running":
|
|
return False
|
|
job._cancelled = True
|
|
return True
|