huyueandClaude Opus 4.7
a1a93cc8e0
新增图集 scraper(core/scraper_photos.py,继承 XChinaScraper 复用导航/反爬 helper):
系列页 .item.photo 翻页 → 详情页解析图片 URL 模板 + 总张数 + 模特/系列/日期/标签 →
construct-or-paginate 收集 images[](默认每套图最多 max_images=100 张)。
- ScrapedItem 加 media_kind(video|photo)/images/image_total/image_capped;to_ingest_dict 按
media_kind 分流(photo → source_type=image / content_type=article / images[] / video_url=None)
- media.py 加 download_image / head_ok;extract_video_id 兼容 /photo/id-...(=extract_detail_id)
- jobs.py _process_uploads 按 media_kind 分流(视频:yt-dlp/ffmpeg;图集:逐张下载上传,边下边传边删)
- app.py /agents/{id}/invoke 按 content_url 路径 / agent_id / 载荷 media_kind 自动选 scraper;
新增 max_images 参数(默认 DEFAULT_MAX_IMAGES=100,上限 MAX_IMAGES_CAP=2000)
- 主站不用改:复用现有采集 Skill,harvester 自动判图集/视频。图集回传 images[] 后主站自动跑图片 OCR 审核
- 已在 dev 实测:单图集详情页 / 系列页 / 视频回归 均通过
Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
309 lines
12 KiB
Python
309 lines
12 KiB
Python
"""内存级作业注册表 + 后台执行。
|
|
|
|
- 单并发:同一时刻只跑一个浏览器作业(xchina 怕风控);用 asyncio.Lock 串行化。
|
|
- 作业状态只存进程内存,重启即丢 —— 主站靠轮询 GET /tasks/{id} 拿结果,够用。
|
|
- 流程:① headed Chrome/xvfb 抓元数据 ② 若给了完整 MinIO 配置则逐条下载 m3u8→mp4 + 封面,
|
|
上传 MinIO,把 video_url/cover_image 改成 MinIO 地址;失败的条目降级保留原 URL。
|
|
- 临时下载目录用完即删,不长期落盘。
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import logging
|
|
import os
|
|
import shutil
|
|
import time
|
|
import uuid
|
|
from dataclasses import dataclass, field
|
|
from pathlib import Path
|
|
from typing import Any, Optional
|
|
|
|
from core.browser import browser_context
|
|
from core.media import (
|
|
MinioConfig,
|
|
download_cover,
|
|
download_image,
|
|
download_video,
|
|
guess_content_type,
|
|
upload_to_minio,
|
|
)
|
|
from core.scraper import ScrapeCancelled, XChinaScraper
|
|
|
|
logger = logging.getLogger("harvester.jobs")
|
|
|
|
_RUN_LOCK = asyncio.Lock()
|
|
|
|
TMP_ROOT = Path(os.environ.get("TMP_DIR", "/tmp")) / "content-harvester"
|
|
CONCURRENT_FRAGMENTS = int(os.environ.get("CONCURRENT_FRAGMENTS", "8"))
|
|
MAX_VIDEO_MB = int(os.environ.get("MAX_VIDEO_MB", "0")) # 0 = 不限
|
|
|
|
|
|
@dataclass
|
|
class Job:
|
|
id: str
|
|
series_url: str
|
|
source_name: str
|
|
start_page: int
|
|
end_page: int
|
|
minio_cfg: MinioConfig = field(default_factory=MinioConfig)
|
|
media_kind: str = "auto" # auto | video | photo
|
|
max_images: int = 100 # 图集每个套图最多抓多少张(media_kind=photo 时)
|
|
status: str = "running" # running | succeeded | failed
|
|
output: Optional[dict[str, Any]] = None
|
|
error: Optional[str] = None
|
|
created_at: float = field(default_factory=time.time)
|
|
finished_at: Optional[float] = None
|
|
log_lines: list[str] = field(default_factory=list)
|
|
_cancelled: bool = False
|
|
_task: Optional[asyncio.Task] = None
|
|
|
|
def to_dict(self) -> dict[str, Any]:
|
|
return {
|
|
"task_id": self.id,
|
|
"status": self.status,
|
|
"output": self.output,
|
|
"error": self.error,
|
|
"created_at": self.created_at,
|
|
"finished_at": self.finished_at,
|
|
"log_tail": self.log_lines[-80:],
|
|
}
|
|
|
|
|
|
_JOBS: dict[str, Job] = {}
|
|
|
|
|
|
def get_job(job_id: str) -> Optional[Job]:
|
|
return _JOBS.get(job_id)
|
|
|
|
|
|
def _log(job: Job, msg: str) -> None:
|
|
# 注意:不要把 secret_key 等敏感值传进来
|
|
line = f"{time.strftime('%H:%M:%S')} {msg}"
|
|
job.log_lines.append(line)
|
|
if len(job.log_lines) > 800:
|
|
del job.log_lines[: len(job.log_lines) - 800]
|
|
logger.info("[job %s] %s", job.id, msg)
|
|
|
|
|
|
def _bytes_h(n: int) -> str:
|
|
f = float(n)
|
|
for unit in ("B", "KB", "MB", "GB"):
|
|
if f < 1024 or unit == "GB":
|
|
return f"{f:.1f}{unit}"
|
|
f /= 1024
|
|
return f"{f:.1f}GB"
|
|
|
|
|
|
async def _process_uploads(job: Job, result, tmp_dir: Path) -> None:
|
|
"""对 result.resources 逐条:下载媒体 → 上传 MinIO → 回写地址。失败降级保留原地址。"""
|
|
total = len(result.resources)
|
|
for idx, item in enumerate(result.resources, 1):
|
|
if job._cancelled:
|
|
raise ScrapeCancelled()
|
|
vid = item.video_id or f"item{idx}"
|
|
item_dir = tmp_dir / vid
|
|
try:
|
|
if getattr(item, "media_kind", "video") == "photo":
|
|
await _process_photo_item(job, item, idx, total, item_dir)
|
|
else:
|
|
await _process_video_item(job, item, idx, total, vid, item_dir)
|
|
except ScrapeCancelled:
|
|
raise
|
|
except Exception as exc: # noqa: BLE001
|
|
reason = f"{type(exc).__name__}: {exc}"
|
|
item.extra["upload_skipped_reason"] = reason
|
|
_log(job, f"[{idx}/{total}] {item.title}: 下载/上传失败,降级保留原地址 -> {reason}")
|
|
finally:
|
|
shutil.rmtree(item_dir, ignore_errors=True)
|
|
|
|
|
|
async def _process_video_item(job: Job, item, idx: int, total: int, vid: str, item_dir: Path) -> None:
|
|
cfg = job.minio_cfg
|
|
if not item.video_url:
|
|
item.extra["upload_skipped_reason"] = "no video url"
|
|
_log(job, f"[{idx}/{total}] {item.title}: 无视频地址,跳过上传")
|
|
return
|
|
_log(job, f"[{idx}/{total}] {item.title}: 下载视频 …")
|
|
mp4_path = await asyncio.to_thread(
|
|
download_video, item.video_url, item.source_url, item_dir, vid,
|
|
CONCURRENT_FRAGMENTS, MAX_VIDEO_MB,
|
|
)
|
|
size = mp4_path.stat().st_size
|
|
video_key = f"xchina/{vid}{mp4_path.suffix.lower() or '.mp4'}"
|
|
_log(job, f"[{idx}/{total}] {item.title}: 视频 {_bytes_h(size)},上传 MinIO {video_key} …")
|
|
video_public = await asyncio.to_thread(upload_to_minio, mp4_path, video_key, "video/mp4", cfg)
|
|
item.extra["original_m3u8"] = item.video_url
|
|
item.extra["minio_object_video"] = video_key
|
|
item.extra["video_bytes"] = size
|
|
item.video_url = video_public
|
|
|
|
if item.cover_image:
|
|
cover_path = await asyncio.to_thread(download_cover, item.cover_image, item.source_url, item_dir, vid)
|
|
if cover_path is not None:
|
|
cover_key = f"xchina/{vid}{cover_path.suffix.lower() or '.jpg'}"
|
|
try:
|
|
cover_public = await asyncio.to_thread(
|
|
upload_to_minio, cover_path, cover_key, guess_content_type(cover_path, "image/jpeg"), cfg,
|
|
)
|
|
item.extra["original_cover"] = item.cover_image
|
|
item.extra["minio_object_cover"] = cover_key
|
|
item.cover_image = cover_public
|
|
except Exception as exc: # noqa: BLE001
|
|
_log(job, f"[{idx}/{total}] {item.title}: 封面上传失败({exc}),保留原图床地址")
|
|
_log(job, f"[{idx}/{total}] {item.title}: OK -> {item.video_url}")
|
|
|
|
|
|
async def _process_photo_item(job: Job, item, idx: int, total: int, item_dir: Path) -> None:
|
|
"""逐张下载图集图片 → 上传 MinIO → 把 item.images 换成 MinIO 地址(边下边传边删,不囤盘)。"""
|
|
cfg = job.minio_cfg
|
|
detail_id = item.video_id or f"item{idx}"
|
|
src_images = list(item.images or [])
|
|
if not src_images:
|
|
item.extra["upload_skipped_reason"] = "no images"
|
|
_log(job, f"[{idx}/{total}] {item.title}: 图集无图片地址,跳过上传")
|
|
return
|
|
n = len(src_images)
|
|
_log(job, f"[{idx}/{total}] {item.title}: 图集 {n} 张,逐张下载并上传 MinIO …")
|
|
out_urls: list[str] = []
|
|
out_keys: list[str] = []
|
|
ok_cnt = fail_cnt = 0
|
|
cover_public: str | None = None
|
|
for i, src in enumerate(src_images, 1):
|
|
if job._cancelled:
|
|
raise ScrapeCancelled()
|
|
path = await asyncio.to_thread(download_image, src, item.source_url, item_dir, f"{i:04d}")
|
|
if path is None:
|
|
out_urls.append(src) # 单张失败:保留原图床地址(img.xchina.io 链接较稳定,不签名)
|
|
fail_cnt += 1
|
|
continue
|
|
ext = path.suffix.lower() or ".webp"
|
|
key = f"xchina/{detail_id}/{i:04d}{ext}"
|
|
try:
|
|
public = await asyncio.to_thread(
|
|
upload_to_minio, path, key, guess_content_type(path, "image/webp"), cfg,
|
|
)
|
|
out_urls.append(public)
|
|
out_keys.append(key)
|
|
if cover_public is None and item.cover_image and item.cover_image == src:
|
|
cover_public = public
|
|
ok_cnt += 1
|
|
except Exception as exc: # noqa: BLE001
|
|
out_urls.append(src)
|
|
fail_cnt += 1
|
|
_log(job, f"[{idx}/{total}] {item.title}: 第 {i} 张上传失败({exc}),保留原图床地址")
|
|
finally:
|
|
path.unlink(missing_ok=True)
|
|
if i % 25 == 0:
|
|
_log(job, f"[{idx}/{total}] {item.title}: 进度 {i}/{n}(成功 {ok_cnt} / 失败 {fail_cnt})")
|
|
item.images = out_urls
|
|
item.extra["original_images_count"] = n
|
|
item.extra["minio_objects_images"] = out_keys
|
|
item.extra["images_uploaded"] = ok_cnt
|
|
if fail_cnt:
|
|
item.extra["images_failed"] = fail_cnt
|
|
# 封面:优先用上传成功的第一张
|
|
if cover_public is None:
|
|
first_minio = next((u for u in out_urls if "://" in u and u not in src_images), None)
|
|
cover_public = first_minio or (out_urls[0] if out_urls else item.cover_image)
|
|
item.extra["original_cover"] = item.cover_image
|
|
item.cover_image = cover_public
|
|
_log(job, f"[{idx}/{total}] {item.title}: OK 图集 {ok_cnt}/{n} 张已上传 MinIO")
|
|
|
|
|
|
def _resolve_media_kind(job: Job) -> str:
|
|
kind = (job.media_kind or "auto").lower()
|
|
if kind in ("video", "photo"):
|
|
return kind
|
|
from core.scraper_photos import is_photo_detail_url, is_photo_series_url
|
|
url = job.series_url
|
|
if is_photo_series_url(url) or is_photo_detail_url(url):
|
|
return "photo"
|
|
return "video"
|
|
|
|
|
|
def _make_scraper(kind: str, ctx, job: Job):
|
|
if kind == "photo":
|
|
from core.scraper_photos import XChinaPhotoScraper
|
|
return XChinaPhotoScraper(
|
|
ctx, is_cancelled=lambda: job._cancelled, log=lambda m: _log(job, m), max_images=job.max_images,
|
|
)
|
|
return XChinaScraper(ctx, is_cancelled=lambda: job._cancelled, log=lambda m: _log(job, m))
|
|
|
|
|
|
async def _run(job: Job, headless: bool) -> None:
|
|
tmp_dir = TMP_ROOT / job.id
|
|
try:
|
|
async with _RUN_LOCK:
|
|
kind = _resolve_media_kind(job)
|
|
extra = f", max_images={job.max_images}" if kind == "photo" else ""
|
|
_log(job, f"开始采集[{kind}]: {job.series_url} (page {job.start_page}-{job.end_page}{extra})")
|
|
if job.minio_cfg.is_complete:
|
|
_log(job, f"启用 MinIO 上传: {job.minio_cfg.safe_repr()}")
|
|
else:
|
|
_log(job, "未提供完整 MinIO 配置,本次不上传(媒体地址保留原始来源地址)")
|
|
|
|
async with browser_context(headless=headless) as ctx:
|
|
scraper = _make_scraper(kind, ctx, job)
|
|
result = await scraper.run(job.series_url, job.start_page, job.end_page)
|
|
# 浏览器已关闭;开始下载+上传(若配置完整)
|
|
if job.minio_cfg.is_complete:
|
|
await _process_uploads(job, result, tmp_dir)
|
|
else:
|
|
for item in result.resources:
|
|
item.extra.setdefault("upload_skipped_reason", "minio config absent")
|
|
|
|
# 统一覆盖 source_name(invoke 里指定的 platform_name)
|
|
for item in result.resources:
|
|
item.source_name = job.source_name
|
|
output = result.to_output()
|
|
job.output = output
|
|
if not output["resources"] and output["item_errors"]:
|
|
job.status = "failed"
|
|
job.error = "no resources scraped; " + "; ".join(
|
|
e.get("reason", "") for e in output["item_errors"][:3]
|
|
)
|
|
else:
|
|
job.status = "succeeded"
|
|
uploaded = sum(
|
|
1 for r in output["resources"]
|
|
if r.get("extra_data", {}).get("minio_object_video") or r.get("extra_data", {}).get("minio_objects_images")
|
|
)
|
|
_log(job, f"完成: {len(output['resources'])} 条(其中 {uploaded} 条已上传 MinIO), 页数 {output['pages_scanned']}, 抓取失败 {len(output['item_errors'])}")
|
|
except ScrapeCancelled:
|
|
job.status = "failed"
|
|
job.error = "cancelled"
|
|
_log(job, "已取消")
|
|
except Exception as exc: # noqa: BLE001
|
|
job.status = "failed"
|
|
job.error = f"{type(exc).__name__}: {exc}"
|
|
_log(job, f"失败: {job.error}")
|
|
finally:
|
|
shutil.rmtree(tmp_dir, ignore_errors=True)
|
|
job.finished_at = time.time()
|
|
|
|
|
|
def submit(series_url: str, source_name: str, start_page: int, end_page: int,
|
|
headless: bool = True, minio_cfg: Optional[MinioConfig] = None,
|
|
media_kind: str = "auto", max_images: int = 100) -> Job:
|
|
job = Job(
|
|
id=uuid.uuid4().hex,
|
|
series_url=series_url,
|
|
source_name=source_name or "xchina",
|
|
start_page=max(1, start_page),
|
|
end_page=max(start_page, end_page),
|
|
minio_cfg=minio_cfg or MinioConfig(),
|
|
media_kind=(media_kind or "auto"),
|
|
max_images=max(1, int(max_images or 100)),
|
|
)
|
|
_JOBS[job.id] = job
|
|
job._task = asyncio.create_task(_run(job, headless))
|
|
return job
|
|
|
|
|
|
def cancel(job_id: str) -> bool:
|
|
job = _JOBS.get(job_id)
|
|
if not job or job.status != "running":
|
|
return False
|
|
job._cancelled = True
|
|
return True
|