content-harvester/server/app.py

"""content-harvester —— 内容采集服务,实现主站 lobster 执行器的调用契约。

当前内置一个 xchina.co 的 scraper(core/scraper.py),以后可加更多来源。
对接 app/services/lobster_client.py::call_lobster_agent:
  POST /agents/{agent_id}/invoke   body = 主站合并后的 payload, 返回 {"task_id": ...}
  GET  /tasks/{task_id}            返回 {"status": running|succeeded|failed, "output": ..., "error": ...}

鉴权: 主站带 `Authorization: Bearer <lobster_api_key>`,需与本服务 env HARVESTER_TOKEN 一致。
SSRF 防护: 只接受 ALLOWED_HOSTS(默认 xchina.co 及其子域) 的 content_url。
"""
from __future__ import annotations

import logging
import os
from urllib.parse import urlparse

from fastapi import FastAPI, Header, HTTPException
from pydantic import BaseModel

from core.media import MinioConfig
from server import jobs

logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(name)s %(message)s")
logger = logging.getLogger("harvester.app")

SERVICE_TOKEN = os.environ.get("HARVESTER_TOKEN", "")
HEADLESS = os.environ.get("HEADLESS", "1") not in ("0", "false", "False")
ALLOWED_HOSTS = tuple(
    h.strip().lower() for h in os.environ.get("ALLOWED_HOSTS", "xchina.co").split(",") if h.strip()
)
MAX_PAGES = int(os.environ.get("MAX_PAGES", "50"))
DEFAULT_MAX_IMAGES = int(os.environ.get("DEFAULT_MAX_IMAGES", "100"))
MAX_IMAGES_CAP = int(os.environ.get("MAX_IMAGES_CAP", "2000"))

app = FastAPI(title="content-harvester", version="1.0")


def _check_auth(authorization: str | None) -> None:
    if not SERVICE_TOKEN:
        # 未配置 token 时不鉴权(仅适合内网/本地);生产务必设 HARVESTER_TOKEN
        return
    expected = f"Bearer {SERVICE_TOKEN}"
    if authorization != expected:
        raise HTTPException(status_code=401, detail="invalid token")


def _host_allowed(url: str) -> bool:
    try:
        host = (urlparse(url).hostname or "").lower()
    except Exception:
        return False
    if not host:
        return False
    return any(host == h or host.endswith("." + h) for h in ALLOWED_HOSTS)


class TaskStatusResponse(BaseModel):
    status: str
    output: dict | None = None
    error: str | None = None


@app.get("/healthz")
async def healthz():
    return {"ok": True, "headless": HEADLESS, "allowed_hosts": list(ALLOWED_HOSTS)}


@app.post("/agents/{agent_id}/invoke")
async def invoke(agent_id: str, payload: dict, authorization: str | None = Header(default=None)):
    _check_auth(authorization)

    # payload 即主站 task.input_params:含 content_url / platform_name / source_kind / start_page / end_page / _crawl_config ...
    source_kind = payload.get("source_kind")
    if source_kind is not None and str(source_kind) != "website":
        raise HTTPException(status_code=400, detail=f"unsupported source_kind: {source_kind}(本服务只处理 website)")

    content_url = (payload.get("content_url") or payload.get("series_url") or payload.get("url") or "").strip()
    if not content_url:
        raise HTTPException(status_code=400, detail="missing content_url")
    if not _host_allowed(content_url):
        raise HTTPException(status_code=400, detail=f"host not allowed: {content_url}")

    def _as_int(v, default):
        try:
            return int(v)
        except (TypeError, ValueError):
            return default

    start_page = max(1, _as_int(payload.get("start_page"), 1))
    end_page = max(start_page, _as_int(payload.get("end_page"), start_page))
    if end_page - start_page + 1 > MAX_PAGES:
        end_page = start_page + MAX_PAGES - 1

    source_name = str(payload.get("platform_name") or "xchina")[:100]

    # 媒体类型:可由 payload.media_kind 显式指定(video/photo);否则按 content_url 自动判断;
    # 再否则按 agent_id 兜底(含 photo → photo)。
    media_kind = str(payload.get("media_kind") or "").strip().lower()
    if media_kind not in ("video", "photo"):
        path_low = urlparse(content_url).path.lower()
        if "/photos/" in path_low or "/photo/id-" in path_low:
            media_kind = "photo"
        elif "/videos/" in path_low or "/video/id-" in path_low:
            media_kind = "video"
        elif "photo" in agent_id.lower():
            media_kind = "photo"
        else:
            media_kind = "auto"
    # 图集每个套图最多抓多少张
    max_images = _as_int(payload.get("max_images"), DEFAULT_MAX_IMAGES)
    max_images = max(1, min(max_images, MAX_IMAGES_CAP))

    # MinIO 配置由主站经 lobster payload 下发(主站把绑定的 minio_* Skill 变量并入 payload);
    # 缺失则降级:不上传,媒体地址保留原始来源地址
    minio_cfg = MinioConfig.from_payload(payload)

    job = jobs.submit(
        content_url, source_name, start_page, end_page,
        headless=HEADLESS, minio_cfg=minio_cfg, media_kind=media_kind, max_images=max_images,
    )
    logger.info(
        "invoke agent=%s job=%s kind=%s url=%s pages=%s-%s max_images=%s minio=%s",
        agent_id, job.id, media_kind, content_url, start_page, end_page, max_images,
        "on" if minio_cfg.is_complete else "off",
    )
    return {"task_id": job.id, "status": job.status}


@app.get("/tasks/{task_id}")
async def get_task(task_id: str, authorization: str | None = Header(default=None)):
    _check_auth(authorization)
    job = jobs.get_job(task_id)
    if not job:
        raise HTTPException(status_code=404, detail="task not found")
    return job.to_dict()


@app.post("/tasks/{task_id}/cancel")
async def cancel_task(task_id: str, authorization: str | None = Header(default=None)):
    _check_auth(authorization)
    ok = jobs.cancel(task_id)
    if not ok:
        raise HTTPException(status_code=409, detail="not cancellable")
    return {"ok": True}