fix(scraper): 视频简介不再抓 .video-detail 的广告位 + 模特/分类做 content
xchina 视频详情页没有真正的简介文本,.video-detail 面板就是 标题/模特/分类/系列名 + 一整块 「推广：…」广告位(a.cpc-text → xchina.click/prepare.html 的联盟跳转,常含 VPN/楼凤/约炮乃至 「幼女」「学生妹」等违规词)。旧 _extract_description 把整段面板 innerText + 那些"标签"链接一锅端 进 content,导致主站内容审核把整批视频资源 block 掉(实测一个 11 视频的系列页 0 条入库)。 改为:视频 content 只抓模特名 + 分类/系列名(限同源 xchina.co,排除 a.cpc-text / xchina.click / prepare.html),拼成「标题\n模特: X · 分类: Y / Z」,抓不到就回退用标题;不再用面板 innerText、 也不用站点级 boilerplate 的 meta/jsonld 描述。图集 scraper 的标签查询也加上同样的广告/外站过滤(防御)。 Co-Authored-By: Claude Opus 4.7 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.7
+32
-32
@@ -464,46 +464,46 @@ class XChinaScraper:
|
||||
# ---- description / content -------------------------------------------
|
||||
|
||||
async def _extract_description(self, page, title: str) -> tuple[str | None, str]:
|
||||
"""返回 (summary, content)。content 保证非空(实在没有就用 title)——
|
||||
主站资源库 ingest 对空 content 直接拒收。"""
|
||||
"""返回 (summary, content)。content 保证非空(实在没有就用 title)——主站资源库 ingest 对空 content 直接拒收。
|
||||
|
||||
xchina 视频详情页**没有真正的简介文本**:`.video-detail` 面板就是 标题 / 模特 / 分类 / 系列名,
|
||||
再加一整块「推广：...」广告位(里面是 `a.cpc-text` → xchina.click/prepare.html 的联盟跳转,常含
|
||||
VPN/楼凤/约炮乃至「幼女」「学生妹」这类违规词,会被主站内容审核 block)。`meta description` 也只是
|
||||
站点级 boilerplate。所以这里只抓干净的结构化字段:模特名 + 分类/系列名(限同源 xchina.co,排除 cpc-text /
|
||||
xchina.click / prepare.html),拼成 content;不要面板 innerText、不要 boilerplate meta/jsonld。
|
||||
"""
|
||||
data = await page.evaluate(
|
||||
"""() => {
|
||||
function clean(t){ return (t || '').replace(/\\s+/g,' ').trim(); }
|
||||
const out = {};
|
||||
const m = document.querySelector('meta[name="description"], meta[property="og:description"], meta[name="twitter:description"]');
|
||||
out.meta_desc = clean(m && m.getAttribute('content'));
|
||||
for (const s of document.querySelectorAll('script[type="application/ld+json"]')) {
|
||||
try { const d = JSON.parse(s.textContent||'null'); const rows = Array.isArray(d)?d:[d];
|
||||
for (const r of rows) { if (r && r.description) { out.jsonld_desc = clean(r.description); break; } } } catch(_){}
|
||||
const out = {model: '', cats: []};
|
||||
const ma = document.querySelector('.video-detail a[href*="/model/id-"], a.model-item, .model-container a[href*="/model/id-"]');
|
||||
out.model = clean(ma && ma.innerText);
|
||||
const seen = new Set();
|
||||
for (const a of document.querySelectorAll('.video-detail a, .video-info a, .breadcrumb a')) {
|
||||
if (a.classList && a.classList.contains('cpc-text')) continue; // 广告位链接
|
||||
let h; try { h = new URL(a.href, location.href); } catch(e){ continue; }
|
||||
if (!h.host.endsWith('xchina.co')) continue; // 排除 xchina.click 等广告域
|
||||
if (h.pathname.indexOf('prepare.html') >= 0) continue;
|
||||
const ok = h.pathname.indexOf('/videos/series-') >= 0 || h.pathname.indexOf('/photos/series-') >= 0
|
||||
|| h.pathname.indexOf('/tags/') === 0 || h.pathname.indexOf('/tag/') === 0;
|
||||
if (!ok) continue;
|
||||
const t = clean(a.innerText);
|
||||
if (t && t.length <= 24 && !seen.has(t)) { seen.add(t); out.cats.push(t); }
|
||||
}
|
||||
// 影片信息 / 详情面板里的文本(键值对、简介等)
|
||||
const panel = document.querySelector('.video-detail') || document.querySelector('.tab-content.active') || document.querySelector('.tab-content');
|
||||
out.panel = panel ? clean(panel.innerText).slice(0, 4000) : '';
|
||||
// 标签
|
||||
const tags = [];
|
||||
for (const a of document.querySelectorAll('.tags a, .tag a, a[href*="/tags/"], .video-detail .item a')) {
|
||||
const t = clean(a.innerText); if (t && t.length<=20 && !tags.includes(t)) tags.push(t);
|
||||
}
|
||||
out.tags = tags.slice(0, 30);
|
||||
out.cats = out.cats.slice(0, 8);
|
||||
return out;
|
||||
}"""
|
||||
)
|
||||
meta_desc = (data.get("meta_desc") or "").strip()
|
||||
jsonld_desc = (data.get("jsonld_desc") or "").strip()
|
||||
panel = (data.get("panel") or "").strip()
|
||||
tags = data.get("tags") or []
|
||||
# content: 优先详情面板文本(信息量最大),否则 og/jsonld 描述,再否则 标签拼接,最后 title
|
||||
model = (data.get("model") or "").strip()
|
||||
cats = data.get("cats") or []
|
||||
parts: list[str] = []
|
||||
body = panel or jsonld_desc or meta_desc
|
||||
if body:
|
||||
parts.append(body)
|
||||
if tags:
|
||||
parts.append("标签: " + " ".join(tags))
|
||||
content = "\n".join(parts).strip()
|
||||
if not content:
|
||||
content = title
|
||||
# summary: og/jsonld 描述,或 content 头一截
|
||||
summary = (meta_desc or jsonld_desc or content)[:300].strip() or None
|
||||
if model:
|
||||
parts.append(f"模特: {model}")
|
||||
if cats:
|
||||
parts.append("分类: " + " / ".join(cats))
|
||||
content = ("\n".join([title, " · ".join(parts)]).strip() if parts else (title or "").strip()) or (title or "")
|
||||
bits = [b for b in [model, *cats[:3]] if b]
|
||||
summary = " · ".join(bits) if bits else None
|
||||
return summary, content
|
||||
|
||||
# ---- cover -----------------------------------------------------------
|
||||
|
||||
@@ -365,7 +365,10 @@ class XChinaPhotoScraper(XChinaScraper):
|
||||
|| document.querySelector('a[href*="/photos/series-"]');
|
||||
out.series = clean(sa && sa.innerText);
|
||||
const tags = [];
|
||||
for (const a of document.querySelectorAll('a[href*="/tag"], .tags a, .tag a')) {
|
||||
for (const a of document.querySelectorAll('a[href*="/tags/"], .tags a, .tag a')) {
|
||||
if (a.classList && a.classList.contains('cpc-text')) continue; // 广告位链接
|
||||
let h; try { h = new URL(a.href, location.href); } catch(e){ continue; }
|
||||
if (!h.host.endsWith('xchina.co') || h.pathname.indexOf('prepare.html') >= 0) continue;
|
||||
const t = clean(a.innerText);
|
||||
if (t && t.length <= 20 && tags.indexOf(t) < 0) tags.push(t);
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user