diff --git a/core/scraper.py b/core/scraper.py index 62b1e65..ba46c66 100644 --- a/core/scraper.py +++ b/core/scraper.py @@ -464,46 +464,46 @@ class XChinaScraper: # ---- description / content ------------------------------------------- async def _extract_description(self, page, title: str) -> tuple[str | None, str]: - """返回 (summary, content)。content 保证非空(实在没有就用 title)—— - 主站资源库 ingest 对空 content 直接拒收。""" + """返回 (summary, content)。content 保证非空(实在没有就用 title)——主站资源库 ingest 对空 content 直接拒收。 + + xchina 视频详情页**没有真正的简介文本**:`.video-detail` 面板就是 标题 / 模特 / 分类 / 系列名, + 再加一整块「推广:...」广告位(里面是 `a.cpc-text` → xchina.click/prepare.html 的联盟跳转,常含 + VPN/楼凤/约炮乃至「幼女」「学生妹」这类违规词,会被主站内容审核 block)。`meta description` 也只是 + 站点级 boilerplate。所以这里只抓干净的结构化字段:模特名 + 分类/系列名(限同源 xchina.co,排除 cpc-text / + xchina.click / prepare.html),拼成 content;不要面板 innerText、不要 boilerplate meta/jsonld。 + """ data = await page.evaluate( """() => { function clean(t){ return (t || '').replace(/\\s+/g,' ').trim(); } - const out = {}; - const m = document.querySelector('meta[name="description"], meta[property="og:description"], meta[name="twitter:description"]'); - out.meta_desc = clean(m && m.getAttribute('content')); - for (const s of document.querySelectorAll('script[type="application/ld+json"]')) { - try { const d = JSON.parse(s.textContent||'null'); const rows = Array.isArray(d)?d:[d]; - for (const r of rows) { if (r && r.description) { out.jsonld_desc = clean(r.description); break; } } } catch(_){} + const out = {model: '', cats: []}; + const ma = document.querySelector('.video-detail a[href*="/model/id-"], a.model-item, .model-container a[href*="/model/id-"]'); + out.model = clean(ma && ma.innerText); + const seen = new Set(); + for (const a of document.querySelectorAll('.video-detail a, .video-info a, .breadcrumb a')) { + if (a.classList && a.classList.contains('cpc-text')) continue; // 广告位链接 + let h; try { h = new URL(a.href, location.href); } catch(e){ continue; } + if (!h.host.endsWith('xchina.co')) continue; // 排除 xchina.click 等广告域 + if (h.pathname.indexOf('prepare.html') >= 0) continue; + const ok = h.pathname.indexOf('/videos/series-') >= 0 || h.pathname.indexOf('/photos/series-') >= 0 + || h.pathname.indexOf('/tags/') === 0 || h.pathname.indexOf('/tag/') === 0; + if (!ok) continue; + const t = clean(a.innerText); + if (t && t.length <= 24 && !seen.has(t)) { seen.add(t); out.cats.push(t); } } - // 影片信息 / 详情面板里的文本(键值对、简介等) - const panel = document.querySelector('.video-detail') || document.querySelector('.tab-content.active') || document.querySelector('.tab-content'); - out.panel = panel ? clean(panel.innerText).slice(0, 4000) : ''; - // 标签 - const tags = []; - for (const a of document.querySelectorAll('.tags a, .tag a, a[href*="/tags/"], .video-detail .item a')) { - const t = clean(a.innerText); if (t && t.length<=20 && !tags.includes(t)) tags.push(t); - } - out.tags = tags.slice(0, 30); + out.cats = out.cats.slice(0, 8); return out; }""" ) - meta_desc = (data.get("meta_desc") or "").strip() - jsonld_desc = (data.get("jsonld_desc") or "").strip() - panel = (data.get("panel") or "").strip() - tags = data.get("tags") or [] - # content: 优先详情面板文本(信息量最大),否则 og/jsonld 描述,再否则 标签拼接,最后 title + model = (data.get("model") or "").strip() + cats = data.get("cats") or [] parts: list[str] = [] - body = panel or jsonld_desc or meta_desc - if body: - parts.append(body) - if tags: - parts.append("标签: " + " ".join(tags)) - content = "\n".join(parts).strip() - if not content: - content = title - # summary: og/jsonld 描述,或 content 头一截 - summary = (meta_desc or jsonld_desc or content)[:300].strip() or None + if model: + parts.append(f"模特: {model}") + if cats: + parts.append("分类: " + " / ".join(cats)) + content = ("\n".join([title, " · ".join(parts)]).strip() if parts else (title or "").strip()) or (title or "") + bits = [b for b in [model, *cats[:3]] if b] + summary = " · ".join(bits) if bits else None return summary, content # ---- cover ----------------------------------------------------------- diff --git a/core/scraper_photos.py b/core/scraper_photos.py index fc24265..0e05c78 100644 --- a/core/scraper_photos.py +++ b/core/scraper_photos.py @@ -365,7 +365,10 @@ class XChinaPhotoScraper(XChinaScraper): || document.querySelector('a[href*="/photos/series-"]'); out.series = clean(sa && sa.innerText); const tags = []; - for (const a of document.querySelectorAll('a[href*="/tag"], .tags a, .tag a')) { + for (const a of document.querySelectorAll('a[href*="/tags/"], .tags a, .tag a')) { + if (a.classList && a.classList.contains('cpc-text')) continue; // 广告位链接 + let h; try { h = new URL(a.href, location.href); } catch(e){ continue; } + if (!h.host.endsWith('xchina.co') || h.pathname.indexOf('prepare.html') >= 0) continue; const t = clean(a.innerText); if (t && t.length <= 20 && tags.indexOf(t) < 0) tags.push(t); }