Merge branch 'hy-dev' into dev

core/scraper_twitter.py

@@ -377,8 +377,9 @@ class XTwitterScraper:
             if time.time() > deadline:
                 self.log("达到总时长上限,停止滚动")
                 break
-            # 先用已收集的响应数粗判,够了就停
-            before_bodies = len(self._bodies)
+            # 用「已处理推文数」判 idle(_seen 比 _items 鲁棒:即使所有新推文都被过滤掉
+            # 我们也知道时间线还在出新内容,不会误判到底)
+            before_seen = len(self._seen)
             try:
                 await page.evaluate("window.scrollBy(0, document.body.scrollHeight)")
             except Exception:  # noqa: BLE001
@@ -392,15 +393,15 @@ class XTwitterScraper:
             if self.since and self._oldest_seen and self._oldest_seen < self.since:
                 self.log(f"已滚动到 {_date_only(self._oldest_seen)},早于 since={_date_only(self.since)},停止滚动")
                 break
-            if len(self._bodies) == before_bodies:
+            if len(self._seen) == before_seen:
                 idle += 1
                 if idle >= self.idle_scrolls:
-                    self.log(f"连续 {idle} 次滚动无新数据,判定到底,停止")
+                    self.log(f"连续 {idle} 次滚动无新数据,判定到底,停止(原创 {len(self._items)} / 已处理 {len(self._seen)})")
                     break
             else:
                 idle = 0
             if n % 10 == 0:
-                self.log(f"滚动 {n} 次,已收集原创推文 {len(self._items)} 条")
+                self.log(f"滚动 {n} 次,原创 {len(self._items)} / 已处理 {len(self._seen)}")
 
     def _reparse_so_far(self) -> None:
         """用目前所有响应体重新解析一遍(幂等,_seen/_items 去重)。"""

server/jobs.py

@@ -37,7 +37,7 @@ _RUN_LOCK = asyncio.Lock()
 
 TMP_ROOT = Path(os.environ.get("TMP_DIR", "/tmp")) / "content-harvester"
 CONCURRENT_FRAGMENTS = int(os.environ.get("CONCURRENT_FRAGMENTS", "8"))
-MAX_VIDEO_MB = int(os.environ.get("MAX_VIDEO_MB", "0"))  # 0 = 不限
+MAX_VIDEO_MB = int(os.environ.get("MAX_VIDEO_MB", "500"))  # 单个视频上限,0=不限;默认 500MB 避免误下载几个 G 的长视频
 
 
 @dataclass