diff --git a/hexi/plugins/nonebot_plugin_video_analysis/cleanup.py b/hexi/plugins/nonebot_plugin_video_analysis/cleanup.py index e8e4d68..b8af76e 100644 --- a/hexi/plugins/nonebot_plugin_video_analysis/cleanup.py +++ b/hexi/plugins/nonebot_plugin_video_analysis/cleanup.py @@ -17,6 +17,7 @@ hexi/data/temp 是下载媒体中转区(见 utils.get_temp_root),发送成功后 import asyncio import time +from collections.abc import Callable from pathlib import Path from typing import Literal @@ -25,9 +26,11 @@ from nonebot.adapters.onebot.v11 import Bot, MessageEvent from pydantic import BaseModel from hexi.core.custom_utils import check_manage +from hexi.core.task_slots import TASKS, Task require("nonebot_plugin_apscheduler") from nonebot_plugin_apscheduler import scheduler +from hexi.core.scheduler import add_job from nonebot_plugin_alconna import UniMessage @@ -101,33 +104,62 @@ def _prune_empty_dirs(root: Path, max_age: float) -> int: return removed -def clean_temp_files(sub: str = "", days: int | None = None) -> tuple[int, int]: +def clean_temp_files( + sub: str = "", + days: int | None = None, + on_progress: Callable[..., None] | None = None, +) -> tuple[int, int]: """清理 temp[/sub] 下超过期限的文件与随之空掉的目录。 + Args: + on_progress: 进度回调 `(done, total, phase, message, warning)`（关键字可省）—— + 扫描阶段 total 未知，之后按"处理过的文件数"上报。跑在哪个线程调用都行 + （任务槽那边会切回事件循环）。**返回 False 表示调用方请求取消**， + 本函数会在处理完当前文件后尽早收尾（清理天然可中断，不硬杀线程）。 + Returns: - (removed, total) — 删除的文件数、统计到的文件总数 + (removed, total) — 删除的文件数、统计到的文件数（被取消时是"取消前"的进度） """ + + def report(**kwargs) -> bool: + if on_progress is None: + return True # 没有槽（纯函数调用）时不存在取消 + return bool(on_progress(**kwargs)) + root = get_temp_root(sub) if not root.is_dir(): return 0, 0 max_age = _max_age_seconds(days) + report(phase="扫描中", unit="个文件", message="正在统计文件…") files = _walk_files(root) - removed = 0 - for path in files: - if not _file_is_stale(path, max_age): - continue - try: - path.unlink(missing_ok=True) - removed += 1 - logger.info(f"temp 清理: 删除 {path}") - except OSError as e: - # 文件被占用(如发送中)等场景,留待下轮 - logger.warning(f"temp 清理: 跳过 {path} ({e})") + report( + done=0, total=len(files), phase="清理中", + unit="个文件", message=f"共 {len(files)} 个文件", + ) - dirs = _prune_empty_dirs(root, max_age) - if dirs: - logger.info(f"temp 清理: 同时收掉 {dirs} 个空目录") + removed = 0 + canceled = False + for index, path in enumerate(files, 1): + if _file_is_stale(path, max_age): + try: + path.unlink(missing_ok=True) + removed += 1 + logger.info(f"temp 清理: 删除 {path}") + except OSError as e: + # 文件被占用(如发送中)等场景,留待下轮 —— 也算"部分失败",报给任务槽 + logger.warning(f"temp 清理: 跳过 {path} ({e})") + report(warning=f"跳过 {path.name}（被占用？）：{e}") + if not report(done=index, total=len(files), message=f"已删除 {removed} 个"): + logger.info("temp 清理: 收到取消请求，提前结束") + canceled = True + break + + if not canceled: + report(phase="回收空目录", done=len(files), total=len(files)) + dirs = _prune_empty_dirs(root, max_age) + if dirs: + logger.info(f"temp 清理: 同时收掉 {dirs} 个空目录") return removed, len(files) @@ -165,7 +197,27 @@ async def _handle_clean(bot: Bot, event: MessageEvent): except ValueError: await UniMessage.text("天数参数不合法,示例:/清理temp 3").send() return - removed, total = await _run_clean(days) + task, reused = submit_clean( + days, + title="清理 temp（命令）", + trigger="command", + actor=event.get_user_id(), + group_id=getattr(event, "group_id", "") or "", + ) + if reused: + await UniMessage.text( + "已经有一个清理任务在跑了,等它跑完再来吧~（进度可在网页端看）" + ).send() + return + await TASKS.wait(task) # 命令要"跑完再回复",与网页端的即提交即返回不同 + + if task.status == "canceled": + await UniMessage.text("清理被取消了（网页端点过取消）。").send() + return + if task.status != "done": + await UniMessage.text(f"清理失败了:{task.error}").send() + return + removed, total = task.result or (0, 0) if total == 0: await UniMessage.text("temp 目录目前是空的,没什么可清理的~").send() else: @@ -174,7 +226,7 @@ async def _handle_clean(bot: Bot, event: MessageEvent): @stats_cmd.handle() async def _handle_stats(event: MessageEvent): - st = await _run_stats() + st = await run_stats() if st["files"] == 0: if st.get("dirs"): await UniMessage.text( @@ -191,13 +243,37 @@ async def _handle_stats(event: MessageEvent): ).send() -async def _run_clean(days: int | None = None) -> tuple[int, int]: - """执行清理并记录日志(供命令与定时任务共用)""" - removed, total = await asyncio.to_thread(clean_temp_files, "", days) - return removed, total +#: 任务槽名:命令 / Web / 每日定时三条路共用同一个槽,同槽只跑一个(防重复触发) +CLEANUP_SLOT = "video_analysis:temp_cleanup" -async def _run_stats() -> dict: +def submit_clean( + days: int | None, + *, + title: str, + trigger: str = "", + actor: str = "", + group_id: str = "", +) -> tuple[Task, bool]: + """把清理提交到任务槽,立即返回 (任务, 是否复用了正在跑的任务)。 + + 任务体跑在线程池里(见 hexi.core.task_slots),进度实时推给网页端; + 同槽已有任务时不会重复清理 —— 服务端这一层才是"防连点"的真正保证。 + trigger / actor / group_id 是审计信息(谁点的:web 用户名 / command 的 QQ 号)。 + """ + return TASKS.submit( + slot=CLEANUP_SLOT, + title=title, + plugin_id="video_analysis", + trigger=trigger, + actor=actor, + group_id=group_id, + func=lambda report: clean_temp_files(days=days, on_progress=report), + ) + + +async def run_stats() -> dict: + """异步统计(命令 / Web 缓存面板共用)""" return await asyncio.to_thread(temp_stats) @@ -213,7 +289,7 @@ def _register_auto_job() -> None: from apscheduler.jobstores.base import JobLookupError # noqa: PLC0415 if cleanup_config.video_analysis_temp_cleanup_mode == "auto": - scheduler.add_job( + add_job( _scheduled_cleanup, "cron", hour=cleanup_config.video_analysis_temp_cleanup_hour, @@ -239,13 +315,18 @@ def reload_cleanup_config() -> None: async def _scheduled_cleanup() -> None: - """每日定时清理 temp 目录(auto 模式)""" + """每日定时清理 temp 目录(auto 模式);同样走任务槽(与手动清理互斥)""" logger.info("temp 自动清理: 开始") - try: - removed, total = await asyncio.to_thread(clean_temp_files) - logger.info(f"temp 自动清理完成:删除 {removed} 个,当前共 {total} 个文件") - except Exception as e: - logger.exception(f"temp 自动清理失败: {e}") + task, reused = submit_clean(None, title="清理 temp（每日定时）", trigger="schedule") + if reused: + logger.info("temp 自动清理: 已有清理任务在跑，跳过本次") + return + await TASKS.wait(task) + if task.status != "done": + logger.error(f"temp 自动清理未完成（{task.status}）: {task.error}") + return + removed, total = task.result or (0, 0) + logger.info(f"temp 自动清理完成:删除 {removed} 个,当前共 {total} 个文件") # 插件加载时按模式注册(auto)或提示(manual) diff --git a/hexi/plugins/nonebot_plugin_video_analysis/handlers/douyin.py b/hexi/plugins/nonebot_plugin_video_analysis/handlers/douyin.py index 5afc0b7..6612cd9 100644 --- a/hexi/plugins/nonebot_plugin_video_analysis/handlers/douyin.py +++ b/hexi/plugins/nonebot_plugin_video_analysis/handlers/douyin.py @@ -1,109 +1,32 @@ -"""抖音视频/图文解析编排层""" +"""抖音视频/图文解析编排层 + +链路（2026-10 起，免浏览器）：短链/全链接 → `douyin_web.fetch_douyin_web` +（短链重定向、签名请求 web detail 接口、最高码率/最高画质下载都在该模块内）。 +旧链路（iesdouyin SSR 静态解析 + playwright 浏览器拦截）已废弃， +详见 services/fetchers/douyin_web.py 顶部说明。 +""" -import re -import urllib.parse from pathlib import Path from typing import Optional, Union -import httpx from nonebot import logger -from ..services.fetchers.douyin_api import fetch_douyin_content -from ..services.fetchers.douyin_ssr import MOBILE_UA, fetch_douyin_note_ssr -from ..models import DouyinFetchError +from ..services.fetchers.douyin_web import fetch_douyin_web from ..utils import get_data_dir, parse_netscape_cookies from ..policy import Policy from .sender import PendingMedia, _as_paths -SHORT_LINK_PATTERN = re.compile(r"(v\.douyin\.com/[A-Za-z0-9_\-]+)") - - -async def _resolve_short_link(url: str) -> Optional[str]: - """短链重定向：httpx 取 Location（最多 3 跳），失败返回 None""" - try: - async with httpx.AsyncClient( - headers={"User-Agent": MOBILE_UA}, - follow_redirects=False, - timeout=10, - ) as client: - current = url - for _ in range(3): - resp = await client.get(current) - if resp.status_code >= 400: - return None - location = resp.headers.get("Location") - if not location: - final = str(resp.url) - # 200 但仍是短链本身（如 JS 挑战壳页）→ 静默失败，显式记录； - # 图文短链若此处失败将退化到 playwright 旧链路 - if "v.douyin.com" in final: - logger.warning(f"短链返回挑战页（未跳转）: {final}") - return final - # Location 可能是相对路径，需拼上当前 URL - current = urllib.parse.urljoin(current, location) - return current - except Exception: - logger.warning(f"短链重定向失败: {url}") - return None - async def parse_douyin( url: str, ) -> tuple[Optional[str], Optional[Union[Path, list[Path]]], bool]: - """解析抖音链接（短链 / 全链接），返回 (title, file_paths, is_image_post) - - 分派规则（不能依赖 URL 路径：短链重定向后图文也统一变成 - iesdouyin.com/share/video/{id} 形态）： - - note 形态链接 → SSR 静态解析，失败不回退 playwright - （其图文链路存在 aweme/post 取到作者其他作品的缺陷） - - 其余形态 → 先试 SSR 按内容判定：有 images 即图文 → SSR 秒级下载； - 真视频 / SSR 失败 → playwright 拦截（保留全部清晰度能力） - """ - # 优先匹配短链接 v.douyin.com/xxx - short = SHORT_LINK_PATTERN.search(url) - if short: - target_url = "https://" + short.group(1) - # 短链先重定向拿到全链接，确定图文/视频类型后分派 - resolved = await _resolve_short_link(target_url) - if resolved: - logger.info(f"短链重定向: {target_url} -> {resolved}") - target_url = resolved - else: - # 全链接: www.douyin.com/note/xxx、www.douyin.com/video/xxx， - # 以及抖音复制出的 iesdouyin.com/share/video|note/xxx（弱网/无网下 App - # 生成的恢复连接同样会是这个形态）。统一提取出作品 id 后再交给 SSR / playwright。 - m = re.search( - r"(https?://(?:www\.)?(?:m\.)?(?:ies)?douyin\.com/(?:share/)?(note|video)/(\d+))", - url, - ) - if not m: - return None, None, False - # 统一规范化为 www.douyin.com/{note|video}/{id}，避免 iesdouyin 分享页 - # 在无登录态/弱网下只返回壳页，直接落到浏览器可正常打开的作品页。 - target_url = f"https://www.douyin.com/{m.group(2)}/{m.group(3)}" - + """解析抖音链接（短链 / 全链接），返回 (title, file_paths, is_image_post)""" try: - is_img_post = False cookies_path = get_data_dir() / "cookies.txt" cookies = parse_netscape_cookies(cookies_path) - # 先试 SSR（秒级、免浏览器），失败一律回退 playwright 链路。 - # 2026-08-13 起抖音对 iesdouyin SSR 端点整体降级（登录态也拿不到 - # videoInfoRes，只返回 33KB 壳页），note/video 形态统一回退， - # 图文作品的"取到作者其他作品"缺陷由 douyin_api 按 aweme_id 精确匹配修复。 - try: - title, file_path = await fetch_douyin_note_ssr(target_url, cookies) - except DouyinFetchError: - logger.info("SSR 解析失败，回退 playwright 链路") - title, file_path = None, None - if file_path is None: - title, file_path = await fetch_douyin_content( - target_url, cookies, 10, False - ) - if isinstance(file_path, list): - is_img_post = True - return title, file_path, is_img_post + return await fetch_douyin_web(url, cookies) except Exception: - logger.exception(f"获取直链失败 {target_url}") + logger.exception(f"获取直链失败 {url}") return None, None, False diff --git a/hexi/plugins/nonebot_plugin_video_analysis/handlers/entry.py b/hexi/plugins/nonebot_plugin_video_analysis/handlers/entry.py index 146bf31..90f5955 100644 --- a/hexi/plugins/nonebot_plugin_video_analysis/handlers/entry.py +++ b/hexi/plugins/nonebot_plugin_video_analysis/handlers/entry.py @@ -11,7 +11,11 @@ from nonebot.rule import to_me from nonebot_plugin_alconna import UniMessage from nonebot_plugin_alconna.uniseg import get_target -from ..services.fetchers.bilibili_content import fetch_bilibili_content, resolve_short_link +from ..services.fetchers.bilibili_content import ( + fetch_bilibili_content, + resolve_short_link, +) +from ..services.fetchers.kuaishou_content import fetch_kuaishou_content from ..services.fetchers.rednote_content import fetch_rednote_content from .douyin import parse_douyin, process_douyin_res from .sender import PendingMedia, send_pending_media @@ -32,6 +36,8 @@ VALID_HOSTS = [ "iesdouyin.com", "m.douyin.com", "jingxuan.douyin.com", + "kuaishou.com", + "chenzhongtech.com", "x.com", "twitter.com", "xiaohongshu.com", @@ -39,7 +45,16 @@ VALID_HOSTS = [ "xhslink.cn", ] -auto_video_handler = on_message(priority=10, block=False) +async def _not_to_me(event: Event) -> bool: + """不是 @bot / 昵称开头的消息（`to_me()` 的反面；Rule 没有 `__invert__`）。""" + return not event.is_tome() + + +# auto 与 active 是互斥的两条路：@bot（或昵称开头）的消息归 active 管，其余归 +# auto。两个 matcher 都是 block=False（不拦其它插件），若 auto 不排除 to_me， +# 一条 @bot 的链接消息会同时命中两个 handler → 解析发送两遍（长视频要等几分钟， +# 重复非常显眼；2026-10-13 排查发送重复时发现）。 +auto_video_handler = on_message(priority=10, block=False, rule=_not_to_me) active_video_handler = on_message(priority=10, block=False, rule=to_me()) @@ -203,14 +218,41 @@ async def dispatch_url( logger.warning(f"媒体解析：未能获取到作品：{url}") return None, None return await process_douyin_res( - title, parsed_path, is_private, image_post, - policy=policy, platform=platform, + title, + parsed_path, + is_private, + image_post, + policy=policy, + platform=platform, ) except Exception: await UniMessage.text(f"无法解析到媒体：{url}").send() logger.exception(f"媒体解析：无法解析到媒体：{url}") return None, None + if "kuaishou.com" in url or "chenzhongtech.com" in url: + await UniMessage.text("检测到链接，正在处理，请稍候...").send() + try: + title, parsed_path = await fetch_kuaishou_content(url) + if not title and parsed_path is None: + await UniMessage.text(f"无法解析到媒体：{url}").send() + return None, None + if parsed_path == []: + await UniMessage.text(title).send() + return None, None + return await process_douyin_res( + title, + parsed_path, + is_private, + isinstance(parsed_path, list), + policy=policy, + platform=platform, + ) + except Exception: + await UniMessage.text(f"无法解析到媒体：{url}").send() + logger.exception(f"快手解析：无法解析：{url}") + return None, None + if any( kw in url for kw in ( @@ -230,8 +272,12 @@ async def dispatch_url( await UniMessage.text(title).send() return None, None return await process_douyin_res( - title, parsed_path, is_private, - isinstance(parsed_path, list), policy=policy, platform=platform, + title, + parsed_path, + is_private, + isinstance(parsed_path, list), + policy=policy, + platform=platform, ) except Exception: await UniMessage.text(f"无法解析到内容：{url}").send() @@ -249,8 +295,12 @@ async def dispatch_url( await UniMessage.text(title).send() return None, None return await process_douyin_res( - title, parsed_path, is_private, - isinstance(parsed_path, list), policy=policy, platform=platform, + title, + parsed_path, + is_private, + isinstance(parsed_path, list), + policy=policy, + platform=platform, ) except Exception: await UniMessage.text(f"无法解析到内容：{url}").send() diff --git a/hexi/plugins/nonebot_plugin_video_analysis/handlers/sender.py b/hexi/plugins/nonebot_plugin_video_analysis/handlers/sender.py index 1a76fc3..59db595 100644 --- a/hexi/plugins/nonebot_plugin_video_analysis/handlers/sender.py +++ b/hexi/plugins/nonebot_plugin_video_analysis/handlers/sender.py @@ -1,15 +1,22 @@ -"""媒体多级发送 — temp 本地文件优先，失败逐级降级 +"""媒体多级发送 — 本地优先，归档与发送解耦（并行），失败逐级降级 发送策略（2026-08-22 用户需求）: 1. temp 本地文件直接发送（最快，不经 S3） -2. 失败 → 上传本地 S3，用预签名链接发送 +2. 失败 → 用**已归档**的 S3 预签名链接发送 3. 再失败 → 回退 temp 本地文件再发一次 +**归档是前提，不是发送的副产品（2026-09-25 用户明确）**：媒体一旦下载成功就要 +进本地 S3——与发送走哪一级、成功与否**无关**。因此 `archive_media` 在 +`send_pending_media` 一进来就无条件开跑，与本地直发**并行**：本地发送不等它 +（保住最快那条路），但它一定会跑完（发送全失败也照传）。原因：temp 只留 7 天 +（cleanup.py 删），桶里那份才是长期存档；顺带产出公网下载链接（`upload_public` +开着时），「发送下载链接」策略直接用它。每个文件只上传一次——第 2 级用的就是 +归档的预签名链接，没有"补传"这一步。 + 本地直发要真的通，得让协议端能拿到文件：2026-09-22 起由 `hexi/core/outbound_media` 的全局钩子把本地路径改写成 `/media/` 链接（此前本地那级因协议端不在本机、 -`file://` 必然 ENOENT 而恒失败，实际每次都落到第 2 级 S3）。因此**媒体不再进 S3 -归档是有意的**——只有群策略真要发下载链接（`policy.sends_link`）时才补传一次拿 -public_url，见 `_local_sent`。 +`file://` 必然 ENOENT 而恒失败，实际每次都落到第 2 级 S3）。注意那条 `/media` +链接**只给协议端拉取用**（TTL 300s、bot 自己的路由），不能当下载链接发给群友。 多媒体（2026-09-08 用户需求）: 多图（图集）、图+视频混合、多视频（动图） 等一个作品落地成多个文件时，直接 UniMessage 拼成一个消息发送会报错 @@ -20,15 +27,25 @@ public_url，见 `_local_sent`。 传到群文件（默认打包成一个 zip，可配解压密码，见 config.group_file_settings）， 失败只记日志、不影响发送结果（打包/上传实现在 services/storage/group_file.py）。 +**超时=结果未知，绝不降级重试**（2026-10-13）：NoneBot 默认 `api_timeout=30s`， +而长视频在协议端要「下载 + 转存到 QQ」，实测 172MB 视频约 2 分钟。超时只是 bot +不再等待响应——**协议端会把这次发送做完**，于是三级降级把同一个视频送了三份 +（群里收到三份 + 一条"媒体发送失败"）。修法有两半：发送调用带 `_timeout` +（`send_timeout()`，默认 300s，`video_analysis_send_timeout` 可调）等它真正 +发完；万一仍超时则按「可能已送达」处理——停在这一级、不发失败提示，只有 +协议端**明确报错**（ActionFailed）才降级到下一级。 + temp 下的文件发送成功后也不清理（用户手动处理 data/temp）。 """ import asyncio +import os from dataclasses import dataclass from pathlib import Path from nonebot import get_bot, get_driver, logger from nonebot.adapters import Event +from nonebot.exception import NetworkError from nonebot_plugin_alconna import UniMessage from ..config import group_file_settings @@ -57,6 +74,56 @@ def _as_paths(file_paths: Path | list[Path]) -> list[Path]: return [Path(file_paths)] +# ─────────────────── 发送超时与「结果未知」判定 ─────────────────── + +#: 媒体发送 per-call 超时（秒）——长视频在协议端「下载 + 转存 QQ」要几十秒 +#: 到几分钟（实测 172MB ≈ 140s），NoneBot 默认 api_timeout=30s 必超时。 +_DEFAULT_SEND_TIMEOUT = 300.0 + + +def send_timeout() -> float: + """媒体发送的 per-call 超时（`_timeout` 关键字，OneBot 适配器原生支持）。 + + 超时只是 bot 不再等响应，**协议端会把这次发送做完**；所以这个值同时是 + 「重试会重复发送」的边界，默认 300s，`video_analysis_send_timeout` 可调。 + """ + raw = ( + os.environ.get("video_analysis_send_timeout") + or os.environ.get("VIDEO_ANALYSIS_SEND_TIMEOUT") + or "" + ).strip() + try: + value = float(raw) + except ValueError: + if raw: + logger.warning( + f"video_analysis_send_timeout={raw!r} 不是数字，" + f"按默认 {_DEFAULT_SEND_TIMEOUT:g}s 处理" + ) + return _DEFAULT_SEND_TIMEOUT + return value if value > 0 else _DEFAULT_SEND_TIMEOUT + + +def unknown_send_outcome(exc: BaseException) -> bool: + """这次发送是否「结果未知」——协议端可能仍在处理并最终送达。 + + `NetworkError` 涵盖调用超时与连接异常：请求已经发出，只是 bot 放弃了 + 等待。长视频超时后协议端照样把视频发进群（2026-10-13 实测：172MB 视频 + 三级降级各超时一次，群里收到三份），所以这种结果**不能再降级重试**， + 否则每试一级就多送一份；对外的"媒体发送失败"提示同理不能再发。 + """ + return isinstance(exc, NetworkError) or isinstance( + exc, (asyncio.TimeoutError, TimeoutError) + ) + + +def _log_unknown(stage: str, exc: BaseException) -> None: + logger.warning( + f"{stage}发送等待超时，结果未知（协议端可能仍在处理并最终送达）；" + f"不再降级重试以免重复发送: {exc}" + ) + + # ─────────────────────── 合并转发（多媒体专用） ─────────────────────── @@ -114,37 +181,81 @@ async def _send_forward_media( *, group_id: int | None = None, user_id: int | None = None, -) -> bool: - """发送合并转发消息（群聊 / 私聊），失败返回 False""" +) -> bool | None: + """发送合并转发消息（群聊 / 私聊）。 + + 返回 True=成功 / False=失败（协议端明确报错，可降级重试）/ + None=结果未知（超时，调用方必须停止重试，见 `unknown_send_outcome`）。 + """ try: bot = get_bot() nodes = _build_forward_nodes(items, sender_name, bot.self_id) + timeout = send_timeout() if group_id is not None: await bot.call_api( - "send_group_forward_msg", group_id=group_id, messages=nodes + "send_group_forward_msg", + group_id=group_id, + messages=nodes, + _timeout=timeout, ) else: - await bot.call_api("send_forward_msg", user_id=user_id, messages=nodes) + await bot.call_api( + "send_forward_msg", user_id=user_id, messages=nodes, _timeout=timeout + ) return True except Exception as e: + if unknown_send_outcome(e): + _log_unknown("合并转发", e) + return None logger.warning(f"合并转发发送失败: {e}") return False -async def _build_s3_forward_items( - media: PendingMedia, -) -> tuple[list[tuple[Path, str | None]], str | None]: - """上传本地 S3，返回 ([(文件, 预签名链接)...], public_url)""" - items: list[tuple[Path, str | None]] = [] - public_url = None +# ─────────────────────── 归档（发送的前提） ─────────────────────── + + +@dataclass +class UploadedFile: + """归档结果：本地文件 → 局域网预签名链接（发送兜底）+ 公网链接（下载链接）""" + + path: Path + local_url: str | None = None + public_url: str | None = None + + +async def archive_media(media: PendingMedia) -> list[UploadedFile]: + """逐文件归档到本地 S3 —— **无条件**执行，与发送成败无关。 + + 这是「媒体必须进桶」这条前提的落点（用户 2026-09-25 定）：temp 只留 7 天， + 桶里那份才是长期存档。boto3 同步阻塞（大视频一次传几十秒），统一丢线程池。 + 单个文件失败只 warning 并**继续**后面的（多图作品不该因第 3 张失败丢掉后 7 张）， + 任何情况都不抛——归档失败不能反过来影响发送，发送失败也不影响归档。 + """ + uploads: list[UploadedFile] = [] for fp in media.files: - local_url, pub = upload_with_plan(fp, policy=media.policy) - if not local_url: - raise RuntimeError(f"上传本地 S3 失败: {fp}") - if pub: - public_url = pub - items.append((fp, local_url)) - return items, public_url + item = UploadedFile(path=fp) + try: + item.local_url, item.public_url = await asyncio.to_thread( + upload_with_plan, fp, policy=media.policy + ) + except Exception as e: # noqa: BLE001 + logger.warning(f"归档到本地 S3 失败 {fp}: {e}") + uploads.append(item) + return uploads + + +def _presigned_items(uploads: list[UploadedFile]) -> list[tuple[Path, str | None]]: + """归档结果 → 链接版消息/合并转发要的 [(文件, 预签名链接)]（缺链接=该文件没传上）""" + return [(item.path, item.local_url) for item in uploads] + + +def _last_public_url(uploads: list[UploadedFile]) -> str | None: + """公网下载链接取最后一个非空值（`upload_public` 关时恒为 None）""" + public_url = None + for item in uploads: + if item.public_url: + public_url = item.public_url + return public_url # ───────────────────────────── 多级发送 ───────────────────────────── @@ -161,68 +272,49 @@ def _build_local_msg(files: list[Path], image_post: bool) -> UniMessage: return msg -def _build_s3_msg( - media: PendingMedia, -) -> tuple[UniMessage, str | None]: - """上传本地 S3 并构建链接版消息，返回 (message, public_url)""" +def _build_s3_msg(items: list[tuple[Path, str | None]]) -> UniMessage: + """用归档好的预签名链接构建链接版消息 + + 有文件没拿到链接（归档时上传失败）就抛，交由调用方降级到下一级。 + """ + urls: list[tuple[Path, str]] = [] + for fp, url in items: + if not url: + raise RuntimeError(f"归档未拿到预签名链接: {fp}") + urls.append((fp, url)) + msg = UniMessage() - public_url = None - for fp in media.files: - local_url, pub = upload_with_plan(fp, policy=media.policy) - if not local_url: - raise RuntimeError(f"上传本地 S3 失败: {fp}") - if pub: - public_url = pub + for fp, url in urls: if _is_video(fp): - msg.video(url=local_url) + msg.video(url=url) else: - msg.image(url=local_url) - return msg, public_url - - -async def _public_link_for(media: PendingMedia) -> str | None: - """发送已由本地通道完成后，为「发送下载链接」策略补一次 S3 上传。 - - 本地直发不经 S3（最快），但群策略开了链接就需要公网 URL——这里**只取链接**， - 不重建消息。失败只记日志并返回 None：发送本身已经成功了，不能反过来判失败。 - """ - public_url = None - for fp in media.files: - try: - _, pub = await asyncio.to_thread(upload_with_plan, fp, policy=media.policy) - except Exception as e: # noqa: BLE001 - logger.warning(f"补传 S3 取下载链接失败 {fp}: {e}") - return None - if pub: - public_url = pub - return public_url - - -async def _local_sent(media: PendingMedia) -> tuple[bool, str | None]: - """本地通道发送成功后的返回。 - - 只有策略真的要发下载链接（`sends_link` = send_link 且已开公网）时才补 S3； - 其余情况一概不碰 S3 —— 媒体不走 S3 归档是有意为之（temp 保留由 cleanup 管）。 - """ - if media.policy is not None and media.policy.sends_link: - return True, await _public_link_for(media) - return True, None + msg.image(url=url) + return msg async def _send_media_core( - media: PendingMedia, event: Event | None = None -) -> tuple[bool, str | None]: - """多级发送主体，返回 (是否成功, public_url) + media: PendingMedia, + event: Event | None, + archive: "asyncio.Task[list[UploadedFile]]", +) -> bool: + """多级发送主体，返回是否成功（public_url 由调用方从归档结果取）。 - public_url 有两种来源：走 S3 链接发送成功，或本地发送成功但策略要链接时 - 补传 S3（见 `_local_sent`）；其余情况为 None（调用方据 `sends_link` 决定是否发文字）。 + `archive` 是**已经在跑**的归档任务（见 `send_pending_media`）：本地直发不等它， + 需要链接时再 `await`——同一个 task 多次 await 安全且结果复用，所以每个文件 + 只上传一次，没有"补传"这一步。 temp 文件发送成功后保留（用户手动清理 data/temp）。 多媒体作品（多图 / 图+视频 / 多视频）先走合并转发（本地文件 → S3 链接）， 失败后仍落回下方普通消息链路兜底。 + + **超时（结果未知）时按"可能已送达"处理**：停在这一级并返回 True——协议端 + 可能仍在把这份发完，降级重试会重复发送、发"发送失败"提示也会误导。只有 + 协议端明确报错（ActionFailed 等）才降级，见 `unknown_send_outcome`。 """ if not media.files: - return False, None + return False + + timeout = send_timeout() # ── 0. 多媒体 → 合并转发（一个图/视频一个节点）───────────────── if event is not None and _needs_forward(media.files): @@ -230,51 +322,66 @@ async def _send_media_core( user_id = None if group_id is not None else getattr(event, "user_id", None) if group_id is not None or user_id is not None: sender_name = _forward_sender_name() - if await _send_forward_media( + sent = await _send_forward_media( [(fp, None) for fp in media.files], sender_name, group_id=group_id, user_id=user_id, - ): + ) + if sent is None: + return True + if sent: logger.info("媒体发送成功（合并转发，本地文件）") - return await _local_sent(media) - try: - s3_items, public_url = await _build_s3_forward_items(media) - if await _send_forward_media( - s3_items, sender_name, group_id=group_id, user_id=user_id - ): + return True + items = _presigned_items(await archive) + if all(url for _, url in items): + sent = await _send_forward_media( + items, sender_name, group_id=group_id, user_id=user_id + ) + if sent is None: + return True + if sent: logger.info("媒体发送成功（合并转发，S3 链接）") - return True, public_url - except Exception as e: - logger.warning(f"S3 链接合并转发失败，走普通消息链路: {e}") + return True + else: + logger.warning("归档未全部成功，跳过 S3 链接合并转发，走普通消息链路") else: logger.warning("无法确定群号/用户号，跳过合并转发，走普通消息链路") # ── 1. temp 本地文件直接发送 ────────────────────────────── try: - await _build_local_msg(media.files, media.image_post).send() + await _build_local_msg(media.files, media.image_post).send(_timeout=timeout) logger.info("媒体发送成功（temp 本地文件直达）") - return await _local_sent(media) + return True except Exception as e: + if unknown_send_outcome(e): + _log_unknown("temp 本地文件直达", e) + return True logger.warning(f"temp 本地文件发送失败，切换本地 S3 链接: {e}") - # ── 2. 上传本地 S3 → 链接发送 ───────────────────────────── + # ── 2. 用归档好的预签名链接发送 ───────────────────────────── try: - s3_msg, public_url = _build_s3_msg(media) - await s3_msg.send() + s3_msg = _build_s3_msg(_presigned_items(await archive)) + await s3_msg.send(_timeout=timeout) logger.info("媒体发送成功（本地 S3 链接）") - return True, public_url + return True except Exception as e: + if unknown_send_outcome(e): + _log_unknown("本地 S3 链接", e) + return True logger.warning(f"本地 S3 链接发送失败，回退 temp 本地文件: {e}") # ── 3. 回退：temp 本地文件再发一次 ──────────────────────── try: - await _build_local_msg(media.files, media.image_post).send() + await _build_local_msg(media.files, media.image_post).send(_timeout=timeout) logger.info("媒体发送成功（回退 temp 本地文件）") - return await _local_sent(media) + return True except Exception as e: + if unknown_send_outcome(e): + _log_unknown("回退 temp 本地文件", e) + return True logger.exception(f"回退发送失败: {e}") - return False, None + return False async def _upload_group_files(media: PendingMedia, event: Event | None) -> None: @@ -312,10 +419,24 @@ async def _upload_group_files(media: PendingMedia, event: Event | None) -> None: async def send_pending_media( media: PendingMedia, event: Event | None = None ) -> tuple[bool, str | None]: - """多级发送 + 群文件并行通道，返回 (是否成功, public_url)。""" + """归档 + 多级发送 + 群文件并行通道，返回 (是否成功, public_url)。 + + **归档无条件且先行开跑**（用户 2026-09-25 定的前提）：媒体进了 temp 就必须 + 再进一次桶，与发送走哪一级、成功与否无关。它与本地直发并行，所以发送仍旧是 + 最快那条路；`public_url` 取自归档结果（`upload_public` 关时为 None），交给 + entry.py 按策略决定是否发链接文本。 + + 「是否成功」对**超时（结果未知）**按 True 算：协议端可能仍在发，调用方 + 不该再给群里发"媒体发送失败"（长视频实测就是超时后照样送达）。 + """ if not media.files: return False, None - ok, public_url = await _send_media_core(media, event) + archive = asyncio.create_task(archive_media(media)) + try: + ok = await _send_media_core(media, event, archive) + finally: + # 发送层任何异常都不能把归档半途丢下：归档自己的失败已在内部吞掉。 + uploads = await archive await _upload_group_files(media, event) - return ok, public_url + return ok, _last_public_url(uploads) diff --git a/hexi/plugins/nonebot_plugin_video_analysis/policy.py b/hexi/plugins/nonebot_plugin_video_analysis/policy.py index e810d04..e622adb 100644 --- a/hexi/plugins/nonebot_plugin_video_analysis/policy.py +++ b/hexi/plugins/nonebot_plugin_video_analysis/policy.py @@ -44,11 +44,12 @@ from nonebot import logger XHS = "小红书" BILIBILI = "哔哩哔哩" DOUYIN = "抖音" +KUAISHOU = "快手" YOUTUBE = "Youtube" X = "X" #: 规范标签（存储值 = 展示值，命令/Web 输入同一个词） -PLATFORMS: tuple[str, ...] = (XHS, BILIBILI, DOUYIN, YOUTUBE, X) +PLATFORMS: tuple[str, ...] = (XHS, BILIBILI, DOUYIN, KUAISHOU, YOUTUBE, X) #: 平台 → 域名片段（判定 auto_link / ban_link 命中用） PLATFORM_DOMAINS: dict[str, tuple[str, ...]] = { @@ -60,6 +61,9 @@ PLATFORM_DOMAINS: dict[str, tuple[str, ...]] = { "m.douyin.com", "jingxuan.douyin.com", ), + # 短链 v.kuaishou.com / 网页版 www.kuaishou.com / + # 分享页 v.m.chenzhongtech.com（QQ 卡片 jumpUrl 就走这个域） + KUAISHOU: ("kuaishou.com", "chenzhongtech.com"), YOUTUBE: ("youtube.com", "youtu.be"), X: ("x.com", "twitter.com"), } @@ -74,6 +78,9 @@ _PLATFORM_ALIASES: dict[str, str] = { "哔哩哔哩": BILIBILI, "douyin": DOUYIN, "抖音": DOUYIN, + "kuaishou": KUAISHOU, + "ks": KUAISHOU, + "快手": KUAISHOU, "yt": YOUTUBE, "youtube": YOUTUBE, "油管": YOUTUBE, diff --git a/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/bilibili_content.py b/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/bilibili_content.py index 34f9782..05431cd 100644 --- a/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/bilibili_content.py +++ b/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/bilibili_content.py @@ -224,7 +224,7 @@ def _build_rel_stem( """ return ( f"{build_author_dir(author, author_id, source=source)}" - f"/{build_work_stem(title)}" + f"/{build_work_stem(title, source=source)}" ) diff --git a/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/douyin_api.py b/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/douyin_api.py index 086f020..7551832 100644 --- a/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/douyin_api.py +++ b/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/douyin_api.py @@ -1,309 +1,17 @@ -"""抖音内容抓取 — 浏览器自动化 + API 拦截 + 媒体下载""" +"""媒体下载辅助 — 图集并发下载（抖音/B站/小红书/快手四条链路共用） + +历史上的 playwright 拦截链路与分轨视频合并已随抖音新链路（douyin_web.py） +废弃移除；这里只保留纯下载工具。 +""" import asyncio -import subprocess from pathlib import Path from typing import Dict, List, Optional import httpx from nonebot import logger -from playwright.async_api import async_playwright -from ...models import DouyinFetchError -from ...utils import get_temp_root, unique_media_path -from .douyin_parser import ( - extract_trailing_digits, - parse_animated_note_videos, - parse_douyin_response, - parse_note_images, - parse_ssr_page, - parse_video_urls, -) - - -def merge_video_audio(video_file: Path, audio_file: Path, out_file: Path): - """分轨视频与音频合并""" - subprocess.run( - [ - "ffmpeg", - "-i", - str(video_file), - "-i", - str(audio_file), - "-c:v", - "copy", - "-c:a", - "copy", - "-movflags", - "faststart", - "-y", - str(out_file), - ], - check=True, - ) - - -# ============================= 主入口 ============================= - - -async def fetch_douyin_content( - douyin_url: str, - cookies: List[dict], - wait_seconds: int = 10, - headless: bool = True, -) -> tuple[str | None, Path | List[Path] | None]: - """ - 获取抖音内容（视频或图文） - - 返回: - (title, content) - - 视频: (title, Path) - - 图文: (title, List[Path]) - - 失败: (None, None) - """ - if not cookies: - raise DouyinFetchError("cookies 为空") - - tmp_root = get_temp_root("douyin") - api_response: Optional[dict] = None - api_response_favorite: Optional[dict] = None - aweme_id = 0 - # 目标作品 id（从入口 URL 提取）：aweme/post 返回的是作者作品列表， - # 按此 id 精确匹配要解析的作品，避免取到作者的其他作品 - target_aweme_id = extract_trailing_digits(douyin_url) or "" - referer_url = None - - async with async_playwright() as p: - browser = await p.chromium.launch( - executable_path="C:/Program Files/Google/Chrome/Application/chrome.exe", - headless=headless, - args=[ - "--autoplay-policy=no-user-gesture-required", - "--disable-features=AutoplayDisableSuppression", - ], - ) - - context = await browser.new_context( - viewport={"width": 1280, "height": 720}, - device_scale_factor=2, - user_agent=( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/122.0.0.0 Safari/537.36" - ), - locale="zh-CN", - ) - - await context.add_cookies(cookies) - page = await context.new_page() - - async def handle_response(response): - nonlocal api_response, api_response_favorite, aweme_id, referer_url - try: - if ( - "https://www.douyin.com/note" in response.url - or "https://www.douyin.com/video" in response.url - ): - referer_url = response.url - logger.info(f"作品链接：{referer_url}") - aweme_id = extract_trailing_digits(response.url) - logger.info(f"作品id：{aweme_id}") - - if "aweme/v1/web/aweme/detail" in response.url: - if not api_response: - logger.info(f"捕获到视频 API: {response.url}") - ct = response.headers.get("content-type", "") - if "json" in ct: - api_response = await response.json() - logger.info("API 响应已捕获") - return - else: - logger.info("API 已有捕获") - return - - elif "aweme/v1/web/aweme/post" in response.url: - logger.info(f"捕获到图文 API: {response.url}") - ct = response.headers.get("content-type", "") - if "json" in ct: - data = await response.json() - aweme_list = data.get("aweme_list", []) - if aweme_list: - # 列表是作者的全部作品，优先按目标作品 id 精确匹配 - # （未登录/风控时目标作品可能不在列表里） - match_id = aweme_id or target_aweme_id - target_aweme = None - if match_id: - target_aweme = next( - ( - a - for a in aweme_list - if str(a.get("aweme_id")) == str(match_id) - ), - None, - ) - if not target_aweme: - target_aweme = aweme_list[0] - if target_aweme.get("images"): - logger.info("找到正确的图文 API 响应（包含图片数据）") - data["aweme_list"] = [target_aweme] - api_response = data - return - else: - logger.warning( - "此响应不包含有效的图片数据，等待下一个请求" - ) - except Exception as e: - logger.warning(f"处理响应失败: {e}") - - page.on("response", handle_response) - - await page.goto(douyin_url, wait_until="domcontentloaded") - - max_wait = wait_seconds - for i in range(max_wait): - if api_response: - logger.info(f"成功在第 {i + 1} 秒捕获 API 响应") - break - await page.wait_for_timeout(1000) - - # 截取页面 HTML（在关闭浏览器前），用于 SSR 回退解析 - page_html = await page.content() if not api_response else None - - await browser.close() - - if not api_response and page_html: - api_response = parse_ssr_page(page_html) - if api_response: - logger.info("通过 SSR 页面回退解析获取到图文数据") - - if not api_response: - raise DouyinFetchError("无法捕获 API 响应，请检查网络或 URL") - - parsed = parse_douyin_response(api_response, referer_url) - - headers = { - "Referer": "https://www.douyin.com/", - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/122.0.0.0 Safari/537.36" - ), - } - - if parsed.media_type == "视频": - content = await _process_video( - api_response, tmp_root, parsed.rel_stem, aweme_id, headers - ) - return parsed.raw_title, content - - elif parsed.media_type == "图片": - # 先解析 images 列表，区分纯动图和图文/图+视频 - images_urls, video_url = parse_note_images( - api_response, api_response_favorite, aweme_id - ) - if images_urls: - # 有图片（纯图文 或 图+视频混合作品） - content = await _process_note_with_parsed( - images_urls, - video_url, - tmp_root, - parsed.rel_stem, - headers, - ) - else: - # 纯动图（所有项都是视频） - content = await _process_animated_note( - api_response, tmp_root, parsed.rel_stem, headers - ) - return parsed.raw_title, content - - return None, None - - -# ============================= 视频下载 ============================= - - -async def _process_video( - api_response: dict, - tmp_root: Path, - rel_stem: str, - aweme_id: str, - headers: Dict[str, str], -) -> Path: - """处理视频内容，返回本地文件路径 - - rel_stem 是相对平台根的路径词干 `{作者目录}/{作品名}`（见 ParsedDouyinContent.rel_stem）。 - """ - groups = parse_video_urls(api_response) - - best_group = None - for g in groups.values(): - if g["FULL"] or (g["VIDEO"] and g["AUDIO"]): - best_group = g - break - - if not best_group: - raise DouyinFetchError("没有可用的视频组合") - - # 完整视频 — 流式下载 - if best_group["FULL"]: - full = best_group["FULL"] - logger.info(f"找到完整视频，数量: {len(full)}") - best = max(full, key=lambda x: x["br"]) - logger.info(f"选择码率: {best['br']} - {best['url'][:60]}...") - - output_path = unique_media_path(tmp_root / f"{rel_stem}.mp4") - async with httpx.AsyncClient(headers=headers) as client: - async with client.stream("GET", best["url"]) as resp: - resp.raise_for_status() - with open(output_path, "wb") as f: - async for chunk in resp.aiter_bytes(8192): - f.write(chunk) - logger.info(f"视频下载完成: {output_path}") - return output_path - - # 分轨视频 — 分别下载后合并 - logger.info( - f"使用分轨模式，视频数: {len(best_group['VIDEO'])}, " - f"音频数: {len(best_group['AUDIO'])}" - ) - video = max(best_group["VIDEO"], key=lambda x: x["br"]) - audio = max(best_group["AUDIO"], key=lambda x: x["br"]) - logger.info(f"选择视频码率: {video['br']}") - logger.info(f"选择音频码率: {audio['br']}") - - # 先定下产物名（顺带建好作者目录），分轨中间文件与产物同目录 - output_path = unique_media_path(tmp_root / f"{rel_stem}.mp4") - video_path = output_path.parent / f"{output_path.stem}_v.mp4" - audio_path = output_path.parent / f"{output_path.stem}_a.mp4" - - async with httpx.AsyncClient(headers=headers) as client: - logger.info("开始下载视频...") - async with client.stream("GET", video["url"]) as v: - v.raise_for_status() - with open(video_path, "wb") as f: - async for chunk in v.aiter_bytes(8192): - f.write(chunk) - - logger.info("开始下载音频...") - async with client.stream("GET", audio["url"]) as a: - a.raise_for_status() - with open(audio_path, "wb") as f: - async for chunk in a.aiter_bytes(8192): - f.write(chunk) - - logger.info("合并视频和音频...") - # ffmpeg 是同步子进程，直接 await 不了：不丢线程池会卡住整个事件循环 - # （合并期间 Web 轮询、群消息全都停摆） - await asyncio.to_thread(merge_video_audio, video_path, audio_path, output_path) - video_path.unlink() - audio_path.unlink() - - logger.info(f"视频下载完成: {output_path}") - return output_path - - -# ============================= 图文下载 ============================= +from ...utils import unique_media_path async def _process_note_with_parsed( @@ -313,41 +21,49 @@ async def _process_note_with_parsed( rel_stem: str, headers: Dict[str, str], ) -> List[Path]: - """根据已解析的图片/视频 URL 列表，并行下载 + """按已解析的图片/视频 URL 列表并发下载 一个作品一个目录：`{平台根}/{作者目录}/{作品名}[_{短码}]/001.jpg…` + - images_urls：每项是该媒体位的 URL 候选列表（按优先级排序，逐个回退） + - video_url：图+视频混合作品中追加到末尾的视频（单 URL，可为 None） """ note_dir = unique_media_path(tmp_root / rel_stem) note_dir.mkdir(parents=True, exist_ok=True) logger.info(f"图文保存目录: {note_dir}") - async def _download_one(idx: int, url: str, ext: str = "") -> Optional[Path]: - if not ext: - ext = _infer_extension(url) - filename = f"{idx:03d}{ext}" - filepath = note_dir / filename - try: - async with httpx.AsyncClient(headers=headers) as client: - async with client.stream("GET", url) as resp: - resp.raise_for_status() - with open(filepath, "wb") as f: - async for chunk in resp.aiter_bytes(8192): - f.write(chunk) - logger.info(f"已保存: {filename}") - return filepath - except Exception as e: - logger.error(f"下载 {idx} 失败: {e}") - return None + async def _download_one( + idx: int, url_list: List[str], ext: str = "" + ) -> Optional[Path]: + for url in url_list: + use_ext = ext or _infer_extension(url) + filename = f"{idx:03d}{use_ext}" + filepath = note_dir / filename + try: + async with httpx.AsyncClient(headers=headers, timeout=300) as client: + async with client.stream("GET", url) as resp: + resp.raise_for_status() + with open(filepath, "wb") as f: + async for chunk in resp.aiter_bytes(65536): + f.write(chunk) + if filepath.stat().st_size == 0: + raise RuntimeError("响应为空（0 字节）") + logger.info(f"已保存: {filename}") + return filepath + except Exception as e: + logger.warning(f"下载 {idx} 失败（尝试下一个候选）: {e}") + logger.error(f"下载 {idx} 失败: 所有候选 URL 均不可用") + return None tasks = [ - _download_one(i, url_list[0]) + _download_one(i, url_list) for i, url_list in enumerate(images_urls, 1) + if url_list ] # 图+视频混合作品：视频追加到下载任务 if video_url: next_idx = len(images_urls) + 1 - tasks.append(_download_one(next_idx, video_url, ext=".mp4")) + tasks.append(_download_one(next_idx, [video_url], ext=".mp4")) results = await asyncio.gather(*tasks) saved_paths: List[Path] = [p for p in results if p is not None] @@ -356,75 +72,9 @@ async def _process_note_with_parsed( return saved_paths -async def _process_note( - api_response: dict, - api_response_favorite: dict, - tmp_root: Path, - rel_stem: str, - aweme_id: str, - headers: Dict[str, str], -) -> List[Path]: - """处理图文内容，并行下载所有图片；图+视频混合作品同时下载视频""" - images_urls, video_url = parse_note_images( - api_response, api_response_favorite, aweme_id - ) - logger.info(f"解析到的图片链接:{images_urls}") - - if not images_urls and not video_url: - raise DouyinFetchError("未找到图文链接") - - return await _process_note_with_parsed( - images_urls, video_url, tmp_root, rel_stem, headers - ) - - -# ============================= 动图下载 ============================= - - -async def _process_animated_note( - api_response: dict, - tmp_root: Path, - rel_stem: str, - headers: Dict[str, str], -) -> List[Path]: - """处理动图内容（media_type=42），并行下载所有无声 mp4 视频""" - video_urls = parse_animated_note_videos(api_response) - logger.info(f"解析到的动图视频链接: {video_urls}") - - note_dir = unique_media_path(tmp_root / rel_stem) - note_dir.mkdir(parents=True, exist_ok=True) - logger.info(f"动图保存目录: {note_dir}") - - async def _download_one(idx: int, url: str) -> Optional[Path]: - filename = f"{idx:03d}.mp4" - filepath = note_dir / filename - try: - async with httpx.AsyncClient(headers=headers) as client: - async with client.stream("GET", url) as resp: - resp.raise_for_status() - with open(filepath, "wb") as f: - async for chunk in resp.aiter_bytes(8192): - f.write(chunk) - logger.info(f"动图视频已保存: {filename}") - return filepath - except Exception as e: - logger.error(f"下载动图视频 {idx} 失败: {e}") - return None - - tasks = [ - _download_one(i, url) - for i, url in enumerate(video_urls, 1) - ] - results = await asyncio.gather(*tasks) - saved_paths: List[Path] = [p for p in results if p is not None] - - logger.info(f"动图下载完成，共 {len(saved_paths)} 个视频") - return saved_paths - - def _infer_extension(url: str) -> str: """从 URL 推断文件扩展名""" - extensions = [".webp", ".jpg", ".jpeg", ".png", ".gif", ".avif"] + extensions = [".webp", ".jpg", ".jpeg", ".png", ".gif", ".avif", ".heic"] url_lower = url.lower() for ext in extensions: if ext in url_lower: diff --git a/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/douyin_parser.py b/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/douyin_parser.py index e5594c7..8c73a60 100644 --- a/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/douyin_parser.py +++ b/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/douyin_parser.py @@ -418,7 +418,7 @@ def parse_douyin_response( author_dir = build_author_dir( raw_nickname, raw_author_id, source=referer_url or aweme_id ) - file_name = build_work_stem(raw_title) + file_name = build_work_stem(raw_title, source=referer_url or aweme_id) logger.info( f"内容标题: {raw_title}, 作者: {raw_nickname}({raw_author_id}), " diff --git a/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/douyin_sign.py b/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/douyin_sign.py new file mode 100644 index 0000000..a537c90 --- /dev/null +++ b/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/douyin_sign.py @@ -0,0 +1,929 @@ +# -*- coding: utf-8 -*- +"""抖音 a_bogus 签名 — 纯 Python 实现（免浏览器/免 Node）。 + +来源：https://github.com/Johnserf-Seed/f2 （Apache-2.0 License， + 许可证全文见 https://github.com/Johnserf-Seed/f2/blob/main/LICENSE） + 文件 f2/utils/crypto/bytedance/abogus.py（2026-10-13 拉取，未做任何修改； + 原文件头部的 f2 内部依赖仅有标准库与 gmssl，vendor 后可直接使用）。 + +与上游保持一致：算法对应 bdms 1.0.1.19，2026-10 实测可过抖音 ArgusSecurityPlugin 网关。 +若抖音再次更换签名算法，更新点就是本文件（对照 f2 上游 diff）。 + +依赖：gmssl（pip install gmssl，纯 Python 的国密 SM3 实现）。 +""" +# fmt: off +# ruff: noqa +# path: f2/utils/crypto/bytedance/abogus.py + +#!/usr/bin/env python +# -*- encoding: utf-8 -*- +""" +@Description:abogus.py +@Date :2024/06/16 11:21:14 +@Author :JohnserfSeed +@version :0.0.4 +@License :Apache License 2.0 +@Github :https://github.com/johnserf-seed +@Mail :support@f2.wiki +------------------------------------------------- +Change Log : +2024/06/16 17:27:47 - Create ABogus algorithm & black style +2024/06/16 17:27:47 - Limit custom ua late open source full version +2024/07/08 21:50:12 - Open the full version of the custom UA +2025/03/05 15:55:42 - Perf Post Method Generate +2025/04/27 17:15:26 - Fix mypy type error +------------------------------------------------- +""" + +import random +import time +from typing import Callable, Dict, List, Union + +from gmssl import func, sm3 # type: ignore[import-untyped] + + +class StringProcessor: + """ + StringProcessor 类用于计算ABogus算法中所需的字符串处理方法。 + 包括字符串和 ASCII 码之间的转换、无符号右移运算等。 + + 类方法: + + to_ord_str(s: str) -> str: + 将字符串转换为 ASCII 码字符串。 + + to_ord_array(s: str) -> List[int]: + 将字符串转换为 ASCII 码列表。 + + to_char_str(s: str) -> str: + 将 ASCII 码列表转换为字符串。 + + to_char_array(s: str) -> List[int]: + 将字符串转换为 ASCII 码列表。 + + js_shift_right(val: int, n: int) -> int: + 实现 JavaScript 中的无符号右移运算。 + + generate_random_bytes(length: int = 3) -> str: + 生成一组伪随机字节字符串，用于混淆数据。 + + 使用示例: + ```python + # 将字符串转换为 ASCII 码字符串 + ord_str = StringProcessor.to_ord_str("Hello, World!") + print(ord_str) + + # 将字符串转换为 ASCII 码列表 + ord_array = StringProcessor.to_ord_array("Hello, World!") + print(ord_array) + + # 将 ASCII 码列表转换为字符串 + char_str = StringProcessor.to_char_str(ord_array) + print(char_str) + + # 将字符串转换为 ASCII 码列表 + char_array = StringProcessor.to_char_array("Hello, World!") + print(char_array) + + # 实现 JavaScript 中的无符号右移运算 + shifted_val = StringProcessor.js_shift_right(10, 2) + print(shifted_val) + + # 生成一组伪随机字节字符串 + random_bytes = StringProcessor.generate_random_bytes(3) + print(random_bytes) + ``` + """ + + @staticmethod + def to_ord_str(s: str) -> str: + """ + 将字符串转换为 ASCII 码字符串 (Convert a string to an ASCII code string). + + Args: + s (str): 输入字符串 (Input string). + + Returns: + str: 转换后的 ASCII 码字符串 (Converted ASCII code string). + """ + return "".join([chr(ord(char)) for char in s]) + + @staticmethod + def to_ord_array(s: str) -> List[int]: + """ + 将字符串转换为 ASCII 码列表 (Convert a string to a list of ASCII codes). + + Args: + s (str): 输入字符串 (Input string). + + Returns: + List[int]: 转换后的 ASCII 码列表 (Converted list of ASCII codes). + """ + return [ord(char) for char in s] + + @staticmethod + def to_char_str(s: Union[str, List[int]]) -> str: + """ + 将 ASCII 码列表转换为字符串 (Convert a list of ASCII codes to a string). + + Args: + s (Union[str, List[int]]): ASCII 码列表或字符串 (List of ASCII codes or string). + + Returns: + str: 转换后的字符串 (Converted string). + """ + if isinstance(s, list): + return "".join([chr(i) for i in s]) + elif isinstance(s, str): + return s + else: + raise TypeError( + "Expected str or List[int], got {}".format(type(s).__name__) + ) + + @staticmethod + def to_char_array(s: str) -> List[int]: + """ + 将字符串转换为 ASCII 码列表 (Convert a string to a list of ASCII codes). + + Args: + s (str): 输入字符串 (Input string). + + Returns: + List[int]: 转换后的 ASCII 码列表 (Converted list of ASCII codes). + """ + return [ord(char) for char in s] + + @staticmethod + def js_shift_right(val: int, n: int) -> int: + """ + 实现 JavaScript 中的无符号右移运算 (Implement the unsigned right shift operation in JavaScript). + + Args: + val (int): 输入值 (Input value). + n (int): 右移位数 (Number of bits to shift right). + + Returns: + int: 右移后的值 (Value after right shift). + """ + return (val % 0x100000000) >> n + + @staticmethod + def generate_random_bytes(length: int = 3) -> str: + """ + 生成一组伪随机字节字符串，用于混淆数据 (Generate a pseudo-random byte string to obfuscate the data). + + Args: + length (int): 生成的字节序列长度 (Length of the byte sequence to generate). + + Returns: + str: 生成的伪随机字节字符串 (Generated pseudo-random byte string). + """ + + def generate_byte_sequence() -> List[str]: + _rd = int(random.random() * 10000) + return [ + chr(((_rd & 255) & 170) | 1), + chr(((_rd & 255) & 85) | 2), + chr((StringProcessor.js_shift_right(_rd, 8) & 170) | 5), + chr((StringProcessor.js_shift_right(_rd, 8) & 85) | 40), + ] + + result = [] + for _ in range(length): + result.extend(generate_byte_sequence()) + + return "".join(result) + + +class CryptoUtility: + """ + CryptoUtility 类用于提供加密和编码的工具方法，包括 SM3 哈希、添加盐值、Base64 编码和 RC4 加密等。 + + 类属性: + salt (str): 加密盐值 (Encryption salt). + base64_alphabet (List[str]): 自定义 Base64 字符表 (Custom Base64 alphabet). + + 类方法: + sm3_to_array(input_data: Union[str, List[int]]) -> List[int]: + 计算请求体的 SM3 哈希值，并将结果转换为整数数组。 + + add_salt(param: str) -> str: + 为字符串参数添加盐值。 + + process_param(param: Union[str, List[int]], add_salt: bool) -> Union[str, List[int]]: + 处理输入参数，根据需要添加盐值。 + + params_to_array(param: Union[str, List[int]], add_salt: bool = True) -> List[int]: + 获取输入参数的哈希数组。 + + transform_bytes(bytes_list: List[int]) -> str: + 对输入的字节列表进行加密/解密操作，返回处理后的字符串。 + + base64_encode(input_string: str, selected_alphabet: int = 0) -> str: + 使用自定义字符表对输入字符串进行 Base64 编码。 + + abogus_encode(abogus_bytes_str: str, selected_alphabet: int) -> str: + 对输入的字节字符串进行自定义 Base64 编码，并添加位移和填充。 + + rc4_encrypt(key: bytes, plaintext: str) -> bytes: + 使用 RC4 算法加密数据。 + + 使用示例: + ```python + # 计算请求体的 SM3 哈希值 + sm3_hash = CryptoUtility.sm3_to_array("Hello, World!") + print(sm3_hash) + + # 为字符串参数添加盐值 + salted_param = CryptoUtility.add_salt("Hello, World!") + print(salted_param) + + # 获取输入参数的哈希数组 + hash_array = CryptoUtility.params_to_array("Hello, World!") + print(hash_array) + + # 对输入的字节列表进行加密/解密操作 + encrypted_str = CryptoUtility.transform_bytes([72, 101, 108, 108, 111, 44, 32, 87, 111, 114, 108, 100, 33]) + print(encrypted_str) + + # 使用自定义字符表对输入字符串进行 Base64 编码 + base64_str = CryptoUtility.base64_encode("Hello, World!") + print(base64_str) + + # 对输入的字节字符串进行自定义 Base64 编码，并添加位移和填充 + abogus_str = CryptoUtility.abogus_encode("Hello, World!", 0) + print(abogus_str) + + # 使用 RC4 算法加密数据 + key = b"key" + plaintext = "Hello, World!" + ciphertext = CryptoUtility.rc4_encrypt(key, plaintext) + print(ciphertext) + ``` + """ + + def __init__(self, salt: str, custom_base64_alphabet: List[str]): + """ + 初始化 CryptoUtility 类 + Initialize the CryptoUtility class. + + Args: + salt (str): 加密盐值 (Encryption salt). + custom_base64_alphabet (List[str]): 自定义 Base64 字符表 (Custom Base64 alphabet). + """ + self.salt = salt + self.base64_alphabet = custom_base64_alphabet + + # fmt: off + self.big_array = [ + 121, 243, 55, 234, 103, 36, 47, 228, 30, 231, 106, 6, 115, 95, 78, 101, 250, 207, 198, 50, + 139, 227, 220, 105, 97, 143, 34, 28, 194, 215, 18, 100, 159, 160, 43, 8, 169, 217, 180, 120, + 247, 45, 90, 11, 27, 197, 46, 3, 84, 72, 5, 68, 62, 56, 221, 75, 144, 79, 73, 161, + 178, 81, 64, 187, 134, 117, 186, 118, 16, 241, 130, 71, 89, 147, 122, 129, 65, 40, 88, 150, + 110, 219, 199, 255, 181, 254, 48, 4, 195, 248, 208, 32, 116, 167, 69, 201, 17, 124, 125, 104, + 96, 83, 80, 127, 236, 108, 154, 126, 204, 15, 20, 135, 112, 158, 13, 1, 188, 164, 210, 237, + 222, 98, 212, 77, 253, 42, 170, 202, 26, 22, 29, 182, 251, 10, 173, 152, 58, 138, 54, 141, + 185, 33, 157, 31, 252, 132, 233, 235, 102, 196, 191, 223, 240, 148, 39, 123, 92, 82, 128, 109, + 57, 24, 38, 113, 209, 245, 2, 119, 153, 229, 189, 214, 230, 174, 232, 63, 52, 205, 86, 140, + 66, 175, 111, 171, 246, 133, 238, 193, 99, 60, 74, 91, 225, 51, 76, 37, 145, 211, 166, 151, + 213, 206, 0, 200, 244, 176, 218, 44, 184, 172, 49, 216, 93, 168, 53, 21, 183, 41, 67, 85, + 224, 155, 226, 242, 87, 177, 146, 70, 190, 12, 162, 19, 137, 114, 25, 165, 163, 192, 23, 59, + 9, 94, 179, 107, 35, 7, 142, 131, 239, 203, 149, 136, 61, 249, 14, 156 + ] + # fmt: on + + @staticmethod + def sm3_to_array(input_data: Union[str, List[int]]) -> List[int]: + """ + 计算请求体的 SM3 哈希值，并将结果转换为整数数组 (Calculate the SM3 hash value of the request body and convert the result to an array of integers). + + Args: + input_data (Union[str, List[int]]): 输入数据 (Input data). + + Returns: + List[int]: 哈希值的整数数组 (Array of integers representing the hash value). + """ + # 如果输入是字符串，则将其编码为字节数组 + if isinstance(input_data, str): + input_data_bytes = input_data.encode("utf-8") + else: + input_data_bytes = bytes(input_data) # 将 List[int] 转换为字节数组 + + # 将字节数组转换为适合 sm3.sm3_hash 函数处理的列表格式 + hex_result = sm3.sm3_hash(func.bytes_to_list(input_data_bytes)) + + # 将十六进制字符串结果转换为十进制整数列表 + return [int(hex_result[i : i + 2], 16) for i in range(0, len(hex_result), 2)] + + def add_salt(self, param: str) -> str: + """ + 为字符串参数添加盐值 (Add salt to the string parameter). + + Args: + param (str): 输入字符串 (Input string). + + Returns: + str: 添加盐值后的字符串 (String with added salt). + """ + return param + self.salt + + def process_param( + self, param: Union[str, List[int]], add_salt: bool + ) -> Union[str, List[int]]: + """ + 处理输入参数，根据需要添加盐值 (Process input parameter and add salt if needed). + + Args: + param (Union[str, List[int]]): 输入参数 (Input parameter). + add_salt (bool): 是否添加盐值 (Whether to add salt). + + Returns: + Union[str, List[int]]: 处理后的参数 (Processed parameter). + """ + if isinstance(param, str) and add_salt: + param = self.add_salt(param) + return param + + def params_to_array( + self, param: Union[str, List[int]], add_salt: bool = True + ) -> List[int]: + """ + 获取输入参数的哈希数组 (Get the hash array of the input parameter). + + Args: + param (Union[str, List[int]]): 输入参数 (Input parameter). + add_salt (bool): 是否添加盐值 (Whether to add salt). + + Returns: + List[int]: 哈希数组 (Hash array). + """ + processed_param = self.process_param(param, add_salt) + return self.sm3_to_array(processed_param) + + def transform_bytes(self, bytes_list: List[int]) -> str: + """ + 对输入的字节列表进行加密/解密操作，返回处理后的字符串 (Encrypt/decrypt the input byte list and return the processed string). + + Args: + bytes_list (List[int]): 输入的字节列表 (Input byte list). + + Returns: + str: 处理后的字符串 (Processed string). + """ + # 将字节列表转换为字符字符串 + bytes_str = StringProcessor.to_char_str(bytes_list) + result_str = [] + index_b = self.big_array[1] + initial_value = 0 + value_e = 0 + + for index, char in enumerate(bytes_str): + if index == 0: + initial_value = self.big_array[index_b] + sum_initial = index_b + initial_value + + self.big_array[1] = initial_value + self.big_array[index_b] = index_b + else: + sum_initial = initial_value + value_e + + char_value = ord(char) + sum_initial %= len(self.big_array) + value_f = self.big_array[sum_initial] + encrypted_char = char_value ^ value_f + result_str.append(chr(encrypted_char)) + + # 交换数组元素 + value_e = self.big_array[(index + 2) % len(self.big_array)] + sum_initial = (index_b + value_e) % len(self.big_array) + initial_value = self.big_array[sum_initial] + self.big_array[sum_initial] = self.big_array[ + (index + 2) % len(self.big_array) + ] + self.big_array[(index + 2) % len(self.big_array)] = initial_value + index_b = sum_initial + + return "".join(result_str) + + def base64_encode(self, input_string: str, selected_alphabet: int = 0) -> str: + """ + 使用自定义字符表对输入字符串进行 Base64 编码 (Encode the input string using a custom Base64 alphabet). + + Args: + input_string (str): 输入字符串 (Input string). + selected_alphabet (int): 选择的自定义 Base64 字符表索引 (Selected custom Base64 alphabet index). + + Returns: + str: 编码后的字符串 (Encoded string). + """ + + # 将输入字符串转换为ASCII码的二进制形式 + binary_string = "".join(["{:08b}".format(ord(char)) for char in input_string]) + + # 补全二进制字符串使其长度为6的倍数 + padding_length = (6 - len(binary_string) % 6) % 6 + binary_string += "0" * padding_length + + # 将二进制字符串分割为6位一组 + base64_indices = [ + int(binary_string[i : i + 6], 2) for i in range(0, len(binary_string), 6) + ] + + # 根据自定义字符表生成输出字符串 + output_string = "".join( + [self.base64_alphabet[selected_alphabet][index] for index in base64_indices] + ) + + # 添加等号填充，使符合 Base64 编码规范 + output_string += "=" * (padding_length // 2) + + return output_string + + def abogus_encode(self, abogus_bytes_str: str, selected_alphabet: int) -> str: + """ + 对输入的字节字符串进行自定义 Base64 编码，并添加位移和填充 (Encode the input byte string using a custom Base64 alphabet, and add shifts and padding). + + Args: + abogus_bytes_str (str): 输入的字节字符串 (Input byte string). + selected_alphabet (int): 选择的自定义 Base64 字符表索引 (Selected custom Base64 alphabet index). + + Returns: + str: 编码后的字符串 (Encoded string). + """ + abogus = [] + + for i in range(0, len(abogus_bytes_str), 3): + if i + 2 < len(abogus_bytes_str): + n = ( + (ord(abogus_bytes_str[i]) << 16) + | (ord(abogus_bytes_str[i + 1]) << 8) + | ord(abogus_bytes_str[i + 2]) + ) + elif i + 1 < len(abogus_bytes_str): + n = (ord(abogus_bytes_str[i]) << 16) | ( + ord(abogus_bytes_str[i + 1]) << 8 + ) + else: + n = ord(abogus_bytes_str[i]) << 16 + + for j, k in zip(range(18, -1, -6), (0xFC0000, 0x03F000, 0x0FC0, 0x3F)): + if j == 6 and i + 1 >= len(abogus_bytes_str): + break + if j == 0 and i + 2 >= len(abogus_bytes_str): + break + abogus.append(self.base64_alphabet[selected_alphabet][(n & k) >> j]) + + abogus.append("=" * ((4 - len(abogus) % 4) % 4)) + return "".join(abogus) + + @staticmethod + def rc4_encrypt(key: bytes, plaintext: str) -> bytes: + """ + 使用 RC4 算法加密数据 (Encrypt data using the RC4 algorithm). + + Args: + key (bytes): 加密密钥 (Encryption key). + plaintext (str): 明文数据 (Plaintext data). + + Returns: + bytes: 加密后的数据 (Encrypted data). + """ + S = list(range(256)) + j = 0 + for i in range(256): + j = (j + S[i] + key[i % len(key)]) % 256 + S[i], S[j] = S[j], S[i] + + i = j = 0 + ciphertext = [] + for char in plaintext: + i = (i + 1) % 256 + j = (j + S[i]) % 256 + S[i], S[j] = S[j], S[i] + K = S[(S[i] + S[j]) % 256] + ciphertext.append(ord(char) ^ K) + + return bytes(ciphertext) + + +class BrowserFingerprintGenerator: + """ + BrowserFingerprintGenerator 用于生成模拟的浏览器指纹信息，用于在不同浏览器环境中进行测试和数据采集。 + + 方法: + generate_fingerprint(browser_type="Edge"): + 根据指定的浏览器类型生成浏览器指纹。 + + generate_chrome_fingerprint(): + 生成 Chrome 浏览器指纹。 + + generate_firefox_fingerprint(): + 生成 Firefox 浏览器指纹。 + + generate_safari_fingerprint(): + 生成 Safari 浏览器指纹。 + + generate_edge_fingerprint(): + 生成 Edge 浏览器指纹。 + + _generate_fingerprint(platform="Win32"): + 根据给定的参数生成浏览器指纹字符串。 + + 使用示例: + ```python + chrome_fp = BrowserFingerprintGenerator.generate_fingerprint("Chrome") + print(chrome_fp) + ``` + """ + + @classmethod + def generate_fingerprint(cls, browser_type: str = "Edge") -> str: + """ + 根据指定的浏览器类型生成浏览器指纹。 (Generate a browser fingerprint based on the specified browser type.) + + Args: + browser_type (str): 浏览器类型 (Browser type). + + Returns: + str: 生成的浏览器指纹字符串 (Generated browser fingerprint string). + """ + browsers: Dict[str, Callable[[], str]] = { + "Chrome": cls.generate_chrome_fingerprint, + "Firefox": cls.generate_firefox_fingerprint, + "Safari": cls.generate_safari_fingerprint, + "Edge": cls.generate_edge_fingerprint, + } + return browsers.get(browser_type, cls.generate_chrome_fingerprint)() + + @classmethod + def generate_chrome_fingerprint(cls) -> str: + return cls._generate_fingerprint(platform="Win32") + + @classmethod + def generate_firefox_fingerprint(cls) -> str: + return cls._generate_fingerprint(platform="Win32") + + @classmethod + def generate_safari_fingerprint(cls) -> str: + return cls._generate_fingerprint(platform="MacIntel") + + @classmethod + def generate_edge_fingerprint(cls) -> str: + return cls._generate_fingerprint(platform="Win32") + + @staticmethod + def _generate_fingerprint(platform: str) -> str: + """ + 根据给定的参数生成浏览器指纹字符串。 (Generate a browser fingerprint string based on the given parameters.) + + Args: + platform (str): 操作系统平台 (Operating system platform). + + Returns: + str: 生成的浏览器指纹字符串 (Generated browser fingerprint string). + """ + inner_width = random.randint(1024, 1920) + inner_height = random.randint(768, 1080) + outer_width = inner_width + random.randint(24, 32) + outer_height = inner_height + random.randint(75, 90) + screen_x = 0 + screen_y = random.choice([0, 30]) + size_width = random.randint(1024, 1920) + size_height = random.randint(768, 1080) + avail_width = random.randint(1280, 1920) + avail_height = random.randint(800, 1080) + + fingerprint = ( + f"{inner_width}|{inner_height}|{outer_width}|{outer_height}|" + f"{screen_x}|{screen_y}|0|0|{size_width}|{size_height}|" + f"{avail_width}|{avail_height}|{inner_width}|{inner_height}|24|24|{platform}" + ) + return fingerprint + + +class ABogus: + """ + ABogus 类用于生成 ABogus 参数。 + + 类属性: + array1 (List[int]): 加密请求体 (Encrypted request body). + array2 (List[int]): 加密请求头 (Encrypted request header). + array3 (List[int]): 加密 UA (Encrypted User-Agent). + aid (int): AID 值 (AID value). + pageId (int): 页面 ID (Page ID). + salt (str): 加密盐值 (Encryption salt). + options (List[int]): 请求选项 (Request options). + ua_key (bytes): UA 加密密钥 (UA encryption key). + character (str): 自定义 Base64 字符表 (Custom Base64 alphabet). + character2 (str): 自定义 Base64 字符表 (Custom Base64 alphabet). + character_list (List[str]): 自定义 Base64 字符表列表 (List of custom Base64 alphabets). + crypto_utility (CryptoUtility): 加密工具类 (Encryption utility). + user_agent (str): 自定义 UA (Custom User-Agent). + browser_fp (str): 浏览器指纹 (Browser fingerprint). + sort_index (List[int]): 排序索引 (Sort index). + sort_index_2 (List[int]): 排序索引 (Sort index). + + 说明： + options 参数用于指定请求的类型，GET 请求使用 [0, 1, 8]，POST 请求使用 [0, 1, 14]。14兼容8，POST同样可以编码params，故写死。 + (The options parameter is used to specify the type of request. GET requests use [0, 1, 8], and POST requests use [0, 1, 14]. 14 is compatible with 8, and POST can also encode params, so it is hardcoded.) + + 方法: + encode_data(data: str, alphabet_index: int = 0) -> str: + 使用指定的字符表对数据进行 Base64 编码 (Encode the data using the specified Base64 alphabet). + + generate_abogus(params: str, request: str = "") -> str: + 生成 ABogus 参数 (Generate the ABogus parameter). + + 使用示例: + ```python + # 生成 ABogus 参数，置空使用默认 UA 和浏览器指纹 + abogus = ABogus(user_agent="xxx", fp="xxx") + abogus_param = abogus.generate_abogus("device_platform=webapp&aid=6383&channel=channel_pc_web&aweme_id=7380308675841297704……省略……") + print(abogus_param[1]) + ``` + """ + + def __init__( + self, + fp: str = "", + user_agent: str = "", + options: List[int] = [0, 1, 14], + ): + self.aid = 6383 + self.pageId = 0 # 1.0.1.19 -> 6241 + self.salt = "cus" # 1.0.1.19 -> 加密盐 # dhzx + self.boe = False + self.ddrt = 8.5 + self.ic = 8.5 + self.paths = [ + "^/webcast/", + "^/aweme/v1/", + "^/aweme/v2/", + "/v1/message/send", + "^/live/", + "^/captcha/", + "^/ecom/", + ] + self.array1: list[int] = [] # 加密请求体 + self.array2: list[int] = [] # 加密请求头 为空 + self.array3: list[int] = [] # 加密UA + self.options = options # GET [0, 1, 8] POST [0, 1, 14] + self.ua_key = b"\x00\x01\x0e" # ua加密key + + self.character = ( + "Dkdpgh2ZmsQB80/MfvV36XI1R45-WUAlEixNLwoqYTOPuzKFjJnry79HbGcaStCe" + ) + self.character2 = ( + "ckdp1h4ZKsUB80/Mfvw36XIgR25+WQAlEi7NLboqYTOPuzmFjJnryx9HVGDaStCe" + ) + self.character_list = [self.character, self.character2] # 自定义base64字符表 + + self.crypto_utility = CryptoUtility( + self.salt, self.character_list + ) # 加密工具类 + + self.user_agent = ( + user_agent + if user_agent is not None and user_agent != "" + else "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0" + ) # 自定义ua，为空则设置一个默认ua + + self.browser_fp = ( + fp + if fp is not None and fp != "" + else BrowserFingerprintGenerator.generate_fingerprint("Edge") + ) # 自定义浏览器指纹，为空则生成Edge指纹 + + # fmt: off + self.sort_index = [ + 18, 20, 52, 26, 30, 34, 58, 38, 40, 53, 42, 21, 27, 54, 55, 31, 35, 57, 39, 41, 43, 22, 28, + 32, 60, 36, 23, 29, 33, 37, 44, 45, 59, 46, 47, 48, 49, 50, 24, 25, 65, 66, 70, 71 + ] + self.sort_index_2 = [ + 18, 20, 26, 30, 34, 38, 40, 42, 21, 27, 31, 35, 39, 41, 43, 22, 28, 32, 36, 23, 29, 33, 37, + 44, 45, 46, 47, 48, 49, 50, 24, 25, 52, 53, 54, 55, 57, 58, 59, 60, 65, 66, 70, 71 + ] + # fmt: on + + def encode_data(self, data: str, alphabet_index: int = 0) -> str: + """ + 使用指定的字符表对数据进行 Base64 编码 (Encode the data using the specified Base64 alphabet). + + Args: + data (str): 输入数据 (Input data). + alphabet_index (int): 自定义字符表索引 (Custom alphabet index). + + Returns: + str: 编码后的数据 (Encoded data). + """ + return self.crypto_utility.abogus_encode(data, alphabet_index) + + def generate_abogus(self, params: str, body: str = "") -> tuple: + """ + 生成 abogus 参数 (Generate the ABogus parameter). + + Args: + params (str): 请求参数 (Request parameters). + body (str): 请求体，GET接口则为空 (Request body, empty for GET interfaces). + + Returns: + tuple: params 生成的 abogus 参数 和 ua (ABogus parameter generated by params and ua). + """ + ab_dir = { + 8: 3, # 固定 + 15: { + "aid": self.aid, + "pageId": self.pageId, + "boe": self.boe, + "ddrt": self.ddrt, + "paths": self.paths, + "track": {"mode": 0, "delay": 300, "paths": []}, + "dump": True, + "rpU": "", + }, + 18: 44, + 19: [1, 0, 1, 0, 1], + 66: 0, # 固定 + 69: 0, # 固定 + 70: 0, # 固定 + 71: 0, # 固定 + } + + # 开始加密时间 + start_encryption = int(time.time() * 1000) + + # params参数加盐加密 + array1 = self.crypto_utility.params_to_array( + self.crypto_utility.params_to_array(params) + ) + array2 = self.crypto_utility.params_to_array( + self.crypto_utility.params_to_array(body) + ) + array3 = self.crypto_utility.params_to_array( + self.crypto_utility.base64_encode( + "".join( + [ + chr(b) + for b in self.crypto_utility.rc4_encrypt( + self.ua_key, self.user_agent + ) + ] + ), + 1, + ), + add_salt=False, + ) + + # 结束加密时间 + end_encryption = int(time.time() * 1000) + + # 插入加密开始时间 + ab_dir[20] = (start_encryption >> 24) & 255 + ab_dir[21] = (start_encryption >> 16) & 255 + ab_dir[22] = (start_encryption >> 8) & 255 + ab_dir[23] = start_encryption & 255 + ab_dir[24] = int(start_encryption / 256 / 256 / 256 / 256) >> 0 + ab_dir[25] = int(start_encryption / 256 / 256 / 256 / 256 / 256) >> 0 + + # 插入请求头配置 + ab_dir[26] = (self.options[0] >> 24) & 255 + ab_dir[27] = (self.options[0] >> 16) & 255 + ab_dir[28] = (self.options[0] >> 8) & 255 + ab_dir[29] = self.options[0] & 255 + + # 插入请求方法 + ab_dir[30] = int(self.options[1] / 256) & 255 + ab_dir[31] = (self.options[1] % 256) & 255 + ab_dir[32] = (self.options[1] >> 24) & 255 + ab_dir[33] = (self.options[1] >> 16) & 255 + + # 插入请求头加密 + ab_dir[34] = (self.options[2] >> 24) & 255 + ab_dir[35] = (self.options[2] >> 16) & 255 + ab_dir[36] = (self.options[2] >> 8) & 255 + ab_dir[37] = self.options[2] & 255 + + # 插入请求体加密 + ab_dir[38] = array1[21] + ab_dir[39] = array1[22] + # 插入body加密 + ab_dir[40] = array2[21] + ab_dir[41] = array2[22] + # 插入ua加密 + ab_dir[42] = array3[23] + ab_dir[43] = array3[24] + + # 插入加密结束时间 + ab_dir[44] = (end_encryption >> 24) & 255 + ab_dir[45] = (end_encryption >> 16) & 255 + ab_dir[46] = (end_encryption >> 8) & 255 + ab_dir[47] = end_encryption & 255 + ab_dir[48] = ab_dir[8] + ab_dir[49] = int(end_encryption / 256 / 256 / 256 / 256) >> 0 + ab_dir[50] = int(end_encryption / 256 / 256 / 256 / 256 / 256) >> 0 + + # 插入固定值 + ab_dir[51] = (self.pageId >> 24) & 255 + ab_dir[52] = (self.pageId >> 16) & 255 + ab_dir[53] = (self.pageId >> 8) & 255 + ab_dir[54] = self.pageId & 255 + ab_dir[55] = self.pageId + ab_dir[56] = self.aid + ab_dir[57] = self.aid & 255 + ab_dir[58] = (self.aid >> 8) & 255 + ab_dir[59] = (self.aid >> 16) & 255 + ab_dir[60] = (self.aid >> 24) & 255 + + # 插入浏览器指纹 + ab_dir[64] = len(self.browser_fp) + ab_dir[65] = len(self.browser_fp) + + # 获取 ab_dir 中 sort_index 的值 + sorted_values: List[int] = [] + for i in self.sort_index: + value = ab_dir.get(i, 0) + if isinstance(value, (int, float)): + sorted_values.append(int(value)) + elif isinstance(value, str) and value.isdigit(): + sorted_values.append(int(value)) + else: + sorted_values.append(0) # 对于不能转换为整数的值使用0作为默认值 + + # 将浏览器指纹转换为 ASCII 码列表 + edge_fp_array = StringProcessor.to_char_array(self.browser_fp) + + # 将浏览器指纹长度的低 8 位作为异或值 + ab_xor = (len(self.browser_fp) & 255) >> 8 & 255 + + # 进行异或计算 + for index in range(len(self.sort_index_2) - 1): + if index == 0: + # 防御性编程，确保返回的是整数 + value = ab_dir.get(self.sort_index_2[index], 0) + if isinstance(value, (int, float)): + ab_xor = int(value) + else: + ab_xor = 0 + # 同样防御性地处理第二个值 + value = ab_dir.get(self.sort_index_2[index + 1], 0) + if isinstance(value, (int, float)): + ab_xor ^= int(value) + + # 安全转换edge_fp_array中的值为整数 + clean_edge_fp = [] + for x in edge_fp_array: + if isinstance(x, (int, float, str)) and str(x).isdigit(): + clean_edge_fp.append(int(x)) + else: + # 对于无法转换的值使用0填充 + clean_edge_fp.append(0) + + sorted_values.extend(clean_edge_fp) + sorted_values.append(ab_xor) + + abogus_bytes_str = ( + StringProcessor.generate_random_bytes() + + self.crypto_utility.transform_bytes(sorted_values) + ) + + abogus = self.crypto_utility.abogus_encode(abogus_bytes_str, 0) + params = "%s&a_bogus=%s" % (params, abogus) + return (params, abogus, self.user_agent, body) + + +if __name__ == "__main__": + # 24/06/16 晚点开源自定义ua + # 24/07/08 支持自定义ua和浏览器指纹 + # 24/11/15 完成1.0.1.19版本abogus算法，择日开源 + # 25/03/05 修复POST请求参数加密错误，修补环境 + # 25/04/27 修复mypy检查错误 + + user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0" + chrome_fp = BrowserFingerprintGenerator.generate_fingerprint("Edge") + abogus = ABogus(user_agent=user_agent, fp=chrome_fp) + + # GET + url = "https://www.douyin.com/aweme/v1/web/aweme/detail/?" + params = "device_platform=webapp&aid=6383&channel=channel_pc_web&sec_user_id=MS4wLjABAAAArDVBosPJF3eIWVEFp0szuJ-e1V_-rK0ieJeWwpE77E8&max_cursor=0&locate_query=false&show_live_replay_strategy=1&need_time_list=1&time_list_query=0&whale_cut_token=&cut_version=1&count=18&publish_video_strategy_type=2&from_user_page=1&update_version_code=170400&pc_client_type=1&pc_libra_divert=Windows&support_h265=1&support_dash=0&version_code=290100&version_name=29.1.0&cookie_enabled=true&screen_width=1920&screen_height=1080&browser_language=zh-CN&browser_platform=Win32&browser_name=Edge&browser_version=131.0.0.0&browser_online=true&engine_name=Blink&engine_version=131.0.0.0&os_name=Windows&os_version=10&cpu_core_num=12&device_memory=8&platform=PC&downlink=10&effective_type=4g&round_trip_time=50" + body = "" + print(url + abogus.generate_abogus(params=params, body=body)[0]) + + # POST + url = "https://www.douyin.com/aweme/v2/web/aweme/stats/?" + params = "device_platform=webapp&aid=6383&channel=channel_pc_web&pc_client_type=1&pc_libra_divert=Windows&update_version_code=170400&support_h265=1&support_dash=0&version_code=170400&version_name=17.4.0&cookie_enabled=true&screen_width=1920&screen_height=1080&browser_language=zh-CN&browser_platform=Win32&browser_name=Edge&browser_version=131.0.0.0&browser_online=true&engine_name=Blink&engine_version=131.0.0.0&os_name=Windows&os_version=10&cpu_core_num=12&device_memory=8&platform=PC&downlink=10&effective_type=4g&round_trip_time=50" + body = "aweme_type=0&item_id=7467485482314763572&play_delta=1&source=0" + print(url + abogus.generate_abogus(params=params, body=body)[0]) + + # # 测试生成100个abogus参数 和 100个指纹所需时间 + # start = time.time() + # for _ in range(100): + # abogus.generate_abogus(params=params, body=body) + # end = time.time() + # print("生成100个abogus参数和指纹所需时间:", end - start) # 生成100个abogus参数和指纹所需时间: 2.203000783920288 + + # start = time.time() + # for _ in range(100): + # BrowserFingerprintGenerator.generate_fingerprint("Chrome") + # end = time.time() + # print("生成100个指纹所需时间:", end - start) # 生成100个指纹所需时间: 0.00400090217590332 diff --git a/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/douyin_ssr.py b/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/douyin_ssr.py deleted file mode 100644 index 61f88e5..0000000 --- a/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/douyin_ssr.py +++ /dev/null @@ -1,153 +0,0 @@ -"""抖音图文 SSR 静态解析 — 不走浏览器，直接解析服务端渲染页面 - -与 douyin_api.py 的 playwright 拦截方案互补： - - 视频作品 → fetch_douyin_content（playwright 拦截，可取全部清晰度） - - 图文作品 → fetch_douyin_note_ssr（本模块，按 aweme_id 精准渲染，秒级返回） - -SSR 数据（window._ROUTER_DATA / __pace_f RSC）是服务端按作品 ID 渲染的页面数据， -不存在 playwright 拦截 aweme/v1/web/aweme/post 时取到作者其他作品的问题。 - -注意：必须用移动端 UA 请求 iesdouyin.com/share 端点，桌面 UA 会被 302 回主站 -并返回 JS 反爬壳（无任何 SSR 数据）。 -""" - -import json -import re -from pathlib import Path -from typing import List, Optional - -import httpx -from nonebot import logger - -from ...models import DouyinFetchError -from ...utils import get_temp_root -from .douyin_api import _process_animated_note, _process_note_with_parsed -from .douyin_parser import parse_douyin_response, parse_note_images, parse_ssr_page - -MOBILE_UA = ( - "Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) " - "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 " - "Mobile/15E148 Safari/604.1" -) -ROUTER_DATA_RE = re.compile(r"window\._ROUTER_DATA\s*=\s*(.*?)", re.S) -# 短链重定向后统一为 iesdouyin.com/share/video/{id} 形态（图文作品也如此）， -# 因此 note / video 两种形态都要能提取 ID -AWEME_ID_RE = re.compile(r"/(?:share/)?(?:note|video)/(\d+)") - - -async def fetch_douyin_note_ssr( - note_url: str, - cookies: List[dict], -) -> tuple[Optional[str], Optional[List[Path]]]: - """ - 解析图文作品（SSR 静态解析，秒级返回） - - 接受 note / video 任意形态的抖音链接（重定向后图文也常是 share/video）， - 按 SSR 内容判定：有 images → 图文/动图下载；无 images（真视频）→ - 返回 (None, None) 交由 playwright 链路。 - - Returns: - (title, file_paths) — 图片/视频本地路径列表；非图文返回 (None, None)； - 失败抛 DouyinFetchError - """ - m = AWEME_ID_RE.search(note_url) - if not m: - raise DouyinFetchError(f"无法从链接提取作品 ID: {note_url}") - vid = m.group(1) - - api_response = await _fetch_ssr_api_response(note_url, vid, cookies) - - # 内容判定：SSR 数据无 images → 真视频作品，回落 playwright - aweme = (api_response.get("aweme_list") or [None])[0] or {} - if not aweme.get("images"): - logger.info("SSR 内容为视频作品（无 images），回退 playwright 链路") - return None, None - - # 复用原有解析链：标题 / 作者 / 文件名（referer 统一用 note 形态， - # 避免输入是 share/video 时被误判为"视频"） - parsed = parse_douyin_response( - api_response, f"https://www.douyin.com/note/{vid}" - ) - logger.info(f"SSR 标题: {parsed.raw_title}") - logger.info(f"SSR 作者: {parsed.raw_nickname}") - - tmp_root = get_temp_root("douyin") - headers = { - "Referer": "https://www.douyin.com/", - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/122.0.0.0 Safari/537.36" - ), - } - - # 复用原有分派：纯图文 / 图+视频混合 → 并发下载；纯动图 → 逐张下载 mp4 - images_urls, video_url = parse_note_images(api_response, None, vid) - if images_urls: - file_paths = await _process_note_with_parsed( - images_urls, video_url, tmp_root, parsed.rel_stem, headers - ) - else: - file_paths = await _process_animated_note( - api_response, tmp_root, parsed.rel_stem, headers - ) - return parsed.raw_title, file_paths - - -async def _fetch_ssr_api_response( - note_url: str, vid: str, cookies: List[dict] -) -> dict: - """请求 douyin.com note 页面并提取 SSR 数据，规范化为 aweme_list 格式 - - 2026-08-13 起 iesdouyin.com/share/note 端点对所有请求（含登录态）只返回 - 33KB 壳页（无任何作品数据），改从 www.douyin.com/note/{id} 提取 - __pace_f RSC 数据——服务端按作品 id 精确渲染，与 URL 一一对应， - 不存在 aweme/post 作者列表"取错作品"的问题。 - 必须用桌面 UA + 登录 cookie（whale 风控对未登录请求截断作品数据）。 - """ - url = f"https://www.douyin.com/note/{vid}" - cookie_map = {c["name"]: c["value"] for c in cookies} if cookies else None - headers = { - "User-Agent": ( - "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " - "AppleWebKit/537.36 (KHTML, like Gecko) " - "Chrome/151.0.0.0 Safari/537.36" - ), - "Referer": "https://www.douyin.com/", - } - async with httpx.AsyncClient( - headers=headers, timeout=20, follow_redirects=True - ) as client: - resp = await client.get(url, cookies=cookie_map) - if resp.status_code >= 400: - raise DouyinFetchError(f"SSR 页面请求失败: status={resp.status_code}") - html = resp.text - logger.info(f"SSR 页面: {url} status={resp.status_code} size={len(html)}") - - # 1. 旧版 SSR: window._ROUTER_DATA（snake_case 结构，可直接复用原有解析函数） - if m := ROUTER_DATA_RE.search(html): - data = _clean_js_json(m.group(1)) - if data: - loader = data.get("loaderData", {}) - page = loader.get("note_(id)/page") or loader.get("video_(id)/page") - item_list = (page.get("videoInfoRes") or {}).get("item_list") or [] - if item_list: - logger.info("SSR 数据来源: _ROUTER_DATA") - return {"aweme_list": [item_list[0]]} - # 2. 新版 SSR: __pace_f.push RSC 流（parse_ssr_page 已规范化为 aweme_list 格式） - if parsed := parse_ssr_page(html): - logger.info("SSR 数据来源: __pace_f RSC") - return parsed - raise DouyinFetchError("SSR 页面无图文数据（可能已被删除或风控）") - - -def _clean_js_json(raw: str) -> Optional[dict]: - """清洗 JS 对象文本为合法 JSON（$undefined / $Lx 占位符）""" - text = raw.strip().rstrip(";").strip() - text = re.sub(r"\$undefined", "null", text) - text = re.sub(r"\$L\d+", "null", text) - try: - return json.loads(text) - except json.JSONDecodeError: - logger.warning("SSR JSON 解析失败") - return None diff --git a/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/douyin_web.py b/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/douyin_web.py new file mode 100644 index 0000000..d092550 --- /dev/null +++ b/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/douyin_web.py @@ -0,0 +1,388 @@ +"""抖音 web 接口抓取 — 免浏览器方案（2026-10 版） + +链路：作品链接 → 提取 aweme_id → 签名请求 `aweme/v1/web/aweme/detail/` + → 解析 aweme_detail → 下载「最高码率视频 / 最高画质图片（含动图）」。 + +网关要求（2026-08 起 ArgusSecurityPlugin）： + 1. 请求头 `x-tt-argus` 必须存在（实测只做存在性校验，占位值 "1" 即可）； + 2. 请求头 `uifid` = cookie 里的 UIFID（未登录为 UIFID_TEMP）； + 3. 查询参数 `a_bogus` 由纯 Python 签名器（douyin_sign.py，源自 f2，Apache-2.0）生成， + UA 必须与签名时传入的 UA 一致（签名对 UA 做了二次哈希）。 +以上三点缺一即被网关 403（报错文案 Uifid Not Found / Signature Not Found）。 + +为什么废弃旧链路（SSR + playwright）： + - iesdouyin 分享页 SSR 已整体降级为壳页（无 videoInfoRes）； + - www.douyin.com 页面 HTML 对非浏览器客户端返回 jsvm 反爬壳，浏览器进页面也会 + 随机触发验证码中间页（headless 同样被拦）； + - web 接口是页面播放器同款数据源，含全部清晰度档（bit_rate 梯）与无水印直链。 +""" + +import asyncio +import re +import urllib.parse +from pathlib import Path +from typing import Dict, List, Optional, Tuple, Union + +import httpx +from nonebot import logger + +from ...models import DouyinFetchError +from ...utils import get_temp_root, unique_media_path +from .douyin_parser import parse_douyin_response +from .douyin_sign import ABogus + +# UA 必须与签名输入一致（ABogus 对 UA 做二次哈希）；browser_name/version 参数同步为 Edge +WEB_UA = ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0" +) +MOBILE_UA = ( + "Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) " + "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 " + "Mobile/15E148 Safari/604.1" +) +DETAIL_API = "https://www.douyin.com/aweme/v1/web/aweme/detail/" +REFERER = "https://www.douyin.com/" +AWEME_ID_RE = re.compile(r"/(?:share/)?(?:note|video)/(\d+)") + +# web 端基础参数（对齐浏览器请求；msToken 实测非必需，不生成） +_BASE_PARAMS: Dict[str, Union[str, int]] = { + "device_platform": "webapp", + "aid": "6383", + "channel": "channel_pc_web", + "pc_client_type": 1, + "publish_video_strategy_type": 2, + "pc_libra_divert": "Windows", + "version_code": "290100", + "version_name": "29.1.0", + "cookie_enabled": "true", + "screen_width": 1920, + "screen_height": 1080, + "browser_language": "zh-CN", + "browser_platform": "Win32", + "browser_name": "Edge", + "browser_version": "130.0.0.0", + "browser_online": "true", + "engine_name": "Blink", + "engine_version": "130.0.0.0", + "os_name": "Windows", + "os_version": "10", + "cpu_core_num": 12, + "device_memory": 8, + "platform": "PC", + "downlink": 10, + "effective_type": "4g", + "round_trip_time": 100, +} + + +def extract_aweme_id(url: str) -> Optional[str]: + """从分享链接 / 全链接里提取作品 id（share/note、share/video、note、video 均支持）""" + m = AWEME_ID_RE.search(url) + if m: + return m.group(1) + m = re.search(r"/(\d+)(?:\?|#|$)", url) + return m.group(1) if m else None + + +async def resolve_short_link(url: str) -> Optional[str]: + """短链重定向：取 Location（最多 3 跳）；失败返回 None + + 只用于拿真实作品 URL（提取 aweme_id），不依赖 iesdouyin 页面内容。 + """ + try: + async with httpx.AsyncClient( + headers={"User-Agent": MOBILE_UA}, + follow_redirects=False, + timeout=10, + ) as client: + current = url + for _ in range(3): + resp = await client.get(current) + if resp.status_code >= 400: + return None + location = resp.headers.get("Location") + if not location: + return str(resp.url) + current = urllib.parse.urljoin(current, location) + return current + except Exception: + logger.warning(f"抖音短链重定向失败: {url}") + return None + + +def build_signed_query(aweme_id: str) -> str: + """构造带 a_bogus 签名的查询串（纯函数，便于测试）""" + params = dict(_BASE_PARAMS) + params["aweme_id"] = aweme_id + param_str = "&".join(f"{k}={v}" for k, v in params.items()) + signed, a_bogus, _, _ = ABogus(user_agent=WEB_UA).generate_abogus(param_str) + logger.debug(f"抖音签名: a_bogus={a_bogus[:16]}... len={len(a_bogus)}") + return signed + + +def _douyin_cookie_map(cookies: List[dict]) -> Dict[str, str]: + """Netscape cookie 列表 → {name: value}（只保留 douyin 域）""" + return { + c["name"]: c["value"] + for c in cookies or [] + if "douyin" in (c.get("domain") or "") + } + + +def _build_headers(cookie_map: Dict[str, str]) -> Dict[str, str]: + headers = { + "User-Agent": WEB_UA, + "Referer": REFERER, + "Accept": "application/json, text/plain, */*", + # 网关只校验存在性，占位值即可（f2 项目 issue #443 实测结论） + "x-tt-argus": "1", + } + uifid = cookie_map.get("UIFID") or cookie_map.get("UIFID_TEMP") + if uifid: + headers["uifid"] = uifid + return headers + + +async def fetch_douyin_detail(aweme_id: str, cookies: List[dict]) -> dict: + """请求作品 detail 接口，返回 aweme_detail dict（失败抛 DouyinFetchError）""" + cookie_map = _douyin_cookie_map(cookies) + if not (cookie_map.get("UIFID") or cookie_map.get("UIFID_TEMP")): + raise DouyinFetchError("cookies.txt 缺少抖音 UIFID（重新登录抖音网页版导出）") + + url = f"{DETAIL_API}?{build_signed_query(aweme_id)}" + headers = _build_headers(cookie_map) + last_error: Optional[Exception] = None + for attempt in range(2): + try: + async with httpx.AsyncClient( + headers=headers, cookies=cookie_map, timeout=20 + ) as client: + resp = await client.get(url) + if resp.status_code == 403: + raise DouyinFetchError(f"detail 接口被网关拦截: {resp.text[:80]}") + if resp.status_code >= 400: + raise DouyinFetchError(f"detail 接口请求失败: {resp.status_code}") + data = resp.json() + detail = data.get("aweme_detail") + if detail: + return detail + # status_code=0 但无数据：反爬空响应（签名偶发不同步），重试一次 + last_error = DouyinFetchError( + f"接口返回空数据 (status_code={data.get('status_code')})" + ) + except DouyinFetchError as e: + last_error = e + except Exception as e: # 网络异常 + last_error = e + if attempt == 0: + logger.warning(f"detail 请求失败，1 秒后重试: {last_error!r}") + await asyncio.sleep(1) + raise DouyinFetchError(f"获取作品详情失败: {last_error}") + + +# ============================= 媒体选择 ============================= + + +def _best_urls_from_video(video: dict) -> Tuple[List[str], dict]: + """从 video 对象选最高码率档的 URL 候选列表（含备选 CDN），返回 (urls, meta)""" + gears = [ + g + for g in (video.get("bit_rate") or []) + if isinstance(g, dict) and (g.get("play_addr") or {}).get("url_list") + ] + if gears: + best = max(gears, key=lambda g: g.get("bit_rate") or 0) + play_addr = best.get("play_addr") or {} + meta = { + "gear_name": best.get("gear_name"), + "bit_rate": best.get("bit_rate"), + "width": play_addr.get("width"), + "height": play_addr.get("height"), + "is_h265": best.get("is_h265"), + "size": play_addr.get("data_size"), + } + return list(play_addr.get("url_list") or []), meta + play_addr = video.get("play_addr") or {} + meta = { + "gear_name": None, + "bit_rate": None, + "width": play_addr.get("width"), + "height": play_addr.get("height"), + "is_h265": video.get("is_h265"), + "size": play_addr.get("data_size"), + } + return list(play_addr.get("url_list") or []), meta + + +def collect_download_items( + detail: dict, +) -> Tuple[List[Tuple[List[str], str]], bool]: + """从 aweme_detail 收集下载项。 + + 返回 (items, is_image_post)： + - 视频作品 → ([(视频URL候选, ".mp4")], False) + - 图文（含图+视频混合）→ 每项一个 (URL候选列表, 扩展名)： + 图片项按 无水印 url_list → 带作者水印的 download_url_list 排序； + 内嵌视频项（动图 / 图集中的视频）走最高码率档。 + is_image_post=True + """ + images = detail.get("images") or [] + if not images: + urls, meta = _best_urls_from_video(detail.get("video") or {}) + if not urls: + raise DouyinFetchError("未找到可用视频直链") + logger.info( + f"抖音视频最高档: {meta.get('gear_name')} " + f"{meta.get('width')}x{meta.get('height')} " + f"br={meta.get('bit_rate')} size={meta.get('size')} " + f"h265={meta.get('is_h265')} 候选={len(urls)}" + ) + return [(urls, ".mp4")], False + + items: List[Tuple[List[str], str]] = [] + for img in images: + if not isinstance(img, dict): + continue + video = img.get("video") + if isinstance(video, dict) and (video.get("play_addr") or {}).get("url_list"): + urls, meta = _best_urls_from_video(video) + if urls: + logger.info( + f"图集内视频项: {meta.get('gear_name')} br={meta.get('bit_rate')}" + ) + items.append((urls, ".mp4")) + continue + # 图片项：url_list 为无水印版（q75 webp/jpeg）； + # download_url_list 是带作者水印的"保存到相册"版（tplv-dy-water-v2， + # base64 里就是作者昵称），只能当兜底绝不能优先 + urls = list(img.get("url_list") or []) + list( + img.get("download_url_list") or [] + ) + if urls: + items.append((urls, "")) + if not items: + raise DouyinFetchError("图文作品未解析到任何图片/视频") + logger.info(f"抖音图文作品: {len(items)} 个下载项") + return items, True + + +# ============================= 下载 ============================= + + +def _infer_extension(url: str) -> str: + """从 URL 推断文件扩展名（douyinpic 图片 URL 无扩展名时默认 jpg）""" + url_lower = url.lower() + for ext in (".webp", ".jpeg", ".jpg", ".png", ".gif", ".avif", ".heic"): + if ext in url_lower: + return ext + return ".jpg" + + +async def _stream_download( + url: str, filepath: Path, headers: dict, timeout: float = 300 +) -> None: + async with httpx.AsyncClient(headers=headers, timeout=timeout) as client: + async with client.stream("GET", url) as resp: + resp.raise_for_status() + with open(filepath, "wb") as f: + async for chunk in resp.aiter_bytes(65536): + f.write(chunk) + + +async def _download_with_fallback( + candidates: List[str], filepath: Path, headers: dict +) -> Optional[Path]: + """按候选 URL 顺序下载（CDN 多备份），全部失败返回 None""" + for url in candidates: + try: + await _stream_download(url, filepath, headers) + if filepath.stat().st_size == 0: # 空响应也当失败，换下一个候选 + raise DouyinFetchError("响应为空（0 字节）") + logger.info(f"已保存: {filepath.name} ({filepath.stat().st_size} 字节)") + return filepath + except Exception as e: + logger.warning(f"下载失败，尝试下一个 CDN: {url[:80]}... ({e!r})") + return None + + +async def _download_video_file( + candidates: List[str], tmp_root: Path, rel_stem: str, headers: dict +) -> Path: + output_path = unique_media_path(tmp_root / f"{rel_stem}.mp4") + result = await _download_with_fallback(candidates, output_path, headers) + if result is None: + raise DouyinFetchError("视频下载失败（所有 CDN 均不可用）") + logger.info(f"视频下载完成: {result}") + return result + + +async def _download_note_items( + items: List[Tuple[List[str], str]], + tmp_root: Path, + rel_stem: str, + headers: dict, +) -> List[Path]: + """图集/动图：一个作品一个目录，`001.jpg / 002.mp4 ...` 并发下载""" + note_dir = unique_media_path(tmp_root / rel_stem) + note_dir.mkdir(parents=True, exist_ok=True) + logger.info(f"图集保存目录: {note_dir}") + + async def _one(idx: int, candidates: List[str], ext: str) -> Optional[Path]: + ext = ext or _infer_extension(candidates[0]) + filepath = note_dir / f"{idx:03d}{ext}" + return await _download_with_fallback(candidates, filepath, headers) + + results = await asyncio.gather( + *(_one(i, urls, ext) for i, (urls, ext) in enumerate(items, 1)) + ) + saved = [p for p in results if p is not None] + if not saved: + raise DouyinFetchError("图集下载失败（所有文件均失败）") + logger.info(f"图集下载完成，共 {len(saved)}/{len(items)} 个文件") + return saved + + +# ============================= 主入口 ============================= + + +async def fetch_douyin_web( + url: str, + cookies: List[dict], +) -> Tuple[str, Union[Path, List[Path]], bool]: + """解析抖音作品（视频 / 图文 / 动图），返回 (title, 本地路径, 是否图文) + + 失败抛 DouyinFetchError（由调用方兜底）。 + """ + aweme_id = extract_aweme_id(url) + if not aweme_id: + # 短链（v.douyin.com/xxx）或异常形态 → 先重定向拿全链接再提取 + resolved = await resolve_short_link(url) + if resolved: + logger.info(f"抖音短链重定向: {url} -> {resolved}") + url = resolved + aweme_id = extract_aweme_id(url) + if not aweme_id: + raise DouyinFetchError(f"无法从链接提取作品 ID: {url}") + + detail = await fetch_douyin_detail(aweme_id, cookies) + items, is_img_post = collect_download_items(detail) + + # 内容形态以接口数据为准（分享短链重定向后图文也常是 share/video 形态） + referer = f"https://www.douyin.com/{'note' if is_img_post else 'video'}/{aweme_id}" + parsed = parse_douyin_response({"aweme_detail": detail}, referer) + logger.info(f"抖音标题: {parsed.raw_title}") + logger.info(f"抖音作者: {parsed.raw_nickname}") + logger.info(f"抖音落盘: {parsed.rel_stem}") + + tmp_root = get_temp_root("douyin") + headers = {"Referer": REFERER, "User-Agent": WEB_UA} + if is_img_post: + paths: Union[Path, List[Path]] = await _download_note_items( + items, tmp_root, parsed.rel_stem, headers + ) + else: + paths = await _download_video_file( + items[0][0], tmp_root, parsed.rel_stem, headers + ) + return parsed.raw_title, paths, is_img_post diff --git a/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/kuaishou_content.py b/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/kuaishou_content.py new file mode 100644 index 0000000..ac991f2 --- /dev/null +++ b/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/kuaishou_content.py @@ -0,0 +1,322 @@ +"""快手作品解析 — 分享页 window.INIT_STATE 静态提取 + +借鉴 nonebot-plugin-parser 的 KuaiShouParser： + - v.kuaishou.com/{code} 短链 → 取 Location 跳到 v.m.chenzhongtech.com/fw/photo/{id} + - www.kuaishou.com/short-video/{id} → 重写为移动端 fw/photo/{id} + （桌面站匿名请求返回 `Intercept-Result: risk-control` 的 JSON，拿不到页面） + - /fw/long-video/ 与 /fw/photo/ 返回结构不同，统一按 /fw/photo/ 请求 + - 页面 HTML 提取 window.INIT_STATE → dict 里第一个非空 photo + - 视频优先从清晰度梯 manifest.adaptationSet 挑最优档（见 `_manifest_video_url`）， + 没有 manifest 才退回 photo.mainMvUrls（默认档） + - 图集取 photo.ext_params.atlas 的 cdnList + list 拼 URL + +必须用移动端（iPhone）UA 请求：桌面 UA 与匿名 www 站都只返回风控壳页。 + +返回 (title, 文件)： + - 视频作品 → (title, 视频文件 Path) + - 图集 / 图+视频混排 → (title, [文件路径列表]，视频排在图片之后) + - 纯文字作品 → (title, []) + - 解析失败 → (None, None) +""" + +import json +import re +import urllib.parse +from pathlib import Path +from typing import Optional, Union + +from nonebot import logger + +from ...models import ContentFetchError +from ...utils import ( + build_author_dir, + build_work_stem, + get_temp_root, + unique_media_path, +) + +#: 移动端 UA（桌面 UA 会被风控挡下，见模块文档） +KUAISHOU_UA = ( + "Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) " + "AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 " + "Safari/604.1 Edg/132.0.0.0" +) +KUAISHOU_REFERER = "https://v.kuaishou.com/" + +SHORT_LINK_RE = re.compile(r"v\.kuaishou\.com/[A-Za-z0-9._?%&+\-=/]+") +WEB_SHORT_VIDEO_RE = re.compile( + r"(?:www\.)?kuaishou\.com/(?:short-video|fw/[a-z-]+)/([A-Za-z0-9]+)" +) +INIT_STATE_RE = re.compile(r"window\.INIT_STATE\s*=\s*(.*?)", re.S) + +#: 昵称缺失时的兜底串（utils.build_author_dir 认它，转成"未知作者"规则） +FALLBACK_AUTHOR = "快手用户" + + +async def fetch_kuaishou_content( + url: str, +) -> tuple[Optional[str], Optional[Union[Path, list[Path]]]]: + """解析快手作品链接（短链 / 分享页 / 网页版短链）""" + try: + return await _parse(url) + except Exception: + logger.exception(f"快手解析失败: {url}") + return None, None + + +async def _parse(url: str): + # 1. 短链 → 单跳重定向（Location 即 v.m.chenzhongtech.com/fw/photo/{id}） + if "v.kuaishou.com" in url: + resolved = await _resolve_redirect(url) + if resolved: + logger.info(f"快手短链重定向: {url} -> {resolved}") + url = resolved + + # 2. www.kuaishou.com/short-video/{id} → 移动端分享页（同一套 INIT_STATE） + m = WEB_SHORT_VIDEO_RE.search(url) + if m and "chenzhongtech.com" not in url: + url = f"https://v.m.chenzhongtech.com/fw/photo/{m.group(1)}" + logger.info(f"快手链接规范化: {url}") + + # 3. long-video 与 photo 返回结构不同，统一按 photo 请求 + url = url.replace("/fw/long-video/", "/fw/photo/") + + html = await _fetch_page(url) + photo = _extract_photo(html) + logger.info(f"快手页面解析: {_describe(photo)}") + return await _build_result(photo) + + +def _describe(photo: dict) -> str: + """日志用的一句话摘要（标题/作者/时长）""" + caption = str(photo.get("caption") or "").strip() + nickname = str(photo.get("userName") or "").strip() + if len(caption) > 30: + caption = caption[:30] + return f"作者={nickname or '未知'}, 标题={caption or '无'}, 时长={photo.get('duration') or 0}ms" + + +def _extract_photo(html: str) -> dict: + """从 window.INIT_STATE 提取作品数据（取第一个非空 photo）""" + m = INIT_STATE_RE.search(html) + if not m: + raise ContentFetchError("快手页面无 INIT_STATE（可能已删除或风控）") + + raw = m.group(1).strip() + try: + data = json.loads(raw) + except json.JSONDecodeError: + # 页面偶有 undefined 之类的非 JSON 字面量（同小红书预处理） + try: + data = json.loads(raw.replace("undefined", "null")) + except json.JSONDecodeError: + raise ContentFetchError("快手 INIT_STATE JSON 解析失败") + + if not isinstance(data, dict): + raise ContentFetchError("快手 INIT_STATE 结构异常") + for value in data.values(): + if isinstance(value, dict) and isinstance(value.get("photo"), dict): + return value["photo"] + raise ContentFetchError("页面数据中未找到作品（可能已删除或风控）") + + +def _first_url(items) -> Optional[str]: + """[{cdn, url}, ...] / [url, ...] → 第一个可用 URL（快手同一资源给多个 CDN）""" + if not isinstance(items, list): + return None + for item in items: + if isinstance(item, dict) and item.get("url"): + return str(item["url"]) + if isinstance(item, str) and item: + return item + return None + + +#: 同分辨率/帧率时优先 avc（QQ 客户端播放、群文件解压后二次剪辑都最稳； +#: hevc 体积小但兼容性差，只在画质更高时才选） +_CODEC_RANK = {"avc": 0, "avc1": 0, "h264": 0} + + +def _manifest_video_url(photo: dict) -> Optional[str]: + """从清晰度梯 `manifest.adaptationSet[].representation[]` 挑最优档 + + 排序键 = 分辨率 → 帧率 → 编码（avc 优先，见 `_CODEC_RANK`）→ 码率。 + 参考实现直接用 mainMvUrls（默认档），多档作品拿不到最高清晰度。 + """ + adaptations = (photo.get("manifest") or {}).get("adaptationSet") + if not isinstance(adaptations, list): + return None + + best_rep = None + best_key = None + total = 0 + for adaptation in adaptations: + if not isinstance(adaptation, dict): + continue + for rep in adaptation.get("representation") or []: + if not isinstance(rep, dict) or not rep.get("url"): + continue + total += 1 + codec = str(rep.get("videoCodec") or "").lower() + key = ( + (rep.get("width") or 0) * (rep.get("height") or 0), + rep.get("frameRate") or 0, + -_CODEC_RANK.get(codec, 1), # 取 max，故用负值让 avc 排前 + rep.get("avgBitrate") or 0, + ) + if best_key is None or key > best_key: + best_key, best_rep = key, rep + + if best_rep is None: + return None + logger.info( + f"快手清晰度: {best_rep.get('width')}x{best_rep.get('height')} " + f"{best_rep.get('frameRate')}fps {best_rep.get('videoCodec')} " + f"avg={best_rep.get('avgBitrate')}kbps " + f"size={best_rep.get('fileSize')}（{total} 档中选最优）" + ) + return str(best_rep["url"]) + + +def _video_url(photo: dict) -> Optional[str]: + """视频直链：清晰度梯最优档 → mainMvUrls 兜底（老页面可能没 manifest）""" + return _manifest_video_url(photo) or _first_url(photo.get("mainMvUrls")) + + +def _atlas_image_urls(photo: dict) -> list[str]: + """图集图片：ext_params.atlas.cdnList（图床域名）+ list（路径）拼 URL + + atlas 缺失/为空即非图集作品（视频的 ext_params 里只有编码参数）。 + """ + ext = photo.get("ext_params") or photo.get("extParams") or {} + atlas = ext.get("atlas") if isinstance(ext, dict) else None + if not isinstance(atlas, dict): + return [] + + routes = atlas.get("list") + if not isinstance(routes, list) or not routes: + return [] + + cdn = "" + for item in atlas.get("cdnList") or []: + if isinstance(item, dict) and item.get("cdn"): + cdn = str(item["cdn"]) + break + if not cdn: + return [] + + return [ + f"https://{cdn}/{route}" + for route in routes + if isinstance(route, str) and route + ] + + +def _build_rel_stem(photo: dict) -> str: + """相对平台根的路径词干：`{作者}_{userId}/{作品名}`""" + author = str(photo.get("userName") or "").replace("ㅤ", "").strip() + author = author or FALLBACK_AUTHOR + source = str(photo.get("photoId") or "") + return ( + f"{build_author_dir(author, photo.get('userId'), source=source)}" + f"/{build_work_stem(str(photo.get('caption') or ''), source=source)}" + ) + + +async def _build_result(photo: dict) -> tuple[str, Union[Path, list[Path]]]: + text = str(photo.get("caption") or "").strip() or FALLBACK_AUTHOR + rel_stem = _build_rel_stem(photo) + + video_url = _video_url(photo) + image_urls = _atlas_image_urls(photo) + + # 图集（含"图 + 视频"混排）：一个作品一个目录，视频排在图片之后 + if image_urls: + files = await _download_images(image_urls, video_url, rel_stem) + if not files: + raise ContentFetchError("快手媒体下载失败") + logger.info( + f"快手图集: 图片={len(image_urls)} 张, " + f"视频={'有' if video_url else '无'}, 落盘={len(files)} 个" + ) + return text, files + + if video_url: + path = await _download_video(video_url, rel_stem) + return text, path + + logger.info(f"快手文字作品（无媒体）: {text[:30]}") + return text, [] + + +async def _download_images( + image_urls: list[str], video_url: Optional[str], rel_stem: str +) -> list[Path]: + """并发下载图集（复用抖音图文的下载流程，视频追加在图片之后）""" + + from .douyin_api import _process_note_with_parsed + + tmp_root = get_temp_root("kuaishou") + headers = {"Referer": KUAISHOU_REFERER, "User-Agent": KUAISHOU_UA} + return await _process_note_with_parsed( + [[u] for u in image_urls], video_url, tmp_root, rel_stem, headers + ) + + +async def _download_video(video_url: str, rel_stem: str) -> Path: + """流式下载视频到 `{平台根}/{作者目录}/{作品名}.mp4`""" + import httpx + + tmp_root = get_temp_root("kuaishou") + output_path = unique_media_path(tmp_root / f"{rel_stem}.mp4") + headers = {"Referer": KUAISHOU_REFERER, "User-Agent": KUAISHOU_UA} + async with httpx.AsyncClient(headers=headers, timeout=300) as client: + async with client.stream("GET", video_url) as resp: + resp.raise_for_status() + with open(output_path, "wb") as f: + async for chunk in resp.aiter_bytes(8192): + f.write(chunk) + logger.info(f"快手视频下载完成: {output_path}") + return output_path + + +async def _fetch_page(url: str) -> str: + import httpx + + headers = {"User-Agent": KUAISHOU_UA, "Referer": KUAISHOU_REFERER} + async with httpx.AsyncClient( + headers=headers, timeout=20, follow_redirects=True + ) as client: + resp = await client.get(url) + if resp.status_code >= 400: + raise ContentFetchError(f"快手页面请求失败: status={resp.status_code}") + return resp.text + + +async def _resolve_redirect(url: str) -> Optional[str]: + """短链重定向（最多 3 跳，取最终 URL） + + v.kuaishou.com 是 302 + Location，单跳即到 chenzhongtech 分享页； + 与小红书一样用移动端 headers，避免被当作非 App 来源拒绝。 + """ + import httpx + + headers = {"User-Agent": KUAISHOU_UA, "Referer": KUAISHOU_REFERER} + try: + async with httpx.AsyncClient( + headers=headers, follow_redirects=False, timeout=10 + ) as client: + current = url + for _ in range(3): + resp = await client.get(current) + if resp.status_code >= 400: + return None + location = resp.headers.get("Location") + if not location: + return str(resp.url) + # Location 可能是相对路径，需拼上当前 URL + current = urllib.parse.urljoin(current, location) + return current + except Exception: + logger.warning(f"快手短链重定向失败: {url}") + return None diff --git a/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/rednote_content.py b/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/rednote_content.py index 49028c1..1caf112 100644 --- a/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/rednote_content.py +++ b/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/rednote_content.py @@ -6,8 +6,9 @@ - xiaohongshu.com/discovery/item/{id}?xsec_token=... → 移动端点（fallback） - 页面 HTML 提取 window.__INITIAL_STATE__ → noteDetailMap[id].note -请求端点依次尝试 国际站 rednote.com（含无水印原片 originVideoKey） -→ 国内站 xiaohongshu.com（国内笔记国际站无数据，需 cookies.txt 登录态）。 +请求端点按链接来源选 host 顺序：国内链接（xiaohongshu.com/xhslink.com）先试 +国内站、国际链接（rednote.com）先试国际站（国际站数据含无水印原片 originVideoKey； +国内笔记在国际站没有数据，反之亦然），失败再换站。需 cookies.txt 登录态。 返回 (title, 文件)： - 图文笔记 → (title, [图片路径列表]) @@ -44,16 +45,15 @@ REDNOTE_UA = ( REDNOTE_REFERER = "https://www.xiaohongshu.com/" REDNOTE_SHORT_RE = re.compile(r"xhslink\.(?:com|cn)/[A-Za-z0-9._?%&+=/#@-]+") -EXPLORE_RE = re.compile( - r"xiaohongshu\.com/explore/([0-9a-zA-Z]+)(\?[^\s\"'<>]+)?" -) +EXPLORE_RE = re.compile(r"xiaohongshu\.com/explore/([0-9a-zA-Z]+)(\?[^\s\"'<>]+)?") DISCOVERY_RE = re.compile( r"xiaohongshu\.com/discovery/item/([0-9a-zA-Z]+)(\?[^\s\"'<>]+)?" ) INITIAL_STATE_RE = re.compile(r"window\.__INITIAL_STATE__=(.*?)", re.S) -# 请求 host 顺序：国际站优先（含无水印原片），失败回退国内站（国内笔记） -REDNOTE_HOSTS = ("www.rednote.com", "www.xiaohongshu.com") +# 请求 host 候选：链接来自哪个站就先用哪个站（国内笔记在国际站没有数据， +# 国际站笔记（rednote.com）带无水印原片 originVideoKey，互不通用） +REDNOTE_HOSTS = ("www.xiaohongshu.com", "www.rednote.com") async def fetch_rednote_content( @@ -75,14 +75,18 @@ async def _parse(url: str): logger.info(f"xhslink 重定向: {url} -> {resolved}") url = resolved - # 2. explore 优先（国际站才有 originVideoKey 无水印原片）， - # 依次尝试 国际站 → 国内站；失败再用同一 id+query 回退 discovery 端点 + # 链接来自哪个站就先试哪个站（国际站 rednote.com / 国内站 xiaohongshu.com） + hosts = REDNOTE_HOSTS + if "rednote.com" in url: + hosts = tuple(reversed(REDNOTE_HOSTS)) + + # 2. explore 优先；失败再用同一 id+query 回退 discovery 端点 if m := EXPLORE_RE.search(url): note_id, query = m.group(1), (m.group(2) or "").lstrip("?") - return await _fetch_with_retry(note_id, query, ("explore", "discovery")) + return await _fetch_with_retry(note_id, query, ("explore", "discovery"), hosts) if m := DISCOVERY_RE.search(url): note_id, query = m.group(1), (m.group(2) or "").lstrip("?") - return await _fetch_with_retry(note_id, query, ("discovery",)) + return await _fetch_with_retry(note_id, query, ("discovery",), hosts) # 3. 短链重定向终态: xiaohongshu.com/explore?target_note_id={id}&xsec_token=... # （discovery/item 会 302 到该形态，note id 在 query 里）， @@ -91,14 +95,10 @@ async def _parse(url: str): qs = urllib.parse.parse_qs(m.group(1)) note_id = (qs.get("target_note_id") or [""])[0] if note_id: - keep = { - k: v - for k, v in qs.items() - if k in ("xsec_token", "xsec_source") - } + keep = {k: v for k, v in qs.items() if k in ("xsec_token", "xsec_source")} query = urllib.parse.urlencode(keep, doseq=True) return await _fetch_with_retry( - note_id, query, ("explore", "discovery") + note_id, query, ("explore", "discovery"), hosts ) logger.warning(f"无法识别的小红书链接: {url}") @@ -106,15 +106,18 @@ async def _parse(url: str): async def _fetch_with_retry( - note_id: str, query: str, paths: tuple[str, ...] + note_id: str, + query: str, + paths: tuple[str, ...], + hosts: tuple[str, ...] = REDNOTE_HOSTS, ) -> tuple[str, Union[Path, list[Path]]] | None: - """依次尝试 国际站 → 国内站 × 端点；全部失败后稍候重试一轮 + """依次尝试 host × 端点；全部失败后稍候重试一轮 小红书对突发请求会节流（页面 200 但 noteDetailMap 为空/无数据）， 重试一轮可绕过大部分瞬时风控；重试仍失败返回 (None, None)。 """ for attempt in range(2): - for host in REDNOTE_HOSTS: + for host in hosts: for path in paths: fetch = _parse_explore if path == "explore" else _parse_discovery try: @@ -211,9 +214,7 @@ def _extract_note(html: str, note_id: str) -> dict: async def _build_result(note: dict) -> tuple[str, Union[Path, list[Path]]]: # 空壳 note（短链重定向带 undertake_note_error=该内容暂时无法查看） # → 笔记已删除/私密，直接报错而不是误判为纯文字笔记 - if not any( - note.get(k) for k in ("title", "desc", "type", "imageList", "video") - ): + if not any(note.get(k) for k in ("title", "desc", "type", "imageList", "video")): raise ContentFetchError("笔记内容不可见（可能已删除/私密），无法解析") title = note.get("title") or "" desc = note.get("desc") or "" @@ -229,18 +230,19 @@ async def _build_result(note: dict) -> tuple[str, Union[Path, list[Path]]]: # 1. 视频笔记 → 无水印原片优先 if note.get("type") == "video" and note.get("video"): # 1a. 无水印原片（国际站数据 video.consumer.originVideoKey） - consumer = (note["video"].get("consumer") or {}) + consumer = note["video"].get("consumer") or {} okey = consumer.get("originVideoKey") if okey: video_url = f"https://sns-video-bd.xhscdn.com/{okey}" logger.info(f"小红书视频: 无水印原片 originVideoKey={okey[:30]}...") - video_path = await _download_video(video_url, rel_stem) + video_path = await _download_video([video_url], rel_stem) return text, video_path - # 1b. 无 originVideoKey（国内站数据）→ 从 stream 分组选无水印原片 - # 国内站 masterUrl 同样是 sns-video-v6 原片 CDN（无水印）， + # 1b. 无 originVideoKey（国内站数据）→ 从 stream 分组选最高档 + # 国内站 masterUrl 同样是 sns-video 原片 CDN（无水印）， # 但不同抓取批次返回的清晰度集合不同（同组多条/分组顺序不定）， - # 因此跨全部编码分组收集候选，取 size 最大（质量最高）的流。 + # 因此跨全部编码分组收集候选，取 size 最大（质量最高）的流； + # 每个流附带 backupUrls（备选 CDN），下载时逐个回退。 stream = ((note["video"].get("media") or {}).get("stream")) or {} candidates = [ it @@ -254,23 +256,34 @@ async def _build_result(note: dict) -> tuple[str, Union[Path, list[Path]]]: candidates, key=lambda it: (it.get("size") or 0, it.get("avgBitrate") or 0), ) - video_url = best["masterUrl"] + urls = [best["masterUrl"]] + [ + u for u in (best.get("backupUrls") or []) if u + ] duration = best.get("duration", 0) logger.info( f"小红书视频: 无水印流 {best.get('qualityType')} " f"{best.get('width')}x{best.get('height')} {best.get('fps')}fps " - f"size={best.get('size')} duration={duration}ms" + f"size={best.get('size')} duration={duration}ms " + f"候选={len(urls)}" ) - video_path = await _download_video(video_url, rel_stem) + video_path = await _download_video(urls, rel_stem) return text, video_path raise ContentFetchError("小红书视频流解析失败") - # 2. 图文笔记 - images = [ - img.get("urlDefault") or img.get("url") - for img in note.get("imageList") or [] - ] - images = [u for u in images if u] + # 2. 图文笔记（urlDefault = WB_DFT 默认档；兜底取 infoList 里 WB_DFT 场景） + images = [] + for img in note.get("imageList") or []: + if not isinstance(img, dict): + continue + url = img.get("urlDefault") + if not url: + for info in img.get("infoList") or []: + if isinstance(info, dict) and info.get("imageScene") == "WB_DFT": + url = info.get("url") + break + url = url or img.get("url") + if url: + images.append(url) if not images: logger.info(f"小红书文字笔记: {text[:30]}") return text, [] @@ -290,16 +303,14 @@ def _extract_author_id(note: dict) -> str: return "" -def _build_rel_stem( - nickname: str, author_id: str, title: str, source: str = "" -) -> str: +def _build_rel_stem(nickname: str, author_id: str, title: str, source: str = "") -> str: """相对平台根的路径词干：`{作者}_{userId}/{作品名}` 昵称/作者 id 都拿不到时用 source（笔记 id）当来源码（见 utils.build_author_dir）。 """ return ( f"{build_author_dir(nickname, author_id, source=source)}" - f"/{build_work_stem(title)}" + f"/{build_work_stem(title, source=source)}" ) @@ -318,8 +329,8 @@ async def _download_images(image_urls: list[str], rel_stem: str) -> list[Path]: ) -async def _download_video(video_url: str, rel_stem: str) -> Path: - """流式下载视频 +async def _download_video(video_urls: list[str], rel_stem: str) -> Path: + """流式下载视频（masterUrl + backupUrls 逐个回退） 注意：sns-video-bd（无水印原片）不带 Referer 或带 xiaohongshu.com 均可，但带 rednote.com Referer 会 403，因此不设 Referer。 @@ -329,14 +340,23 @@ async def _download_video(video_url: str, rel_stem: str) -> Path: tmp_root = get_temp_root("xiaohongshu") output_path = unique_media_path(tmp_root / f"{rel_stem}.mp4") headers = {"User-Agent": REDNOTE_UA} - async with httpx.AsyncClient(headers=headers, timeout=300) as client: - async with client.stream("GET", video_url) as resp: - resp.raise_for_status() - with open(output_path, "wb") as f: - async for chunk in resp.aiter_bytes(8192): - f.write(chunk) - logger.info(f"小红书视频下载完成: {output_path}") - return output_path + last_error: Exception | None = None + for video_url in video_urls: + try: + async with httpx.AsyncClient(headers=headers, timeout=300) as client: + async with client.stream("GET", video_url) as resp: + resp.raise_for_status() + with open(output_path, "wb") as f: + async for chunk in resp.aiter_bytes(65536): + f.write(chunk) + if output_path.stat().st_size == 0: + raise ContentFetchError("响应为空（0 字节）") + logger.info(f"小红书视频下载完成: {output_path}") + return output_path + except Exception as e: + last_error = e + logger.warning(f"小红书视频下载失败（尝试下一个候选 CDN）: {e}") + raise ContentFetchError(f"小红书视频下载失败: {last_error}") async def _fetch_page(url: str, headers: dict) -> str: diff --git a/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/video_downloader.py b/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/video_downloader.py index cae3a90..a84aee2 100644 --- a/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/video_downloader.py +++ b/hexi/plugins/nonebot_plugin_video_analysis/services/fetchers/video_downloader.py @@ -19,6 +19,7 @@ from ...utils import ( build_work_stem, get_data_dir, get_temp_root, + short_source_code, slugify, unique_media_path, ) @@ -145,7 +146,8 @@ async def download_video( ext = m.group(1) url_stem = Path(url.split("?")[0]).stem or "video" - slug_stem = slugify(url_stem, max_length=15) or "视频" + # 兜底名同 build_work_stem：用来源短码，别让不同直链挤在同一个 `视频` 下 + slug_stem = slugify(url_stem, max_length=15) or f"视频_{short_source_code(url)}" # 直链拿不到作者信息 → `未知作者_{来源短码}`（同一链接稳定、不同链接不撞） author_dir = build_author_dir(author, author_id, source=url) final_path = unique_media_path(temp_root / author_dir / f"{slug_stem}.{ext}") @@ -277,8 +279,10 @@ async def download_video( title = ((info or {}).get("title") or "").strip() author_dir = build_author_dir(uploader, uploader_id, source=url) + # 标题空时的兜底名用来源短码（作品 id 优先，其次作品链接） + work_stem = build_work_stem(title, source=(info or {}).get("id") or url) new_path = unique_media_path( - temp_root / author_dir / f"{build_work_stem(title)}{original_file.suffix}" + temp_root / author_dir / f"{work_stem}{original_file.suffix}" ) shutil.move(str(original_file), str(new_path)) # 下载用的 scratch 目录已空，顺手收掉（cleanup 不删目录） diff --git a/hexi/plugins/nonebot_plugin_video_analysis/services/storage/group_file.py b/hexi/plugins/nonebot_plugin_video_analysis/services/storage/group_file.py index 8aa275d..3df9847 100644 --- a/hexi/plugins/nonebot_plugin_video_analysis/services/storage/group_file.py +++ b/hexi/plugins/nonebot_plugin_video_analysis/services/storage/group_file.py @@ -15,6 +15,7 @@ from __future__ import annotations import asyncio +import os import re import zipfile from collections.abc import Iterable @@ -23,6 +24,7 @@ from pathlib import Path from typing import TYPE_CHECKING, Any from nonebot import get_bot, logger +from nonebot.exception import NetworkError if TYPE_CHECKING: # 仅类型检查：运行期不导入，单测可裸加载本模块 from ..policy import Policy @@ -154,12 +156,42 @@ def _file_uri(path: Path) -> str: return "file:///" + str(path.resolve()).replace("\\", "/") +#: 群文件上传 per-call 超时（秒）——大文件在协议端「下载 + 上传群文件」会超过 +#: NoneBot 默认的 30s（同 handlers/sender.py 的 `send_timeout`，就地实现以保持 +#: 本模块可裸加载；配置键同一个 `video_analysis_send_timeout`）。 +_DEFAULT_SEND_TIMEOUT = 300.0 + + +def _send_timeout() -> float: + raw = ( + os.environ.get("video_analysis_send_timeout") + or os.environ.get("VIDEO_ANALYSIS_SEND_TIMEOUT") + or "" + ).strip() + try: + value = float(raw) + except ValueError: + return _DEFAULT_SEND_TIMEOUT + return value if value > 0 else _DEFAULT_SEND_TIMEOUT + + +def _unknown_outcome(exc: BaseException) -> bool: + """超时类错误 = 结果未知：协议端可能仍在处理并最终送达（同 sender 的判定）。 + + 这种失败**不能**再降级到本地直传重试——那会让群文件里出现两份。 + """ + return isinstance(exc, NetworkError) or isinstance( + exc, (asyncio.TimeoutError, TimeoutError) + ) + + async def _call_upload( file_value: str, group_id: int, name: str, folder_id: str | None = None ) -> None: params: dict = {"group_id": group_id, "file": file_value, "name": name} if folder_id: params["folder"] = folder_id + params["_timeout"] = _send_timeout() await get_bot().call_api("upload_group_file", **params) @@ -181,7 +213,9 @@ async def upload_group_file( ) -> bool: """上传单个文件到群文件，返回是否成功。 - 两级投递，**S3 链接优先、本地直传兜底**： + 两级投递，**S3 链接优先、本地直传兜底**；只有第 1 级**明确失败**才降级， + 超时（结果未知）直接收手——协议端可能还在传，再直传一份就重复了 + （见 `_unknown_outcome`）。 本环境实测 NapCat 读不到 bot 进程写的本地文件（`realpath ... ENOENT`， 媒体消息的本地直发 55 次全失败、换 S3 链接后次次成功），所以直传只留作 @@ -206,6 +240,14 @@ async def upload_group_file( try: await _call_upload(url, group_id, display_name, folder_id) except Exception as e: + if _unknown_outcome(e): + # 超时=结果未知：协议端可能还在传，再走本地直传会出两份群文件 + logger.warning( + f"群文件 S3 链接上传超时，结果未知（协议端可能仍在传输），" + f"不再降级直传以免重复（群 {group_id} / {display_name}）: " + f"{str(e)[:160]}" + ) + return True logger.warning( f"群文件 S3 链接上传失败（群 {group_id} / {display_name}）: " f"{str(e)[:160]}；改用本地路径重试" @@ -220,6 +262,12 @@ async def upload_group_file( try: await _call_upload(_file_uri(path), group_id, display_name, folder_id) except Exception as e: + if _unknown_outcome(e): + logger.warning( + f"群文件上传超时，结果未知（协议端可能仍在传输）" + f"（群 {group_id} / {display_name}）: {str(e)[:160]}" + ) + return True logger.warning( f"群文件上传失败（群 {group_id} / {display_name}）: {str(e)[:160]}" ) diff --git a/hexi/plugins/nonebot_plugin_video_analysis/services/web_jobs.py b/hexi/plugins/nonebot_plugin_video_analysis/services/web_jobs.py index 684f61a..c65b0d6 100644 --- a/hexi/plugins/nonebot_plugin_video_analysis/services/web_jobs.py +++ b/hexi/plugins/nonebot_plugin_video_analysis/services/web_jobs.py @@ -66,6 +66,10 @@ POSTER_WIDTH = 480 POSTER_SUFFIX = "_封面.jpg" POSTER_TIMEOUT = 20 +#: bot 跑在 pythonw.exe（无控制台）下，spawn 控制台程序（ffmpeg）不带这个标志 +#: 会弹出一个 cmd 黑窗口；只在 Windows 有该常量，其它平台取 0 不影响调用 +CREATE_NO_WINDOW = getattr(subprocess, "CREATE_NO_WINDOW", 0) + #: 状态机：queued → running → done / failed ACTIVE_STATUSES = ("queued", "running") @@ -136,9 +140,14 @@ async def make_poster(video: Path) -> Optional[Path]: str(out), ] try: - # 同步子进程必须丢线程池，否则抽帧期间整个事件循环都停摆 + # 同步子进程必须丢线程池，否则抽帧期间整个事件循环都停摆； + # CREATE_NO_WINDOW 防止 pythonw 下弹 cmd 黑窗口 await asyncio.to_thread( - subprocess.run, cmd, capture_output=True, timeout=POSTER_TIMEOUT + subprocess.run, + cmd, + capture_output=True, + timeout=POSTER_TIMEOUT, + creationflags=CREATE_NO_WINDOW, ) except Exception: # noqa: BLE001 — 封面失败不该拖累任务 logger.warning(f"抽关键帧失败：{video.name}") @@ -167,12 +176,14 @@ async def fetch_media(url: str) -> FetchResult: 分派规则与 `handlers/entry.py::dispatch_url` 一致：抖音 → parse_douyin； B站动态/专栏 → fetch_bilibili_content；其余 B站链接 → download_video（yt-dlp）； - 小红书 → fetch_rednote_content；其它平台 → download_video。 - b23 / xhslink / 抖音短链的重定向在各 fetcher 内部自己处理。 + 小红书 → fetch_rednote_content；快手 → fetch_kuaishou_content； + 其它平台 → download_video。 + b23 / xhslink / 快手短链的重定向在各 fetcher 内部自己处理。 """ from ..handlers.douyin import parse_douyin - from ..policy import BILIBILI, DOUYIN, XHS + from ..policy import BILIBILI, DOUYIN, KUAISHOU, XHS from .fetchers.bilibili_content import fetch_bilibili_content + from .fetchers.kuaishou_content import fetch_kuaishou_content from .fetchers.rednote_content import fetch_rednote_content from .fetchers.video_downloader import download_video @@ -183,6 +194,9 @@ async def fetch_media(url: str) -> FetchResult: if platform == DOUYIN: title, parsed, image_post = await parse_douyin(url) + elif platform == KUAISHOU: + title, parsed = await fetch_kuaishou_content(url) + image_post = isinstance(parsed, list) elif platform == XHS: title, parsed = await fetch_rednote_content(url) image_post = isinstance(parsed, list) @@ -344,7 +358,7 @@ class JobManager: # ── 清理 ────────────────────────────────────────────── def remove(self, job_id: str) -> bool: - """删掉一条任务；还在跑的一并取消（Chrome 那头由 playwright 自己收尾）。""" + """删掉一条任务；还在跑的一并取消（进行中的下载协程由取消信号收尾）。""" job = self._jobs.pop(job_id, None) if job is None: return False diff --git a/hexi/plugins/nonebot_plugin_video_analysis/utils.py b/hexi/plugins/nonebot_plugin_video_analysis/utils.py index 1bda10a..59fcad6 100644 --- a/hexi/plugins/nonebot_plugin_video_analysis/utils.py +++ b/hexi/plugins/nonebot_plugin_video_analysis/utils.py @@ -118,7 +118,7 @@ def short_time_code(at: Optional[float] = None) -> str: _JUNK_AUTHOR_IDS = {"", "0", "none", "null", "na", "nan", "undefined"} #: 抓取层拿不到昵称时的兜底串（slugify 后）—— 它们等于"没有作者信息" -_PLACEHOLDER_AUTHORS = {"未知作者", "小红书用户", "b站用户"} +_PLACEHOLDER_AUTHORS = {"未知作者", "小红书用户", "b站用户", "快手用户"} def _clean_author_id(author_id: Optional[str]) -> str: @@ -174,9 +174,18 @@ def build_author_dir( return f"未知作者_{short_time_code(at)}" -def build_work_stem(title: Optional[str]) -> str: - """作品名做文件名/子目录名：slugify（沿用 15 字上限），空则 `作品`""" - return slugify(title or "", max_length=15) or "作品" +def build_work_stem(title: Optional[str], *, source: Optional[str] = None) -> str: + """作品名做文件名/子目录名：slugify（沿用 15 字上限），空则 `作品_{来源短码}` + + 标题为空（或只有 `#话题`，slugify 后为空）时**不能**固定用 `作品`：同一作者的 + 多个无名作品会全挤在一个名字下，只能靠 4 位**时间**短码区分，每次重解析名字都 + 变，S3 key 与群文件成员名都不可复现。改用 `作品_{short_source_code(source)}`： + 同一来源稳定、不同来源不撞，与 `未知作者_{来源码}` 同一套约定。 + `source` 也没给才退回固定的 `作品`（保住旧契约）。 + """ + return slugify(title or "", max_length=15) or ( + f"作品_{short_source_code(source)}" if source else "作品" + ) def unique_media_path(path: Path, *, at: Optional[float] = None) -> Path: @@ -227,7 +236,7 @@ def media_rel_dir_of(file_path: Path | str) -> str: return "/".join(parts[:-1]) if len(parts) > 1 else "" -def parse_netscape_cookies(file_path: str) -> List[dict]: +def parse_netscape_cookies(file_path: str | Path) -> List[dict]: """解析 Netscape 格式 cookies 文件""" cookies = [] with open(file_path, "r", encoding="utf-8") as f: diff --git a/hexi/plugins/nonebot_plugin_video_analysis/web_hub.py b/hexi/plugins/nonebot_plugin_video_analysis/web_hub.py index 3d14b33..8c50ba3 100644 --- a/hexi/plugins/nonebot_plugin_video_analysis/web_hub.py +++ b/hexi/plugins/nonebot_plugin_video_analysis/web_hub.py @@ -15,14 +15,15 @@ from __future__ import annotations -from fastapi import FastAPI +from fastapi import Depends, FastAPI from fastapi.responses import JSONResponse from nonebot import get_adapter from nonebot.adapters.onebot.v11 import Adapter -from hexi.web_hub.web_auth import require_admin +from hexi.web_hub.web_auth import get_current_user, require_admin +from .cleanup import cleanup_config, run_stats, submit_clean from .policy import PLANS, PLATFORMS, STORE, Policy from .services.web_jobs import JOBS, extract_urls @@ -167,6 +168,50 @@ def build_admin_app() -> FastAPI | None: removed = JOBS.clear_finished() return _ok({"removed": removed}, f"已清理 {removed} 条任务") + # ── temp 缓存：占用统计 + 手动清理 ────────────────────── + # 与「清理temp / temp统计」命令、每日定时任务同一套函数（cleanup.py）， + # 不重写清理逻辑；文件系统操作丢线程池，别卡住事件循环。 + + @app.get("/temp/stats", response_class=JSONResponse, dependencies=[auth]) + async def temp_stats_api(): + """temp 目录占用（文件数/总大小/目录数）+ 当前自动清理配置。""" + st = await run_stats() + st["mode"] = cleanup_config.video_analysis_temp_cleanup_mode + st["max_age_days"] = cleanup_config.video_analysis_temp_max_age_days + st["hour"] = cleanup_config.video_analysis_temp_cleanup_hour + return _ok(st) + + @app.post("/temp/cleanup", response_class=JSONResponse, dependencies=[auth]) + async def temp_cleanup_api(data: dict, user: dict = Depends(get_current_user)): + """手动清理 temp：**提交到任务槽后立即返回**，进度走 /hub/api/tasks/stream。 + + `days=0` 表示全部清空，不传则用配置的保留天数。清理本身是几十秒级的阻塞 + 任务，放请求里会把页面交互一起卡住；同槽已有任务时返回它并带 `reused=true` + （前端据此提示"已在运行"，服务端这层才是防连点的真正保证）。 + """ + raw = data.get("days") + if raw is None or str(raw).strip() == "": + days: int | None = None # 用配置的保留天数 + shown = cleanup_config.video_analysis_temp_max_age_days + else: + try: + days = max(0, int(raw)) + except (TypeError, ValueError): + return _fail("天数参数不合法") + shown = days + + title = "清理 temp（全部）" if shown == 0 else f"清理 temp（保留 {shown} 天）" + task, reused = submit_clean( + days, + title=title, + trigger="web", + actor=str((user or {}).get("username") or ""), + ) + return _ok( + {"task": task.to_dict(), "reused": reused}, + "已有清理任务在跑" if reused else "清理已开始", + ) + return app diff --git a/tests/test_douyin_web.py b/tests/test_douyin_web.py new file mode 100644 index 0000000..60dff7e --- /dev/null +++ b/tests/test_douyin_web.py @@ -0,0 +1,309 @@ +"""抖音免浏览器链路（douyin_web.py）单元测试 + +覆盖：作品 id 提取 / 签名查询串 / 网关头构造 / 最高码率档选择 / 图集下载项收集。 +签名与媒体选择都是纯函数，不碰网络（真实链路实测见 debug 目录脚本）。 + +douyin_web.py 有插件内相对导入（`...models` / `...utils` / `.douyin_sign` 等）， +按 test_kuaishou_parse.py 同款姿势：sys.modules 预置包替身 + 真实加载依赖后裸加载。 +""" + +import importlib.util +import sys +import types +from pathlib import Path + +import pytest + +_REPO = Path(__file__).resolve().parents[1] +_PLUGIN = _REPO / "hexi" / "plugins" / "nonebot_plugin_video_analysis" +_PKG = "hexi.plugins.nonebot_plugin_video_analysis" + +_REAL_MODULES = { + f"{_PKG}.models": _PLUGIN / "models.py", + f"{_PKG}.utils": _PLUGIN / "utils.py", + f"{_PKG}.services.fetchers.douyin_parser": _PLUGIN + / "services/fetchers/douyin_parser.py", + f"{_PKG}.services.fetchers.douyin_sign": _PLUGIN + / "services/fetchers/douyin_sign.py", + f"{_PKG}.services.fetchers.douyin_api": _PLUGIN / "services/fetchers/douyin_api.py", + f"{_PKG}.services.fetchers.douyin_web": _PLUGIN / "services/fetchers/douyin_web.py", +} +_STUB_PACKAGES = (_PKG, f"{_PKG}.services", f"{_PKG}.services.fetchers") + + +def _load_real(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def dw(): + saved = {name: sys.modules.get(name) for name in _STUB_PACKAGES} + for name in _STUB_PACKAGES: + sys.modules[name] = types.ModuleType(name) + for name, path in _REAL_MODULES.items(): + saved[name] = sys.modules.get(name) + _load_real(name, path) + + yield sys.modules[f"{_PKG}.services.fetchers.douyin_web"] + + for name, previous in saved.items(): + if previous is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = previous + + +# ───────────────────────── 样本 ───────────────────────── + + +def _image(url_list=None, download_url_list=None, video=None) -> dict: + img = { + "uri": "img-uri", + "width": 2160, + "height": 2880, + "url_list": ( + url_list + if url_list is not None + else ["https://p3.douyinpic.com/a~q75.webp"] + ), + "download_url_list": ( + download_url_list + if download_url_list is not None + else ["https://p9.douyinpic.com/a~tplv-dy-water-v2:xxx.webp"] + ), + } + if video is not None: + img["video"] = video + return img + + +def _video(bit_rate=None, play_urls=("https://v95.douyinvod.com/full.mp4",)) -> dict: + v = {"play_addr": {"url_list": list(play_urls), "width": 1080, "height": 1440}} + if bit_rate is not None: + v["bit_rate"] = bit_rate + return v + + +def _gear(br, urls, gear_name="gear", h265=0, size=100): + return { + "gear_name": gear_name, + "bit_rate": br, + "is_h265": h265, + "play_addr": { + "url_list": list(urls), + "width": 1080, + "height": 1440, + "data_size": size, + }, + } + + +# ───────────────────────── extract_aweme_id ───────────────────────── + + +@pytest.mark.parametrize( + "url,expected", + [ + ("https://www.douyin.com/note/7696029938990839419", "7696029938990839419"), + ( + "https://www.douyin.com/video/7618117951811949541?foo=1", + "7618117951811949541", + ), + ( + "https://www.iesdouyin.com/share/video/7412345678901234567/?region=CN", + "7412345678901234567", + ), + ( + "https://www.iesdouyin.com/share/note/7412345678901234567/", + "7412345678901234567", + ), + ("https://v.douyin.com/abc123/", None), + ("https://github.com/x/y", None), + ], +) +def test_extract_aweme_id(dw, url, expected): + assert dw.extract_aweme_id(url) == expected + + +# ───────────────────────── 签名 ───────────────────────── + + +def test_build_signed_query(dw): + qs = dw.build_signed_query("7618117951811949541") + assert "aweme_id=7618117951811949541" in qs + assert "a_bogus=" in qs + a_bogus = qs.split("a_bogus=")[-1] + assert len(a_bogus) >= 100 + # 两次签名都有效（签名带随机熵，不要求逐字节一致） + qs2 = dw.build_signed_query("7618117951811949541") + assert "a_bogus=" in qs2 + + +def test_build_signed_query_requires_aweme_id_last(dw): + """aweme_id 应在查询串末尾、a_bogus 追加在其后（与浏览器请求形态一致）""" + qs = dw.build_signed_query("123") + assert qs.index("aweme_id=123") < qs.index("a_bogus=") + + +# ───────────────────────── 网关头 ───────────────────────── + + +def test_build_headers_always_argus_and_uifid(dw): + headers = dw._build_headers({"UIFID": "u-123"}) + assert headers["x-tt-argus"] == "1" + assert headers["uifid"] == "u-123" + assert headers["User-Agent"] == dw.WEB_UA + + +def test_build_headers_uifid_temp_fallback(dw): + headers = dw._build_headers({"UIFID_TEMP": "tmp"}) + assert headers["uifid"] == "tmp" + assert "uifid" not in dw._build_headers({}) + + +def test_cookie_map_filters_domain(dw): + cookies = [ + {"name": "UIFID", "value": "u1", "domain": ".douyin.com"}, + {"name": "a1", "value": "x", "domain": ".xiaohongshu.com"}, + {"name": "ttwid", "value": "t", "domain": "www.douyin.com"}, + ] + m = dw._douyin_cookie_map(cookies) + assert m == {"UIFID": "u1", "ttwid": "t"} + + +# ───────────────────────── 最高码率档选择 ───────────────────────── + + +def test_best_gear_picks_max_bitrate(dw): + video = { + "bit_rate": [ + _gear(773939, ["https://cdn/1080.mp4"], gear_name="normal_1080_0"), + _gear(385679, ["https://cdn/720.mp4"], gear_name="720_1_1", h265=1), + _gear( + 1419088, + ["https://cdn/best.mp4", "https://cdn/best-bak.mp4"], + gear_name="1080_1_1", + ), + ], + "play_addr": {"url_list": ["https://cdn/main.mp4"]}, + } + urls, meta = dw._best_urls_from_video(video) + assert urls == ["https://cdn/best.mp4", "https://cdn/best-bak.mp4"] + assert meta["bit_rate"] == 1419088 + assert meta["width"] == 1080 + + +def test_best_gear_falls_back_to_play_addr(dw): + video = _video(bit_rate=[], play_urls=["https://cdn/only.mp4"]) + urls, meta = dw._best_urls_from_video(video) + assert urls == ["https://cdn/only.mp4"] + assert meta["gear_name"] is None + + +def test_best_gear_skips_gear_without_urls(dw): + video = { + "bit_rate": [ + {"bit_rate": 9999999, "play_addr": {"url_list": []}}, + _gear(100, ["https://cdn/ok.mp4"]), + ], + "play_addr": {"url_list": ["https://cdn/main.mp4"]}, + } + urls, _ = dw._best_urls_from_video(video) + assert urls == ["https://cdn/ok.mp4"] + + +# ───────────────────────── 下载项收集 ───────────────────────── + + +def test_collect_video_post(dw): + detail = { + "aweme_id": "1", + "desc": "x", + "video": _video(bit_rate=[_gear(500, ["https://cdn/v.mp4"])]), + } + items, is_img = dw.collect_download_items(detail) + assert is_img is False + assert items == [(["https://cdn/v.mp4"], ".mp4")] + + +def test_collect_video_post_without_urls_raises(dw): + with pytest.raises(Exception): + dw.collect_download_items({"video": {"play_addr": {"url_list": []}}}) + + +def test_collect_images_prefers_no_watermark_urls(dw): + detail = { + "images": [ + _image( + url_list=["https://p3/a-q75.webp", "https://p3/a-q75.jpeg"], + download_url_list=["https://p9/a-water.webp"], + ) + ] + } + items, is_img = dw.collect_download_items(detail) + assert is_img is True + assert items[0][0] == [ + "https://p3/a-q75.webp", + "https://p3/a-q75.jpeg", # 无水印在前 + "https://p9/a-water.webp", # 带作者水印的保存版兜底 + ] + assert items[0][1] == "" + + +def test_collect_animated_note_uses_video_gears(dw): + anim_video = { + "bit_rate": [_gear(300, ["https://cdn/anim-best.mp4"])], + "play_addr": {"url_list": ["https://cdn/anim.mp4"]}, + } + detail = { + "images": [ + _image(url_list=[], download_url_list=[], video=anim_video), + _image(url_list=[], download_url_list=[], video=anim_video), + ] + } + items, is_img = dw.collect_download_items(detail) + assert is_img is True + assert items == [ + (["https://cdn/anim-best.mp4"], ".mp4"), + (["https://cdn/anim-best.mp4"], ".mp4"), + ] + + +def test_collect_mixed_note(dw): + """图 + 内嵌视频混排：各自按类型收集""" + detail = { + "images": [ + _image(), + _image( + url_list=[], + download_url_list=[], + video={"play_addr": {"url_list": ["https://cdn/clip.mp4"]}}, + ), + ] + } + items, is_img = dw.collect_download_items(detail) + assert is_img is True + assert len(items) == 2 + assert items[0][1] == "" # 图片 + assert items[1] == (["https://cdn/clip.mp4"], ".mp4") + + +# ───────────────────────── 扩展名推断 ───────────────────────── + + +@pytest.mark.parametrize( + "url,expected", + [ + ("https://x/a.jpg?x=1", ".jpg"), + ("https://x/a~q75.webp?y=2", ".webp"), + ("https://x/a.jpeg", ".jpeg"), + ("https://x/a", ".jpg"), + ("https://x/a.heic", ".heic"), + ], +) +def test_infer_extension(dw, url, expected): + assert dw._infer_extension(url) == expected diff --git a/tests/test_kuaishou_parse.py b/tests/test_kuaishou_parse.py new file mode 100644 index 0000000..0e9bb6d --- /dev/null +++ b/tests/test_kuaishou_parse.py @@ -0,0 +1,302 @@ +"""快手分享页解析（kuaishou_content.py 纯函数）单元测试 + +只测不碰网络的解析/命名部分：INIT_STATE 提取、视频/图集 URL 组装、媒体命名。 +样本仿自 2026-10-08 实测的分享页（v.kuaishou.com 短链 → v.m.chenzhongtech.com/fw/photo）。 + +kuaishou_content.py 有插件内相对导入（`...models` / `...utils`），无法裸加载， +故在 sys.modules 里预置父包替身 + 真实加载 utils/models 后按路径加载目标模块 +（与 test_video_sender_link.py 同法）。 +""" + +import importlib.util +import json +import sys +import types +from pathlib import Path + +import pytest + +_REPO = Path(__file__).resolve().parents[1] +_PLUGIN = _REPO / "hexi" / "plugins" / "nonebot_plugin_video_analysis" +_PKG = "hexi.plugins.nonebot_plugin_video_analysis" + +#: 真实加载的插件内模块（utils 纯标准库、models 只有一个异常类） +_REAL_MODULES = { + f"{_PKG}.models": _PLUGIN / "models.py", + f"{_PKG}.utils": _PLUGIN / "utils.py", +} +#: 只当命名空间用的父包（避免触发插件包 __init__ 的 NoneBot 初始化） +_STUB_PACKAGES = (_PKG, f"{_PKG}.services", f"{_PKG}.services.fetchers") + + +def _load_real(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +@pytest.fixture(scope="module") +def ks(): + saved = {name: sys.modules.get(name) for name in _STUB_PACKAGES} + for name in _STUB_PACKAGES: + sys.modules[name] = types.ModuleType(name) + for name, path in _REAL_MODULES.items(): + saved[name] = sys.modules.get(name) + _load_real(name, path) + + target = f"{_PKG}.services.fetchers.kuaishou_content" + saved[target] = sys.modules.get(target) + module = _load_real(target, _PLUGIN / "services" / "fetchers" / "kuaishou_content.py") + + yield module + + for name, previous in saved.items(): + if previous is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = previous + + +@pytest.fixture(scope="module") +def u(ks): + """插件 utils（命名断言用同一套函数，别在测试里手拼）""" + return sys.modules[f"{_PKG}.utils"] + + +# ───────────────────────── 样本 ───────────────────────── + +KEY = "string/rest/wd/ugH5App/photo/simple/info@kpn>KUAISHOU" + + +def _photo(**over) -> dict: + photo = { + "caption": "欧巴欧巴哦 #蔚蓝档案 #砂狼白子", + "userName": "砂狼白子", + "userId": 2329823245, + "photoId": "5206161376908999738", + "duration": 15150, + "timestamp": 1789916802288, + "coverUrls": [ + {"cdn": "p2.a.yximgs.com", "url": "https://p2.a.yximgs.com/cover.jpg"} + ], + "mainMvUrls": [ + {"cdn": "tymov2.a.kwimgs.com", "url": "https://tymov2.a.kwimgs.com/a_b.mp4"}, + {"cdn": "txmov2.a.kwimgs.com", "url": "https://txmov2.a.kwimgs.com/a_b.mp4"}, + ], + "ext_params": {"mtype": 3, "video": 15150}, + } + photo.update(over) + return photo + + +def _page(data: dict) -> str: + payload = json.dumps(data, ensure_ascii=False) + return f"" + + +def _page_with(photo: dict | None, prelude: dict | None = None) -> str: + data = dict(prelude or {}) + data[KEY] = {"result": 1, "photo": photo} + return _page(data) + + +# ───────────────────────── INIT_STATE 提取 ───────────────────────── + + +def test_extract_photo_skips_empty_entries(ks): + """INIT_STATE 是一堆缓存键的 dict，photo 在第一个非空项里（顺序不定）""" + page = _page( + { + "some/cache/key": {"result": 1, "photo": None}, + "another/key": {"result": 2}, + KEY: {"result": 1, "photo": _photo()}, + } + ) + assert ks._extract_photo(page)["photoId"] == "5206161376908999738" + + +def test_extract_photo_tolerates_js_literals(ks): + """页面偶有 undefined 之类非 JSON 字面量（同小红书预处理）""" + page = ( + "" + ) + photo = ks._extract_photo(page) + assert photo["photoId"] == "1" + assert photo["userName"] is None + + +def test_extract_photo_missing_state(ks): + """风控壳页（桌面 UA / www 站）没有 INIT_STATE → 报错而不是静默返回空""" + err = sys.modules[f"{_PKG}.models"].ContentFetchError + with pytest.raises(err): + ks._extract_photo("risk control") + + +def test_extract_photo_without_photo(ks): + """INIT_STATE 在但没有作品（已删除/私密）→ 报错""" + err = sys.modules[f"{_PKG}.models"].ContentFetchError + with pytest.raises(err): + ks._extract_photo(_page({"k": {"result": 2, "photo": None}})) + + +# ───────────────────────── 媒体 URL ───────────────────────── + + +def test_first_url_prefers_first_available(ks): + assert ks._first_url([{"cdn": "a", "url": "https://a/1.mp4"}]) == "https://a/1.mp4" + # 首项没有 url（快手同一资源的备用项可能缺字段）时继续往后找 + assert ks._first_url([{"cdn": "a"}, {"url": "https://b/2.mp4"}]) == "https://b/2.mp4" + assert ks._first_url([]) is None + assert ks._first_url(None) is None + + +def _rep(width, height, fps, codec, bitrate, url, **over) -> dict: + rep = { + "width": width, + "height": height, + "frameRate": fps, + "videoCodec": codec, + "avgBitrate": bitrate, + "url": url, + } + rep.update(over) + return rep + + +def test_manifest_video_url_picks_highest_quality(ks): + """清晰度梯取最优：先比分辨率，再帧率、编码（avc 优先）、码率""" + photo = _photo( + manifest={ + "adaptationSet": [ + { + "representation": [ + # 1080p 30fps hevc 也要赢过 720p 60fps avc（分辨率优先） + _rep(1920, 1080, 30, "hevc", 2000, "https://c/1080.mp4"), + _rep(1280, 720, 60, "avc", 1597, "https://c/720p60.mp4"), + _rep(1280, 720, 30, "hevc", 726, "https://c/720p30.mp4"), + ] + } + ] + } + ) + assert ks._manifest_video_url(photo) == "https://c/1080.mp4" + + +def test_manifest_video_url_prefers_avc_on_tie(ks): + """同分辨率同帧率 → avc 优先（兼容性），hevc 只在画质更高时才选""" + photo = _photo( + manifest={ + "adaptationSet": [ + { + "representation": [ + _rep(1280, 720, 60, "avc", 1597, "https://c/avc.mp4"), + # 码率更高但编码是 hevc，仍输给 avc + _rep(1280, 720, 60, "hevc", 2000, "https://c/hevc.mp4"), + ] + } + ] + } + ) + assert ks._manifest_video_url(photo) == "https://c/avc.mp4" + + +def test_manifest_video_url_skips_broken_entries(ks): + """缺 url / 结构不对的档跳过，不因为一条脏数据整体失败""" + photo = _photo( + manifest={ + "adaptationSet": [ + {"representation": [{"width": 1920, "height": 1080}, "junk"]}, + {"representation": [_rep(1280, 720, 30, "avc", 900, "https://c/720.mp4")]}, + ] + } + ) + assert ks._manifest_video_url(photo) == "https://c/720.mp4" + assert ks._manifest_video_url(_photo()) is None + assert ks._manifest_video_url(_photo(manifest={"adaptationSet": []})) is None + + +def test_video_url_falls_back_to_main_mv_urls(ks): + """老页面没有 manifest → 退回 mainMvUrls（参考实现的默认档路径）""" + assert ks._video_url(_photo()) == "https://tymov2.a.kwimgs.com/a_b.mp4" + photo = _photo( + manifest={"adaptationSet": [{"representation": [_rep(1920, 1080, 30, "avc", 2000, "https://c/best.mp4")]}]} + ) + assert ks._video_url(photo) == "https://c/best.mp4" + + +def test_atlas_image_urls(ks): + photo = _photo( + ext_params={ + "atlas": { + "cdnList": [{"cdn": "p1.a.yximgs.com"}, {"cdn": "p2.a.yximgs.com"}], + "list": ["upic/a.jpg", "upic/b.jpg"], + } + } + ) + assert ks._atlas_image_urls(photo) == [ + "https://p1.a.yximgs.com/upic/a.jpg", + "https://p1.a.yximgs.com/upic/b.jpg", + ] + + +def test_atlas_image_urls_absent(ks): + """视频作品的 ext_params 里没有 atlas → 不是图集""" + assert ks._atlas_image_urls(_photo()) == [] + assert ks._atlas_image_urls(_photo(ext_params={})) == [] + # 有 atlas 但缺图床/路径 → 空（宁可当视频处理也不要拼出坏链接） + assert ks._atlas_image_urls(_photo(ext_params={"atlas": {"list": ["a.jpg"]}})) == [] + assert ks._atlas_image_urls( + _photo(ext_params={"atlas": {"cdnList": [{"cdn": "c"}]}}) + ) == [] + # 老字段名 extParams 也认（ext_params 缺失时才会去看它） + assert ks._atlas_image_urls( + _photo( + ext_params=None, + extParams={"atlas": {"cdnList": [{"cdn": "c"}], "list": ["a.jpg"]}}, + ) + ) == ["https://c/a.jpg"] + + +# ───────────────────────── 命名 ───────────────────────── + + +def test_rel_stem_uses_author_id(ks, u): + """有 userId（数字）→ 稳定作者目录，标题去掉 #话题""" + assert ks._build_rel_stem(_photo()) == "砂狼白子_2329823245/欧巴欧巴哦" + + +def test_rel_stem_without_author(ks, u): + """昵称是兜底串（"快手用户"）→ 按"未知作者"规则 + 作品来源码""" + stem = ks._build_rel_stem(_photo(userName="快手用户", userId=0)) + code = u.short_source_code("5206161376908999738") + assert stem == f"未知作者_{code}/欧巴欧巴哦" + + +def test_rel_stem_empty_caption(ks, u): + """标题只有 #话题（slugify 后为空）→ 作品_来源码，同一作品稳定可复现""" + stem = ks._build_rel_stem(_photo(caption="#蔚蓝档案")) + code = u.short_source_code("5206161376908999738") + assert stem == f"砂狼白子_2329823245/作品_{code}" + + +# ───────────────────────── URL 识别 ───────────────────────── + + +def test_web_short_video_regex(ks): + """www 站形态（分享出来的网页链接，实际会重写成移动端分享页）""" + for url, expected in ( + ("https://www.kuaishou.com/short-video/3xhjgcmir24m4nm", "3xhjgcmir24m4nm"), + ("https://www.kuaishou.com/short-video/5206161376908999738", "5206161376908999738"), + ("https://kuaishou.com/short-video/abc123", "abc123"), + ): + m = ks.WEB_SHORT_VIDEO_RE.search(url) + assert m and m.group(1) == expected + + +def test_short_link_regex(ks): + m = ks.SHORT_LINK_RE.search("看看这个 https://v.kuaishou.com/nUqhAiDI 好看") + assert m and m.group(0) == "v.kuaishou.com/nUqhAiDI" diff --git a/tests/test_video_cleanup.py b/tests/test_video_cleanup.py new file mode 100644 index 0000000..1d0c784 --- /dev/null +++ b/tests/test_video_cleanup.py @@ -0,0 +1,214 @@ +"""temp 目录清理逻辑（video_analysis/cleanup.py）单元测试 + +这组函数现在同时被「/清理temp 命令」「每日定时」「Web 缓存面板的手动清理」 +三条路复用，是**真的会删文件**的代码，所以单测覆盖"删什么/不删什么"。 + +cleanup.py 有插件内相对导入（`.utils`）和 import 期副作用（`get_plugin_config` +建配置对象、`_register_auto_job()` 注册定时任务），无法裸加载 —— 按 +test_video_sender_link.py 的办法：sys.modules 预置替身 + monkeypatch nonebot 的 +几个工厂函数，再按路径加载目标模块。 + +文件一律用 tmp_path 造真实文件/目录 + `os.utime` 改 mtime，**不 mock 文件系统**： +这些判断（mtime 过期、目录为空、60s 目录下限）全在真实 stat 语义上。 +""" + +import importlib.util +import os +import sys +import time +import types +from pathlib import Path + +import pytest + +_REPO = Path(__file__).resolve().parents[1] +_PLUGIN = _REPO / "hexi" / "plugins" / "nonebot_plugin_video_analysis" +_PKG = "hexi.plugins.nonebot_plugin_video_analysis" + +_STUBS = { + "hexi": {}, + "hexi.core": {}, + "hexi.core.custom_utils": {"check_manage": None}, + "hexi.core.scheduler": {"add_job": lambda *a, **k: None}, + # cleanup.py 现在把清理提交到任务槽；这里只测 clean_temp_files / temp_stats， + # 槽本身由 tests/test_task_slots.py 覆盖，所以给个够 import 的替身即可。 + "hexi.core.task_slots": {"TASKS": None, "Task": type("Task", (), {})}, + "nonebot_plugin_apscheduler": {"scheduler": types.SimpleNamespace(remove_job=None)}, + "nonebot_plugin_alconna": {"UniMessage": type("UniMessage", (), {})}, +} + + +class _FakeMatcher: + """`on_command(...)` 的替身：`.handle()` 当装饰器用。""" + + def handle(self, *args, **kwargs): + def deco(func): + return func + + return deco + + +class _TempRoot: + """`utils.get_temp_root` 的替身：指向测试用的临时根。""" + + def __init__(self) -> None: + self.root: Path | None = None + + def __call__(self, sub: str = "") -> Path: + assert self.root is not None, "测试里要先 sched.get_temp_root.root = ..." + return self.root / sub if sub else self.root + + +@pytest.fixture +def cleanup(monkeypatch, tmp_path): + saved = {name: sys.modules.get(name) for name in _STUBS} + for name, attrs in _STUBS.items(): + module = types.ModuleType(name) + for key, value in attrs.items(): + setattr(module, key, value) + sys.modules[name] = module + + # 插件包本体也用替身，避免触发 __init__ 的 NoneBot 初始化 + saved[_PKG] = sys.modules.get(_PKG) + sys.modules[_PKG] = types.ModuleType(_PKG) + temp_root = _TempRoot() + utils_mod = types.ModuleType(f"{_PKG}.utils") + utils_mod.get_temp_root = temp_root + saved[f"{_PKG}.utils"] = sys.modules.get(f"{_PKG}.utils") + sys.modules[f"{_PKG}.utils"] = utils_mod + + import nonebot + + monkeypatch.setattr(nonebot, "require", lambda name: None, raising=False) + monkeypatch.setattr( + nonebot, "get_plugin_config", lambda cls: cls(), raising=False + ) # 默认配置：auto / 7 天 / 4 点 + monkeypatch.setattr(nonebot, "on_command", lambda *a, **k: _FakeMatcher(), raising=False) + + target = f"{_PKG}.cleanup" + saved[target] = sys.modules.get(target) + spec = importlib.util.spec_from_file_location(target, _PLUGIN / "cleanup.py") + module = importlib.util.module_from_spec(spec) + sys.modules[target] = module + spec.loader.exec_module(module) + + temp_root.root = tmp_path / "temp" + temp_root.root.mkdir() + module._tmp = temp_root.root # 方便测试直接拿到根 + + yield module + + for name, previous in saved.items(): + if previous is None: + sys.modules.pop(name, None) + else: + sys.modules[name] = previous + + +def _make(root: Path, rel: str, age_days: float = 0.0, data: bytes = b"x") -> Path: + """造一个文件，并把 mtime 往回拨 age_days 天。""" + path = root / rel + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(data) + if age_days: + t = time.time() - age_days * 86400 + os.utime(path, (t, t)) + return path + + +# ───────────────────────── 清理 ───────────────────────── + + +def test_removes_stale_keeps_fresh(cleanup): + root = cleanup._tmp + stale = _make(root, "douyin/作者_1/旧作品.mp4", age_days=2) + fresh = _make(root, "douyin/作者_1/新作品.mp4", age_days=0) + + removed, total = cleanup.clean_temp_files(days=1) + + assert (removed, total) == (1, 2) + assert not stale.exists() + assert fresh.exists() + + +def test_days_zero_clears_files_but_keeps_fresh_dirs(cleanup): + """`/清理temp 0`（= Web 的「全部清空」）：文件全删，但刚建的目录留着。 + + 目录另有 60s 下限（见 `_prune_empty_dirs`）—— 下载落盘是先 mkdir 再写文件， + 把目录一起删掉会让正在进行的下载失败。 + """ + root = cleanup._tmp + a = _make(root, "kuaishou/作者_2/作品.mp4", age_days=0) + b = _make(root, "xiaohongshu/作者_3/笔记/001.jpg", age_days=0) + + removed, total = cleanup.clean_temp_files(days=0) + + assert (removed, total) == (2, 2) + assert not a.exists() and not b.exists() + assert (root / "kuaishou/作者_2").is_dir() # 新鲜目录保留 + assert (root / "xiaohongshu/作者_3/笔记").is_dir() + + +def test_prunes_stale_empty_dirs_bottom_up(cleanup): + """过期空目录会被收掉，但是**逐轮**回收而不是同一轮全清。 + + 删文件（或删子目录）会把所在目录的 mtime 刷成"现在"，当轮它就不再"过期"， + 要等下一轮 —— 这正是 `_prune_empty_dirs` 注释里写的行为（也被这条测试钉住）。 + """ + root = cleanup._tmp + _make(root, "bilibili/作者_4/作品/001.jpg", age_days=3) + nested = root / "bilibili/作者_4/作品" + author = root / "bilibili/作者_4" + + removed, _ = cleanup.clean_temp_files(days=1) + assert removed == 1 + assert nested.is_dir() and author.is_dir() # 文件刚删，目录 mtime 是新的 + + old = time.time() - 3 * 86400 # 模拟又过了几天 + os.utime(nested, (old, old)) + os.utime(author, (old, old)) + cleanup.clean_temp_files(days=1) + assert not nested.exists() # 深的先走 + + os.utime(author, (old, old)) # 上一轮删子目录，又把父目录的 mtime 刷新了 + cleanup.clean_temp_files(days=1) + assert not author.exists() + + +def test_keeps_stale_but_nonempty_dirs(cleanup): + """过期但里面还有新鲜文件的目录不能删。""" + root = cleanup._tmp + _make(root, "douyin/作者_5/新片.mp4", age_days=0) + old_dir = root / "douyin/作者_5" + os.utime(old_dir, (time.time() - 3 * 86400,) * 2) + + cleanup.clean_temp_files(days=1) + + assert old_dir.is_dir() + + +def test_missing_root_is_noop(cleanup): + """根目录不存在（还没下载过任何东西）→ (0, 0)，不抛。""" + cleanup._tmp.rmdir() + assert cleanup.clean_temp_files(days=1) == (0, 0) + + +# ───────────────────────── 统计 ───────────────────────── + + +def test_stats_counts_files_bytes_dirs(cleanup): + root = cleanup._tmp + _make(root, "douyin/作者_6/a.mp4", age_days=2, data=b"12345") + _make(root, "douyin/作者_6/b.mp4", age_days=0, data=b"123") + _make(root, "douyin/作者_7/c.jpg", age_days=5, data=b"1") + + st = cleanup.temp_stats() + + assert st["files"] == 3 + assert st["bytes"] == 9 + assert st["dirs"] == 3 # douyin / 作者_6 / 作者_7（子目录各算一层） + + +def test_stats_missing_root(cleanup): + cleanup._tmp.rmdir() + assert cleanup.temp_stats() == {"files": 0, "bytes": 0, "dirs": 0} diff --git a/tests/test_video_group_file.py b/tests/test_video_group_file.py index 5834938..41d7103 100644 --- a/tests/test_video_group_file.py +++ b/tests/test_video_group_file.py @@ -11,6 +11,7 @@ import zipfile from pathlib import Path import pytest +from nonebot.adapters.onebot.v11.exception import NetworkError _MODULE_PATH = ( Path(__file__).resolve().parents[1] @@ -223,6 +224,75 @@ async def test_missing_file_short_circuits(gf, tmp_path, monkeypatch): assert called == [] +# ────────── 传输超时=结果未知：不降级直传（防两份群文件） ────────── +# 与 handlers/sender.py 同一物理事实：超时只是 bot 不再等待，协议端会把这次 +# 上传做完（大文件远超默认 30s）；此时再走本地直传就是重复投递。 + + +def test_send_timeout_default_and_env(gf, monkeypatch): + monkeypatch.delenv("video_analysis_send_timeout", raising=False) + monkeypatch.delenv("VIDEO_ANALYSIS_SEND_TIMEOUT", raising=False) + assert gf._send_timeout() == 300.0 + + monkeypatch.setenv("video_analysis_send_timeout", "77") + assert gf._send_timeout() == 77.0 + + monkeypatch.setenv("video_analysis_send_timeout", "bad") + assert gf._send_timeout() == 300.0 + + +async def test_upload_call_carries_per_call_timeout(gf, monkeypatch): + """upload_group_file API 带 `_timeout`（适配器 per-call 超时）。""" + monkeypatch.setenv("video_analysis_send_timeout", "77") + calls: list[dict] = [] + + class _Bot: + async def call_api(self, api, **kwargs): + calls.append({"api": api, **kwargs}) + + monkeypatch.setattr(gf, "get_bot", lambda: _Bot()) + await gf._call_upload("file:///D:/x.zip", 123, "x.zip") + assert calls == [ + { + "api": "upload_group_file", + "group_id": 123, + "file": "file:///D:/x.zip", + "name": "x.zip", + "_timeout": 77.0, + } + ] + + +async def test_s3_timeout_does_not_fall_back_to_local(gf, tmp_path, monkeypatch): + """S3 链接上传超时=结果未知（协议端可能还在传）→ 不再直传，避免两份。""" + seen: list[str] = [] + link = "http://192.168.2.15:5246/PLANA/x.zip?sig=1" + + async def fake_call(file_value, group_id, name, folder_id=None): + seen.append(file_value) + raise NetworkError("WebSocket call api upload_group_file timeout") + + monkeypatch.setattr(gf, "_call_upload", fake_call) + monkeypatch.setattr(gf, "_s3_url", lambda path, policy: link) + + file = _make_files(tmp_path / "src", ["a.zip"])[0] + assert await gf.upload_group_file(file, 123) is True + assert seen == [link] # 只有 S3 那一次，没有第二次 file:// 直传 + + +async def test_local_upload_timeout_reports_unknown(gf, tmp_path, monkeypatch): + """本地直传超时同样按「可能已送达」处理（不误报失败即可）。""" + + async def fake_call(*args, **kwargs): + raise NetworkError("timeout") + + monkeypatch.setattr(gf, "_call_upload", fake_call) + monkeypatch.setattr(gf, "_s3_url", lambda path, policy: "") + + file = _make_files(tmp_path / "src", ["a.zip"])[0] + assert await gf.upload_group_file(file, 123) is True + + # ───────────────────────── 投递编排 ───────────────────────── @@ -241,9 +311,7 @@ async def test_zip_mode_uploads_single_archive(gf, tmp_path, monkeypatch): archive = tmp_path / "pack.zip" archive.write_bytes(b"zip") calls = _record_uploads(gf, monkeypatch, archive) - monkeypatch.setattr( - gf, "build_archive", lambda *a, **kw: archive - ) + monkeypatch.setattr(gf, "build_archive", lambda *a, **kw: archive) files = _make_files(tmp_path / "src", ["a.txt", "b.txt"]) ok = await gf.upload_group_files(files, 123, title="t", password="pw") @@ -271,9 +339,7 @@ async def test_raw_mode_prefixes_work_name_for_multi_files(gf, tmp_path, monkeyp monkeypatch.setattr(gf, "upload_group_file", fake_upload) images = _make_files(tmp_path / "src", ["001.jpg", "002.jpg"]) - await gf.upload_group_files( - images, 123, zip_files=False, rel_dir="作者_9/海边日落" - ) + await gf.upload_group_files(images, 123, zip_files=False, rel_dir="作者_9/海边日落") assert seen == ["海边日落_001.jpg", "海边日落_002.jpg"] seen.clear() diff --git a/tests/test_video_media_naming.py b/tests/test_video_media_naming.py index d71f6cf..b7fac8e 100644 --- a/tests/test_video_media_naming.py +++ b/tests/test_video_media_naming.py @@ -6,6 +6,7 @@ get_temp_root 的返回值，测试里 monkeypatch 成临时目录。 """ import importlib.util +import re import sys from pathlib import Path @@ -127,11 +128,31 @@ def test_work_stem_strips_hashtags_and_spaces(u): def test_work_stem_empty_falls_back(u): + """没给来源时保持旧契约（固定 `作品`）""" assert u.build_work_stem("") == "作品" assert u.build_work_stem(None) == "作品" assert u.build_work_stem("🎬") == "作品" +def test_work_stem_empty_uses_source_code(u): + """给了来源 → 空标题兜底成 `作品_{来源短码}`：同作品稳定、不同作品不撞 + + 固定 `作品` 会让同一作者的多个无名作品挤在一个名字下，靠 4 位**时间**短码 + 区分，每次重解析名字都变（S3 key / 群文件成员名不可复现）。 + """ + url = "https://www.douyin.com/video/7412345678901234567" + stem = u.build_work_stem("#话题", source=url) + assert stem == f"作品_{u.short_source_code(url)}" + assert re.fullmatch(r"作品_[0-9a-z]{4}", stem) + assert stem == u.build_work_stem("", source=url) # 同来源稳定 + assert stem != u.build_work_stem("", source=url + "8") # 不同来源不同码 + + +def test_work_stem_source_ignored_when_title_present(u): + """有标题时来源短码不参与（标题本身已有区分度，别把名字改花）""" + assert u.build_work_stem("旅行 #随手拍", source="whatever") == "旅行" + + def test_work_stem_truncates(u): assert len(u.build_work_stem("标题" * 20)) <= 15 diff --git a/tests/test_video_policy.py b/tests/test_video_policy.py index 13adaad..f1e7d07 100644 --- a/tests/test_video_policy.py +++ b/tests/test_video_policy.py @@ -40,6 +40,8 @@ def test_normalize_platform_aliases(policy): assert policy.normalize_platform("小红书") == "小红书" assert policy.normalize_platform("b23") == "哔哩哔哩" assert policy.normalize_platform("B站") == "哔哩哔哩" + assert policy.normalize_platform("ks") == "快手" + assert policy.normalize_platform("快手") == "快手" assert policy.normalize_platform("yt") == "Youtube" assert policy.normalize_platform("twitter") == "X" assert policy.normalize_platform(" X ") == "X" @@ -81,6 +83,10 @@ def test_match_platform_domains(policy): assert policy.match_platform("https://twitter.com/a/status/1") == "X" assert policy.match_platform("https://b23.tv/abc") == "哔哩哔哩" assert policy.match_platform("https://v.douyin.com/abc") == "抖音" + assert policy.match_platform("https://v.kuaishou.com/nUqhAiDI") == "快手" + assert policy.match_platform("https://www.kuaishou.com/short-video/abc") == "快手" + # QQ 卡片 jumpUrl 走 v.m.chenzhongtech.com（快手分享页域） + assert policy.match_platform("https://v.m.chenzhongtech.com/fw/photo/abc") == "快手" assert policy.match_platform("https://youtu.be/abc") == "Youtube" assert policy.match_platform("https://example.com/a.mp4") is None diff --git a/tests/test_video_sender_link.py b/tests/test_video_sender_link.py index bbd21fc..5c6acd2 100644 --- a/tests/test_video_sender_link.py +++ b/tests/test_video_sender_link.py @@ -1,24 +1,27 @@ -"""媒体发送成功后「补传 S3 取下载链接」的单元测试 +"""sender.py 归档语义的单元测试（「媒体必须进本地 S3」是前提） -背景：2026-09-22 起 `hexi/core/outbound_media` 的全局钩子把本地路径改写成 -`/media/`，本地直发那一级才真正可用。此前它因协议端不在本机、`file://` -必然 ENOENT 而**恒失败**，所以每次都落到第 2 级 S3 —— `public_url` 因此总有值。 -本地直发通了以后，如果什么都不做，`public_url` 会恒为 None，`entry.py` 的 -`if pub and policy.sends_link` 就会让「发送下载链接」**静默失效**。 +用户 2026-09-25 定的前提：**无论发不发送消息**，下载成功的媒体都要进本地 S3 +（temp 只留 7 天，cleanup.py 删；桶里那份才是长期存档）。因此 sender 的结构是 +「归档与发送解耦且并行」： -`_local_sent` / `_public_link_for` 就是为此把「取链接」与「发送」解耦： -只有策略真要链接（`sends_link`）时才补一次 S3，其余情况一概不碰 S3。 +- `archive_media` 无条件逐文件上传：单个文件失败只 warning 并继续，任何情况都不抛； +- `send_pending_media` 一进来就 `create_task` 归档，与本地直发并行开跑， + 发送全失败也一定 await 它跑完；`public_url` 从归档结果取（最后一个非空）； +- 第 2 级（S3 链接发送）复用归档的预签名链接 —— 每个文件只上传一次，没有"补传"。 sender.py 有插件内相对导入（`..config` / `..policy` / `..services...`）， 无法裸加载，故在 sys.modules 里预置这些依赖的替身再按路径加载目标模块。 """ +import asyncio import importlib.util import sys +import time import types from pathlib import Path import pytest +from nonebot.adapters.onebot.v11.exception import ActionFailed, NetworkError _REPO = Path(__file__).resolve().parents[1] _PKG = "hexi.plugins.nonebot_plugin_video_analysis" @@ -71,23 +74,31 @@ def sender(): class _Policy: - """只带 sends_link 的策略替身（真实 Policy 需要一堆字段/归一化）。""" - - def __init__(self, sends_link: bool): - self.sends_link = sends_link + """群策略替身（归档只是把它透传给 upload_with_plan，不需要真实字段）。""" @pytest.fixture def uploads(monkeypatch, sender): - """记录 upload_with_plan 的调用；返回 (调用列表, 设置返回值的钩子)。""" - calls: list[Path] = [] - result: dict = {"pub": "https://pub.example/x.mp4", "raise": None} + """记录 upload_with_plan 的调用；返回 (调用列表, 结果配置)。 + + result 可覆盖的字段： + - `pub`：常量，或按调用顺序消费的列表（元素为 None = 没传公网）； + - `raises`：文件名 → 该文件上传时抛出的异常（其余文件照常上传）。 + """ + calls: list[tuple[Path, object]] = [] + result: dict = {"pub": "https://pub.example/x.mp4", "raises": {}} def fake_upload(file_path, *, policy=None): - calls.append(Path(file_path)) - if result["raise"]: - raise result["raise"] - return "local-url", result["pub"] + fp = Path(file_path) + calls.append((fp, policy)) + pub = result["pub"] + if isinstance(pub, list): + # 列表按下标对应「第几个文件」（先取值再判失败：元素与文件一一对应） + pub = pub.pop(0) if pub else None + error = result["raises"].get(fp.name) + if error: + raise error + return "local-url", pub monkeypatch.setattr(sender, "upload_with_plan", fake_upload) return calls, result @@ -95,61 +106,315 @@ def uploads(monkeypatch, sender): @pytest.fixture def media(sender): - def build(sends_link: bool, n: int = 1, policy=None): + def build(n: int = 1, policy=_Policy()): files = [Path(f"D:/temp/f{i}.mp4") for i in range(n)] - return sender.PendingMedia( - files=files, - policy=_Policy(sends_link) if policy is None else policy, - ) + return sender.PendingMedia(files=files, policy=policy) return build -async def test_no_s3_when_link_disabled(sender, media, uploads): - """默认路径：本地发成功就够了，绝不碰 S3（媒体不走 S3 归档是有意的）。""" - calls, _ = uploads - ok, pub = await sender._local_sent(media(sends_link=False)) - assert (ok, pub) == (True, None) - assert calls == [] +# ───────────────────────── 归档本体（无条件） ───────────────────────── -async def test_s3_fetched_when_link_enabled(sender, media, uploads): - """策略要下载链接时：补传 S3 并把 public_url 交出去（本次修的回归）。""" +async def test_archive_uploads_every_file(sender, media, uploads): + """逐文件归档，拿到 (预签名链接, 公网链接)。""" calls, _ = uploads - ok, pub = await sender._local_sent(media(sends_link=True)) + result = await sender.archive_media(media(n=3)) + assert len(calls) == 3 + assert [(r.path.name, r.local_url, r.public_url) for r in result] == [ + (f"f{i}.mp4", "local-url", "https://pub.example/x.mp4") for i in range(3) + ] + + +async def test_archive_continues_after_single_failure(sender, media, uploads): + """单个文件失败只跳过它，后面的照常传（多图作品不早退）。""" + calls, result = uploads + result["raises"] = {"f1.mp4": RuntimeError("这个文件 S3 挂了")} + + archived = await sender.archive_media(media(n=3)) + assert [r.local_url for r in archived] == ["local-url", None, "local-url"] + assert len(calls) == 3 # 失败的那个也试过了，f2 没被跳过 + + +async def test_archive_never_raises(sender, media, uploads): + """全部上传失败也不抛（归档失败不能反过来影响发送）。""" + calls, result = uploads + error = RuntimeError("S3 挂了") + result["raises"] = {"f0.mp4": error, "f1.mp4": error} + archived = await sender.archive_media(media(n=2)) + assert [r.local_url for r in archived] == [None, None] + assert len(calls) == 2 + + +async def test_archive_passes_policy_through(sender, media, uploads): + """policy=None（如 Web 面板路径）也照常归档：upload_with_plan 自己兜 Policy()。""" + calls, _ = uploads + await sender.archive_media(media(policy=None)) + assert [policy for _, policy in calls] == [None] + + +async def test_last_public_url_takes_last_non_empty(sender, media, uploads): + """公网链接取最后一个非空值；只有本地链接的文件不影响结果。""" + calls, result = uploads + result["pub"] = [None, "https://pub.example/b.mp4", None] + archived = await sender.archive_media(media(n=3)) + assert sender._last_public_url(archived) == "https://pub.example/b.mp4" + + +async def test_last_public_url_none_when_upload_public_off(sender, media, uploads): + """S3 有本地链接但没公网（upload_public 关）→ None，且算不上失败。""" + _, result = uploads + result["pub"] = None + archived = await sender.archive_media(media()) + assert archived[0].local_url == "local-url" + assert sender._last_public_url(archived) is None + + +# ─────────────────── 发送入口：归档无条件且与之并行 ─────────────────── + + +async def test_archives_even_when_send_fails(monkeypatch, sender, media, uploads): + """前提的核心：发送全失败也照样归档，public_url 照常交出去。""" + calls, _ = uploads + + async def fake_core(m, event, archive): + return False # 三级全失败 + + monkeypatch.setattr(sender, "_send_media_core", fake_core) + ok, pub = await sender.send_pending_media(media(n=2)) + assert (ok, pub) == (False, "https://pub.example/x.mp4") + assert len(calls) == 2 + + +async def test_archives_when_send_succeeds(monkeypatch, sender, media, uploads): + """发送成功也归档（归档不是发送的副产品）。""" + calls, _ = uploads + + async def fake_core(m, event, archive): + return True + + monkeypatch.setattr(sender, "_send_media_core", fake_core) + ok, pub = await sender.send_pending_media(media()) assert (ok, pub) == (True, "https://pub.example/x.mp4") assert len(calls) == 1 -async def test_multi_file_uploads_each(sender, media, uploads): - calls, _ = uploads - ok, pub = await sender._local_sent(media(sends_link=True, n=3)) +async def test_archive_runs_in_parallel_with_send(monkeypatch, sender, media, uploads): + """归档与发送并行开跑，不是"发送跑完才传"（本地直发不等它）。""" + started: list[str] = [] + + async def fake_archive(m): + started.append("archive") + return [sender.UploadedFile(path=fp) for fp in m.files] + + async def fake_core(m, event, archive): + await asyncio.sleep(0) # 让出一个事件循环回合 + assert started, "归档应当已在跑，而不是等发送结束才开始" + return True + + monkeypatch.setattr(sender, "archive_media", fake_archive) + monkeypatch.setattr(sender, "_send_media_core", fake_core) + ok, _ = await sender.send_pending_media(media()) assert ok is True - assert pub == "https://pub.example/x.mp4" - assert len(calls) == 3 -async def test_no_public_url_keeps_none(sender, media, uploads): - """S3 有本地链接但没公网（upload_public 关）→ 不带链接，但也不算失败。""" - calls, result = uploads - result["pub"] = None - ok, pub = await sender._local_sent(media(sends_link=True)) - assert (ok, pub) == (True, None) - assert len(calls) == 1 - - -async def test_upload_failure_never_breaks_send(sender, media, uploads): - """补传失败只记日志：发送本身已经成功了，不能反过来判失败。""" - calls, result = uploads - result["raise"] = RuntimeError("S3 挂了") - ok, pub = await sender._local_sent(media(sends_link=True)) - assert (ok, pub) == (True, None) - assert len(calls) == 1 - - -async def test_policy_none_is_safe(sender, media, uploads): - """无策略（如 Web 面板路径）时不补传也不炸。""" +async def test_send_failure_still_waits_for_archive( + monkeypatch, sender, media, uploads +): + """发送层抛异常也不能把归档半途丢下（finally 里 await 它跑完）。""" calls, _ = uploads - ok, pub = await sender._local_sent(media(sends_link=False, policy=None)) - assert (ok, pub) == (True, None) - assert calls == [] + finished: list[str] = [] + + async def fake_core(m, event, archive): + raise RuntimeError("发送层炸了") + + def slow_upload(file_path, *, policy=None): # to_thread 里跑的是同步函数 + calls.append((Path(file_path), policy)) + time.sleep(0.01) + finished.append(Path(file_path).name) + return "local-url", "https://pub.example/x.mp4" + + monkeypatch.setattr(sender, "upload_with_plan", slow_upload) + monkeypatch.setattr(sender, "_send_media_core", fake_core) + + with pytest.raises(RuntimeError): + await sender.send_pending_media(media(n=2)) + assert finished == ["f0.mp4", "f1.mp4"] # 归档完整跑完才抛出去 + + +# ─────────────────── 第 2 级复用归档结果（不重复上传） ─────────────────── + + +async def test_s3_level_reuses_archived_urls(monkeypatch, sender, media, uploads): + """本地直发失败 → 用归档的预签名链接发送，每个文件只上传一次。""" + calls, _ = uploads + built: list = [] + + def fake_local_msg(files, image_post): + class _Msg: + async def send(self, **_kw): # 本地直发失败 + raise RuntimeError("协议端拿不到文件") + + return _Msg() + + def fake_s3_msg(items): + built.append(items) + + class _Msg: + async def send(self, **_kw): # 链接版发送成功 + pass + + return _Msg() + + monkeypatch.setattr(sender, "_build_local_msg", fake_local_msg) + monkeypatch.setattr(sender, "_build_s3_msg", fake_s3_msg) + + ok, pub = await sender.send_pending_media(media(n=2)) + assert (ok, pub) == (True, "https://pub.example/x.mp4") + assert len(calls) == 2 # 只有归档那次上传，第 2 级没有重复传 + assert built == [ + [(Path("D:/temp/f0.mp4"), "local-url"), (Path("D:/temp/f1.mp4"), "local-url")] + ] + + +async def test_no_files_is_noop(sender): + """没有媒体文件时既不归档也不发送。""" + ok, pub = await sender.send_pending_media(sender.PendingMedia(files=[])) + assert (ok, pub) == (False, None) + + +# ─────────── 超时=结果未知：不降级重试（长视频防重复发送） ─────────── +# 2026-10-13 实测：172MB 视频三级降级各超时一次（NoneBot 默认 api_timeout=30s， +# 而协议端下载+转存要 ~2 分钟），但协议端在 bot 超时后照样把三份全发进群。 +# 超时只是 bot 放弃等待 ⇒ 结果未知 ⇒ 重试必重复。 + + +def test_send_timeout_default_and_env(sender, monkeypatch): + """默认 300s；`video_analysis_send_timeout` 可调，非法/非正值回退默认。""" + monkeypatch.delenv("video_analysis_send_timeout", raising=False) + monkeypatch.delenv("VIDEO_ANALYSIS_SEND_TIMEOUT", raising=False) + assert sender.send_timeout() == 300.0 + + monkeypatch.setenv("video_analysis_send_timeout", "120") + assert sender.send_timeout() == 120.0 + + monkeypatch.setenv("video_analysis_send_timeout", "abc") + assert sender.send_timeout() == 300.0 + + monkeypatch.setenv("video_analysis_send_timeout", "0") + assert sender.send_timeout() == 300.0 + + +def test_unknown_outcome_classification(sender): + """超时/网络错误=结果未知；ActionFailed（协议端明确报错）=确定失败。""" + assert sender.unknown_send_outcome(NetworkError("timeout")) is True + assert sender.unknown_send_outcome(asyncio.TimeoutError()) is True + assert sender.unknown_send_outcome(ActionFailed(error="fetch failed")) is False + assert sender.unknown_send_outcome(RuntimeError("本地错误")) is False + + +async def test_media_send_carries_per_call_timeout(monkeypatch, sender, media, uploads): + """媒体发送带 `_timeout`（适配器 per-call 超时），长视频才等得到真结果。""" + monkeypatch.setenv("video_analysis_send_timeout", "42") + seen: list[dict] = [] + + def fake_local_msg(files, image_post): + class _Msg: + async def send(self, **kwargs): + seen.append(kwargs) + + return _Msg() + + monkeypatch.setattr(sender, "_build_local_msg", fake_local_msg) + ok, _ = await sender.send_pending_media(media()) + assert ok is True + assert seen == [{"_timeout": 42.0}] + + +async def test_local_timeout_does_not_resend(monkeypatch, sender, media, uploads): + """本地直发超时（结果未知）→ 不走 S3 降级、按「可能已送达」返回。""" + s3_built: list = [] + + def fake_local_msg(files, image_post): + class _Msg: + async def send(self, **_kw): + raise NetworkError("WebSocket call api send_msg timeout") + + return _Msg() + + def fake_s3_msg(items): + s3_built.append(items) + + class _Msg: + async def send(self, **_kw): + pass + + return _Msg() + + monkeypatch.setattr(sender, "_build_local_msg", fake_local_msg) + monkeypatch.setattr(sender, "_build_s3_msg", fake_s3_msg) + + ok, pub = await sender.send_pending_media(media()) + assert ok is True # 结果未知按可能已送达：不触发「媒体发送失败」提示 + assert pub == "https://pub.example/x.mp4" # 归档照常完成 + assert s3_built == [] # 关键：没有第二次发送 + + +async def test_action_failed_still_cascades_to_s3(monkeypatch, sender, media, uploads): + """协议端明确报错（如 fetch failed）= 确定没发出去，照旧降级到 S3 链接。""" + sent: list = [] + + def fake_local_msg(files, image_post): + class _Msg: + async def send(self, **_kw): + raise ActionFailed(error="fetch failed") + + return _Msg() + + def fake_s3_msg(items): + class _Msg: + async def send(self, **_kw): # 链接版成功 + sent.append(items) + + return _Msg() + + monkeypatch.setattr(sender, "_build_local_msg", fake_local_msg) + monkeypatch.setattr(sender, "_build_s3_msg", fake_s3_msg) + + ok, _ = await sender.send_pending_media(media()) + assert ok is True + assert len(sent) == 1 # 恰好降级一次 + + +async def test_forward_timeout_does_not_fall_through( + monkeypatch, sender, media, uploads +): + """合并转发超时（结果未知）→ 不再落回普通消息链路重发。""" + two = media(n=2) + fallback_built: list = [] + + async def fake_forward(items, sender_name, *, group_id=None, user_id=None): + return None # 结果未知 + + def fake_local_msg(files, image_post): + fallback_built.append(files) + + class _Msg: + async def send(self, **_kw): + raise AssertionError("不该走到普通消息链路") + + return _Msg() + + monkeypatch.setattr(sender, "_send_forward_media", fake_forward) + monkeypatch.setattr(sender, "_forward_sender_name", lambda: "Bot") + monkeypatch.setattr(sender, "_build_local_msg", fake_local_msg) + + archive = asyncio.create_task(sender.archive_media(two)) + try: + ok = await sender._send_media_core( + two, types.SimpleNamespace(group_id=123), archive + ) + finally: + await archive + assert ok is True + assert fallback_built == []