feat(video-analysis): 抖音免浏览器解析链路 + 快手支持 + 小红书加固 + 发送防重复
- 抖音（2026-10-13）：删除 SSR + playwright 旧链路（douyin_ssr.py 移除， douyin_api.py 精简为下载工具），新增 douyin_web.py（x-tt-argus 网关头 + web detail 接口 + 最高码率梯/多 CDN 回退下载）与 douyin_sign.py （纯 Python a_bogus 签名，vendor 自 f2（Apache-2.0），依赖 gmssl SM3）； 实测 10/10 成功率、单作品 <1s（旧浏览器链路 5-15s 且随机撞验证码） - 快手（2026-10-08）：kuaishou_content.py 分享页 INIT_STATE 静态解析（无登录态）， 清晰度梯最佳档 + 图集 URL 组装，入口/策略/Web 面板同步接入 - 小红书：按链接来源定国际/国内站优先序、视频 backupUrls 逐候选回退、 图片 urlDefault→infoList WB_DFT→url 三档回退 - 长视频防重复发送（2026-10-13）：媒体发送带 per-call _timeout（默认 300s）， 超时=结果未知不再降级重试；auto/active 两条 handler 互斥 - Web 解析封面抽帧 ffmpeg 加 CREATE_NO_WINDOW（pythonw 下不再弹 cmd 黑窗） - 依赖：+gmssl；测试：新增 test_douyin_web / test_kuaishou_parse / test_video_cleanup， 扩充 test_video_policy / test_video_sender_link / test_video_group_file / test_video_media_naming Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
sansenhoshiandClaude Code committed
1 parent
a31aab00e8
commit
a789a44f5d
25 files changed
+3508
-867
No files matched your search
@@ -17,6 +17,7 @@ hexi/data/temp 是下载媒体中转区(见 utils.get_temp_root),发送成功后
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from collections.abc import Callable
|
||||
from pathlib import Path
|
||||
from typing import Literal
|
||||
|
||||
@@ -25,9 +26,11 @@ from nonebot.adapters.onebot.v11 import Bot, MessageEvent
|
||||
from pydantic import BaseModel
|
||||
|
||||
from hexi.core.custom_utils import check_manage
|
||||
from hexi.core.task_slots import TASKS, Task
|
||||
|
||||
require("nonebot_plugin_apscheduler")
|
||||
from nonebot_plugin_apscheduler import scheduler
|
||||
from hexi.core.scheduler import add_job
|
||||
|
||||
from nonebot_plugin_alconna import UniMessage
|
||||
|
||||
@@ -101,30 +104,59 @@ def _prune_empty_dirs(root: Path, max_age: float) -> int:
|
||||
return removed
|
||||
|
||||
|
||||
def clean_temp_files(sub: str = "", days: int | None = None) -> tuple[int, int]:
|
||||
def clean_temp_files(
|
||||
sub: str = "",
|
||||
days: int | None = None,
|
||||
on_progress: Callable[..., None] | None = None,
|
||||
) -> tuple[int, int]:
|
||||
"""清理 temp[/sub] 下超过期限的文件与随之空掉的目录。
|
||||
|
||||
Args:
|
||||
on_progress: 进度回调 `(done, total, phase, message, warning)`（关键字可省）——
|
||||
扫描阶段 total 未知，之后按"处理过的文件数"上报。跑在哪个线程调用都行
|
||||
（任务槽那边会切回事件循环）。**返回 False 表示调用方请求取消**，
|
||||
本函数会在处理完当前文件后尽早收尾（清理天然可中断，不硬杀线程）。
|
||||
|
||||
Returns:
|
||||
(removed, total) — 删除的文件数、统计到的文件总数
|
||||
(removed, total) — 删除的文件数、统计到的文件数（被取消时是"取消前"的进度）
|
||||
"""
|
||||
|
||||
def report(**kwargs) -> bool:
|
||||
if on_progress is None:
|
||||
return True # 没有槽（纯函数调用）时不存在取消
|
||||
return bool(on_progress(**kwargs))
|
||||
|
||||
root = get_temp_root(sub)
|
||||
if not root.is_dir():
|
||||
return 0, 0
|
||||
|
||||
max_age = _max_age_seconds(days)
|
||||
report(phase="扫描中", unit="个文件", message="正在统计文件…")
|
||||
files = _walk_files(root)
|
||||
report(
|
||||
done=0, total=len(files), phase="清理中",
|
||||
unit="个文件", message=f"共 {len(files)} 个文件",
|
||||
)
|
||||
|
||||
removed = 0
|
||||
for path in files:
|
||||
if not _file_is_stale(path, max_age):
|
||||
continue
|
||||
canceled = False
|
||||
for index, path in enumerate(files, 1):
|
||||
if _file_is_stale(path, max_age):
|
||||
try:
|
||||
path.unlink(missing_ok=True)
|
||||
removed += 1
|
||||
logger.info(f"temp 清理: 删除 {path}")
|
||||
except OSError as e:
|
||||
# 文件被占用(如发送中)等场景,留待下轮
|
||||
# 文件被占用(如发送中)等场景,留待下轮 —— 也算"部分失败",报给任务槽
|
||||
logger.warning(f"temp 清理: 跳过 {path} ({e})")
|
||||
report(warning=f"跳过 {path.name}（被占用？）：{e}")
|
||||
if not report(done=index, total=len(files), message=f"已删除 {removed} 个"):
|
||||
logger.info("temp 清理: 收到取消请求，提前结束")
|
||||
canceled = True
|
||||
break
|
||||
|
||||
if not canceled:
|
||||
report(phase="回收空目录", done=len(files), total=len(files))
|
||||
dirs = _prune_empty_dirs(root, max_age)
|
||||
if dirs:
|
||||
logger.info(f"temp 清理: 同时收掉 {dirs} 个空目录")
|
||||
@@ -165,7 +197,27 @@ async def _handle_clean(bot: Bot, event: MessageEvent):
|
||||
except ValueError:
|
||||
await UniMessage.text("天数参数不合法,示例:/清理temp 3").send()
|
||||
return
|
||||
removed, total = await _run_clean(days)
|
||||
task, reused = submit_clean(
|
||||
days,
|
||||
title="清理 temp（命令）",
|
||||
trigger="command",
|
||||
actor=event.get_user_id(),
|
||||
group_id=getattr(event, "group_id", "") or "",
|
||||
)
|
||||
if reused:
|
||||
await UniMessage.text(
|
||||
"已经有一个清理任务在跑了,等它跑完再来吧~（进度可在网页端看）"
|
||||
).send()
|
||||
return
|
||||
await TASKS.wait(task) # 命令要"跑完再回复",与网页端的即提交即返回不同
|
||||
|
||||
if task.status == "canceled":
|
||||
await UniMessage.text("清理被取消了（网页端点过取消）。").send()
|
||||
return
|
||||
if task.status != "done":
|
||||
await UniMessage.text(f"清理失败了:{task.error}").send()
|
||||
return
|
||||
removed, total = task.result or (0, 0)
|
||||
if total == 0:
|
||||
await UniMessage.text("temp 目录目前是空的,没什么可清理的~").send()
|
||||
else:
|
||||
@@ -174,7 +226,7 @@ async def _handle_clean(bot: Bot, event: MessageEvent):
|
||||
|
||||
@stats_cmd.handle()
|
||||
async def _handle_stats(event: MessageEvent):
|
||||
st = await _run_stats()
|
||||
st = await run_stats()
|
||||
if st["files"] == 0:
|
||||
if st.get("dirs"):
|
||||
await UniMessage.text(
|
||||
@@ -191,13 +243,37 @@ async def _handle_stats(event: MessageEvent):
|
||||
).send()
|
||||
|
||||
|
||||
async def _run_clean(days: int | None = None) -> tuple[int, int]:
|
||||
"""执行清理并记录日志(供命令与定时任务共用)"""
|
||||
removed, total = await asyncio.to_thread(clean_temp_files, "", days)
|
||||
return removed, total
|
||||
#: 任务槽名:命令 / Web / 每日定时三条路共用同一个槽,同槽只跑一个(防重复触发)
|
||||
CLEANUP_SLOT = "video_analysis:temp_cleanup"
|
||||
|
||||
|
||||
async def _run_stats() -> dict:
|
||||
def submit_clean(
|
||||
days: int | None,
|
||||
*,
|
||||
title: str,
|
||||
trigger: str = "",
|
||||
actor: str = "",
|
||||
group_id: str = "",
|
||||
) -> tuple[Task, bool]:
|
||||
"""把清理提交到任务槽,立即返回 (任务, 是否复用了正在跑的任务)。
|
||||
|
||||
任务体跑在线程池里(见 hexi.core.task_slots),进度实时推给网页端;
|
||||
同槽已有任务时不会重复清理 —— 服务端这一层才是"防连点"的真正保证。
|
||||
trigger / actor / group_id 是审计信息(谁点的:web 用户名 / command 的 QQ 号)。
|
||||
"""
|
||||
return TASKS.submit(
|
||||
slot=CLEANUP_SLOT,
|
||||
title=title,
|
||||
plugin_id="video_analysis",
|
||||
trigger=trigger,
|
||||
actor=actor,
|
||||
group_id=group_id,
|
||||
func=lambda report: clean_temp_files(days=days, on_progress=report),
|
||||
)
|
||||
|
||||
|
||||
async def run_stats() -> dict:
|
||||
"""异步统计(命令 / Web 缓存面板共用)"""
|
||||
return await asyncio.to_thread(temp_stats)
|
||||
|
||||
|
||||
@@ -213,7 +289,7 @@ def _register_auto_job() -> None:
|
||||
from apscheduler.jobstores.base import JobLookupError # noqa: PLC0415
|
||||
|
||||
if cleanup_config.video_analysis_temp_cleanup_mode == "auto":
|
||||
scheduler.add_job(
|
||||
add_job(
|
||||
_scheduled_cleanup,
|
||||
"cron",
|
||||
hour=cleanup_config.video_analysis_temp_cleanup_hour,
|
||||
@@ -239,13 +315,18 @@ def reload_cleanup_config() -> None:
|
||||
|
||||
|
||||
async def _scheduled_cleanup() -> None:
|
||||
"""每日定时清理 temp 目录(auto 模式)"""
|
||||
"""每日定时清理 temp 目录(auto 模式);同样走任务槽(与手动清理互斥)"""
|
||||
logger.info("temp 自动清理: 开始")
|
||||
try:
|
||||
removed, total = await asyncio.to_thread(clean_temp_files)
|
||||
task, reused = submit_clean(None, title="清理 temp（每日定时）", trigger="schedule")
|
||||
if reused:
|
||||
logger.info("temp 自动清理: 已有清理任务在跑，跳过本次")
|
||||
return
|
||||
await TASKS.wait(task)
|
||||
if task.status != "done":
|
||||
logger.error(f"temp 自动清理未完成（{task.status}）: {task.error}")
|
||||
return
|
||||
removed, total = task.result or (0, 0)
|
||||
logger.info(f"temp 自动清理完成:删除 {removed} 个,当前共 {total} 个文件")
|
||||
except Exception as e:
|
||||
logger.exception(f"temp 自动清理失败: {e}")
|
||||
|
||||
|
||||
# 插件加载时按模式注册(auto)或提示(manual)
|
||||
|
||||
@@ -1,109 +1,32 @@
|
||||
"""抖音视频/图文解析编排层"""
|
||||
"""抖音视频/图文解析编排层
|
||||
|
||||
链路（2026-10 起，免浏览器）：短链/全链接 → `douyin_web.fetch_douyin_web`
|
||||
（短链重定向、签名请求 web detail 接口、最高码率/最高画质下载都在该模块内）。
|
||||
旧链路（iesdouyin SSR 静态解析 + playwright 浏览器拦截）已废弃，
|
||||
详见 services/fetchers/douyin_web.py 顶部说明。
|
||||
"""
|
||||
|
||||
import re
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
import httpx
|
||||
from nonebot import logger
|
||||
|
||||
from ..services.fetchers.douyin_api import fetch_douyin_content
|
||||
from ..services.fetchers.douyin_ssr import MOBILE_UA, fetch_douyin_note_ssr
|
||||
from ..models import DouyinFetchError
|
||||
from ..services.fetchers.douyin_web import fetch_douyin_web
|
||||
from ..utils import get_data_dir, parse_netscape_cookies
|
||||
from ..policy import Policy
|
||||
from .sender import PendingMedia, _as_paths
|
||||
|
||||
SHORT_LINK_PATTERN = re.compile(r"(v\.douyin\.com/[A-Za-z0-9_\-]+)")
|
||||
|
||||
|
||||
async def _resolve_short_link(url: str) -> Optional[str]:
|
||||
"""短链重定向：httpx 取 Location（最多 3 跳），失败返回 None"""
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
headers={"User-Agent": MOBILE_UA},
|
||||
follow_redirects=False,
|
||||
timeout=10,
|
||||
) as client:
|
||||
current = url
|
||||
for _ in range(3):
|
||||
resp = await client.get(current)
|
||||
if resp.status_code >= 400:
|
||||
return None
|
||||
location = resp.headers.get("Location")
|
||||
if not location:
|
||||
final = str(resp.url)
|
||||
# 200 但仍是短链本身（如 JS 挑战壳页）→ 静默失败，显式记录；
|
||||
# 图文短链若此处失败将退化到 playwright 旧链路
|
||||
if "v.douyin.com" in final:
|
||||
logger.warning(f"短链返回挑战页（未跳转）: {final}")
|
||||
return final
|
||||
# Location 可能是相对路径，需拼上当前 URL
|
||||
current = urllib.parse.urljoin(current, location)
|
||||
return current
|
||||
except Exception:
|
||||
logger.warning(f"短链重定向失败: {url}")
|
||||
return None
|
||||
|
||||
|
||||
async def parse_douyin(
|
||||
url: str,
|
||||
) -> tuple[Optional[str], Optional[Union[Path, list[Path]]], bool]:
|
||||
"""解析抖音链接（短链 / 全链接），返回 (title, file_paths, is_image_post)
|
||||
|
||||
分派规则（不能依赖 URL 路径：短链重定向后图文也统一变成
|
||||
iesdouyin.com/share/video/{id} 形态）：
|
||||
- note 形态链接 → SSR 静态解析，失败不回退 playwright
|
||||
（其图文链路存在 aweme/post 取到作者其他作品的缺陷）
|
||||
- 其余形态 → 先试 SSR 按内容判定：有 images 即图文 → SSR 秒级下载；
|
||||
真视频 / SSR 失败 → playwright 拦截（保留全部清晰度能力）
|
||||
"""
|
||||
# 优先匹配短链接 v.douyin.com/xxx
|
||||
short = SHORT_LINK_PATTERN.search(url)
|
||||
if short:
|
||||
target_url = "https://" + short.group(1)
|
||||
# 短链先重定向拿到全链接，确定图文/视频类型后分派
|
||||
resolved = await _resolve_short_link(target_url)
|
||||
if resolved:
|
||||
logger.info(f"短链重定向: {target_url} -> {resolved}")
|
||||
target_url = resolved
|
||||
else:
|
||||
# 全链接: www.douyin.com/note/xxx、www.douyin.com/video/xxx，
|
||||
# 以及抖音复制出的 iesdouyin.com/share/video|note/xxx（弱网/无网下 App
|
||||
# 生成的恢复连接同样会是这个形态）。统一提取出作品 id 后再交给 SSR / playwright。
|
||||
m = re.search(
|
||||
r"(https?://(?:www\.)?(?:m\.)?(?:ies)?douyin\.com/(?:share/)?(note|video)/(\d+))",
|
||||
url,
|
||||
)
|
||||
if not m:
|
||||
return None, None, False
|
||||
# 统一规范化为 www.douyin.com/{note|video}/{id}，避免 iesdouyin 分享页
|
||||
# 在无登录态/弱网下只返回壳页，直接落到浏览器可正常打开的作品页。
|
||||
target_url = f"https://www.douyin.com/{m.group(2)}/{m.group(3)}"
|
||||
|
||||
"""解析抖音链接（短链 / 全链接），返回 (title, file_paths, is_image_post)"""
|
||||
try:
|
||||
is_img_post = False
|
||||
cookies_path = get_data_dir() / "cookies.txt"
|
||||
cookies = parse_netscape_cookies(cookies_path)
|
||||
# 先试 SSR（秒级、免浏览器），失败一律回退 playwright 链路。
|
||||
# 2026-08-13 起抖音对 iesdouyin SSR 端点整体降级（登录态也拿不到
|
||||
# videoInfoRes，只返回 33KB 壳页），note/video 形态统一回退，
|
||||
# 图文作品的"取到作者其他作品"缺陷由 douyin_api 按 aweme_id 精确匹配修复。
|
||||
try:
|
||||
title, file_path = await fetch_douyin_note_ssr(target_url, cookies)
|
||||
except DouyinFetchError:
|
||||
logger.info("SSR 解析失败，回退 playwright 链路")
|
||||
title, file_path = None, None
|
||||
if file_path is None:
|
||||
title, file_path = await fetch_douyin_content(
|
||||
target_url, cookies, 10, False
|
||||
)
|
||||
if isinstance(file_path, list):
|
||||
is_img_post = True
|
||||
return title, file_path, is_img_post
|
||||
return await fetch_douyin_web(url, cookies)
|
||||
except Exception:
|
||||
logger.exception(f"获取直链失败 {target_url}")
|
||||
logger.exception(f"获取直链失败 {url}")
|
||||
return None, None, False
|
||||
|
||||
|
||||
|
||||
@@ -11,7 +11,11 @@ from nonebot.rule import to_me
|
||||
from nonebot_plugin_alconna import UniMessage
|
||||
from nonebot_plugin_alconna.uniseg import get_target
|
||||
|
||||
from ..services.fetchers.bilibili_content import fetch_bilibili_content, resolve_short_link
|
||||
from ..services.fetchers.bilibili_content import (
|
||||
fetch_bilibili_content,
|
||||
resolve_short_link,
|
||||
)
|
||||
from ..services.fetchers.kuaishou_content import fetch_kuaishou_content
|
||||
from ..services.fetchers.rednote_content import fetch_rednote_content
|
||||
from .douyin import parse_douyin, process_douyin_res
|
||||
from .sender import PendingMedia, send_pending_media
|
||||
@@ -32,6 +36,8 @@ VALID_HOSTS = [
|
||||
"iesdouyin.com",
|
||||
"m.douyin.com",
|
||||
"jingxuan.douyin.com",
|
||||
"kuaishou.com",
|
||||
"chenzhongtech.com",
|
||||
"x.com",
|
||||
"twitter.com",
|
||||
"xiaohongshu.com",
|
||||
@@ -39,7 +45,16 @@ VALID_HOSTS = [
|
||||
"xhslink.cn",
|
||||
]
|
||||
|
||||
auto_video_handler = on_message(priority=10, block=False)
|
||||
async def _not_to_me(event: Event) -> bool:
|
||||
"""不是 @bot / 昵称开头的消息（`to_me()` 的反面；Rule 没有 `__invert__`）。"""
|
||||
return not event.is_tome()
|
||||
|
||||
|
||||
# auto 与 active 是互斥的两条路：@bot（或昵称开头）的消息归 active 管，其余归
|
||||
# auto。两个 matcher 都是 block=False（不拦其它插件），若 auto 不排除 to_me，
|
||||
# 一条 @bot 的链接消息会同时命中两个 handler → 解析发送两遍（长视频要等几分钟，
|
||||
# 重复非常显眼；2026-10-13 排查发送重复时发现）。
|
||||
auto_video_handler = on_message(priority=10, block=False, rule=_not_to_me)
|
||||
active_video_handler = on_message(priority=10, block=False, rule=to_me())
|
||||
|
||||
|
||||
@@ -203,14 +218,41 @@ async def dispatch_url(
|
||||
logger.warning(f"媒体解析：未能获取到作品：{url}")
|
||||
return None, None
|
||||
return await process_douyin_res(
|
||||
title, parsed_path, is_private, image_post,
|
||||
policy=policy, platform=platform,
|
||||
title,
|
||||
parsed_path,
|
||||
is_private,
|
||||
image_post,
|
||||
policy=policy,
|
||||
platform=platform,
|
||||
)
|
||||
except Exception:
|
||||
await UniMessage.text(f"无法解析到媒体：{url}").send()
|
||||
logger.exception(f"媒体解析：无法解析到媒体：{url}")
|
||||
return None, None
|
||||
|
||||
if "kuaishou.com" in url or "chenzhongtech.com" in url:
|
||||
await UniMessage.text("检测到链接，正在处理，请稍候...").send()
|
||||
try:
|
||||
title, parsed_path = await fetch_kuaishou_content(url)
|
||||
if not title and parsed_path is None:
|
||||
await UniMessage.text(f"无法解析到媒体：{url}").send()
|
||||
return None, None
|
||||
if parsed_path == []:
|
||||
await UniMessage.text(title).send()
|
||||
return None, None
|
||||
return await process_douyin_res(
|
||||
title,
|
||||
parsed_path,
|
||||
is_private,
|
||||
isinstance(parsed_path, list),
|
||||
policy=policy,
|
||||
platform=platform,
|
||||
)
|
||||
except Exception:
|
||||
await UniMessage.text(f"无法解析到媒体：{url}").send()
|
||||
logger.exception(f"快手解析：无法解析：{url}")
|
||||
return None, None
|
||||
|
||||
if any(
|
||||
kw in url
|
||||
for kw in (
|
||||
@@ -230,8 +272,12 @@ async def dispatch_url(
|
||||
await UniMessage.text(title).send()
|
||||
return None, None
|
||||
return await process_douyin_res(
|
||||
title, parsed_path, is_private,
|
||||
isinstance(parsed_path, list), policy=policy, platform=platform,
|
||||
title,
|
||||
parsed_path,
|
||||
is_private,
|
||||
isinstance(parsed_path, list),
|
||||
policy=policy,
|
||||
platform=platform,
|
||||
)
|
||||
except Exception:
|
||||
await UniMessage.text(f"无法解析到内容：{url}").send()
|
||||
@@ -249,8 +295,12 @@ async def dispatch_url(
|
||||
await UniMessage.text(title).send()
|
||||
return None, None
|
||||
return await process_douyin_res(
|
||||
title, parsed_path, is_private,
|
||||
isinstance(parsed_path, list), policy=policy, platform=platform,
|
||||
title,
|
||||
parsed_path,
|
||||
is_private,
|
||||
isinstance(parsed_path, list),
|
||||
policy=policy,
|
||||
platform=platform,
|
||||
)
|
||||
except Exception:
|
||||
await UniMessage.text(f"无法解析到内容：{url}").send()
|
||||
|
||||
@@ -1,15 +1,22 @@
|
||||
"""媒体多级发送 — temp 本地文件优先，失败逐级降级
|
||||
"""媒体多级发送 — 本地优先，归档与发送解耦（并行），失败逐级降级
|
||||
|
||||
发送策略（2026-08-22 用户需求）:
|
||||
1. temp 本地文件直接发送（最快，不经 S3）
|
||||
2. 失败 → 上传本地 S3，用预签名链接发送
|
||||
2. 失败 → 用**已归档**的 S3 预签名链接发送
|
||||
3. 再失败 → 回退 temp 本地文件再发一次
|
||||
|
||||
**归档是前提，不是发送的副产品（2026-09-25 用户明确）**：媒体一旦下载成功就要
|
||||
进本地 S3——与发送走哪一级、成功与否**无关**。因此 `archive_media` 在
|
||||
`send_pending_media` 一进来就无条件开跑，与本地直发**并行**：本地发送不等它
|
||||
（保住最快那条路），但它一定会跑完（发送全失败也照传）。原因：temp 只留 7 天
|
||||
（cleanup.py 删），桶里那份才是长期存档；顺带产出公网下载链接（`upload_public`
|
||||
开着时），「发送下载链接」策略直接用它。每个文件只上传一次——第 2 级用的就是
|
||||
归档的预签名链接，没有"补传"这一步。
|
||||
|
||||
本地直发要真的通，得让协议端能拿到文件：2026-09-22 起由 `hexi/core/outbound_media`
|
||||
的全局钩子把本地路径改写成 `/media/<token>` 链接（此前本地那级因协议端不在本机、
|
||||
`file://` 必然 ENOENT 而恒失败，实际每次都落到第 2 级 S3）。因此**媒体不再进 S3
|
||||
归档是有意的**——只有群策略真要发下载链接（`policy.sends_link`）时才补传一次拿
|
||||
public_url，见 `_local_sent`。
|
||||
`file://` 必然 ENOENT 而恒失败，实际每次都落到第 2 级 S3）。注意那条 `/media`
|
||||
链接**只给协议端拉取用**（TTL 300s、bot 自己的路由），不能当下载链接发给群友。
|
||||
|
||||
多媒体（2026-09-08 用户需求）: 多图（图集）、图+视频混合、多视频（动图）
|
||||
等一个作品落地成多个文件时，直接 UniMessage 拼成一个消息发送会报错
|
||||
@@ -20,15 +27,25 @@ public_url，见 `_local_sent`。
|
||||
传到群文件（默认打包成一个 zip，可配解压密码，见 config.group_file_settings），
|
||||
失败只记日志、不影响发送结果（打包/上传实现在 services/storage/group_file.py）。
|
||||
|
||||
**超时=结果未知，绝不降级重试**（2026-10-13）：NoneBot 默认 `api_timeout=30s`，
|
||||
而长视频在协议端要「下载 + 转存到 QQ」，实测 172MB 视频约 2 分钟。超时只是 bot
|
||||
不再等待响应——**协议端会把这次发送做完**，于是三级降级把同一个视频送了三份
|
||||
（群里收到三份 + 一条"媒体发送失败"）。修法有两半：发送调用带 `_timeout`
|
||||
（`send_timeout()`，默认 300s，`video_analysis_send_timeout` 可调）等它真正
|
||||
发完；万一仍超时则按「可能已送达」处理——停在这一级、不发失败提示，只有
|
||||
协议端**明确报错**（ActionFailed）才降级到下一级。
|
||||
|
||||
temp 下的文件发送成功后也不清理（用户手动处理 data/temp）。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
|
||||
from nonebot import get_bot, get_driver, logger
|
||||
from nonebot.adapters import Event
|
||||
from nonebot.exception import NetworkError
|
||||
from nonebot_plugin_alconna import UniMessage
|
||||
|
||||
from ..config import group_file_settings
|
||||
@@ -57,6 +74,56 @@ def _as_paths(file_paths: Path | list[Path]) -> list[Path]:
|
||||
return [Path(file_paths)]
|
||||
|
||||
|
||||
# ─────────────────── 发送超时与「结果未知」判定 ───────────────────
|
||||
|
||||
#: 媒体发送 per-call 超时（秒）——长视频在协议端「下载 + 转存 QQ」要几十秒
|
||||
#: 到几分钟（实测 172MB ≈ 140s），NoneBot 默认 api_timeout=30s 必超时。
|
||||
_DEFAULT_SEND_TIMEOUT = 300.0
|
||||
|
||||
|
||||
def send_timeout() -> float:
|
||||
"""媒体发送的 per-call 超时（`_timeout` 关键字，OneBot 适配器原生支持）。
|
||||
|
||||
超时只是 bot 不再等响应，**协议端会把这次发送做完**；所以这个值同时是
|
||||
「重试会重复发送」的边界，默认 300s，`video_analysis_send_timeout` 可调。
|
||||
"""
|
||||
raw = (
|
||||
os.environ.get("video_analysis_send_timeout")
|
||||
or os.environ.get("VIDEO_ANALYSIS_SEND_TIMEOUT")
|
||||
or ""
|
||||
).strip()
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
if raw:
|
||||
logger.warning(
|
||||
f"video_analysis_send_timeout={raw!r} 不是数字，"
|
||||
f"按默认 {_DEFAULT_SEND_TIMEOUT:g}s 处理"
|
||||
)
|
||||
return _DEFAULT_SEND_TIMEOUT
|
||||
return value if value > 0 else _DEFAULT_SEND_TIMEOUT
|
||||
|
||||
|
||||
def unknown_send_outcome(exc: BaseException) -> bool:
|
||||
"""这次发送是否「结果未知」——协议端可能仍在处理并最终送达。
|
||||
|
||||
`NetworkError` 涵盖调用超时与连接异常：请求已经发出，只是 bot 放弃了
|
||||
等待。长视频超时后协议端照样把视频发进群（2026-10-13 实测：172MB 视频
|
||||
三级降级各超时一次，群里收到三份），所以这种结果**不能再降级重试**，
|
||||
否则每试一级就多送一份；对外的"媒体发送失败"提示同理不能再发。
|
||||
"""
|
||||
return isinstance(exc, NetworkError) or isinstance(
|
||||
exc, (asyncio.TimeoutError, TimeoutError)
|
||||
)
|
||||
|
||||
|
||||
def _log_unknown(stage: str, exc: BaseException) -> None:
|
||||
logger.warning(
|
||||
f"{stage}发送等待超时，结果未知（协议端可能仍在处理并最终送达）；"
|
||||
f"不再降级重试以免重复发送: {exc}"
|
||||
)
|
||||
|
||||
|
||||
# ─────────────────────── 合并转发（多媒体专用） ───────────────────────
|
||||
|
||||
|
||||
@@ -114,37 +181,81 @@ async def _send_forward_media(
|
||||
*,
|
||||
group_id: int | None = None,
|
||||
user_id: int | None = None,
|
||||
) -> bool:
|
||||
"""发送合并转发消息（群聊 / 私聊），失败返回 False"""
|
||||
) -> bool | None:
|
||||
"""发送合并转发消息（群聊 / 私聊）。
|
||||
|
||||
返回 True=成功 / False=失败（协议端明确报错，可降级重试）/
|
||||
None=结果未知（超时，调用方必须停止重试，见 `unknown_send_outcome`）。
|
||||
"""
|
||||
try:
|
||||
bot = get_bot()
|
||||
nodes = _build_forward_nodes(items, sender_name, bot.self_id)
|
||||
timeout = send_timeout()
|
||||
if group_id is not None:
|
||||
await bot.call_api(
|
||||
"send_group_forward_msg", group_id=group_id, messages=nodes
|
||||
"send_group_forward_msg",
|
||||
group_id=group_id,
|
||||
messages=nodes,
|
||||
_timeout=timeout,
|
||||
)
|
||||
else:
|
||||
await bot.call_api("send_forward_msg", user_id=user_id, messages=nodes)
|
||||
await bot.call_api(
|
||||
"send_forward_msg", user_id=user_id, messages=nodes, _timeout=timeout
|
||||
)
|
||||
return True
|
||||
except Exception as e:
|
||||
if unknown_send_outcome(e):
|
||||
_log_unknown("合并转发", e)
|
||||
return None
|
||||
logger.warning(f"合并转发发送失败: {e}")
|
||||
return False
|
||||
|
||||
|
||||
async def _build_s3_forward_items(
|
||||
media: PendingMedia,
|
||||
) -> tuple[list[tuple[Path, str | None]], str | None]:
|
||||
"""上传本地 S3，返回 ([(文件, 预签名链接)...], public_url)"""
|
||||
items: list[tuple[Path, str | None]] = []
|
||||
public_url = None
|
||||
# ─────────────────────── 归档（发送的前提） ───────────────────────
|
||||
|
||||
|
||||
@dataclass
|
||||
class UploadedFile:
|
||||
"""归档结果：本地文件 → 局域网预签名链接（发送兜底）+ 公网链接（下载链接）"""
|
||||
|
||||
path: Path
|
||||
local_url: str | None = None
|
||||
public_url: str | None = None
|
||||
|
||||
|
||||
async def archive_media(media: PendingMedia) -> list[UploadedFile]:
|
||||
"""逐文件归档到本地 S3 —— **无条件**执行，与发送成败无关。
|
||||
|
||||
这是「媒体必须进桶」这条前提的落点（用户 2026-09-25 定）：temp 只留 7 天，
|
||||
桶里那份才是长期存档。boto3 同步阻塞（大视频一次传几十秒），统一丢线程池。
|
||||
单个文件失败只 warning 并**继续**后面的（多图作品不该因第 3 张失败丢掉后 7 张），
|
||||
任何情况都不抛——归档失败不能反过来影响发送，发送失败也不影响归档。
|
||||
"""
|
||||
uploads: list[UploadedFile] = []
|
||||
for fp in media.files:
|
||||
local_url, pub = upload_with_plan(fp, policy=media.policy)
|
||||
if not local_url:
|
||||
raise RuntimeError(f"上传本地 S3 失败: {fp}")
|
||||
if pub:
|
||||
public_url = pub
|
||||
items.append((fp, local_url))
|
||||
return items, public_url
|
||||
item = UploadedFile(path=fp)
|
||||
try:
|
||||
item.local_url, item.public_url = await asyncio.to_thread(
|
||||
upload_with_plan, fp, policy=media.policy
|
||||
)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning(f"归档到本地 S3 失败 {fp}: {e}")
|
||||
uploads.append(item)
|
||||
return uploads
|
||||
|
||||
|
||||
def _presigned_items(uploads: list[UploadedFile]) -> list[tuple[Path, str | None]]:
|
||||
"""归档结果 → 链接版消息/合并转发要的 [(文件, 预签名链接)]（缺链接=该文件没传上）"""
|
||||
return [(item.path, item.local_url) for item in uploads]
|
||||
|
||||
|
||||
def _last_public_url(uploads: list[UploadedFile]) -> str | None:
|
||||
"""公网下载链接取最后一个非空值（`upload_public` 关时恒为 None）"""
|
||||
public_url = None
|
||||
for item in uploads:
|
||||
if item.public_url:
|
||||
public_url = item.public_url
|
||||
return public_url
|
||||
|
||||
|
||||
# ───────────────────────────── 多级发送 ─────────────────────────────
|
||||
@@ -161,68 +272,49 @@ def _build_local_msg(files: list[Path], image_post: bool) -> UniMessage:
|
||||
return msg
|
||||
|
||||
|
||||
def _build_s3_msg(
|
||||
media: PendingMedia,
|
||||
) -> tuple[UniMessage, str | None]:
|
||||
"""上传本地 S3 并构建链接版消息，返回 (message, public_url)"""
|
||||
def _build_s3_msg(items: list[tuple[Path, str | None]]) -> UniMessage:
|
||||
"""用归档好的预签名链接构建链接版消息
|
||||
|
||||
有文件没拿到链接（归档时上传失败）就抛，交由调用方降级到下一级。
|
||||
"""
|
||||
urls: list[tuple[Path, str]] = []
|
||||
for fp, url in items:
|
||||
if not url:
|
||||
raise RuntimeError(f"归档未拿到预签名链接: {fp}")
|
||||
urls.append((fp, url))
|
||||
|
||||
msg = UniMessage()
|
||||
public_url = None
|
||||
for fp in media.files:
|
||||
local_url, pub = upload_with_plan(fp, policy=media.policy)
|
||||
if not local_url:
|
||||
raise RuntimeError(f"上传本地 S3 失败: {fp}")
|
||||
if pub:
|
||||
public_url = pub
|
||||
for fp, url in urls:
|
||||
if _is_video(fp):
|
||||
msg.video(url=local_url)
|
||||
msg.video(url=url)
|
||||
else:
|
||||
msg.image(url=local_url)
|
||||
return msg, public_url
|
||||
|
||||
|
||||
async def _public_link_for(media: PendingMedia) -> str | None:
|
||||
"""发送已由本地通道完成后，为「发送下载链接」策略补一次 S3 上传。
|
||||
|
||||
本地直发不经 S3（最快），但群策略开了链接就需要公网 URL——这里**只取链接**，
|
||||
不重建消息。失败只记日志并返回 None：发送本身已经成功了，不能反过来判失败。
|
||||
"""
|
||||
public_url = None
|
||||
for fp in media.files:
|
||||
try:
|
||||
_, pub = await asyncio.to_thread(upload_with_plan, fp, policy=media.policy)
|
||||
except Exception as e: # noqa: BLE001
|
||||
logger.warning(f"补传 S3 取下载链接失败 {fp}: {e}")
|
||||
return None
|
||||
if pub:
|
||||
public_url = pub
|
||||
return public_url
|
||||
|
||||
|
||||
async def _local_sent(media: PendingMedia) -> tuple[bool, str | None]:
|
||||
"""本地通道发送成功后的返回。
|
||||
|
||||
只有策略真的要发下载链接（`sends_link` = send_link 且已开公网）时才补 S3；
|
||||
其余情况一概不碰 S3 —— 媒体不走 S3 归档是有意为之（temp 保留由 cleanup 管）。
|
||||
"""
|
||||
if media.policy is not None and media.policy.sends_link:
|
||||
return True, await _public_link_for(media)
|
||||
return True, None
|
||||
msg.image(url=url)
|
||||
return msg
|
||||
|
||||
|
||||
async def _send_media_core(
|
||||
media: PendingMedia, event: Event | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
"""多级发送主体，返回 (是否成功, public_url)
|
||||
media: PendingMedia,
|
||||
event: Event | None,
|
||||
archive: "asyncio.Task[list[UploadedFile]]",
|
||||
) -> bool:
|
||||
"""多级发送主体，返回是否成功（public_url 由调用方从归档结果取）。
|
||||
|
||||
public_url 有两种来源：走 S3 链接发送成功，或本地发送成功但策略要链接时
|
||||
补传 S3（见 `_local_sent`）；其余情况为 None（调用方据 `sends_link` 决定是否发文字）。
|
||||
`archive` 是**已经在跑**的归档任务（见 `send_pending_media`）：本地直发不等它，
|
||||
需要链接时再 `await`——同一个 task 多次 await 安全且结果复用，所以每个文件
|
||||
只上传一次，没有"补传"这一步。
|
||||
temp 文件发送成功后保留（用户手动清理 data/temp）。
|
||||
|
||||
多媒体作品（多图 / 图+视频 / 多视频）先走合并转发（本地文件 → S3 链接），
|
||||
失败后仍落回下方普通消息链路兜底。
|
||||
|
||||
**超时（结果未知）时按"可能已送达"处理**：停在这一级并返回 True——协议端
|
||||
可能仍在把这份发完，降级重试会重复发送、发"发送失败"提示也会误导。只有
|
||||
协议端明确报错（ActionFailed 等）才降级，见 `unknown_send_outcome`。
|
||||
"""
|
||||
if not media.files:
|
||||
return False, None
|
||||
return False
|
||||
|
||||
timeout = send_timeout()
|
||||
|
||||
# ── 0. 多媒体 → 合并转发（一个图/视频一个节点）─────────────────
|
||||
if event is not None and _needs_forward(media.files):
|
||||
@@ -230,51 +322,66 @@ async def _send_media_core(
|
||||
user_id = None if group_id is not None else getattr(event, "user_id", None)
|
||||
if group_id is not None or user_id is not None:
|
||||
sender_name = _forward_sender_name()
|
||||
if await _send_forward_media(
|
||||
sent = await _send_forward_media(
|
||||
[(fp, None) for fp in media.files],
|
||||
sender_name,
|
||||
group_id=group_id,
|
||||
user_id=user_id,
|
||||
):
|
||||
)
|
||||
if sent is None:
|
||||
return True
|
||||
if sent:
|
||||
logger.info("媒体发送成功（合并转发，本地文件）")
|
||||
return await _local_sent(media)
|
||||
try:
|
||||
s3_items, public_url = await _build_s3_forward_items(media)
|
||||
if await _send_forward_media(
|
||||
s3_items, sender_name, group_id=group_id, user_id=user_id
|
||||
):
|
||||
return True
|
||||
items = _presigned_items(await archive)
|
||||
if all(url for _, url in items):
|
||||
sent = await _send_forward_media(
|
||||
items, sender_name, group_id=group_id, user_id=user_id
|
||||
)
|
||||
if sent is None:
|
||||
return True
|
||||
if sent:
|
||||
logger.info("媒体发送成功（合并转发，S3 链接）")
|
||||
return True, public_url
|
||||
except Exception as e:
|
||||
logger.warning(f"S3 链接合并转发失败，走普通消息链路: {e}")
|
||||
return True
|
||||
else:
|
||||
logger.warning("归档未全部成功，跳过 S3 链接合并转发，走普通消息链路")
|
||||
else:
|
||||
logger.warning("无法确定群号/用户号，跳过合并转发，走普通消息链路")
|
||||
|
||||
# ── 1. temp 本地文件直接发送 ──────────────────────────────
|
||||
try:
|
||||
await _build_local_msg(media.files, media.image_post).send()
|
||||
await _build_local_msg(media.files, media.image_post).send(_timeout=timeout)
|
||||
logger.info("媒体发送成功（temp 本地文件直达）")
|
||||
return await _local_sent(media)
|
||||
return True
|
||||
except Exception as e:
|
||||
if unknown_send_outcome(e):
|
||||
_log_unknown("temp 本地文件直达", e)
|
||||
return True
|
||||
logger.warning(f"temp 本地文件发送失败，切换本地 S3 链接: {e}")
|
||||
|
||||
# ── 2. 上传本地 S3 → 链接发送 ─────────────────────────────
|
||||
# ── 2. 用归档好的预签名链接发送 ─────────────────────────────
|
||||
try:
|
||||
s3_msg, public_url = _build_s3_msg(media)
|
||||
await s3_msg.send()
|
||||
s3_msg = _build_s3_msg(_presigned_items(await archive))
|
||||
await s3_msg.send(_timeout=timeout)
|
||||
logger.info("媒体发送成功（本地 S3 链接）")
|
||||
return True, public_url
|
||||
return True
|
||||
except Exception as e:
|
||||
if unknown_send_outcome(e):
|
||||
_log_unknown("本地 S3 链接", e)
|
||||
return True
|
||||
logger.warning(f"本地 S3 链接发送失败，回退 temp 本地文件: {e}")
|
||||
|
||||
# ── 3. 回退：temp 本地文件再发一次 ────────────────────────
|
||||
try:
|
||||
await _build_local_msg(media.files, media.image_post).send()
|
||||
await _build_local_msg(media.files, media.image_post).send(_timeout=timeout)
|
||||
logger.info("媒体发送成功（回退 temp 本地文件）")
|
||||
return await _local_sent(media)
|
||||
return True
|
||||
except Exception as e:
|
||||
if unknown_send_outcome(e):
|
||||
_log_unknown("回退 temp 本地文件", e)
|
||||
return True
|
||||
logger.exception(f"回退发送失败: {e}")
|
||||
return False, None
|
||||
return False
|
||||
|
||||
|
||||
async def _upload_group_files(media: PendingMedia, event: Event | None) -> None:
|
||||
@@ -312,10 +419,24 @@ async def _upload_group_files(media: PendingMedia, event: Event | None) -> None:
|
||||
async def send_pending_media(
|
||||
media: PendingMedia, event: Event | None = None
|
||||
) -> tuple[bool, str | None]:
|
||||
"""多级发送 + 群文件并行通道，返回 (是否成功, public_url)。"""
|
||||
"""归档 + 多级发送 + 群文件并行通道，返回 (是否成功, public_url)。
|
||||
|
||||
**归档无条件且先行开跑**（用户 2026-09-25 定的前提）：媒体进了 temp 就必须
|
||||
再进一次桶，与发送走哪一级、成功与否无关。它与本地直发并行，所以发送仍旧是
|
||||
最快那条路；`public_url` 取自归档结果（`upload_public` 关时为 None），交给
|
||||
entry.py 按策略决定是否发链接文本。
|
||||
|
||||
「是否成功」对**超时（结果未知）**按 True 算：协议端可能仍在发，调用方
|
||||
不该再给群里发"媒体发送失败"（长视频实测就是超时后照样送达）。
|
||||
"""
|
||||
if not media.files:
|
||||
return False, None
|
||||
|
||||
ok, public_url = await _send_media_core(media, event)
|
||||
archive = asyncio.create_task(archive_media(media))
|
||||
try:
|
||||
ok = await _send_media_core(media, event, archive)
|
||||
finally:
|
||||
# 发送层任何异常都不能把归档半途丢下：归档自己的失败已在内部吞掉。
|
||||
uploads = await archive
|
||||
await _upload_group_files(media, event)
|
||||
return ok, public_url
|
||||
return ok, _last_public_url(uploads)
|
||||
@@ -44,11 +44,12 @@ from nonebot import logger
|
||||
XHS = "小红书"
|
||||
BILIBILI = "哔哩哔哩"
|
||||
DOUYIN = "抖音"
|
||||
KUAISHOU = "快手"
|
||||
YOUTUBE = "Youtube"
|
||||
X = "X"
|
||||
|
||||
#: 规范标签（存储值 = 展示值，命令/Web 输入同一个词）
|
||||
PLATFORMS: tuple[str, ...] = (XHS, BILIBILI, DOUYIN, YOUTUBE, X)
|
||||
PLATFORMS: tuple[str, ...] = (XHS, BILIBILI, DOUYIN, KUAISHOU, YOUTUBE, X)
|
||||
|
||||
#: 平台 → 域名片段（判定 auto_link / ban_link 命中用）
|
||||
PLATFORM_DOMAINS: dict[str, tuple[str, ...]] = {
|
||||
@@ -60,6 +61,9 @@ PLATFORM_DOMAINS: dict[str, tuple[str, ...]] = {
|
||||
"m.douyin.com",
|
||||
"jingxuan.douyin.com",
|
||||
),
|
||||
# 短链 v.kuaishou.com / 网页版 www.kuaishou.com /
|
||||
# 分享页 v.m.chenzhongtech.com（QQ 卡片 jumpUrl 就走这个域）
|
||||
KUAISHOU: ("kuaishou.com", "chenzhongtech.com"),
|
||||
YOUTUBE: ("youtube.com", "youtu.be"),
|
||||
X: ("x.com", "twitter.com"),
|
||||
}
|
||||
@@ -74,6 +78,9 @@ _PLATFORM_ALIASES: dict[str, str] = {
|
||||
"哔哩哔哩": BILIBILI,
|
||||
"douyin": DOUYIN,
|
||||
"抖音": DOUYIN,
|
||||
"kuaishou": KUAISHOU,
|
||||
"ks": KUAISHOU,
|
||||
"快手": KUAISHOU,
|
||||
"yt": YOUTUBE,
|
||||
"youtube": YOUTUBE,
|
||||
"油管": YOUTUBE,
|
||||
|
||||
@@ -224,7 +224,7 @@ def _build_rel_stem(
|
||||
"""
|
||||
return (
|
||||
f"{build_author_dir(author, author_id, source=source)}"
|
||||
f"/{build_work_stem(title)}"
|
||||
f"/{build_work_stem(title, source=source)}"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,309 +1,17 @@
|
||||
"""抖音内容抓取 — 浏览器自动化 + API 拦截 + 媒体下载"""
|
||||
"""媒体下载辅助 — 图集并发下载（抖音/B站/小红书/快手四条链路共用）
|
||||
|
||||
历史上的 playwright 拦截链路与分轨视频合并已随抖音新链路（douyin_web.py）
|
||||
废弃移除；这里只保留纯下载工具。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import httpx
|
||||
from nonebot import logger
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
from ...models import DouyinFetchError
|
||||
from ...utils import get_temp_root, unique_media_path
|
||||
from .douyin_parser import (
|
||||
extract_trailing_digits,
|
||||
parse_animated_note_videos,
|
||||
parse_douyin_response,
|
||||
parse_note_images,
|
||||
parse_ssr_page,
|
||||
parse_video_urls,
|
||||
)
|
||||
|
||||
|
||||
def merge_video_audio(video_file: Path, audio_file: Path, out_file: Path):
|
||||
"""分轨视频与音频合并"""
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg",
|
||||
"-i",
|
||||
str(video_file),
|
||||
"-i",
|
||||
str(audio_file),
|
||||
"-c:v",
|
||||
"copy",
|
||||
"-c:a",
|
||||
"copy",
|
||||
"-movflags",
|
||||
"faststart",
|
||||
"-y",
|
||||
str(out_file),
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
# ============================= 主入口 =============================
|
||||
|
||||
|
||||
async def fetch_douyin_content(
|
||||
douyin_url: str,
|
||||
cookies: List[dict],
|
||||
wait_seconds: int = 10,
|
||||
headless: bool = True,
|
||||
) -> tuple[str | None, Path | List[Path] | None]:
|
||||
"""
|
||||
获取抖音内容（视频或图文）
|
||||
|
||||
返回:
|
||||
(title, content)
|
||||
- 视频: (title, Path)
|
||||
- 图文: (title, List[Path])
|
||||
- 失败: (None, None)
|
||||
"""
|
||||
if not cookies:
|
||||
raise DouyinFetchError("cookies 为空")
|
||||
|
||||
tmp_root = get_temp_root("douyin")
|
||||
api_response: Optional[dict] = None
|
||||
api_response_favorite: Optional[dict] = None
|
||||
aweme_id = 0
|
||||
# 目标作品 id（从入口 URL 提取）：aweme/post 返回的是作者作品列表，
|
||||
# 按此 id 精确匹配要解析的作品，避免取到作者的其他作品
|
||||
target_aweme_id = extract_trailing_digits(douyin_url) or ""
|
||||
referer_url = None
|
||||
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch(
|
||||
executable_path="C:/Program Files/Google/Chrome/Application/chrome.exe",
|
||||
headless=headless,
|
||||
args=[
|
||||
"--autoplay-policy=no-user-gesture-required",
|
||||
"--disable-features=AutoplayDisableSuppression",
|
||||
],
|
||||
)
|
||||
|
||||
context = await browser.new_context(
|
||||
viewport={"width": 1280, "height": 720},
|
||||
device_scale_factor=2,
|
||||
user_agent=(
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/122.0.0.0 Safari/537.36"
|
||||
),
|
||||
locale="zh-CN",
|
||||
)
|
||||
|
||||
await context.add_cookies(cookies)
|
||||
page = await context.new_page()
|
||||
|
||||
async def handle_response(response):
|
||||
nonlocal api_response, api_response_favorite, aweme_id, referer_url
|
||||
try:
|
||||
if (
|
||||
"https://www.douyin.com/note" in response.url
|
||||
or "https://www.douyin.com/video" in response.url
|
||||
):
|
||||
referer_url = response.url
|
||||
logger.info(f"作品链接：{referer_url}")
|
||||
aweme_id = extract_trailing_digits(response.url)
|
||||
logger.info(f"作品id：{aweme_id}")
|
||||
|
||||
if "aweme/v1/web/aweme/detail" in response.url:
|
||||
if not api_response:
|
||||
logger.info(f"捕获到视频 API: {response.url}")
|
||||
ct = response.headers.get("content-type", "")
|
||||
if "json" in ct:
|
||||
api_response = await response.json()
|
||||
logger.info("API 响应已捕获")
|
||||
return
|
||||
else:
|
||||
logger.info("API 已有捕获")
|
||||
return
|
||||
|
||||
elif "aweme/v1/web/aweme/post" in response.url:
|
||||
logger.info(f"捕获到图文 API: {response.url}")
|
||||
ct = response.headers.get("content-type", "")
|
||||
if "json" in ct:
|
||||
data = await response.json()
|
||||
aweme_list = data.get("aweme_list", [])
|
||||
if aweme_list:
|
||||
# 列表是作者的全部作品，优先按目标作品 id 精确匹配
|
||||
# （未登录/风控时目标作品可能不在列表里）
|
||||
match_id = aweme_id or target_aweme_id
|
||||
target_aweme = None
|
||||
if match_id:
|
||||
target_aweme = next(
|
||||
(
|
||||
a
|
||||
for a in aweme_list
|
||||
if str(a.get("aweme_id")) == str(match_id)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not target_aweme:
|
||||
target_aweme = aweme_list[0]
|
||||
if target_aweme.get("images"):
|
||||
logger.info("找到正确的图文 API 响应（包含图片数据）")
|
||||
data["aweme_list"] = [target_aweme]
|
||||
api_response = data
|
||||
return
|
||||
else:
|
||||
logger.warning(
|
||||
"此响应不包含有效的图片数据，等待下一个请求"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"处理响应失败: {e}")
|
||||
|
||||
page.on("response", handle_response)
|
||||
|
||||
await page.goto(douyin_url, wait_until="domcontentloaded")
|
||||
|
||||
max_wait = wait_seconds
|
||||
for i in range(max_wait):
|
||||
if api_response:
|
||||
logger.info(f"成功在第 {i + 1} 秒捕获 API 响应")
|
||||
break
|
||||
await page.wait_for_timeout(1000)
|
||||
|
||||
# 截取页面 HTML（在关闭浏览器前），用于 SSR 回退解析
|
||||
page_html = await page.content() if not api_response else None
|
||||
|
||||
await browser.close()
|
||||
|
||||
if not api_response and page_html:
|
||||
api_response = parse_ssr_page(page_html)
|
||||
if api_response:
|
||||
logger.info("通过 SSR 页面回退解析获取到图文数据")
|
||||
|
||||
if not api_response:
|
||||
raise DouyinFetchError("无法捕获 API 响应，请检查网络或 URL")
|
||||
|
||||
parsed = parse_douyin_response(api_response, referer_url)
|
||||
|
||||
headers = {
|
||||
"Referer": "https://www.douyin.com/",
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/122.0.0.0 Safari/537.36"
|
||||
),
|
||||
}
|
||||
|
||||
if parsed.media_type == "视频":
|
||||
content = await _process_video(
|
||||
api_response, tmp_root, parsed.rel_stem, aweme_id, headers
|
||||
)
|
||||
return parsed.raw_title, content
|
||||
|
||||
elif parsed.media_type == "图片":
|
||||
# 先解析 images 列表，区分纯动图和图文/图+视频
|
||||
images_urls, video_url = parse_note_images(
|
||||
api_response, api_response_favorite, aweme_id
|
||||
)
|
||||
if images_urls:
|
||||
# 有图片（纯图文 或 图+视频混合作品）
|
||||
content = await _process_note_with_parsed(
|
||||
images_urls,
|
||||
video_url,
|
||||
tmp_root,
|
||||
parsed.rel_stem,
|
||||
headers,
|
||||
)
|
||||
else:
|
||||
# 纯动图（所有项都是视频）
|
||||
content = await _process_animated_note(
|
||||
api_response, tmp_root, parsed.rel_stem, headers
|
||||
)
|
||||
return parsed.raw_title, content
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
# ============================= 视频下载 =============================
|
||||
|
||||
|
||||
async def _process_video(
|
||||
api_response: dict,
|
||||
tmp_root: Path,
|
||||
rel_stem: str,
|
||||
aweme_id: str,
|
||||
headers: Dict[str, str],
|
||||
) -> Path:
|
||||
"""处理视频内容，返回本地文件路径
|
||||
|
||||
rel_stem 是相对平台根的路径词干 `{作者目录}/{作品名}`（见 ParsedDouyinContent.rel_stem）。
|
||||
"""
|
||||
groups = parse_video_urls(api_response)
|
||||
|
||||
best_group = None
|
||||
for g in groups.values():
|
||||
if g["FULL"] or (g["VIDEO"] and g["AUDIO"]):
|
||||
best_group = g
|
||||
break
|
||||
|
||||
if not best_group:
|
||||
raise DouyinFetchError("没有可用的视频组合")
|
||||
|
||||
# 完整视频 — 流式下载
|
||||
if best_group["FULL"]:
|
||||
full = best_group["FULL"]
|
||||
logger.info(f"找到完整视频，数量: {len(full)}")
|
||||
best = max(full, key=lambda x: x["br"])
|
||||
logger.info(f"选择码率: {best['br']} - {best['url'][:60]}...")
|
||||
|
||||
output_path = unique_media_path(tmp_root / f"{rel_stem}.mp4")
|
||||
async with httpx.AsyncClient(headers=headers) as client:
|
||||
async with client.stream("GET", best["url"]) as resp:
|
||||
resp.raise_for_status()
|
||||
with open(output_path, "wb") as f:
|
||||
async for chunk in resp.aiter_bytes(8192):
|
||||
f.write(chunk)
|
||||
logger.info(f"视频下载完成: {output_path}")
|
||||
return output_path
|
||||
|
||||
# 分轨视频 — 分别下载后合并
|
||||
logger.info(
|
||||
f"使用分轨模式，视频数: {len(best_group['VIDEO'])}, "
|
||||
f"音频数: {len(best_group['AUDIO'])}"
|
||||
)
|
||||
video = max(best_group["VIDEO"], key=lambda x: x["br"])
|
||||
audio = max(best_group["AUDIO"], key=lambda x: x["br"])
|
||||
logger.info(f"选择视频码率: {video['br']}")
|
||||
logger.info(f"选择音频码率: {audio['br']}")
|
||||
|
||||
# 先定下产物名（顺带建好作者目录），分轨中间文件与产物同目录
|
||||
output_path = unique_media_path(tmp_root / f"{rel_stem}.mp4")
|
||||
video_path = output_path.parent / f"{output_path.stem}_v.mp4"
|
||||
audio_path = output_path.parent / f"{output_path.stem}_a.mp4"
|
||||
|
||||
async with httpx.AsyncClient(headers=headers) as client:
|
||||
logger.info("开始下载视频...")
|
||||
async with client.stream("GET", video["url"]) as v:
|
||||
v.raise_for_status()
|
||||
with open(video_path, "wb") as f:
|
||||
async for chunk in v.aiter_bytes(8192):
|
||||
f.write(chunk)
|
||||
|
||||
logger.info("开始下载音频...")
|
||||
async with client.stream("GET", audio["url"]) as a:
|
||||
a.raise_for_status()
|
||||
with open(audio_path, "wb") as f:
|
||||
async for chunk in a.aiter_bytes(8192):
|
||||
f.write(chunk)
|
||||
|
||||
logger.info("合并视频和音频...")
|
||||
# ffmpeg 是同步子进程，直接 await 不了：不丢线程池会卡住整个事件循环
|
||||
# （合并期间 Web 轮询、群消息全都停摆）
|
||||
await asyncio.to_thread(merge_video_audio, video_path, audio_path, output_path)
|
||||
video_path.unlink()
|
||||
audio_path.unlink()
|
||||
|
||||
logger.info(f"视频下载完成: {output_path}")
|
||||
return output_path
|
||||
|
||||
|
||||
# ============================= 图文下载 =============================
|
||||
from ...utils import unique_media_path
|
||||
|
||||
|
||||
async def _process_note_with_parsed(
|
||||
@@ -313,41 +21,49 @@ async def _process_note_with_parsed(
|
||||
rel_stem: str,
|
||||
headers: Dict[str, str],
|
||||
) -> List[Path]:
|
||||
"""根据已解析的图片/视频 URL 列表，并行下载
|
||||
"""按已解析的图片/视频 URL 列表并发下载
|
||||
|
||||
一个作品一个目录：`{平台根}/{作者目录}/{作品名}[_{短码}]/001.jpg…`
|
||||
- images_urls：每项是该媒体位的 URL 候选列表（按优先级排序，逐个回退）
|
||||
- video_url：图+视频混合作品中追加到末尾的视频（单 URL，可为 None）
|
||||
"""
|
||||
note_dir = unique_media_path(tmp_root / rel_stem)
|
||||
note_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"图文保存目录: {note_dir}")
|
||||
|
||||
async def _download_one(idx: int, url: str, ext: str = "") -> Optional[Path]:
|
||||
if not ext:
|
||||
ext = _infer_extension(url)
|
||||
filename = f"{idx:03d}{ext}"
|
||||
async def _download_one(
|
||||
idx: int, url_list: List[str], ext: str = ""
|
||||
) -> Optional[Path]:
|
||||
for url in url_list:
|
||||
use_ext = ext or _infer_extension(url)
|
||||
filename = f"{idx:03d}{use_ext}"
|
||||
filepath = note_dir / filename
|
||||
try:
|
||||
async with httpx.AsyncClient(headers=headers) as client:
|
||||
async with httpx.AsyncClient(headers=headers, timeout=300) as client:
|
||||
async with client.stream("GET", url) as resp:
|
||||
resp.raise_for_status()
|
||||
with open(filepath, "wb") as f:
|
||||
async for chunk in resp.aiter_bytes(8192):
|
||||
async for chunk in resp.aiter_bytes(65536):
|
||||
f.write(chunk)
|
||||
if filepath.stat().st_size == 0:
|
||||
raise RuntimeError("响应为空（0 字节）")
|
||||
logger.info(f"已保存: {filename}")
|
||||
return filepath
|
||||
except Exception as e:
|
||||
logger.error(f"下载 {idx} 失败: {e}")
|
||||
logger.warning(f"下载 {idx} 失败（尝试下一个候选）: {e}")
|
||||
logger.error(f"下载 {idx} 失败: 所有候选 URL 均不可用")
|
||||
return None
|
||||
|
||||
tasks = [
|
||||
_download_one(i, url_list[0])
|
||||
_download_one(i, url_list)
|
||||
for i, url_list in enumerate(images_urls, 1)
|
||||
if url_list
|
||||
]
|
||||
|
||||
# 图+视频混合作品：视频追加到下载任务
|
||||
if video_url:
|
||||
next_idx = len(images_urls) + 1
|
||||
tasks.append(_download_one(next_idx, video_url, ext=".mp4"))
|
||||
tasks.append(_download_one(next_idx, [video_url], ext=".mp4"))
|
||||
|
||||
results = await asyncio.gather(*tasks)
|
||||
saved_paths: List[Path] = [p for p in results if p is not None]
|
||||
@@ -356,75 +72,9 @@ async def _process_note_with_parsed(
|
||||
return saved_paths
|
||||
|
||||
|
||||
async def _process_note(
|
||||
api_response: dict,
|
||||
api_response_favorite: dict,
|
||||
tmp_root: Path,
|
||||
rel_stem: str,
|
||||
aweme_id: str,
|
||||
headers: Dict[str, str],
|
||||
) -> List[Path]:
|
||||
"""处理图文内容，并行下载所有图片；图+视频混合作品同时下载视频"""
|
||||
images_urls, video_url = parse_note_images(
|
||||
api_response, api_response_favorite, aweme_id
|
||||
)
|
||||
logger.info(f"解析到的图片链接:{images_urls}")
|
||||
|
||||
if not images_urls and not video_url:
|
||||
raise DouyinFetchError("未找到图文链接")
|
||||
|
||||
return await _process_note_with_parsed(
|
||||
images_urls, video_url, tmp_root, rel_stem, headers
|
||||
)
|
||||
|
||||
|
||||
# ============================= 动图下载 =============================
|
||||
|
||||
|
||||
async def _process_animated_note(
|
||||
api_response: dict,
|
||||
tmp_root: Path,
|
||||
rel_stem: str,
|
||||
headers: Dict[str, str],
|
||||
) -> List[Path]:
|
||||
"""处理动图内容（media_type=42），并行下载所有无声 mp4 视频"""
|
||||
video_urls = parse_animated_note_videos(api_response)
|
||||
logger.info(f"解析到的动图视频链接: {video_urls}")
|
||||
|
||||
note_dir = unique_media_path(tmp_root / rel_stem)
|
||||
note_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"动图保存目录: {note_dir}")
|
||||
|
||||
async def _download_one(idx: int, url: str) -> Optional[Path]:
|
||||
filename = f"{idx:03d}.mp4"
|
||||
filepath = note_dir / filename
|
||||
try:
|
||||
async with httpx.AsyncClient(headers=headers) as client:
|
||||
async with client.stream("GET", url) as resp:
|
||||
resp.raise_for_status()
|
||||
with open(filepath, "wb") as f:
|
||||
async for chunk in resp.aiter_bytes(8192):
|
||||
f.write(chunk)
|
||||
logger.info(f"动图视频已保存: {filename}")
|
||||
return filepath
|
||||
except Exception as e:
|
||||
logger.error(f"下载动图视频 {idx} 失败: {e}")
|
||||
return None
|
||||
|
||||
tasks = [
|
||||
_download_one(i, url)
|
||||
for i, url in enumerate(video_urls, 1)
|
||||
]
|
||||
results = await asyncio.gather(*tasks)
|
||||
saved_paths: List[Path] = [p for p in results if p is not None]
|
||||
|
||||
logger.info(f"动图下载完成，共 {len(saved_paths)} 个视频")
|
||||
return saved_paths
|
||||
|
||||
|
||||
def _infer_extension(url: str) -> str:
|
||||
"""从 URL 推断文件扩展名"""
|
||||
extensions = [".webp", ".jpg", ".jpeg", ".png", ".gif", ".avif"]
|
||||
extensions = [".webp", ".jpg", ".jpeg", ".png", ".gif", ".avif", ".heic"]
|
||||
url_lower = url.lower()
|
||||
for ext in extensions:
|
||||
if ext in url_lower:
|
||||
|
||||
@@ -418,7 +418,7 @@ def parse_douyin_response(
|
||||
author_dir = build_author_dir(
|
||||
raw_nickname, raw_author_id, source=referer_url or aweme_id
|
||||
)
|
||||
file_name = build_work_stem(raw_title)
|
||||
file_name = build_work_stem(raw_title, source=referer_url or aweme_id)
|
||||
|
||||
logger.info(
|
||||
f"内容标题: {raw_title}, 作者: {raw_nickname}({raw_author_id}), "
|
||||
|
||||
@@ -0,0 +1,929 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""抖音 a_bogus 签名 — 纯 Python 实现（免浏览器/免 Node）。
|
||||
|
||||
来源：https://github.com/Johnserf-Seed/f2 （Apache-2.0 License，
|
||||
许可证全文见 https://github.com/Johnserf-Seed/f2/blob/main/LICENSE）
|
||||
文件 f2/utils/crypto/bytedance/abogus.py（2026-10-13 拉取，未做任何修改；
|
||||
原文件头部的 f2 内部依赖仅有标准库与 gmssl，vendor 后可直接使用）。
|
||||
|
||||
与上游保持一致：算法对应 bdms 1.0.1.19，2026-10 实测可过抖音 ArgusSecurityPlugin 网关。
|
||||
若抖音再次更换签名算法，更新点就是本文件（对照 f2 上游 diff）。
|
||||
|
||||
依赖：gmssl（pip install gmssl，纯 Python 的国密 SM3 实现）。
|
||||
"""
|
||||
# fmt: off
|
||||
# ruff: noqa
|
||||
# path: f2/utils/crypto/bytedance/abogus.py
|
||||
|
||||
#!/usr/bin/env python
|
||||
# -*- encoding: utf-8 -*-
|
||||
"""
|
||||
@Description:abogus.py
|
||||
@Date :2024/06/16 11:21:14
|
||||
@Author :JohnserfSeed
|
||||
@version :0.0.4
|
||||
@License :Apache License 2.0
|
||||
@Github :https://github.com/johnserf-seed
|
||||
@Mail :support@f2.wiki
|
||||
-------------------------------------------------
|
||||
Change Log :
|
||||
2024/06/16 17:27:47 - Create ABogus algorithm & black style
|
||||
2024/06/16 17:27:47 - Limit custom ua late open source full version
|
||||
2024/07/08 21:50:12 - Open the full version of the custom UA
|
||||
2025/03/05 15:55:42 - Perf Post Method Generate
|
||||
2025/04/27 17:15:26 - Fix mypy type error
|
||||
-------------------------------------------------
|
||||
"""
|
||||
|
||||
import random
|
||||
import time
|
||||
from typing import Callable, Dict, List, Union
|
||||
|
||||
from gmssl import func, sm3 # type: ignore[import-untyped]
|
||||
|
||||
|
||||
class StringProcessor:
|
||||
"""
|
||||
StringProcessor 类用于计算ABogus算法中所需的字符串处理方法。
|
||||
包括字符串和 ASCII 码之间的转换、无符号右移运算等。
|
||||
|
||||
类方法:
|
||||
|
||||
to_ord_str(s: str) -> str:
|
||||
将字符串转换为 ASCII 码字符串。
|
||||
|
||||
to_ord_array(s: str) -> List[int]:
|
||||
将字符串转换为 ASCII 码列表。
|
||||
|
||||
to_char_str(s: str) -> str:
|
||||
将 ASCII 码列表转换为字符串。
|
||||
|
||||
to_char_array(s: str) -> List[int]:
|
||||
将字符串转换为 ASCII 码列表。
|
||||
|
||||
js_shift_right(val: int, n: int) -> int:
|
||||
实现 JavaScript 中的无符号右移运算。
|
||||
|
||||
generate_random_bytes(length: int = 3) -> str:
|
||||
生成一组伪随机字节字符串，用于混淆数据。
|
||||
|
||||
使用示例:
|
||||
```python
|
||||
# 将字符串转换为 ASCII 码字符串
|
||||
ord_str = StringProcessor.to_ord_str("Hello, World!")
|
||||
print(ord_str)
|
||||
|
||||
# 将字符串转换为 ASCII 码列表
|
||||
ord_array = StringProcessor.to_ord_array("Hello, World!")
|
||||
print(ord_array)
|
||||
|
||||
# 将 ASCII 码列表转换为字符串
|
||||
char_str = StringProcessor.to_char_str(ord_array)
|
||||
print(char_str)
|
||||
|
||||
# 将字符串转换为 ASCII 码列表
|
||||
char_array = StringProcessor.to_char_array("Hello, World!")
|
||||
print(char_array)
|
||||
|
||||
# 实现 JavaScript 中的无符号右移运算
|
||||
shifted_val = StringProcessor.js_shift_right(10, 2)
|
||||
print(shifted_val)
|
||||
|
||||
# 生成一组伪随机字节字符串
|
||||
random_bytes = StringProcessor.generate_random_bytes(3)
|
||||
print(random_bytes)
|
||||
```
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def to_ord_str(s: str) -> str:
|
||||
"""
|
||||
将字符串转换为 ASCII 码字符串 (Convert a string to an ASCII code string).
|
||||
|
||||
Args:
|
||||
s (str): 输入字符串 (Input string).
|
||||
|
||||
Returns:
|
||||
str: 转换后的 ASCII 码字符串 (Converted ASCII code string).
|
||||
"""
|
||||
return "".join([chr(ord(char)) for char in s])
|
||||
|
||||
@staticmethod
|
||||
def to_ord_array(s: str) -> List[int]:
|
||||
"""
|
||||
将字符串转换为 ASCII 码列表 (Convert a string to a list of ASCII codes).
|
||||
|
||||
Args:
|
||||
s (str): 输入字符串 (Input string).
|
||||
|
||||
Returns:
|
||||
List[int]: 转换后的 ASCII 码列表 (Converted list of ASCII codes).
|
||||
"""
|
||||
return [ord(char) for char in s]
|
||||
|
||||
@staticmethod
|
||||
def to_char_str(s: Union[str, List[int]]) -> str:
|
||||
"""
|
||||
将 ASCII 码列表转换为字符串 (Convert a list of ASCII codes to a string).
|
||||
|
||||
Args:
|
||||
s (Union[str, List[int]]): ASCII 码列表或字符串 (List of ASCII codes or string).
|
||||
|
||||
Returns:
|
||||
str: 转换后的字符串 (Converted string).
|
||||
"""
|
||||
if isinstance(s, list):
|
||||
return "".join([chr(i) for i in s])
|
||||
elif isinstance(s, str):
|
||||
return s
|
||||
else:
|
||||
raise TypeError(
|
||||
"Expected str or List[int], got {}".format(type(s).__name__)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def to_char_array(s: str) -> List[int]:
|
||||
"""
|
||||
将字符串转换为 ASCII 码列表 (Convert a string to a list of ASCII codes).
|
||||
|
||||
Args:
|
||||
s (str): 输入字符串 (Input string).
|
||||
|
||||
Returns:
|
||||
List[int]: 转换后的 ASCII 码列表 (Converted list of ASCII codes).
|
||||
"""
|
||||
return [ord(char) for char in s]
|
||||
|
||||
@staticmethod
|
||||
def js_shift_right(val: int, n: int) -> int:
|
||||
"""
|
||||
实现 JavaScript 中的无符号右移运算 (Implement the unsigned right shift operation in JavaScript).
|
||||
|
||||
Args:
|
||||
val (int): 输入值 (Input value).
|
||||
n (int): 右移位数 (Number of bits to shift right).
|
||||
|
||||
Returns:
|
||||
int: 右移后的值 (Value after right shift).
|
||||
"""
|
||||
return (val % 0x100000000) >> n
|
||||
|
||||
@staticmethod
|
||||
def generate_random_bytes(length: int = 3) -> str:
|
||||
"""
|
||||
生成一组伪随机字节字符串，用于混淆数据 (Generate a pseudo-random byte string to obfuscate the data).
|
||||
|
||||
Args:
|
||||
length (int): 生成的字节序列长度 (Length of the byte sequence to generate).
|
||||
|
||||
Returns:
|
||||
str: 生成的伪随机字节字符串 (Generated pseudo-random byte string).
|
||||
"""
|
||||
|
||||
def generate_byte_sequence() -> List[str]:
|
||||
_rd = int(random.random() * 10000)
|
||||
return [
|
||||
chr(((_rd & 255) & 170) | 1),
|
||||
chr(((_rd & 255) & 85) | 2),
|
||||
chr((StringProcessor.js_shift_right(_rd, 8) & 170) | 5),
|
||||
chr((StringProcessor.js_shift_right(_rd, 8) & 85) | 40),
|
||||
]
|
||||
|
||||
result = []
|
||||
for _ in range(length):
|
||||
result.extend(generate_byte_sequence())
|
||||
|
||||
return "".join(result)
|
||||
|
||||
|
||||
class CryptoUtility:
|
||||
"""
|
||||
CryptoUtility 类用于提供加密和编码的工具方法，包括 SM3 哈希、添加盐值、Base64 编码和 RC4 加密等。
|
||||
|
||||
类属性:
|
||||
salt (str): 加密盐值 (Encryption salt).
|
||||
base64_alphabet (List[str]): 自定义 Base64 字符表 (Custom Base64 alphabet).
|
||||
|
||||
类方法:
|
||||
sm3_to_array(input_data: Union[str, List[int]]) -> List[int]:
|
||||
计算请求体的 SM3 哈希值，并将结果转换为整数数组。
|
||||
|
||||
add_salt(param: str) -> str:
|
||||
为字符串参数添加盐值。
|
||||
|
||||
process_param(param: Union[str, List[int]], add_salt: bool) -> Union[str, List[int]]:
|
||||
处理输入参数，根据需要添加盐值。
|
||||
|
||||
params_to_array(param: Union[str, List[int]], add_salt: bool = True) -> List[int]:
|
||||
获取输入参数的哈希数组。
|
||||
|
||||
transform_bytes(bytes_list: List[int]) -> str:
|
||||
对输入的字节列表进行加密/解密操作，返回处理后的字符串。
|
||||
|
||||
base64_encode(input_string: str, selected_alphabet: int = 0) -> str:
|
||||
使用自定义字符表对输入字符串进行 Base64 编码。
|
||||
|
||||
abogus_encode(abogus_bytes_str: str, selected_alphabet: int) -> str:
|
||||
对输入的字节字符串进行自定义 Base64 编码，并添加位移和填充。
|
||||
|
||||
rc4_encrypt(key: bytes, plaintext: str) -> bytes:
|
||||
使用 RC4 算法加密数据。
|
||||
|
||||
使用示例:
|
||||
```python
|
||||
# 计算请求体的 SM3 哈希值
|
||||
sm3_hash = CryptoUtility.sm3_to_array("Hello, World!")
|
||||
print(sm3_hash)
|
||||
|
||||
# 为字符串参数添加盐值
|
||||
salted_param = CryptoUtility.add_salt("Hello, World!")
|
||||
print(salted_param)
|
||||
|
||||
# 获取输入参数的哈希数组
|
||||
hash_array = CryptoUtility.params_to_array("Hello, World!")
|
||||
print(hash_array)
|
||||
|
||||
# 对输入的字节列表进行加密/解密操作
|
||||
encrypted_str = CryptoUtility.transform_bytes([72, 101, 108, 108, 111, 44, 32, 87, 111, 114, 108, 100, 33])
|
||||
print(encrypted_str)
|
||||
|
||||
# 使用自定义字符表对输入字符串进行 Base64 编码
|
||||
base64_str = CryptoUtility.base64_encode("Hello, World!")
|
||||
print(base64_str)
|
||||
|
||||
# 对输入的字节字符串进行自定义 Base64 编码，并添加位移和填充
|
||||
abogus_str = CryptoUtility.abogus_encode("Hello, World!", 0)
|
||||
print(abogus_str)
|
||||
|
||||
# 使用 RC4 算法加密数据
|
||||
key = b"key"
|
||||
plaintext = "Hello, World!"
|
||||
ciphertext = CryptoUtility.rc4_encrypt(key, plaintext)
|
||||
print(ciphertext)
|
||||
```
|
||||
"""
|
||||
|
||||
def __init__(self, salt: str, custom_base64_alphabet: List[str]):
|
||||
"""
|
||||
初始化 CryptoUtility 类
|
||||
Initialize the CryptoUtility class.
|
||||
|
||||
Args:
|
||||
salt (str): 加密盐值 (Encryption salt).
|
||||
custom_base64_alphabet (List[str]): 自定义 Base64 字符表 (Custom Base64 alphabet).
|
||||
"""
|
||||
self.salt = salt
|
||||
self.base64_alphabet = custom_base64_alphabet
|
||||
|
||||
# fmt: off
|
||||
self.big_array = [
|
||||
121, 243, 55, 234, 103, 36, 47, 228, 30, 231, 106, 6, 115, 95, 78, 101, 250, 207, 198, 50,
|
||||
139, 227, 220, 105, 97, 143, 34, 28, 194, 215, 18, 100, 159, 160, 43, 8, 169, 217, 180, 120,
|
||||
247, 45, 90, 11, 27, 197, 46, 3, 84, 72, 5, 68, 62, 56, 221, 75, 144, 79, 73, 161,
|
||||
178, 81, 64, 187, 134, 117, 186, 118, 16, 241, 130, 71, 89, 147, 122, 129, 65, 40, 88, 150,
|
||||
110, 219, 199, 255, 181, 254, 48, 4, 195, 248, 208, 32, 116, 167, 69, 201, 17, 124, 125, 104,
|
||||
96, 83, 80, 127, 236, 108, 154, 126, 204, 15, 20, 135, 112, 158, 13, 1, 188, 164, 210, 237,
|
||||
222, 98, 212, 77, 253, 42, 170, 202, 26, 22, 29, 182, 251, 10, 173, 152, 58, 138, 54, 141,
|
||||
185, 33, 157, 31, 252, 132, 233, 235, 102, 196, 191, 223, 240, 148, 39, 123, 92, 82, 128, 109,
|
||||
57, 24, 38, 113, 209, 245, 2, 119, 153, 229, 189, 214, 230, 174, 232, 63, 52, 205, 86, 140,
|
||||
66, 175, 111, 171, 246, 133, 238, 193, 99, 60, 74, 91, 225, 51, 76, 37, 145, 211, 166, 151,
|
||||
213, 206, 0, 200, 244, 176, 218, 44, 184, 172, 49, 216, 93, 168, 53, 21, 183, 41, 67, 85,
|
||||
224, 155, 226, 242, 87, 177, 146, 70, 190, 12, 162, 19, 137, 114, 25, 165, 163, 192, 23, 59,
|
||||
9, 94, 179, 107, 35, 7, 142, 131, 239, 203, 149, 136, 61, 249, 14, 156
|
||||
]
|
||||
# fmt: on
|
||||
|
||||
@staticmethod
|
||||
def sm3_to_array(input_data: Union[str, List[int]]) -> List[int]:
|
||||
"""
|
||||
计算请求体的 SM3 哈希值，并将结果转换为整数数组 (Calculate the SM3 hash value of the request body and convert the result to an array of integers).
|
||||
|
||||
Args:
|
||||
input_data (Union[str, List[int]]): 输入数据 (Input data).
|
||||
|
||||
Returns:
|
||||
List[int]: 哈希值的整数数组 (Array of integers representing the hash value).
|
||||
"""
|
||||
# 如果输入是字符串，则将其编码为字节数组
|
||||
if isinstance(input_data, str):
|
||||
input_data_bytes = input_data.encode("utf-8")
|
||||
else:
|
||||
input_data_bytes = bytes(input_data) # 将 List[int] 转换为字节数组
|
||||
|
||||
# 将字节数组转换为适合 sm3.sm3_hash 函数处理的列表格式
|
||||
hex_result = sm3.sm3_hash(func.bytes_to_list(input_data_bytes))
|
||||
|
||||
# 将十六进制字符串结果转换为十进制整数列表
|
||||
return [int(hex_result[i : i + 2], 16) for i in range(0, len(hex_result), 2)]
|
||||
|
||||
def add_salt(self, param: str) -> str:
|
||||
"""
|
||||
为字符串参数添加盐值 (Add salt to the string parameter).
|
||||
|
||||
Args:
|
||||
param (str): 输入字符串 (Input string).
|
||||
|
||||
Returns:
|
||||
str: 添加盐值后的字符串 (String with added salt).
|
||||
"""
|
||||
return param + self.salt
|
||||
|
||||
def process_param(
|
||||
self, param: Union[str, List[int]], add_salt: bool
|
||||
) -> Union[str, List[int]]:
|
||||
"""
|
||||
处理输入参数，根据需要添加盐值 (Process input parameter and add salt if needed).
|
||||
|
||||
Args:
|
||||
param (Union[str, List[int]]): 输入参数 (Input parameter).
|
||||
add_salt (bool): 是否添加盐值 (Whether to add salt).
|
||||
|
||||
Returns:
|
||||
Union[str, List[int]]: 处理后的参数 (Processed parameter).
|
||||
"""
|
||||
if isinstance(param, str) and add_salt:
|
||||
param = self.add_salt(param)
|
||||
return param
|
||||
|
||||
def params_to_array(
|
||||
self, param: Union[str, List[int]], add_salt: bool = True
|
||||
) -> List[int]:
|
||||
"""
|
||||
获取输入参数的哈希数组 (Get the hash array of the input parameter).
|
||||
|
||||
Args:
|
||||
param (Union[str, List[int]]): 输入参数 (Input parameter).
|
||||
add_salt (bool): 是否添加盐值 (Whether to add salt).
|
||||
|
||||
Returns:
|
||||
List[int]: 哈希数组 (Hash array).
|
||||
"""
|
||||
processed_param = self.process_param(param, add_salt)
|
||||
return self.sm3_to_array(processed_param)
|
||||
|
||||
def transform_bytes(self, bytes_list: List[int]) -> str:
|
||||
"""
|
||||
对输入的字节列表进行加密/解密操作，返回处理后的字符串 (Encrypt/decrypt the input byte list and return the processed string).
|
||||
|
||||
Args:
|
||||
bytes_list (List[int]): 输入的字节列表 (Input byte list).
|
||||
|
||||
Returns:
|
||||
str: 处理后的字符串 (Processed string).
|
||||
"""
|
||||
# 将字节列表转换为字符字符串
|
||||
bytes_str = StringProcessor.to_char_str(bytes_list)
|
||||
result_str = []
|
||||
index_b = self.big_array[1]
|
||||
initial_value = 0
|
||||
value_e = 0
|
||||
|
||||
for index, char in enumerate(bytes_str):
|
||||
if index == 0:
|
||||
initial_value = self.big_array[index_b]
|
||||
sum_initial = index_b + initial_value
|
||||
|
||||
self.big_array[1] = initial_value
|
||||
self.big_array[index_b] = index_b
|
||||
else:
|
||||
sum_initial = initial_value + value_e
|
||||
|
||||
char_value = ord(char)
|
||||
sum_initial %= len(self.big_array)
|
||||
value_f = self.big_array[sum_initial]
|
||||
encrypted_char = char_value ^ value_f
|
||||
result_str.append(chr(encrypted_char))
|
||||
|
||||
# 交换数组元素
|
||||
value_e = self.big_array[(index + 2) % len(self.big_array)]
|
||||
sum_initial = (index_b + value_e) % len(self.big_array)
|
||||
initial_value = self.big_array[sum_initial]
|
||||
self.big_array[sum_initial] = self.big_array[
|
||||
(index + 2) % len(self.big_array)
|
||||
]
|
||||
self.big_array[(index + 2) % len(self.big_array)] = initial_value
|
||||
index_b = sum_initial
|
||||
|
||||
return "".join(result_str)
|
||||
|
||||
def base64_encode(self, input_string: str, selected_alphabet: int = 0) -> str:
|
||||
"""
|
||||
使用自定义字符表对输入字符串进行 Base64 编码 (Encode the input string using a custom Base64 alphabet).
|
||||
|
||||
Args:
|
||||
input_string (str): 输入字符串 (Input string).
|
||||
selected_alphabet (int): 选择的自定义 Base64 字符表索引 (Selected custom Base64 alphabet index).
|
||||
|
||||
Returns:
|
||||
str: 编码后的字符串 (Encoded string).
|
||||
"""
|
||||
|
||||
# 将输入字符串转换为ASCII码的二进制形式
|
||||
binary_string = "".join(["{:08b}".format(ord(char)) for char in input_string])
|
||||
|
||||
# 补全二进制字符串使其长度为6的倍数
|
||||
padding_length = (6 - len(binary_string) % 6) % 6
|
||||
binary_string += "0" * padding_length
|
||||
|
||||
# 将二进制字符串分割为6位一组
|
||||
base64_indices = [
|
||||
int(binary_string[i : i + 6], 2) for i in range(0, len(binary_string), 6)
|
||||
]
|
||||
|
||||
# 根据自定义字符表生成输出字符串
|
||||
output_string = "".join(
|
||||
[self.base64_alphabet[selected_alphabet][index] for index in base64_indices]
|
||||
)
|
||||
|
||||
# 添加等号填充，使符合 Base64 编码规范
|
||||
output_string += "=" * (padding_length // 2)
|
||||
|
||||
return output_string
|
||||
|
||||
def abogus_encode(self, abogus_bytes_str: str, selected_alphabet: int) -> str:
|
||||
"""
|
||||
对输入的字节字符串进行自定义 Base64 编码，并添加位移和填充 (Encode the input byte string using a custom Base64 alphabet, and add shifts and padding).
|
||||
|
||||
Args:
|
||||
abogus_bytes_str (str): 输入的字节字符串 (Input byte string).
|
||||
selected_alphabet (int): 选择的自定义 Base64 字符表索引 (Selected custom Base64 alphabet index).
|
||||
|
||||
Returns:
|
||||
str: 编码后的字符串 (Encoded string).
|
||||
"""
|
||||
abogus = []
|
||||
|
||||
for i in range(0, len(abogus_bytes_str), 3):
|
||||
if i + 2 < len(abogus_bytes_str):
|
||||
n = (
|
||||
(ord(abogus_bytes_str[i]) << 16)
|
||||
| (ord(abogus_bytes_str[i + 1]) << 8)
|
||||
| ord(abogus_bytes_str[i + 2])
|
||||
)
|
||||
elif i + 1 < len(abogus_bytes_str):
|
||||
n = (ord(abogus_bytes_str[i]) << 16) | (
|
||||
ord(abogus_bytes_str[i + 1]) << 8
|
||||
)
|
||||
else:
|
||||
n = ord(abogus_bytes_str[i]) << 16
|
||||
|
||||
for j, k in zip(range(18, -1, -6), (0xFC0000, 0x03F000, 0x0FC0, 0x3F)):
|
||||
if j == 6 and i + 1 >= len(abogus_bytes_str):
|
||||
break
|
||||
if j == 0 and i + 2 >= len(abogus_bytes_str):
|
||||
break
|
||||
abogus.append(self.base64_alphabet[selected_alphabet][(n & k) >> j])
|
||||
|
||||
abogus.append("=" * ((4 - len(abogus) % 4) % 4))
|
||||
return "".join(abogus)
|
||||
|
||||
@staticmethod
|
||||
def rc4_encrypt(key: bytes, plaintext: str) -> bytes:
|
||||
"""
|
||||
使用 RC4 算法加密数据 (Encrypt data using the RC4 algorithm).
|
||||
|
||||
Args:
|
||||
key (bytes): 加密密钥 (Encryption key).
|
||||
plaintext (str): 明文数据 (Plaintext data).
|
||||
|
||||
Returns:
|
||||
bytes: 加密后的数据 (Encrypted data).
|
||||
"""
|
||||
S = list(range(256))
|
||||
j = 0
|
||||
for i in range(256):
|
||||
j = (j + S[i] + key[i % len(key)]) % 256
|
||||
S[i], S[j] = S[j], S[i]
|
||||
|
||||
i = j = 0
|
||||
ciphertext = []
|
||||
for char in plaintext:
|
||||
i = (i + 1) % 256
|
||||
j = (j + S[i]) % 256
|
||||
S[i], S[j] = S[j], S[i]
|
||||
K = S[(S[i] + S[j]) % 256]
|
||||
ciphertext.append(ord(char) ^ K)
|
||||
|
||||
return bytes(ciphertext)
|
||||
|
||||
|
||||
class BrowserFingerprintGenerator:
|
||||
"""
|
||||
BrowserFingerprintGenerator 用于生成模拟的浏览器指纹信息，用于在不同浏览器环境中进行测试和数据采集。
|
||||
|
||||
方法:
|
||||
generate_fingerprint(browser_type="Edge"):
|
||||
根据指定的浏览器类型生成浏览器指纹。
|
||||
|
||||
generate_chrome_fingerprint():
|
||||
生成 Chrome 浏览器指纹。
|
||||
|
||||
generate_firefox_fingerprint():
|
||||
生成 Firefox 浏览器指纹。
|
||||
|
||||
generate_safari_fingerprint():
|
||||
生成 Safari 浏览器指纹。
|
||||
|
||||
generate_edge_fingerprint():
|
||||
生成 Edge 浏览器指纹。
|
||||
|
||||
_generate_fingerprint(platform="Win32"):
|
||||
根据给定的参数生成浏览器指纹字符串。
|
||||
|
||||
使用示例:
|
||||
```python
|
||||
chrome_fp = BrowserFingerprintGenerator.generate_fingerprint("Chrome")
|
||||
print(chrome_fp)
|
||||
```
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def generate_fingerprint(cls, browser_type: str = "Edge") -> str:
|
||||
"""
|
||||
根据指定的浏览器类型生成浏览器指纹。 (Generate a browser fingerprint based on the specified browser type.)
|
||||
|
||||
Args:
|
||||
browser_type (str): 浏览器类型 (Browser type).
|
||||
|
||||
Returns:
|
||||
str: 生成的浏览器指纹字符串 (Generated browser fingerprint string).
|
||||
"""
|
||||
browsers: Dict[str, Callable[[], str]] = {
|
||||
"Chrome": cls.generate_chrome_fingerprint,
|
||||
"Firefox": cls.generate_firefox_fingerprint,
|
||||
"Safari": cls.generate_safari_fingerprint,
|
||||
"Edge": cls.generate_edge_fingerprint,
|
||||
}
|
||||
return browsers.get(browser_type, cls.generate_chrome_fingerprint)()
|
||||
|
||||
@classmethod
|
||||
def generate_chrome_fingerprint(cls) -> str:
|
||||
return cls._generate_fingerprint(platform="Win32")
|
||||
|
||||
@classmethod
|
||||
def generate_firefox_fingerprint(cls) -> str:
|
||||
return cls._generate_fingerprint(platform="Win32")
|
||||
|
||||
@classmethod
|
||||
def generate_safari_fingerprint(cls) -> str:
|
||||
return cls._generate_fingerprint(platform="MacIntel")
|
||||
|
||||
@classmethod
|
||||
def generate_edge_fingerprint(cls) -> str:
|
||||
return cls._generate_fingerprint(platform="Win32")
|
||||
|
||||
@staticmethod
|
||||
def _generate_fingerprint(platform: str) -> str:
|
||||
"""
|
||||
根据给定的参数生成浏览器指纹字符串。 (Generate a browser fingerprint string based on the given parameters.)
|
||||
|
||||
Args:
|
||||
platform (str): 操作系统平台 (Operating system platform).
|
||||
|
||||
Returns:
|
||||
str: 生成的浏览器指纹字符串 (Generated browser fingerprint string).
|
||||
"""
|
||||
inner_width = random.randint(1024, 1920)
|
||||
inner_height = random.randint(768, 1080)
|
||||
outer_width = inner_width + random.randint(24, 32)
|
||||
outer_height = inner_height + random.randint(75, 90)
|
||||
screen_x = 0
|
||||
screen_y = random.choice([0, 30])
|
||||
size_width = random.randint(1024, 1920)
|
||||
size_height = random.randint(768, 1080)
|
||||
avail_width = random.randint(1280, 1920)
|
||||
avail_height = random.randint(800, 1080)
|
||||
|
||||
fingerprint = (
|
||||
f"{inner_width}|{inner_height}|{outer_width}|{outer_height}|"
|
||||
f"{screen_x}|{screen_y}|0|0|{size_width}|{size_height}|"
|
||||
f"{avail_width}|{avail_height}|{inner_width}|{inner_height}|24|24|{platform}"
|
||||
)
|
||||
return fingerprint
|
||||
|
||||
|
||||
class ABogus:
|
||||
"""
|
||||
ABogus 类用于生成 ABogus 参数。
|
||||
|
||||
类属性:
|
||||
array1 (List[int]): 加密请求体 (Encrypted request body).
|
||||
array2 (List[int]): 加密请求头 (Encrypted request header).
|
||||
array3 (List[int]): 加密 UA (Encrypted User-Agent).
|
||||
aid (int): AID 值 (AID value).
|
||||
pageId (int): 页面 ID (Page ID).
|
||||
salt (str): 加密盐值 (Encryption salt).
|
||||
options (List[int]): 请求选项 (Request options).
|
||||
ua_key (bytes): UA 加密密钥 (UA encryption key).
|
||||
character (str): 自定义 Base64 字符表 (Custom Base64 alphabet).
|
||||
character2 (str): 自定义 Base64 字符表 (Custom Base64 alphabet).
|
||||
character_list (List[str]): 自定义 Base64 字符表列表 (List of custom Base64 alphabets).
|
||||
crypto_utility (CryptoUtility): 加密工具类 (Encryption utility).
|
||||
user_agent (str): 自定义 UA (Custom User-Agent).
|
||||
browser_fp (str): 浏览器指纹 (Browser fingerprint).
|
||||
sort_index (List[int]): 排序索引 (Sort index).
|
||||
sort_index_2 (List[int]): 排序索引 (Sort index).
|
||||
|
||||
说明：
|
||||
options 参数用于指定请求的类型，GET 请求使用 [0, 1, 8]，POST 请求使用 [0, 1, 14]。14兼容8，POST同样可以编码params，故写死。
|
||||
(The options parameter is used to specify the type of request. GET requests use [0, 1, 8], and POST requests use [0, 1, 14]. 14 is compatible with 8, and POST can also encode params, so it is hardcoded.)
|
||||
|
||||
方法:
|
||||
encode_data(data: str, alphabet_index: int = 0) -> str:
|
||||
使用指定的字符表对数据进行 Base64 编码 (Encode the data using the specified Base64 alphabet).
|
||||
|
||||
generate_abogus(params: str, request: str = "") -> str:
|
||||
生成 ABogus 参数 (Generate the ABogus parameter).
|
||||
|
||||
使用示例:
|
||||
```python
|
||||
# 生成 ABogus 参数，置空使用默认 UA 和浏览器指纹
|
||||
abogus = ABogus(user_agent="xxx", fp="xxx")
|
||||
abogus_param = abogus.generate_abogus("device_platform=webapp&aid=6383&channel=channel_pc_web&aweme_id=7380308675841297704……省略……")
|
||||
print(abogus_param[1])
|
||||
```
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
fp: str = "",
|
||||
user_agent: str = "",
|
||||
options: List[int] = [0, 1, 14],
|
||||
):
|
||||
self.aid = 6383
|
||||
self.pageId = 0 # 1.0.1.19 -> 6241
|
||||
self.salt = "cus" # 1.0.1.19 -> 加密盐 # dhzx
|
||||
self.boe = False
|
||||
self.ddrt = 8.5
|
||||
self.ic = 8.5
|
||||
self.paths = [
|
||||
"^/webcast/",
|
||||
"^/aweme/v1/",
|
||||
"^/aweme/v2/",
|
||||
"/v1/message/send",
|
||||
"^/live/",
|
||||
"^/captcha/",
|
||||
"^/ecom/",
|
||||
]
|
||||
self.array1: list[int] = [] # 加密请求体
|
||||
self.array2: list[int] = [] # 加密请求头 为空
|
||||
self.array3: list[int] = [] # 加密UA
|
||||
self.options = options # GET [0, 1, 8] POST [0, 1, 14]
|
||||
self.ua_key = b"\x00\x01\x0e" # ua加密key
|
||||
|
||||
self.character = (
|
||||
"Dkdpgh2ZmsQB80/MfvV36XI1R45-WUAlEixNLwoqYTOPuzKFjJnry79HbGcaStCe"
|
||||
)
|
||||
self.character2 = (
|
||||
"ckdp1h4ZKsUB80/Mfvw36XIgR25+WQAlEi7NLboqYTOPuzmFjJnryx9HVGDaStCe"
|
||||
)
|
||||
self.character_list = [self.character, self.character2] # 自定义base64字符表
|
||||
|
||||
self.crypto_utility = CryptoUtility(
|
||||
self.salt, self.character_list
|
||||
) # 加密工具类
|
||||
|
||||
self.user_agent = (
|
||||
user_agent
|
||||
if user_agent is not None and user_agent != ""
|
||||
else "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0"
|
||||
) # 自定义ua，为空则设置一个默认ua
|
||||
|
||||
self.browser_fp = (
|
||||
fp
|
||||
if fp is not None and fp != ""
|
||||
else BrowserFingerprintGenerator.generate_fingerprint("Edge")
|
||||
) # 自定义浏览器指纹，为空则生成Edge指纹
|
||||
|
||||
# fmt: off
|
||||
self.sort_index = [
|
||||
18, 20, 52, 26, 30, 34, 58, 38, 40, 53, 42, 21, 27, 54, 55, 31, 35, 57, 39, 41, 43, 22, 28,
|
||||
32, 60, 36, 23, 29, 33, 37, 44, 45, 59, 46, 47, 48, 49, 50, 24, 25, 65, 66, 70, 71
|
||||
]
|
||||
self.sort_index_2 = [
|
||||
18, 20, 26, 30, 34, 38, 40, 42, 21, 27, 31, 35, 39, 41, 43, 22, 28, 32, 36, 23, 29, 33, 37,
|
||||
44, 45, 46, 47, 48, 49, 50, 24, 25, 52, 53, 54, 55, 57, 58, 59, 60, 65, 66, 70, 71
|
||||
]
|
||||
# fmt: on
|
||||
|
||||
def encode_data(self, data: str, alphabet_index: int = 0) -> str:
|
||||
"""
|
||||
使用指定的字符表对数据进行 Base64 编码 (Encode the data using the specified Base64 alphabet).
|
||||
|
||||
Args:
|
||||
data (str): 输入数据 (Input data).
|
||||
alphabet_index (int): 自定义字符表索引 (Custom alphabet index).
|
||||
|
||||
Returns:
|
||||
str: 编码后的数据 (Encoded data).
|
||||
"""
|
||||
return self.crypto_utility.abogus_encode(data, alphabet_index)
|
||||
|
||||
def generate_abogus(self, params: str, body: str = "") -> tuple:
|
||||
"""
|
||||
生成 abogus 参数 (Generate the ABogus parameter).
|
||||
|
||||
Args:
|
||||
params (str): 请求参数 (Request parameters).
|
||||
body (str): 请求体，GET接口则为空 (Request body, empty for GET interfaces).
|
||||
|
||||
Returns:
|
||||
tuple: params 生成的 abogus 参数 和 ua (ABogus parameter generated by params and ua).
|
||||
"""
|
||||
ab_dir = {
|
||||
8: 3, # 固定
|
||||
15: {
|
||||
"aid": self.aid,
|
||||
"pageId": self.pageId,
|
||||
"boe": self.boe,
|
||||
"ddrt": self.ddrt,
|
||||
"paths": self.paths,
|
||||
"track": {"mode": 0, "delay": 300, "paths": []},
|
||||
"dump": True,
|
||||
"rpU": "",
|
||||
},
|
||||
18: 44,
|
||||
19: [1, 0, 1, 0, 1],
|
||||
66: 0, # 固定
|
||||
69: 0, # 固定
|
||||
70: 0, # 固定
|
||||
71: 0, # 固定
|
||||
}
|
||||
|
||||
# 开始加密时间
|
||||
start_encryption = int(time.time() * 1000)
|
||||
|
||||
# params参数加盐加密
|
||||
array1 = self.crypto_utility.params_to_array(
|
||||
self.crypto_utility.params_to_array(params)
|
||||
)
|
||||
array2 = self.crypto_utility.params_to_array(
|
||||
self.crypto_utility.params_to_array(body)
|
||||
)
|
||||
array3 = self.crypto_utility.params_to_array(
|
||||
self.crypto_utility.base64_encode(
|
||||
"".join(
|
||||
[
|
||||
chr(b)
|
||||
for b in self.crypto_utility.rc4_encrypt(
|
||||
self.ua_key, self.user_agent
|
||||
)
|
||||
]
|
||||
),
|
||||
1,
|
||||
),
|
||||
add_salt=False,
|
||||
)
|
||||
|
||||
# 结束加密时间
|
||||
end_encryption = int(time.time() * 1000)
|
||||
|
||||
# 插入加密开始时间
|
||||
ab_dir[20] = (start_encryption >> 24) & 255
|
||||
ab_dir[21] = (start_encryption >> 16) & 255
|
||||
ab_dir[22] = (start_encryption >> 8) & 255
|
||||
ab_dir[23] = start_encryption & 255
|
||||
ab_dir[24] = int(start_encryption / 256 / 256 / 256 / 256) >> 0
|
||||
ab_dir[25] = int(start_encryption / 256 / 256 / 256 / 256 / 256) >> 0
|
||||
|
||||
# 插入请求头配置
|
||||
ab_dir[26] = (self.options[0] >> 24) & 255
|
||||
ab_dir[27] = (self.options[0] >> 16) & 255
|
||||
ab_dir[28] = (self.options[0] >> 8) & 255
|
||||
ab_dir[29] = self.options[0] & 255
|
||||
|
||||
# 插入请求方法
|
||||
ab_dir[30] = int(self.options[1] / 256) & 255
|
||||
ab_dir[31] = (self.options[1] % 256) & 255
|
||||
ab_dir[32] = (self.options[1] >> 24) & 255
|
||||
ab_dir[33] = (self.options[1] >> 16) & 255
|
||||
|
||||
# 插入请求头加密
|
||||
ab_dir[34] = (self.options[2] >> 24) & 255
|
||||
ab_dir[35] = (self.options[2] >> 16) & 255
|
||||
ab_dir[36] = (self.options[2] >> 8) & 255
|
||||
ab_dir[37] = self.options[2] & 255
|
||||
|
||||
# 插入请求体加密
|
||||
ab_dir[38] = array1[21]
|
||||
ab_dir[39] = array1[22]
|
||||
# 插入body加密
|
||||
ab_dir[40] = array2[21]
|
||||
ab_dir[41] = array2[22]
|
||||
# 插入ua加密
|
||||
ab_dir[42] = array3[23]
|
||||
ab_dir[43] = array3[24]
|
||||
|
||||
# 插入加密结束时间
|
||||
ab_dir[44] = (end_encryption >> 24) & 255
|
||||
ab_dir[45] = (end_encryption >> 16) & 255
|
||||
ab_dir[46] = (end_encryption >> 8) & 255
|
||||
ab_dir[47] = end_encryption & 255
|
||||
ab_dir[48] = ab_dir[8]
|
||||
ab_dir[49] = int(end_encryption / 256 / 256 / 256 / 256) >> 0
|
||||
ab_dir[50] = int(end_encryption / 256 / 256 / 256 / 256 / 256) >> 0
|
||||
|
||||
# 插入固定值
|
||||
ab_dir[51] = (self.pageId >> 24) & 255
|
||||
ab_dir[52] = (self.pageId >> 16) & 255
|
||||
ab_dir[53] = (self.pageId >> 8) & 255
|
||||
ab_dir[54] = self.pageId & 255
|
||||
ab_dir[55] = self.pageId
|
||||
ab_dir[56] = self.aid
|
||||
ab_dir[57] = self.aid & 255
|
||||
ab_dir[58] = (self.aid >> 8) & 255
|
||||
ab_dir[59] = (self.aid >> 16) & 255
|
||||
ab_dir[60] = (self.aid >> 24) & 255
|
||||
|
||||
# 插入浏览器指纹
|
||||
ab_dir[64] = len(self.browser_fp)
|
||||
ab_dir[65] = len(self.browser_fp)
|
||||
|
||||
# 获取 ab_dir 中 sort_index 的值
|
||||
sorted_values: List[int] = []
|
||||
for i in self.sort_index:
|
||||
value = ab_dir.get(i, 0)
|
||||
if isinstance(value, (int, float)):
|
||||
sorted_values.append(int(value))
|
||||
elif isinstance(value, str) and value.isdigit():
|
||||
sorted_values.append(int(value))
|
||||
else:
|
||||
sorted_values.append(0) # 对于不能转换为整数的值使用0作为默认值
|
||||
|
||||
# 将浏览器指纹转换为 ASCII 码列表
|
||||
edge_fp_array = StringProcessor.to_char_array(self.browser_fp)
|
||||
|
||||
# 将浏览器指纹长度的低 8 位作为异或值
|
||||
ab_xor = (len(self.browser_fp) & 255) >> 8 & 255
|
||||
|
||||
# 进行异或计算
|
||||
for index in range(len(self.sort_index_2) - 1):
|
||||
if index == 0:
|
||||
# 防御性编程，确保返回的是整数
|
||||
value = ab_dir.get(self.sort_index_2[index], 0)
|
||||
if isinstance(value, (int, float)):
|
||||
ab_xor = int(value)
|
||||
else:
|
||||
ab_xor = 0
|
||||
# 同样防御性地处理第二个值
|
||||
value = ab_dir.get(self.sort_index_2[index + 1], 0)
|
||||
if isinstance(value, (int, float)):
|
||||
ab_xor ^= int(value)
|
||||
|
||||
# 安全转换edge_fp_array中的值为整数
|
||||
clean_edge_fp = []
|
||||
for x in edge_fp_array:
|
||||
if isinstance(x, (int, float, str)) and str(x).isdigit():
|
||||
clean_edge_fp.append(int(x))
|
||||
else:
|
||||
# 对于无法转换的值使用0填充
|
||||
clean_edge_fp.append(0)
|
||||
|
||||
sorted_values.extend(clean_edge_fp)
|
||||
sorted_values.append(ab_xor)
|
||||
|
||||
abogus_bytes_str = (
|
||||
StringProcessor.generate_random_bytes()
|
||||
+ self.crypto_utility.transform_bytes(sorted_values)
|
||||
)
|
||||
|
||||
abogus = self.crypto_utility.abogus_encode(abogus_bytes_str, 0)
|
||||
params = "%s&a_bogus=%s" % (params, abogus)
|
||||
return (params, abogus, self.user_agent, body)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 24/06/16 晚点开源自定义ua
|
||||
# 24/07/08 支持自定义ua和浏览器指纹
|
||||
# 24/11/15 完成1.0.1.19版本abogus算法，择日开源
|
||||
# 25/03/05 修复POST请求参数加密错误，修补环境
|
||||
# 25/04/27 修复mypy检查错误
|
||||
|
||||
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0"
|
||||
chrome_fp = BrowserFingerprintGenerator.generate_fingerprint("Edge")
|
||||
abogus = ABogus(user_agent=user_agent, fp=chrome_fp)
|
||||
|
||||
# GET
|
||||
url = "https://www.douyin.com/aweme/v1/web/aweme/detail/?"
|
||||
params = "device_platform=webapp&aid=6383&channel=channel_pc_web&sec_user_id=MS4wLjABAAAArDVBosPJF3eIWVEFp0szuJ-e1V_-rK0ieJeWwpE77E8&max_cursor=0&locate_query=false&show_live_replay_strategy=1&need_time_list=1&time_list_query=0&whale_cut_token=&cut_version=1&count=18&publish_video_strategy_type=2&from_user_page=1&update_version_code=170400&pc_client_type=1&pc_libra_divert=Windows&support_h265=1&support_dash=0&version_code=290100&version_name=29.1.0&cookie_enabled=true&screen_width=1920&screen_height=1080&browser_language=zh-CN&browser_platform=Win32&browser_name=Edge&browser_version=131.0.0.0&browser_online=true&engine_name=Blink&engine_version=131.0.0.0&os_name=Windows&os_version=10&cpu_core_num=12&device_memory=8&platform=PC&downlink=10&effective_type=4g&round_trip_time=50"
|
||||
body = ""
|
||||
print(url + abogus.generate_abogus(params=params, body=body)[0])
|
||||
|
||||
# POST
|
||||
url = "https://www.douyin.com/aweme/v2/web/aweme/stats/?"
|
||||
params = "device_platform=webapp&aid=6383&channel=channel_pc_web&pc_client_type=1&pc_libra_divert=Windows&update_version_code=170400&support_h265=1&support_dash=0&version_code=170400&version_name=17.4.0&cookie_enabled=true&screen_width=1920&screen_height=1080&browser_language=zh-CN&browser_platform=Win32&browser_name=Edge&browser_version=131.0.0.0&browser_online=true&engine_name=Blink&engine_version=131.0.0.0&os_name=Windows&os_version=10&cpu_core_num=12&device_memory=8&platform=PC&downlink=10&effective_type=4g&round_trip_time=50"
|
||||
body = "aweme_type=0&item_id=7467485482314763572&play_delta=1&source=0"
|
||||
print(url + abogus.generate_abogus(params=params, body=body)[0])
|
||||
|
||||
# # 测试生成100个abogus参数 和 100个指纹所需时间
|
||||
# start = time.time()
|
||||
# for _ in range(100):
|
||||
# abogus.generate_abogus(params=params, body=body)
|
||||
# end = time.time()
|
||||
# print("生成100个abogus参数和指纹所需时间:", end - start) # 生成100个abogus参数和指纹所需时间: 2.203000783920288
|
||||
|
||||
# start = time.time()
|
||||
# for _ in range(100):
|
||||
# BrowserFingerprintGenerator.generate_fingerprint("Chrome")
|
||||
# end = time.time()
|
||||
# print("生成100个指纹所需时间:", end - start) # 生成100个指纹所需时间: 0.00400090217590332
|
||||
@@ -1,153 +0,0 @@
|
||||
"""抖音图文 SSR 静态解析 — 不走浏览器，直接解析服务端渲染页面
|
||||
|
||||
与 douyin_api.py 的 playwright 拦截方案互补：
|
||||
- 视频作品 → fetch_douyin_content（playwright 拦截，可取全部清晰度）
|
||||
- 图文作品 → fetch_douyin_note_ssr（本模块，按 aweme_id 精准渲染，秒级返回）
|
||||
|
||||
SSR 数据（window._ROUTER_DATA / __pace_f RSC）是服务端按作品 ID 渲染的页面数据，
|
||||
不存在 playwright 拦截 aweme/v1/web/aweme/post 时取到作者其他作品的问题。
|
||||
|
||||
注意：必须用移动端 UA 请求 iesdouyin.com/share 端点，桌面 UA 会被 302 回主站
|
||||
并返回 JS 反爬壳（无任何 SSR 数据）。
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
import httpx
|
||||
from nonebot import logger
|
||||
|
||||
from ...models import DouyinFetchError
|
||||
from ...utils import get_temp_root
|
||||
from .douyin_api import _process_animated_note, _process_note_with_parsed
|
||||
from .douyin_parser import parse_douyin_response, parse_note_images, parse_ssr_page
|
||||
|
||||
MOBILE_UA = (
|
||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) "
|
||||
"AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 "
|
||||
"Mobile/15E148 Safari/604.1"
|
||||
)
|
||||
ROUTER_DATA_RE = re.compile(r"window\._ROUTER_DATA\s*=\s*(.*?)</script>", re.S)
|
||||
# 短链重定向后统一为 iesdouyin.com/share/video/{id} 形态（图文作品也如此），
|
||||
# 因此 note / video 两种形态都要能提取 ID
|
||||
AWEME_ID_RE = re.compile(r"/(?:share/)?(?:note|video)/(\d+)")
|
||||
|
||||
|
||||
async def fetch_douyin_note_ssr(
|
||||
note_url: str,
|
||||
cookies: List[dict],
|
||||
) -> tuple[Optional[str], Optional[List[Path]]]:
|
||||
"""
|
||||
解析图文作品（SSR 静态解析，秒级返回）
|
||||
|
||||
接受 note / video 任意形态的抖音链接（重定向后图文也常是 share/video），
|
||||
按 SSR 内容判定：有 images → 图文/动图下载；无 images（真视频）→
|
||||
返回 (None, None) 交由 playwright 链路。
|
||||
|
||||
Returns:
|
||||
(title, file_paths) — 图片/视频本地路径列表；非图文返回 (None, None)；
|
||||
失败抛 DouyinFetchError
|
||||
"""
|
||||
m = AWEME_ID_RE.search(note_url)
|
||||
if not m:
|
||||
raise DouyinFetchError(f"无法从链接提取作品 ID: {note_url}")
|
||||
vid = m.group(1)
|
||||
|
||||
api_response = await _fetch_ssr_api_response(note_url, vid, cookies)
|
||||
|
||||
# 内容判定：SSR 数据无 images → 真视频作品，回落 playwright
|
||||
aweme = (api_response.get("aweme_list") or [None])[0] or {}
|
||||
if not aweme.get("images"):
|
||||
logger.info("SSR 内容为视频作品（无 images），回退 playwright 链路")
|
||||
return None, None
|
||||
|
||||
# 复用原有解析链：标题 / 作者 / 文件名（referer 统一用 note 形态，
|
||||
# 避免输入是 share/video 时被误判为"视频"）
|
||||
parsed = parse_douyin_response(
|
||||
api_response, f"https://www.douyin.com/note/{vid}"
|
||||
)
|
||||
logger.info(f"SSR 标题: {parsed.raw_title}")
|
||||
logger.info(f"SSR 作者: {parsed.raw_nickname}")
|
||||
|
||||
tmp_root = get_temp_root("douyin")
|
||||
headers = {
|
||||
"Referer": "https://www.douyin.com/",
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/122.0.0.0 Safari/537.36"
|
||||
),
|
||||
}
|
||||
|
||||
# 复用原有分派：纯图文 / 图+视频混合 → 并发下载；纯动图 → 逐张下载 mp4
|
||||
images_urls, video_url = parse_note_images(api_response, None, vid)
|
||||
if images_urls:
|
||||
file_paths = await _process_note_with_parsed(
|
||||
images_urls, video_url, tmp_root, parsed.rel_stem, headers
|
||||
)
|
||||
else:
|
||||
file_paths = await _process_animated_note(
|
||||
api_response, tmp_root, parsed.rel_stem, headers
|
||||
)
|
||||
return parsed.raw_title, file_paths
|
||||
|
||||
|
||||
async def _fetch_ssr_api_response(
|
||||
note_url: str, vid: str, cookies: List[dict]
|
||||
) -> dict:
|
||||
"""请求 douyin.com note 页面并提取 SSR 数据，规范化为 aweme_list 格式
|
||||
|
||||
2026-08-13 起 iesdouyin.com/share/note 端点对所有请求（含登录态）只返回
|
||||
33KB 壳页（无任何作品数据），改从 www.douyin.com/note/{id} 提取
|
||||
__pace_f RSC 数据——服务端按作品 id 精确渲染，与 URL 一一对应，
|
||||
不存在 aweme/post 作者列表"取错作品"的问题。
|
||||
必须用桌面 UA + 登录 cookie（whale 风控对未登录请求截断作品数据）。
|
||||
"""
|
||||
url = f"https://www.douyin.com/note/{vid}"
|
||||
cookie_map = {c["name"]: c["value"] for c in cookies} if cookies else None
|
||||
headers = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/151.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Referer": "https://www.douyin.com/",
|
||||
}
|
||||
async with httpx.AsyncClient(
|
||||
headers=headers, timeout=20, follow_redirects=True
|
||||
) as client:
|
||||
resp = await client.get(url, cookies=cookie_map)
|
||||
if resp.status_code >= 400:
|
||||
raise DouyinFetchError(f"SSR 页面请求失败: status={resp.status_code}")
|
||||
html = resp.text
|
||||
logger.info(f"SSR 页面: {url} status={resp.status_code} size={len(html)}")
|
||||
|
||||
# 1. 旧版 SSR: window._ROUTER_DATA（snake_case 结构，可直接复用原有解析函数）
|
||||
if m := ROUTER_DATA_RE.search(html):
|
||||
data = _clean_js_json(m.group(1))
|
||||
if data:
|
||||
loader = data.get("loaderData", {})
|
||||
page = loader.get("note_(id)/page") or loader.get("video_(id)/page")
|
||||
item_list = (page.get("videoInfoRes") or {}).get("item_list") or []
|
||||
if item_list:
|
||||
logger.info("SSR 数据来源: _ROUTER_DATA")
|
||||
return {"aweme_list": [item_list[0]]}
|
||||
# 2. 新版 SSR: __pace_f.push RSC 流（parse_ssr_page 已规范化为 aweme_list 格式）
|
||||
if parsed := parse_ssr_page(html):
|
||||
logger.info("SSR 数据来源: __pace_f RSC")
|
||||
return parsed
|
||||
raise DouyinFetchError("SSR 页面无图文数据（可能已被删除或风控）")
|
||||
|
||||
|
||||
def _clean_js_json(raw: str) -> Optional[dict]:
|
||||
"""清洗 JS 对象文本为合法 JSON（$undefined / $Lx 占位符）"""
|
||||
text = raw.strip().rstrip(";").strip()
|
||||
text = re.sub(r"\$undefined", "null", text)
|
||||
text = re.sub(r"\$L\d+", "null", text)
|
||||
try:
|
||||
return json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
logger.warning("SSR JSON 解析失败")
|
||||
return None
|
||||
@@ -0,0 +1,388 @@
|
||||
"""抖音 web 接口抓取 — 免浏览器方案（2026-10 版）
|
||||
|
||||
链路：作品链接 → 提取 aweme_id → 签名请求 `aweme/v1/web/aweme/detail/`
|
||||
→ 解析 aweme_detail → 下载「最高码率视频 / 最高画质图片（含动图）」。
|
||||
|
||||
网关要求（2026-08 起 ArgusSecurityPlugin）：
|
||||
1. 请求头 `x-tt-argus` 必须存在（实测只做存在性校验，占位值 "1" 即可）；
|
||||
2. 请求头 `uifid` = cookie 里的 UIFID（未登录为 UIFID_TEMP）；
|
||||
3. 查询参数 `a_bogus` 由纯 Python 签名器（douyin_sign.py，源自 f2，Apache-2.0）生成，
|
||||
UA 必须与签名时传入的 UA 一致（签名对 UA 做了二次哈希）。
|
||||
以上三点缺一即被网关 403（报错文案 Uifid Not Found / Signature Not Found）。
|
||||
|
||||
为什么废弃旧链路（SSR + playwright）：
|
||||
- iesdouyin 分享页 SSR 已整体降级为壳页（无 videoInfoRes）；
|
||||
- www.douyin.com 页面 HTML 对非浏览器客户端返回 jsvm 反爬壳，浏览器进页面也会
|
||||
随机触发验证码中间页（headless 同样被拦）；
|
||||
- web 接口是页面播放器同款数据源，含全部清晰度档（bit_rate 梯）与无水印直链。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple, Union
|
||||
|
||||
import httpx
|
||||
from nonebot import logger
|
||||
|
||||
from ...models import DouyinFetchError
|
||||
from ...utils import get_temp_root, unique_media_path
|
||||
from .douyin_parser import parse_douyin_response
|
||||
from .douyin_sign import ABogus
|
||||
|
||||
# UA 必须与签名输入一致（ABogus 对 UA 做二次哈希）；browser_name/version 参数同步为 Edge
|
||||
WEB_UA = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0"
|
||||
)
|
||||
MOBILE_UA = (
|
||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) "
|
||||
"AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 "
|
||||
"Mobile/15E148 Safari/604.1"
|
||||
)
|
||||
DETAIL_API = "https://www.douyin.com/aweme/v1/web/aweme/detail/"
|
||||
REFERER = "https://www.douyin.com/"
|
||||
AWEME_ID_RE = re.compile(r"/(?:share/)?(?:note|video)/(\d+)")
|
||||
|
||||
# web 端基础参数（对齐浏览器请求；msToken 实测非必需，不生成）
|
||||
_BASE_PARAMS: Dict[str, Union[str, int]] = {
|
||||
"device_platform": "webapp",
|
||||
"aid": "6383",
|
||||
"channel": "channel_pc_web",
|
||||
"pc_client_type": 1,
|
||||
"publish_video_strategy_type": 2,
|
||||
"pc_libra_divert": "Windows",
|
||||
"version_code": "290100",
|
||||
"version_name": "29.1.0",
|
||||
"cookie_enabled": "true",
|
||||
"screen_width": 1920,
|
||||
"screen_height": 1080,
|
||||
"browser_language": "zh-CN",
|
||||
"browser_platform": "Win32",
|
||||
"browser_name": "Edge",
|
||||
"browser_version": "130.0.0.0",
|
||||
"browser_online": "true",
|
||||
"engine_name": "Blink",
|
||||
"engine_version": "130.0.0.0",
|
||||
"os_name": "Windows",
|
||||
"os_version": "10",
|
||||
"cpu_core_num": 12,
|
||||
"device_memory": 8,
|
||||
"platform": "PC",
|
||||
"downlink": 10,
|
||||
"effective_type": "4g",
|
||||
"round_trip_time": 100,
|
||||
}
|
||||
|
||||
|
||||
def extract_aweme_id(url: str) -> Optional[str]:
|
||||
"""从分享链接 / 全链接里提取作品 id（share/note、share/video、note、video 均支持）"""
|
||||
m = AWEME_ID_RE.search(url)
|
||||
if m:
|
||||
return m.group(1)
|
||||
m = re.search(r"/(\d+)(?:\?|#|$)", url)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
async def resolve_short_link(url: str) -> Optional[str]:
|
||||
"""短链重定向：取 Location（最多 3 跳）；失败返回 None
|
||||
|
||||
只用于拿真实作品 URL（提取 aweme_id），不依赖 iesdouyin 页面内容。
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
headers={"User-Agent": MOBILE_UA},
|
||||
follow_redirects=False,
|
||||
timeout=10,
|
||||
) as client:
|
||||
current = url
|
||||
for _ in range(3):
|
||||
resp = await client.get(current)
|
||||
if resp.status_code >= 400:
|
||||
return None
|
||||
location = resp.headers.get("Location")
|
||||
if not location:
|
||||
return str(resp.url)
|
||||
current = urllib.parse.urljoin(current, location)
|
||||
return current
|
||||
except Exception:
|
||||
logger.warning(f"抖音短链重定向失败: {url}")
|
||||
return None
|
||||
|
||||
|
||||
def build_signed_query(aweme_id: str) -> str:
|
||||
"""构造带 a_bogus 签名的查询串（纯函数，便于测试）"""
|
||||
params = dict(_BASE_PARAMS)
|
||||
params["aweme_id"] = aweme_id
|
||||
param_str = "&".join(f"{k}={v}" for k, v in params.items())
|
||||
signed, a_bogus, _, _ = ABogus(user_agent=WEB_UA).generate_abogus(param_str)
|
||||
logger.debug(f"抖音签名: a_bogus={a_bogus[:16]}... len={len(a_bogus)}")
|
||||
return signed
|
||||
|
||||
|
||||
def _douyin_cookie_map(cookies: List[dict]) -> Dict[str, str]:
|
||||
"""Netscape cookie 列表 → {name: value}（只保留 douyin 域）"""
|
||||
return {
|
||||
c["name"]: c["value"]
|
||||
for c in cookies or []
|
||||
if "douyin" in (c.get("domain") or "")
|
||||
}
|
||||
|
||||
|
||||
def _build_headers(cookie_map: Dict[str, str]) -> Dict[str, str]:
|
||||
headers = {
|
||||
"User-Agent": WEB_UA,
|
||||
"Referer": REFERER,
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
# 网关只校验存在性，占位值即可（f2 项目 issue #443 实测结论）
|
||||
"x-tt-argus": "1",
|
||||
}
|
||||
uifid = cookie_map.get("UIFID") or cookie_map.get("UIFID_TEMP")
|
||||
if uifid:
|
||||
headers["uifid"] = uifid
|
||||
return headers
|
||||
|
||||
|
||||
async def fetch_douyin_detail(aweme_id: str, cookies: List[dict]) -> dict:
|
||||
"""请求作品 detail 接口，返回 aweme_detail dict（失败抛 DouyinFetchError）"""
|
||||
cookie_map = _douyin_cookie_map(cookies)
|
||||
if not (cookie_map.get("UIFID") or cookie_map.get("UIFID_TEMP")):
|
||||
raise DouyinFetchError("cookies.txt 缺少抖音 UIFID（重新登录抖音网页版导出）")
|
||||
|
||||
url = f"{DETAIL_API}?{build_signed_query(aweme_id)}"
|
||||
headers = _build_headers(cookie_map)
|
||||
last_error: Optional[Exception] = None
|
||||
for attempt in range(2):
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
headers=headers, cookies=cookie_map, timeout=20
|
||||
) as client:
|
||||
resp = await client.get(url)
|
||||
if resp.status_code == 403:
|
||||
raise DouyinFetchError(f"detail 接口被网关拦截: {resp.text[:80]}")
|
||||
if resp.status_code >= 400:
|
||||
raise DouyinFetchError(f"detail 接口请求失败: {resp.status_code}")
|
||||
data = resp.json()
|
||||
detail = data.get("aweme_detail")
|
||||
if detail:
|
||||
return detail
|
||||
# status_code=0 但无数据：反爬空响应（签名偶发不同步），重试一次
|
||||
last_error = DouyinFetchError(
|
||||
f"接口返回空数据 (status_code={data.get('status_code')})"
|
||||
)
|
||||
except DouyinFetchError as e:
|
||||
last_error = e
|
||||
except Exception as e: # 网络异常
|
||||
last_error = e
|
||||
if attempt == 0:
|
||||
logger.warning(f"detail 请求失败，1 秒后重试: {last_error!r}")
|
||||
await asyncio.sleep(1)
|
||||
raise DouyinFetchError(f"获取作品详情失败: {last_error}")
|
||||
|
||||
|
||||
# ============================= 媒体选择 =============================
|
||||
|
||||
|
||||
def _best_urls_from_video(video: dict) -> Tuple[List[str], dict]:
|
||||
"""从 video 对象选最高码率档的 URL 候选列表（含备选 CDN），返回 (urls, meta)"""
|
||||
gears = [
|
||||
g
|
||||
for g in (video.get("bit_rate") or [])
|
||||
if isinstance(g, dict) and (g.get("play_addr") or {}).get("url_list")
|
||||
]
|
||||
if gears:
|
||||
best = max(gears, key=lambda g: g.get("bit_rate") or 0)
|
||||
play_addr = best.get("play_addr") or {}
|
||||
meta = {
|
||||
"gear_name": best.get("gear_name"),
|
||||
"bit_rate": best.get("bit_rate"),
|
||||
"width": play_addr.get("width"),
|
||||
"height": play_addr.get("height"),
|
||||
"is_h265": best.get("is_h265"),
|
||||
"size": play_addr.get("data_size"),
|
||||
}
|
||||
return list(play_addr.get("url_list") or []), meta
|
||||
play_addr = video.get("play_addr") or {}
|
||||
meta = {
|
||||
"gear_name": None,
|
||||
"bit_rate": None,
|
||||
"width": play_addr.get("width"),
|
||||
"height": play_addr.get("height"),
|
||||
"is_h265": video.get("is_h265"),
|
||||
"size": play_addr.get("data_size"),
|
||||
}
|
||||
return list(play_addr.get("url_list") or []), meta
|
||||
|
||||
|
||||
def collect_download_items(
|
||||
detail: dict,
|
||||
) -> Tuple[List[Tuple[List[str], str]], bool]:
|
||||
"""从 aweme_detail 收集下载项。
|
||||
|
||||
返回 (items, is_image_post)：
|
||||
- 视频作品 → ([(视频URL候选, ".mp4")], False)
|
||||
- 图文（含图+视频混合）→ 每项一个 (URL候选列表, 扩展名)：
|
||||
图片项按 无水印 url_list → 带作者水印的 download_url_list 排序；
|
||||
内嵌视频项（动图 / 图集中的视频）走最高码率档。
|
||||
is_image_post=True
|
||||
"""
|
||||
images = detail.get("images") or []
|
||||
if not images:
|
||||
urls, meta = _best_urls_from_video(detail.get("video") or {})
|
||||
if not urls:
|
||||
raise DouyinFetchError("未找到可用视频直链")
|
||||
logger.info(
|
||||
f"抖音视频最高档: {meta.get('gear_name')} "
|
||||
f"{meta.get('width')}x{meta.get('height')} "
|
||||
f"br={meta.get('bit_rate')} size={meta.get('size')} "
|
||||
f"h265={meta.get('is_h265')} 候选={len(urls)}"
|
||||
)
|
||||
return [(urls, ".mp4")], False
|
||||
|
||||
items: List[Tuple[List[str], str]] = []
|
||||
for img in images:
|
||||
if not isinstance(img, dict):
|
||||
continue
|
||||
video = img.get("video")
|
||||
if isinstance(video, dict) and (video.get("play_addr") or {}).get("url_list"):
|
||||
urls, meta = _best_urls_from_video(video)
|
||||
if urls:
|
||||
logger.info(
|
||||
f"图集内视频项: {meta.get('gear_name')} br={meta.get('bit_rate')}"
|
||||
)
|
||||
items.append((urls, ".mp4"))
|
||||
continue
|
||||
# 图片项：url_list 为无水印版（q75 webp/jpeg）；
|
||||
# download_url_list 是带作者水印的"保存到相册"版（tplv-dy-water-v2，
|
||||
# base64 里就是作者昵称），只能当兜底绝不能优先
|
||||
urls = list(img.get("url_list") or []) + list(
|
||||
img.get("download_url_list") or []
|
||||
)
|
||||
if urls:
|
||||
items.append((urls, ""))
|
||||
if not items:
|
||||
raise DouyinFetchError("图文作品未解析到任何图片/视频")
|
||||
logger.info(f"抖音图文作品: {len(items)} 个下载项")
|
||||
return items, True
|
||||
|
||||
|
||||
# ============================= 下载 =============================
|
||||
|
||||
|
||||
def _infer_extension(url: str) -> str:
|
||||
"""从 URL 推断文件扩展名（douyinpic 图片 URL 无扩展名时默认 jpg）"""
|
||||
url_lower = url.lower()
|
||||
for ext in (".webp", ".jpeg", ".jpg", ".png", ".gif", ".avif", ".heic"):
|
||||
if ext in url_lower:
|
||||
return ext
|
||||
return ".jpg"
|
||||
|
||||
|
||||
async def _stream_download(
|
||||
url: str, filepath: Path, headers: dict, timeout: float = 300
|
||||
) -> None:
|
||||
async with httpx.AsyncClient(headers=headers, timeout=timeout) as client:
|
||||
async with client.stream("GET", url) as resp:
|
||||
resp.raise_for_status()
|
||||
with open(filepath, "wb") as f:
|
||||
async for chunk in resp.aiter_bytes(65536):
|
||||
f.write(chunk)
|
||||
|
||||
|
||||
async def _download_with_fallback(
|
||||
candidates: List[str], filepath: Path, headers: dict
|
||||
) -> Optional[Path]:
|
||||
"""按候选 URL 顺序下载（CDN 多备份），全部失败返回 None"""
|
||||
for url in candidates:
|
||||
try:
|
||||
await _stream_download(url, filepath, headers)
|
||||
if filepath.stat().st_size == 0: # 空响应也当失败，换下一个候选
|
||||
raise DouyinFetchError("响应为空（0 字节）")
|
||||
logger.info(f"已保存: {filepath.name} ({filepath.stat().st_size} 字节)")
|
||||
return filepath
|
||||
except Exception as e:
|
||||
logger.warning(f"下载失败，尝试下一个 CDN: {url[:80]}... ({e!r})")
|
||||
return None
|
||||
|
||||
|
||||
async def _download_video_file(
|
||||
candidates: List[str], tmp_root: Path, rel_stem: str, headers: dict
|
||||
) -> Path:
|
||||
output_path = unique_media_path(tmp_root / f"{rel_stem}.mp4")
|
||||
result = await _download_with_fallback(candidates, output_path, headers)
|
||||
if result is None:
|
||||
raise DouyinFetchError("视频下载失败（所有 CDN 均不可用）")
|
||||
logger.info(f"视频下载完成: {result}")
|
||||
return result
|
||||
|
||||
|
||||
async def _download_note_items(
|
||||
items: List[Tuple[List[str], str]],
|
||||
tmp_root: Path,
|
||||
rel_stem: str,
|
||||
headers: dict,
|
||||
) -> List[Path]:
|
||||
"""图集/动图：一个作品一个目录，`001.jpg / 002.mp4 ...` 并发下载"""
|
||||
note_dir = unique_media_path(tmp_root / rel_stem)
|
||||
note_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"图集保存目录: {note_dir}")
|
||||
|
||||
async def _one(idx: int, candidates: List[str], ext: str) -> Optional[Path]:
|
||||
ext = ext or _infer_extension(candidates[0])
|
||||
filepath = note_dir / f"{idx:03d}{ext}"
|
||||
return await _download_with_fallback(candidates, filepath, headers)
|
||||
|
||||
results = await asyncio.gather(
|
||||
*(_one(i, urls, ext) for i, (urls, ext) in enumerate(items, 1))
|
||||
)
|
||||
saved = [p for p in results if p is not None]
|
||||
if not saved:
|
||||
raise DouyinFetchError("图集下载失败（所有文件均失败）")
|
||||
logger.info(f"图集下载完成，共 {len(saved)}/{len(items)} 个文件")
|
||||
return saved
|
||||
|
||||
|
||||
# ============================= 主入口 =============================
|
||||
|
||||
|
||||
async def fetch_douyin_web(
|
||||
url: str,
|
||||
cookies: List[dict],
|
||||
) -> Tuple[str, Union[Path, List[Path]], bool]:
|
||||
"""解析抖音作品（视频 / 图文 / 动图），返回 (title, 本地路径, 是否图文)
|
||||
|
||||
失败抛 DouyinFetchError（由调用方兜底）。
|
||||
"""
|
||||
aweme_id = extract_aweme_id(url)
|
||||
if not aweme_id:
|
||||
# 短链（v.douyin.com/xxx）或异常形态 → 先重定向拿全链接再提取
|
||||
resolved = await resolve_short_link(url)
|
||||
if resolved:
|
||||
logger.info(f"抖音短链重定向: {url} -> {resolved}")
|
||||
url = resolved
|
||||
aweme_id = extract_aweme_id(url)
|
||||
if not aweme_id:
|
||||
raise DouyinFetchError(f"无法从链接提取作品 ID: {url}")
|
||||
|
||||
detail = await fetch_douyin_detail(aweme_id, cookies)
|
||||
items, is_img_post = collect_download_items(detail)
|
||||
|
||||
# 内容形态以接口数据为准（分享短链重定向后图文也常是 share/video 形态）
|
||||
referer = f"https://www.douyin.com/{'note' if is_img_post else 'video'}/{aweme_id}"
|
||||
parsed = parse_douyin_response({"aweme_detail": detail}, referer)
|
||||
logger.info(f"抖音标题: {parsed.raw_title}")
|
||||
logger.info(f"抖音作者: {parsed.raw_nickname}")
|
||||
logger.info(f"抖音落盘: {parsed.rel_stem}")
|
||||
|
||||
tmp_root = get_temp_root("douyin")
|
||||
headers = {"Referer": REFERER, "User-Agent": WEB_UA}
|
||||
if is_img_post:
|
||||
paths: Union[Path, List[Path]] = await _download_note_items(
|
||||
items, tmp_root, parsed.rel_stem, headers
|
||||
)
|
||||
else:
|
||||
paths = await _download_video_file(
|
||||
items[0][0], tmp_root, parsed.rel_stem, headers
|
||||
)
|
||||
return parsed.raw_title, paths, is_img_post
|
||||
@@ -0,0 +1,322 @@
|
||||
"""快手作品解析 — 分享页 window.INIT_STATE 静态提取
|
||||
|
||||
借鉴 nonebot-plugin-parser 的 KuaiShouParser：
|
||||
- v.kuaishou.com/{code} 短链 → 取 Location 跳到 v.m.chenzhongtech.com/fw/photo/{id}
|
||||
- www.kuaishou.com/short-video/{id} → 重写为移动端 fw/photo/{id}
|
||||
（桌面站匿名请求返回 `Intercept-Result: risk-control` 的 JSON，拿不到页面）
|
||||
- /fw/long-video/ 与 /fw/photo/ 返回结构不同，统一按 /fw/photo/ 请求
|
||||
- 页面 HTML 提取 window.INIT_STATE → dict 里第一个非空 photo
|
||||
- 视频优先从清晰度梯 manifest.adaptationSet 挑最优档（见 `_manifest_video_url`），
|
||||
没有 manifest 才退回 photo.mainMvUrls（默认档）
|
||||
- 图集取 photo.ext_params.atlas 的 cdnList + list 拼 URL
|
||||
|
||||
必须用移动端（iPhone）UA 请求：桌面 UA 与匿名 www 站都只返回风控壳页。
|
||||
|
||||
返回 (title, 文件)：
|
||||
- 视频作品 → (title, 视频文件 Path)
|
||||
- 图集 / 图+视频混排 → (title, [文件路径列表]，视频排在图片之后)
|
||||
- 纯文字作品 → (title, [])
|
||||
- 解析失败 → (None, None)
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
from nonebot import logger
|
||||
|
||||
from ...models import ContentFetchError
|
||||
from ...utils import (
|
||||
build_author_dir,
|
||||
build_work_stem,
|
||||
get_temp_root,
|
||||
unique_media_path,
|
||||
)
|
||||
|
||||
#: 移动端 UA（桌面 UA 会被风控挡下，见模块文档）
|
||||
KUAISHOU_UA = (
|
||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) "
|
||||
"AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 "
|
||||
"Safari/604.1 Edg/132.0.0.0"
|
||||
)
|
||||
KUAISHOU_REFERER = "https://v.kuaishou.com/"
|
||||
|
||||
SHORT_LINK_RE = re.compile(r"v\.kuaishou\.com/[A-Za-z0-9._?%&+\-=/]+")
|
||||
WEB_SHORT_VIDEO_RE = re.compile(
|
||||
r"(?:www\.)?kuaishou\.com/(?:short-video|fw/[a-z-]+)/([A-Za-z0-9]+)"
|
||||
)
|
||||
INIT_STATE_RE = re.compile(r"window\.INIT_STATE\s*=\s*(.*?)</script>", re.S)
|
||||
|
||||
#: 昵称缺失时的兜底串（utils.build_author_dir 认它，转成"未知作者"规则）
|
||||
FALLBACK_AUTHOR = "快手用户"
|
||||
|
||||
|
||||
async def fetch_kuaishou_content(
|
||||
url: str,
|
||||
) -> tuple[Optional[str], Optional[Union[Path, list[Path]]]]:
|
||||
"""解析快手作品链接（短链 / 分享页 / 网页版短链）"""
|
||||
try:
|
||||
return await _parse(url)
|
||||
except Exception:
|
||||
logger.exception(f"快手解析失败: {url}")
|
||||
return None, None
|
||||
|
||||
|
||||
async def _parse(url: str):
|
||||
# 1. 短链 → 单跳重定向（Location 即 v.m.chenzhongtech.com/fw/photo/{id}）
|
||||
if "v.kuaishou.com" in url:
|
||||
resolved = await _resolve_redirect(url)
|
||||
if resolved:
|
||||
logger.info(f"快手短链重定向: {url} -> {resolved}")
|
||||
url = resolved
|
||||
|
||||
# 2. www.kuaishou.com/short-video/{id} → 移动端分享页（同一套 INIT_STATE）
|
||||
m = WEB_SHORT_VIDEO_RE.search(url)
|
||||
if m and "chenzhongtech.com" not in url:
|
||||
url = f"https://v.m.chenzhongtech.com/fw/photo/{m.group(1)}"
|
||||
logger.info(f"快手链接规范化: {url}")
|
||||
|
||||
# 3. long-video 与 photo 返回结构不同，统一按 photo 请求
|
||||
url = url.replace("/fw/long-video/", "/fw/photo/")
|
||||
|
||||
html = await _fetch_page(url)
|
||||
photo = _extract_photo(html)
|
||||
logger.info(f"快手页面解析: {_describe(photo)}")
|
||||
return await _build_result(photo)
|
||||
|
||||
|
||||
def _describe(photo: dict) -> str:
|
||||
"""日志用的一句话摘要（标题/作者/时长）"""
|
||||
caption = str(photo.get("caption") or "").strip()
|
||||
nickname = str(photo.get("userName") or "").strip()
|
||||
if len(caption) > 30:
|
||||
caption = caption[:30]
|
||||
return f"作者={nickname or '未知'}, 标题={caption or '无'}, 时长={photo.get('duration') or 0}ms"
|
||||
|
||||
|
||||
def _extract_photo(html: str) -> dict:
|
||||
"""从 window.INIT_STATE 提取作品数据（取第一个非空 photo）"""
|
||||
m = INIT_STATE_RE.search(html)
|
||||
if not m:
|
||||
raise ContentFetchError("快手页面无 INIT_STATE（可能已删除或风控）")
|
||||
|
||||
raw = m.group(1).strip()
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
# 页面偶有 undefined 之类的非 JSON 字面量（同小红书预处理）
|
||||
try:
|
||||
data = json.loads(raw.replace("undefined", "null"))
|
||||
except json.JSONDecodeError:
|
||||
raise ContentFetchError("快手 INIT_STATE JSON 解析失败")
|
||||
|
||||
if not isinstance(data, dict):
|
||||
raise ContentFetchError("快手 INIT_STATE 结构异常")
|
||||
for value in data.values():
|
||||
if isinstance(value, dict) and isinstance(value.get("photo"), dict):
|
||||
return value["photo"]
|
||||
raise ContentFetchError("页面数据中未找到作品（可能已删除或风控）")
|
||||
|
||||
|
||||
def _first_url(items) -> Optional[str]:
|
||||
"""[{cdn, url}, ...] / [url, ...] → 第一个可用 URL（快手同一资源给多个 CDN）"""
|
||||
if not isinstance(items, list):
|
||||
return None
|
||||
for item in items:
|
||||
if isinstance(item, dict) and item.get("url"):
|
||||
return str(item["url"])
|
||||
if isinstance(item, str) and item:
|
||||
return item
|
||||
return None
|
||||
|
||||
|
||||
#: 同分辨率/帧率时优先 avc（QQ 客户端播放、群文件解压后二次剪辑都最稳；
|
||||
#: hevc 体积小但兼容性差，只在画质更高时才选）
|
||||
_CODEC_RANK = {"avc": 0, "avc1": 0, "h264": 0}
|
||||
|
||||
|
||||
def _manifest_video_url(photo: dict) -> Optional[str]:
|
||||
"""从清晰度梯 `manifest.adaptationSet[].representation[]` 挑最优档
|
||||
|
||||
排序键 = 分辨率 → 帧率 → 编码（avc 优先，见 `_CODEC_RANK`）→ 码率。
|
||||
参考实现直接用 mainMvUrls（默认档），多档作品拿不到最高清晰度。
|
||||
"""
|
||||
adaptations = (photo.get("manifest") or {}).get("adaptationSet")
|
||||
if not isinstance(adaptations, list):
|
||||
return None
|
||||
|
||||
best_rep = None
|
||||
best_key = None
|
||||
total = 0
|
||||
for adaptation in adaptations:
|
||||
if not isinstance(adaptation, dict):
|
||||
continue
|
||||
for rep in adaptation.get("representation") or []:
|
||||
if not isinstance(rep, dict) or not rep.get("url"):
|
||||
continue
|
||||
total += 1
|
||||
codec = str(rep.get("videoCodec") or "").lower()
|
||||
key = (
|
||||
(rep.get("width") or 0) * (rep.get("height") or 0),
|
||||
rep.get("frameRate") or 0,
|
||||
-_CODEC_RANK.get(codec, 1), # 取 max，故用负值让 avc 排前
|
||||
rep.get("avgBitrate") or 0,
|
||||
)
|
||||
if best_key is None or key > best_key:
|
||||
best_key, best_rep = key, rep
|
||||
|
||||
if best_rep is None:
|
||||
return None
|
||||
logger.info(
|
||||
f"快手清晰度: {best_rep.get('width')}x{best_rep.get('height')} "
|
||||
f"{best_rep.get('frameRate')}fps {best_rep.get('videoCodec')} "
|
||||
f"avg={best_rep.get('avgBitrate')}kbps "
|
||||
f"size={best_rep.get('fileSize')}（{total} 档中选最优）"
|
||||
)
|
||||
return str(best_rep["url"])
|
||||
|
||||
|
||||
def _video_url(photo: dict) -> Optional[str]:
|
||||
"""视频直链：清晰度梯最优档 → mainMvUrls 兜底（老页面可能没 manifest）"""
|
||||
return _manifest_video_url(photo) or _first_url(photo.get("mainMvUrls"))
|
||||
|
||||
|
||||
def _atlas_image_urls(photo: dict) -> list[str]:
|
||||
"""图集图片：ext_params.atlas.cdnList（图床域名）+ list（路径）拼 URL
|
||||
|
||||
atlas 缺失/为空即非图集作品（视频的 ext_params 里只有编码参数）。
|
||||
"""
|
||||
ext = photo.get("ext_params") or photo.get("extParams") or {}
|
||||
atlas = ext.get("atlas") if isinstance(ext, dict) else None
|
||||
if not isinstance(atlas, dict):
|
||||
return []
|
||||
|
||||
routes = atlas.get("list")
|
||||
if not isinstance(routes, list) or not routes:
|
||||
return []
|
||||
|
||||
cdn = ""
|
||||
for item in atlas.get("cdnList") or []:
|
||||
if isinstance(item, dict) and item.get("cdn"):
|
||||
cdn = str(item["cdn"])
|
||||
break
|
||||
if not cdn:
|
||||
return []
|
||||
|
||||
return [
|
||||
f"https://{cdn}/{route}"
|
||||
for route in routes
|
||||
if isinstance(route, str) and route
|
||||
]
|
||||
|
||||
|
||||
def _build_rel_stem(photo: dict) -> str:
|
||||
"""相对平台根的路径词干：`{作者}_{userId}/{作品名}`"""
|
||||
author = str(photo.get("userName") or "").replace("ㅤ", "").strip()
|
||||
author = author or FALLBACK_AUTHOR
|
||||
source = str(photo.get("photoId") or "")
|
||||
return (
|
||||
f"{build_author_dir(author, photo.get('userId'), source=source)}"
|
||||
f"/{build_work_stem(str(photo.get('caption') or ''), source=source)}"
|
||||
)
|
||||
|
||||
|
||||
async def _build_result(photo: dict) -> tuple[str, Union[Path, list[Path]]]:
|
||||
text = str(photo.get("caption") or "").strip() or FALLBACK_AUTHOR
|
||||
rel_stem = _build_rel_stem(photo)
|
||||
|
||||
video_url = _video_url(photo)
|
||||
image_urls = _atlas_image_urls(photo)
|
||||
|
||||
# 图集（含"图 + 视频"混排）：一个作品一个目录，视频排在图片之后
|
||||
if image_urls:
|
||||
files = await _download_images(image_urls, video_url, rel_stem)
|
||||
if not files:
|
||||
raise ContentFetchError("快手媒体下载失败")
|
||||
logger.info(
|
||||
f"快手图集: 图片={len(image_urls)} 张, "
|
||||
f"视频={'有' if video_url else '无'}, 落盘={len(files)} 个"
|
||||
)
|
||||
return text, files
|
||||
|
||||
if video_url:
|
||||
path = await _download_video(video_url, rel_stem)
|
||||
return text, path
|
||||
|
||||
logger.info(f"快手文字作品（无媒体）: {text[:30]}")
|
||||
return text, []
|
||||
|
||||
|
||||
async def _download_images(
|
||||
image_urls: list[str], video_url: Optional[str], rel_stem: str
|
||||
) -> list[Path]:
|
||||
"""并发下载图集（复用抖音图文的下载流程，视频追加在图片之后）"""
|
||||
|
||||
from .douyin_api import _process_note_with_parsed
|
||||
|
||||
tmp_root = get_temp_root("kuaishou")
|
||||
headers = {"Referer": KUAISHOU_REFERER, "User-Agent": KUAISHOU_UA}
|
||||
return await _process_note_with_parsed(
|
||||
[[u] for u in image_urls], video_url, tmp_root, rel_stem, headers
|
||||
)
|
||||
|
||||
|
||||
async def _download_video(video_url: str, rel_stem: str) -> Path:
|
||||
"""流式下载视频到 `{平台根}/{作者目录}/{作品名}.mp4`"""
|
||||
import httpx
|
||||
|
||||
tmp_root = get_temp_root("kuaishou")
|
||||
output_path = unique_media_path(tmp_root / f"{rel_stem}.mp4")
|
||||
headers = {"Referer": KUAISHOU_REFERER, "User-Agent": KUAISHOU_UA}
|
||||
async with httpx.AsyncClient(headers=headers, timeout=300) as client:
|
||||
async with client.stream("GET", video_url) as resp:
|
||||
resp.raise_for_status()
|
||||
with open(output_path, "wb") as f:
|
||||
async for chunk in resp.aiter_bytes(8192):
|
||||
f.write(chunk)
|
||||
logger.info(f"快手视频下载完成: {output_path}")
|
||||
return output_path
|
||||
|
||||
|
||||
async def _fetch_page(url: str) -> str:
|
||||
import httpx
|
||||
|
||||
headers = {"User-Agent": KUAISHOU_UA, "Referer": KUAISHOU_REFERER}
|
||||
async with httpx.AsyncClient(
|
||||
headers=headers, timeout=20, follow_redirects=True
|
||||
) as client:
|
||||
resp = await client.get(url)
|
||||
if resp.status_code >= 400:
|
||||
raise ContentFetchError(f"快手页面请求失败: status={resp.status_code}")
|
||||
return resp.text
|
||||
|
||||
|
||||
async def _resolve_redirect(url: str) -> Optional[str]:
|
||||
"""短链重定向（最多 3 跳，取最终 URL）
|
||||
|
||||
v.kuaishou.com 是 302 + Location，单跳即到 chenzhongtech 分享页；
|
||||
与小红书一样用移动端 headers，避免被当作非 App 来源拒绝。
|
||||
"""
|
||||
import httpx
|
||||
|
||||
headers = {"User-Agent": KUAISHOU_UA, "Referer": KUAISHOU_REFERER}
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
headers=headers, follow_redirects=False, timeout=10
|
||||
) as client:
|
||||
current = url
|
||||
for _ in range(3):
|
||||
resp = await client.get(current)
|
||||
if resp.status_code >= 400:
|
||||
return None
|
||||
location = resp.headers.get("Location")
|
||||
if not location:
|
||||
return str(resp.url)
|
||||
# Location 可能是相对路径，需拼上当前 URL
|
||||
current = urllib.parse.urljoin(current, location)
|
||||
return current
|
||||
except Exception:
|
||||
logger.warning(f"快手短链重定向失败: {url}")
|
||||
return None
|
||||
@@ -6,8 +6,9 @@
|
||||
- xiaohongshu.com/discovery/item/{id}?xsec_token=... → 移动端点（fallback）
|
||||
- 页面 HTML 提取 window.__INITIAL_STATE__ → noteDetailMap[id].note
|
||||
|
||||
请求端点依次尝试 国际站 rednote.com（含无水印原片 originVideoKey）
|
||||
→ 国内站 xiaohongshu.com（国内笔记国际站无数据，需 cookies.txt 登录态）。
|
||||
请求端点按链接来源选 host 顺序：国内链接（xiaohongshu.com/xhslink.com）先试
|
||||
国内站、国际链接（rednote.com）先试国际站（国际站数据含无水印原片 originVideoKey；
|
||||
国内笔记在国际站没有数据，反之亦然），失败再换站。需 cookies.txt 登录态。
|
||||
|
||||
返回 (title, 文件)：
|
||||
- 图文笔记 → (title, [图片路径列表])
|
||||
@@ -44,16 +45,15 @@ REDNOTE_UA = (
|
||||
REDNOTE_REFERER = "https://www.xiaohongshu.com/"
|
||||
|
||||
REDNOTE_SHORT_RE = re.compile(r"xhslink\.(?:com|cn)/[A-Za-z0-9._?%&+=/#@-]+")
|
||||
EXPLORE_RE = re.compile(
|
||||
r"xiaohongshu\.com/explore/([0-9a-zA-Z]+)(\?[^\s\"'<>]+)?"
|
||||
)
|
||||
EXPLORE_RE = re.compile(r"xiaohongshu\.com/explore/([0-9a-zA-Z]+)(\?[^\s\"'<>]+)?")
|
||||
DISCOVERY_RE = re.compile(
|
||||
r"xiaohongshu\.com/discovery/item/([0-9a-zA-Z]+)(\?[^\s\"'<>]+)?"
|
||||
)
|
||||
INITIAL_STATE_RE = re.compile(r"window\.__INITIAL_STATE__=(.*?)</script>", re.S)
|
||||
|
||||
# 请求 host 顺序：国际站优先（含无水印原片），失败回退国内站（国内笔记）
|
||||
REDNOTE_HOSTS = ("www.rednote.com", "www.xiaohongshu.com")
|
||||
# 请求 host 候选：链接来自哪个站就先用哪个站（国内笔记在国际站没有数据，
|
||||
# 国际站笔记（rednote.com）带无水印原片 originVideoKey，互不通用）
|
||||
REDNOTE_HOSTS = ("www.xiaohongshu.com", "www.rednote.com")
|
||||
|
||||
|
||||
async def fetch_rednote_content(
|
||||
@@ -75,14 +75,18 @@ async def _parse(url: str):
|
||||
logger.info(f"xhslink 重定向: {url} -> {resolved}")
|
||||
url = resolved
|
||||
|
||||
# 2. explore 优先（国际站才有 originVideoKey 无水印原片），
|
||||
# 依次尝试 国际站 → 国内站；失败再用同一 id+query 回退 discovery 端点
|
||||
# 链接来自哪个站就先试哪个站（国际站 rednote.com / 国内站 xiaohongshu.com）
|
||||
hosts = REDNOTE_HOSTS
|
||||
if "rednote.com" in url:
|
||||
hosts = tuple(reversed(REDNOTE_HOSTS))
|
||||
|
||||
# 2. explore 优先；失败再用同一 id+query 回退 discovery 端点
|
||||
if m := EXPLORE_RE.search(url):
|
||||
note_id, query = m.group(1), (m.group(2) or "").lstrip("?")
|
||||
return await _fetch_with_retry(note_id, query, ("explore", "discovery"))
|
||||
return await _fetch_with_retry(note_id, query, ("explore", "discovery"), hosts)
|
||||
if m := DISCOVERY_RE.search(url):
|
||||
note_id, query = m.group(1), (m.group(2) or "").lstrip("?")
|
||||
return await _fetch_with_retry(note_id, query, ("discovery",))
|
||||
return await _fetch_with_retry(note_id, query, ("discovery",), hosts)
|
||||
|
||||
# 3. 短链重定向终态: xiaohongshu.com/explore?target_note_id={id}&xsec_token=...
|
||||
# （discovery/item 会 302 到该形态，note id 在 query 里），
|
||||
@@ -91,14 +95,10 @@ async def _parse(url: str):
|
||||
qs = urllib.parse.parse_qs(m.group(1))
|
||||
note_id = (qs.get("target_note_id") or [""])[0]
|
||||
if note_id:
|
||||
keep = {
|
||||
k: v
|
||||
for k, v in qs.items()
|
||||
if k in ("xsec_token", "xsec_source")
|
||||
}
|
||||
keep = {k: v for k, v in qs.items() if k in ("xsec_token", "xsec_source")}
|
||||
query = urllib.parse.urlencode(keep, doseq=True)
|
||||
return await _fetch_with_retry(
|
||||
note_id, query, ("explore", "discovery")
|
||||
note_id, query, ("explore", "discovery"), hosts
|
||||
)
|
||||
|
||||
logger.warning(f"无法识别的小红书链接: {url}")
|
||||
@@ -106,15 +106,18 @@ async def _parse(url: str):
|
||||
|
||||
|
||||
async def _fetch_with_retry(
|
||||
note_id: str, query: str, paths: tuple[str, ...]
|
||||
note_id: str,
|
||||
query: str,
|
||||
paths: tuple[str, ...],
|
||||
hosts: tuple[str, ...] = REDNOTE_HOSTS,
|
||||
) -> tuple[str, Union[Path, list[Path]]] | None:
|
||||
"""依次尝试 国际站 → 国内站 × 端点；全部失败后稍候重试一轮
|
||||
"""依次尝试 host × 端点；全部失败后稍候重试一轮
|
||||
|
||||
小红书对突发请求会节流（页面 200 但 noteDetailMap 为空/无数据），
|
||||
重试一轮可绕过大部分瞬时风控；重试仍失败返回 (None, None)。
|
||||
"""
|
||||
for attempt in range(2):
|
||||
for host in REDNOTE_HOSTS:
|
||||
for host in hosts:
|
||||
for path in paths:
|
||||
fetch = _parse_explore if path == "explore" else _parse_discovery
|
||||
try:
|
||||
@@ -211,9 +214,7 @@ def _extract_note(html: str, note_id: str) -> dict:
|
||||
async def _build_result(note: dict) -> tuple[str, Union[Path, list[Path]]]:
|
||||
# 空壳 note（短链重定向带 undertake_note_error=该内容暂时无法查看）
|
||||
# → 笔记已删除/私密，直接报错而不是误判为纯文字笔记
|
||||
if not any(
|
||||
note.get(k) for k in ("title", "desc", "type", "imageList", "video")
|
||||
):
|
||||
if not any(note.get(k) for k in ("title", "desc", "type", "imageList", "video")):
|
||||
raise ContentFetchError("笔记内容不可见（可能已删除/私密），无法解析")
|
||||
title = note.get("title") or ""
|
||||
desc = note.get("desc") or ""
|
||||
@@ -229,18 +230,19 @@ async def _build_result(note: dict) -> tuple[str, Union[Path, list[Path]]]:
|
||||
# 1. 视频笔记 → 无水印原片优先
|
||||
if note.get("type") == "video" and note.get("video"):
|
||||
# 1a. 无水印原片（国际站数据 video.consumer.originVideoKey）
|
||||
consumer = (note["video"].get("consumer") or {})
|
||||
consumer = note["video"].get("consumer") or {}
|
||||
okey = consumer.get("originVideoKey")
|
||||
if okey:
|
||||
video_url = f"https://sns-video-bd.xhscdn.com/{okey}"
|
||||
logger.info(f"小红书视频: 无水印原片 originVideoKey={okey[:30]}...")
|
||||
video_path = await _download_video(video_url, rel_stem)
|
||||
video_path = await _download_video([video_url], rel_stem)
|
||||
return text, video_path
|
||||
|
||||
# 1b. 无 originVideoKey（国内站数据）→ 从 stream 分组选无水印原片
|
||||
# 国内站 masterUrl 同样是 sns-video-v6 原片 CDN（无水印），
|
||||
# 1b. 无 originVideoKey（国内站数据）→ 从 stream 分组选最高档
|
||||
# 国内站 masterUrl 同样是 sns-video 原片 CDN（无水印），
|
||||
# 但不同抓取批次返回的清晰度集合不同（同组多条/分组顺序不定），
|
||||
# 因此跨全部编码分组收集候选，取 size 最大（质量最高）的流。
|
||||
# 因此跨全部编码分组收集候选，取 size 最大（质量最高）的流；
|
||||
# 每个流附带 backupUrls（备选 CDN），下载时逐个回退。
|
||||
stream = ((note["video"].get("media") or {}).get("stream")) or {}
|
||||
candidates = [
|
||||
it
|
||||
@@ -254,23 +256,34 @@ async def _build_result(note: dict) -> tuple[str, Union[Path, list[Path]]]:
|
||||
candidates,
|
||||
key=lambda it: (it.get("size") or 0, it.get("avgBitrate") or 0),
|
||||
)
|
||||
video_url = best["masterUrl"]
|
||||
urls = [best["masterUrl"]] + [
|
||||
u for u in (best.get("backupUrls") or []) if u
|
||||
]
|
||||
duration = best.get("duration", 0)
|
||||
logger.info(
|
||||
f"小红书视频: 无水印流 {best.get('qualityType')} "
|
||||
f"{best.get('width')}x{best.get('height')} {best.get('fps')}fps "
|
||||
f"size={best.get('size')} duration={duration}ms"
|
||||
f"size={best.get('size')} duration={duration}ms "
|
||||
f"候选={len(urls)}"
|
||||
)
|
||||
video_path = await _download_video(video_url, rel_stem)
|
||||
video_path = await _download_video(urls, rel_stem)
|
||||
return text, video_path
|
||||
raise ContentFetchError("小红书视频流解析失败")
|
||||
|
||||
# 2. 图文笔记
|
||||
images = [
|
||||
img.get("urlDefault") or img.get("url")
|
||||
for img in note.get("imageList") or []
|
||||
]
|
||||
images = [u for u in images if u]
|
||||
# 2. 图文笔记（urlDefault = WB_DFT 默认档；兜底取 infoList 里 WB_DFT 场景）
|
||||
images = []
|
||||
for img in note.get("imageList") or []:
|
||||
if not isinstance(img, dict):
|
||||
continue
|
||||
url = img.get("urlDefault")
|
||||
if not url:
|
||||
for info in img.get("infoList") or []:
|
||||
if isinstance(info, dict) and info.get("imageScene") == "WB_DFT":
|
||||
url = info.get("url")
|
||||
break
|
||||
url = url or img.get("url")
|
||||
if url:
|
||||
images.append(url)
|
||||
if not images:
|
||||
logger.info(f"小红书文字笔记: {text[:30]}")
|
||||
return text, []
|
||||
@@ -290,16 +303,14 @@ def _extract_author_id(note: dict) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def _build_rel_stem(
|
||||
nickname: str, author_id: str, title: str, source: str = ""
|
||||
) -> str:
|
||||
def _build_rel_stem(nickname: str, author_id: str, title: str, source: str = "") -> str:
|
||||
"""相对平台根的路径词干：`{作者}_{userId}/{作品名}`
|
||||
|
||||
昵称/作者 id 都拿不到时用 source（笔记 id）当来源码（见 utils.build_author_dir）。
|
||||
"""
|
||||
return (
|
||||
f"{build_author_dir(nickname, author_id, source=source)}"
|
||||
f"/{build_work_stem(title)}"
|
||||
f"/{build_work_stem(title, source=source)}"
|
||||
)
|
||||
|
||||
|
||||
@@ -318,8 +329,8 @@ async def _download_images(image_urls: list[str], rel_stem: str) -> list[Path]:
|
||||
)
|
||||
|
||||
|
||||
async def _download_video(video_url: str, rel_stem: str) -> Path:
|
||||
"""流式下载视频
|
||||
async def _download_video(video_urls: list[str], rel_stem: str) -> Path:
|
||||
"""流式下载视频（masterUrl + backupUrls 逐个回退）
|
||||
|
||||
注意：sns-video-bd（无水印原片）不带 Referer 或带 xiaohongshu.com
|
||||
均可，但带 rednote.com Referer 会 403，因此不设 Referer。
|
||||
@@ -329,14 +340,23 @@ async def _download_video(video_url: str, rel_stem: str) -> Path:
|
||||
tmp_root = get_temp_root("xiaohongshu")
|
||||
output_path = unique_media_path(tmp_root / f"{rel_stem}.mp4")
|
||||
headers = {"User-Agent": REDNOTE_UA}
|
||||
last_error: Exception | None = None
|
||||
for video_url in video_urls:
|
||||
try:
|
||||
async with httpx.AsyncClient(headers=headers, timeout=300) as client:
|
||||
async with client.stream("GET", video_url) as resp:
|
||||
resp.raise_for_status()
|
||||
with open(output_path, "wb") as f:
|
||||
async for chunk in resp.aiter_bytes(8192):
|
||||
async for chunk in resp.aiter_bytes(65536):
|
||||
f.write(chunk)
|
||||
if output_path.stat().st_size == 0:
|
||||
raise ContentFetchError("响应为空（0 字节）")
|
||||
logger.info(f"小红书视频下载完成: {output_path}")
|
||||
return output_path
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
logger.warning(f"小红书视频下载失败（尝试下一个候选 CDN）: {e}")
|
||||
raise ContentFetchError(f"小红书视频下载失败: {last_error}")
|
||||
|
||||
|
||||
async def _fetch_page(url: str, headers: dict) -> str:
|
||||
|
||||
@@ -19,6 +19,7 @@ from ...utils import (
|
||||
build_work_stem,
|
||||
get_data_dir,
|
||||
get_temp_root,
|
||||
short_source_code,
|
||||
slugify,
|
||||
unique_media_path,
|
||||
)
|
||||
@@ -145,7 +146,8 @@ async def download_video(
|
||||
ext = m.group(1)
|
||||
|
||||
url_stem = Path(url.split("?")[0]).stem or "video"
|
||||
slug_stem = slugify(url_stem, max_length=15) or "视频"
|
||||
# 兜底名同 build_work_stem：用来源短码，别让不同直链挤在同一个 `视频` 下
|
||||
slug_stem = slugify(url_stem, max_length=15) or f"视频_{short_source_code(url)}"
|
||||
# 直链拿不到作者信息 → `未知作者_{来源短码}`（同一链接稳定、不同链接不撞）
|
||||
author_dir = build_author_dir(author, author_id, source=url)
|
||||
final_path = unique_media_path(temp_root / author_dir / f"{slug_stem}.{ext}")
|
||||
@@ -277,8 +279,10 @@ async def download_video(
|
||||
title = ((info or {}).get("title") or "").strip()
|
||||
|
||||
author_dir = build_author_dir(uploader, uploader_id, source=url)
|
||||
# 标题空时的兜底名用来源短码（作品 id 优先，其次作品链接）
|
||||
work_stem = build_work_stem(title, source=(info or {}).get("id") or url)
|
||||
new_path = unique_media_path(
|
||||
temp_root / author_dir / f"{build_work_stem(title)}{original_file.suffix}"
|
||||
temp_root / author_dir / f"{work_stem}{original_file.suffix}"
|
||||
)
|
||||
shutil.move(str(original_file), str(new_path))
|
||||
# 下载用的 scratch 目录已空，顺手收掉（cleanup 不删目录）
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
import zipfile
|
||||
from collections.abc import Iterable
|
||||
@@ -23,6 +24,7 @@ from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from nonebot import get_bot, logger
|
||||
from nonebot.exception import NetworkError
|
||||
|
||||
if TYPE_CHECKING: # 仅类型检查：运行期不导入，单测可裸加载本模块
|
||||
from ..policy import Policy
|
||||
@@ -154,12 +156,42 @@ def _file_uri(path: Path) -> str:
|
||||
return "file:///" + str(path.resolve()).replace("\\", "/")
|
||||
|
||||
|
||||
#: 群文件上传 per-call 超时（秒）——大文件在协议端「下载 + 上传群文件」会超过
|
||||
#: NoneBot 默认的 30s（同 handlers/sender.py 的 `send_timeout`，就地实现以保持
|
||||
#: 本模块可裸加载；配置键同一个 `video_analysis_send_timeout`）。
|
||||
_DEFAULT_SEND_TIMEOUT = 300.0
|
||||
|
||||
|
||||
def _send_timeout() -> float:
|
||||
raw = (
|
||||
os.environ.get("video_analysis_send_timeout")
|
||||
or os.environ.get("VIDEO_ANALYSIS_SEND_TIMEOUT")
|
||||
or ""
|
||||
).strip()
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
return _DEFAULT_SEND_TIMEOUT
|
||||
return value if value > 0 else _DEFAULT_SEND_TIMEOUT
|
||||
|
||||
|
||||
def _unknown_outcome(exc: BaseException) -> bool:
|
||||
"""超时类错误 = 结果未知：协议端可能仍在处理并最终送达（同 sender 的判定）。
|
||||
|
||||
这种失败**不能**再降级到本地直传重试——那会让群文件里出现两份。
|
||||
"""
|
||||
return isinstance(exc, NetworkError) or isinstance(
|
||||
exc, (asyncio.TimeoutError, TimeoutError)
|
||||
)
|
||||
|
||||
|
||||
async def _call_upload(
|
||||
file_value: str, group_id: int, name: str, folder_id: str | None = None
|
||||
) -> None:
|
||||
params: dict = {"group_id": group_id, "file": file_value, "name": name}
|
||||
if folder_id:
|
||||
params["folder"] = folder_id
|
||||
params["_timeout"] = _send_timeout()
|
||||
await get_bot().call_api("upload_group_file", **params)
|
||||
|
||||
|
||||
@@ -181,7 +213,9 @@ async def upload_group_file(
|
||||
) -> bool:
|
||||
"""上传单个文件到群文件，返回是否成功。
|
||||
|
||||
两级投递，**S3 链接优先、本地直传兜底**：
|
||||
两级投递，**S3 链接优先、本地直传兜底**；只有第 1 级**明确失败**才降级，
|
||||
超时（结果未知）直接收手——协议端可能还在传，再直传一份就重复了
|
||||
（见 `_unknown_outcome`）。
|
||||
|
||||
本环境实测 NapCat 读不到 bot 进程写的本地文件（`realpath ... ENOENT`，
|
||||
媒体消息的本地直发 55 次全失败、换 S3 链接后次次成功），所以直传只留作
|
||||
@@ -206,6 +240,14 @@ async def upload_group_file(
|
||||
try:
|
||||
await _call_upload(url, group_id, display_name, folder_id)
|
||||
except Exception as e:
|
||||
if _unknown_outcome(e):
|
||||
# 超时=结果未知：协议端可能还在传，再走本地直传会出两份群文件
|
||||
logger.warning(
|
||||
f"群文件 S3 链接上传超时，结果未知（协议端可能仍在传输），"
|
||||
f"不再降级直传以免重复（群 {group_id} / {display_name}）: "
|
||||
f"{str(e)[:160]}"
|
||||
)
|
||||
return True
|
||||
logger.warning(
|
||||
f"群文件 S3 链接上传失败（群 {group_id} / {display_name}）: "
|
||||
f"{str(e)[:160]}；改用本地路径重试"
|
||||
@@ -220,6 +262,12 @@ async def upload_group_file(
|
||||
try:
|
||||
await _call_upload(_file_uri(path), group_id, display_name, folder_id)
|
||||
except Exception as e:
|
||||
if _unknown_outcome(e):
|
||||
logger.warning(
|
||||
f"群文件上传超时，结果未知（协议端可能仍在传输）"
|
||||
f"（群 {group_id} / {display_name}）: {str(e)[:160]}"
|
||||
)
|
||||
return True
|
||||
logger.warning(
|
||||
f"群文件上传失败（群 {group_id} / {display_name}）: {str(e)[:160]}"
|
||||
)
|
||||
|
||||
@@ -66,6 +66,10 @@ POSTER_WIDTH = 480
|
||||
POSTER_SUFFIX = "_封面.jpg"
|
||||
POSTER_TIMEOUT = 20
|
||||
|
||||
#: bot 跑在 pythonw.exe（无控制台）下，spawn 控制台程序（ffmpeg）不带这个标志
|
||||
#: 会弹出一个 cmd 黑窗口；只在 Windows 有该常量，其它平台取 0 不影响调用
|
||||
CREATE_NO_WINDOW = getattr(subprocess, "CREATE_NO_WINDOW", 0)
|
||||
|
||||
#: 状态机：queued → running → done / failed
|
||||
ACTIVE_STATUSES = ("queued", "running")
|
||||
|
||||
@@ -136,9 +140,14 @@ async def make_poster(video: Path) -> Optional[Path]:
|
||||
str(out),
|
||||
]
|
||||
try:
|
||||
# 同步子进程必须丢线程池，否则抽帧期间整个事件循环都停摆
|
||||
# 同步子进程必须丢线程池，否则抽帧期间整个事件循环都停摆；
|
||||
# CREATE_NO_WINDOW 防止 pythonw 下弹 cmd 黑窗口
|
||||
await asyncio.to_thread(
|
||||
subprocess.run, cmd, capture_output=True, timeout=POSTER_TIMEOUT
|
||||
subprocess.run,
|
||||
cmd,
|
||||
capture_output=True,
|
||||
timeout=POSTER_TIMEOUT,
|
||||
creationflags=CREATE_NO_WINDOW,
|
||||
)
|
||||
except Exception: # noqa: BLE001 — 封面失败不该拖累任务
|
||||
logger.warning(f"抽关键帧失败：{video.name}")
|
||||
@@ -167,12 +176,14 @@ async def fetch_media(url: str) -> FetchResult:
|
||||
|
||||
分派规则与 `handlers/entry.py::dispatch_url` 一致：抖音 → parse_douyin；
|
||||
B站动态/专栏 → fetch_bilibili_content；其余 B站链接 → download_video（yt-dlp）；
|
||||
小红书 → fetch_rednote_content；其它平台 → download_video。
|
||||
b23 / xhslink / 抖音短链的重定向在各 fetcher 内部自己处理。
|
||||
小红书 → fetch_rednote_content；快手 → fetch_kuaishou_content；
|
||||
其它平台 → download_video。
|
||||
b23 / xhslink / 快手短链的重定向在各 fetcher 内部自己处理。
|
||||
"""
|
||||
from ..handlers.douyin import parse_douyin
|
||||
from ..policy import BILIBILI, DOUYIN, XHS
|
||||
from ..policy import BILIBILI, DOUYIN, KUAISHOU, XHS
|
||||
from .fetchers.bilibili_content import fetch_bilibili_content
|
||||
from .fetchers.kuaishou_content import fetch_kuaishou_content
|
||||
from .fetchers.rednote_content import fetch_rednote_content
|
||||
from .fetchers.video_downloader import download_video
|
||||
|
||||
@@ -183,6 +194,9 @@ async def fetch_media(url: str) -> FetchResult:
|
||||
|
||||
if platform == DOUYIN:
|
||||
title, parsed, image_post = await parse_douyin(url)
|
||||
elif platform == KUAISHOU:
|
||||
title, parsed = await fetch_kuaishou_content(url)
|
||||
image_post = isinstance(parsed, list)
|
||||
elif platform == XHS:
|
||||
title, parsed = await fetch_rednote_content(url)
|
||||
image_post = isinstance(parsed, list)
|
||||
@@ -344,7 +358,7 @@ class JobManager:
|
||||
# ── 清理 ──────────────────────────────────────────────
|
||||
|
||||
def remove(self, job_id: str) -> bool:
|
||||
"""删掉一条任务；还在跑的一并取消（Chrome 那头由 playwright 自己收尾）。"""
|
||||
"""删掉一条任务；还在跑的一并取消（进行中的下载协程由取消信号收尾）。"""
|
||||
job = self._jobs.pop(job_id, None)
|
||||
if job is None:
|
||||
return False
|
||||
|
||||
@@ -118,7 +118,7 @@ def short_time_code(at: Optional[float] = None) -> str:
|
||||
_JUNK_AUTHOR_IDS = {"", "0", "none", "null", "na", "nan", "undefined"}
|
||||
|
||||
#: 抓取层拿不到昵称时的兜底串（slugify 后）—— 它们等于"没有作者信息"
|
||||
_PLACEHOLDER_AUTHORS = {"未知作者", "小红书用户", "b站用户"}
|
||||
_PLACEHOLDER_AUTHORS = {"未知作者", "小红书用户", "b站用户", "快手用户"}
|
||||
|
||||
|
||||
def _clean_author_id(author_id: Optional[str]) -> str:
|
||||
@@ -174,9 +174,18 @@ def build_author_dir(
|
||||
return f"未知作者_{short_time_code(at)}"
|
||||
|
||||
|
||||
def build_work_stem(title: Optional[str]) -> str:
|
||||
"""作品名做文件名/子目录名：slugify（沿用 15 字上限），空则 `作品`"""
|
||||
return slugify(title or "", max_length=15) or "作品"
|
||||
def build_work_stem(title: Optional[str], *, source: Optional[str] = None) -> str:
|
||||
"""作品名做文件名/子目录名：slugify（沿用 15 字上限），空则 `作品_{来源短码}`
|
||||
|
||||
标题为空（或只有 `#话题`，slugify 后为空）时**不能**固定用 `作品`：同一作者的
|
||||
多个无名作品会全挤在一个名字下，只能靠 4 位**时间**短码区分，每次重解析名字都
|
||||
变，S3 key 与群文件成员名都不可复现。改用 `作品_{short_source_code(source)}`：
|
||||
同一来源稳定、不同来源不撞，与 `未知作者_{来源码}` 同一套约定。
|
||||
`source` 也没给才退回固定的 `作品`（保住旧契约）。
|
||||
"""
|
||||
return slugify(title or "", max_length=15) or (
|
||||
f"作品_{short_source_code(source)}" if source else "作品"
|
||||
)
|
||||
|
||||
|
||||
def unique_media_path(path: Path, *, at: Optional[float] = None) -> Path:
|
||||
@@ -227,7 +236,7 @@ def media_rel_dir_of(file_path: Path | str) -> str:
|
||||
return "/".join(parts[:-1]) if len(parts) > 1 else ""
|
||||
|
||||
|
||||
def parse_netscape_cookies(file_path: str) -> List[dict]:
|
||||
def parse_netscape_cookies(file_path: str | Path) -> List[dict]:
|
||||
"""解析 Netscape 格式 cookies 文件"""
|
||||
cookies = []
|
||||
with open(file_path, "r", encoding="utf-8") as f:
|
||||
|
||||
@@ -15,14 +15,15 @@
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi import Depends, FastAPI
|
||||
from fastapi.responses import JSONResponse
|
||||
|
||||
from nonebot import get_adapter
|
||||
from nonebot.adapters.onebot.v11 import Adapter
|
||||
|
||||
from hexi.web_hub.web_auth import require_admin
|
||||
from hexi.web_hub.web_auth import get_current_user, require_admin
|
||||
|
||||
from .cleanup import cleanup_config, run_stats, submit_clean
|
||||
from .policy import PLANS, PLATFORMS, STORE, Policy
|
||||
from .services.web_jobs import JOBS, extract_urls
|
||||
|
||||
@@ -167,6 +168,50 @@ def build_admin_app() -> FastAPI | None:
|
||||
removed = JOBS.clear_finished()
|
||||
return _ok({"removed": removed}, f"已清理 {removed} 条任务")
|
||||
|
||||
# ── temp 缓存：占用统计 + 手动清理 ──────────────────────
|
||||
# 与「清理temp / temp统计」命令、每日定时任务同一套函数（cleanup.py），
|
||||
# 不重写清理逻辑；文件系统操作丢线程池，别卡住事件循环。
|
||||
|
||||
@app.get("/temp/stats", response_class=JSONResponse, dependencies=[auth])
|
||||
async def temp_stats_api():
|
||||
"""temp 目录占用（文件数/总大小/目录数）+ 当前自动清理配置。"""
|
||||
st = await run_stats()
|
||||
st["mode"] = cleanup_config.video_analysis_temp_cleanup_mode
|
||||
st["max_age_days"] = cleanup_config.video_analysis_temp_max_age_days
|
||||
st["hour"] = cleanup_config.video_analysis_temp_cleanup_hour
|
||||
return _ok(st)
|
||||
|
||||
@app.post("/temp/cleanup", response_class=JSONResponse, dependencies=[auth])
|
||||
async def temp_cleanup_api(data: dict, user: dict = Depends(get_current_user)):
|
||||
"""手动清理 temp：**提交到任务槽后立即返回**，进度走 /hub/api/tasks/stream。
|
||||
|
||||
`days=0` 表示全部清空，不传则用配置的保留天数。清理本身是几十秒级的阻塞
|
||||
任务，放请求里会把页面交互一起卡住；同槽已有任务时返回它并带 `reused=true`
|
||||
（前端据此提示"已在运行"，服务端这层才是防连点的真正保证）。
|
||||
"""
|
||||
raw = data.get("days")
|
||||
if raw is None or str(raw).strip() == "":
|
||||
days: int | None = None # 用配置的保留天数
|
||||
shown = cleanup_config.video_analysis_temp_max_age_days
|
||||
else:
|
||||
try:
|
||||
days = max(0, int(raw))
|
||||
except (TypeError, ValueError):
|
||||
return _fail("天数参数不合法")
|
||||
shown = days
|
||||
|
||||
title = "清理 temp（全部）" if shown == 0 else f"清理 temp（保留 {shown} 天）"
|
||||
task, reused = submit_clean(
|
||||
days,
|
||||
title=title,
|
||||
trigger="web",
|
||||
actor=str((user or {}).get("username") or ""),
|
||||
)
|
||||
return _ok(
|
||||
{"task": task.to_dict(), "reused": reused},
|
||||
"已有清理任务在跑" if reused else "清理已开始",
|
||||
)
|
||||
|
||||
return app
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,309 @@
|
||||
"""抖音免浏览器链路（douyin_web.py）单元测试
|
||||
|
||||
覆盖：作品 id 提取 / 签名查询串 / 网关头构造 / 最高码率档选择 / 图集下载项收集。
|
||||
签名与媒体选择都是纯函数，不碰网络（真实链路实测见 debug 目录脚本）。
|
||||
|
||||
douyin_web.py 有插件内相对导入（`...models` / `...utils` / `.douyin_sign` 等），
|
||||
按 test_kuaishou_parse.py 同款姿势：sys.modules 预置包替身 + 真实加载依赖后裸加载。
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
_REPO = Path(__file__).resolve().parents[1]
|
||||
_PLUGIN = _REPO / "hexi" / "plugins" / "nonebot_plugin_video_analysis"
|
||||
_PKG = "hexi.plugins.nonebot_plugin_video_analysis"
|
||||
|
||||
_REAL_MODULES = {
|
||||
f"{_PKG}.models": _PLUGIN / "models.py",
|
||||
f"{_PKG}.utils": _PLUGIN / "utils.py",
|
||||
f"{_PKG}.services.fetchers.douyin_parser": _PLUGIN
|
||||
/ "services/fetchers/douyin_parser.py",
|
||||
f"{_PKG}.services.fetchers.douyin_sign": _PLUGIN
|
||||
/ "services/fetchers/douyin_sign.py",
|
||||
f"{_PKG}.services.fetchers.douyin_api": _PLUGIN / "services/fetchers/douyin_api.py",
|
||||
f"{_PKG}.services.fetchers.douyin_web": _PLUGIN / "services/fetchers/douyin_web.py",
|
||||
}
|
||||
_STUB_PACKAGES = (_PKG, f"{_PKG}.services", f"{_PKG}.services.fetchers")
|
||||
|
||||
|
||||
def _load_real(name: str, path: Path):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def dw():
|
||||
saved = {name: sys.modules.get(name) for name in _STUB_PACKAGES}
|
||||
for name in _STUB_PACKAGES:
|
||||
sys.modules[name] = types.ModuleType(name)
|
||||
for name, path in _REAL_MODULES.items():
|
||||
saved[name] = sys.modules.get(name)
|
||||
_load_real(name, path)
|
||||
|
||||
yield sys.modules[f"{_PKG}.services.fetchers.douyin_web"]
|
||||
|
||||
for name, previous in saved.items():
|
||||
if previous is None:
|
||||
sys.modules.pop(name, None)
|
||||
else:
|
||||
sys.modules[name] = previous
|
||||
|
||||
|
||||
# ───────────────────────── 样本 ─────────────────────────
|
||||
|
||||
|
||||
def _image(url_list=None, download_url_list=None, video=None) -> dict:
|
||||
img = {
|
||||
"uri": "img-uri",
|
||||
"width": 2160,
|
||||
"height": 2880,
|
||||
"url_list": (
|
||||
url_list
|
||||
if url_list is not None
|
||||
else ["https://p3.douyinpic.com/a~q75.webp"]
|
||||
),
|
||||
"download_url_list": (
|
||||
download_url_list
|
||||
if download_url_list is not None
|
||||
else ["https://p9.douyinpic.com/a~tplv-dy-water-v2:xxx.webp"]
|
||||
),
|
||||
}
|
||||
if video is not None:
|
||||
img["video"] = video
|
||||
return img
|
||||
|
||||
|
||||
def _video(bit_rate=None, play_urls=("https://v95.douyinvod.com/full.mp4",)) -> dict:
|
||||
v = {"play_addr": {"url_list": list(play_urls), "width": 1080, "height": 1440}}
|
||||
if bit_rate is not None:
|
||||
v["bit_rate"] = bit_rate
|
||||
return v
|
||||
|
||||
|
||||
def _gear(br, urls, gear_name="gear", h265=0, size=100):
|
||||
return {
|
||||
"gear_name": gear_name,
|
||||
"bit_rate": br,
|
||||
"is_h265": h265,
|
||||
"play_addr": {
|
||||
"url_list": list(urls),
|
||||
"width": 1080,
|
||||
"height": 1440,
|
||||
"data_size": size,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ───────────────────────── extract_aweme_id ─────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url,expected",
|
||||
[
|
||||
("https://www.douyin.com/note/7696029938990839419", "7696029938990839419"),
|
||||
(
|
||||
"https://www.douyin.com/video/7618117951811949541?foo=1",
|
||||
"7618117951811949541",
|
||||
),
|
||||
(
|
||||
"https://www.iesdouyin.com/share/video/7412345678901234567/?region=CN",
|
||||
"7412345678901234567",
|
||||
),
|
||||
(
|
||||
"https://www.iesdouyin.com/share/note/7412345678901234567/",
|
||||
"7412345678901234567",
|
||||
),
|
||||
("https://v.douyin.com/abc123/", None),
|
||||
("https://github.com/x/y", None),
|
||||
],
|
||||
)
|
||||
def test_extract_aweme_id(dw, url, expected):
|
||||
assert dw.extract_aweme_id(url) == expected
|
||||
|
||||
|
||||
# ───────────────────────── 签名 ─────────────────────────
|
||||
|
||||
|
||||
def test_build_signed_query(dw):
|
||||
qs = dw.build_signed_query("7618117951811949541")
|
||||
assert "aweme_id=7618117951811949541" in qs
|
||||
assert "a_bogus=" in qs
|
||||
a_bogus = qs.split("a_bogus=")[-1]
|
||||
assert len(a_bogus) >= 100
|
||||
# 两次签名都有效（签名带随机熵，不要求逐字节一致）
|
||||
qs2 = dw.build_signed_query("7618117951811949541")
|
||||
assert "a_bogus=" in qs2
|
||||
|
||||
|
||||
def test_build_signed_query_requires_aweme_id_last(dw):
|
||||
"""aweme_id 应在查询串末尾、a_bogus 追加在其后（与浏览器请求形态一致）"""
|
||||
qs = dw.build_signed_query("123")
|
||||
assert qs.index("aweme_id=123") < qs.index("a_bogus=")
|
||||
|
||||
|
||||
# ───────────────────────── 网关头 ─────────────────────────
|
||||
|
||||
|
||||
def test_build_headers_always_argus_and_uifid(dw):
|
||||
headers = dw._build_headers({"UIFID": "u-123"})
|
||||
assert headers["x-tt-argus"] == "1"
|
||||
assert headers["uifid"] == "u-123"
|
||||
assert headers["User-Agent"] == dw.WEB_UA
|
||||
|
||||
|
||||
def test_build_headers_uifid_temp_fallback(dw):
|
||||
headers = dw._build_headers({"UIFID_TEMP": "tmp"})
|
||||
assert headers["uifid"] == "tmp"
|
||||
assert "uifid" not in dw._build_headers({})
|
||||
|
||||
|
||||
def test_cookie_map_filters_domain(dw):
|
||||
cookies = [
|
||||
{"name": "UIFID", "value": "u1", "domain": ".douyin.com"},
|
||||
{"name": "a1", "value": "x", "domain": ".xiaohongshu.com"},
|
||||
{"name": "ttwid", "value": "t", "domain": "www.douyin.com"},
|
||||
]
|
||||
m = dw._douyin_cookie_map(cookies)
|
||||
assert m == {"UIFID": "u1", "ttwid": "t"}
|
||||
|
||||
|
||||
# ───────────────────────── 最高码率档选择 ─────────────────────────
|
||||
|
||||
|
||||
def test_best_gear_picks_max_bitrate(dw):
|
||||
video = {
|
||||
"bit_rate": [
|
||||
_gear(773939, ["https://cdn/1080.mp4"], gear_name="normal_1080_0"),
|
||||
_gear(385679, ["https://cdn/720.mp4"], gear_name="720_1_1", h265=1),
|
||||
_gear(
|
||||
1419088,
|
||||
["https://cdn/best.mp4", "https://cdn/best-bak.mp4"],
|
||||
gear_name="1080_1_1",
|
||||
),
|
||||
],
|
||||
"play_addr": {"url_list": ["https://cdn/main.mp4"]},
|
||||
}
|
||||
urls, meta = dw._best_urls_from_video(video)
|
||||
assert urls == ["https://cdn/best.mp4", "https://cdn/best-bak.mp4"]
|
||||
assert meta["bit_rate"] == 1419088
|
||||
assert meta["width"] == 1080
|
||||
|
||||
|
||||
def test_best_gear_falls_back_to_play_addr(dw):
|
||||
video = _video(bit_rate=[], play_urls=["https://cdn/only.mp4"])
|
||||
urls, meta = dw._best_urls_from_video(video)
|
||||
assert urls == ["https://cdn/only.mp4"]
|
||||
assert meta["gear_name"] is None
|
||||
|
||||
|
||||
def test_best_gear_skips_gear_without_urls(dw):
|
||||
video = {
|
||||
"bit_rate": [
|
||||
{"bit_rate": 9999999, "play_addr": {"url_list": []}},
|
||||
_gear(100, ["https://cdn/ok.mp4"]),
|
||||
],
|
||||
"play_addr": {"url_list": ["https://cdn/main.mp4"]},
|
||||
}
|
||||
urls, _ = dw._best_urls_from_video(video)
|
||||
assert urls == ["https://cdn/ok.mp4"]
|
||||
|
||||
|
||||
# ───────────────────────── 下载项收集 ─────────────────────────
|
||||
|
||||
|
||||
def test_collect_video_post(dw):
|
||||
detail = {
|
||||
"aweme_id": "1",
|
||||
"desc": "x",
|
||||
"video": _video(bit_rate=[_gear(500, ["https://cdn/v.mp4"])]),
|
||||
}
|
||||
items, is_img = dw.collect_download_items(detail)
|
||||
assert is_img is False
|
||||
assert items == [(["https://cdn/v.mp4"], ".mp4")]
|
||||
|
||||
|
||||
def test_collect_video_post_without_urls_raises(dw):
|
||||
with pytest.raises(Exception):
|
||||
dw.collect_download_items({"video": {"play_addr": {"url_list": []}}})
|
||||
|
||||
|
||||
def test_collect_images_prefers_no_watermark_urls(dw):
|
||||
detail = {
|
||||
"images": [
|
||||
_image(
|
||||
url_list=["https://p3/a-q75.webp", "https://p3/a-q75.jpeg"],
|
||||
download_url_list=["https://p9/a-water.webp"],
|
||||
)
|
||||
]
|
||||
}
|
||||
items, is_img = dw.collect_download_items(detail)
|
||||
assert is_img is True
|
||||
assert items[0][0] == [
|
||||
"https://p3/a-q75.webp",
|
||||
"https://p3/a-q75.jpeg", # 无水印在前
|
||||
"https://p9/a-water.webp", # 带作者水印的保存版兜底
|
||||
]
|
||||
assert items[0][1] == ""
|
||||
|
||||
|
||||
def test_collect_animated_note_uses_video_gears(dw):
|
||||
anim_video = {
|
||||
"bit_rate": [_gear(300, ["https://cdn/anim-best.mp4"])],
|
||||
"play_addr": {"url_list": ["https://cdn/anim.mp4"]},
|
||||
}
|
||||
detail = {
|
||||
"images": [
|
||||
_image(url_list=[], download_url_list=[], video=anim_video),
|
||||
_image(url_list=[], download_url_list=[], video=anim_video),
|
||||
]
|
||||
}
|
||||
items, is_img = dw.collect_download_items(detail)
|
||||
assert is_img is True
|
||||
assert items == [
|
||||
(["https://cdn/anim-best.mp4"], ".mp4"),
|
||||
(["https://cdn/anim-best.mp4"], ".mp4"),
|
||||
]
|
||||
|
||||
|
||||
def test_collect_mixed_note(dw):
|
||||
"""图 + 内嵌视频混排：各自按类型收集"""
|
||||
detail = {
|
||||
"images": [
|
||||
_image(),
|
||||
_image(
|
||||
url_list=[],
|
||||
download_url_list=[],
|
||||
video={"play_addr": {"url_list": ["https://cdn/clip.mp4"]}},
|
||||
),
|
||||
]
|
||||
}
|
||||
items, is_img = dw.collect_download_items(detail)
|
||||
assert is_img is True
|
||||
assert len(items) == 2
|
||||
assert items[0][1] == "" # 图片
|
||||
assert items[1] == (["https://cdn/clip.mp4"], ".mp4")
|
||||
|
||||
|
||||
# ───────────────────────── 扩展名推断 ─────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url,expected",
|
||||
[
|
||||
("https://x/a.jpg?x=1", ".jpg"),
|
||||
("https://x/a~q75.webp?y=2", ".webp"),
|
||||
("https://x/a.jpeg", ".jpeg"),
|
||||
("https://x/a", ".jpg"),
|
||||
("https://x/a.heic", ".heic"),
|
||||
],
|
||||
)
|
||||
def test_infer_extension(dw, url, expected):
|
||||
assert dw._infer_extension(url) == expected
|
||||
@@ -0,0 +1,302 @@
|
||||
"""快手分享页解析（kuaishou_content.py 纯函数）单元测试
|
||||
|
||||
只测不碰网络的解析/命名部分：INIT_STATE 提取、视频/图集 URL 组装、媒体命名。
|
||||
样本仿自 2026-10-08 实测的分享页（v.kuaishou.com 短链 → v.m.chenzhongtech.com/fw/photo）。
|
||||
|
||||
kuaishou_content.py 有插件内相对导入（`...models` / `...utils`），无法裸加载，
|
||||
故在 sys.modules 里预置父包替身 + 真实加载 utils/models 后按路径加载目标模块
|
||||
（与 test_video_sender_link.py 同法）。
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
_REPO = Path(__file__).resolve().parents[1]
|
||||
_PLUGIN = _REPO / "hexi" / "plugins" / "nonebot_plugin_video_analysis"
|
||||
_PKG = "hexi.plugins.nonebot_plugin_video_analysis"
|
||||
|
||||
#: 真实加载的插件内模块（utils 纯标准库、models 只有一个异常类）
|
||||
_REAL_MODULES = {
|
||||
f"{_PKG}.models": _PLUGIN / "models.py",
|
||||
f"{_PKG}.utils": _PLUGIN / "utils.py",
|
||||
}
|
||||
#: 只当命名空间用的父包（避免触发插件包 __init__ 的 NoneBot 初始化）
|
||||
_STUB_PACKAGES = (_PKG, f"{_PKG}.services", f"{_PKG}.services.fetchers")
|
||||
|
||||
|
||||
def _load_real(name: str, path: Path):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def ks():
|
||||
saved = {name: sys.modules.get(name) for name in _STUB_PACKAGES}
|
||||
for name in _STUB_PACKAGES:
|
||||
sys.modules[name] = types.ModuleType(name)
|
||||
for name, path in _REAL_MODULES.items():
|
||||
saved[name] = sys.modules.get(name)
|
||||
_load_real(name, path)
|
||||
|
||||
target = f"{_PKG}.services.fetchers.kuaishou_content"
|
||||
saved[target] = sys.modules.get(target)
|
||||
module = _load_real(target, _PLUGIN / "services" / "fetchers" / "kuaishou_content.py")
|
||||
|
||||
yield module
|
||||
|
||||
for name, previous in saved.items():
|
||||
if previous is None:
|
||||
sys.modules.pop(name, None)
|
||||
else:
|
||||
sys.modules[name] = previous
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def u(ks):
|
||||
"""插件 utils（命名断言用同一套函数，别在测试里手拼）"""
|
||||
return sys.modules[f"{_PKG}.utils"]
|
||||
|
||||
|
||||
# ───────────────────────── 样本 ─────────────────────────
|
||||
|
||||
KEY = "string/rest/wd/ugH5App/photo/simple/info@kpn>KUAISHOU"
|
||||
|
||||
|
||||
def _photo(**over) -> dict:
|
||||
photo = {
|
||||
"caption": "欧巴欧巴哦 #蔚蓝档案 #砂狼白子",
|
||||
"userName": "砂狼白子",
|
||||
"userId": 2329823245,
|
||||
"photoId": "5206161376908999738",
|
||||
"duration": 15150,
|
||||
"timestamp": 1789916802288,
|
||||
"coverUrls": [
|
||||
{"cdn": "p2.a.yximgs.com", "url": "https://p2.a.yximgs.com/cover.jpg"}
|
||||
],
|
||||
"mainMvUrls": [
|
||||
{"cdn": "tymov2.a.kwimgs.com", "url": "https://tymov2.a.kwimgs.com/a_b.mp4"},
|
||||
{"cdn": "txmov2.a.kwimgs.com", "url": "https://txmov2.a.kwimgs.com/a_b.mp4"},
|
||||
],
|
||||
"ext_params": {"mtype": 3, "video": 15150},
|
||||
}
|
||||
photo.update(over)
|
||||
return photo
|
||||
|
||||
|
||||
def _page(data: dict) -> str:
|
||||
payload = json.dumps(data, ensure_ascii=False)
|
||||
return f"<!doctype html><html><script>window.INIT_STATE={payload}</script></html>"
|
||||
|
||||
|
||||
def _page_with(photo: dict | None, prelude: dict | None = None) -> str:
|
||||
data = dict(prelude or {})
|
||||
data[KEY] = {"result": 1, "photo": photo}
|
||||
return _page(data)
|
||||
|
||||
|
||||
# ───────────────────────── INIT_STATE 提取 ─────────────────────────
|
||||
|
||||
|
||||
def test_extract_photo_skips_empty_entries(ks):
|
||||
"""INIT_STATE 是一堆缓存键的 dict，photo 在第一个非空项里（顺序不定）"""
|
||||
page = _page(
|
||||
{
|
||||
"some/cache/key": {"result": 1, "photo": None},
|
||||
"another/key": {"result": 2},
|
||||
KEY: {"result": 1, "photo": _photo()},
|
||||
}
|
||||
)
|
||||
assert ks._extract_photo(page)["photoId"] == "5206161376908999738"
|
||||
|
||||
|
||||
def test_extract_photo_tolerates_js_literals(ks):
|
||||
"""页面偶有 undefined 之类非 JSON 字面量（同小红书预处理）"""
|
||||
page = (
|
||||
"<script>window.INIT_STATE = "
|
||||
'{"k": {"result": 1, "photo": {"photoId": "1", "userName": undefined}}}'
|
||||
"</script>"
|
||||
)
|
||||
photo = ks._extract_photo(page)
|
||||
assert photo["photoId"] == "1"
|
||||
assert photo["userName"] is None
|
||||
|
||||
|
||||
def test_extract_photo_missing_state(ks):
|
||||
"""风控壳页（桌面 UA / www 站）没有 INIT_STATE → 报错而不是静默返回空"""
|
||||
err = sys.modules[f"{_PKG}.models"].ContentFetchError
|
||||
with pytest.raises(err):
|
||||
ks._extract_photo("<html><body>risk control</body></html>")
|
||||
|
||||
|
||||
def test_extract_photo_without_photo(ks):
|
||||
"""INIT_STATE 在但没有作品（已删除/私密）→ 报错"""
|
||||
err = sys.modules[f"{_PKG}.models"].ContentFetchError
|
||||
with pytest.raises(err):
|
||||
ks._extract_photo(_page({"k": {"result": 2, "photo": None}}))
|
||||
|
||||
|
||||
# ───────────────────────── 媒体 URL ─────────────────────────
|
||||
|
||||
|
||||
def test_first_url_prefers_first_available(ks):
|
||||
assert ks._first_url([{"cdn": "a", "url": "https://a/1.mp4"}]) == "https://a/1.mp4"
|
||||
# 首项没有 url（快手同一资源的备用项可能缺字段）时继续往后找
|
||||
assert ks._first_url([{"cdn": "a"}, {"url": "https://b/2.mp4"}]) == "https://b/2.mp4"
|
||||
assert ks._first_url([]) is None
|
||||
assert ks._first_url(None) is None
|
||||
|
||||
|
||||
def _rep(width, height, fps, codec, bitrate, url, **over) -> dict:
|
||||
rep = {
|
||||
"width": width,
|
||||
"height": height,
|
||||
"frameRate": fps,
|
||||
"videoCodec": codec,
|
||||
"avgBitrate": bitrate,
|
||||
"url": url,
|
||||
}
|
||||
rep.update(over)
|
||||
return rep
|
||||
|
||||
|
||||
def test_manifest_video_url_picks_highest_quality(ks):
|
||||
"""清晰度梯取最优：先比分辨率，再帧率、编码（avc 优先）、码率"""
|
||||
photo = _photo(
|
||||
manifest={
|
||||
"adaptationSet": [
|
||||
{
|
||||
"representation": [
|
||||
# 1080p 30fps hevc 也要赢过 720p 60fps avc（分辨率优先）
|
||||
_rep(1920, 1080, 30, "hevc", 2000, "https://c/1080.mp4"),
|
||||
_rep(1280, 720, 60, "avc", 1597, "https://c/720p60.mp4"),
|
||||
_rep(1280, 720, 30, "hevc", 726, "https://c/720p30.mp4"),
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
assert ks._manifest_video_url(photo) == "https://c/1080.mp4"
|
||||
|
||||
|
||||
def test_manifest_video_url_prefers_avc_on_tie(ks):
|
||||
"""同分辨率同帧率 → avc 优先（兼容性），hevc 只在画质更高时才选"""
|
||||
photo = _photo(
|
||||
manifest={
|
||||
"adaptationSet": [
|
||||
{
|
||||
"representation": [
|
||||
_rep(1280, 720, 60, "avc", 1597, "https://c/avc.mp4"),
|
||||
# 码率更高但编码是 hevc，仍输给 avc
|
||||
_rep(1280, 720, 60, "hevc", 2000, "https://c/hevc.mp4"),
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
assert ks._manifest_video_url(photo) == "https://c/avc.mp4"
|
||||
|
||||
|
||||
def test_manifest_video_url_skips_broken_entries(ks):
|
||||
"""缺 url / 结构不对的档跳过，不因为一条脏数据整体失败"""
|
||||
photo = _photo(
|
||||
manifest={
|
||||
"adaptationSet": [
|
||||
{"representation": [{"width": 1920, "height": 1080}, "junk"]},
|
||||
{"representation": [_rep(1280, 720, 30, "avc", 900, "https://c/720.mp4")]},
|
||||
]
|
||||
}
|
||||
)
|
||||
assert ks._manifest_video_url(photo) == "https://c/720.mp4"
|
||||
assert ks._manifest_video_url(_photo()) is None
|
||||
assert ks._manifest_video_url(_photo(manifest={"adaptationSet": []})) is None
|
||||
|
||||
|
||||
def test_video_url_falls_back_to_main_mv_urls(ks):
|
||||
"""老页面没有 manifest → 退回 mainMvUrls（参考实现的默认档路径）"""
|
||||
assert ks._video_url(_photo()) == "https://tymov2.a.kwimgs.com/a_b.mp4"
|
||||
photo = _photo(
|
||||
manifest={"adaptationSet": [{"representation": [_rep(1920, 1080, 30, "avc", 2000, "https://c/best.mp4")]}]}
|
||||
)
|
||||
assert ks._video_url(photo) == "https://c/best.mp4"
|
||||
|
||||
|
||||
def test_atlas_image_urls(ks):
|
||||
photo = _photo(
|
||||
ext_params={
|
||||
"atlas": {
|
||||
"cdnList": [{"cdn": "p1.a.yximgs.com"}, {"cdn": "p2.a.yximgs.com"}],
|
||||
"list": ["upic/a.jpg", "upic/b.jpg"],
|
||||
}
|
||||
}
|
||||
)
|
||||
assert ks._atlas_image_urls(photo) == [
|
||||
"https://p1.a.yximgs.com/upic/a.jpg",
|
||||
"https://p1.a.yximgs.com/upic/b.jpg",
|
||||
]
|
||||
|
||||
|
||||
def test_atlas_image_urls_absent(ks):
|
||||
"""视频作品的 ext_params 里没有 atlas → 不是图集"""
|
||||
assert ks._atlas_image_urls(_photo()) == []
|
||||
assert ks._atlas_image_urls(_photo(ext_params={})) == []
|
||||
# 有 atlas 但缺图床/路径 → 空（宁可当视频处理也不要拼出坏链接）
|
||||
assert ks._atlas_image_urls(_photo(ext_params={"atlas": {"list": ["a.jpg"]}})) == []
|
||||
assert ks._atlas_image_urls(
|
||||
_photo(ext_params={"atlas": {"cdnList": [{"cdn": "c"}]}})
|
||||
) == []
|
||||
# 老字段名 extParams 也认（ext_params 缺失时才会去看它）
|
||||
assert ks._atlas_image_urls(
|
||||
_photo(
|
||||
ext_params=None,
|
||||
extParams={"atlas": {"cdnList": [{"cdn": "c"}], "list": ["a.jpg"]}},
|
||||
)
|
||||
) == ["https://c/a.jpg"]
|
||||
|
||||
|
||||
# ───────────────────────── 命名 ─────────────────────────
|
||||
|
||||
|
||||
def test_rel_stem_uses_author_id(ks, u):
|
||||
"""有 userId（数字）→ 稳定作者目录，标题去掉 #话题"""
|
||||
assert ks._build_rel_stem(_photo()) == "砂狼白子_2329823245/欧巴欧巴哦"
|
||||
|
||||
|
||||
def test_rel_stem_without_author(ks, u):
|
||||
"""昵称是兜底串（"快手用户"）→ 按"未知作者"规则 + 作品来源码"""
|
||||
stem = ks._build_rel_stem(_photo(userName="快手用户", userId=0))
|
||||
code = u.short_source_code("5206161376908999738")
|
||||
assert stem == f"未知作者_{code}/欧巴欧巴哦"
|
||||
|
||||
|
||||
def test_rel_stem_empty_caption(ks, u):
|
||||
"""标题只有 #话题（slugify 后为空）→ 作品_来源码，同一作品稳定可复现"""
|
||||
stem = ks._build_rel_stem(_photo(caption="#蔚蓝档案"))
|
||||
code = u.short_source_code("5206161376908999738")
|
||||
assert stem == f"砂狼白子_2329823245/作品_{code}"
|
||||
|
||||
|
||||
# ───────────────────────── URL 识别 ─────────────────────────
|
||||
|
||||
|
||||
def test_web_short_video_regex(ks):
|
||||
"""www 站形态（分享出来的网页链接，实际会重写成移动端分享页）"""
|
||||
for url, expected in (
|
||||
("https://www.kuaishou.com/short-video/3xhjgcmir24m4nm", "3xhjgcmir24m4nm"),
|
||||
("https://www.kuaishou.com/short-video/5206161376908999738", "5206161376908999738"),
|
||||
("https://kuaishou.com/short-video/abc123", "abc123"),
|
||||
):
|
||||
m = ks.WEB_SHORT_VIDEO_RE.search(url)
|
||||
assert m and m.group(1) == expected
|
||||
|
||||
|
||||
def test_short_link_regex(ks):
|
||||
m = ks.SHORT_LINK_RE.search("看看这个 https://v.kuaishou.com/nUqhAiDI 好看")
|
||||
assert m and m.group(0) == "v.kuaishou.com/nUqhAiDI"
|
||||
@@ -0,0 +1,214 @@
|
||||
"""temp 目录清理逻辑（video_analysis/cleanup.py）单元测试
|
||||
|
||||
这组函数现在同时被「/清理temp 命令」「每日定时」「Web 缓存面板的手动清理」
|
||||
三条路复用，是**真的会删文件**的代码，所以单测覆盖"删什么/不删什么"。
|
||||
|
||||
cleanup.py 有插件内相对导入（`.utils`）和 import 期副作用（`get_plugin_config`
|
||||
建配置对象、`_register_auto_job()` 注册定时任务），无法裸加载 —— 按
|
||||
test_video_sender_link.py 的办法：sys.modules 预置替身 + monkeypatch nonebot 的
|
||||
几个工厂函数，再按路径加载目标模块。
|
||||
|
||||
文件一律用 tmp_path 造真实文件/目录 + `os.utime` 改 mtime，**不 mock 文件系统**：
|
||||
这些判断（mtime 过期、目录为空、60s 目录下限）全在真实 stat 语义上。
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
_REPO = Path(__file__).resolve().parents[1]
|
||||
_PLUGIN = _REPO / "hexi" / "plugins" / "nonebot_plugin_video_analysis"
|
||||
_PKG = "hexi.plugins.nonebot_plugin_video_analysis"
|
||||
|
||||
_STUBS = {
|
||||
"hexi": {},
|
||||
"hexi.core": {},
|
||||
"hexi.core.custom_utils": {"check_manage": None},
|
||||
"hexi.core.scheduler": {"add_job": lambda *a, **k: None},
|
||||
# cleanup.py 现在把清理提交到任务槽；这里只测 clean_temp_files / temp_stats，
|
||||
# 槽本身由 tests/test_task_slots.py 覆盖，所以给个够 import 的替身即可。
|
||||
"hexi.core.task_slots": {"TASKS": None, "Task": type("Task", (), {})},
|
||||
"nonebot_plugin_apscheduler": {"scheduler": types.SimpleNamespace(remove_job=None)},
|
||||
"nonebot_plugin_alconna": {"UniMessage": type("UniMessage", (), {})},
|
||||
}
|
||||
|
||||
|
||||
class _FakeMatcher:
|
||||
"""`on_command(...)` 的替身：`.handle()` 当装饰器用。"""
|
||||
|
||||
def handle(self, *args, **kwargs):
|
||||
def deco(func):
|
||||
return func
|
||||
|
||||
return deco
|
||||
|
||||
|
||||
class _TempRoot:
|
||||
"""`utils.get_temp_root` 的替身：指向测试用的临时根。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.root: Path | None = None
|
||||
|
||||
def __call__(self, sub: str = "") -> Path:
|
||||
assert self.root is not None, "测试里要先 sched.get_temp_root.root = ..."
|
||||
return self.root / sub if sub else self.root
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cleanup(monkeypatch, tmp_path):
|
||||
saved = {name: sys.modules.get(name) for name in _STUBS}
|
||||
for name, attrs in _STUBS.items():
|
||||
module = types.ModuleType(name)
|
||||
for key, value in attrs.items():
|
||||
setattr(module, key, value)
|
||||
sys.modules[name] = module
|
||||
|
||||
# 插件包本体也用替身，避免触发 __init__ 的 NoneBot 初始化
|
||||
saved[_PKG] = sys.modules.get(_PKG)
|
||||
sys.modules[_PKG] = types.ModuleType(_PKG)
|
||||
temp_root = _TempRoot()
|
||||
utils_mod = types.ModuleType(f"{_PKG}.utils")
|
||||
utils_mod.get_temp_root = temp_root
|
||||
saved[f"{_PKG}.utils"] = sys.modules.get(f"{_PKG}.utils")
|
||||
sys.modules[f"{_PKG}.utils"] = utils_mod
|
||||
|
||||
import nonebot
|
||||
|
||||
monkeypatch.setattr(nonebot, "require", lambda name: None, raising=False)
|
||||
monkeypatch.setattr(
|
||||
nonebot, "get_plugin_config", lambda cls: cls(), raising=False
|
||||
) # 默认配置：auto / 7 天 / 4 点
|
||||
monkeypatch.setattr(nonebot, "on_command", lambda *a, **k: _FakeMatcher(), raising=False)
|
||||
|
||||
target = f"{_PKG}.cleanup"
|
||||
saved[target] = sys.modules.get(target)
|
||||
spec = importlib.util.spec_from_file_location(target, _PLUGIN / "cleanup.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[target] = module
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
temp_root.root = tmp_path / "temp"
|
||||
temp_root.root.mkdir()
|
||||
module._tmp = temp_root.root # 方便测试直接拿到根
|
||||
|
||||
yield module
|
||||
|
||||
for name, previous in saved.items():
|
||||
if previous is None:
|
||||
sys.modules.pop(name, None)
|
||||
else:
|
||||
sys.modules[name] = previous
|
||||
|
||||
|
||||
def _make(root: Path, rel: str, age_days: float = 0.0, data: bytes = b"x") -> Path:
|
||||
"""造一个文件，并把 mtime 往回拨 age_days 天。"""
|
||||
path = root / rel
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
if age_days:
|
||||
t = time.time() - age_days * 86400
|
||||
os.utime(path, (t, t))
|
||||
return path
|
||||
|
||||
|
||||
# ───────────────────────── 清理 ─────────────────────────
|
||||
|
||||
|
||||
def test_removes_stale_keeps_fresh(cleanup):
|
||||
root = cleanup._tmp
|
||||
stale = _make(root, "douyin/作者_1/旧作品.mp4", age_days=2)
|
||||
fresh = _make(root, "douyin/作者_1/新作品.mp4", age_days=0)
|
||||
|
||||
removed, total = cleanup.clean_temp_files(days=1)
|
||||
|
||||
assert (removed, total) == (1, 2)
|
||||
assert not stale.exists()
|
||||
assert fresh.exists()
|
||||
|
||||
|
||||
def test_days_zero_clears_files_but_keeps_fresh_dirs(cleanup):
|
||||
"""`/清理temp 0`（= Web 的「全部清空」）：文件全删，但刚建的目录留着。
|
||||
|
||||
目录另有 60s 下限（见 `_prune_empty_dirs`）—— 下载落盘是先 mkdir 再写文件，
|
||||
把目录一起删掉会让正在进行的下载失败。
|
||||
"""
|
||||
root = cleanup._tmp
|
||||
a = _make(root, "kuaishou/作者_2/作品.mp4", age_days=0)
|
||||
b = _make(root, "xiaohongshu/作者_3/笔记/001.jpg", age_days=0)
|
||||
|
||||
removed, total = cleanup.clean_temp_files(days=0)
|
||||
|
||||
assert (removed, total) == (2, 2)
|
||||
assert not a.exists() and not b.exists()
|
||||
assert (root / "kuaishou/作者_2").is_dir() # 新鲜目录保留
|
||||
assert (root / "xiaohongshu/作者_3/笔记").is_dir()
|
||||
|
||||
|
||||
def test_prunes_stale_empty_dirs_bottom_up(cleanup):
|
||||
"""过期空目录会被收掉，但是**逐轮**回收而不是同一轮全清。
|
||||
|
||||
删文件（或删子目录）会把所在目录的 mtime 刷成"现在"，当轮它就不再"过期"，
|
||||
要等下一轮 —— 这正是 `_prune_empty_dirs` 注释里写的行为（也被这条测试钉住）。
|
||||
"""
|
||||
root = cleanup._tmp
|
||||
_make(root, "bilibili/作者_4/作品/001.jpg", age_days=3)
|
||||
nested = root / "bilibili/作者_4/作品"
|
||||
author = root / "bilibili/作者_4"
|
||||
|
||||
removed, _ = cleanup.clean_temp_files(days=1)
|
||||
assert removed == 1
|
||||
assert nested.is_dir() and author.is_dir() # 文件刚删，目录 mtime 是新的
|
||||
|
||||
old = time.time() - 3 * 86400 # 模拟又过了几天
|
||||
os.utime(nested, (old, old))
|
||||
os.utime(author, (old, old))
|
||||
cleanup.clean_temp_files(days=1)
|
||||
assert not nested.exists() # 深的先走
|
||||
|
||||
os.utime(author, (old, old)) # 上一轮删子目录，又把父目录的 mtime 刷新了
|
||||
cleanup.clean_temp_files(days=1)
|
||||
assert not author.exists()
|
||||
|
||||
|
||||
def test_keeps_stale_but_nonempty_dirs(cleanup):
|
||||
"""过期但里面还有新鲜文件的目录不能删。"""
|
||||
root = cleanup._tmp
|
||||
_make(root, "douyin/作者_5/新片.mp4", age_days=0)
|
||||
old_dir = root / "douyin/作者_5"
|
||||
os.utime(old_dir, (time.time() - 3 * 86400,) * 2)
|
||||
|
||||
cleanup.clean_temp_files(days=1)
|
||||
|
||||
assert old_dir.is_dir()
|
||||
|
||||
|
||||
def test_missing_root_is_noop(cleanup):
|
||||
"""根目录不存在（还没下载过任何东西）→ (0, 0)，不抛。"""
|
||||
cleanup._tmp.rmdir()
|
||||
assert cleanup.clean_temp_files(days=1) == (0, 0)
|
||||
|
||||
|
||||
# ───────────────────────── 统计 ─────────────────────────
|
||||
|
||||
|
||||
def test_stats_counts_files_bytes_dirs(cleanup):
|
||||
root = cleanup._tmp
|
||||
_make(root, "douyin/作者_6/a.mp4", age_days=2, data=b"12345")
|
||||
_make(root, "douyin/作者_6/b.mp4", age_days=0, data=b"123")
|
||||
_make(root, "douyin/作者_7/c.jpg", age_days=5, data=b"1")
|
||||
|
||||
st = cleanup.temp_stats()
|
||||
|
||||
assert st["files"] == 3
|
||||
assert st["bytes"] == 9
|
||||
assert st["dirs"] == 3 # douyin / 作者_6 / 作者_7（子目录各算一层）
|
||||
|
||||
|
||||
def test_stats_missing_root(cleanup):
|
||||
cleanup._tmp.rmdir()
|
||||
assert cleanup.temp_stats() == {"files": 0, "bytes": 0, "dirs": 0}
|
||||
@@ -11,6 +11,7 @@ import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from nonebot.adapters.onebot.v11.exception import NetworkError
|
||||
|
||||
_MODULE_PATH = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
@@ -223,6 +224,75 @@ async def test_missing_file_short_circuits(gf, tmp_path, monkeypatch):
|
||||
assert called == []
|
||||
|
||||
|
||||
# ────────── 传输超时=结果未知：不降级直传（防两份群文件） ──────────
|
||||
# 与 handlers/sender.py 同一物理事实：超时只是 bot 不再等待，协议端会把这次
|
||||
# 上传做完（大文件远超默认 30s）；此时再走本地直传就是重复投递。
|
||||
|
||||
|
||||
def test_send_timeout_default_and_env(gf, monkeypatch):
|
||||
monkeypatch.delenv("video_analysis_send_timeout", raising=False)
|
||||
monkeypatch.delenv("VIDEO_ANALYSIS_SEND_TIMEOUT", raising=False)
|
||||
assert gf._send_timeout() == 300.0
|
||||
|
||||
monkeypatch.setenv("video_analysis_send_timeout", "77")
|
||||
assert gf._send_timeout() == 77.0
|
||||
|
||||
monkeypatch.setenv("video_analysis_send_timeout", "bad")
|
||||
assert gf._send_timeout() == 300.0
|
||||
|
||||
|
||||
async def test_upload_call_carries_per_call_timeout(gf, monkeypatch):
|
||||
"""upload_group_file API 带 `_timeout`（适配器 per-call 超时）。"""
|
||||
monkeypatch.setenv("video_analysis_send_timeout", "77")
|
||||
calls: list[dict] = []
|
||||
|
||||
class _Bot:
|
||||
async def call_api(self, api, **kwargs):
|
||||
calls.append({"api": api, **kwargs})
|
||||
|
||||
monkeypatch.setattr(gf, "get_bot", lambda: _Bot())
|
||||
await gf._call_upload("file:///D:/x.zip", 123, "x.zip")
|
||||
assert calls == [
|
||||
{
|
||||
"api": "upload_group_file",
|
||||
"group_id": 123,
|
||||
"file": "file:///D:/x.zip",
|
||||
"name": "x.zip",
|
||||
"_timeout": 77.0,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
async def test_s3_timeout_does_not_fall_back_to_local(gf, tmp_path, monkeypatch):
|
||||
"""S3 链接上传超时=结果未知（协议端可能还在传）→ 不再直传，避免两份。"""
|
||||
seen: list[str] = []
|
||||
link = "http://192.168.2.15:5246/PLANA/x.zip?sig=1"
|
||||
|
||||
async def fake_call(file_value, group_id, name, folder_id=None):
|
||||
seen.append(file_value)
|
||||
raise NetworkError("WebSocket call api upload_group_file timeout")
|
||||
|
||||
monkeypatch.setattr(gf, "_call_upload", fake_call)
|
||||
monkeypatch.setattr(gf, "_s3_url", lambda path, policy: link)
|
||||
|
||||
file = _make_files(tmp_path / "src", ["a.zip"])[0]
|
||||
assert await gf.upload_group_file(file, 123) is True
|
||||
assert seen == [link] # 只有 S3 那一次，没有第二次 file:// 直传
|
||||
|
||||
|
||||
async def test_local_upload_timeout_reports_unknown(gf, tmp_path, monkeypatch):
|
||||
"""本地直传超时同样按「可能已送达」处理（不误报失败即可）。"""
|
||||
|
||||
async def fake_call(*args, **kwargs):
|
||||
raise NetworkError("timeout")
|
||||
|
||||
monkeypatch.setattr(gf, "_call_upload", fake_call)
|
||||
monkeypatch.setattr(gf, "_s3_url", lambda path, policy: "")
|
||||
|
||||
file = _make_files(tmp_path / "src", ["a.zip"])[0]
|
||||
assert await gf.upload_group_file(file, 123) is True
|
||||
|
||||
|
||||
# ───────────────────────── 投递编排 ─────────────────────────
|
||||
|
||||
|
||||
@@ -241,9 +311,7 @@ async def test_zip_mode_uploads_single_archive(gf, tmp_path, monkeypatch):
|
||||
archive = tmp_path / "pack.zip"
|
||||
archive.write_bytes(b"zip")
|
||||
calls = _record_uploads(gf, monkeypatch, archive)
|
||||
monkeypatch.setattr(
|
||||
gf, "build_archive", lambda *a, **kw: archive
|
||||
)
|
||||
monkeypatch.setattr(gf, "build_archive", lambda *a, **kw: archive)
|
||||
|
||||
files = _make_files(tmp_path / "src", ["a.txt", "b.txt"])
|
||||
ok = await gf.upload_group_files(files, 123, title="t", password="pw")
|
||||
@@ -271,9 +339,7 @@ async def test_raw_mode_prefixes_work_name_for_multi_files(gf, tmp_path, monkeyp
|
||||
monkeypatch.setattr(gf, "upload_group_file", fake_upload)
|
||||
|
||||
images = _make_files(tmp_path / "src", ["001.jpg", "002.jpg"])
|
||||
await gf.upload_group_files(
|
||||
images, 123, zip_files=False, rel_dir="作者_9/海边日落"
|
||||
)
|
||||
await gf.upload_group_files(images, 123, zip_files=False, rel_dir="作者_9/海边日落")
|
||||
assert seen == ["海边日落_001.jpg", "海边日落_002.jpg"]
|
||||
|
||||
seen.clear()
|
||||
|
||||
@@ -6,6 +6,7 @@ get_temp_root 的返回值，测试里 monkeypatch 成临时目录。
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
@@ -127,11 +128,31 @@ def test_work_stem_strips_hashtags_and_spaces(u):
|
||||
|
||||
|
||||
def test_work_stem_empty_falls_back(u):
|
||||
"""没给来源时保持旧契约（固定 `作品`）"""
|
||||
assert u.build_work_stem("") == "作品"
|
||||
assert u.build_work_stem(None) == "作品"
|
||||
assert u.build_work_stem("🎬") == "作品"
|
||||
|
||||
|
||||
def test_work_stem_empty_uses_source_code(u):
|
||||
"""给了来源 → 空标题兜底成 `作品_{来源短码}`：同作品稳定、不同作品不撞
|
||||
|
||||
固定 `作品` 会让同一作者的多个无名作品挤在一个名字下，靠 4 位**时间**短码
|
||||
区分，每次重解析名字都变（S3 key / 群文件成员名不可复现）。
|
||||
"""
|
||||
url = "https://www.douyin.com/video/7412345678901234567"
|
||||
stem = u.build_work_stem("#话题", source=url)
|
||||
assert stem == f"作品_{u.short_source_code(url)}"
|
||||
assert re.fullmatch(r"作品_[0-9a-z]{4}", stem)
|
||||
assert stem == u.build_work_stem("", source=url) # 同来源稳定
|
||||
assert stem != u.build_work_stem("", source=url + "8") # 不同来源不同码
|
||||
|
||||
|
||||
def test_work_stem_source_ignored_when_title_present(u):
|
||||
"""有标题时来源短码不参与（标题本身已有区分度，别把名字改花）"""
|
||||
assert u.build_work_stem("旅行 #随手拍", source="whatever") == "旅行"
|
||||
|
||||
|
||||
def test_work_stem_truncates(u):
|
||||
assert len(u.build_work_stem("标题" * 20)) <= 15
|
||||
|
||||
|
||||
@@ -40,6 +40,8 @@ def test_normalize_platform_aliases(policy):
|
||||
assert policy.normalize_platform("小红书") == "小红书"
|
||||
assert policy.normalize_platform("b23") == "哔哩哔哩"
|
||||
assert policy.normalize_platform("B站") == "哔哩哔哩"
|
||||
assert policy.normalize_platform("ks") == "快手"
|
||||
assert policy.normalize_platform("快手") == "快手"
|
||||
assert policy.normalize_platform("yt") == "Youtube"
|
||||
assert policy.normalize_platform("twitter") == "X"
|
||||
assert policy.normalize_platform(" X ") == "X"
|
||||
@@ -81,6 +83,10 @@ def test_match_platform_domains(policy):
|
||||
assert policy.match_platform("https://twitter.com/a/status/1") == "X"
|
||||
assert policy.match_platform("https://b23.tv/abc") == "哔哩哔哩"
|
||||
assert policy.match_platform("https://v.douyin.com/abc") == "抖音"
|
||||
assert policy.match_platform("https://v.kuaishou.com/nUqhAiDI") == "快手"
|
||||
assert policy.match_platform("https://www.kuaishou.com/short-video/abc") == "快手"
|
||||
# QQ 卡片 jumpUrl 走 v.m.chenzhongtech.com（快手分享页域）
|
||||
assert policy.match_platform("https://v.m.chenzhongtech.com/fw/photo/abc") == "快手"
|
||||
assert policy.match_platform("https://youtu.be/abc") == "Youtube"
|
||||
assert policy.match_platform("https://example.com/a.mp4") is None
|
||||
|
||||
|
||||
+326
-61
@@ -1,24 +1,27 @@
|
||||
"""媒体发送成功后「补传 S3 取下载链接」的单元测试
|
||||
"""sender.py 归档语义的单元测试（「媒体必须进本地 S3」是前提）
|
||||
|
||||
背景：2026-09-22 起 `hexi/core/outbound_media` 的全局钩子把本地路径改写成
|
||||
`/media/<token>`，本地直发那一级才真正可用。此前它因协议端不在本机、`file://`
|
||||
必然 ENOENT 而**恒失败**，所以每次都落到第 2 级 S3 —— `public_url` 因此总有值。
|
||||
本地直发通了以后，如果什么都不做，`public_url` 会恒为 None，`entry.py` 的
|
||||
`if pub and policy.sends_link` 就会让「发送下载链接」**静默失效**。
|
||||
用户 2026-09-25 定的前提：**无论发不发送消息**，下载成功的媒体都要进本地 S3
|
||||
（temp 只留 7 天，cleanup.py 删；桶里那份才是长期存档）。因此 sender 的结构是
|
||||
「归档与发送解耦且并行」：
|
||||
|
||||
`_local_sent` / `_public_link_for` 就是为此把「取链接」与「发送」解耦：
|
||||
只有策略真要链接（`sends_link`）时才补一次 S3，其余情况一概不碰 S3。
|
||||
- `archive_media` 无条件逐文件上传：单个文件失败只 warning 并继续，任何情况都不抛；
|
||||
- `send_pending_media` 一进来就 `create_task` 归档，与本地直发并行开跑，
|
||||
发送全失败也一定 await 它跑完；`public_url` 从归档结果取（最后一个非空）；
|
||||
- 第 2 级（S3 链接发送）复用归档的预签名链接 —— 每个文件只上传一次，没有"补传"。
|
||||
|
||||
sender.py 有插件内相对导入（`..config` / `..policy` / `..services...`），
|
||||
无法裸加载，故在 sys.modules 里预置这些依赖的替身再按路径加载目标模块。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from nonebot.adapters.onebot.v11.exception import ActionFailed, NetworkError
|
||||
|
||||
_REPO = Path(__file__).resolve().parents[1]
|
||||
_PKG = "hexi.plugins.nonebot_plugin_video_analysis"
|
||||
@@ -71,23 +74,31 @@ def sender():
|
||||
|
||||
|
||||
class _Policy:
|
||||
"""只带 sends_link 的策略替身（真实 Policy 需要一堆字段/归一化）。"""
|
||||
|
||||
def __init__(self, sends_link: bool):
|
||||
self.sends_link = sends_link
|
||||
"""群策略替身（归档只是把它透传给 upload_with_plan，不需要真实字段）。"""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def uploads(monkeypatch, sender):
|
||||
"""记录 upload_with_plan 的调用；返回 (调用列表, 设置返回值的钩子)。"""
|
||||
calls: list[Path] = []
|
||||
result: dict = {"pub": "https://pub.example/x.mp4", "raise": None}
|
||||
"""记录 upload_with_plan 的调用；返回 (调用列表, 结果配置)。
|
||||
|
||||
result 可覆盖的字段：
|
||||
- `pub`：常量，或按调用顺序消费的列表（元素为 None = 没传公网）；
|
||||
- `raises`：文件名 → 该文件上传时抛出的异常（其余文件照常上传）。
|
||||
"""
|
||||
calls: list[tuple[Path, object]] = []
|
||||
result: dict = {"pub": "https://pub.example/x.mp4", "raises": {}}
|
||||
|
||||
def fake_upload(file_path, *, policy=None):
|
||||
calls.append(Path(file_path))
|
||||
if result["raise"]:
|
||||
raise result["raise"]
|
||||
return "local-url", result["pub"]
|
||||
fp = Path(file_path)
|
||||
calls.append((fp, policy))
|
||||
pub = result["pub"]
|
||||
if isinstance(pub, list):
|
||||
# 列表按下标对应「第几个文件」（先取值再判失败：元素与文件一一对应）
|
||||
pub = pub.pop(0) if pub else None
|
||||
error = result["raises"].get(fp.name)
|
||||
if error:
|
||||
raise error
|
||||
return "local-url", pub
|
||||
|
||||
monkeypatch.setattr(sender, "upload_with_plan", fake_upload)
|
||||
return calls, result
|
||||
@@ -95,61 +106,315 @@ def uploads(monkeypatch, sender):
|
||||
|
||||
@pytest.fixture
|
||||
def media(sender):
|
||||
def build(sends_link: bool, n: int = 1, policy=None):
|
||||
def build(n: int = 1, policy=_Policy()):
|
||||
files = [Path(f"D:/temp/f{i}.mp4") for i in range(n)]
|
||||
return sender.PendingMedia(
|
||||
files=files,
|
||||
policy=_Policy(sends_link) if policy is None else policy,
|
||||
)
|
||||
return sender.PendingMedia(files=files, policy=policy)
|
||||
|
||||
return build
|
||||
|
||||
|
||||
async def test_no_s3_when_link_disabled(sender, media, uploads):
|
||||
"""默认路径：本地发成功就够了，绝不碰 S3（媒体不走 S3 归档是有意的）。"""
|
||||
calls, _ = uploads
|
||||
ok, pub = await sender._local_sent(media(sends_link=False))
|
||||
assert (ok, pub) == (True, None)
|
||||
assert calls == []
|
||||
# ───────────────────────── 归档本体（无条件） ─────────────────────────
|
||||
|
||||
|
||||
async def test_s3_fetched_when_link_enabled(sender, media, uploads):
|
||||
"""策略要下载链接时：补传 S3 并把 public_url 交出去（本次修的回归）。"""
|
||||
async def test_archive_uploads_every_file(sender, media, uploads):
|
||||
"""逐文件归档，拿到 (预签名链接, 公网链接)。"""
|
||||
calls, _ = uploads
|
||||
ok, pub = await sender._local_sent(media(sends_link=True))
|
||||
result = await sender.archive_media(media(n=3))
|
||||
assert len(calls) == 3
|
||||
assert [(r.path.name, r.local_url, r.public_url) for r in result] == [
|
||||
(f"f{i}.mp4", "local-url", "https://pub.example/x.mp4") for i in range(3)
|
||||
]
|
||||
|
||||
|
||||
async def test_archive_continues_after_single_failure(sender, media, uploads):
|
||||
"""单个文件失败只跳过它，后面的照常传（多图作品不早退）。"""
|
||||
calls, result = uploads
|
||||
result["raises"] = {"f1.mp4": RuntimeError("这个文件 S3 挂了")}
|
||||
|
||||
archived = await sender.archive_media(media(n=3))
|
||||
assert [r.local_url for r in archived] == ["local-url", None, "local-url"]
|
||||
assert len(calls) == 3 # 失败的那个也试过了，f2 没被跳过
|
||||
|
||||
|
||||
async def test_archive_never_raises(sender, media, uploads):
|
||||
"""全部上传失败也不抛（归档失败不能反过来影响发送）。"""
|
||||
calls, result = uploads
|
||||
error = RuntimeError("S3 挂了")
|
||||
result["raises"] = {"f0.mp4": error, "f1.mp4": error}
|
||||
archived = await sender.archive_media(media(n=2))
|
||||
assert [r.local_url for r in archived] == [None, None]
|
||||
assert len(calls) == 2
|
||||
|
||||
|
||||
async def test_archive_passes_policy_through(sender, media, uploads):
|
||||
"""policy=None（如 Web 面板路径）也照常归档：upload_with_plan 自己兜 Policy()。"""
|
||||
calls, _ = uploads
|
||||
await sender.archive_media(media(policy=None))
|
||||
assert [policy for _, policy in calls] == [None]
|
||||
|
||||
|
||||
async def test_last_public_url_takes_last_non_empty(sender, media, uploads):
|
||||
"""公网链接取最后一个非空值；只有本地链接的文件不影响结果。"""
|
||||
calls, result = uploads
|
||||
result["pub"] = [None, "https://pub.example/b.mp4", None]
|
||||
archived = await sender.archive_media(media(n=3))
|
||||
assert sender._last_public_url(archived) == "https://pub.example/b.mp4"
|
||||
|
||||
|
||||
async def test_last_public_url_none_when_upload_public_off(sender, media, uploads):
|
||||
"""S3 有本地链接但没公网（upload_public 关）→ None，且算不上失败。"""
|
||||
_, result = uploads
|
||||
result["pub"] = None
|
||||
archived = await sender.archive_media(media())
|
||||
assert archived[0].local_url == "local-url"
|
||||
assert sender._last_public_url(archived) is None
|
||||
|
||||
|
||||
# ─────────────────── 发送入口：归档无条件且与之并行 ───────────────────
|
||||
|
||||
|
||||
async def test_archives_even_when_send_fails(monkeypatch, sender, media, uploads):
|
||||
"""前提的核心：发送全失败也照样归档，public_url 照常交出去。"""
|
||||
calls, _ = uploads
|
||||
|
||||
async def fake_core(m, event, archive):
|
||||
return False # 三级全失败
|
||||
|
||||
monkeypatch.setattr(sender, "_send_media_core", fake_core)
|
||||
ok, pub = await sender.send_pending_media(media(n=2))
|
||||
assert (ok, pub) == (False, "https://pub.example/x.mp4")
|
||||
assert len(calls) == 2
|
||||
|
||||
|
||||
async def test_archives_when_send_succeeds(monkeypatch, sender, media, uploads):
|
||||
"""发送成功也归档（归档不是发送的副产品）。"""
|
||||
calls, _ = uploads
|
||||
|
||||
async def fake_core(m, event, archive):
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(sender, "_send_media_core", fake_core)
|
||||
ok, pub = await sender.send_pending_media(media())
|
||||
assert (ok, pub) == (True, "https://pub.example/x.mp4")
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
async def test_multi_file_uploads_each(sender, media, uploads):
|
||||
calls, _ = uploads
|
||||
ok, pub = await sender._local_sent(media(sends_link=True, n=3))
|
||||
async def test_archive_runs_in_parallel_with_send(monkeypatch, sender, media, uploads):
|
||||
"""归档与发送并行开跑，不是"发送跑完才传"（本地直发不等它）。"""
|
||||
started: list[str] = []
|
||||
|
||||
async def fake_archive(m):
|
||||
started.append("archive")
|
||||
return [sender.UploadedFile(path=fp) for fp in m.files]
|
||||
|
||||
async def fake_core(m, event, archive):
|
||||
await asyncio.sleep(0) # 让出一个事件循环回合
|
||||
assert started, "归档应当已在跑，而不是等发送结束才开始"
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(sender, "archive_media", fake_archive)
|
||||
monkeypatch.setattr(sender, "_send_media_core", fake_core)
|
||||
ok, _ = await sender.send_pending_media(media())
|
||||
assert ok is True
|
||||
assert pub == "https://pub.example/x.mp4"
|
||||
assert len(calls) == 3
|
||||
|
||||
|
||||
async def test_no_public_url_keeps_none(sender, media, uploads):
|
||||
"""S3 有本地链接但没公网（upload_public 关）→ 不带链接，但也不算失败。"""
|
||||
calls, result = uploads
|
||||
result["pub"] = None
|
||||
ok, pub = await sender._local_sent(media(sends_link=True))
|
||||
assert (ok, pub) == (True, None)
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
async def test_upload_failure_never_breaks_send(sender, media, uploads):
|
||||
"""补传失败只记日志：发送本身已经成功了，不能反过来判失败。"""
|
||||
calls, result = uploads
|
||||
result["raise"] = RuntimeError("S3 挂了")
|
||||
ok, pub = await sender._local_sent(media(sends_link=True))
|
||||
assert (ok, pub) == (True, None)
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
async def test_policy_none_is_safe(sender, media, uploads):
|
||||
"""无策略（如 Web 面板路径）时不补传也不炸。"""
|
||||
async def test_send_failure_still_waits_for_archive(
|
||||
monkeypatch, sender, media, uploads
|
||||
):
|
||||
"""发送层抛异常也不能把归档半途丢下（finally 里 await 它跑完）。"""
|
||||
calls, _ = uploads
|
||||
ok, pub = await sender._local_sent(media(sends_link=False, policy=None))
|
||||
assert (ok, pub) == (True, None)
|
||||
assert calls == []
|
||||
finished: list[str] = []
|
||||
|
||||
async def fake_core(m, event, archive):
|
||||
raise RuntimeError("发送层炸了")
|
||||
|
||||
def slow_upload(file_path, *, policy=None): # to_thread 里跑的是同步函数
|
||||
calls.append((Path(file_path), policy))
|
||||
time.sleep(0.01)
|
||||
finished.append(Path(file_path).name)
|
||||
return "local-url", "https://pub.example/x.mp4"
|
||||
|
||||
monkeypatch.setattr(sender, "upload_with_plan", slow_upload)
|
||||
monkeypatch.setattr(sender, "_send_media_core", fake_core)
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
await sender.send_pending_media(media(n=2))
|
||||
assert finished == ["f0.mp4", "f1.mp4"] # 归档完整跑完才抛出去
|
||||
|
||||
|
||||
# ─────────────────── 第 2 级复用归档结果（不重复上传） ───────────────────
|
||||
|
||||
|
||||
async def test_s3_level_reuses_archived_urls(monkeypatch, sender, media, uploads):
|
||||
"""本地直发失败 → 用归档的预签名链接发送，每个文件只上传一次。"""
|
||||
calls, _ = uploads
|
||||
built: list = []
|
||||
|
||||
def fake_local_msg(files, image_post):
|
||||
class _Msg:
|
||||
async def send(self, **_kw): # 本地直发失败
|
||||
raise RuntimeError("协议端拿不到文件")
|
||||
|
||||
return _Msg()
|
||||
|
||||
def fake_s3_msg(items):
|
||||
built.append(items)
|
||||
|
||||
class _Msg:
|
||||
async def send(self, **_kw): # 链接版发送成功
|
||||
pass
|
||||
|
||||
return _Msg()
|
||||
|
||||
monkeypatch.setattr(sender, "_build_local_msg", fake_local_msg)
|
||||
monkeypatch.setattr(sender, "_build_s3_msg", fake_s3_msg)
|
||||
|
||||
ok, pub = await sender.send_pending_media(media(n=2))
|
||||
assert (ok, pub) == (True, "https://pub.example/x.mp4")
|
||||
assert len(calls) == 2 # 只有归档那次上传，第 2 级没有重复传
|
||||
assert built == [
|
||||
[(Path("D:/temp/f0.mp4"), "local-url"), (Path("D:/temp/f1.mp4"), "local-url")]
|
||||
]
|
||||
|
||||
|
||||
async def test_no_files_is_noop(sender):
|
||||
"""没有媒体文件时既不归档也不发送。"""
|
||||
ok, pub = await sender.send_pending_media(sender.PendingMedia(files=[]))
|
||||
assert (ok, pub) == (False, None)
|
||||
|
||||
|
||||
# ─────────── 超时=结果未知：不降级重试（长视频防重复发送） ───────────
|
||||
# 2026-10-13 实测：172MB 视频三级降级各超时一次（NoneBot 默认 api_timeout=30s，
|
||||
# 而协议端下载+转存要 ~2 分钟），但协议端在 bot 超时后照样把三份全发进群。
|
||||
# 超时只是 bot 放弃等待 ⇒ 结果未知 ⇒ 重试必重复。
|
||||
|
||||
|
||||
def test_send_timeout_default_and_env(sender, monkeypatch):
|
||||
"""默认 300s；`video_analysis_send_timeout` 可调，非法/非正值回退默认。"""
|
||||
monkeypatch.delenv("video_analysis_send_timeout", raising=False)
|
||||
monkeypatch.delenv("VIDEO_ANALYSIS_SEND_TIMEOUT", raising=False)
|
||||
assert sender.send_timeout() == 300.0
|
||||
|
||||
monkeypatch.setenv("video_analysis_send_timeout", "120")
|
||||
assert sender.send_timeout() == 120.0
|
||||
|
||||
monkeypatch.setenv("video_analysis_send_timeout", "abc")
|
||||
assert sender.send_timeout() == 300.0
|
||||
|
||||
monkeypatch.setenv("video_analysis_send_timeout", "0")
|
||||
assert sender.send_timeout() == 300.0
|
||||
|
||||
|
||||
def test_unknown_outcome_classification(sender):
|
||||
"""超时/网络错误=结果未知；ActionFailed（协议端明确报错）=确定失败。"""
|
||||
assert sender.unknown_send_outcome(NetworkError("timeout")) is True
|
||||
assert sender.unknown_send_outcome(asyncio.TimeoutError()) is True
|
||||
assert sender.unknown_send_outcome(ActionFailed(error="fetch failed")) is False
|
||||
assert sender.unknown_send_outcome(RuntimeError("本地错误")) is False
|
||||
|
||||
|
||||
async def test_media_send_carries_per_call_timeout(monkeypatch, sender, media, uploads):
|
||||
"""媒体发送带 `_timeout`（适配器 per-call 超时），长视频才等得到真结果。"""
|
||||
monkeypatch.setenv("video_analysis_send_timeout", "42")
|
||||
seen: list[dict] = []
|
||||
|
||||
def fake_local_msg(files, image_post):
|
||||
class _Msg:
|
||||
async def send(self, **kwargs):
|
||||
seen.append(kwargs)
|
||||
|
||||
return _Msg()
|
||||
|
||||
monkeypatch.setattr(sender, "_build_local_msg", fake_local_msg)
|
||||
ok, _ = await sender.send_pending_media(media())
|
||||
assert ok is True
|
||||
assert seen == [{"_timeout": 42.0}]
|
||||
|
||||
|
||||
async def test_local_timeout_does_not_resend(monkeypatch, sender, media, uploads):
|
||||
"""本地直发超时（结果未知）→ 不走 S3 降级、按「可能已送达」返回。"""
|
||||
s3_built: list = []
|
||||
|
||||
def fake_local_msg(files, image_post):
|
||||
class _Msg:
|
||||
async def send(self, **_kw):
|
||||
raise NetworkError("WebSocket call api send_msg timeout")
|
||||
|
||||
return _Msg()
|
||||
|
||||
def fake_s3_msg(items):
|
||||
s3_built.append(items)
|
||||
|
||||
class _Msg:
|
||||
async def send(self, **_kw):
|
||||
pass
|
||||
|
||||
return _Msg()
|
||||
|
||||
monkeypatch.setattr(sender, "_build_local_msg", fake_local_msg)
|
||||
monkeypatch.setattr(sender, "_build_s3_msg", fake_s3_msg)
|
||||
|
||||
ok, pub = await sender.send_pending_media(media())
|
||||
assert ok is True # 结果未知按可能已送达：不触发「媒体发送失败」提示
|
||||
assert pub == "https://pub.example/x.mp4" # 归档照常完成
|
||||
assert s3_built == [] # 关键：没有第二次发送
|
||||
|
||||
|
||||
async def test_action_failed_still_cascades_to_s3(monkeypatch, sender, media, uploads):
|
||||
"""协议端明确报错（如 fetch failed）= 确定没发出去，照旧降级到 S3 链接。"""
|
||||
sent: list = []
|
||||
|
||||
def fake_local_msg(files, image_post):
|
||||
class _Msg:
|
||||
async def send(self, **_kw):
|
||||
raise ActionFailed(error="fetch failed")
|
||||
|
||||
return _Msg()
|
||||
|
||||
def fake_s3_msg(items):
|
||||
class _Msg:
|
||||
async def send(self, **_kw): # 链接版成功
|
||||
sent.append(items)
|
||||
|
||||
return _Msg()
|
||||
|
||||
monkeypatch.setattr(sender, "_build_local_msg", fake_local_msg)
|
||||
monkeypatch.setattr(sender, "_build_s3_msg", fake_s3_msg)
|
||||
|
||||
ok, _ = await sender.send_pending_media(media())
|
||||
assert ok is True
|
||||
assert len(sent) == 1 # 恰好降级一次
|
||||
|
||||
|
||||
async def test_forward_timeout_does_not_fall_through(
|
||||
monkeypatch, sender, media, uploads
|
||||
):
|
||||
"""合并转发超时（结果未知）→ 不再落回普通消息链路重发。"""
|
||||
two = media(n=2)
|
||||
fallback_built: list = []
|
||||
|
||||
async def fake_forward(items, sender_name, *, group_id=None, user_id=None):
|
||||
return None # 结果未知
|
||||
|
||||
def fake_local_msg(files, image_post):
|
||||
fallback_built.append(files)
|
||||
|
||||
class _Msg:
|
||||
async def send(self, **_kw):
|
||||
raise AssertionError("不该走到普通消息链路")
|
||||
|
||||
return _Msg()
|
||||
|
||||
monkeypatch.setattr(sender, "_send_forward_media", fake_forward)
|
||||
monkeypatch.setattr(sender, "_forward_sender_name", lambda: "Bot")
|
||||
monkeypatch.setattr(sender, "_build_local_msg", fake_local_msg)
|
||||
|
||||
archive = asyncio.create_task(sender.archive_media(two))
|
||||
try:
|
||||
ok = await sender._send_media_core(
|
||||
two, types.SimpleNamespace(group_id=123), archive
|
||||
)
|
||||
finally:
|
||||
await archive
|
||||
assert ok is True
|
||||
assert fallback_built == []
|
||||
Reference in new issue
Block a user