sansenhoshiandClaude Code
a789a44f5d
- 抖音（2026-10-13）：删除 SSR + playwright 旧链路（douyin_ssr.py 移除， douyin_api.py 精简为下载工具），新增 douyin_web.py（x-tt-argus 网关头 + web detail 接口 + 最高码率梯/多 CDN 回退下载）与 douyin_sign.py （纯 Python a_bogus 签名，vendor 自 f2（Apache-2.0），依赖 gmssl SM3）； 实测 10/10 成功率、单作品 <1s（旧浏览器链路 5-15s 且随机撞验证码） - 快手（2026-10-08）：kuaishou_content.py 分享页 INIT_STATE 静态解析（无登录态）， 清晰度梯最佳档 + 图集 URL 组装，入口/策略/Web 面板同步接入 - 小红书：按链接来源定国际/国内站优先序、视频 backupUrls 逐候选回退、 图片 urlDefault→infoList WB_DFT→url 三档回退 - 长视频防重复发送（2026-10-13）：媒体发送带 per-call _timeout（默认 300s）， 超时=结果未知不再降级重试；auto/active 两条 handler 互斥 - Web 解析封面抽帧 ffmpeg 加 CREATE_NO_WINDOW（pythonw 下不再弹 cmd 黑窗） - 依赖：+gmssl；测试：新增 test_douyin_web / test_kuaishou_parse / test_video_cleanup， 扩充 test_video_policy / test_video_sender_link / test_video_group_file / test_video_media_naming Co-Authored-By: Claude Code <noreply@anthropic.com>
421 lines
16 KiB
Python
421 lines
16 KiB
Python
"""sender.py 归档语义的单元测试（「媒体必须进本地 S3」是前提）
|
||
|
||
用户 2026-09-25 定的前提：**无论发不发送消息**，下载成功的媒体都要进本地 S3
|
||
（temp 只留 7 天，cleanup.py 删；桶里那份才是长期存档）。因此 sender 的结构是
|
||
「归档与发送解耦且并行」：
|
||
|
||
- `archive_media` 无条件逐文件上传：单个文件失败只 warning 并继续，任何情况都不抛；
|
||
- `send_pending_media` 一进来就 `create_task` 归档，与本地直发并行开跑，
|
||
发送全失败也一定 await 它跑完；`public_url` 从归档结果取（最后一个非空）；
|
||
- 第 2 级（S3 链接发送）复用归档的预签名链接 —— 每个文件只上传一次，没有"补传"。
|
||
|
||
sender.py 有插件内相对导入（`..config` / `..policy` / `..services...`），
|
||
无法裸加载，故在 sys.modules 里预置这些依赖的替身再按路径加载目标模块。
|
||
"""
|
||
|
||
import asyncio
|
||
import importlib.util
|
||
import sys
|
||
import time
|
||
import types
|
||
from pathlib import Path
|
||
|
||
import pytest
|
||
from nonebot.adapters.onebot.v11.exception import ActionFailed, NetworkError
|
||
|
||
_REPO = Path(__file__).resolve().parents[1]
|
||
_PKG = "hexi.plugins.nonebot_plugin_video_analysis"
|
||
_SENDER_PATH = (
|
||
_REPO
|
||
/ "hexi"
|
||
/ "plugins"
|
||
/ "nonebot_plugin_video_analysis"
|
||
/ "handlers"
|
||
/ "sender.py"
|
||
)
|
||
|
||
#: 预置的替身模块名 → 该模块要暴露的属性占位（真实函数在测试里替换）
|
||
_STUBS = {
|
||
_PKG: {},
|
||
f"{_PKG}.handlers": {},
|
||
f"{_PKG}.config": {"group_file_settings": lambda: ([], None)},
|
||
f"{_PKG}.policy": {"Policy": type("Policy", (), {})},
|
||
f"{_PKG}.services": {},
|
||
f"{_PKG}.services.storage": {},
|
||
f"{_PKG}.services.storage.group_file": {"upload_group_files": None},
|
||
f"{_PKG}.services.storage.s3": {"upload_with_plan": None},
|
||
f"{_PKG}.utils": {"media_rel_dir_of": lambda _p: ""},
|
||
}
|
||
|
||
|
||
@pytest.fixture(scope="module")
|
||
def sender():
|
||
saved = {name: sys.modules.get(name) for name in _STUBS}
|
||
for name, attrs in _STUBS.items():
|
||
module = types.ModuleType(name)
|
||
for key, value in attrs.items():
|
||
setattr(module, key, value)
|
||
sys.modules[name] = module
|
||
|
||
target = f"{_PKG}.handlers.sender"
|
||
saved[target] = sys.modules.get(target)
|
||
spec = importlib.util.spec_from_file_location(target, _SENDER_PATH)
|
||
module = importlib.util.module_from_spec(spec)
|
||
sys.modules[target] = module
|
||
spec.loader.exec_module(module)
|
||
|
||
yield module
|
||
|
||
for name, previous in saved.items():
|
||
if previous is None:
|
||
sys.modules.pop(name, None)
|
||
else:
|
||
sys.modules[name] = previous
|
||
|
||
|
||
class _Policy:
|
||
"""群策略替身（归档只是把它透传给 upload_with_plan，不需要真实字段）。"""
|
||
|
||
|
||
@pytest.fixture
|
||
def uploads(monkeypatch, sender):
|
||
"""记录 upload_with_plan 的调用；返回 (调用列表, 结果配置)。
|
||
|
||
result 可覆盖的字段：
|
||
- `pub`：常量，或按调用顺序消费的列表（元素为 None = 没传公网）；
|
||
- `raises`：文件名 → 该文件上传时抛出的异常（其余文件照常上传）。
|
||
"""
|
||
calls: list[tuple[Path, object]] = []
|
||
result: dict = {"pub": "https://pub.example/x.mp4", "raises": {}}
|
||
|
||
def fake_upload(file_path, *, policy=None):
|
||
fp = Path(file_path)
|
||
calls.append((fp, policy))
|
||
pub = result["pub"]
|
||
if isinstance(pub, list):
|
||
# 列表按下标对应「第几个文件」（先取值再判失败：元素与文件一一对应）
|
||
pub = pub.pop(0) if pub else None
|
||
error = result["raises"].get(fp.name)
|
||
if error:
|
||
raise error
|
||
return "local-url", pub
|
||
|
||
monkeypatch.setattr(sender, "upload_with_plan", fake_upload)
|
||
return calls, result
|
||
|
||
|
||
@pytest.fixture
|
||
def media(sender):
|
||
def build(n: int = 1, policy=_Policy()):
|
||
files = [Path(f"D:/temp/f{i}.mp4") for i in range(n)]
|
||
return sender.PendingMedia(files=files, policy=policy)
|
||
|
||
return build
|
||
|
||
|
||
# ───────────────────────── 归档本体（无条件） ─────────────────────────
|
||
|
||
|
||
async def test_archive_uploads_every_file(sender, media, uploads):
|
||
"""逐文件归档，拿到 (预签名链接, 公网链接)。"""
|
||
calls, _ = uploads
|
||
result = await sender.archive_media(media(n=3))
|
||
assert len(calls) == 3
|
||
assert [(r.path.name, r.local_url, r.public_url) for r in result] == [
|
||
(f"f{i}.mp4", "local-url", "https://pub.example/x.mp4") for i in range(3)
|
||
]
|
||
|
||
|
||
async def test_archive_continues_after_single_failure(sender, media, uploads):
|
||
"""单个文件失败只跳过它，后面的照常传（多图作品不早退）。"""
|
||
calls, result = uploads
|
||
result["raises"] = {"f1.mp4": RuntimeError("这个文件 S3 挂了")}
|
||
|
||
archived = await sender.archive_media(media(n=3))
|
||
assert [r.local_url for r in archived] == ["local-url", None, "local-url"]
|
||
assert len(calls) == 3 # 失败的那个也试过了，f2 没被跳过
|
||
|
||
|
||
async def test_archive_never_raises(sender, media, uploads):
|
||
"""全部上传失败也不抛（归档失败不能反过来影响发送）。"""
|
||
calls, result = uploads
|
||
error = RuntimeError("S3 挂了")
|
||
result["raises"] = {"f0.mp4": error, "f1.mp4": error}
|
||
archived = await sender.archive_media(media(n=2))
|
||
assert [r.local_url for r in archived] == [None, None]
|
||
assert len(calls) == 2
|
||
|
||
|
||
async def test_archive_passes_policy_through(sender, media, uploads):
|
||
"""policy=None（如 Web 面板路径）也照常归档：upload_with_plan 自己兜 Policy()。"""
|
||
calls, _ = uploads
|
||
await sender.archive_media(media(policy=None))
|
||
assert [policy for _, policy in calls] == [None]
|
||
|
||
|
||
async def test_last_public_url_takes_last_non_empty(sender, media, uploads):
|
||
"""公网链接取最后一个非空值；只有本地链接的文件不影响结果。"""
|
||
calls, result = uploads
|
||
result["pub"] = [None, "https://pub.example/b.mp4", None]
|
||
archived = await sender.archive_media(media(n=3))
|
||
assert sender._last_public_url(archived) == "https://pub.example/b.mp4"
|
||
|
||
|
||
async def test_last_public_url_none_when_upload_public_off(sender, media, uploads):
|
||
"""S3 有本地链接但没公网（upload_public 关）→ None，且算不上失败。"""
|
||
_, result = uploads
|
||
result["pub"] = None
|
||
archived = await sender.archive_media(media())
|
||
assert archived[0].local_url == "local-url"
|
||
assert sender._last_public_url(archived) is None
|
||
|
||
|
||
# ─────────────────── 发送入口：归档无条件且与之并行 ───────────────────
|
||
|
||
|
||
async def test_archives_even_when_send_fails(monkeypatch, sender, media, uploads):
|
||
"""前提的核心：发送全失败也照样归档，public_url 照常交出去。"""
|
||
calls, _ = uploads
|
||
|
||
async def fake_core(m, event, archive):
|
||
return False # 三级全失败
|
||
|
||
monkeypatch.setattr(sender, "_send_media_core", fake_core)
|
||
ok, pub = await sender.send_pending_media(media(n=2))
|
||
assert (ok, pub) == (False, "https://pub.example/x.mp4")
|
||
assert len(calls) == 2
|
||
|
||
|
||
async def test_archives_when_send_succeeds(monkeypatch, sender, media, uploads):
|
||
"""发送成功也归档（归档不是发送的副产品）。"""
|
||
calls, _ = uploads
|
||
|
||
async def fake_core(m, event, archive):
|
||
return True
|
||
|
||
monkeypatch.setattr(sender, "_send_media_core", fake_core)
|
||
ok, pub = await sender.send_pending_media(media())
|
||
assert (ok, pub) == (True, "https://pub.example/x.mp4")
|
||
assert len(calls) == 1
|
||
|
||
|
||
async def test_archive_runs_in_parallel_with_send(monkeypatch, sender, media, uploads):
|
||
"""归档与发送并行开跑，不是"发送跑完才传"（本地直发不等它）。"""
|
||
started: list[str] = []
|
||
|
||
async def fake_archive(m):
|
||
started.append("archive")
|
||
return [sender.UploadedFile(path=fp) for fp in m.files]
|
||
|
||
async def fake_core(m, event, archive):
|
||
await asyncio.sleep(0) # 让出一个事件循环回合
|
||
assert started, "归档应当已在跑，而不是等发送结束才开始"
|
||
return True
|
||
|
||
monkeypatch.setattr(sender, "archive_media", fake_archive)
|
||
monkeypatch.setattr(sender, "_send_media_core", fake_core)
|
||
ok, _ = await sender.send_pending_media(media())
|
||
assert ok is True
|
||
|
||
|
||
async def test_send_failure_still_waits_for_archive(
|
||
monkeypatch, sender, media, uploads
|
||
):
|
||
"""发送层抛异常也不能把归档半途丢下（finally 里 await 它跑完）。"""
|
||
calls, _ = uploads
|
||
finished: list[str] = []
|
||
|
||
async def fake_core(m, event, archive):
|
||
raise RuntimeError("发送层炸了")
|
||
|
||
def slow_upload(file_path, *, policy=None): # to_thread 里跑的是同步函数
|
||
calls.append((Path(file_path), policy))
|
||
time.sleep(0.01)
|
||
finished.append(Path(file_path).name)
|
||
return "local-url", "https://pub.example/x.mp4"
|
||
|
||
monkeypatch.setattr(sender, "upload_with_plan", slow_upload)
|
||
monkeypatch.setattr(sender, "_send_media_core", fake_core)
|
||
|
||
with pytest.raises(RuntimeError):
|
||
await sender.send_pending_media(media(n=2))
|
||
assert finished == ["f0.mp4", "f1.mp4"] # 归档完整跑完才抛出去
|
||
|
||
|
||
# ─────────────────── 第 2 级复用归档结果（不重复上传） ───────────────────
|
||
|
||
|
||
async def test_s3_level_reuses_archived_urls(monkeypatch, sender, media, uploads):
|
||
"""本地直发失败 → 用归档的预签名链接发送，每个文件只上传一次。"""
|
||
calls, _ = uploads
|
||
built: list = []
|
||
|
||
def fake_local_msg(files, image_post):
|
||
class _Msg:
|
||
async def send(self, **_kw): # 本地直发失败
|
||
raise RuntimeError("协议端拿不到文件")
|
||
|
||
return _Msg()
|
||
|
||
def fake_s3_msg(items):
|
||
built.append(items)
|
||
|
||
class _Msg:
|
||
async def send(self, **_kw): # 链接版发送成功
|
||
pass
|
||
|
||
return _Msg()
|
||
|
||
monkeypatch.setattr(sender, "_build_local_msg", fake_local_msg)
|
||
monkeypatch.setattr(sender, "_build_s3_msg", fake_s3_msg)
|
||
|
||
ok, pub = await sender.send_pending_media(media(n=2))
|
||
assert (ok, pub) == (True, "https://pub.example/x.mp4")
|
||
assert len(calls) == 2 # 只有归档那次上传，第 2 级没有重复传
|
||
assert built == [
|
||
[(Path("D:/temp/f0.mp4"), "local-url"), (Path("D:/temp/f1.mp4"), "local-url")]
|
||
]
|
||
|
||
|
||
async def test_no_files_is_noop(sender):
|
||
"""没有媒体文件时既不归档也不发送。"""
|
||
ok, pub = await sender.send_pending_media(sender.PendingMedia(files=[]))
|
||
assert (ok, pub) == (False, None)
|
||
|
||
|
||
# ─────────── 超时=结果未知：不降级重试（长视频防重复发送） ───────────
|
||
# 2026-10-13 实测：172MB 视频三级降级各超时一次（NoneBot 默认 api_timeout=30s，
|
||
# 而协议端下载+转存要 ~2 分钟），但协议端在 bot 超时后照样把三份全发进群。
|
||
# 超时只是 bot 放弃等待 ⇒ 结果未知 ⇒ 重试必重复。
|
||
|
||
|
||
def test_send_timeout_default_and_env(sender, monkeypatch):
|
||
"""默认 300s；`video_analysis_send_timeout` 可调，非法/非正值回退默认。"""
|
||
monkeypatch.delenv("video_analysis_send_timeout", raising=False)
|
||
monkeypatch.delenv("VIDEO_ANALYSIS_SEND_TIMEOUT", raising=False)
|
||
assert sender.send_timeout() == 300.0
|
||
|
||
monkeypatch.setenv("video_analysis_send_timeout", "120")
|
||
assert sender.send_timeout() == 120.0
|
||
|
||
monkeypatch.setenv("video_analysis_send_timeout", "abc")
|
||
assert sender.send_timeout() == 300.0
|
||
|
||
monkeypatch.setenv("video_analysis_send_timeout", "0")
|
||
assert sender.send_timeout() == 300.0
|
||
|
||
|
||
def test_unknown_outcome_classification(sender):
|
||
"""超时/网络错误=结果未知；ActionFailed（协议端明确报错）=确定失败。"""
|
||
assert sender.unknown_send_outcome(NetworkError("timeout")) is True
|
||
assert sender.unknown_send_outcome(asyncio.TimeoutError()) is True
|
||
assert sender.unknown_send_outcome(ActionFailed(error="fetch failed")) is False
|
||
assert sender.unknown_send_outcome(RuntimeError("本地错误")) is False
|
||
|
||
|
||
async def test_media_send_carries_per_call_timeout(monkeypatch, sender, media, uploads):
|
||
"""媒体发送带 `_timeout`（适配器 per-call 超时），长视频才等得到真结果。"""
|
||
monkeypatch.setenv("video_analysis_send_timeout", "42")
|
||
seen: list[dict] = []
|
||
|
||
def fake_local_msg(files, image_post):
|
||
class _Msg:
|
||
async def send(self, **kwargs):
|
||
seen.append(kwargs)
|
||
|
||
return _Msg()
|
||
|
||
monkeypatch.setattr(sender, "_build_local_msg", fake_local_msg)
|
||
ok, _ = await sender.send_pending_media(media())
|
||
assert ok is True
|
||
assert seen == [{"_timeout": 42.0}]
|
||
|
||
|
||
async def test_local_timeout_does_not_resend(monkeypatch, sender, media, uploads):
|
||
"""本地直发超时（结果未知）→ 不走 S3 降级、按「可能已送达」返回。"""
|
||
s3_built: list = []
|
||
|
||
def fake_local_msg(files, image_post):
|
||
class _Msg:
|
||
async def send(self, **_kw):
|
||
raise NetworkError("WebSocket call api send_msg timeout")
|
||
|
||
return _Msg()
|
||
|
||
def fake_s3_msg(items):
|
||
s3_built.append(items)
|
||
|
||
class _Msg:
|
||
async def send(self, **_kw):
|
||
pass
|
||
|
||
return _Msg()
|
||
|
||
monkeypatch.setattr(sender, "_build_local_msg", fake_local_msg)
|
||
monkeypatch.setattr(sender, "_build_s3_msg", fake_s3_msg)
|
||
|
||
ok, pub = await sender.send_pending_media(media())
|
||
assert ok is True # 结果未知按可能已送达：不触发「媒体发送失败」提示
|
||
assert pub == "https://pub.example/x.mp4" # 归档照常完成
|
||
assert s3_built == [] # 关键：没有第二次发送
|
||
|
||
|
||
async def test_action_failed_still_cascades_to_s3(monkeypatch, sender, media, uploads):
|
||
"""协议端明确报错（如 fetch failed）= 确定没发出去，照旧降级到 S3 链接。"""
|
||
sent: list = []
|
||
|
||
def fake_local_msg(files, image_post):
|
||
class _Msg:
|
||
async def send(self, **_kw):
|
||
raise ActionFailed(error="fetch failed")
|
||
|
||
return _Msg()
|
||
|
||
def fake_s3_msg(items):
|
||
class _Msg:
|
||
async def send(self, **_kw): # 链接版成功
|
||
sent.append(items)
|
||
|
||
return _Msg()
|
||
|
||
monkeypatch.setattr(sender, "_build_local_msg", fake_local_msg)
|
||
monkeypatch.setattr(sender, "_build_s3_msg", fake_s3_msg)
|
||
|
||
ok, _ = await sender.send_pending_media(media())
|
||
assert ok is True
|
||
assert len(sent) == 1 # 恰好降级一次
|
||
|
||
|
||
async def test_forward_timeout_does_not_fall_through(
|
||
monkeypatch, sender, media, uploads
|
||
):
|
||
"""合并转发超时（结果未知）→ 不再落回普通消息链路重发。"""
|
||
two = media(n=2)
|
||
fallback_built: list = []
|
||
|
||
async def fake_forward(items, sender_name, *, group_id=None, user_id=None):
|
||
return None # 结果未知
|
||
|
||
def fake_local_msg(files, image_post):
|
||
fallback_built.append(files)
|
||
|
||
class _Msg:
|
||
async def send(self, **_kw):
|
||
raise AssertionError("不该走到普通消息链路")
|
||
|
||
return _Msg()
|
||
|
||
monkeypatch.setattr(sender, "_send_forward_media", fake_forward)
|
||
monkeypatch.setattr(sender, "_forward_sender_name", lambda: "Bot")
|
||
monkeypatch.setattr(sender, "_build_local_msg", fake_local_msg)
|
||
|
||
archive = asyncio.create_task(sender.archive_media(two))
|
||
try:
|
||
ok = await sender._send_media_core(
|
||
two, types.SimpleNamespace(group_id=123), archive
|
||
)
|
||
finally:
|
||
await archive
|
||
assert ok is True
|
||
assert fallback_built == []
|