feat(video-analysis): 抖音免浏览器解析链路 + 快手支持 + 小红书加固 + 发送防重复
- 抖音(2026-10-13):删除 SSR + playwright 旧链路(douyin_ssr.py 移除, douyin_api.py 精简为下载工具),新增 douyin_web.py(x-tt-argus 网关头 + web detail 接口 + 最高码率梯/多 CDN 回退下载)与 douyin_sign.py (纯 Python a_bogus 签名,vendor 自 f2(Apache-2.0),依赖 gmssl SM3); 实测 10/10 成功率、单作品 <1s(旧浏览器链路 5-15s 且随机撞验证码) - 快手(2026-10-08):kuaishou_content.py 分享页 INIT_STATE 静态解析(无登录态), 清晰度梯最佳档 + 图集 URL 组装,入口/策略/Web 面板同步接入 - 小红书:按链接来源定国际/国内站优先序、视频 backupUrls 逐候选回退、 图片 urlDefault→infoList WB_DFT→url 三档回退 - 长视频防重复发送(2026-10-13):媒体发送带 per-call _timeout(默认 300s), 超时=结果未知不再降级重试;auto/active 两条 handler 互斥 - Web 解析封面抽帧 ffmpeg 加 CREATE_NO_WINDOW(pythonw 下不再弹 cmd 黑窗) - 依赖:+gmssl;测试:新增 test_douyin_web / test_kuaishou_parse / test_video_cleanup, 扩充 test_video_policy / test_video_sender_link / test_video_group_file / test_video_media_naming Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
1 parent
a31aab00e8
commit
a789a44f5d
25 files changed
+3536
-895
No files matched your search
@@ -0,0 +1,309 @@
|
||||
"""抖音免浏览器链路(douyin_web.py)单元测试
|
||||
|
||||
覆盖:作品 id 提取 / 签名查询串 / 网关头构造 / 最高码率档选择 / 图集下载项收集。
|
||||
签名与媒体选择都是纯函数,不碰网络(真实链路实测见 debug 目录脚本)。
|
||||
|
||||
douyin_web.py 有插件内相对导入(`...models` / `...utils` / `.douyin_sign` 等),
|
||||
按 test_kuaishou_parse.py 同款姿势:sys.modules 预置包替身 + 真实加载依赖后裸加载。
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
_REPO = Path(__file__).resolve().parents[1]
|
||||
_PLUGIN = _REPO / "hexi" / "plugins" / "nonebot_plugin_video_analysis"
|
||||
_PKG = "hexi.plugins.nonebot_plugin_video_analysis"
|
||||
|
||||
_REAL_MODULES = {
|
||||
f"{_PKG}.models": _PLUGIN / "models.py",
|
||||
f"{_PKG}.utils": _PLUGIN / "utils.py",
|
||||
f"{_PKG}.services.fetchers.douyin_parser": _PLUGIN
|
||||
/ "services/fetchers/douyin_parser.py",
|
||||
f"{_PKG}.services.fetchers.douyin_sign": _PLUGIN
|
||||
/ "services/fetchers/douyin_sign.py",
|
||||
f"{_PKG}.services.fetchers.douyin_api": _PLUGIN / "services/fetchers/douyin_api.py",
|
||||
f"{_PKG}.services.fetchers.douyin_web": _PLUGIN / "services/fetchers/douyin_web.py",
|
||||
}
|
||||
_STUB_PACKAGES = (_PKG, f"{_PKG}.services", f"{_PKG}.services.fetchers")
|
||||
|
||||
|
||||
def _load_real(name: str, path: Path):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def dw():
|
||||
saved = {name: sys.modules.get(name) for name in _STUB_PACKAGES}
|
||||
for name in _STUB_PACKAGES:
|
||||
sys.modules[name] = types.ModuleType(name)
|
||||
for name, path in _REAL_MODULES.items():
|
||||
saved[name] = sys.modules.get(name)
|
||||
_load_real(name, path)
|
||||
|
||||
yield sys.modules[f"{_PKG}.services.fetchers.douyin_web"]
|
||||
|
||||
for name, previous in saved.items():
|
||||
if previous is None:
|
||||
sys.modules.pop(name, None)
|
||||
else:
|
||||
sys.modules[name] = previous
|
||||
|
||||
|
||||
# ───────────────────────── 样本 ─────────────────────────
|
||||
|
||||
|
||||
def _image(url_list=None, download_url_list=None, video=None) -> dict:
|
||||
img = {
|
||||
"uri": "img-uri",
|
||||
"width": 2160,
|
||||
"height": 2880,
|
||||
"url_list": (
|
||||
url_list
|
||||
if url_list is not None
|
||||
else ["https://p3.douyinpic.com/a~q75.webp"]
|
||||
),
|
||||
"download_url_list": (
|
||||
download_url_list
|
||||
if download_url_list is not None
|
||||
else ["https://p9.douyinpic.com/a~tplv-dy-water-v2:xxx.webp"]
|
||||
),
|
||||
}
|
||||
if video is not None:
|
||||
img["video"] = video
|
||||
return img
|
||||
|
||||
|
||||
def _video(bit_rate=None, play_urls=("https://v95.douyinvod.com/full.mp4",)) -> dict:
|
||||
v = {"play_addr": {"url_list": list(play_urls), "width": 1080, "height": 1440}}
|
||||
if bit_rate is not None:
|
||||
v["bit_rate"] = bit_rate
|
||||
return v
|
||||
|
||||
|
||||
def _gear(br, urls, gear_name="gear", h265=0, size=100):
|
||||
return {
|
||||
"gear_name": gear_name,
|
||||
"bit_rate": br,
|
||||
"is_h265": h265,
|
||||
"play_addr": {
|
||||
"url_list": list(urls),
|
||||
"width": 1080,
|
||||
"height": 1440,
|
||||
"data_size": size,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
# ───────────────────────── extract_aweme_id ─────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url,expected",
|
||||
[
|
||||
("https://www.douyin.com/note/7696029938990839419", "7696029938990839419"),
|
||||
(
|
||||
"https://www.douyin.com/video/7618117951811949541?foo=1",
|
||||
"7618117951811949541",
|
||||
),
|
||||
(
|
||||
"https://www.iesdouyin.com/share/video/7412345678901234567/?region=CN",
|
||||
"7412345678901234567",
|
||||
),
|
||||
(
|
||||
"https://www.iesdouyin.com/share/note/7412345678901234567/",
|
||||
"7412345678901234567",
|
||||
),
|
||||
("https://v.douyin.com/abc123/", None),
|
||||
("https://github.com/x/y", None),
|
||||
],
|
||||
)
|
||||
def test_extract_aweme_id(dw, url, expected):
|
||||
assert dw.extract_aweme_id(url) == expected
|
||||
|
||||
|
||||
# ───────────────────────── 签名 ─────────────────────────
|
||||
|
||||
|
||||
def test_build_signed_query(dw):
|
||||
qs = dw.build_signed_query("7618117951811949541")
|
||||
assert "aweme_id=7618117951811949541" in qs
|
||||
assert "a_bogus=" in qs
|
||||
a_bogus = qs.split("a_bogus=")[-1]
|
||||
assert len(a_bogus) >= 100
|
||||
# 两次签名都有效(签名带随机熵,不要求逐字节一致)
|
||||
qs2 = dw.build_signed_query("7618117951811949541")
|
||||
assert "a_bogus=" in qs2
|
||||
|
||||
|
||||
def test_build_signed_query_requires_aweme_id_last(dw):
|
||||
"""aweme_id 应在查询串末尾、a_bogus 追加在其后(与浏览器请求形态一致)"""
|
||||
qs = dw.build_signed_query("123")
|
||||
assert qs.index("aweme_id=123") < qs.index("a_bogus=")
|
||||
|
||||
|
||||
# ───────────────────────── 网关头 ─────────────────────────
|
||||
|
||||
|
||||
def test_build_headers_always_argus_and_uifid(dw):
|
||||
headers = dw._build_headers({"UIFID": "u-123"})
|
||||
assert headers["x-tt-argus"] == "1"
|
||||
assert headers["uifid"] == "u-123"
|
||||
assert headers["User-Agent"] == dw.WEB_UA
|
||||
|
||||
|
||||
def test_build_headers_uifid_temp_fallback(dw):
|
||||
headers = dw._build_headers({"UIFID_TEMP": "tmp"})
|
||||
assert headers["uifid"] == "tmp"
|
||||
assert "uifid" not in dw._build_headers({})
|
||||
|
||||
|
||||
def test_cookie_map_filters_domain(dw):
|
||||
cookies = [
|
||||
{"name": "UIFID", "value": "u1", "domain": ".douyin.com"},
|
||||
{"name": "a1", "value": "x", "domain": ".xiaohongshu.com"},
|
||||
{"name": "ttwid", "value": "t", "domain": "www.douyin.com"},
|
||||
]
|
||||
m = dw._douyin_cookie_map(cookies)
|
||||
assert m == {"UIFID": "u1", "ttwid": "t"}
|
||||
|
||||
|
||||
# ───────────────────────── 最高码率档选择 ─────────────────────────
|
||||
|
||||
|
||||
def test_best_gear_picks_max_bitrate(dw):
|
||||
video = {
|
||||
"bit_rate": [
|
||||
_gear(773939, ["https://cdn/1080.mp4"], gear_name="normal_1080_0"),
|
||||
_gear(385679, ["https://cdn/720.mp4"], gear_name="720_1_1", h265=1),
|
||||
_gear(
|
||||
1419088,
|
||||
["https://cdn/best.mp4", "https://cdn/best-bak.mp4"],
|
||||
gear_name="1080_1_1",
|
||||
),
|
||||
],
|
||||
"play_addr": {"url_list": ["https://cdn/main.mp4"]},
|
||||
}
|
||||
urls, meta = dw._best_urls_from_video(video)
|
||||
assert urls == ["https://cdn/best.mp4", "https://cdn/best-bak.mp4"]
|
||||
assert meta["bit_rate"] == 1419088
|
||||
assert meta["width"] == 1080
|
||||
|
||||
|
||||
def test_best_gear_falls_back_to_play_addr(dw):
|
||||
video = _video(bit_rate=[], play_urls=["https://cdn/only.mp4"])
|
||||
urls, meta = dw._best_urls_from_video(video)
|
||||
assert urls == ["https://cdn/only.mp4"]
|
||||
assert meta["gear_name"] is None
|
||||
|
||||
|
||||
def test_best_gear_skips_gear_without_urls(dw):
|
||||
video = {
|
||||
"bit_rate": [
|
||||
{"bit_rate": 9999999, "play_addr": {"url_list": []}},
|
||||
_gear(100, ["https://cdn/ok.mp4"]),
|
||||
],
|
||||
"play_addr": {"url_list": ["https://cdn/main.mp4"]},
|
||||
}
|
||||
urls, _ = dw._best_urls_from_video(video)
|
||||
assert urls == ["https://cdn/ok.mp4"]
|
||||
|
||||
|
||||
# ───────────────────────── 下载项收集 ─────────────────────────
|
||||
|
||||
|
||||
def test_collect_video_post(dw):
|
||||
detail = {
|
||||
"aweme_id": "1",
|
||||
"desc": "x",
|
||||
"video": _video(bit_rate=[_gear(500, ["https://cdn/v.mp4"])]),
|
||||
}
|
||||
items, is_img = dw.collect_download_items(detail)
|
||||
assert is_img is False
|
||||
assert items == [(["https://cdn/v.mp4"], ".mp4")]
|
||||
|
||||
|
||||
def test_collect_video_post_without_urls_raises(dw):
|
||||
with pytest.raises(Exception):
|
||||
dw.collect_download_items({"video": {"play_addr": {"url_list": []}}})
|
||||
|
||||
|
||||
def test_collect_images_prefers_no_watermark_urls(dw):
|
||||
detail = {
|
||||
"images": [
|
||||
_image(
|
||||
url_list=["https://p3/a-q75.webp", "https://p3/a-q75.jpeg"],
|
||||
download_url_list=["https://p9/a-water.webp"],
|
||||
)
|
||||
]
|
||||
}
|
||||
items, is_img = dw.collect_download_items(detail)
|
||||
assert is_img is True
|
||||
assert items[0][0] == [
|
||||
"https://p3/a-q75.webp",
|
||||
"https://p3/a-q75.jpeg", # 无水印在前
|
||||
"https://p9/a-water.webp", # 带作者水印的保存版兜底
|
||||
]
|
||||
assert items[0][1] == ""
|
||||
|
||||
|
||||
def test_collect_animated_note_uses_video_gears(dw):
|
||||
anim_video = {
|
||||
"bit_rate": [_gear(300, ["https://cdn/anim-best.mp4"])],
|
||||
"play_addr": {"url_list": ["https://cdn/anim.mp4"]},
|
||||
}
|
||||
detail = {
|
||||
"images": [
|
||||
_image(url_list=[], download_url_list=[], video=anim_video),
|
||||
_image(url_list=[], download_url_list=[], video=anim_video),
|
||||
]
|
||||
}
|
||||
items, is_img = dw.collect_download_items(detail)
|
||||
assert is_img is True
|
||||
assert items == [
|
||||
(["https://cdn/anim-best.mp4"], ".mp4"),
|
||||
(["https://cdn/anim-best.mp4"], ".mp4"),
|
||||
]
|
||||
|
||||
|
||||
def test_collect_mixed_note(dw):
|
||||
"""图 + 内嵌视频混排:各自按类型收集"""
|
||||
detail = {
|
||||
"images": [
|
||||
_image(),
|
||||
_image(
|
||||
url_list=[],
|
||||
download_url_list=[],
|
||||
video={"play_addr": {"url_list": ["https://cdn/clip.mp4"]}},
|
||||
),
|
||||
]
|
||||
}
|
||||
items, is_img = dw.collect_download_items(detail)
|
||||
assert is_img is True
|
||||
assert len(items) == 2
|
||||
assert items[0][1] == "" # 图片
|
||||
assert items[1] == (["https://cdn/clip.mp4"], ".mp4")
|
||||
|
||||
|
||||
# ───────────────────────── 扩展名推断 ─────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"url,expected",
|
||||
[
|
||||
("https://x/a.jpg?x=1", ".jpg"),
|
||||
("https://x/a~q75.webp?y=2", ".webp"),
|
||||
("https://x/a.jpeg", ".jpeg"),
|
||||
("https://x/a", ".jpg"),
|
||||
("https://x/a.heic", ".heic"),
|
||||
],
|
||||
)
|
||||
def test_infer_extension(dw, url, expected):
|
||||
assert dw._infer_extension(url) == expected
|
||||
@@ -0,0 +1,302 @@
|
||||
"""快手分享页解析(kuaishou_content.py 纯函数)单元测试
|
||||
|
||||
只测不碰网络的解析/命名部分:INIT_STATE 提取、视频/图集 URL 组装、媒体命名。
|
||||
样本仿自 2026-10-08 实测的分享页(v.kuaishou.com 短链 → v.m.chenzhongtech.com/fw/photo)。
|
||||
|
||||
kuaishou_content.py 有插件内相对导入(`...models` / `...utils`),无法裸加载,
|
||||
故在 sys.modules 里预置父包替身 + 真实加载 utils/models 后按路径加载目标模块
|
||||
(与 test_video_sender_link.py 同法)。
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
_REPO = Path(__file__).resolve().parents[1]
|
||||
_PLUGIN = _REPO / "hexi" / "plugins" / "nonebot_plugin_video_analysis"
|
||||
_PKG = "hexi.plugins.nonebot_plugin_video_analysis"
|
||||
|
||||
#: 真实加载的插件内模块(utils 纯标准库、models 只有一个异常类)
|
||||
_REAL_MODULES = {
|
||||
f"{_PKG}.models": _PLUGIN / "models.py",
|
||||
f"{_PKG}.utils": _PLUGIN / "utils.py",
|
||||
}
|
||||
#: 只当命名空间用的父包(避免触发插件包 __init__ 的 NoneBot 初始化)
|
||||
_STUB_PACKAGES = (_PKG, f"{_PKG}.services", f"{_PKG}.services.fetchers")
|
||||
|
||||
|
||||
def _load_real(name: str, path: Path):
|
||||
spec = importlib.util.spec_from_file_location(name, path)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[name] = module
|
||||
spec.loader.exec_module(module)
|
||||
return module
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def ks():
|
||||
saved = {name: sys.modules.get(name) for name in _STUB_PACKAGES}
|
||||
for name in _STUB_PACKAGES:
|
||||
sys.modules[name] = types.ModuleType(name)
|
||||
for name, path in _REAL_MODULES.items():
|
||||
saved[name] = sys.modules.get(name)
|
||||
_load_real(name, path)
|
||||
|
||||
target = f"{_PKG}.services.fetchers.kuaishou_content"
|
||||
saved[target] = sys.modules.get(target)
|
||||
module = _load_real(target, _PLUGIN / "services" / "fetchers" / "kuaishou_content.py")
|
||||
|
||||
yield module
|
||||
|
||||
for name, previous in saved.items():
|
||||
if previous is None:
|
||||
sys.modules.pop(name, None)
|
||||
else:
|
||||
sys.modules[name] = previous
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def u(ks):
|
||||
"""插件 utils(命名断言用同一套函数,别在测试里手拼)"""
|
||||
return sys.modules[f"{_PKG}.utils"]
|
||||
|
||||
|
||||
# ───────────────────────── 样本 ─────────────────────────
|
||||
|
||||
KEY = "string/rest/wd/ugH5App/photo/simple/info@kpn>KUAISHOU"
|
||||
|
||||
|
||||
def _photo(**over) -> dict:
|
||||
photo = {
|
||||
"caption": "欧巴欧巴哦 #蔚蓝档案 #砂狼白子",
|
||||
"userName": "砂狼白子",
|
||||
"userId": 2329823245,
|
||||
"photoId": "5206161376908999738",
|
||||
"duration": 15150,
|
||||
"timestamp": 1789916802288,
|
||||
"coverUrls": [
|
||||
{"cdn": "p2.a.yximgs.com", "url": "https://p2.a.yximgs.com/cover.jpg"}
|
||||
],
|
||||
"mainMvUrls": [
|
||||
{"cdn": "tymov2.a.kwimgs.com", "url": "https://tymov2.a.kwimgs.com/a_b.mp4"},
|
||||
{"cdn": "txmov2.a.kwimgs.com", "url": "https://txmov2.a.kwimgs.com/a_b.mp4"},
|
||||
],
|
||||
"ext_params": {"mtype": 3, "video": 15150},
|
||||
}
|
||||
photo.update(over)
|
||||
return photo
|
||||
|
||||
|
||||
def _page(data: dict) -> str:
|
||||
payload = json.dumps(data, ensure_ascii=False)
|
||||
return f"<!doctype html><html><script>window.INIT_STATE={payload}</script></html>"
|
||||
|
||||
|
||||
def _page_with(photo: dict | None, prelude: dict | None = None) -> str:
|
||||
data = dict(prelude or {})
|
||||
data[KEY] = {"result": 1, "photo": photo}
|
||||
return _page(data)
|
||||
|
||||
|
||||
# ───────────────────────── INIT_STATE 提取 ─────────────────────────
|
||||
|
||||
|
||||
def test_extract_photo_skips_empty_entries(ks):
|
||||
"""INIT_STATE 是一堆缓存键的 dict,photo 在第一个非空项里(顺序不定)"""
|
||||
page = _page(
|
||||
{
|
||||
"some/cache/key": {"result": 1, "photo": None},
|
||||
"another/key": {"result": 2},
|
||||
KEY: {"result": 1, "photo": _photo()},
|
||||
}
|
||||
)
|
||||
assert ks._extract_photo(page)["photoId"] == "5206161376908999738"
|
||||
|
||||
|
||||
def test_extract_photo_tolerates_js_literals(ks):
|
||||
"""页面偶有 undefined 之类非 JSON 字面量(同小红书预处理)"""
|
||||
page = (
|
||||
"<script>window.INIT_STATE = "
|
||||
'{"k": {"result": 1, "photo": {"photoId": "1", "userName": undefined}}}'
|
||||
"</script>"
|
||||
)
|
||||
photo = ks._extract_photo(page)
|
||||
assert photo["photoId"] == "1"
|
||||
assert photo["userName"] is None
|
||||
|
||||
|
||||
def test_extract_photo_missing_state(ks):
|
||||
"""风控壳页(桌面 UA / www 站)没有 INIT_STATE → 报错而不是静默返回空"""
|
||||
err = sys.modules[f"{_PKG}.models"].ContentFetchError
|
||||
with pytest.raises(err):
|
||||
ks._extract_photo("<html><body>risk control</body></html>")
|
||||
|
||||
|
||||
def test_extract_photo_without_photo(ks):
|
||||
"""INIT_STATE 在但没有作品(已删除/私密)→ 报错"""
|
||||
err = sys.modules[f"{_PKG}.models"].ContentFetchError
|
||||
with pytest.raises(err):
|
||||
ks._extract_photo(_page({"k": {"result": 2, "photo": None}}))
|
||||
|
||||
|
||||
# ───────────────────────── 媒体 URL ─────────────────────────
|
||||
|
||||
|
||||
def test_first_url_prefers_first_available(ks):
|
||||
assert ks._first_url([{"cdn": "a", "url": "https://a/1.mp4"}]) == "https://a/1.mp4"
|
||||
# 首项没有 url(快手同一资源的备用项可能缺字段)时继续往后找
|
||||
assert ks._first_url([{"cdn": "a"}, {"url": "https://b/2.mp4"}]) == "https://b/2.mp4"
|
||||
assert ks._first_url([]) is None
|
||||
assert ks._first_url(None) is None
|
||||
|
||||
|
||||
def _rep(width, height, fps, codec, bitrate, url, **over) -> dict:
|
||||
rep = {
|
||||
"width": width,
|
||||
"height": height,
|
||||
"frameRate": fps,
|
||||
"videoCodec": codec,
|
||||
"avgBitrate": bitrate,
|
||||
"url": url,
|
||||
}
|
||||
rep.update(over)
|
||||
return rep
|
||||
|
||||
|
||||
def test_manifest_video_url_picks_highest_quality(ks):
|
||||
"""清晰度梯取最优:先比分辨率,再帧率、编码(avc 优先)、码率"""
|
||||
photo = _photo(
|
||||
manifest={
|
||||
"adaptationSet": [
|
||||
{
|
||||
"representation": [
|
||||
# 1080p 30fps hevc 也要赢过 720p 60fps avc(分辨率优先)
|
||||
_rep(1920, 1080, 30, "hevc", 2000, "https://c/1080.mp4"),
|
||||
_rep(1280, 720, 60, "avc", 1597, "https://c/720p60.mp4"),
|
||||
_rep(1280, 720, 30, "hevc", 726, "https://c/720p30.mp4"),
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
assert ks._manifest_video_url(photo) == "https://c/1080.mp4"
|
||||
|
||||
|
||||
def test_manifest_video_url_prefers_avc_on_tie(ks):
|
||||
"""同分辨率同帧率 → avc 优先(兼容性),hevc 只在画质更高时才选"""
|
||||
photo = _photo(
|
||||
manifest={
|
||||
"adaptationSet": [
|
||||
{
|
||||
"representation": [
|
||||
_rep(1280, 720, 60, "avc", 1597, "https://c/avc.mp4"),
|
||||
# 码率更高但编码是 hevc,仍输给 avc
|
||||
_rep(1280, 720, 60, "hevc", 2000, "https://c/hevc.mp4"),
|
||||
]
|
||||
}
|
||||
]
|
||||
}
|
||||
)
|
||||
assert ks._manifest_video_url(photo) == "https://c/avc.mp4"
|
||||
|
||||
|
||||
def test_manifest_video_url_skips_broken_entries(ks):
|
||||
"""缺 url / 结构不对的档跳过,不因为一条脏数据整体失败"""
|
||||
photo = _photo(
|
||||
manifest={
|
||||
"adaptationSet": [
|
||||
{"representation": [{"width": 1920, "height": 1080}, "junk"]},
|
||||
{"representation": [_rep(1280, 720, 30, "avc", 900, "https://c/720.mp4")]},
|
||||
]
|
||||
}
|
||||
)
|
||||
assert ks._manifest_video_url(photo) == "https://c/720.mp4"
|
||||
assert ks._manifest_video_url(_photo()) is None
|
||||
assert ks._manifest_video_url(_photo(manifest={"adaptationSet": []})) is None
|
||||
|
||||
|
||||
def test_video_url_falls_back_to_main_mv_urls(ks):
|
||||
"""老页面没有 manifest → 退回 mainMvUrls(参考实现的默认档路径)"""
|
||||
assert ks._video_url(_photo()) == "https://tymov2.a.kwimgs.com/a_b.mp4"
|
||||
photo = _photo(
|
||||
manifest={"adaptationSet": [{"representation": [_rep(1920, 1080, 30, "avc", 2000, "https://c/best.mp4")]}]}
|
||||
)
|
||||
assert ks._video_url(photo) == "https://c/best.mp4"
|
||||
|
||||
|
||||
def test_atlas_image_urls(ks):
|
||||
photo = _photo(
|
||||
ext_params={
|
||||
"atlas": {
|
||||
"cdnList": [{"cdn": "p1.a.yximgs.com"}, {"cdn": "p2.a.yximgs.com"}],
|
||||
"list": ["upic/a.jpg", "upic/b.jpg"],
|
||||
}
|
||||
}
|
||||
)
|
||||
assert ks._atlas_image_urls(photo) == [
|
||||
"https://p1.a.yximgs.com/upic/a.jpg",
|
||||
"https://p1.a.yximgs.com/upic/b.jpg",
|
||||
]
|
||||
|
||||
|
||||
def test_atlas_image_urls_absent(ks):
|
||||
"""视频作品的 ext_params 里没有 atlas → 不是图集"""
|
||||
assert ks._atlas_image_urls(_photo()) == []
|
||||
assert ks._atlas_image_urls(_photo(ext_params={})) == []
|
||||
# 有 atlas 但缺图床/路径 → 空(宁可当视频处理也不要拼出坏链接)
|
||||
assert ks._atlas_image_urls(_photo(ext_params={"atlas": {"list": ["a.jpg"]}})) == []
|
||||
assert ks._atlas_image_urls(
|
||||
_photo(ext_params={"atlas": {"cdnList": [{"cdn": "c"}]}})
|
||||
) == []
|
||||
# 老字段名 extParams 也认(ext_params 缺失时才会去看它)
|
||||
assert ks._atlas_image_urls(
|
||||
_photo(
|
||||
ext_params=None,
|
||||
extParams={"atlas": {"cdnList": [{"cdn": "c"}], "list": ["a.jpg"]}},
|
||||
)
|
||||
) == ["https://c/a.jpg"]
|
||||
|
||||
|
||||
# ───────────────────────── 命名 ─────────────────────────
|
||||
|
||||
|
||||
def test_rel_stem_uses_author_id(ks, u):
|
||||
"""有 userId(数字)→ 稳定作者目录,标题去掉 #话题"""
|
||||
assert ks._build_rel_stem(_photo()) == "砂狼白子_2329823245/欧巴欧巴哦"
|
||||
|
||||
|
||||
def test_rel_stem_without_author(ks, u):
|
||||
"""昵称是兜底串("快手用户")→ 按"未知作者"规则 + 作品来源码"""
|
||||
stem = ks._build_rel_stem(_photo(userName="快手用户", userId=0))
|
||||
code = u.short_source_code("5206161376908999738")
|
||||
assert stem == f"未知作者_{code}/欧巴欧巴哦"
|
||||
|
||||
|
||||
def test_rel_stem_empty_caption(ks, u):
|
||||
"""标题只有 #话题(slugify 后为空)→ 作品_来源码,同一作品稳定可复现"""
|
||||
stem = ks._build_rel_stem(_photo(caption="#蔚蓝档案"))
|
||||
code = u.short_source_code("5206161376908999738")
|
||||
assert stem == f"砂狼白子_2329823245/作品_{code}"
|
||||
|
||||
|
||||
# ───────────────────────── URL 识别 ─────────────────────────
|
||||
|
||||
|
||||
def test_web_short_video_regex(ks):
|
||||
"""www 站形态(分享出来的网页链接,实际会重写成移动端分享页)"""
|
||||
for url, expected in (
|
||||
("https://www.kuaishou.com/short-video/3xhjgcmir24m4nm", "3xhjgcmir24m4nm"),
|
||||
("https://www.kuaishou.com/short-video/5206161376908999738", "5206161376908999738"),
|
||||
("https://kuaishou.com/short-video/abc123", "abc123"),
|
||||
):
|
||||
m = ks.WEB_SHORT_VIDEO_RE.search(url)
|
||||
assert m and m.group(1) == expected
|
||||
|
||||
|
||||
def test_short_link_regex(ks):
|
||||
m = ks.SHORT_LINK_RE.search("看看这个 https://v.kuaishou.com/nUqhAiDI 好看")
|
||||
assert m and m.group(0) == "v.kuaishou.com/nUqhAiDI"
|
||||
@@ -0,0 +1,214 @@
|
||||
"""temp 目录清理逻辑(video_analysis/cleanup.py)单元测试
|
||||
|
||||
这组函数现在同时被「/清理temp 命令」「每日定时」「Web 缓存面板的手动清理」
|
||||
三条路复用,是**真的会删文件**的代码,所以单测覆盖"删什么/不删什么"。
|
||||
|
||||
cleanup.py 有插件内相对导入(`.utils`)和 import 期副作用(`get_plugin_config`
|
||||
建配置对象、`_register_auto_job()` 注册定时任务),无法裸加载 —— 按
|
||||
test_video_sender_link.py 的办法:sys.modules 预置替身 + monkeypatch nonebot 的
|
||||
几个工厂函数,再按路径加载目标模块。
|
||||
|
||||
文件一律用 tmp_path 造真实文件/目录 + `os.utime` 改 mtime,**不 mock 文件系统**:
|
||||
这些判断(mtime 过期、目录为空、60s 目录下限)全在真实 stat 语义上。
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
_REPO = Path(__file__).resolve().parents[1]
|
||||
_PLUGIN = _REPO / "hexi" / "plugins" / "nonebot_plugin_video_analysis"
|
||||
_PKG = "hexi.plugins.nonebot_plugin_video_analysis"
|
||||
|
||||
_STUBS = {
|
||||
"hexi": {},
|
||||
"hexi.core": {},
|
||||
"hexi.core.custom_utils": {"check_manage": None},
|
||||
"hexi.core.scheduler": {"add_job": lambda *a, **k: None},
|
||||
# cleanup.py 现在把清理提交到任务槽;这里只测 clean_temp_files / temp_stats,
|
||||
# 槽本身由 tests/test_task_slots.py 覆盖,所以给个够 import 的替身即可。
|
||||
"hexi.core.task_slots": {"TASKS": None, "Task": type("Task", (), {})},
|
||||
"nonebot_plugin_apscheduler": {"scheduler": types.SimpleNamespace(remove_job=None)},
|
||||
"nonebot_plugin_alconna": {"UniMessage": type("UniMessage", (), {})},
|
||||
}
|
||||
|
||||
|
||||
class _FakeMatcher:
|
||||
"""`on_command(...)` 的替身:`.handle()` 当装饰器用。"""
|
||||
|
||||
def handle(self, *args, **kwargs):
|
||||
def deco(func):
|
||||
return func
|
||||
|
||||
return deco
|
||||
|
||||
|
||||
class _TempRoot:
|
||||
"""`utils.get_temp_root` 的替身:指向测试用的临时根。"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.root: Path | None = None
|
||||
|
||||
def __call__(self, sub: str = "") -> Path:
|
||||
assert self.root is not None, "测试里要先 sched.get_temp_root.root = ..."
|
||||
return self.root / sub if sub else self.root
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def cleanup(monkeypatch, tmp_path):
|
||||
saved = {name: sys.modules.get(name) for name in _STUBS}
|
||||
for name, attrs in _STUBS.items():
|
||||
module = types.ModuleType(name)
|
||||
for key, value in attrs.items():
|
||||
setattr(module, key, value)
|
||||
sys.modules[name] = module
|
||||
|
||||
# 插件包本体也用替身,避免触发 __init__ 的 NoneBot 初始化
|
||||
saved[_PKG] = sys.modules.get(_PKG)
|
||||
sys.modules[_PKG] = types.ModuleType(_PKG)
|
||||
temp_root = _TempRoot()
|
||||
utils_mod = types.ModuleType(f"{_PKG}.utils")
|
||||
utils_mod.get_temp_root = temp_root
|
||||
saved[f"{_PKG}.utils"] = sys.modules.get(f"{_PKG}.utils")
|
||||
sys.modules[f"{_PKG}.utils"] = utils_mod
|
||||
|
||||
import nonebot
|
||||
|
||||
monkeypatch.setattr(nonebot, "require", lambda name: None, raising=False)
|
||||
monkeypatch.setattr(
|
||||
nonebot, "get_plugin_config", lambda cls: cls(), raising=False
|
||||
) # 默认配置:auto / 7 天 / 4 点
|
||||
monkeypatch.setattr(nonebot, "on_command", lambda *a, **k: _FakeMatcher(), raising=False)
|
||||
|
||||
target = f"{_PKG}.cleanup"
|
||||
saved[target] = sys.modules.get(target)
|
||||
spec = importlib.util.spec_from_file_location(target, _PLUGIN / "cleanup.py")
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[target] = module
|
||||
spec.loader.exec_module(module)
|
||||
|
||||
temp_root.root = tmp_path / "temp"
|
||||
temp_root.root.mkdir()
|
||||
module._tmp = temp_root.root # 方便测试直接拿到根
|
||||
|
||||
yield module
|
||||
|
||||
for name, previous in saved.items():
|
||||
if previous is None:
|
||||
sys.modules.pop(name, None)
|
||||
else:
|
||||
sys.modules[name] = previous
|
||||
|
||||
|
||||
def _make(root: Path, rel: str, age_days: float = 0.0, data: bytes = b"x") -> Path:
|
||||
"""造一个文件,并把 mtime 往回拨 age_days 天。"""
|
||||
path = root / rel
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
path.write_bytes(data)
|
||||
if age_days:
|
||||
t = time.time() - age_days * 86400
|
||||
os.utime(path, (t, t))
|
||||
return path
|
||||
|
||||
|
||||
# ───────────────────────── 清理 ─────────────────────────
|
||||
|
||||
|
||||
def test_removes_stale_keeps_fresh(cleanup):
|
||||
root = cleanup._tmp
|
||||
stale = _make(root, "douyin/作者_1/旧作品.mp4", age_days=2)
|
||||
fresh = _make(root, "douyin/作者_1/新作品.mp4", age_days=0)
|
||||
|
||||
removed, total = cleanup.clean_temp_files(days=1)
|
||||
|
||||
assert (removed, total) == (1, 2)
|
||||
assert not stale.exists()
|
||||
assert fresh.exists()
|
||||
|
||||
|
||||
def test_days_zero_clears_files_but_keeps_fresh_dirs(cleanup):
|
||||
"""`/清理temp 0`(= Web 的「全部清空」):文件全删,但刚建的目录留着。
|
||||
|
||||
目录另有 60s 下限(见 `_prune_empty_dirs`)—— 下载落盘是先 mkdir 再写文件,
|
||||
把目录一起删掉会让正在进行的下载失败。
|
||||
"""
|
||||
root = cleanup._tmp
|
||||
a = _make(root, "kuaishou/作者_2/作品.mp4", age_days=0)
|
||||
b = _make(root, "xiaohongshu/作者_3/笔记/001.jpg", age_days=0)
|
||||
|
||||
removed, total = cleanup.clean_temp_files(days=0)
|
||||
|
||||
assert (removed, total) == (2, 2)
|
||||
assert not a.exists() and not b.exists()
|
||||
assert (root / "kuaishou/作者_2").is_dir() # 新鲜目录保留
|
||||
assert (root / "xiaohongshu/作者_3/笔记").is_dir()
|
||||
|
||||
|
||||
def test_prunes_stale_empty_dirs_bottom_up(cleanup):
|
||||
"""过期空目录会被收掉,但是**逐轮**回收而不是同一轮全清。
|
||||
|
||||
删文件(或删子目录)会把所在目录的 mtime 刷成"现在",当轮它就不再"过期",
|
||||
要等下一轮 —— 这正是 `_prune_empty_dirs` 注释里写的行为(也被这条测试钉住)。
|
||||
"""
|
||||
root = cleanup._tmp
|
||||
_make(root, "bilibili/作者_4/作品/001.jpg", age_days=3)
|
||||
nested = root / "bilibili/作者_4/作品"
|
||||
author = root / "bilibili/作者_4"
|
||||
|
||||
removed, _ = cleanup.clean_temp_files(days=1)
|
||||
assert removed == 1
|
||||
assert nested.is_dir() and author.is_dir() # 文件刚删,目录 mtime 是新的
|
||||
|
||||
old = time.time() - 3 * 86400 # 模拟又过了几天
|
||||
os.utime(nested, (old, old))
|
||||
os.utime(author, (old, old))
|
||||
cleanup.clean_temp_files(days=1)
|
||||
assert not nested.exists() # 深的先走
|
||||
|
||||
os.utime(author, (old, old)) # 上一轮删子目录,又把父目录的 mtime 刷新了
|
||||
cleanup.clean_temp_files(days=1)
|
||||
assert not author.exists()
|
||||
|
||||
|
||||
def test_keeps_stale_but_nonempty_dirs(cleanup):
|
||||
"""过期但里面还有新鲜文件的目录不能删。"""
|
||||
root = cleanup._tmp
|
||||
_make(root, "douyin/作者_5/新片.mp4", age_days=0)
|
||||
old_dir = root / "douyin/作者_5"
|
||||
os.utime(old_dir, (time.time() - 3 * 86400,) * 2)
|
||||
|
||||
cleanup.clean_temp_files(days=1)
|
||||
|
||||
assert old_dir.is_dir()
|
||||
|
||||
|
||||
def test_missing_root_is_noop(cleanup):
|
||||
"""根目录不存在(还没下载过任何东西)→ (0, 0),不抛。"""
|
||||
cleanup._tmp.rmdir()
|
||||
assert cleanup.clean_temp_files(days=1) == (0, 0)
|
||||
|
||||
|
||||
# ───────────────────────── 统计 ─────────────────────────
|
||||
|
||||
|
||||
def test_stats_counts_files_bytes_dirs(cleanup):
|
||||
root = cleanup._tmp
|
||||
_make(root, "douyin/作者_6/a.mp4", age_days=2, data=b"12345")
|
||||
_make(root, "douyin/作者_6/b.mp4", age_days=0, data=b"123")
|
||||
_make(root, "douyin/作者_7/c.jpg", age_days=5, data=b"1")
|
||||
|
||||
st = cleanup.temp_stats()
|
||||
|
||||
assert st["files"] == 3
|
||||
assert st["bytes"] == 9
|
||||
assert st["dirs"] == 3 # douyin / 作者_6 / 作者_7(子目录各算一层)
|
||||
|
||||
|
||||
def test_stats_missing_root(cleanup):
|
||||
cleanup._tmp.rmdir()
|
||||
assert cleanup.temp_stats() == {"files": 0, "bytes": 0, "dirs": 0}
|
||||
@@ -11,6 +11,7 @@ import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from nonebot.adapters.onebot.v11.exception import NetworkError
|
||||
|
||||
_MODULE_PATH = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
@@ -223,6 +224,75 @@ async def test_missing_file_short_circuits(gf, tmp_path, monkeypatch):
|
||||
assert called == []
|
||||
|
||||
|
||||
# ────────── 传输超时=结果未知:不降级直传(防两份群文件) ──────────
|
||||
# 与 handlers/sender.py 同一物理事实:超时只是 bot 不再等待,协议端会把这次
|
||||
# 上传做完(大文件远超默认 30s);此时再走本地直传就是重复投递。
|
||||
|
||||
|
||||
def test_send_timeout_default_and_env(gf, monkeypatch):
|
||||
monkeypatch.delenv("video_analysis_send_timeout", raising=False)
|
||||
monkeypatch.delenv("VIDEO_ANALYSIS_SEND_TIMEOUT", raising=False)
|
||||
assert gf._send_timeout() == 300.0
|
||||
|
||||
monkeypatch.setenv("video_analysis_send_timeout", "77")
|
||||
assert gf._send_timeout() == 77.0
|
||||
|
||||
monkeypatch.setenv("video_analysis_send_timeout", "bad")
|
||||
assert gf._send_timeout() == 300.0
|
||||
|
||||
|
||||
async def test_upload_call_carries_per_call_timeout(gf, monkeypatch):
|
||||
"""upload_group_file API 带 `_timeout`(适配器 per-call 超时)。"""
|
||||
monkeypatch.setenv("video_analysis_send_timeout", "77")
|
||||
calls: list[dict] = []
|
||||
|
||||
class _Bot:
|
||||
async def call_api(self, api, **kwargs):
|
||||
calls.append({"api": api, **kwargs})
|
||||
|
||||
monkeypatch.setattr(gf, "get_bot", lambda: _Bot())
|
||||
await gf._call_upload("file:///D:/x.zip", 123, "x.zip")
|
||||
assert calls == [
|
||||
{
|
||||
"api": "upload_group_file",
|
||||
"group_id": 123,
|
||||
"file": "file:///D:/x.zip",
|
||||
"name": "x.zip",
|
||||
"_timeout": 77.0,
|
||||
}
|
||||
]
|
||||
|
||||
|
||||
async def test_s3_timeout_does_not_fall_back_to_local(gf, tmp_path, monkeypatch):
|
||||
"""S3 链接上传超时=结果未知(协议端可能还在传)→ 不再直传,避免两份。"""
|
||||
seen: list[str] = []
|
||||
link = "http://192.168.2.15:5246/PLANA/x.zip?sig=1"
|
||||
|
||||
async def fake_call(file_value, group_id, name, folder_id=None):
|
||||
seen.append(file_value)
|
||||
raise NetworkError("WebSocket call api upload_group_file timeout")
|
||||
|
||||
monkeypatch.setattr(gf, "_call_upload", fake_call)
|
||||
monkeypatch.setattr(gf, "_s3_url", lambda path, policy: link)
|
||||
|
||||
file = _make_files(tmp_path / "src", ["a.zip"])[0]
|
||||
assert await gf.upload_group_file(file, 123) is True
|
||||
assert seen == [link] # 只有 S3 那一次,没有第二次 file:// 直传
|
||||
|
||||
|
||||
async def test_local_upload_timeout_reports_unknown(gf, tmp_path, monkeypatch):
|
||||
"""本地直传超时同样按「可能已送达」处理(不误报失败即可)。"""
|
||||
|
||||
async def fake_call(*args, **kwargs):
|
||||
raise NetworkError("timeout")
|
||||
|
||||
monkeypatch.setattr(gf, "_call_upload", fake_call)
|
||||
monkeypatch.setattr(gf, "_s3_url", lambda path, policy: "")
|
||||
|
||||
file = _make_files(tmp_path / "src", ["a.zip"])[0]
|
||||
assert await gf.upload_group_file(file, 123) is True
|
||||
|
||||
|
||||
# ───────────────────────── 投递编排 ─────────────────────────
|
||||
|
||||
|
||||
@@ -241,9 +311,7 @@ async def test_zip_mode_uploads_single_archive(gf, tmp_path, monkeypatch):
|
||||
archive = tmp_path / "pack.zip"
|
||||
archive.write_bytes(b"zip")
|
||||
calls = _record_uploads(gf, monkeypatch, archive)
|
||||
monkeypatch.setattr(
|
||||
gf, "build_archive", lambda *a, **kw: archive
|
||||
)
|
||||
monkeypatch.setattr(gf, "build_archive", lambda *a, **kw: archive)
|
||||
|
||||
files = _make_files(tmp_path / "src", ["a.txt", "b.txt"])
|
||||
ok = await gf.upload_group_files(files, 123, title="t", password="pw")
|
||||
@@ -271,9 +339,7 @@ async def test_raw_mode_prefixes_work_name_for_multi_files(gf, tmp_path, monkeyp
|
||||
monkeypatch.setattr(gf, "upload_group_file", fake_upload)
|
||||
|
||||
images = _make_files(tmp_path / "src", ["001.jpg", "002.jpg"])
|
||||
await gf.upload_group_files(
|
||||
images, 123, zip_files=False, rel_dir="作者_9/海边日落"
|
||||
)
|
||||
await gf.upload_group_files(images, 123, zip_files=False, rel_dir="作者_9/海边日落")
|
||||
assert seen == ["海边日落_001.jpg", "海边日落_002.jpg"]
|
||||
|
||||
seen.clear()
|
||||
|
||||
@@ -6,6 +6,7 @@ get_temp_root 的返回值,测试里 monkeypatch 成临时目录。
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import re
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
@@ -127,11 +128,31 @@ def test_work_stem_strips_hashtags_and_spaces(u):
|
||||
|
||||
|
||||
def test_work_stem_empty_falls_back(u):
|
||||
"""没给来源时保持旧契约(固定 `作品`)"""
|
||||
assert u.build_work_stem("") == "作品"
|
||||
assert u.build_work_stem(None) == "作品"
|
||||
assert u.build_work_stem("🎬") == "作品"
|
||||
|
||||
|
||||
def test_work_stem_empty_uses_source_code(u):
|
||||
"""给了来源 → 空标题兜底成 `作品_{来源短码}`:同作品稳定、不同作品不撞
|
||||
|
||||
固定 `作品` 会让同一作者的多个无名作品挤在一个名字下,靠 4 位**时间**短码
|
||||
区分,每次重解析名字都变(S3 key / 群文件成员名不可复现)。
|
||||
"""
|
||||
url = "https://www.douyin.com/video/7412345678901234567"
|
||||
stem = u.build_work_stem("#话题", source=url)
|
||||
assert stem == f"作品_{u.short_source_code(url)}"
|
||||
assert re.fullmatch(r"作品_[0-9a-z]{4}", stem)
|
||||
assert stem == u.build_work_stem("", source=url) # 同来源稳定
|
||||
assert stem != u.build_work_stem("", source=url + "8") # 不同来源不同码
|
||||
|
||||
|
||||
def test_work_stem_source_ignored_when_title_present(u):
|
||||
"""有标题时来源短码不参与(标题本身已有区分度,别把名字改花)"""
|
||||
assert u.build_work_stem("旅行 #随手拍", source="whatever") == "旅行"
|
||||
|
||||
|
||||
def test_work_stem_truncates(u):
|
||||
assert len(u.build_work_stem("标题" * 20)) <= 15
|
||||
|
||||
|
||||
@@ -40,6 +40,8 @@ def test_normalize_platform_aliases(policy):
|
||||
assert policy.normalize_platform("小红书") == "小红书"
|
||||
assert policy.normalize_platform("b23") == "哔哩哔哩"
|
||||
assert policy.normalize_platform("B站") == "哔哩哔哩"
|
||||
assert policy.normalize_platform("ks") == "快手"
|
||||
assert policy.normalize_platform("快手") == "快手"
|
||||
assert policy.normalize_platform("yt") == "Youtube"
|
||||
assert policy.normalize_platform("twitter") == "X"
|
||||
assert policy.normalize_platform(" X ") == "X"
|
||||
@@ -81,6 +83,10 @@ def test_match_platform_domains(policy):
|
||||
assert policy.match_platform("https://twitter.com/a/status/1") == "X"
|
||||
assert policy.match_platform("https://b23.tv/abc") == "哔哩哔哩"
|
||||
assert policy.match_platform("https://v.douyin.com/abc") == "抖音"
|
||||
assert policy.match_platform("https://v.kuaishou.com/nUqhAiDI") == "快手"
|
||||
assert policy.match_platform("https://www.kuaishou.com/short-video/abc") == "快手"
|
||||
# QQ 卡片 jumpUrl 走 v.m.chenzhongtech.com(快手分享页域)
|
||||
assert policy.match_platform("https://v.m.chenzhongtech.com/fw/photo/abc") == "快手"
|
||||
assert policy.match_platform("https://youtu.be/abc") == "Youtube"
|
||||
assert policy.match_platform("https://example.com/a.mp4") is None
|
||||
|
||||
|
||||
+326
-61
@@ -1,24 +1,27 @@
|
||||
"""媒体发送成功后「补传 S3 取下载链接」的单元测试
|
||||
"""sender.py 归档语义的单元测试(「媒体必须进本地 S3」是前提)
|
||||
|
||||
背景:2026-09-22 起 `hexi/core/outbound_media` 的全局钩子把本地路径改写成
|
||||
`/media/<token>`,本地直发那一级才真正可用。此前它因协议端不在本机、`file://`
|
||||
必然 ENOENT 而**恒失败**,所以每次都落到第 2 级 S3 —— `public_url` 因此总有值。
|
||||
本地直发通了以后,如果什么都不做,`public_url` 会恒为 None,`entry.py` 的
|
||||
`if pub and policy.sends_link` 就会让「发送下载链接」**静默失效**。
|
||||
用户 2026-09-25 定的前提:**无论发不发送消息**,下载成功的媒体都要进本地 S3
|
||||
(temp 只留 7 天,cleanup.py 删;桶里那份才是长期存档)。因此 sender 的结构是
|
||||
「归档与发送解耦且并行」:
|
||||
|
||||
`_local_sent` / `_public_link_for` 就是为此把「取链接」与「发送」解耦:
|
||||
只有策略真要链接(`sends_link`)时才补一次 S3,其余情况一概不碰 S3。
|
||||
- `archive_media` 无条件逐文件上传:单个文件失败只 warning 并继续,任何情况都不抛;
|
||||
- `send_pending_media` 一进来就 `create_task` 归档,与本地直发并行开跑,
|
||||
发送全失败也一定 await 它跑完;`public_url` 从归档结果取(最后一个非空);
|
||||
- 第 2 级(S3 链接发送)复用归档的预签名链接 —— 每个文件只上传一次,没有"补传"。
|
||||
|
||||
sender.py 有插件内相对导入(`..config` / `..policy` / `..services...`),
|
||||
无法裸加载,故在 sys.modules 里预置这些依赖的替身再按路径加载目标模块。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import sys
|
||||
import time
|
||||
import types
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from nonebot.adapters.onebot.v11.exception import ActionFailed, NetworkError
|
||||
|
||||
_REPO = Path(__file__).resolve().parents[1]
|
||||
_PKG = "hexi.plugins.nonebot_plugin_video_analysis"
|
||||
@@ -71,23 +74,31 @@ def sender():
|
||||
|
||||
|
||||
class _Policy:
|
||||
"""只带 sends_link 的策略替身(真实 Policy 需要一堆字段/归一化)。"""
|
||||
|
||||
def __init__(self, sends_link: bool):
|
||||
self.sends_link = sends_link
|
||||
"""群策略替身(归档只是把它透传给 upload_with_plan,不需要真实字段)。"""
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def uploads(monkeypatch, sender):
|
||||
"""记录 upload_with_plan 的调用;返回 (调用列表, 设置返回值的钩子)。"""
|
||||
calls: list[Path] = []
|
||||
result: dict = {"pub": "https://pub.example/x.mp4", "raise": None}
|
||||
"""记录 upload_with_plan 的调用;返回 (调用列表, 结果配置)。
|
||||
|
||||
result 可覆盖的字段:
|
||||
- `pub`:常量,或按调用顺序消费的列表(元素为 None = 没传公网);
|
||||
- `raises`:文件名 → 该文件上传时抛出的异常(其余文件照常上传)。
|
||||
"""
|
||||
calls: list[tuple[Path, object]] = []
|
||||
result: dict = {"pub": "https://pub.example/x.mp4", "raises": {}}
|
||||
|
||||
def fake_upload(file_path, *, policy=None):
|
||||
calls.append(Path(file_path))
|
||||
if result["raise"]:
|
||||
raise result["raise"]
|
||||
return "local-url", result["pub"]
|
||||
fp = Path(file_path)
|
||||
calls.append((fp, policy))
|
||||
pub = result["pub"]
|
||||
if isinstance(pub, list):
|
||||
# 列表按下标对应「第几个文件」(先取值再判失败:元素与文件一一对应)
|
||||
pub = pub.pop(0) if pub else None
|
||||
error = result["raises"].get(fp.name)
|
||||
if error:
|
||||
raise error
|
||||
return "local-url", pub
|
||||
|
||||
monkeypatch.setattr(sender, "upload_with_plan", fake_upload)
|
||||
return calls, result
|
||||
@@ -95,61 +106,315 @@ def uploads(monkeypatch, sender):
|
||||
|
||||
@pytest.fixture
|
||||
def media(sender):
|
||||
def build(sends_link: bool, n: int = 1, policy=None):
|
||||
def build(n: int = 1, policy=_Policy()):
|
||||
files = [Path(f"D:/temp/f{i}.mp4") for i in range(n)]
|
||||
return sender.PendingMedia(
|
||||
files=files,
|
||||
policy=_Policy(sends_link) if policy is None else policy,
|
||||
)
|
||||
return sender.PendingMedia(files=files, policy=policy)
|
||||
|
||||
return build
|
||||
|
||||
|
||||
async def test_no_s3_when_link_disabled(sender, media, uploads):
|
||||
"""默认路径:本地发成功就够了,绝不碰 S3(媒体不走 S3 归档是有意的)。"""
|
||||
calls, _ = uploads
|
||||
ok, pub = await sender._local_sent(media(sends_link=False))
|
||||
assert (ok, pub) == (True, None)
|
||||
assert calls == []
|
||||
# ───────────────────────── 归档本体(无条件) ─────────────────────────
|
||||
|
||||
|
||||
async def test_s3_fetched_when_link_enabled(sender, media, uploads):
|
||||
"""策略要下载链接时:补传 S3 并把 public_url 交出去(本次修的回归)。"""
|
||||
async def test_archive_uploads_every_file(sender, media, uploads):
|
||||
"""逐文件归档,拿到 (预签名链接, 公网链接)。"""
|
||||
calls, _ = uploads
|
||||
ok, pub = await sender._local_sent(media(sends_link=True))
|
||||
result = await sender.archive_media(media(n=3))
|
||||
assert len(calls) == 3
|
||||
assert [(r.path.name, r.local_url, r.public_url) for r in result] == [
|
||||
(f"f{i}.mp4", "local-url", "https://pub.example/x.mp4") for i in range(3)
|
||||
]
|
||||
|
||||
|
||||
async def test_archive_continues_after_single_failure(sender, media, uploads):
|
||||
"""单个文件失败只跳过它,后面的照常传(多图作品不早退)。"""
|
||||
calls, result = uploads
|
||||
result["raises"] = {"f1.mp4": RuntimeError("这个文件 S3 挂了")}
|
||||
|
||||
archived = await sender.archive_media(media(n=3))
|
||||
assert [r.local_url for r in archived] == ["local-url", None, "local-url"]
|
||||
assert len(calls) == 3 # 失败的那个也试过了,f2 没被跳过
|
||||
|
||||
|
||||
async def test_archive_never_raises(sender, media, uploads):
|
||||
"""全部上传失败也不抛(归档失败不能反过来影响发送)。"""
|
||||
calls, result = uploads
|
||||
error = RuntimeError("S3 挂了")
|
||||
result["raises"] = {"f0.mp4": error, "f1.mp4": error}
|
||||
archived = await sender.archive_media(media(n=2))
|
||||
assert [r.local_url for r in archived] == [None, None]
|
||||
assert len(calls) == 2
|
||||
|
||||
|
||||
async def test_archive_passes_policy_through(sender, media, uploads):
|
||||
"""policy=None(如 Web 面板路径)也照常归档:upload_with_plan 自己兜 Policy()。"""
|
||||
calls, _ = uploads
|
||||
await sender.archive_media(media(policy=None))
|
||||
assert [policy for _, policy in calls] == [None]
|
||||
|
||||
|
||||
async def test_last_public_url_takes_last_non_empty(sender, media, uploads):
|
||||
"""公网链接取最后一个非空值;只有本地链接的文件不影响结果。"""
|
||||
calls, result = uploads
|
||||
result["pub"] = [None, "https://pub.example/b.mp4", None]
|
||||
archived = await sender.archive_media(media(n=3))
|
||||
assert sender._last_public_url(archived) == "https://pub.example/b.mp4"
|
||||
|
||||
|
||||
async def test_last_public_url_none_when_upload_public_off(sender, media, uploads):
|
||||
"""S3 有本地链接但没公网(upload_public 关)→ None,且算不上失败。"""
|
||||
_, result = uploads
|
||||
result["pub"] = None
|
||||
archived = await sender.archive_media(media())
|
||||
assert archived[0].local_url == "local-url"
|
||||
assert sender._last_public_url(archived) is None
|
||||
|
||||
|
||||
# ─────────────────── 发送入口:归档无条件且与之并行 ───────────────────
|
||||
|
||||
|
||||
async def test_archives_even_when_send_fails(monkeypatch, sender, media, uploads):
|
||||
"""前提的核心:发送全失败也照样归档,public_url 照常交出去。"""
|
||||
calls, _ = uploads
|
||||
|
||||
async def fake_core(m, event, archive):
|
||||
return False # 三级全失败
|
||||
|
||||
monkeypatch.setattr(sender, "_send_media_core", fake_core)
|
||||
ok, pub = await sender.send_pending_media(media(n=2))
|
||||
assert (ok, pub) == (False, "https://pub.example/x.mp4")
|
||||
assert len(calls) == 2
|
||||
|
||||
|
||||
async def test_archives_when_send_succeeds(monkeypatch, sender, media, uploads):
|
||||
"""发送成功也归档(归档不是发送的副产品)。"""
|
||||
calls, _ = uploads
|
||||
|
||||
async def fake_core(m, event, archive):
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(sender, "_send_media_core", fake_core)
|
||||
ok, pub = await sender.send_pending_media(media())
|
||||
assert (ok, pub) == (True, "https://pub.example/x.mp4")
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
async def test_multi_file_uploads_each(sender, media, uploads):
|
||||
calls, _ = uploads
|
||||
ok, pub = await sender._local_sent(media(sends_link=True, n=3))
|
||||
async def test_archive_runs_in_parallel_with_send(monkeypatch, sender, media, uploads):
|
||||
"""归档与发送并行开跑,不是"发送跑完才传"(本地直发不等它)。"""
|
||||
started: list[str] = []
|
||||
|
||||
async def fake_archive(m):
|
||||
started.append("archive")
|
||||
return [sender.UploadedFile(path=fp) for fp in m.files]
|
||||
|
||||
async def fake_core(m, event, archive):
|
||||
await asyncio.sleep(0) # 让出一个事件循环回合
|
||||
assert started, "归档应当已在跑,而不是等发送结束才开始"
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(sender, "archive_media", fake_archive)
|
||||
monkeypatch.setattr(sender, "_send_media_core", fake_core)
|
||||
ok, _ = await sender.send_pending_media(media())
|
||||
assert ok is True
|
||||
assert pub == "https://pub.example/x.mp4"
|
||||
assert len(calls) == 3
|
||||
|
||||
|
||||
async def test_no_public_url_keeps_none(sender, media, uploads):
|
||||
"""S3 有本地链接但没公网(upload_public 关)→ 不带链接,但也不算失败。"""
|
||||
calls, result = uploads
|
||||
result["pub"] = None
|
||||
ok, pub = await sender._local_sent(media(sends_link=True))
|
||||
assert (ok, pub) == (True, None)
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
async def test_upload_failure_never_breaks_send(sender, media, uploads):
|
||||
"""补传失败只记日志:发送本身已经成功了,不能反过来判失败。"""
|
||||
calls, result = uploads
|
||||
result["raise"] = RuntimeError("S3 挂了")
|
||||
ok, pub = await sender._local_sent(media(sends_link=True))
|
||||
assert (ok, pub) == (True, None)
|
||||
assert len(calls) == 1
|
||||
|
||||
|
||||
async def test_policy_none_is_safe(sender, media, uploads):
|
||||
"""无策略(如 Web 面板路径)时不补传也不炸。"""
|
||||
async def test_send_failure_still_waits_for_archive(
|
||||
monkeypatch, sender, media, uploads
|
||||
):
|
||||
"""发送层抛异常也不能把归档半途丢下(finally 里 await 它跑完)。"""
|
||||
calls, _ = uploads
|
||||
ok, pub = await sender._local_sent(media(sends_link=False, policy=None))
|
||||
assert (ok, pub) == (True, None)
|
||||
assert calls == []
|
||||
finished: list[str] = []
|
||||
|
||||
async def fake_core(m, event, archive):
|
||||
raise RuntimeError("发送层炸了")
|
||||
|
||||
def slow_upload(file_path, *, policy=None): # to_thread 里跑的是同步函数
|
||||
calls.append((Path(file_path), policy))
|
||||
time.sleep(0.01)
|
||||
finished.append(Path(file_path).name)
|
||||
return "local-url", "https://pub.example/x.mp4"
|
||||
|
||||
monkeypatch.setattr(sender, "upload_with_plan", slow_upload)
|
||||
monkeypatch.setattr(sender, "_send_media_core", fake_core)
|
||||
|
||||
with pytest.raises(RuntimeError):
|
||||
await sender.send_pending_media(media(n=2))
|
||||
assert finished == ["f0.mp4", "f1.mp4"] # 归档完整跑完才抛出去
|
||||
|
||||
|
||||
# ─────────────────── 第 2 级复用归档结果(不重复上传) ───────────────────
|
||||
|
||||
|
||||
async def test_s3_level_reuses_archived_urls(monkeypatch, sender, media, uploads):
|
||||
"""本地直发失败 → 用归档的预签名链接发送,每个文件只上传一次。"""
|
||||
calls, _ = uploads
|
||||
built: list = []
|
||||
|
||||
def fake_local_msg(files, image_post):
|
||||
class _Msg:
|
||||
async def send(self, **_kw): # 本地直发失败
|
||||
raise RuntimeError("协议端拿不到文件")
|
||||
|
||||
return _Msg()
|
||||
|
||||
def fake_s3_msg(items):
|
||||
built.append(items)
|
||||
|
||||
class _Msg:
|
||||
async def send(self, **_kw): # 链接版发送成功
|
||||
pass
|
||||
|
||||
return _Msg()
|
||||
|
||||
monkeypatch.setattr(sender, "_build_local_msg", fake_local_msg)
|
||||
monkeypatch.setattr(sender, "_build_s3_msg", fake_s3_msg)
|
||||
|
||||
ok, pub = await sender.send_pending_media(media(n=2))
|
||||
assert (ok, pub) == (True, "https://pub.example/x.mp4")
|
||||
assert len(calls) == 2 # 只有归档那次上传,第 2 级没有重复传
|
||||
assert built == [
|
||||
[(Path("D:/temp/f0.mp4"), "local-url"), (Path("D:/temp/f1.mp4"), "local-url")]
|
||||
]
|
||||
|
||||
|
||||
async def test_no_files_is_noop(sender):
|
||||
"""没有媒体文件时既不归档也不发送。"""
|
||||
ok, pub = await sender.send_pending_media(sender.PendingMedia(files=[]))
|
||||
assert (ok, pub) == (False, None)
|
||||
|
||||
|
||||
# ─────────── 超时=结果未知:不降级重试(长视频防重复发送) ───────────
|
||||
# 2026-10-13 实测:172MB 视频三级降级各超时一次(NoneBot 默认 api_timeout=30s,
|
||||
# 而协议端下载+转存要 ~2 分钟),但协议端在 bot 超时后照样把三份全发进群。
|
||||
# 超时只是 bot 放弃等待 ⇒ 结果未知 ⇒ 重试必重复。
|
||||
|
||||
|
||||
def test_send_timeout_default_and_env(sender, monkeypatch):
|
||||
"""默认 300s;`video_analysis_send_timeout` 可调,非法/非正值回退默认。"""
|
||||
monkeypatch.delenv("video_analysis_send_timeout", raising=False)
|
||||
monkeypatch.delenv("VIDEO_ANALYSIS_SEND_TIMEOUT", raising=False)
|
||||
assert sender.send_timeout() == 300.0
|
||||
|
||||
monkeypatch.setenv("video_analysis_send_timeout", "120")
|
||||
assert sender.send_timeout() == 120.0
|
||||
|
||||
monkeypatch.setenv("video_analysis_send_timeout", "abc")
|
||||
assert sender.send_timeout() == 300.0
|
||||
|
||||
monkeypatch.setenv("video_analysis_send_timeout", "0")
|
||||
assert sender.send_timeout() == 300.0
|
||||
|
||||
|
||||
def test_unknown_outcome_classification(sender):
|
||||
"""超时/网络错误=结果未知;ActionFailed(协议端明确报错)=确定失败。"""
|
||||
assert sender.unknown_send_outcome(NetworkError("timeout")) is True
|
||||
assert sender.unknown_send_outcome(asyncio.TimeoutError()) is True
|
||||
assert sender.unknown_send_outcome(ActionFailed(error="fetch failed")) is False
|
||||
assert sender.unknown_send_outcome(RuntimeError("本地错误")) is False
|
||||
|
||||
|
||||
async def test_media_send_carries_per_call_timeout(monkeypatch, sender, media, uploads):
|
||||
"""媒体发送带 `_timeout`(适配器 per-call 超时),长视频才等得到真结果。"""
|
||||
monkeypatch.setenv("video_analysis_send_timeout", "42")
|
||||
seen: list[dict] = []
|
||||
|
||||
def fake_local_msg(files, image_post):
|
||||
class _Msg:
|
||||
async def send(self, **kwargs):
|
||||
seen.append(kwargs)
|
||||
|
||||
return _Msg()
|
||||
|
||||
monkeypatch.setattr(sender, "_build_local_msg", fake_local_msg)
|
||||
ok, _ = await sender.send_pending_media(media())
|
||||
assert ok is True
|
||||
assert seen == [{"_timeout": 42.0}]
|
||||
|
||||
|
||||
async def test_local_timeout_does_not_resend(monkeypatch, sender, media, uploads):
|
||||
"""本地直发超时(结果未知)→ 不走 S3 降级、按「可能已送达」返回。"""
|
||||
s3_built: list = []
|
||||
|
||||
def fake_local_msg(files, image_post):
|
||||
class _Msg:
|
||||
async def send(self, **_kw):
|
||||
raise NetworkError("WebSocket call api send_msg timeout")
|
||||
|
||||
return _Msg()
|
||||
|
||||
def fake_s3_msg(items):
|
||||
s3_built.append(items)
|
||||
|
||||
class _Msg:
|
||||
async def send(self, **_kw):
|
||||
pass
|
||||
|
||||
return _Msg()
|
||||
|
||||
monkeypatch.setattr(sender, "_build_local_msg", fake_local_msg)
|
||||
monkeypatch.setattr(sender, "_build_s3_msg", fake_s3_msg)
|
||||
|
||||
ok, pub = await sender.send_pending_media(media())
|
||||
assert ok is True # 结果未知按可能已送达:不触发「媒体发送失败」提示
|
||||
assert pub == "https://pub.example/x.mp4" # 归档照常完成
|
||||
assert s3_built == [] # 关键:没有第二次发送
|
||||
|
||||
|
||||
async def test_action_failed_still_cascades_to_s3(monkeypatch, sender, media, uploads):
|
||||
"""协议端明确报错(如 fetch failed)= 确定没发出去,照旧降级到 S3 链接。"""
|
||||
sent: list = []
|
||||
|
||||
def fake_local_msg(files, image_post):
|
||||
class _Msg:
|
||||
async def send(self, **_kw):
|
||||
raise ActionFailed(error="fetch failed")
|
||||
|
||||
return _Msg()
|
||||
|
||||
def fake_s3_msg(items):
|
||||
class _Msg:
|
||||
async def send(self, **_kw): # 链接版成功
|
||||
sent.append(items)
|
||||
|
||||
return _Msg()
|
||||
|
||||
monkeypatch.setattr(sender, "_build_local_msg", fake_local_msg)
|
||||
monkeypatch.setattr(sender, "_build_s3_msg", fake_s3_msg)
|
||||
|
||||
ok, _ = await sender.send_pending_media(media())
|
||||
assert ok is True
|
||||
assert len(sent) == 1 # 恰好降级一次
|
||||
|
||||
|
||||
async def test_forward_timeout_does_not_fall_through(
|
||||
monkeypatch, sender, media, uploads
|
||||
):
|
||||
"""合并转发超时(结果未知)→ 不再落回普通消息链路重发。"""
|
||||
two = media(n=2)
|
||||
fallback_built: list = []
|
||||
|
||||
async def fake_forward(items, sender_name, *, group_id=None, user_id=None):
|
||||
return None # 结果未知
|
||||
|
||||
def fake_local_msg(files, image_post):
|
||||
fallback_built.append(files)
|
||||
|
||||
class _Msg:
|
||||
async def send(self, **_kw):
|
||||
raise AssertionError("不该走到普通消息链路")
|
||||
|
||||
return _Msg()
|
||||
|
||||
monkeypatch.setattr(sender, "_send_forward_media", fake_forward)
|
||||
monkeypatch.setattr(sender, "_forward_sender_name", lambda: "Bot")
|
||||
monkeypatch.setattr(sender, "_build_local_msg", fake_local_msg)
|
||||
|
||||
archive = asyncio.create_task(sender.archive_media(two))
|
||||
try:
|
||||
ok = await sender._send_media_core(
|
||||
two, types.SimpleNamespace(group_id=123), archive
|
||||
)
|
||||
finally:
|
||||
await archive
|
||||
assert ok is True
|
||||
assert fallback_built == []
|
||||
Reference in new issue
Block a user