feat(video-analysis): 群策略 v3 / 群文件投递通道 / Web 管理页
- policy.py:per-group 正交策略(自动解析 / 自动策略 / 禁用策略 / 存储 A·B·C /
公网 / 链接 / 群文件 + 平台限定),list.json v1/v2 → v3 自动迁移,
写入统一走 PolicyStore(加锁 + .tmp 原子替换 + 字段归一)
- 群文件并行通道 group_file.py:打包 zip(可选 pyzipper AES-256)后优先走 S3 预签名、
本地直传兜底;设了密码但 pyzipper 不可用就放弃上传,不退化成明文
- list_proc.py 收敛到「视频策略」统一入口,权限判定改走 policy
- Web 管理页 /hub/video_analysis(群策略 + 链接解析面板)与 services/web_jobs.py
(只复用纯函数层,Web 上下文不发消息;内存任务表 + 并发闸门 + 超时)
- 媒体命名统一到 utils.py({作者}_{作者id}/{作品名}[_短码]),cleanup 回收空目录
- 测试:policy / 命名 / 群文件 / web_jobs 四组
顺带 pyproject 的 pytest 加 testpaths=tests(避免收进 debug/ 下的调试脚本)。
Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,303 @@
|
||||
"""群文件打包/投递 (group_file.py) 单元测试
|
||||
|
||||
group_file.py 只依赖标准库 + nonebot.logger,故用 importlib 按文件路径裸加载
|
||||
(与 test_video_policy.py 同法)。build_archive 一律显式传 out_dir —— 默认目录
|
||||
要经相对导入取 utils.get_temp_root(),裸加载模块没有包上下文。
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
_MODULE_PATH = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "hexi"
|
||||
/ "plugins"
|
||||
/ "nonebot_plugin_video_analysis"
|
||||
/ "services"
|
||||
/ "storage"
|
||||
/ "group_file.py"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def gf():
|
||||
"""以独立模块名加载 group_file.py,避免与插件包 __init__ 冲突"""
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"video_group_file_under_test", _MODULE_PATH
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
yield module
|
||||
sys.modules.pop(spec.name, None)
|
||||
|
||||
|
||||
def _make_files(directory: Path, names: list[str]) -> list[Path]:
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
made = []
|
||||
for name in names:
|
||||
path = directory / name
|
||||
path.write_bytes(f"hello {name}".encode())
|
||||
made.append(path)
|
||||
return made
|
||||
|
||||
|
||||
def test_plain_archive(gf, tmp_path):
|
||||
files = _make_files(tmp_path / "src", ["a.txt", "b.txt"])
|
||||
archive = gf.build_archive(files, "我的作品", out_dir=tmp_path / "out")
|
||||
|
||||
assert archive.exists() and archive.suffix == ".zip"
|
||||
assert "我的作品" in archive.name
|
||||
with zipfile.ZipFile(archive) as zf:
|
||||
assert sorted(zf.namelist()) == ["a.txt", "b.txt"]
|
||||
assert zf.read("a.txt") == b"hello a.txt"
|
||||
|
||||
|
||||
def test_encrypted_archive_requires_password(gf, tmp_path):
|
||||
pyzipper = pytest.importorskip("pyzipper")
|
||||
files = _make_files(tmp_path / "src", ["v.mp4"])
|
||||
archive = gf.build_archive(
|
||||
files, "加密作品", password="pw123", out_dir=tmp_path / "out"
|
||||
)
|
||||
|
||||
# 标准库打不开加密包
|
||||
with zipfile.ZipFile(archive) as zf:
|
||||
with pytest.raises(RuntimeError):
|
||||
zf.read("v.mp4")
|
||||
|
||||
# 正确密码可读,错误密码不行
|
||||
with pyzipper.AESZipFile(archive) as zf:
|
||||
zf.setpassword(b"pw123")
|
||||
assert zf.read("v.mp4") == b"hello v.mp4"
|
||||
with pyzipper.AESZipFile(archive) as zf:
|
||||
zf.setpassword(b"wrong")
|
||||
with pytest.raises(RuntimeError):
|
||||
zf.read("v.mp4")
|
||||
|
||||
|
||||
def test_duplicate_names_indexed(gf, tmp_path):
|
||||
first = _make_files(tmp_path / "p1", ["same.txt"])[0]
|
||||
second = _make_files(tmp_path / "p2", ["same.txt"])[0]
|
||||
archive = gf.build_archive([first, second], "t", out_dir=tmp_path / "out")
|
||||
|
||||
with zipfile.ZipFile(archive) as zf:
|
||||
assert sorted(zf.namelist()) == ["same.txt", "same_2.txt"]
|
||||
assert zf.read("same_2.txt") == b"hello same.txt"
|
||||
|
||||
|
||||
def test_output_path_is_unique(gf, tmp_path):
|
||||
files = _make_files(tmp_path / "src", ["a.txt"])
|
||||
out = tmp_path / "out"
|
||||
one = gf.build_archive(files, "t", out_dir=out)
|
||||
two = gf.build_archive(files, "t", out_dir=out)
|
||||
assert one != two and one.exists() and two.exists()
|
||||
|
||||
|
||||
def test_placeholder_title_falls_back(gf, tmp_path):
|
||||
"""universal.py 传的 title 是占位符 "title",不该出现在文件名里"""
|
||||
files = _make_files(tmp_path / "src", ["a.txt"])
|
||||
archive = gf.build_archive(files, "title", out_dir=tmp_path / "out")
|
||||
assert archive.name.startswith("群文件_")
|
||||
assert "群文件_群文件" not in archive.name
|
||||
|
||||
|
||||
def test_title_with_path_separators_is_sanitized(gf, tmp_path):
|
||||
"""原始标题(抖音文案/YouTube 标题)可能带 /、换行、#话题,不能进文件名"""
|
||||
files = _make_files(tmp_path / "src", ["a.txt"])
|
||||
archive = gf.build_archive(files, "和/或 #话题\n测试", out_dir=tmp_path / "out")
|
||||
|
||||
assert "/" not in archive.name and "\\" not in archive.name
|
||||
assert "\n" not in archive.name and "#" not in archive.name
|
||||
assert archive.name.startswith("和_或_测试")
|
||||
|
||||
|
||||
def test_title_starting_with_archive_prefix_falls_back(gf, tmp_path):
|
||||
"""标题本身以「群文件」开头时不要产出 群文件_xxx_群文件_yyy.zip"""
|
||||
files = _make_files(tmp_path / "src", ["a.txt"])
|
||||
archive = gf.build_archive(files, "群文件_120606", out_dir=tmp_path / "out")
|
||||
assert archive.name.startswith("群文件_")
|
||||
assert "群文件_120606_群文件" not in archive.name
|
||||
|
||||
|
||||
def test_empty_or_missing_input(gf, tmp_path):
|
||||
with pytest.raises(ValueError):
|
||||
gf.build_archive([], "t", out_dir=tmp_path / "out")
|
||||
with pytest.raises(FileNotFoundError):
|
||||
gf.build_archive([tmp_path / "nope.mp4"], "t", out_dir=tmp_path / "out")
|
||||
|
||||
|
||||
def test_password_without_pyzipper_fails_loud(gf, tmp_path, monkeypatch):
|
||||
"""设了密码但加密库不可用 → 报错(调用方会放弃上传,绝不能退化传明文)"""
|
||||
monkeypatch.setattr(gf, "pyzipper", None)
|
||||
assert gf.encryption_available() is False
|
||||
|
||||
files = _make_files(tmp_path / "src", ["a.txt"])
|
||||
with pytest.raises(RuntimeError):
|
||||
gf.build_archive(files, "t", password="pw", out_dir=tmp_path / "out")
|
||||
# 不加密那一路不受影响
|
||||
assert gf.build_archive(files, "t", out_dir=tmp_path / "out").exists()
|
||||
|
||||
|
||||
# ───────────────────── 上传:URI 直传 + S3 降级 ─────────────────────
|
||||
|
||||
|
||||
def test_file_uri_keeps_unicode_and_uses_slashes(gf, tmp_path):
|
||||
target = tmp_path / "dir with space" / "群文件_120606.zip"
|
||||
target.parent.mkdir(parents=True)
|
||||
target.write_bytes(b"x")
|
||||
|
||||
uri = gf._file_uri(target)
|
||||
assert uri.startswith("file:///")
|
||||
assert "\\" not in uri
|
||||
# 中文/空格保持原样(不做百分号转义,NapCat 侧直接当路径解析)
|
||||
assert uri.endswith("/dir with space/群文件_120606.zip")
|
||||
|
||||
|
||||
async def test_s3_link_is_the_primary_channel(gf, tmp_path, monkeypatch):
|
||||
"""默认通道就是 S3 链接(本环境本地直传必挂,不先浪费一次调用)"""
|
||||
calls: list = []
|
||||
link = "http://192.168.2.15:5246/PLANA/x.zip?sig=1"
|
||||
|
||||
async def fake_call(file_value, group_id, name, folder_id=None):
|
||||
calls.append((file_value, group_id, name))
|
||||
|
||||
monkeypatch.setattr(gf, "_call_upload", fake_call)
|
||||
monkeypatch.setattr(gf, "_s3_url", lambda path, policy: link)
|
||||
|
||||
file = _make_files(tmp_path / "src", ["a.zip"])[0]
|
||||
assert await gf.upload_group_file(file, 123) is True
|
||||
assert calls == [(link, 123, "a.zip")]
|
||||
|
||||
|
||||
async def test_s3_call_failure_falls_back_to_local(gf, tmp_path, monkeypatch):
|
||||
seen: list[str] = []
|
||||
link = "http://192.168.2.15:5246/PLANA/x.zip?sig=1"
|
||||
|
||||
async def fake_call(file_value, group_id, name, folder_id=None):
|
||||
seen.append(file_value)
|
||||
if file_value.startswith("http"):
|
||||
raise RuntimeError("链接不可达")
|
||||
|
||||
monkeypatch.setattr(gf, "_call_upload", fake_call)
|
||||
monkeypatch.setattr(gf, "_s3_url", lambda path, policy: link)
|
||||
|
||||
file = _make_files(tmp_path / "src", ["a.zip"])[0]
|
||||
assert await gf.upload_group_file(file, 123) is True
|
||||
assert seen[0] == link
|
||||
assert seen[1] == gf._file_uri(file)
|
||||
|
||||
|
||||
async def test_no_s3_link_falls_back_to_local(gf, tmp_path, monkeypatch):
|
||||
seen: list[str] = []
|
||||
|
||||
async def fake_call(file_value, group_id, name, folder_id=None):
|
||||
seen.append(file_value)
|
||||
|
||||
monkeypatch.setattr(gf, "_call_upload", fake_call)
|
||||
monkeypatch.setattr(gf, "_s3_url", lambda path, policy: "")
|
||||
|
||||
file = _make_files(tmp_path / "src", ["a.zip"])[0]
|
||||
assert await gf.upload_group_file(file, 123) is True
|
||||
assert seen == [gf._file_uri(file)]
|
||||
|
||||
|
||||
async def test_both_transports_fail(gf, tmp_path, monkeypatch):
|
||||
async def always_fail(*args, **kwargs):
|
||||
raise RuntimeError("nope")
|
||||
|
||||
monkeypatch.setattr(gf, "_call_upload", always_fail)
|
||||
monkeypatch.setattr(gf, "_s3_url", lambda path, policy: "http://x/y.zip")
|
||||
|
||||
file = _make_files(tmp_path / "src", ["a.zip"])[0]
|
||||
assert await gf.upload_group_file(file, 123) is False
|
||||
|
||||
|
||||
async def test_missing_file_short_circuits(gf, tmp_path, monkeypatch):
|
||||
called = []
|
||||
monkeypatch.setattr(gf, "_call_upload", lambda *a, **kw: called.append(1))
|
||||
assert await gf.upload_group_file(tmp_path / "nope.zip", 123) is False
|
||||
assert called == []
|
||||
|
||||
|
||||
# ───────────────────────── 投递编排 ─────────────────────────
|
||||
|
||||
|
||||
def _record_uploads(gf, monkeypatch, path: Path, ok: bool = True) -> list:
|
||||
calls: list = []
|
||||
|
||||
async def fake_upload(file_path, group_id, **kwargs):
|
||||
calls.append((str(file_path), group_id))
|
||||
return ok
|
||||
|
||||
monkeypatch.setattr(gf, "upload_group_file", fake_upload)
|
||||
return calls
|
||||
|
||||
|
||||
async def test_zip_mode_uploads_single_archive(gf, tmp_path, monkeypatch):
|
||||
archive = tmp_path / "pack.zip"
|
||||
archive.write_bytes(b"zip")
|
||||
calls = _record_uploads(gf, monkeypatch, archive)
|
||||
monkeypatch.setattr(
|
||||
gf, "build_archive", lambda *a, **kw: archive
|
||||
)
|
||||
|
||||
files = _make_files(tmp_path / "src", ["a.txt", "b.txt"])
|
||||
ok = await gf.upload_group_files(files, 123, title="t", password="pw")
|
||||
assert ok is True
|
||||
assert calls == [(str(archive), 123)]
|
||||
|
||||
|
||||
async def test_raw_mode_uploads_each_file(gf, tmp_path, monkeypatch):
|
||||
calls = _record_uploads(gf, monkeypatch, tmp_path)
|
||||
files = _make_files(tmp_path / "src", ["a.txt", "b.txt"])
|
||||
|
||||
ok = await gf.upload_group_files(files, 123, zip_files=False)
|
||||
assert ok is True
|
||||
assert [c[0] for c in calls] == [str(p) for p in files]
|
||||
|
||||
|
||||
async def test_raw_mode_prefixes_work_name_for_multi_files(gf, tmp_path, monkeypatch):
|
||||
"""多图作品逐个传时,001.jpg 这类成员名要带上作品名(单文件不加)"""
|
||||
seen: list = []
|
||||
|
||||
async def fake_upload(file_path, group_id, **kwargs):
|
||||
seen.append(kwargs.get("name"))
|
||||
return True
|
||||
|
||||
monkeypatch.setattr(gf, "upload_group_file", fake_upload)
|
||||
|
||||
images = _make_files(tmp_path / "src", ["001.jpg", "002.jpg"])
|
||||
await gf.upload_group_files(
|
||||
images, 123, zip_files=False, rel_dir="作者_9/海边日落"
|
||||
)
|
||||
assert seen == ["海边日落_001.jpg", "海边日落_002.jpg"]
|
||||
|
||||
seen.clear()
|
||||
single = _make_files(tmp_path / "src2", ["作品.mp4"])
|
||||
await gf.upload_group_files(single, 123, zip_files=False, rel_dir="作者_9")
|
||||
assert seen == ["作品.mp4"] # 单文件时文件名本身就是作品名,不再加前缀
|
||||
|
||||
|
||||
async def test_pack_failure_skips_upload_entirely(gf, tmp_path, monkeypatch):
|
||||
"""打包失败(例如缺 pyzipper)时不能退化成上传原文件"""
|
||||
calls = _record_uploads(gf, monkeypatch, tmp_path)
|
||||
|
||||
def boom(*args, **kwargs):
|
||||
raise RuntimeError("pyzipper 未安装")
|
||||
|
||||
monkeypatch.setattr(gf, "build_archive", boom)
|
||||
files = _make_files(tmp_path / "src", ["a.txt"])
|
||||
|
||||
ok = await gf.upload_group_files(files, 123, title="t", password="pw")
|
||||
assert ok is False
|
||||
assert calls == []
|
||||
|
||||
|
||||
async def test_empty_file_list(gf, tmp_path, monkeypatch):
|
||||
calls = _record_uploads(gf, monkeypatch, tmp_path)
|
||||
assert await gf.upload_group_files([], 123) is False
|
||||
assert calls == []
|
||||
@@ -0,0 +1,214 @@
|
||||
"""媒体命名 / S3 key (utils.py 纯函数) 单元测试
|
||||
|
||||
utils.py 只依赖标准库,故用 importlib 按文件路径裸加载(与
|
||||
test_video_group_file.py 同法)。media_key_of / media_rel_dir_of 依赖
|
||||
get_temp_root 的返回值,测试里 monkeypatch 成临时目录。
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
_MODULE_PATH = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "hexi"
|
||||
/ "plugins"
|
||||
/ "nonebot_plugin_video_analysis"
|
||||
/ "utils.py"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def u():
|
||||
"""以独立模块名加载 utils.py,避免与插件包 __init__ 冲突"""
|
||||
spec = importlib.util.spec_from_file_location("video_utils_under_test", _MODULE_PATH)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
yield module
|
||||
sys.modules.pop(spec.name, None)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def temp_root(u, tmp_path, monkeypatch):
|
||||
"""把 get_temp_root 指到 tmp_path(media_key_of 只做路径解析,不需 mkdir)"""
|
||||
root = tmp_path / "temp"
|
||||
root.mkdir()
|
||||
monkeypatch.setattr(u, "get_temp_root", lambda sub="": root / sub if sub else root)
|
||||
return root
|
||||
|
||||
|
||||
# ───────────────────────── 作者目录 ─────────────────────────
|
||||
|
||||
|
||||
def test_author_dir_with_id(u):
|
||||
"""有作者 id 就是稳定的 `{昵称}_{id}`,不带码"""
|
||||
assert u.build_author_dir("示例作者", "12345") == "示例作者_12345"
|
||||
assert u.build_author_dir("示例作者", "12345", source="x", at=1) == (
|
||||
"示例作者_12345"
|
||||
)
|
||||
|
||||
|
||||
def test_author_dir_nickname_only_gets_time_code(u):
|
||||
"""只有昵称 → 追加 4 位时间短码:分不清同名作者,宁可不聚合也不混目录"""
|
||||
at = 1_700_000_000
|
||||
code = u.short_time_code(at)
|
||||
assert u.build_author_dir("示例作者", "", at=at) == f"示例作者_{code}"
|
||||
assert u.build_author_dir("示例作者", at=at) == f"示例作者_{code}"
|
||||
for junk in ("0", "NA", "none", "None", " null "):
|
||||
assert u.build_author_dir("示例作者", junk, at=at) == f"示例作者_{code}"
|
||||
|
||||
# 不同时刻 = 不同目录(同一作者的不同作品不会互相覆盖)
|
||||
other = u.build_author_dir("示例作者", at=at + 3600)
|
||||
assert other != f"示例作者_{code}"
|
||||
assert other.startswith("示例作者_") and len(other.split("_")[-1]) == 4
|
||||
|
||||
|
||||
def test_author_dir_uses_source_code_when_anonymous(u):
|
||||
"""连昵称都没有 → 未知作者 + 来源短码(同一来源稳定,不同来源不撞)"""
|
||||
url = "https://www.douyin.com/video/7412345678901234567"
|
||||
first = u.build_author_dir(None, None, source=url)
|
||||
assert first.startswith("未知作者_")
|
||||
assert first == u.build_author_dir("", "", source=url) # 同来源稳定
|
||||
|
||||
second = u.build_author_dir(None, None, source=url + "8")
|
||||
assert second != first
|
||||
|
||||
# 没有来源时才退化成时间短码
|
||||
at = 1_700_000_000
|
||||
assert u.build_author_dir(None, None, at=at) == (
|
||||
f"未知作者_{u.short_time_code(at)}"
|
||||
)
|
||||
|
||||
|
||||
def test_author_dir_treats_placeholder_nicknames_as_anonymous(u):
|
||||
"""抓取层拿不到昵称时的兜底串不算作者名,走来源码"""
|
||||
url = "https://www.bilibili.com/opus/123"
|
||||
expected = u.build_author_dir(None, None, source=url)
|
||||
for placeholder in ("未知作者", "小红书用户", "B站用户", "b站用户"):
|
||||
assert u.build_author_dir(placeholder, "", source=url) == expected
|
||||
# 纯 emoji 昵称 slugify 后为空 → 同样按匿名处理
|
||||
assert u.build_author_dir("🎬🎬", "", source=url) == expected
|
||||
|
||||
|
||||
def test_short_source_code_is_stable_base36(u):
|
||||
url = "https://www.douyin.com/video/7412345678901234567"
|
||||
code = u.short_source_code(url)
|
||||
assert len(code) == 4 and code.isalnum()
|
||||
assert code == u.short_source_code(url) # 可复现(不能用带盐的 hash())
|
||||
assert code != u.short_source_code(url + "8")
|
||||
|
||||
|
||||
def test_author_dir_sanitizes_illegal_chars(u):
|
||||
"""Windows 非法字符不能进目录名(slugify 直接删掉,空白转 -)"""
|
||||
assert u.build_author_dir("a/b:c*d?", "12 3") == "abcd_12-3"
|
||||
assert "/" not in u.build_author_dir("a/b", "1/2")
|
||||
|
||||
|
||||
def test_author_dir_dedupes_same_handle(u):
|
||||
"""X 这类站点上传者名就是 handle,别产出 someone_someone"""
|
||||
assert u.build_author_dir("@Someone", "@Someone") == "someone"
|
||||
|
||||
|
||||
def test_author_dir_keeps_long_sec_uid_distinct(u):
|
||||
"""sec_uid 公共前缀就有 17 字符,截断到 20 会让不同作者撞同一目录"""
|
||||
first = "MS4wLjABAAAA" + "x" * 45
|
||||
second = "MS4wLjABAAAA" + "y" * 45
|
||||
assert u.build_author_dir("n", first) != u.build_author_dir("n", second)
|
||||
|
||||
|
||||
# ───────────────────────── 作品名 ─────────────────────────
|
||||
|
||||
|
||||
def test_work_stem_strips_hashtags_and_spaces(u):
|
||||
assert u.build_work_stem("旅行 #随手拍 vlog") == "旅行-vlog"
|
||||
|
||||
|
||||
def test_work_stem_empty_falls_back(u):
|
||||
assert u.build_work_stem("") == "作品"
|
||||
assert u.build_work_stem(None) == "作品"
|
||||
assert u.build_work_stem("🎬") == "作品"
|
||||
|
||||
|
||||
def test_work_stem_truncates(u):
|
||||
assert len(u.build_work_stem("标题" * 20)) <= 15
|
||||
|
||||
|
||||
# ───────────────────────── 短码与重名 ─────────────────────────
|
||||
|
||||
|
||||
def test_short_time_code_is_four_char_base36(u):
|
||||
assert u.short_time_code(0) == "0000"
|
||||
code = u.short_time_code(1_700_000_000)
|
||||
assert len(code) == 4 and code.isalnum()
|
||||
# 相邻秒不同码
|
||||
assert code != u.short_time_code(1_700_000_001)
|
||||
|
||||
|
||||
def test_unique_media_path_without_collision(u, tmp_path):
|
||||
target = tmp_path / "作品.mp4"
|
||||
assert u.unique_media_path(target) == target
|
||||
|
||||
|
||||
def test_unique_media_path_adds_code_then_index(u, tmp_path):
|
||||
target = tmp_path / "作品.mp4"
|
||||
target.write_bytes(b"x")
|
||||
code = u.short_time_code(1_700_000_000)
|
||||
|
||||
second = u.unique_media_path(target, at=1_700_000_000)
|
||||
assert second.name == f"作品_{code}.mp4"
|
||||
|
||||
second.write_bytes(b"x")
|
||||
third = u.unique_media_path(target, at=1_700_000_000)
|
||||
assert third.name == f"作品_{code}_2.mp4"
|
||||
|
||||
|
||||
def test_unique_media_path_creates_author_dir(u, tmp_path):
|
||||
"""落盘前才建作者目录:命名函数顺带确保父目录存在"""
|
||||
target = tmp_path / "作者_123" / "作品.mp4"
|
||||
assert u.unique_media_path(target) == target
|
||||
assert target.parent.is_dir()
|
||||
|
||||
|
||||
def test_unique_media_path_works_for_directories(u, tmp_path):
|
||||
"""多图作品目录同名时同样加短码"""
|
||||
note_dir = tmp_path / "作者_123" / "作品"
|
||||
note_dir.mkdir(parents=True)
|
||||
renamed = u.unique_media_path(note_dir, at=1_700_000_000)
|
||||
assert renamed.name == f"作品_{u.short_time_code(1_700_000_000)}"
|
||||
|
||||
|
||||
# ───────────────────────── S3 key ─────────────────────────
|
||||
|
||||
|
||||
def test_media_key_keeps_author_dir_and_drops_platform(u, temp_root):
|
||||
f = temp_root / "douyin" / "作者_123" / "作品.mp4"
|
||||
assert u.media_key_of(f) == "作者_123/作品.mp4"
|
||||
assert u.media_rel_dir_of(f) == "作者_123"
|
||||
|
||||
|
||||
def test_media_key_of_multi_image_work(u, temp_root):
|
||||
f = temp_root / "bilibili" / "作者_9" / "作品_ab12" / "001.jpg"
|
||||
assert u.media_key_of(f) == "作者_9/作品_ab12/001.jpg"
|
||||
assert u.media_rel_dir_of(f) == "作者_9/作品_ab12"
|
||||
|
||||
|
||||
def test_media_key_of_archive_follows_same_rule(u, temp_root):
|
||||
"""群文件 zip 落在 archive/{作者目录} 下,key 规则与媒体一致"""
|
||||
f = temp_root / "archive" / "作者_9" / "作品_群文件_120606.zip"
|
||||
assert u.media_key_of(f) == "作者_9/作品_群文件_120606.zip"
|
||||
|
||||
|
||||
def test_media_key_of_outside_temp_falls_back_to_name(u, temp_root, tmp_path):
|
||||
f = tmp_path / "elsewhere" / "x.mp4"
|
||||
assert u.media_key_of(f) == "x.mp4"
|
||||
assert u.media_rel_dir_of(f) == ""
|
||||
|
||||
|
||||
def test_media_key_of_flat_file_under_platform(u, temp_root):
|
||||
"""老数据/第三方产物:平台层下没有作者层 → 只用文件名"""
|
||||
f = temp_root / "douyin" / "x.mp4"
|
||||
assert u.media_key_of(f) == "x.mp4"
|
||||
assert u.media_rel_dir_of(f) == ""
|
||||
@@ -0,0 +1,315 @@
|
||||
"""视频解析插件群策略 (policy.py) 单元测试
|
||||
|
||||
policy.py 只依赖标准库 + nonebot.logger,故用 importlib 按文件路径裸加载
|
||||
(与 test_group_daily_history_store.py 同法),避免触发插件包 __init__ 的
|
||||
NoneBot 初始化。
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
_MODULE_PATH = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "hexi"
|
||||
/ "plugins"
|
||||
/ "nonebot_plugin_video_analysis"
|
||||
/ "policy.py"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def policy():
|
||||
"""以独立模块名加载 policy.py,避免与插件包 __init__ 冲突"""
|
||||
spec = importlib.util.spec_from_file_location("video_policy_under_test", _MODULE_PATH)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
yield module
|
||||
sys.modules.pop(spec.name, None)
|
||||
|
||||
|
||||
# ───────────────────────── 平台标签 ─────────────────────────
|
||||
|
||||
|
||||
def test_normalize_platform_aliases(policy):
|
||||
assert policy.normalize_platform("xhs") == "小红书"
|
||||
assert policy.normalize_platform("小红书") == "小红书"
|
||||
assert policy.normalize_platform("b23") == "哔哩哔哩"
|
||||
assert policy.normalize_platform("B站") == "哔哩哔哩"
|
||||
assert policy.normalize_platform("yt") == "Youtube"
|
||||
assert policy.normalize_platform("twitter") == "X"
|
||||
assert policy.normalize_platform(" X ") == "X"
|
||||
assert policy.normalize_platform("微博") is None
|
||||
assert policy.normalize_platform(None) is None
|
||||
|
||||
|
||||
def test_normalize_platforms_dedupe_and_order(policy):
|
||||
# 别名去重 + 按 PLATFORMS 顺序排列(与输入顺序无关)
|
||||
assert policy.normalize_platforms(["x", "xhs", "twitter"]) == ["小红书", "X"]
|
||||
# 字符串按分隔符拆
|
||||
assert policy.normalize_platforms("小红书,xhs 抖音") == ["小红书", "抖音"]
|
||||
# 非法值丢弃
|
||||
assert policy.normalize_platforms(["微博", "xhs"]) == ["小红书"]
|
||||
assert policy.normalize_platforms(None) == []
|
||||
assert policy.normalize_platforms(123) == []
|
||||
|
||||
|
||||
def test_apply_platform_diff_incremental(policy):
|
||||
# 全部带 +/- → 增量
|
||||
assert policy.apply_platform_diff([], ["+xhs"]) == ["小红书"]
|
||||
assert policy.apply_platform_diff(["小红书"], ["+X"]) == ["小红书", "X"]
|
||||
assert policy.apply_platform_diff(["小红书", "X"], ["-X"]) == ["小红书"]
|
||||
assert policy.apply_platform_diff(["小红书"], ["+xhs"]) == ["小红书"] # 去重
|
||||
assert policy.apply_platform_diff(["小红书"], ["-X"]) == ["小红书"] # 删不存在的
|
||||
assert policy.apply_platform_diff([], ["+xhs", "+b23"]) == ["小红书", "哔哩哔哩"]
|
||||
|
||||
|
||||
def test_apply_platform_diff_replace_and_invalid(policy):
|
||||
# 出现裸平台名 → 整体覆盖
|
||||
assert policy.apply_platform_diff(["小红书"], ["x"]) == ["X"]
|
||||
assert policy.apply_platform_diff(["小红书"], []) is None
|
||||
assert policy.apply_platform_diff(["小红书"], ["微博"]) is None
|
||||
assert policy.apply_platform_diff(["小红书"], ["+微博"]) is None
|
||||
|
||||
|
||||
def test_match_platform_domains(policy):
|
||||
assert policy.match_platform("https://x.com/a/status/1") == "X"
|
||||
assert policy.match_platform("https://twitter.com/a/status/1") == "X"
|
||||
assert policy.match_platform("https://b23.tv/abc") == "哔哩哔哩"
|
||||
assert policy.match_platform("https://v.douyin.com/abc") == "抖音"
|
||||
assert policy.match_platform("https://youtu.be/abc") == "Youtube"
|
||||
assert policy.match_platform("https://example.com/a.mp4") is None
|
||||
|
||||
|
||||
# ───────────────────────── 策略模型 ─────────────────────────
|
||||
|
||||
|
||||
def test_policy_from_dict_sanitizes(policy):
|
||||
p = policy.Policy.from_dict(
|
||||
{
|
||||
"auto": 1,
|
||||
"auto_link": ["xhs", "xhs", "微博"],
|
||||
"ban_link": ["twitter"],
|
||||
"plan": "b",
|
||||
"upload_public": True,
|
||||
"send_link": True,
|
||||
"upload_group_file": True,
|
||||
"unknown_key": "ignored",
|
||||
}
|
||||
)
|
||||
assert p.auto is True
|
||||
assert p.auto_link == ["小红书"]
|
||||
assert p.ban_link == ["X"]
|
||||
assert p.plan == "B"
|
||||
assert p.to_dict()["upload_group_file"] is True
|
||||
assert "unknown_key" not in p.to_dict()
|
||||
|
||||
|
||||
def test_policy_from_dict_bad_values_fall_back(policy):
|
||||
p = policy.Policy.from_dict({"plan": "Z", "auto_link": "b23"})
|
||||
assert p.plan == "C" # 非法 plan 回退默认
|
||||
assert p.auto_link == ["哔哩哔哩"]
|
||||
# 空/非 dict 输入不炸
|
||||
assert policy.Policy.from_dict(None) == policy.Policy()
|
||||
assert policy.Policy.from_dict([1, 2]) == policy.Policy()
|
||||
|
||||
|
||||
def test_sends_link_requires_public(policy):
|
||||
assert policy.Policy(upload_public=True, send_link=True).sends_link is True
|
||||
# 没开公网就没有链接可发
|
||||
assert policy.Policy(upload_public=False, send_link=True).sends_link is False
|
||||
|
||||
|
||||
def test_allows_group_file(policy):
|
||||
off = policy.Policy(upload_group_file=False)
|
||||
assert off.allows_group_file("抖音") is False
|
||||
assert off.allows_group_file(None) is False
|
||||
|
||||
# 开了但没限定平台 → 所有平台(含识别不出的)都传
|
||||
every = policy.Policy(upload_group_file=True)
|
||||
assert every.allows_group_file("抖音") is True
|
||||
assert every.allows_group_file(None) is True
|
||||
|
||||
# 限定了平台 → 只传清单里的,识别不出平台时按"不在清单里"处理
|
||||
only_douyin = policy.Policy(
|
||||
upload_group_file=True, group_file_platforms=["抖音"]
|
||||
)
|
||||
assert only_douyin.allows_group_file("抖音") is True
|
||||
assert only_douyin.allows_group_file("X") is False
|
||||
assert only_douyin.allows_group_file(None) is False
|
||||
|
||||
|
||||
def test_policy_construct_normalizes(policy):
|
||||
"""直接构造 Policy(...) 也要归一:绕过 from_dict 传别名不该静默失效"""
|
||||
p = policy.Policy(
|
||||
plan="z",
|
||||
auto_link=["xhs"],
|
||||
upload_group_file=True,
|
||||
group_file_platforms=["douyin"],
|
||||
)
|
||||
assert p.plan == "C"
|
||||
assert p.auto_link == ["小红书"]
|
||||
assert p.group_file_platforms == ["抖音"]
|
||||
assert p.allows_group_file("抖音") is True
|
||||
|
||||
|
||||
def test_group_file_platforms_normalized(policy):
|
||||
p = policy.Policy.from_dict(
|
||||
{"group_file_platforms": ["douyin", "b23", "微博"], "upload_group_file": True}
|
||||
)
|
||||
# 别名归一成规范标签、非法值丢弃、顺序按 PLATFORMS
|
||||
assert p.group_file_platforms == ["哔哩哔哩", "抖音"]
|
||||
|
||||
|
||||
def test_banned_and_auto_matched(policy):
|
||||
p = policy.Policy(auto_link=["小红书"], ban_link=["X"])
|
||||
assert p.auto_matched("https://xhslink.com/a") == "小红书"
|
||||
assert p.auto_matched("https://x.com/a") is None
|
||||
assert p.banned("https://x.com/a") == "X"
|
||||
assert p.banned("https://twitter.com/a") == "X"
|
||||
assert p.banned("https://douyin.com/a") is None
|
||||
|
||||
|
||||
# ───────────────────────── 迁移 ─────────────────────────
|
||||
|
||||
|
||||
def test_migrate_v1_to_v3(policy):
|
||||
data, changed = policy.migrate(
|
||||
{
|
||||
"WHITE_LIST": ["111", "222"],
|
||||
"AUTO_ANALYSIS": ["111"],
|
||||
"PLANA": ["111"],
|
||||
"PLANB": ["222"],
|
||||
"BLACK_LIST": ["999"],
|
||||
}
|
||||
)
|
||||
assert changed is True
|
||||
assert set(data["groups"]) == {"111", "222"}
|
||||
# v1 的 plan=B 隐含公网,迁移后显式补上
|
||||
assert data["groups"]["222"]["upload_public"] is True
|
||||
assert data["groups"]["111"]["plan"] == "A"
|
||||
assert data["groups"]["111"]["auto"] is True
|
||||
assert data["blacklist"] == ["999"]
|
||||
assert data["default"] == policy.Policy().to_dict()
|
||||
|
||||
|
||||
def test_migrate_v2_plan_b_gets_public(policy):
|
||||
data, changed = policy.migrate(
|
||||
{
|
||||
"groups": {
|
||||
"111": {"auto": True, "plan": "B"},
|
||||
"222": {"plan": "C"},
|
||||
"333": {},
|
||||
},
|
||||
"blacklist": [],
|
||||
}
|
||||
)
|
||||
assert changed is True
|
||||
assert data["groups"]["111"]["upload_public"] is True
|
||||
assert data["groups"]["222"]["upload_public"] is False
|
||||
assert data["groups"]["333"]["auto"] is False
|
||||
|
||||
|
||||
def test_migrate_keeps_explicit_public_flag(policy):
|
||||
# 显式写了 upload_public 的条目不被迁移覆盖
|
||||
data, _ = policy.migrate(
|
||||
{"groups": {"111": {"plan": "B", "upload_public": False}}, "blacklist": []}
|
||||
)
|
||||
assert data["groups"]["111"]["upload_public"] is False
|
||||
|
||||
|
||||
def test_migrate_idempotent(policy):
|
||||
first, _ = policy.migrate({"groups": {"111": {"plan": "B"}}, "blacklist": []})
|
||||
second, changed = policy.migrate(first)
|
||||
assert changed is False
|
||||
assert second == first
|
||||
|
||||
|
||||
# ───────────────────────── 存储 ─────────────────────────
|
||||
|
||||
|
||||
async def test_store_update_and_persist(policy, tmp_path):
|
||||
path = tmp_path / "list.json"
|
||||
store = policy.PolicyStore(path)
|
||||
store.load()
|
||||
assert path.exists() # 首次 load 落盘
|
||||
|
||||
await store.update_group("111", create=True, auto=True, plan="A")
|
||||
await store.update_group("222", create=True, auto_link=["xhs"])
|
||||
|
||||
# 内存读
|
||||
assert store.get("111").auto is True
|
||||
assert store.get("222").auto_link == ["小红书"]
|
||||
# 未配置的群回落 default
|
||||
assert store.get("999") == store.default_policy()
|
||||
assert store.is_whitelisted("111") is True
|
||||
assert store.is_whitelisted("999") is False
|
||||
|
||||
on_disk = json.loads(path.read_text(encoding="utf-8"))
|
||||
assert on_disk["groups"]["111"]["auto"] is True
|
||||
assert on_disk["groups"]["222"]["auto_link"] == ["小红书"]
|
||||
assert not path.with_name("list.json.tmp").exists()
|
||||
|
||||
|
||||
async def test_store_update_requires_whitelist_unless_create(policy, tmp_path):
|
||||
store = policy.PolicyStore(tmp_path / "list.json")
|
||||
assert await store.update_group("111", auto=True) is None
|
||||
assert store.is_whitelisted("111") is False
|
||||
|
||||
|
||||
async def test_store_remove_group_and_blacklist(policy, tmp_path):
|
||||
store = policy.PolicyStore(tmp_path / "list.json")
|
||||
await store.update_group("111", create=True)
|
||||
assert await store.remove_group("111") is True
|
||||
assert await store.remove_group("111") is False
|
||||
assert store.is_whitelisted("111") is False
|
||||
|
||||
assert await store.add_blacklist(999) is True
|
||||
assert await store.add_blacklist("999") is False # 幂等(int/str 同一个人)
|
||||
assert store.is_blacklisted(999) is True
|
||||
assert store.blacklist() == ["999"]
|
||||
assert await store.remove_blacklist("999") is True
|
||||
assert store.blacklist() == []
|
||||
|
||||
|
||||
async def test_store_concurrent_updates_keep_all(policy, tmp_path):
|
||||
"""并发写不能丢群:每次写都从事件循环线程快照,写线程不碰共享状态"""
|
||||
path = tmp_path / "list.json"
|
||||
store = policy.PolicyStore(path)
|
||||
store.load()
|
||||
|
||||
import asyncio
|
||||
|
||||
await asyncio.gather(
|
||||
*(store.update_group(str(1000 + i), create=True, auto=True) for i in range(30))
|
||||
)
|
||||
|
||||
on_disk = json.loads(path.read_text(encoding="utf-8"))
|
||||
assert len(on_disk["groups"]) == 30
|
||||
assert len(store.all_groups()) == 30
|
||||
|
||||
|
||||
async def test_store_persists_group_file_platforms(policy, tmp_path):
|
||||
"""POLICY_FIELDS 里漏了新字段的话这次更新会被静默丢掉"""
|
||||
path = tmp_path / "list.json"
|
||||
store = policy.PolicyStore(path)
|
||||
await store.update_group(
|
||||
"111", create=True, upload_group_file=True, group_file_platforms=["xhs"]
|
||||
)
|
||||
assert store.get("111").group_file_platforms == ["小红书"]
|
||||
on_disk = json.loads(path.read_text(encoding="utf-8"))
|
||||
assert on_disk["groups"]["111"]["group_file_platforms"] == ["小红书"]
|
||||
assert on_disk["groups"]["111"]["upload_group_file"] is True
|
||||
|
||||
|
||||
async def test_store_default_policy_edit(policy, tmp_path):
|
||||
store = policy.PolicyStore(tmp_path / "list.json")
|
||||
await store.set_default(policy.Policy(plan="B", upload_public=True))
|
||||
assert store.default_policy().plan == "B"
|
||||
assert store.default_policy().upload_public is True
|
||||
# 群未配置时读到的就是 default
|
||||
assert store.get("111").plan == "B"
|
||||
@@ -0,0 +1,571 @@
|
||||
"""Web 管理台解析任务 (services/web_jobs.py) 单元测试
|
||||
|
||||
模块本身只依赖标准库 + nonebot.logger,故用 importlib 按文件路径裸加载
|
||||
(与 test_video_policy.py / test_video_group_file.py 同法)。fetchers 与
|
||||
上传都是函数内延迟导入的注入点,这里全部塞假实现 —— 不碰网络、不碰 S3。
|
||||
`platform_of` 走 `..policy` 的包内相对导入,裸加载下不可用,测试里一并替换。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import importlib.util
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
_MODULE_PATH = (
|
||||
Path(__file__).resolve().parents[1]
|
||||
/ "hexi"
|
||||
/ "plugins"
|
||||
/ "nonebot_plugin_video_analysis"
|
||||
/ "services"
|
||||
/ "web_jobs.py"
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture(scope="module")
|
||||
def wj():
|
||||
"""以独立模块名加载 web_jobs.py,避免与插件包 __init__ 冲突"""
|
||||
spec = importlib.util.spec_from_file_location(
|
||||
"video_web_jobs_under_test", _MODULE_PATH
|
||||
)
|
||||
module = importlib.util.module_from_spec(spec)
|
||||
sys.modules[spec.name] = module
|
||||
spec.loader.exec_module(module)
|
||||
yield module
|
||||
sys.modules.pop(spec.name, None)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def manager(wj, monkeypatch):
|
||||
"""造一个依赖全注入的 JobManager(fetch / upload / poster / policy 都传进来)"""
|
||||
monkeypatch.setattr(wj, "platform_of", lambda url: "抖音")
|
||||
|
||||
def build(*, fetch, upload=None, poster=None, policy=None):
|
||||
return wj.JobManager(
|
||||
fetch=fetch,
|
||||
upload=upload or _ok_upload,
|
||||
poster=poster or _fake_poster,
|
||||
policy=policy or (lambda: None),
|
||||
)
|
||||
|
||||
return build
|
||||
|
||||
|
||||
async def _fake_poster(video):
|
||||
"""假抽帧:真的写一个封面文件出来(上传那一环会 stat 它)"""
|
||||
out = video.with_name(f"{video.stem}_封面.jpg")
|
||||
out.write_bytes(b"fake-jpeg")
|
||||
return out
|
||||
|
||||
|
||||
async def _no_poster(video):
|
||||
return None
|
||||
|
||||
|
||||
# ─────────────────────────── 假实现 ───────────────────────────
|
||||
|
||||
|
||||
async def _ok_upload(path, policy):
|
||||
return f"http://lan/{path.name}", None
|
||||
|
||||
|
||||
def _touch(directory: Path, name: str, content: bytes = b"x") -> Path:
|
||||
directory.mkdir(parents=True, exist_ok=True)
|
||||
path = directory / name
|
||||
path.write_bytes(content)
|
||||
return path
|
||||
|
||||
|
||||
def _ok_fetch(*paths, title="作品"):
|
||||
async def fetch(url):
|
||||
return title, [Path(p) for p in paths], len(paths) > 1
|
||||
|
||||
return fetch
|
||||
|
||||
|
||||
async def _settle(mgr, timeout=5.0):
|
||||
"""等所有任务进入终态,返回 list_jobs() 的结果。"""
|
||||
deadline = time.monotonic() + timeout
|
||||
while True:
|
||||
jobs = mgr.list_jobs()
|
||||
if jobs and all(not j.active for j in jobs):
|
||||
return jobs
|
||||
if time.monotonic() > deadline:
|
||||
raise AssertionError("任务没有在超时内结束")
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
|
||||
# ─────────────────────── URL 提取 / 文件类型 ───────────────────────
|
||||
|
||||
|
||||
def test_extract_urls_from_multiline_text(wj):
|
||||
text = (
|
||||
"看看这个 https://v.douyin.com/abc123。\n"
|
||||
"还有 https://b23.tv/xyz,https://v.douyin.com/abc123(重复的只留一条)"
|
||||
)
|
||||
assert wj.extract_urls(text) == [
|
||||
"https://v.douyin.com/abc123",
|
||||
"https://b23.tv/xyz",
|
||||
]
|
||||
|
||||
|
||||
def test_extract_urls_keeps_query_string(wj):
|
||||
"""小红书 xsec_token / 抖音带参数的分享链,query 不能被截掉"""
|
||||
url = "https://www.xiaohongshu.com/explore/abc?xsec_token=XYZ&xsec_source=pc_feed"
|
||||
assert wj.extract_urls(f"看这个 {url}") == [url]
|
||||
|
||||
|
||||
def test_extract_urls_without_links(wj):
|
||||
assert wj.extract_urls("这里没有链接,纯文本一段") == []
|
||||
assert wj.extract_urls("") == []
|
||||
|
||||
|
||||
def test_kind_of(wj):
|
||||
assert wj.kind_of("001.JPG") == "image"
|
||||
assert wj.kind_of(Path("作品.mp4")) == "video"
|
||||
assert wj.kind_of("note.txt") == "file"
|
||||
|
||||
|
||||
# ───────────────────────── 提交与复用 ─────────────────────────
|
||||
|
||||
|
||||
async def test_same_url_reuses_active_job(wj, manager, tmp_path):
|
||||
"""同 URL 连点两次不能起两个任务(抖音双开浏览器 + 落盘路径撞车)"""
|
||||
calls: list[str] = []
|
||||
|
||||
async def fetch(url):
|
||||
calls.append(url)
|
||||
await asyncio.sleep(0.05) # 保持 running,让第二次提交能命中
|
||||
return "作品", [_touch(tmp_path, "a.jpg")], False
|
||||
|
||||
mgr = manager(fetch=fetch)
|
||||
url = "https://v.douyin.com/abc123"
|
||||
first = (await mgr.submit([url]))[0]
|
||||
second = (await mgr.submit([url]))[0]
|
||||
|
||||
assert first.id == second.id
|
||||
assert first.platform == "抖音"
|
||||
await _settle(mgr)
|
||||
assert calls == [url]
|
||||
|
||||
# 跑完之后再提交同一条 → 是新任务(复用只针对在跑的)
|
||||
third = (await mgr.submit([url]))[0]
|
||||
assert third.id != first.id
|
||||
|
||||
|
||||
async def test_force_creates_new_job(wj, manager, tmp_path):
|
||||
calls: list[str] = []
|
||||
|
||||
async def fetch(url):
|
||||
calls.append(url)
|
||||
await asyncio.sleep(0.05)
|
||||
return "作品", [_touch(tmp_path, "a.jpg")], False
|
||||
|
||||
mgr = manager(fetch=fetch)
|
||||
url = "https://v.douyin.com/abc123"
|
||||
old = (await mgr.submit([url]))[0]
|
||||
new = (await mgr.submit([url], force=True))[0]
|
||||
|
||||
assert new.id != old.id
|
||||
jobs = await _settle(mgr)
|
||||
assert len(calls) == 2 and len(jobs) == 2
|
||||
|
||||
|
||||
async def test_batch_submit_keeps_input_order(wj, manager, tmp_path):
|
||||
mgr = manager(fetch=_ok_fetch(_touch(tmp_path, "a.jpg")))
|
||||
jobs = await mgr.submit(["https://a/1", "https://a/2", "https://a/3"])
|
||||
assert [j.url for j in jobs] == ["https://a/1", "https://a/2", "https://a/3"]
|
||||
await _settle(mgr)
|
||||
|
||||
|
||||
# ───────────────────────── 失败与超时 ─────────────────────────
|
||||
|
||||
|
||||
async def test_no_media_marks_job_failed(wj, manager):
|
||||
async def fetch(url):
|
||||
return None, [], False
|
||||
|
||||
mgr = manager(fetch=fetch)
|
||||
url = "https://v.douyin.com/gone"
|
||||
(await mgr.submit([url]))[0]
|
||||
|
||||
job = (await _settle(mgr))[0]
|
||||
assert job.status == "failed"
|
||||
assert "无法解析到媒体" in job.error
|
||||
assert job.title == url # 没标题时退回 URL,前端不至于只显示空白
|
||||
|
||||
|
||||
async def test_fetch_exception_marks_job_failed(wj, manager):
|
||||
async def fetch(url):
|
||||
raise RuntimeError("被风控了")
|
||||
|
||||
mgr = manager(fetch=fetch)
|
||||
(await mgr.submit(["https://v.douyin.com/x"]))[0]
|
||||
|
||||
job = (await _settle(mgr))[0]
|
||||
assert job.status == "failed" and job.error == "被风控了"
|
||||
|
||||
|
||||
async def test_job_timeout(wj, manager, monkeypatch):
|
||||
monkeypatch.setattr(wj, "JOB_TIMEOUT", 0.05)
|
||||
|
||||
async def fetch(url):
|
||||
await asyncio.sleep(5)
|
||||
|
||||
mgr = manager(fetch=fetch)
|
||||
(await mgr.submit(["https://v.douyin.com/slow"]))[0]
|
||||
|
||||
job = (await _settle(mgr))[0]
|
||||
assert job.status == "failed" and "解析超时" in job.error
|
||||
|
||||
|
||||
# ───────────────────────── 上传 ─────────────────────────
|
||||
|
||||
|
||||
async def test_single_upload_failure_does_not_fail_job(wj, manager, tmp_path):
|
||||
"""多图作品挂一张 → 只标那一张,其余照常出链接"""
|
||||
files = [_touch(tmp_path, "001.jpg"), _touch(tmp_path, "002.jpg")]
|
||||
|
||||
async def upload(path, policy):
|
||||
if path.name == "002.jpg":
|
||||
raise RuntimeError("S3 连接超时")
|
||||
return "http://lan/001.jpg?sig=1", "https://pub/001.jpg"
|
||||
|
||||
mgr = manager(fetch=_ok_fetch(*files), upload=upload)
|
||||
(await mgr.submit(["https://v.douyin.com/x"]))[0]
|
||||
|
||||
job = (await _settle(mgr))[0]
|
||||
assert job.status == "done"
|
||||
assert job.files[0]["local_url"].endswith("sig=1")
|
||||
assert job.files[0]["public_url"] == "https://pub/001.jpg"
|
||||
assert job.files[0]["error"] == ""
|
||||
assert "S3 连接超时" in job.files[1]["error"]
|
||||
assert job.files[1]["kind"] == "image" and job.files[1]["size"] == 1
|
||||
|
||||
|
||||
async def test_upload_without_links_is_marked(wj, manager, tmp_path):
|
||||
async def upload(path, policy):
|
||||
return "", None # s3.py 失败时就是返回空串
|
||||
|
||||
mgr = manager(fetch=_ok_fetch(_touch(tmp_path, "a.mp4")), upload=upload)
|
||||
(await mgr.submit(["https://v.douyin.com/x"]))[0]
|
||||
|
||||
job = (await _settle(mgr))[0]
|
||||
assert job.status == "done" and "上传失败" in job.files[0]["error"]
|
||||
assert job.files[0]["kind"] == "video"
|
||||
|
||||
|
||||
async def test_upload_gets_default_policy(wj, manager, tmp_path):
|
||||
"""存储链路按默认策略走:policy 原样传给 upload_with_plan"""
|
||||
seen: list = []
|
||||
sentinel = object()
|
||||
|
||||
async def upload(path, policy):
|
||||
seen.append(policy)
|
||||
return "http://lan/a.jpg", None
|
||||
|
||||
mgr = manager(
|
||||
fetch=_ok_fetch(_touch(tmp_path, "a.jpg")),
|
||||
upload=upload,
|
||||
policy=lambda: sentinel,
|
||||
)
|
||||
(await mgr.submit(["https://v.douyin.com/x"]))[0]
|
||||
await _settle(mgr)
|
||||
assert seen == [sentinel]
|
||||
|
||||
|
||||
# ───────────────────────── 刷新链接 ─────────────────────────
|
||||
|
||||
|
||||
async def test_refresh_reruns_upload(wj, manager, tmp_path):
|
||||
files = [_touch(tmp_path, "001.jpg"), _touch(tmp_path, "002.jpg")]
|
||||
links = iter(
|
||||
[
|
||||
"http://lan/a?sig=1",
|
||||
"http://lan/b?sig=1",
|
||||
"http://lan/a?sig=2",
|
||||
"http://lan/b?sig=2",
|
||||
]
|
||||
)
|
||||
|
||||
async def upload(path, policy):
|
||||
return next(links), None
|
||||
|
||||
mgr = manager(fetch=_ok_fetch(*files), upload=upload)
|
||||
job = (await mgr.submit(["https://v.douyin.com/x"]))[0]
|
||||
job = (await _settle(mgr))[0]
|
||||
assert [f["local_url"] for f in job.files] == [
|
||||
"http://lan/a?sig=1",
|
||||
"http://lan/b?sig=1",
|
||||
]
|
||||
|
||||
refreshed = await mgr.refresh(job.id)
|
||||
assert refreshed is not None
|
||||
assert [f["local_url"] for f in refreshed.files] == [
|
||||
"http://lan/a?sig=2",
|
||||
"http://lan/b?sig=2",
|
||||
]
|
||||
|
||||
|
||||
async def test_refresh_marks_files_gone_from_disk(wj, manager, tmp_path):
|
||||
"""temp 被 cleanup 清掉后再点刷新 → 只标文件没了,不炸"""
|
||||
path = _touch(tmp_path, "001.jpg")
|
||||
|
||||
async def upload(p, policy):
|
||||
return "http://lan/a.jpg", None
|
||||
|
||||
mgr = manager(fetch=_ok_fetch(path), upload=upload)
|
||||
(await mgr.submit(["https://v.douyin.com/x"]))[0]
|
||||
job = (await _settle(mgr))[0]
|
||||
|
||||
path.unlink()
|
||||
refreshed = await mgr.refresh(job.id)
|
||||
assert refreshed is not None
|
||||
assert "本地文件不存在" in refreshed.files[0]["error"]
|
||||
assert refreshed.files[0]["local_url"] == ""
|
||||
|
||||
|
||||
async def test_refresh_unknown_job_returns_none(wj, manager):
|
||||
mgr = manager(fetch=_ok_fetch())
|
||||
assert await mgr.refresh("nope") is None
|
||||
|
||||
async def fetch(url):
|
||||
return None, [], False
|
||||
|
||||
(await mgr.submit(["https://a/nothing"]))[0]
|
||||
job = (await _settle(mgr))[0]
|
||||
assert job.files == []
|
||||
assert await mgr.refresh(job.id) is None
|
||||
|
||||
|
||||
# ───────────────────────── 封面 ─────────────────────────
|
||||
|
||||
|
||||
async def test_cover_is_first_image(wj, manager, tmp_path):
|
||||
"""图文作品:封面直接取第一张图,不抽帧"""
|
||||
poster_calls: list = []
|
||||
|
||||
async def poster(video):
|
||||
poster_calls.append(video)
|
||||
return await _fake_poster(video)
|
||||
|
||||
files = [_touch(tmp_path, "001.jpg"), _touch(tmp_path, "002.jpg")]
|
||||
mgr = manager(fetch=_ok_fetch(*files), poster=poster)
|
||||
(await mgr.submit(["https://v.douyin.com/x"]))[0]
|
||||
|
||||
job = (await _settle(mgr))[0]
|
||||
assert job.cover_url == "http://lan/001.jpg"
|
||||
assert poster_calls == []
|
||||
|
||||
|
||||
async def test_cover_is_extracted_for_video_only_post(wj, manager, tmp_path):
|
||||
"""纯视频作品:抽第一帧、上传、当封面"""
|
||||
uploaded: list[str] = []
|
||||
|
||||
async def upload(path, policy):
|
||||
uploaded.append(path.name)
|
||||
return f"http://lan/{path.name}", None
|
||||
|
||||
mgr = manager(fetch=_ok_fetch(_touch(tmp_path, "作品.mp4")), upload=upload)
|
||||
(await mgr.submit(["https://v.douyin.com/x"]))[0]
|
||||
|
||||
job = (await _settle(mgr))[0]
|
||||
assert job.cover_url == "http://lan/作品_封面.jpg"
|
||||
assert uploaded == ["作品.mp4", "作品_封面.jpg"]
|
||||
assert job.poster_path is not None and job.poster_path.exists()
|
||||
# 封面是单独的文件,不能混进 files(否则播放器里会多出一个"文件")
|
||||
assert [f["name"] for f in job.files] == ["作品.mp4"]
|
||||
|
||||
|
||||
async def test_cover_prefers_image_over_video_in_mixed_post(wj, manager, tmp_path):
|
||||
"""混排(图 + 视频):按用户定的规则取第一张图"""
|
||||
files = [_touch(tmp_path, "图文.jpg"), _touch(tmp_path, "动图.mp4")]
|
||||
mgr = manager(fetch=_ok_fetch(*files))
|
||||
(await mgr.submit(["https://v.douyin.com/x"]))[0]
|
||||
|
||||
job = (await _settle(mgr))[0]
|
||||
assert job.cover_url == "http://lan/图文.jpg"
|
||||
assert job.poster_path is None
|
||||
|
||||
|
||||
async def test_poster_failure_keeps_job_done(wj, manager, tmp_path):
|
||||
"""抽帧失败 → 封面留空,任务照常完成"""
|
||||
mgr = manager(fetch=_ok_fetch(_touch(tmp_path, "作品.mp4")), poster=_no_poster)
|
||||
(await mgr.submit(["https://v.douyin.com/x"]))[0]
|
||||
|
||||
job = (await _settle(mgr))[0]
|
||||
assert job.status == "done"
|
||||
assert job.cover_url == "" and job.poster_path is None
|
||||
|
||||
|
||||
async def test_make_poster_extracts_real_frame(wj, tmp_path, monkeypatch):
|
||||
"""真跑一次 ffmpeg:确认真能抽出 JPEG(假 poster 只覆盖接缝,覆盖不到命令本身)"""
|
||||
ffmpeg = Path(__file__).resolve().parents[1] / ".venv" / "Scripts" / "ffmpeg.exe"
|
||||
if not ffmpeg.exists():
|
||||
pytest.skip("没有 .venv/Scripts/ffmpeg.exe")
|
||||
monkeypatch.setattr(wj, "ffmpeg_path", lambda: str(ffmpeg))
|
||||
|
||||
video = tmp_path / "作品.mp4"
|
||||
subprocess.run(
|
||||
[str(ffmpeg), "-f", "lavfi", "-i", "testsrc=size=320x240:rate=10",
|
||||
"-t", "2", "-pix_fmt", "yuv420p", "-y", str(video)],
|
||||
capture_output=True, check=True,
|
||||
)
|
||||
|
||||
poster = await wj.make_poster(video)
|
||||
assert poster is not None
|
||||
assert poster.name == "作品_封面.jpg"
|
||||
assert poster.read_bytes()[:2] == b"\xff\xd8" # JPEG magic
|
||||
|
||||
|
||||
async def test_make_poster_falls_back_to_first_frame(wj, tmp_path, monkeypatch):
|
||||
"""不足 1 秒的视频取不到"第 1 秒"那一帧,要退回第 0 秒再试"""
|
||||
ffmpeg = Path(__file__).resolve().parents[1] / ".venv" / "Scripts" / "ffmpeg.exe"
|
||||
if not ffmpeg.exists():
|
||||
pytest.skip("没有 .venv/Scripts/ffmpeg.exe")
|
||||
monkeypatch.setattr(wj, "ffmpeg_path", lambda: str(ffmpeg))
|
||||
|
||||
video = tmp_path / "短片.mp4"
|
||||
subprocess.run(
|
||||
[str(ffmpeg), "-f", "lavfi", "-i", "testsrc=size=160x120:rate=10",
|
||||
"-t", "0.5", "-pix_fmt", "yuv420p", "-y", str(video)],
|
||||
capture_output=True, check=True,
|
||||
)
|
||||
|
||||
poster = await wj.make_poster(video)
|
||||
assert poster is not None and poster.stat().st_size > 0
|
||||
|
||||
|
||||
async def test_make_poster_missing_video_returns_none(wj, tmp_path, monkeypatch):
|
||||
ffmpeg = Path(__file__).resolve().parents[1] / ".venv" / "Scripts" / "ffmpeg.exe"
|
||||
if not ffmpeg.exists():
|
||||
pytest.skip("没有 .venv/Scripts/ffmpeg.exe")
|
||||
monkeypatch.setattr(wj, "ffmpeg_path", lambda: str(ffmpeg))
|
||||
|
||||
assert await wj.make_poster(tmp_path / "不存在.mp4") is None
|
||||
|
||||
|
||||
async def test_refresh_refreshes_cover(wj, manager, tmp_path):
|
||||
"""刷新链接时封面一起换新(否则卡片封面 1 小时后变裂图)"""
|
||||
links = iter(["http://lan/a?sig=1", "http://lan/p?sig=1", "http://lan/a?sig=2", "http://lan/p?sig=2"])
|
||||
|
||||
async def upload(path, policy):
|
||||
return next(links), None
|
||||
|
||||
mgr = manager(fetch=_ok_fetch(_touch(tmp_path, "作品.mp4")), upload=upload)
|
||||
(await mgr.submit(["https://v.douyin.com/x"]))[0]
|
||||
job = (await _settle(mgr))[0]
|
||||
assert job.cover_url == "http://lan/p?sig=1"
|
||||
|
||||
refreshed = await mgr.refresh(job.id)
|
||||
assert refreshed is not None
|
||||
assert refreshed.cover_url == "http://lan/p?sig=2"
|
||||
|
||||
|
||||
# ───────────────────────── 清理 ─────────────────────────
|
||||
|
||||
|
||||
async def test_remove_job(wj, manager, tmp_path):
|
||||
mgr = manager(fetch=_ok_fetch(_touch(tmp_path, "a.jpg")))
|
||||
job = (await mgr.submit(["https://v.douyin.com/x"]))[0]
|
||||
await _settle(mgr)
|
||||
|
||||
assert mgr.remove(job.id) is True
|
||||
assert mgr.list_jobs() == []
|
||||
assert mgr.remove(job.id) is False # 再删一次:不存在
|
||||
|
||||
|
||||
async def test_remove_running_job_cancels_it(wj, manager, tmp_path):
|
||||
"""删掉在跑的任务要顺手取消,不能让它继续跑完再写回任务表"""
|
||||
started = asyncio.Event()
|
||||
cancelled = asyncio.Event()
|
||||
|
||||
async def fetch(url):
|
||||
started.set()
|
||||
try:
|
||||
await asyncio.sleep(30)
|
||||
except asyncio.CancelledError:
|
||||
cancelled.set()
|
||||
raise
|
||||
return "作品", [_touch(tmp_path, "a.jpg")], False
|
||||
|
||||
mgr = manager(fetch=fetch)
|
||||
job = (await mgr.submit(["https://v.douyin.com/x"]))[0]
|
||||
await asyncio.wait_for(started.wait(), timeout=2)
|
||||
|
||||
assert mgr.remove(job.id) is True
|
||||
await asyncio.wait_for(cancelled.wait(), timeout=2)
|
||||
assert mgr.list_jobs() == []
|
||||
|
||||
|
||||
async def test_clear_finished_keeps_running(wj, manager, tmp_path):
|
||||
release = asyncio.Event()
|
||||
|
||||
async def fetch(url):
|
||||
if "slow" in url:
|
||||
await release.wait()
|
||||
return "作品", [_touch(tmp_path, "a.jpg")], False
|
||||
|
||||
mgr = manager(fetch=fetch)
|
||||
fast = (await mgr.submit(["https://v.douyin.com/fast"]))[0]
|
||||
slow = (await mgr.submit(["https://v.douyin.com/slow"]))[0]
|
||||
for _ in range(500):
|
||||
if not fast.active:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
|
||||
assert mgr.clear_finished() == 1
|
||||
assert [j.id for j in mgr.list_jobs()] == [slow.id]
|
||||
|
||||
release.set()
|
||||
await _settle(mgr)
|
||||
|
||||
|
||||
# ───────────────────────── 上限与回收 ─────────────────────────
|
||||
|
||||
|
||||
async def test_job_count_is_capped(wj, manager, tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(wj, "MAX_JOBS", 3)
|
||||
mgr = manager(fetch=_ok_fetch(_touch(tmp_path, "a.jpg")))
|
||||
|
||||
await mgr.submit([f"https://v.douyin.com/{i}" for i in range(6)])
|
||||
jobs = await _settle(mgr)
|
||||
assert len(jobs) == 3
|
||||
|
||||
|
||||
async def test_expired_terminal_jobs_are_pruned(wj, manager, tmp_path):
|
||||
mgr = manager(fetch=_ok_fetch(_touch(tmp_path, "a.jpg")))
|
||||
job = (await mgr.submit(["https://v.douyin.com/x"]))[0]
|
||||
await _settle(mgr)
|
||||
assert len(mgr.list_jobs()) == 1
|
||||
|
||||
job.updated_at -= wj.TERMINAL_TTL + 1
|
||||
assert mgr.list_jobs() == []
|
||||
|
||||
|
||||
async def test_running_jobs_survive_prune(wj, manager, monkeypatch):
|
||||
"""在跑的任务不受上限/TTL 影响(TTL 设为负数,终态一律该清掉)"""
|
||||
monkeypatch.setattr(wj, "TERMINAL_TTL", -1)
|
||||
release = asyncio.Event()
|
||||
|
||||
async def fetch(url):
|
||||
await release.wait()
|
||||
return None, [], False
|
||||
|
||||
mgr = manager(fetch=fetch)
|
||||
job = (await mgr.submit(["https://v.douyin.com/x"]))[0]
|
||||
await asyncio.sleep(0.05)
|
||||
|
||||
assert [j.id for j in mgr.list_jobs()] == [job.id]
|
||||
assert job.status == "running" and job.stage == "解析中"
|
||||
|
||||
release.set()
|
||||
# 终态任务会被 TTL=-1 立即回收,所以这里盯 job 本身而不是 list_jobs()
|
||||
for _ in range(500):
|
||||
if not job.active:
|
||||
break
|
||||
await asyncio.sleep(0.01)
|
||||
assert job.status == "failed"
|
||||
Reference in New Issue
Block a user