feat(video-analysis): 群策略 v3 / 群文件投递通道 / Web 管理页

- policy.py:per-group 正交策略(自动解析 / 自动策略 / 禁用策略 / 存储 A·B·C /
  公网 / 链接 / 群文件 + 平台限定),list.json v1/v2 → v3 自动迁移,
  写入统一走 PolicyStore(加锁 + .tmp 原子替换 + 字段归一)
- 群文件并行通道 group_file.py:打包 zip(可选 pyzipper AES-256)后优先走 S3 预签名、
  本地直传兜底;设了密码但 pyzipper 不可用就放弃上传,不退化成明文
- list_proc.py 收敛到「视频策略」统一入口,权限判定改走 policy
- Web 管理页 /hub/video_analysis(群策略 + 链接解析面板)与 services/web_jobs.py
  (只复用纯函数层,Web 上下文不发消息;内存任务表 + 并发闸门 + 超时)
- 媒体命名统一到 utils.py({作者}_{作者id}/{作品名}[_短码]),cleanup 回收空目录
- 测试:policy / 命名 / 群文件 / web_jobs 四组

顺带 pyproject 的 pytest 加 testpaths=tests(避免收进 debug/ 下的调试脚本)。

Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
2026-09-22 14:23:32 +08:00
co-authored by Claude Code
parent 51b08ccb68
commit 4badcfcf32
29 changed files with 4838 additions and 670 deletions
+303
View File
@@ -0,0 +1,303 @@
"""群文件打包/投递 (group_file.py) 单元测试
group_file.py 只依赖标准库 + nonebot.logger,故用 importlib 按文件路径裸加载
(与 test_video_policy.py 同法)。build_archive 一律显式传 out_dir —— 默认目录
要经相对导入取 utils.get_temp_root(),裸加载模块没有包上下文。
"""
import importlib.util
import sys
import zipfile
from pathlib import Path
import pytest
_MODULE_PATH = (
Path(__file__).resolve().parents[1]
/ "hexi"
/ "plugins"
/ "nonebot_plugin_video_analysis"
/ "services"
/ "storage"
/ "group_file.py"
)
@pytest.fixture(scope="module")
def gf():
"""以独立模块名加载 group_file.py,避免与插件包 __init__ 冲突"""
spec = importlib.util.spec_from_file_location(
"video_group_file_under_test", _MODULE_PATH
)
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
yield module
sys.modules.pop(spec.name, None)
def _make_files(directory: Path, names: list[str]) -> list[Path]:
directory.mkdir(parents=True, exist_ok=True)
made = []
for name in names:
path = directory / name
path.write_bytes(f"hello {name}".encode())
made.append(path)
return made
def test_plain_archive(gf, tmp_path):
files = _make_files(tmp_path / "src", ["a.txt", "b.txt"])
archive = gf.build_archive(files, "我的作品", out_dir=tmp_path / "out")
assert archive.exists() and archive.suffix == ".zip"
assert "我的作品" in archive.name
with zipfile.ZipFile(archive) as zf:
assert sorted(zf.namelist()) == ["a.txt", "b.txt"]
assert zf.read("a.txt") == b"hello a.txt"
def test_encrypted_archive_requires_password(gf, tmp_path):
pyzipper = pytest.importorskip("pyzipper")
files = _make_files(tmp_path / "src", ["v.mp4"])
archive = gf.build_archive(
files, "加密作品", password="pw123", out_dir=tmp_path / "out"
)
# 标准库打不开加密包
with zipfile.ZipFile(archive) as zf:
with pytest.raises(RuntimeError):
zf.read("v.mp4")
# 正确密码可读,错误密码不行
with pyzipper.AESZipFile(archive) as zf:
zf.setpassword(b"pw123")
assert zf.read("v.mp4") == b"hello v.mp4"
with pyzipper.AESZipFile(archive) as zf:
zf.setpassword(b"wrong")
with pytest.raises(RuntimeError):
zf.read("v.mp4")
def test_duplicate_names_indexed(gf, tmp_path):
first = _make_files(tmp_path / "p1", ["same.txt"])[0]
second = _make_files(tmp_path / "p2", ["same.txt"])[0]
archive = gf.build_archive([first, second], "t", out_dir=tmp_path / "out")
with zipfile.ZipFile(archive) as zf:
assert sorted(zf.namelist()) == ["same.txt", "same_2.txt"]
assert zf.read("same_2.txt") == b"hello same.txt"
def test_output_path_is_unique(gf, tmp_path):
files = _make_files(tmp_path / "src", ["a.txt"])
out = tmp_path / "out"
one = gf.build_archive(files, "t", out_dir=out)
two = gf.build_archive(files, "t", out_dir=out)
assert one != two and one.exists() and two.exists()
def test_placeholder_title_falls_back(gf, tmp_path):
"""universal.py 传的 title 是占位符 "title",不该出现在文件名里"""
files = _make_files(tmp_path / "src", ["a.txt"])
archive = gf.build_archive(files, "title", out_dir=tmp_path / "out")
assert archive.name.startswith("群文件_")
assert "群文件_群文件" not in archive.name
def test_title_with_path_separators_is_sanitized(gf, tmp_path):
"""原始标题(抖音文案/YouTube 标题)可能带 /、换行、#话题,不能进文件名"""
files = _make_files(tmp_path / "src", ["a.txt"])
archive = gf.build_archive(files, "和/或 #话题\n测试", out_dir=tmp_path / "out")
assert "/" not in archive.name and "\\" not in archive.name
assert "\n" not in archive.name and "#" not in archive.name
assert archive.name.startswith("和_或_测试")
def test_title_starting_with_archive_prefix_falls_back(gf, tmp_path):
"""标题本身以「群文件」开头时不要产出 群文件_xxx_群文件_yyy.zip"""
files = _make_files(tmp_path / "src", ["a.txt"])
archive = gf.build_archive(files, "群文件_120606", out_dir=tmp_path / "out")
assert archive.name.startswith("群文件_")
assert "群文件_120606_群文件" not in archive.name
def test_empty_or_missing_input(gf, tmp_path):
with pytest.raises(ValueError):
gf.build_archive([], "t", out_dir=tmp_path / "out")
with pytest.raises(FileNotFoundError):
gf.build_archive([tmp_path / "nope.mp4"], "t", out_dir=tmp_path / "out")
def test_password_without_pyzipper_fails_loud(gf, tmp_path, monkeypatch):
"""设了密码但加密库不可用 → 报错(调用方会放弃上传,绝不能退化传明文)"""
monkeypatch.setattr(gf, "pyzipper", None)
assert gf.encryption_available() is False
files = _make_files(tmp_path / "src", ["a.txt"])
with pytest.raises(RuntimeError):
gf.build_archive(files, "t", password="pw", out_dir=tmp_path / "out")
# 不加密那一路不受影响
assert gf.build_archive(files, "t", out_dir=tmp_path / "out").exists()
# ───────────────────── 上传:URI 直传 + S3 降级 ─────────────────────
def test_file_uri_keeps_unicode_and_uses_slashes(gf, tmp_path):
target = tmp_path / "dir with space" / "群文件_120606.zip"
target.parent.mkdir(parents=True)
target.write_bytes(b"x")
uri = gf._file_uri(target)
assert uri.startswith("file:///")
assert "\\" not in uri
# 中文/空格保持原样(不做百分号转义,NapCat 侧直接当路径解析)
assert uri.endswith("/dir with space/群文件_120606.zip")
async def test_s3_link_is_the_primary_channel(gf, tmp_path, monkeypatch):
"""默认通道就是 S3 链接(本环境本地直传必挂,不先浪费一次调用)"""
calls: list = []
link = "http://192.168.2.15:5246/PLANA/x.zip?sig=1"
async def fake_call(file_value, group_id, name, folder_id=None):
calls.append((file_value, group_id, name))
monkeypatch.setattr(gf, "_call_upload", fake_call)
monkeypatch.setattr(gf, "_s3_url", lambda path, policy: link)
file = _make_files(tmp_path / "src", ["a.zip"])[0]
assert await gf.upload_group_file(file, 123) is True
assert calls == [(link, 123, "a.zip")]
async def test_s3_call_failure_falls_back_to_local(gf, tmp_path, monkeypatch):
seen: list[str] = []
link = "http://192.168.2.15:5246/PLANA/x.zip?sig=1"
async def fake_call(file_value, group_id, name, folder_id=None):
seen.append(file_value)
if file_value.startswith("http"):
raise RuntimeError("链接不可达")
monkeypatch.setattr(gf, "_call_upload", fake_call)
monkeypatch.setattr(gf, "_s3_url", lambda path, policy: link)
file = _make_files(tmp_path / "src", ["a.zip"])[0]
assert await gf.upload_group_file(file, 123) is True
assert seen[0] == link
assert seen[1] == gf._file_uri(file)
async def test_no_s3_link_falls_back_to_local(gf, tmp_path, monkeypatch):
seen: list[str] = []
async def fake_call(file_value, group_id, name, folder_id=None):
seen.append(file_value)
monkeypatch.setattr(gf, "_call_upload", fake_call)
monkeypatch.setattr(gf, "_s3_url", lambda path, policy: "")
file = _make_files(tmp_path / "src", ["a.zip"])[0]
assert await gf.upload_group_file(file, 123) is True
assert seen == [gf._file_uri(file)]
async def test_both_transports_fail(gf, tmp_path, monkeypatch):
async def always_fail(*args, **kwargs):
raise RuntimeError("nope")
monkeypatch.setattr(gf, "_call_upload", always_fail)
monkeypatch.setattr(gf, "_s3_url", lambda path, policy: "http://x/y.zip")
file = _make_files(tmp_path / "src", ["a.zip"])[0]
assert await gf.upload_group_file(file, 123) is False
async def test_missing_file_short_circuits(gf, tmp_path, monkeypatch):
called = []
monkeypatch.setattr(gf, "_call_upload", lambda *a, **kw: called.append(1))
assert await gf.upload_group_file(tmp_path / "nope.zip", 123) is False
assert called == []
# ───────────────────────── 投递编排 ─────────────────────────
def _record_uploads(gf, monkeypatch, path: Path, ok: bool = True) -> list:
calls: list = []
async def fake_upload(file_path, group_id, **kwargs):
calls.append((str(file_path), group_id))
return ok
monkeypatch.setattr(gf, "upload_group_file", fake_upload)
return calls
async def test_zip_mode_uploads_single_archive(gf, tmp_path, monkeypatch):
archive = tmp_path / "pack.zip"
archive.write_bytes(b"zip")
calls = _record_uploads(gf, monkeypatch, archive)
monkeypatch.setattr(
gf, "build_archive", lambda *a, **kw: archive
)
files = _make_files(tmp_path / "src", ["a.txt", "b.txt"])
ok = await gf.upload_group_files(files, 123, title="t", password="pw")
assert ok is True
assert calls == [(str(archive), 123)]
async def test_raw_mode_uploads_each_file(gf, tmp_path, monkeypatch):
calls = _record_uploads(gf, monkeypatch, tmp_path)
files = _make_files(tmp_path / "src", ["a.txt", "b.txt"])
ok = await gf.upload_group_files(files, 123, zip_files=False)
assert ok is True
assert [c[0] for c in calls] == [str(p) for p in files]
async def test_raw_mode_prefixes_work_name_for_multi_files(gf, tmp_path, monkeypatch):
"""多图作品逐个传时,001.jpg 这类成员名要带上作品名(单文件不加)"""
seen: list = []
async def fake_upload(file_path, group_id, **kwargs):
seen.append(kwargs.get("name"))
return True
monkeypatch.setattr(gf, "upload_group_file", fake_upload)
images = _make_files(tmp_path / "src", ["001.jpg", "002.jpg"])
await gf.upload_group_files(
images, 123, zip_files=False, rel_dir="作者_9/海边日落"
)
assert seen == ["海边日落_001.jpg", "海边日落_002.jpg"]
seen.clear()
single = _make_files(tmp_path / "src2", ["作品.mp4"])
await gf.upload_group_files(single, 123, zip_files=False, rel_dir="作者_9")
assert seen == ["作品.mp4"] # 单文件时文件名本身就是作品名,不再加前缀
async def test_pack_failure_skips_upload_entirely(gf, tmp_path, monkeypatch):
"""打包失败(例如缺 pyzipper)时不能退化成上传原文件"""
calls = _record_uploads(gf, monkeypatch, tmp_path)
def boom(*args, **kwargs):
raise RuntimeError("pyzipper 未安装")
monkeypatch.setattr(gf, "build_archive", boom)
files = _make_files(tmp_path / "src", ["a.txt"])
ok = await gf.upload_group_files(files, 123, title="t", password="pw")
assert ok is False
assert calls == []
async def test_empty_file_list(gf, tmp_path, monkeypatch):
calls = _record_uploads(gf, monkeypatch, tmp_path)
assert await gf.upload_group_files([], 123) is False
assert calls == []
+214
View File
@@ -0,0 +1,214 @@
"""媒体命名 / S3 key (utils.py 纯函数) 单元测试
utils.py 只依赖标准库,故用 importlib 按文件路径裸加载(与
test_video_group_file.py 同法)。media_key_of / media_rel_dir_of 依赖
get_temp_root 的返回值,测试里 monkeypatch 成临时目录。
"""
import importlib.util
import sys
from pathlib import Path
import pytest
_MODULE_PATH = (
Path(__file__).resolve().parents[1]
/ "hexi"
/ "plugins"
/ "nonebot_plugin_video_analysis"
/ "utils.py"
)
@pytest.fixture(scope="module")
def u():
"""以独立模块名加载 utils.py,避免与插件包 __init__ 冲突"""
spec = importlib.util.spec_from_file_location("video_utils_under_test", _MODULE_PATH)
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
yield module
sys.modules.pop(spec.name, None)
@pytest.fixture
def temp_root(u, tmp_path, monkeypatch):
"""把 get_temp_root 指到 tmp_path(media_key_of 只做路径解析,不需 mkdir)"""
root = tmp_path / "temp"
root.mkdir()
monkeypatch.setattr(u, "get_temp_root", lambda sub="": root / sub if sub else root)
return root
# ───────────────────────── 作者目录 ─────────────────────────
def test_author_dir_with_id(u):
"""有作者 id 就是稳定的 `{昵称}_{id}`,不带码"""
assert u.build_author_dir("示例作者", "12345") == "示例作者_12345"
assert u.build_author_dir("示例作者", "12345", source="x", at=1) == (
"示例作者_12345"
)
def test_author_dir_nickname_only_gets_time_code(u):
"""只有昵称 → 追加 4 位时间短码:分不清同名作者,宁可不聚合也不混目录"""
at = 1_700_000_000
code = u.short_time_code(at)
assert u.build_author_dir("示例作者", "", at=at) == f"示例作者_{code}"
assert u.build_author_dir("示例作者", at=at) == f"示例作者_{code}"
for junk in ("0", "NA", "none", "None", " null "):
assert u.build_author_dir("示例作者", junk, at=at) == f"示例作者_{code}"
# 不同时刻 = 不同目录(同一作者的不同作品不会互相覆盖)
other = u.build_author_dir("示例作者", at=at + 3600)
assert other != f"示例作者_{code}"
assert other.startswith("示例作者_") and len(other.split("_")[-1]) == 4
def test_author_dir_uses_source_code_when_anonymous(u):
"""连昵称都没有 → 未知作者 + 来源短码(同一来源稳定,不同来源不撞)"""
url = "https://www.douyin.com/video/7412345678901234567"
first = u.build_author_dir(None, None, source=url)
assert first.startswith("未知作者_")
assert first == u.build_author_dir("", "", source=url) # 同来源稳定
second = u.build_author_dir(None, None, source=url + "8")
assert second != first
# 没有来源时才退化成时间短码
at = 1_700_000_000
assert u.build_author_dir(None, None, at=at) == (
f"未知作者_{u.short_time_code(at)}"
)
def test_author_dir_treats_placeholder_nicknames_as_anonymous(u):
"""抓取层拿不到昵称时的兜底串不算作者名,走来源码"""
url = "https://www.bilibili.com/opus/123"
expected = u.build_author_dir(None, None, source=url)
for placeholder in ("未知作者", "小红书用户", "B站用户", "b站用户"):
assert u.build_author_dir(placeholder, "", source=url) == expected
# 纯 emoji 昵称 slugify 后为空 → 同样按匿名处理
assert u.build_author_dir("🎬🎬", "", source=url) == expected
def test_short_source_code_is_stable_base36(u):
url = "https://www.douyin.com/video/7412345678901234567"
code = u.short_source_code(url)
assert len(code) == 4 and code.isalnum()
assert code == u.short_source_code(url) # 可复现(不能用带盐的 hash())
assert code != u.short_source_code(url + "8")
def test_author_dir_sanitizes_illegal_chars(u):
"""Windows 非法字符不能进目录名(slugify 直接删掉,空白转 -)"""
assert u.build_author_dir("a/b:c*d?", "12 3") == "abcd_12-3"
assert "/" not in u.build_author_dir("a/b", "1/2")
def test_author_dir_dedupes_same_handle(u):
"""X 这类站点上传者名就是 handle,别产出 someone_someone"""
assert u.build_author_dir("@Someone", "@Someone") == "someone"
def test_author_dir_keeps_long_sec_uid_distinct(u):
"""sec_uid 公共前缀就有 17 字符,截断到 20 会让不同作者撞同一目录"""
first = "MS4wLjABAAAA" + "x" * 45
second = "MS4wLjABAAAA" + "y" * 45
assert u.build_author_dir("n", first) != u.build_author_dir("n", second)
# ───────────────────────── 作品名 ─────────────────────────
def test_work_stem_strips_hashtags_and_spaces(u):
assert u.build_work_stem("旅行 #随手拍 vlog") == "旅行-vlog"
def test_work_stem_empty_falls_back(u):
assert u.build_work_stem("") == "作品"
assert u.build_work_stem(None) == "作品"
assert u.build_work_stem("🎬") == "作品"
def test_work_stem_truncates(u):
assert len(u.build_work_stem("标题" * 20)) <= 15
# ───────────────────────── 短码与重名 ─────────────────────────
def test_short_time_code_is_four_char_base36(u):
assert u.short_time_code(0) == "0000"
code = u.short_time_code(1_700_000_000)
assert len(code) == 4 and code.isalnum()
# 相邻秒不同码
assert code != u.short_time_code(1_700_000_001)
def test_unique_media_path_without_collision(u, tmp_path):
target = tmp_path / "作品.mp4"
assert u.unique_media_path(target) == target
def test_unique_media_path_adds_code_then_index(u, tmp_path):
target = tmp_path / "作品.mp4"
target.write_bytes(b"x")
code = u.short_time_code(1_700_000_000)
second = u.unique_media_path(target, at=1_700_000_000)
assert second.name == f"作品_{code}.mp4"
second.write_bytes(b"x")
third = u.unique_media_path(target, at=1_700_000_000)
assert third.name == f"作品_{code}_2.mp4"
def test_unique_media_path_creates_author_dir(u, tmp_path):
"""落盘前才建作者目录:命名函数顺带确保父目录存在"""
target = tmp_path / "作者_123" / "作品.mp4"
assert u.unique_media_path(target) == target
assert target.parent.is_dir()
def test_unique_media_path_works_for_directories(u, tmp_path):
"""多图作品目录同名时同样加短码"""
note_dir = tmp_path / "作者_123" / "作品"
note_dir.mkdir(parents=True)
renamed = u.unique_media_path(note_dir, at=1_700_000_000)
assert renamed.name == f"作品_{u.short_time_code(1_700_000_000)}"
# ───────────────────────── S3 key ─────────────────────────
def test_media_key_keeps_author_dir_and_drops_platform(u, temp_root):
f = temp_root / "douyin" / "作者_123" / "作品.mp4"
assert u.media_key_of(f) == "作者_123/作品.mp4"
assert u.media_rel_dir_of(f) == "作者_123"
def test_media_key_of_multi_image_work(u, temp_root):
f = temp_root / "bilibili" / "作者_9" / "作品_ab12" / "001.jpg"
assert u.media_key_of(f) == "作者_9/作品_ab12/001.jpg"
assert u.media_rel_dir_of(f) == "作者_9/作品_ab12"
def test_media_key_of_archive_follows_same_rule(u, temp_root):
"""群文件 zip 落在 archive/{作者目录} 下,key 规则与媒体一致"""
f = temp_root / "archive" / "作者_9" / "作品_群文件_120606.zip"
assert u.media_key_of(f) == "作者_9/作品_群文件_120606.zip"
def test_media_key_of_outside_temp_falls_back_to_name(u, temp_root, tmp_path):
f = tmp_path / "elsewhere" / "x.mp4"
assert u.media_key_of(f) == "x.mp4"
assert u.media_rel_dir_of(f) == ""
def test_media_key_of_flat_file_under_platform(u, temp_root):
"""老数据/第三方产物:平台层下没有作者层 → 只用文件名"""
f = temp_root / "douyin" / "x.mp4"
assert u.media_key_of(f) == "x.mp4"
assert u.media_rel_dir_of(f) == ""
+315
View File
@@ -0,0 +1,315 @@
"""视频解析插件群策略 (policy.py) 单元测试
policy.py 只依赖标准库 + nonebot.logger,故用 importlib 按文件路径裸加载
(与 test_group_daily_history_store.py 同法),避免触发插件包 __init__ 的
NoneBot 初始化。
"""
import importlib.util
import json
import sys
from pathlib import Path
import pytest
_MODULE_PATH = (
Path(__file__).resolve().parents[1]
/ "hexi"
/ "plugins"
/ "nonebot_plugin_video_analysis"
/ "policy.py"
)
@pytest.fixture(scope="module")
def policy():
"""以独立模块名加载 policy.py,避免与插件包 __init__ 冲突"""
spec = importlib.util.spec_from_file_location("video_policy_under_test", _MODULE_PATH)
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
yield module
sys.modules.pop(spec.name, None)
# ───────────────────────── 平台标签 ─────────────────────────
def test_normalize_platform_aliases(policy):
assert policy.normalize_platform("xhs") == "小红书"
assert policy.normalize_platform("小红书") == "小红书"
assert policy.normalize_platform("b23") == "哔哩哔哩"
assert policy.normalize_platform("B站") == "哔哩哔哩"
assert policy.normalize_platform("yt") == "Youtube"
assert policy.normalize_platform("twitter") == "X"
assert policy.normalize_platform(" X ") == "X"
assert policy.normalize_platform("微博") is None
assert policy.normalize_platform(None) is None
def test_normalize_platforms_dedupe_and_order(policy):
# 别名去重 + 按 PLATFORMS 顺序排列(与输入顺序无关)
assert policy.normalize_platforms(["x", "xhs", "twitter"]) == ["小红书", "X"]
# 字符串按分隔符拆
assert policy.normalize_platforms("小红书,xhs 抖音") == ["小红书", "抖音"]
# 非法值丢弃
assert policy.normalize_platforms(["微博", "xhs"]) == ["小红书"]
assert policy.normalize_platforms(None) == []
assert policy.normalize_platforms(123) == []
def test_apply_platform_diff_incremental(policy):
# 全部带 +/- → 增量
assert policy.apply_platform_diff([], ["+xhs"]) == ["小红书"]
assert policy.apply_platform_diff(["小红书"], ["+X"]) == ["小红书", "X"]
assert policy.apply_platform_diff(["小红书", "X"], ["-X"]) == ["小红书"]
assert policy.apply_platform_diff(["小红书"], ["+xhs"]) == ["小红书"] # 去重
assert policy.apply_platform_diff(["小红书"], ["-X"]) == ["小红书"] # 删不存在的
assert policy.apply_platform_diff([], ["+xhs", "+b23"]) == ["小红书", "哔哩哔哩"]
def test_apply_platform_diff_replace_and_invalid(policy):
# 出现裸平台名 → 整体覆盖
assert policy.apply_platform_diff(["小红书"], ["x"]) == ["X"]
assert policy.apply_platform_diff(["小红书"], []) is None
assert policy.apply_platform_diff(["小红书"], ["微博"]) is None
assert policy.apply_platform_diff(["小红书"], ["+微博"]) is None
def test_match_platform_domains(policy):
assert policy.match_platform("https://x.com/a/status/1") == "X"
assert policy.match_platform("https://twitter.com/a/status/1") == "X"
assert policy.match_platform("https://b23.tv/abc") == "哔哩哔哩"
assert policy.match_platform("https://v.douyin.com/abc") == "抖音"
assert policy.match_platform("https://youtu.be/abc") == "Youtube"
assert policy.match_platform("https://example.com/a.mp4") is None
# ───────────────────────── 策略模型 ─────────────────────────
def test_policy_from_dict_sanitizes(policy):
p = policy.Policy.from_dict(
{
"auto": 1,
"auto_link": ["xhs", "xhs", "微博"],
"ban_link": ["twitter"],
"plan": "b",
"upload_public": True,
"send_link": True,
"upload_group_file": True,
"unknown_key": "ignored",
}
)
assert p.auto is True
assert p.auto_link == ["小红书"]
assert p.ban_link == ["X"]
assert p.plan == "B"
assert p.to_dict()["upload_group_file"] is True
assert "unknown_key" not in p.to_dict()
def test_policy_from_dict_bad_values_fall_back(policy):
p = policy.Policy.from_dict({"plan": "Z", "auto_link": "b23"})
assert p.plan == "C" # 非法 plan 回退默认
assert p.auto_link == ["哔哩哔哩"]
# 空/非 dict 输入不炸
assert policy.Policy.from_dict(None) == policy.Policy()
assert policy.Policy.from_dict([1, 2]) == policy.Policy()
def test_sends_link_requires_public(policy):
assert policy.Policy(upload_public=True, send_link=True).sends_link is True
# 没开公网就没有链接可发
assert policy.Policy(upload_public=False, send_link=True).sends_link is False
def test_allows_group_file(policy):
off = policy.Policy(upload_group_file=False)
assert off.allows_group_file("抖音") is False
assert off.allows_group_file(None) is False
# 开了但没限定平台 → 所有平台(含识别不出的)都传
every = policy.Policy(upload_group_file=True)
assert every.allows_group_file("抖音") is True
assert every.allows_group_file(None) is True
# 限定了平台 → 只传清单里的,识别不出平台时按"不在清单里"处理
only_douyin = policy.Policy(
upload_group_file=True, group_file_platforms=["抖音"]
)
assert only_douyin.allows_group_file("抖音") is True
assert only_douyin.allows_group_file("X") is False
assert only_douyin.allows_group_file(None) is False
def test_policy_construct_normalizes(policy):
"""直接构造 Policy(...) 也要归一:绕过 from_dict 传别名不该静默失效"""
p = policy.Policy(
plan="z",
auto_link=["xhs"],
upload_group_file=True,
group_file_platforms=["douyin"],
)
assert p.plan == "C"
assert p.auto_link == ["小红书"]
assert p.group_file_platforms == ["抖音"]
assert p.allows_group_file("抖音") is True
def test_group_file_platforms_normalized(policy):
p = policy.Policy.from_dict(
{"group_file_platforms": ["douyin", "b23", "微博"], "upload_group_file": True}
)
# 别名归一成规范标签、非法值丢弃、顺序按 PLATFORMS
assert p.group_file_platforms == ["哔哩哔哩", "抖音"]
def test_banned_and_auto_matched(policy):
p = policy.Policy(auto_link=["小红书"], ban_link=["X"])
assert p.auto_matched("https://xhslink.com/a") == "小红书"
assert p.auto_matched("https://x.com/a") is None
assert p.banned("https://x.com/a") == "X"
assert p.banned("https://twitter.com/a") == "X"
assert p.banned("https://douyin.com/a") is None
# ───────────────────────── 迁移 ─────────────────────────
def test_migrate_v1_to_v3(policy):
data, changed = policy.migrate(
{
"WHITE_LIST": ["111", "222"],
"AUTO_ANALYSIS": ["111"],
"PLANA": ["111"],
"PLANB": ["222"],
"BLACK_LIST": ["999"],
}
)
assert changed is True
assert set(data["groups"]) == {"111", "222"}
# v1 的 plan=B 隐含公网,迁移后显式补上
assert data["groups"]["222"]["upload_public"] is True
assert data["groups"]["111"]["plan"] == "A"
assert data["groups"]["111"]["auto"] is True
assert data["blacklist"] == ["999"]
assert data["default"] == policy.Policy().to_dict()
def test_migrate_v2_plan_b_gets_public(policy):
data, changed = policy.migrate(
{
"groups": {
"111": {"auto": True, "plan": "B"},
"222": {"plan": "C"},
"333": {},
},
"blacklist": [],
}
)
assert changed is True
assert data["groups"]["111"]["upload_public"] is True
assert data["groups"]["222"]["upload_public"] is False
assert data["groups"]["333"]["auto"] is False
def test_migrate_keeps_explicit_public_flag(policy):
# 显式写了 upload_public 的条目不被迁移覆盖
data, _ = policy.migrate(
{"groups": {"111": {"plan": "B", "upload_public": False}}, "blacklist": []}
)
assert data["groups"]["111"]["upload_public"] is False
def test_migrate_idempotent(policy):
first, _ = policy.migrate({"groups": {"111": {"plan": "B"}}, "blacklist": []})
second, changed = policy.migrate(first)
assert changed is False
assert second == first
# ───────────────────────── 存储 ─────────────────────────
async def test_store_update_and_persist(policy, tmp_path):
path = tmp_path / "list.json"
store = policy.PolicyStore(path)
store.load()
assert path.exists() # 首次 load 落盘
await store.update_group("111", create=True, auto=True, plan="A")
await store.update_group("222", create=True, auto_link=["xhs"])
# 内存读
assert store.get("111").auto is True
assert store.get("222").auto_link == ["小红书"]
# 未配置的群回落 default
assert store.get("999") == store.default_policy()
assert store.is_whitelisted("111") is True
assert store.is_whitelisted("999") is False
on_disk = json.loads(path.read_text(encoding="utf-8"))
assert on_disk["groups"]["111"]["auto"] is True
assert on_disk["groups"]["222"]["auto_link"] == ["小红书"]
assert not path.with_name("list.json.tmp").exists()
async def test_store_update_requires_whitelist_unless_create(policy, tmp_path):
store = policy.PolicyStore(tmp_path / "list.json")
assert await store.update_group("111", auto=True) is None
assert store.is_whitelisted("111") is False
async def test_store_remove_group_and_blacklist(policy, tmp_path):
store = policy.PolicyStore(tmp_path / "list.json")
await store.update_group("111", create=True)
assert await store.remove_group("111") is True
assert await store.remove_group("111") is False
assert store.is_whitelisted("111") is False
assert await store.add_blacklist(999) is True
assert await store.add_blacklist("999") is False # 幂等(int/str 同一个人)
assert store.is_blacklisted(999) is True
assert store.blacklist() == ["999"]
assert await store.remove_blacklist("999") is True
assert store.blacklist() == []
async def test_store_concurrent_updates_keep_all(policy, tmp_path):
"""并发写不能丢群:每次写都从事件循环线程快照,写线程不碰共享状态"""
path = tmp_path / "list.json"
store = policy.PolicyStore(path)
store.load()
import asyncio
await asyncio.gather(
*(store.update_group(str(1000 + i), create=True, auto=True) for i in range(30))
)
on_disk = json.loads(path.read_text(encoding="utf-8"))
assert len(on_disk["groups"]) == 30
assert len(store.all_groups()) == 30
async def test_store_persists_group_file_platforms(policy, tmp_path):
"""POLICY_FIELDS 里漏了新字段的话这次更新会被静默丢掉"""
path = tmp_path / "list.json"
store = policy.PolicyStore(path)
await store.update_group(
"111", create=True, upload_group_file=True, group_file_platforms=["xhs"]
)
assert store.get("111").group_file_platforms == ["小红书"]
on_disk = json.loads(path.read_text(encoding="utf-8"))
assert on_disk["groups"]["111"]["group_file_platforms"] == ["小红书"]
assert on_disk["groups"]["111"]["upload_group_file"] is True
async def test_store_default_policy_edit(policy, tmp_path):
store = policy.PolicyStore(tmp_path / "list.json")
await store.set_default(policy.Policy(plan="B", upload_public=True))
assert store.default_policy().plan == "B"
assert store.default_policy().upload_public is True
# 群未配置时读到的就是 default
assert store.get("111").plan == "B"
+571
View File
@@ -0,0 +1,571 @@
"""Web 管理台解析任务 (services/web_jobs.py) 单元测试
模块本身只依赖标准库 + nonebot.logger,故用 importlib 按文件路径裸加载
(与 test_video_policy.py / test_video_group_file.py 同法)。fetchers 与
上传都是函数内延迟导入的注入点,这里全部塞假实现 —— 不碰网络、不碰 S3。
`platform_of` 走 `..policy` 的包内相对导入,裸加载下不可用,测试里一并替换。
"""
import asyncio
import importlib.util
import subprocess
import sys
import time
from pathlib import Path
import pytest
_MODULE_PATH = (
Path(__file__).resolve().parents[1]
/ "hexi"
/ "plugins"
/ "nonebot_plugin_video_analysis"
/ "services"
/ "web_jobs.py"
)
@pytest.fixture(scope="module")
def wj():
"""以独立模块名加载 web_jobs.py,避免与插件包 __init__ 冲突"""
spec = importlib.util.spec_from_file_location(
"video_web_jobs_under_test", _MODULE_PATH
)
module = importlib.util.module_from_spec(spec)
sys.modules[spec.name] = module
spec.loader.exec_module(module)
yield module
sys.modules.pop(spec.name, None)
@pytest.fixture
def manager(wj, monkeypatch):
"""造一个依赖全注入的 JobManager(fetch / upload / poster / policy 都传进来)"""
monkeypatch.setattr(wj, "platform_of", lambda url: "抖音")
def build(*, fetch, upload=None, poster=None, policy=None):
return wj.JobManager(
fetch=fetch,
upload=upload or _ok_upload,
poster=poster or _fake_poster,
policy=policy or (lambda: None),
)
return build
async def _fake_poster(video):
"""假抽帧:真的写一个封面文件出来(上传那一环会 stat 它)"""
out = video.with_name(f"{video.stem}_封面.jpg")
out.write_bytes(b"fake-jpeg")
return out
async def _no_poster(video):
return None
# ─────────────────────────── 假实现 ───────────────────────────
async def _ok_upload(path, policy):
return f"http://lan/{path.name}", None
def _touch(directory: Path, name: str, content: bytes = b"x") -> Path:
directory.mkdir(parents=True, exist_ok=True)
path = directory / name
path.write_bytes(content)
return path
def _ok_fetch(*paths, title="作品"):
async def fetch(url):
return title, [Path(p) for p in paths], len(paths) > 1
return fetch
async def _settle(mgr, timeout=5.0):
"""等所有任务进入终态,返回 list_jobs() 的结果。"""
deadline = time.monotonic() + timeout
while True:
jobs = mgr.list_jobs()
if jobs and all(not j.active for j in jobs):
return jobs
if time.monotonic() > deadline:
raise AssertionError("任务没有在超时内结束")
await asyncio.sleep(0.01)
# ─────────────────────── URL 提取 / 文件类型 ───────────────────────
def test_extract_urls_from_multiline_text(wj):
text = (
"看看这个 https://v.douyin.com/abc123。\n"
"还有 https://b23.tv/xyz,https://v.douyin.com/abc123(重复的只留一条)"
)
assert wj.extract_urls(text) == [
"https://v.douyin.com/abc123",
"https://b23.tv/xyz",
]
def test_extract_urls_keeps_query_string(wj):
"""小红书 xsec_token / 抖音带参数的分享链,query 不能被截掉"""
url = "https://www.xiaohongshu.com/explore/abc?xsec_token=XYZ&xsec_source=pc_feed"
assert wj.extract_urls(f"看这个 {url}") == [url]
def test_extract_urls_without_links(wj):
assert wj.extract_urls("这里没有链接,纯文本一段") == []
assert wj.extract_urls("") == []
def test_kind_of(wj):
assert wj.kind_of("001.JPG") == "image"
assert wj.kind_of(Path("作品.mp4")) == "video"
assert wj.kind_of("note.txt") == "file"
# ───────────────────────── 提交与复用 ─────────────────────────
async def test_same_url_reuses_active_job(wj, manager, tmp_path):
"""同 URL 连点两次不能起两个任务(抖音双开浏览器 + 落盘路径撞车)"""
calls: list[str] = []
async def fetch(url):
calls.append(url)
await asyncio.sleep(0.05) # 保持 running,让第二次提交能命中
return "作品", [_touch(tmp_path, "a.jpg")], False
mgr = manager(fetch=fetch)
url = "https://v.douyin.com/abc123"
first = (await mgr.submit([url]))[0]
second = (await mgr.submit([url]))[0]
assert first.id == second.id
assert first.platform == "抖音"
await _settle(mgr)
assert calls == [url]
# 跑完之后再提交同一条 → 是新任务(复用只针对在跑的)
third = (await mgr.submit([url]))[0]
assert third.id != first.id
async def test_force_creates_new_job(wj, manager, tmp_path):
calls: list[str] = []
async def fetch(url):
calls.append(url)
await asyncio.sleep(0.05)
return "作品", [_touch(tmp_path, "a.jpg")], False
mgr = manager(fetch=fetch)
url = "https://v.douyin.com/abc123"
old = (await mgr.submit([url]))[0]
new = (await mgr.submit([url], force=True))[0]
assert new.id != old.id
jobs = await _settle(mgr)
assert len(calls) == 2 and len(jobs) == 2
async def test_batch_submit_keeps_input_order(wj, manager, tmp_path):
mgr = manager(fetch=_ok_fetch(_touch(tmp_path, "a.jpg")))
jobs = await mgr.submit(["https://a/1", "https://a/2", "https://a/3"])
assert [j.url for j in jobs] == ["https://a/1", "https://a/2", "https://a/3"]
await _settle(mgr)
# ───────────────────────── 失败与超时 ─────────────────────────
async def test_no_media_marks_job_failed(wj, manager):
async def fetch(url):
return None, [], False
mgr = manager(fetch=fetch)
url = "https://v.douyin.com/gone"
(await mgr.submit([url]))[0]
job = (await _settle(mgr))[0]
assert job.status == "failed"
assert "无法解析到媒体" in job.error
assert job.title == url # 没标题时退回 URL,前端不至于只显示空白
async def test_fetch_exception_marks_job_failed(wj, manager):
async def fetch(url):
raise RuntimeError("被风控了")
mgr = manager(fetch=fetch)
(await mgr.submit(["https://v.douyin.com/x"]))[0]
job = (await _settle(mgr))[0]
assert job.status == "failed" and job.error == "被风控了"
async def test_job_timeout(wj, manager, monkeypatch):
monkeypatch.setattr(wj, "JOB_TIMEOUT", 0.05)
async def fetch(url):
await asyncio.sleep(5)
mgr = manager(fetch=fetch)
(await mgr.submit(["https://v.douyin.com/slow"]))[0]
job = (await _settle(mgr))[0]
assert job.status == "failed" and "解析超时" in job.error
# ───────────────────────── 上传 ─────────────────────────
async def test_single_upload_failure_does_not_fail_job(wj, manager, tmp_path):
"""多图作品挂一张 → 只标那一张,其余照常出链接"""
files = [_touch(tmp_path, "001.jpg"), _touch(tmp_path, "002.jpg")]
async def upload(path, policy):
if path.name == "002.jpg":
raise RuntimeError("S3 连接超时")
return "http://lan/001.jpg?sig=1", "https://pub/001.jpg"
mgr = manager(fetch=_ok_fetch(*files), upload=upload)
(await mgr.submit(["https://v.douyin.com/x"]))[0]
job = (await _settle(mgr))[0]
assert job.status == "done"
assert job.files[0]["local_url"].endswith("sig=1")
assert job.files[0]["public_url"] == "https://pub/001.jpg"
assert job.files[0]["error"] == ""
assert "S3 连接超时" in job.files[1]["error"]
assert job.files[1]["kind"] == "image" and job.files[1]["size"] == 1
async def test_upload_without_links_is_marked(wj, manager, tmp_path):
async def upload(path, policy):
return "", None # s3.py 失败时就是返回空串
mgr = manager(fetch=_ok_fetch(_touch(tmp_path, "a.mp4")), upload=upload)
(await mgr.submit(["https://v.douyin.com/x"]))[0]
job = (await _settle(mgr))[0]
assert job.status == "done" and "上传失败" in job.files[0]["error"]
assert job.files[0]["kind"] == "video"
async def test_upload_gets_default_policy(wj, manager, tmp_path):
"""存储链路按默认策略走:policy 原样传给 upload_with_plan"""
seen: list = []
sentinel = object()
async def upload(path, policy):
seen.append(policy)
return "http://lan/a.jpg", None
mgr = manager(
fetch=_ok_fetch(_touch(tmp_path, "a.jpg")),
upload=upload,
policy=lambda: sentinel,
)
(await mgr.submit(["https://v.douyin.com/x"]))[0]
await _settle(mgr)
assert seen == [sentinel]
# ───────────────────────── 刷新链接 ─────────────────────────
async def test_refresh_reruns_upload(wj, manager, tmp_path):
files = [_touch(tmp_path, "001.jpg"), _touch(tmp_path, "002.jpg")]
links = iter(
[
"http://lan/a?sig=1",
"http://lan/b?sig=1",
"http://lan/a?sig=2",
"http://lan/b?sig=2",
]
)
async def upload(path, policy):
return next(links), None
mgr = manager(fetch=_ok_fetch(*files), upload=upload)
job = (await mgr.submit(["https://v.douyin.com/x"]))[0]
job = (await _settle(mgr))[0]
assert [f["local_url"] for f in job.files] == [
"http://lan/a?sig=1",
"http://lan/b?sig=1",
]
refreshed = await mgr.refresh(job.id)
assert refreshed is not None
assert [f["local_url"] for f in refreshed.files] == [
"http://lan/a?sig=2",
"http://lan/b?sig=2",
]
async def test_refresh_marks_files_gone_from_disk(wj, manager, tmp_path):
"""temp 被 cleanup 清掉后再点刷新 → 只标文件没了,不炸"""
path = _touch(tmp_path, "001.jpg")
async def upload(p, policy):
return "http://lan/a.jpg", None
mgr = manager(fetch=_ok_fetch(path), upload=upload)
(await mgr.submit(["https://v.douyin.com/x"]))[0]
job = (await _settle(mgr))[0]
path.unlink()
refreshed = await mgr.refresh(job.id)
assert refreshed is not None
assert "本地文件不存在" in refreshed.files[0]["error"]
assert refreshed.files[0]["local_url"] == ""
async def test_refresh_unknown_job_returns_none(wj, manager):
mgr = manager(fetch=_ok_fetch())
assert await mgr.refresh("nope") is None
async def fetch(url):
return None, [], False
(await mgr.submit(["https://a/nothing"]))[0]
job = (await _settle(mgr))[0]
assert job.files == []
assert await mgr.refresh(job.id) is None
# ───────────────────────── 封面 ─────────────────────────
async def test_cover_is_first_image(wj, manager, tmp_path):
"""图文作品:封面直接取第一张图,不抽帧"""
poster_calls: list = []
async def poster(video):
poster_calls.append(video)
return await _fake_poster(video)
files = [_touch(tmp_path, "001.jpg"), _touch(tmp_path, "002.jpg")]
mgr = manager(fetch=_ok_fetch(*files), poster=poster)
(await mgr.submit(["https://v.douyin.com/x"]))[0]
job = (await _settle(mgr))[0]
assert job.cover_url == "http://lan/001.jpg"
assert poster_calls == []
async def test_cover_is_extracted_for_video_only_post(wj, manager, tmp_path):
"""纯视频作品:抽第一帧、上传、当封面"""
uploaded: list[str] = []
async def upload(path, policy):
uploaded.append(path.name)
return f"http://lan/{path.name}", None
mgr = manager(fetch=_ok_fetch(_touch(tmp_path, "作品.mp4")), upload=upload)
(await mgr.submit(["https://v.douyin.com/x"]))[0]
job = (await _settle(mgr))[0]
assert job.cover_url == "http://lan/作品_封面.jpg"
assert uploaded == ["作品.mp4", "作品_封面.jpg"]
assert job.poster_path is not None and job.poster_path.exists()
# 封面是单独的文件,不能混进 files(否则播放器里会多出一个"文件")
assert [f["name"] for f in job.files] == ["作品.mp4"]
async def test_cover_prefers_image_over_video_in_mixed_post(wj, manager, tmp_path):
"""混排(图 + 视频):按用户定的规则取第一张图"""
files = [_touch(tmp_path, "图文.jpg"), _touch(tmp_path, "动图.mp4")]
mgr = manager(fetch=_ok_fetch(*files))
(await mgr.submit(["https://v.douyin.com/x"]))[0]
job = (await _settle(mgr))[0]
assert job.cover_url == "http://lan/图文.jpg"
assert job.poster_path is None
async def test_poster_failure_keeps_job_done(wj, manager, tmp_path):
"""抽帧失败 → 封面留空,任务照常完成"""
mgr = manager(fetch=_ok_fetch(_touch(tmp_path, "作品.mp4")), poster=_no_poster)
(await mgr.submit(["https://v.douyin.com/x"]))[0]
job = (await _settle(mgr))[0]
assert job.status == "done"
assert job.cover_url == "" and job.poster_path is None
async def test_make_poster_extracts_real_frame(wj, tmp_path, monkeypatch):
"""真跑一次 ffmpeg:确认真能抽出 JPEG(假 poster 只覆盖接缝,覆盖不到命令本身)"""
ffmpeg = Path(__file__).resolve().parents[1] / ".venv" / "Scripts" / "ffmpeg.exe"
if not ffmpeg.exists():
pytest.skip("没有 .venv/Scripts/ffmpeg.exe")
monkeypatch.setattr(wj, "ffmpeg_path", lambda: str(ffmpeg))
video = tmp_path / "作品.mp4"
subprocess.run(
[str(ffmpeg), "-f", "lavfi", "-i", "testsrc=size=320x240:rate=10",
"-t", "2", "-pix_fmt", "yuv420p", "-y", str(video)],
capture_output=True, check=True,
)
poster = await wj.make_poster(video)
assert poster is not None
assert poster.name == "作品_封面.jpg"
assert poster.read_bytes()[:2] == b"\xff\xd8" # JPEG magic
async def test_make_poster_falls_back_to_first_frame(wj, tmp_path, monkeypatch):
"""不足 1 秒的视频取不到"第 1 秒"那一帧,要退回第 0 秒再试"""
ffmpeg = Path(__file__).resolve().parents[1] / ".venv" / "Scripts" / "ffmpeg.exe"
if not ffmpeg.exists():
pytest.skip("没有 .venv/Scripts/ffmpeg.exe")
monkeypatch.setattr(wj, "ffmpeg_path", lambda: str(ffmpeg))
video = tmp_path / "短片.mp4"
subprocess.run(
[str(ffmpeg), "-f", "lavfi", "-i", "testsrc=size=160x120:rate=10",
"-t", "0.5", "-pix_fmt", "yuv420p", "-y", str(video)],
capture_output=True, check=True,
)
poster = await wj.make_poster(video)
assert poster is not None and poster.stat().st_size > 0
async def test_make_poster_missing_video_returns_none(wj, tmp_path, monkeypatch):
ffmpeg = Path(__file__).resolve().parents[1] / ".venv" / "Scripts" / "ffmpeg.exe"
if not ffmpeg.exists():
pytest.skip("没有 .venv/Scripts/ffmpeg.exe")
monkeypatch.setattr(wj, "ffmpeg_path", lambda: str(ffmpeg))
assert await wj.make_poster(tmp_path / "不存在.mp4") is None
async def test_refresh_refreshes_cover(wj, manager, tmp_path):
"""刷新链接时封面一起换新(否则卡片封面 1 小时后变裂图)"""
links = iter(["http://lan/a?sig=1", "http://lan/p?sig=1", "http://lan/a?sig=2", "http://lan/p?sig=2"])
async def upload(path, policy):
return next(links), None
mgr = manager(fetch=_ok_fetch(_touch(tmp_path, "作品.mp4")), upload=upload)
(await mgr.submit(["https://v.douyin.com/x"]))[0]
job = (await _settle(mgr))[0]
assert job.cover_url == "http://lan/p?sig=1"
refreshed = await mgr.refresh(job.id)
assert refreshed is not None
assert refreshed.cover_url == "http://lan/p?sig=2"
# ───────────────────────── 清理 ─────────────────────────
async def test_remove_job(wj, manager, tmp_path):
mgr = manager(fetch=_ok_fetch(_touch(tmp_path, "a.jpg")))
job = (await mgr.submit(["https://v.douyin.com/x"]))[0]
await _settle(mgr)
assert mgr.remove(job.id) is True
assert mgr.list_jobs() == []
assert mgr.remove(job.id) is False # 再删一次:不存在
async def test_remove_running_job_cancels_it(wj, manager, tmp_path):
"""删掉在跑的任务要顺手取消,不能让它继续跑完再写回任务表"""
started = asyncio.Event()
cancelled = asyncio.Event()
async def fetch(url):
started.set()
try:
await asyncio.sleep(30)
except asyncio.CancelledError:
cancelled.set()
raise
return "作品", [_touch(tmp_path, "a.jpg")], False
mgr = manager(fetch=fetch)
job = (await mgr.submit(["https://v.douyin.com/x"]))[0]
await asyncio.wait_for(started.wait(), timeout=2)
assert mgr.remove(job.id) is True
await asyncio.wait_for(cancelled.wait(), timeout=2)
assert mgr.list_jobs() == []
async def test_clear_finished_keeps_running(wj, manager, tmp_path):
release = asyncio.Event()
async def fetch(url):
if "slow" in url:
await release.wait()
return "作品", [_touch(tmp_path, "a.jpg")], False
mgr = manager(fetch=fetch)
fast = (await mgr.submit(["https://v.douyin.com/fast"]))[0]
slow = (await mgr.submit(["https://v.douyin.com/slow"]))[0]
for _ in range(500):
if not fast.active:
break
await asyncio.sleep(0.01)
assert mgr.clear_finished() == 1
assert [j.id for j in mgr.list_jobs()] == [slow.id]
release.set()
await _settle(mgr)
# ───────────────────────── 上限与回收 ─────────────────────────
async def test_job_count_is_capped(wj, manager, tmp_path, monkeypatch):
monkeypatch.setattr(wj, "MAX_JOBS", 3)
mgr = manager(fetch=_ok_fetch(_touch(tmp_path, "a.jpg")))
await mgr.submit([f"https://v.douyin.com/{i}" for i in range(6)])
jobs = await _settle(mgr)
assert len(jobs) == 3
async def test_expired_terminal_jobs_are_pruned(wj, manager, tmp_path):
mgr = manager(fetch=_ok_fetch(_touch(tmp_path, "a.jpg")))
job = (await mgr.submit(["https://v.douyin.com/x"]))[0]
await _settle(mgr)
assert len(mgr.list_jobs()) == 1
job.updated_at -= wj.TERMINAL_TTL + 1
assert mgr.list_jobs() == []
async def test_running_jobs_survive_prune(wj, manager, monkeypatch):
"""在跑的任务不受上限/TTL 影响(TTL 设为负数,终态一律该清掉)"""
monkeypatch.setattr(wj, "TERMINAL_TTL", -1)
release = asyncio.Event()
async def fetch(url):
await release.wait()
return None, [], False
mgr = manager(fetch=fetch)
job = (await mgr.submit(["https://v.douyin.com/x"]))[0]
await asyncio.sleep(0.05)
assert [j.id for j in mgr.list_jobs()] == [job.id]
assert job.status == "running" and job.stage == "解析中"
release.set()
# 终态任务会被 TTL=-1 立即回收,所以这里盯 job 本身而不是 list_jobs()
for _ in range(500):
if not job.active:
break
await asyncio.sleep(0.01)
assert job.status == "failed"