feat(video-analysis): 抖音免浏览器解析链路 + 快手支持 + 小红书加固 + 发送防重复
- 抖音（2026-10-13）：删除 SSR + playwright 旧链路（douyin_ssr.py 移除， douyin_api.py 精简为下载工具），新增 douyin_web.py（x-tt-argus 网关头 + web detail 接口 + 最高码率梯/多 CDN 回退下载）与 douyin_sign.py （纯 Python a_bogus 签名，vendor 自 f2（Apache-2.0），依赖 gmssl SM3）； 实测 10/10 成功率、单作品 <1s（旧浏览器链路 5-15s 且随机撞验证码） - 快手（2026-10-08）：kuaishou_content.py 分享页 INIT_STATE 静态解析（无登录态）， 清晰度梯最佳档 + 图集 URL 组装，入口/策略/Web 面板同步接入 - 小红书：按链接来源定国际/国内站优先序、视频 backupUrls 逐候选回退、 图片 urlDefault→infoList WB_DFT→url 三档回退 - 长视频防重复发送（2026-10-13）：媒体发送带 per-call _timeout（默认 300s）， 超时=结果未知不再降级重试；auto/active 两条 handler 互斥 - Web 解析封面抽帧 ffmpeg 加 CREATE_NO_WINDOW（pythonw 下不再弹 cmd 黑窗） - 依赖：+gmssl；测试：新增 test_douyin_web / test_kuaishou_parse / test_video_cleanup， 扩充 test_video_policy / test_video_sender_link / test_video_group_file / test_video_media_naming Co-Authored-By: Claude Code <noreply@anthropic.com>
This commit is contained in:
sansenhoshiandClaude Code committed
1 parent
a31aab00e8
commit
a789a44f5d
25 files changed
+3536
-895
No files matched your search
@@ -224,7 +224,7 @@ def _build_rel_stem(
|
||||
"""
|
||||
return (
|
||||
f"{build_author_dir(author, author_id, source=source)}"
|
||||
f"/{build_work_stem(title)}"
|
||||
f"/{build_work_stem(title, source=source)}"
|
||||
)
|
||||
|
||||
|
||||
|
||||
@@ -1,309 +1,17 @@
|
||||
"""抖音内容抓取 — 浏览器自动化 + API 拦截 + 媒体下载"""
|
||||
"""媒体下载辅助 — 图集并发下载（抖音/B站/小红书/快手四条链路共用）
|
||||
|
||||
历史上的 playwright 拦截链路与分轨视频合并已随抖音新链路（douyin_web.py）
|
||||
废弃移除；这里只保留纯下载工具。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import subprocess
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import httpx
|
||||
from nonebot import logger
|
||||
from playwright.async_api import async_playwright
|
||||
|
||||
from ...models import DouyinFetchError
|
||||
from ...utils import get_temp_root, unique_media_path
|
||||
from .douyin_parser import (
|
||||
extract_trailing_digits,
|
||||
parse_animated_note_videos,
|
||||
parse_douyin_response,
|
||||
parse_note_images,
|
||||
parse_ssr_page,
|
||||
parse_video_urls,
|
||||
)
|
||||
|
||||
|
||||
def merge_video_audio(video_file: Path, audio_file: Path, out_file: Path):
|
||||
"""分轨视频与音频合并"""
|
||||
subprocess.run(
|
||||
[
|
||||
"ffmpeg",
|
||||
"-i",
|
||||
str(video_file),
|
||||
"-i",
|
||||
str(audio_file),
|
||||
"-c:v",
|
||||
"copy",
|
||||
"-c:a",
|
||||
"copy",
|
||||
"-movflags",
|
||||
"faststart",
|
||||
"-y",
|
||||
str(out_file),
|
||||
],
|
||||
check=True,
|
||||
)
|
||||
|
||||
|
||||
# ============================= 主入口 =============================
|
||||
|
||||
|
||||
async def fetch_douyin_content(
|
||||
douyin_url: str,
|
||||
cookies: List[dict],
|
||||
wait_seconds: int = 10,
|
||||
headless: bool = True,
|
||||
) -> tuple[str | None, Path | List[Path] | None]:
|
||||
"""
|
||||
获取抖音内容（视频或图文）
|
||||
|
||||
返回:
|
||||
(title, content)
|
||||
- 视频: (title, Path)
|
||||
- 图文: (title, List[Path])
|
||||
- 失败: (None, None)
|
||||
"""
|
||||
if not cookies:
|
||||
raise DouyinFetchError("cookies 为空")
|
||||
|
||||
tmp_root = get_temp_root("douyin")
|
||||
api_response: Optional[dict] = None
|
||||
api_response_favorite: Optional[dict] = None
|
||||
aweme_id = 0
|
||||
# 目标作品 id（从入口 URL 提取）：aweme/post 返回的是作者作品列表，
|
||||
# 按此 id 精确匹配要解析的作品，避免取到作者的其他作品
|
||||
target_aweme_id = extract_trailing_digits(douyin_url) or ""
|
||||
referer_url = None
|
||||
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.launch(
|
||||
executable_path="C:/Program Files/Google/Chrome/Application/chrome.exe",
|
||||
headless=headless,
|
||||
args=[
|
||||
"--autoplay-policy=no-user-gesture-required",
|
||||
"--disable-features=AutoplayDisableSuppression",
|
||||
],
|
||||
)
|
||||
|
||||
context = await browser.new_context(
|
||||
viewport={"width": 1280, "height": 720},
|
||||
device_scale_factor=2,
|
||||
user_agent=(
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/122.0.0.0 Safari/537.36"
|
||||
),
|
||||
locale="zh-CN",
|
||||
)
|
||||
|
||||
await context.add_cookies(cookies)
|
||||
page = await context.new_page()
|
||||
|
||||
async def handle_response(response):
|
||||
nonlocal api_response, api_response_favorite, aweme_id, referer_url
|
||||
try:
|
||||
if (
|
||||
"https://www.douyin.com/note" in response.url
|
||||
or "https://www.douyin.com/video" in response.url
|
||||
):
|
||||
referer_url = response.url
|
||||
logger.info(f"作品链接：{referer_url}")
|
||||
aweme_id = extract_trailing_digits(response.url)
|
||||
logger.info(f"作品id：{aweme_id}")
|
||||
|
||||
if "aweme/v1/web/aweme/detail" in response.url:
|
||||
if not api_response:
|
||||
logger.info(f"捕获到视频 API: {response.url}")
|
||||
ct = response.headers.get("content-type", "")
|
||||
if "json" in ct:
|
||||
api_response = await response.json()
|
||||
logger.info("API 响应已捕获")
|
||||
return
|
||||
else:
|
||||
logger.info("API 已有捕获")
|
||||
return
|
||||
|
||||
elif "aweme/v1/web/aweme/post" in response.url:
|
||||
logger.info(f"捕获到图文 API: {response.url}")
|
||||
ct = response.headers.get("content-type", "")
|
||||
if "json" in ct:
|
||||
data = await response.json()
|
||||
aweme_list = data.get("aweme_list", [])
|
||||
if aweme_list:
|
||||
# 列表是作者的全部作品，优先按目标作品 id 精确匹配
|
||||
# （未登录/风控时目标作品可能不在列表里）
|
||||
match_id = aweme_id or target_aweme_id
|
||||
target_aweme = None
|
||||
if match_id:
|
||||
target_aweme = next(
|
||||
(
|
||||
a
|
||||
for a in aweme_list
|
||||
if str(a.get("aweme_id")) == str(match_id)
|
||||
),
|
||||
None,
|
||||
)
|
||||
if not target_aweme:
|
||||
target_aweme = aweme_list[0]
|
||||
if target_aweme.get("images"):
|
||||
logger.info("找到正确的图文 API 响应（包含图片数据）")
|
||||
data["aweme_list"] = [target_aweme]
|
||||
api_response = data
|
||||
return
|
||||
else:
|
||||
logger.warning(
|
||||
"此响应不包含有效的图片数据，等待下一个请求"
|
||||
)
|
||||
except Exception as e:
|
||||
logger.warning(f"处理响应失败: {e}")
|
||||
|
||||
page.on("response", handle_response)
|
||||
|
||||
await page.goto(douyin_url, wait_until="domcontentloaded")
|
||||
|
||||
max_wait = wait_seconds
|
||||
for i in range(max_wait):
|
||||
if api_response:
|
||||
logger.info(f"成功在第 {i + 1} 秒捕获 API 响应")
|
||||
break
|
||||
await page.wait_for_timeout(1000)
|
||||
|
||||
# 截取页面 HTML（在关闭浏览器前），用于 SSR 回退解析
|
||||
page_html = await page.content() if not api_response else None
|
||||
|
||||
await browser.close()
|
||||
|
||||
if not api_response and page_html:
|
||||
api_response = parse_ssr_page(page_html)
|
||||
if api_response:
|
||||
logger.info("通过 SSR 页面回退解析获取到图文数据")
|
||||
|
||||
if not api_response:
|
||||
raise DouyinFetchError("无法捕获 API 响应，请检查网络或 URL")
|
||||
|
||||
parsed = parse_douyin_response(api_response, referer_url)
|
||||
|
||||
headers = {
|
||||
"Referer": "https://www.douyin.com/",
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/122.0.0.0 Safari/537.36"
|
||||
),
|
||||
}
|
||||
|
||||
if parsed.media_type == "视频":
|
||||
content = await _process_video(
|
||||
api_response, tmp_root, parsed.rel_stem, aweme_id, headers
|
||||
)
|
||||
return parsed.raw_title, content
|
||||
|
||||
elif parsed.media_type == "图片":
|
||||
# 先解析 images 列表，区分纯动图和图文/图+视频
|
||||
images_urls, video_url = parse_note_images(
|
||||
api_response, api_response_favorite, aweme_id
|
||||
)
|
||||
if images_urls:
|
||||
# 有图片（纯图文 或 图+视频混合作品）
|
||||
content = await _process_note_with_parsed(
|
||||
images_urls,
|
||||
video_url,
|
||||
tmp_root,
|
||||
parsed.rel_stem,
|
||||
headers,
|
||||
)
|
||||
else:
|
||||
# 纯动图（所有项都是视频）
|
||||
content = await _process_animated_note(
|
||||
api_response, tmp_root, parsed.rel_stem, headers
|
||||
)
|
||||
return parsed.raw_title, content
|
||||
|
||||
return None, None
|
||||
|
||||
|
||||
# ============================= 视频下载 =============================
|
||||
|
||||
|
||||
async def _process_video(
|
||||
api_response: dict,
|
||||
tmp_root: Path,
|
||||
rel_stem: str,
|
||||
aweme_id: str,
|
||||
headers: Dict[str, str],
|
||||
) -> Path:
|
||||
"""处理视频内容，返回本地文件路径
|
||||
|
||||
rel_stem 是相对平台根的路径词干 `{作者目录}/{作品名}`（见 ParsedDouyinContent.rel_stem）。
|
||||
"""
|
||||
groups = parse_video_urls(api_response)
|
||||
|
||||
best_group = None
|
||||
for g in groups.values():
|
||||
if g["FULL"] or (g["VIDEO"] and g["AUDIO"]):
|
||||
best_group = g
|
||||
break
|
||||
|
||||
if not best_group:
|
||||
raise DouyinFetchError("没有可用的视频组合")
|
||||
|
||||
# 完整视频 — 流式下载
|
||||
if best_group["FULL"]:
|
||||
full = best_group["FULL"]
|
||||
logger.info(f"找到完整视频，数量: {len(full)}")
|
||||
best = max(full, key=lambda x: x["br"])
|
||||
logger.info(f"选择码率: {best['br']} - {best['url'][:60]}...")
|
||||
|
||||
output_path = unique_media_path(tmp_root / f"{rel_stem}.mp4")
|
||||
async with httpx.AsyncClient(headers=headers) as client:
|
||||
async with client.stream("GET", best["url"]) as resp:
|
||||
resp.raise_for_status()
|
||||
with open(output_path, "wb") as f:
|
||||
async for chunk in resp.aiter_bytes(8192):
|
||||
f.write(chunk)
|
||||
logger.info(f"视频下载完成: {output_path}")
|
||||
return output_path
|
||||
|
||||
# 分轨视频 — 分别下载后合并
|
||||
logger.info(
|
||||
f"使用分轨模式，视频数: {len(best_group['VIDEO'])}, "
|
||||
f"音频数: {len(best_group['AUDIO'])}"
|
||||
)
|
||||
video = max(best_group["VIDEO"], key=lambda x: x["br"])
|
||||
audio = max(best_group["AUDIO"], key=lambda x: x["br"])
|
||||
logger.info(f"选择视频码率: {video['br']}")
|
||||
logger.info(f"选择音频码率: {audio['br']}")
|
||||
|
||||
# 先定下产物名（顺带建好作者目录），分轨中间文件与产物同目录
|
||||
output_path = unique_media_path(tmp_root / f"{rel_stem}.mp4")
|
||||
video_path = output_path.parent / f"{output_path.stem}_v.mp4"
|
||||
audio_path = output_path.parent / f"{output_path.stem}_a.mp4"
|
||||
|
||||
async with httpx.AsyncClient(headers=headers) as client:
|
||||
logger.info("开始下载视频...")
|
||||
async with client.stream("GET", video["url"]) as v:
|
||||
v.raise_for_status()
|
||||
with open(video_path, "wb") as f:
|
||||
async for chunk in v.aiter_bytes(8192):
|
||||
f.write(chunk)
|
||||
|
||||
logger.info("开始下载音频...")
|
||||
async with client.stream("GET", audio["url"]) as a:
|
||||
a.raise_for_status()
|
||||
with open(audio_path, "wb") as f:
|
||||
async for chunk in a.aiter_bytes(8192):
|
||||
f.write(chunk)
|
||||
|
||||
logger.info("合并视频和音频...")
|
||||
# ffmpeg 是同步子进程，直接 await 不了：不丢线程池会卡住整个事件循环
|
||||
# （合并期间 Web 轮询、群消息全都停摆）
|
||||
await asyncio.to_thread(merge_video_audio, video_path, audio_path, output_path)
|
||||
video_path.unlink()
|
||||
audio_path.unlink()
|
||||
|
||||
logger.info(f"视频下载完成: {output_path}")
|
||||
return output_path
|
||||
|
||||
|
||||
# ============================= 图文下载 =============================
|
||||
from ...utils import unique_media_path
|
||||
|
||||
|
||||
async def _process_note_with_parsed(
|
||||
@@ -313,41 +21,49 @@ async def _process_note_with_parsed(
|
||||
rel_stem: str,
|
||||
headers: Dict[str, str],
|
||||
) -> List[Path]:
|
||||
"""根据已解析的图片/视频 URL 列表，并行下载
|
||||
"""按已解析的图片/视频 URL 列表并发下载
|
||||
|
||||
一个作品一个目录：`{平台根}/{作者目录}/{作品名}[_{短码}]/001.jpg…`
|
||||
- images_urls：每项是该媒体位的 URL 候选列表（按优先级排序，逐个回退）
|
||||
- video_url：图+视频混合作品中追加到末尾的视频（单 URL，可为 None）
|
||||
"""
|
||||
note_dir = unique_media_path(tmp_root / rel_stem)
|
||||
note_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"图文保存目录: {note_dir}")
|
||||
|
||||
async def _download_one(idx: int, url: str, ext: str = "") -> Optional[Path]:
|
||||
if not ext:
|
||||
ext = _infer_extension(url)
|
||||
filename = f"{idx:03d}{ext}"
|
||||
filepath = note_dir / filename
|
||||
try:
|
||||
async with httpx.AsyncClient(headers=headers) as client:
|
||||
async with client.stream("GET", url) as resp:
|
||||
resp.raise_for_status()
|
||||
with open(filepath, "wb") as f:
|
||||
async for chunk in resp.aiter_bytes(8192):
|
||||
f.write(chunk)
|
||||
logger.info(f"已保存: {filename}")
|
||||
return filepath
|
||||
except Exception as e:
|
||||
logger.error(f"下载 {idx} 失败: {e}")
|
||||
return None
|
||||
async def _download_one(
|
||||
idx: int, url_list: List[str], ext: str = ""
|
||||
) -> Optional[Path]:
|
||||
for url in url_list:
|
||||
use_ext = ext or _infer_extension(url)
|
||||
filename = f"{idx:03d}{use_ext}"
|
||||
filepath = note_dir / filename
|
||||
try:
|
||||
async with httpx.AsyncClient(headers=headers, timeout=300) as client:
|
||||
async with client.stream("GET", url) as resp:
|
||||
resp.raise_for_status()
|
||||
with open(filepath, "wb") as f:
|
||||
async for chunk in resp.aiter_bytes(65536):
|
||||
f.write(chunk)
|
||||
if filepath.stat().st_size == 0:
|
||||
raise RuntimeError("响应为空（0 字节）")
|
||||
logger.info(f"已保存: {filename}")
|
||||
return filepath
|
||||
except Exception as e:
|
||||
logger.warning(f"下载 {idx} 失败（尝试下一个候选）: {e}")
|
||||
logger.error(f"下载 {idx} 失败: 所有候选 URL 均不可用")
|
||||
return None
|
||||
|
||||
tasks = [
|
||||
_download_one(i, url_list[0])
|
||||
_download_one(i, url_list)
|
||||
for i, url_list in enumerate(images_urls, 1)
|
||||
if url_list
|
||||
]
|
||||
|
||||
# 图+视频混合作品：视频追加到下载任务
|
||||
if video_url:
|
||||
next_idx = len(images_urls) + 1
|
||||
tasks.append(_download_one(next_idx, video_url, ext=".mp4"))
|
||||
tasks.append(_download_one(next_idx, [video_url], ext=".mp4"))
|
||||
|
||||
results = await asyncio.gather(*tasks)
|
||||
saved_paths: List[Path] = [p for p in results if p is not None]
|
||||
@@ -356,75 +72,9 @@ async def _process_note_with_parsed(
|
||||
return saved_paths
|
||||
|
||||
|
||||
async def _process_note(
|
||||
api_response: dict,
|
||||
api_response_favorite: dict,
|
||||
tmp_root: Path,
|
||||
rel_stem: str,
|
||||
aweme_id: str,
|
||||
headers: Dict[str, str],
|
||||
) -> List[Path]:
|
||||
"""处理图文内容，并行下载所有图片；图+视频混合作品同时下载视频"""
|
||||
images_urls, video_url = parse_note_images(
|
||||
api_response, api_response_favorite, aweme_id
|
||||
)
|
||||
logger.info(f"解析到的图片链接:{images_urls}")
|
||||
|
||||
if not images_urls and not video_url:
|
||||
raise DouyinFetchError("未找到图文链接")
|
||||
|
||||
return await _process_note_with_parsed(
|
||||
images_urls, video_url, tmp_root, rel_stem, headers
|
||||
)
|
||||
|
||||
|
||||
# ============================= 动图下载 =============================
|
||||
|
||||
|
||||
async def _process_animated_note(
|
||||
api_response: dict,
|
||||
tmp_root: Path,
|
||||
rel_stem: str,
|
||||
headers: Dict[str, str],
|
||||
) -> List[Path]:
|
||||
"""处理动图内容（media_type=42），并行下载所有无声 mp4 视频"""
|
||||
video_urls = parse_animated_note_videos(api_response)
|
||||
logger.info(f"解析到的动图视频链接: {video_urls}")
|
||||
|
||||
note_dir = unique_media_path(tmp_root / rel_stem)
|
||||
note_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"动图保存目录: {note_dir}")
|
||||
|
||||
async def _download_one(idx: int, url: str) -> Optional[Path]:
|
||||
filename = f"{idx:03d}.mp4"
|
||||
filepath = note_dir / filename
|
||||
try:
|
||||
async with httpx.AsyncClient(headers=headers) as client:
|
||||
async with client.stream("GET", url) as resp:
|
||||
resp.raise_for_status()
|
||||
with open(filepath, "wb") as f:
|
||||
async for chunk in resp.aiter_bytes(8192):
|
||||
f.write(chunk)
|
||||
logger.info(f"动图视频已保存: {filename}")
|
||||
return filepath
|
||||
except Exception as e:
|
||||
logger.error(f"下载动图视频 {idx} 失败: {e}")
|
||||
return None
|
||||
|
||||
tasks = [
|
||||
_download_one(i, url)
|
||||
for i, url in enumerate(video_urls, 1)
|
||||
]
|
||||
results = await asyncio.gather(*tasks)
|
||||
saved_paths: List[Path] = [p for p in results if p is not None]
|
||||
|
||||
logger.info(f"动图下载完成，共 {len(saved_paths)} 个视频")
|
||||
return saved_paths
|
||||
|
||||
|
||||
def _infer_extension(url: str) -> str:
|
||||
"""从 URL 推断文件扩展名"""
|
||||
extensions = [".webp", ".jpg", ".jpeg", ".png", ".gif", ".avif"]
|
||||
extensions = [".webp", ".jpg", ".jpeg", ".png", ".gif", ".avif", ".heic"]
|
||||
url_lower = url.lower()
|
||||
for ext in extensions:
|
||||
if ext in url_lower:
|
||||
|
||||
@@ -418,7 +418,7 @@ def parse_douyin_response(
|
||||
author_dir = build_author_dir(
|
||||
raw_nickname, raw_author_id, source=referer_url or aweme_id
|
||||
)
|
||||
file_name = build_work_stem(raw_title)
|
||||
file_name = build_work_stem(raw_title, source=referer_url or aweme_id)
|
||||
|
||||
logger.info(
|
||||
f"内容标题: {raw_title}, 作者: {raw_nickname}({raw_author_id}), "
|
||||
|
||||
@@ -0,0 +1,929 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
"""抖音 a_bogus 签名 — 纯 Python 实现（免浏览器/免 Node）。
|
||||
|
||||
来源：https://github.com/Johnserf-Seed/f2 （Apache-2.0 License，
|
||||
许可证全文见 https://github.com/Johnserf-Seed/f2/blob/main/LICENSE）
|
||||
文件 f2/utils/crypto/bytedance/abogus.py（2026-10-13 拉取，未做任何修改；
|
||||
原文件头部的 f2 内部依赖仅有标准库与 gmssl，vendor 后可直接使用）。
|
||||
|
||||
与上游保持一致：算法对应 bdms 1.0.1.19，2026-10 实测可过抖音 ArgusSecurityPlugin 网关。
|
||||
若抖音再次更换签名算法，更新点就是本文件（对照 f2 上游 diff）。
|
||||
|
||||
依赖：gmssl（pip install gmssl，纯 Python 的国密 SM3 实现）。
|
||||
"""
|
||||
# fmt: off
|
||||
# ruff: noqa
|
||||
# path: f2/utils/crypto/bytedance/abogus.py
|
||||
|
||||
#!/usr/bin/env python
|
||||
# -*- encoding: utf-8 -*-
|
||||
"""
|
||||
@Description:abogus.py
|
||||
@Date :2024/06/16 11:21:14
|
||||
@Author :JohnserfSeed
|
||||
@version :0.0.4
|
||||
@License :Apache License 2.0
|
||||
@Github :https://github.com/johnserf-seed
|
||||
@Mail :support@f2.wiki
|
||||
-------------------------------------------------
|
||||
Change Log :
|
||||
2024/06/16 17:27:47 - Create ABogus algorithm & black style
|
||||
2024/06/16 17:27:47 - Limit custom ua late open source full version
|
||||
2024/07/08 21:50:12 - Open the full version of the custom UA
|
||||
2025/03/05 15:55:42 - Perf Post Method Generate
|
||||
2025/04/27 17:15:26 - Fix mypy type error
|
||||
-------------------------------------------------
|
||||
"""
|
||||
|
||||
import random
|
||||
import time
|
||||
from typing import Callable, Dict, List, Union
|
||||
|
||||
from gmssl import func, sm3 # type: ignore[import-untyped]
|
||||
|
||||
|
||||
class StringProcessor:
|
||||
"""
|
||||
StringProcessor 类用于计算ABogus算法中所需的字符串处理方法。
|
||||
包括字符串和 ASCII 码之间的转换、无符号右移运算等。
|
||||
|
||||
类方法:
|
||||
|
||||
to_ord_str(s: str) -> str:
|
||||
将字符串转换为 ASCII 码字符串。
|
||||
|
||||
to_ord_array(s: str) -> List[int]:
|
||||
将字符串转换为 ASCII 码列表。
|
||||
|
||||
to_char_str(s: str) -> str:
|
||||
将 ASCII 码列表转换为字符串。
|
||||
|
||||
to_char_array(s: str) -> List[int]:
|
||||
将字符串转换为 ASCII 码列表。
|
||||
|
||||
js_shift_right(val: int, n: int) -> int:
|
||||
实现 JavaScript 中的无符号右移运算。
|
||||
|
||||
generate_random_bytes(length: int = 3) -> str:
|
||||
生成一组伪随机字节字符串，用于混淆数据。
|
||||
|
||||
使用示例:
|
||||
```python
|
||||
# 将字符串转换为 ASCII 码字符串
|
||||
ord_str = StringProcessor.to_ord_str("Hello, World!")
|
||||
print(ord_str)
|
||||
|
||||
# 将字符串转换为 ASCII 码列表
|
||||
ord_array = StringProcessor.to_ord_array("Hello, World!")
|
||||
print(ord_array)
|
||||
|
||||
# 将 ASCII 码列表转换为字符串
|
||||
char_str = StringProcessor.to_char_str(ord_array)
|
||||
print(char_str)
|
||||
|
||||
# 将字符串转换为 ASCII 码列表
|
||||
char_array = StringProcessor.to_char_array("Hello, World!")
|
||||
print(char_array)
|
||||
|
||||
# 实现 JavaScript 中的无符号右移运算
|
||||
shifted_val = StringProcessor.js_shift_right(10, 2)
|
||||
print(shifted_val)
|
||||
|
||||
# 生成一组伪随机字节字符串
|
||||
random_bytes = StringProcessor.generate_random_bytes(3)
|
||||
print(random_bytes)
|
||||
```
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def to_ord_str(s: str) -> str:
|
||||
"""
|
||||
将字符串转换为 ASCII 码字符串 (Convert a string to an ASCII code string).
|
||||
|
||||
Args:
|
||||
s (str): 输入字符串 (Input string).
|
||||
|
||||
Returns:
|
||||
str: 转换后的 ASCII 码字符串 (Converted ASCII code string).
|
||||
"""
|
||||
return "".join([chr(ord(char)) for char in s])
|
||||
|
||||
@staticmethod
|
||||
def to_ord_array(s: str) -> List[int]:
|
||||
"""
|
||||
将字符串转换为 ASCII 码列表 (Convert a string to a list of ASCII codes).
|
||||
|
||||
Args:
|
||||
s (str): 输入字符串 (Input string).
|
||||
|
||||
Returns:
|
||||
List[int]: 转换后的 ASCII 码列表 (Converted list of ASCII codes).
|
||||
"""
|
||||
return [ord(char) for char in s]
|
||||
|
||||
@staticmethod
|
||||
def to_char_str(s: Union[str, List[int]]) -> str:
|
||||
"""
|
||||
将 ASCII 码列表转换为字符串 (Convert a list of ASCII codes to a string).
|
||||
|
||||
Args:
|
||||
s (Union[str, List[int]]): ASCII 码列表或字符串 (List of ASCII codes or string).
|
||||
|
||||
Returns:
|
||||
str: 转换后的字符串 (Converted string).
|
||||
"""
|
||||
if isinstance(s, list):
|
||||
return "".join([chr(i) for i in s])
|
||||
elif isinstance(s, str):
|
||||
return s
|
||||
else:
|
||||
raise TypeError(
|
||||
"Expected str or List[int], got {}".format(type(s).__name__)
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def to_char_array(s: str) -> List[int]:
|
||||
"""
|
||||
将字符串转换为 ASCII 码列表 (Convert a string to a list of ASCII codes).
|
||||
|
||||
Args:
|
||||
s (str): 输入字符串 (Input string).
|
||||
|
||||
Returns:
|
||||
List[int]: 转换后的 ASCII 码列表 (Converted list of ASCII codes).
|
||||
"""
|
||||
return [ord(char) for char in s]
|
||||
|
||||
@staticmethod
|
||||
def js_shift_right(val: int, n: int) -> int:
|
||||
"""
|
||||
实现 JavaScript 中的无符号右移运算 (Implement the unsigned right shift operation in JavaScript).
|
||||
|
||||
Args:
|
||||
val (int): 输入值 (Input value).
|
||||
n (int): 右移位数 (Number of bits to shift right).
|
||||
|
||||
Returns:
|
||||
int: 右移后的值 (Value after right shift).
|
||||
"""
|
||||
return (val % 0x100000000) >> n
|
||||
|
||||
@staticmethod
|
||||
def generate_random_bytes(length: int = 3) -> str:
|
||||
"""
|
||||
生成一组伪随机字节字符串，用于混淆数据 (Generate a pseudo-random byte string to obfuscate the data).
|
||||
|
||||
Args:
|
||||
length (int): 生成的字节序列长度 (Length of the byte sequence to generate).
|
||||
|
||||
Returns:
|
||||
str: 生成的伪随机字节字符串 (Generated pseudo-random byte string).
|
||||
"""
|
||||
|
||||
def generate_byte_sequence() -> List[str]:
|
||||
_rd = int(random.random() * 10000)
|
||||
return [
|
||||
chr(((_rd & 255) & 170) | 1),
|
||||
chr(((_rd & 255) & 85) | 2),
|
||||
chr((StringProcessor.js_shift_right(_rd, 8) & 170) | 5),
|
||||
chr((StringProcessor.js_shift_right(_rd, 8) & 85) | 40),
|
||||
]
|
||||
|
||||
result = []
|
||||
for _ in range(length):
|
||||
result.extend(generate_byte_sequence())
|
||||
|
||||
return "".join(result)
|
||||
|
||||
|
||||
class CryptoUtility:
|
||||
"""
|
||||
CryptoUtility 类用于提供加密和编码的工具方法，包括 SM3 哈希、添加盐值、Base64 编码和 RC4 加密等。
|
||||
|
||||
类属性:
|
||||
salt (str): 加密盐值 (Encryption salt).
|
||||
base64_alphabet (List[str]): 自定义 Base64 字符表 (Custom Base64 alphabet).
|
||||
|
||||
类方法:
|
||||
sm3_to_array(input_data: Union[str, List[int]]) -> List[int]:
|
||||
计算请求体的 SM3 哈希值，并将结果转换为整数数组。
|
||||
|
||||
add_salt(param: str) -> str:
|
||||
为字符串参数添加盐值。
|
||||
|
||||
process_param(param: Union[str, List[int]], add_salt: bool) -> Union[str, List[int]]:
|
||||
处理输入参数，根据需要添加盐值。
|
||||
|
||||
params_to_array(param: Union[str, List[int]], add_salt: bool = True) -> List[int]:
|
||||
获取输入参数的哈希数组。
|
||||
|
||||
transform_bytes(bytes_list: List[int]) -> str:
|
||||
对输入的字节列表进行加密/解密操作，返回处理后的字符串。
|
||||
|
||||
base64_encode(input_string: str, selected_alphabet: int = 0) -> str:
|
||||
使用自定义字符表对输入字符串进行 Base64 编码。
|
||||
|
||||
abogus_encode(abogus_bytes_str: str, selected_alphabet: int) -> str:
|
||||
对输入的字节字符串进行自定义 Base64 编码，并添加位移和填充。
|
||||
|
||||
rc4_encrypt(key: bytes, plaintext: str) -> bytes:
|
||||
使用 RC4 算法加密数据。
|
||||
|
||||
使用示例:
|
||||
```python
|
||||
# 计算请求体的 SM3 哈希值
|
||||
sm3_hash = CryptoUtility.sm3_to_array("Hello, World!")
|
||||
print(sm3_hash)
|
||||
|
||||
# 为字符串参数添加盐值
|
||||
salted_param = CryptoUtility.add_salt("Hello, World!")
|
||||
print(salted_param)
|
||||
|
||||
# 获取输入参数的哈希数组
|
||||
hash_array = CryptoUtility.params_to_array("Hello, World!")
|
||||
print(hash_array)
|
||||
|
||||
# 对输入的字节列表进行加密/解密操作
|
||||
encrypted_str = CryptoUtility.transform_bytes([72, 101, 108, 108, 111, 44, 32, 87, 111, 114, 108, 100, 33])
|
||||
print(encrypted_str)
|
||||
|
||||
# 使用自定义字符表对输入字符串进行 Base64 编码
|
||||
base64_str = CryptoUtility.base64_encode("Hello, World!")
|
||||
print(base64_str)
|
||||
|
||||
# 对输入的字节字符串进行自定义 Base64 编码，并添加位移和填充
|
||||
abogus_str = CryptoUtility.abogus_encode("Hello, World!", 0)
|
||||
print(abogus_str)
|
||||
|
||||
# 使用 RC4 算法加密数据
|
||||
key = b"key"
|
||||
plaintext = "Hello, World!"
|
||||
ciphertext = CryptoUtility.rc4_encrypt(key, plaintext)
|
||||
print(ciphertext)
|
||||
```
|
||||
"""
|
||||
|
||||
def __init__(self, salt: str, custom_base64_alphabet: List[str]):
|
||||
"""
|
||||
初始化 CryptoUtility 类
|
||||
Initialize the CryptoUtility class.
|
||||
|
||||
Args:
|
||||
salt (str): 加密盐值 (Encryption salt).
|
||||
custom_base64_alphabet (List[str]): 自定义 Base64 字符表 (Custom Base64 alphabet).
|
||||
"""
|
||||
self.salt = salt
|
||||
self.base64_alphabet = custom_base64_alphabet
|
||||
|
||||
# fmt: off
|
||||
self.big_array = [
|
||||
121, 243, 55, 234, 103, 36, 47, 228, 30, 231, 106, 6, 115, 95, 78, 101, 250, 207, 198, 50,
|
||||
139, 227, 220, 105, 97, 143, 34, 28, 194, 215, 18, 100, 159, 160, 43, 8, 169, 217, 180, 120,
|
||||
247, 45, 90, 11, 27, 197, 46, 3, 84, 72, 5, 68, 62, 56, 221, 75, 144, 79, 73, 161,
|
||||
178, 81, 64, 187, 134, 117, 186, 118, 16, 241, 130, 71, 89, 147, 122, 129, 65, 40, 88, 150,
|
||||
110, 219, 199, 255, 181, 254, 48, 4, 195, 248, 208, 32, 116, 167, 69, 201, 17, 124, 125, 104,
|
||||
96, 83, 80, 127, 236, 108, 154, 126, 204, 15, 20, 135, 112, 158, 13, 1, 188, 164, 210, 237,
|
||||
222, 98, 212, 77, 253, 42, 170, 202, 26, 22, 29, 182, 251, 10, 173, 152, 58, 138, 54, 141,
|
||||
185, 33, 157, 31, 252, 132, 233, 235, 102, 196, 191, 223, 240, 148, 39, 123, 92, 82, 128, 109,
|
||||
57, 24, 38, 113, 209, 245, 2, 119, 153, 229, 189, 214, 230, 174, 232, 63, 52, 205, 86, 140,
|
||||
66, 175, 111, 171, 246, 133, 238, 193, 99, 60, 74, 91, 225, 51, 76, 37, 145, 211, 166, 151,
|
||||
213, 206, 0, 200, 244, 176, 218, 44, 184, 172, 49, 216, 93, 168, 53, 21, 183, 41, 67, 85,
|
||||
224, 155, 226, 242, 87, 177, 146, 70, 190, 12, 162, 19, 137, 114, 25, 165, 163, 192, 23, 59,
|
||||
9, 94, 179, 107, 35, 7, 142, 131, 239, 203, 149, 136, 61, 249, 14, 156
|
||||
]
|
||||
# fmt: on
|
||||
|
||||
@staticmethod
|
||||
def sm3_to_array(input_data: Union[str, List[int]]) -> List[int]:
|
||||
"""
|
||||
计算请求体的 SM3 哈希值，并将结果转换为整数数组 (Calculate the SM3 hash value of the request body and convert the result to an array of integers).
|
||||
|
||||
Args:
|
||||
input_data (Union[str, List[int]]): 输入数据 (Input data).
|
||||
|
||||
Returns:
|
||||
List[int]: 哈希值的整数数组 (Array of integers representing the hash value).
|
||||
"""
|
||||
# 如果输入是字符串，则将其编码为字节数组
|
||||
if isinstance(input_data, str):
|
||||
input_data_bytes = input_data.encode("utf-8")
|
||||
else:
|
||||
input_data_bytes = bytes(input_data) # 将 List[int] 转换为字节数组
|
||||
|
||||
# 将字节数组转换为适合 sm3.sm3_hash 函数处理的列表格式
|
||||
hex_result = sm3.sm3_hash(func.bytes_to_list(input_data_bytes))
|
||||
|
||||
# 将十六进制字符串结果转换为十进制整数列表
|
||||
return [int(hex_result[i : i + 2], 16) for i in range(0, len(hex_result), 2)]
|
||||
|
||||
def add_salt(self, param: str) -> str:
|
||||
"""
|
||||
为字符串参数添加盐值 (Add salt to the string parameter).
|
||||
|
||||
Args:
|
||||
param (str): 输入字符串 (Input string).
|
||||
|
||||
Returns:
|
||||
str: 添加盐值后的字符串 (String with added salt).
|
||||
"""
|
||||
return param + self.salt
|
||||
|
||||
def process_param(
|
||||
self, param: Union[str, List[int]], add_salt: bool
|
||||
) -> Union[str, List[int]]:
|
||||
"""
|
||||
处理输入参数，根据需要添加盐值 (Process input parameter and add salt if needed).
|
||||
|
||||
Args:
|
||||
param (Union[str, List[int]]): 输入参数 (Input parameter).
|
||||
add_salt (bool): 是否添加盐值 (Whether to add salt).
|
||||
|
||||
Returns:
|
||||
Union[str, List[int]]: 处理后的参数 (Processed parameter).
|
||||
"""
|
||||
if isinstance(param, str) and add_salt:
|
||||
param = self.add_salt(param)
|
||||
return param
|
||||
|
||||
def params_to_array(
|
||||
self, param: Union[str, List[int]], add_salt: bool = True
|
||||
) -> List[int]:
|
||||
"""
|
||||
获取输入参数的哈希数组 (Get the hash array of the input parameter).
|
||||
|
||||
Args:
|
||||
param (Union[str, List[int]]): 输入参数 (Input parameter).
|
||||
add_salt (bool): 是否添加盐值 (Whether to add salt).
|
||||
|
||||
Returns:
|
||||
List[int]: 哈希数组 (Hash array).
|
||||
"""
|
||||
processed_param = self.process_param(param, add_salt)
|
||||
return self.sm3_to_array(processed_param)
|
||||
|
||||
def transform_bytes(self, bytes_list: List[int]) -> str:
|
||||
"""
|
||||
对输入的字节列表进行加密/解密操作，返回处理后的字符串 (Encrypt/decrypt the input byte list and return the processed string).
|
||||
|
||||
Args:
|
||||
bytes_list (List[int]): 输入的字节列表 (Input byte list).
|
||||
|
||||
Returns:
|
||||
str: 处理后的字符串 (Processed string).
|
||||
"""
|
||||
# 将字节列表转换为字符字符串
|
||||
bytes_str = StringProcessor.to_char_str(bytes_list)
|
||||
result_str = []
|
||||
index_b = self.big_array[1]
|
||||
initial_value = 0
|
||||
value_e = 0
|
||||
|
||||
for index, char in enumerate(bytes_str):
|
||||
if index == 0:
|
||||
initial_value = self.big_array[index_b]
|
||||
sum_initial = index_b + initial_value
|
||||
|
||||
self.big_array[1] = initial_value
|
||||
self.big_array[index_b] = index_b
|
||||
else:
|
||||
sum_initial = initial_value + value_e
|
||||
|
||||
char_value = ord(char)
|
||||
sum_initial %= len(self.big_array)
|
||||
value_f = self.big_array[sum_initial]
|
||||
encrypted_char = char_value ^ value_f
|
||||
result_str.append(chr(encrypted_char))
|
||||
|
||||
# 交换数组元素
|
||||
value_e = self.big_array[(index + 2) % len(self.big_array)]
|
||||
sum_initial = (index_b + value_e) % len(self.big_array)
|
||||
initial_value = self.big_array[sum_initial]
|
||||
self.big_array[sum_initial] = self.big_array[
|
||||
(index + 2) % len(self.big_array)
|
||||
]
|
||||
self.big_array[(index + 2) % len(self.big_array)] = initial_value
|
||||
index_b = sum_initial
|
||||
|
||||
return "".join(result_str)
|
||||
|
||||
def base64_encode(self, input_string: str, selected_alphabet: int = 0) -> str:
|
||||
"""
|
||||
使用自定义字符表对输入字符串进行 Base64 编码 (Encode the input string using a custom Base64 alphabet).
|
||||
|
||||
Args:
|
||||
input_string (str): 输入字符串 (Input string).
|
||||
selected_alphabet (int): 选择的自定义 Base64 字符表索引 (Selected custom Base64 alphabet index).
|
||||
|
||||
Returns:
|
||||
str: 编码后的字符串 (Encoded string).
|
||||
"""
|
||||
|
||||
# 将输入字符串转换为ASCII码的二进制形式
|
||||
binary_string = "".join(["{:08b}".format(ord(char)) for char in input_string])
|
||||
|
||||
# 补全二进制字符串使其长度为6的倍数
|
||||
padding_length = (6 - len(binary_string) % 6) % 6
|
||||
binary_string += "0" * padding_length
|
||||
|
||||
# 将二进制字符串分割为6位一组
|
||||
base64_indices = [
|
||||
int(binary_string[i : i + 6], 2) for i in range(0, len(binary_string), 6)
|
||||
]
|
||||
|
||||
# 根据自定义字符表生成输出字符串
|
||||
output_string = "".join(
|
||||
[self.base64_alphabet[selected_alphabet][index] for index in base64_indices]
|
||||
)
|
||||
|
||||
# 添加等号填充，使符合 Base64 编码规范
|
||||
output_string += "=" * (padding_length // 2)
|
||||
|
||||
return output_string
|
||||
|
||||
def abogus_encode(self, abogus_bytes_str: str, selected_alphabet: int) -> str:
|
||||
"""
|
||||
对输入的字节字符串进行自定义 Base64 编码，并添加位移和填充 (Encode the input byte string using a custom Base64 alphabet, and add shifts and padding).
|
||||
|
||||
Args:
|
||||
abogus_bytes_str (str): 输入的字节字符串 (Input byte string).
|
||||
selected_alphabet (int): 选择的自定义 Base64 字符表索引 (Selected custom Base64 alphabet index).
|
||||
|
||||
Returns:
|
||||
str: 编码后的字符串 (Encoded string).
|
||||
"""
|
||||
abogus = []
|
||||
|
||||
for i in range(0, len(abogus_bytes_str), 3):
|
||||
if i + 2 < len(abogus_bytes_str):
|
||||
n = (
|
||||
(ord(abogus_bytes_str[i]) << 16)
|
||||
| (ord(abogus_bytes_str[i + 1]) << 8)
|
||||
| ord(abogus_bytes_str[i + 2])
|
||||
)
|
||||
elif i + 1 < len(abogus_bytes_str):
|
||||
n = (ord(abogus_bytes_str[i]) << 16) | (
|
||||
ord(abogus_bytes_str[i + 1]) << 8
|
||||
)
|
||||
else:
|
||||
n = ord(abogus_bytes_str[i]) << 16
|
||||
|
||||
for j, k in zip(range(18, -1, -6), (0xFC0000, 0x03F000, 0x0FC0, 0x3F)):
|
||||
if j == 6 and i + 1 >= len(abogus_bytes_str):
|
||||
break
|
||||
if j == 0 and i + 2 >= len(abogus_bytes_str):
|
||||
break
|
||||
abogus.append(self.base64_alphabet[selected_alphabet][(n & k) >> j])
|
||||
|
||||
abogus.append("=" * ((4 - len(abogus) % 4) % 4))
|
||||
return "".join(abogus)
|
||||
|
||||
@staticmethod
|
||||
def rc4_encrypt(key: bytes, plaintext: str) -> bytes:
|
||||
"""
|
||||
使用 RC4 算法加密数据 (Encrypt data using the RC4 algorithm).
|
||||
|
||||
Args:
|
||||
key (bytes): 加密密钥 (Encryption key).
|
||||
plaintext (str): 明文数据 (Plaintext data).
|
||||
|
||||
Returns:
|
||||
bytes: 加密后的数据 (Encrypted data).
|
||||
"""
|
||||
S = list(range(256))
|
||||
j = 0
|
||||
for i in range(256):
|
||||
j = (j + S[i] + key[i % len(key)]) % 256
|
||||
S[i], S[j] = S[j], S[i]
|
||||
|
||||
i = j = 0
|
||||
ciphertext = []
|
||||
for char in plaintext:
|
||||
i = (i + 1) % 256
|
||||
j = (j + S[i]) % 256
|
||||
S[i], S[j] = S[j], S[i]
|
||||
K = S[(S[i] + S[j]) % 256]
|
||||
ciphertext.append(ord(char) ^ K)
|
||||
|
||||
return bytes(ciphertext)
|
||||
|
||||
|
||||
class BrowserFingerprintGenerator:
|
||||
"""
|
||||
BrowserFingerprintGenerator 用于生成模拟的浏览器指纹信息，用于在不同浏览器环境中进行测试和数据采集。
|
||||
|
||||
方法:
|
||||
generate_fingerprint(browser_type="Edge"):
|
||||
根据指定的浏览器类型生成浏览器指纹。
|
||||
|
||||
generate_chrome_fingerprint():
|
||||
生成 Chrome 浏览器指纹。
|
||||
|
||||
generate_firefox_fingerprint():
|
||||
生成 Firefox 浏览器指纹。
|
||||
|
||||
generate_safari_fingerprint():
|
||||
生成 Safari 浏览器指纹。
|
||||
|
||||
generate_edge_fingerprint():
|
||||
生成 Edge 浏览器指纹。
|
||||
|
||||
_generate_fingerprint(platform="Win32"):
|
||||
根据给定的参数生成浏览器指纹字符串。
|
||||
|
||||
使用示例:
|
||||
```python
|
||||
chrome_fp = BrowserFingerprintGenerator.generate_fingerprint("Chrome")
|
||||
print(chrome_fp)
|
||||
```
|
||||
"""
|
||||
|
||||
@classmethod
|
||||
def generate_fingerprint(cls, browser_type: str = "Edge") -> str:
|
||||
"""
|
||||
根据指定的浏览器类型生成浏览器指纹。 (Generate a browser fingerprint based on the specified browser type.)
|
||||
|
||||
Args:
|
||||
browser_type (str): 浏览器类型 (Browser type).
|
||||
|
||||
Returns:
|
||||
str: 生成的浏览器指纹字符串 (Generated browser fingerprint string).
|
||||
"""
|
||||
browsers: Dict[str, Callable[[], str]] = {
|
||||
"Chrome": cls.generate_chrome_fingerprint,
|
||||
"Firefox": cls.generate_firefox_fingerprint,
|
||||
"Safari": cls.generate_safari_fingerprint,
|
||||
"Edge": cls.generate_edge_fingerprint,
|
||||
}
|
||||
return browsers.get(browser_type, cls.generate_chrome_fingerprint)()
|
||||
|
||||
@classmethod
|
||||
def generate_chrome_fingerprint(cls) -> str:
|
||||
return cls._generate_fingerprint(platform="Win32")
|
||||
|
||||
@classmethod
|
||||
def generate_firefox_fingerprint(cls) -> str:
|
||||
return cls._generate_fingerprint(platform="Win32")
|
||||
|
||||
@classmethod
|
||||
def generate_safari_fingerprint(cls) -> str:
|
||||
return cls._generate_fingerprint(platform="MacIntel")
|
||||
|
||||
@classmethod
|
||||
def generate_edge_fingerprint(cls) -> str:
|
||||
return cls._generate_fingerprint(platform="Win32")
|
||||
|
||||
@staticmethod
|
||||
def _generate_fingerprint(platform: str) -> str:
|
||||
"""
|
||||
根据给定的参数生成浏览器指纹字符串。 (Generate a browser fingerprint string based on the given parameters.)
|
||||
|
||||
Args:
|
||||
platform (str): 操作系统平台 (Operating system platform).
|
||||
|
||||
Returns:
|
||||
str: 生成的浏览器指纹字符串 (Generated browser fingerprint string).
|
||||
"""
|
||||
inner_width = random.randint(1024, 1920)
|
||||
inner_height = random.randint(768, 1080)
|
||||
outer_width = inner_width + random.randint(24, 32)
|
||||
outer_height = inner_height + random.randint(75, 90)
|
||||
screen_x = 0
|
||||
screen_y = random.choice([0, 30])
|
||||
size_width = random.randint(1024, 1920)
|
||||
size_height = random.randint(768, 1080)
|
||||
avail_width = random.randint(1280, 1920)
|
||||
avail_height = random.randint(800, 1080)
|
||||
|
||||
fingerprint = (
|
||||
f"{inner_width}|{inner_height}|{outer_width}|{outer_height}|"
|
||||
f"{screen_x}|{screen_y}|0|0|{size_width}|{size_height}|"
|
||||
f"{avail_width}|{avail_height}|{inner_width}|{inner_height}|24|24|{platform}"
|
||||
)
|
||||
return fingerprint
|
||||
|
||||
|
||||
class ABogus:
|
||||
"""
|
||||
ABogus 类用于生成 ABogus 参数。
|
||||
|
||||
类属性:
|
||||
array1 (List[int]): 加密请求体 (Encrypted request body).
|
||||
array2 (List[int]): 加密请求头 (Encrypted request header).
|
||||
array3 (List[int]): 加密 UA (Encrypted User-Agent).
|
||||
aid (int): AID 值 (AID value).
|
||||
pageId (int): 页面 ID (Page ID).
|
||||
salt (str): 加密盐值 (Encryption salt).
|
||||
options (List[int]): 请求选项 (Request options).
|
||||
ua_key (bytes): UA 加密密钥 (UA encryption key).
|
||||
character (str): 自定义 Base64 字符表 (Custom Base64 alphabet).
|
||||
character2 (str): 自定义 Base64 字符表 (Custom Base64 alphabet).
|
||||
character_list (List[str]): 自定义 Base64 字符表列表 (List of custom Base64 alphabets).
|
||||
crypto_utility (CryptoUtility): 加密工具类 (Encryption utility).
|
||||
user_agent (str): 自定义 UA (Custom User-Agent).
|
||||
browser_fp (str): 浏览器指纹 (Browser fingerprint).
|
||||
sort_index (List[int]): 排序索引 (Sort index).
|
||||
sort_index_2 (List[int]): 排序索引 (Sort index).
|
||||
|
||||
说明：
|
||||
options 参数用于指定请求的类型，GET 请求使用 [0, 1, 8]，POST 请求使用 [0, 1, 14]。14兼容8，POST同样可以编码params，故写死。
|
||||
(The options parameter is used to specify the type of request. GET requests use [0, 1, 8], and POST requests use [0, 1, 14]. 14 is compatible with 8, and POST can also encode params, so it is hardcoded.)
|
||||
|
||||
方法:
|
||||
encode_data(data: str, alphabet_index: int = 0) -> str:
|
||||
使用指定的字符表对数据进行 Base64 编码 (Encode the data using the specified Base64 alphabet).
|
||||
|
||||
generate_abogus(params: str, request: str = "") -> str:
|
||||
生成 ABogus 参数 (Generate the ABogus parameter).
|
||||
|
||||
使用示例:
|
||||
```python
|
||||
# 生成 ABogus 参数，置空使用默认 UA 和浏览器指纹
|
||||
abogus = ABogus(user_agent="xxx", fp="xxx")
|
||||
abogus_param = abogus.generate_abogus("device_platform=webapp&aid=6383&channel=channel_pc_web&aweme_id=7380308675841297704……省略……")
|
||||
print(abogus_param[1])
|
||||
```
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
fp: str = "",
|
||||
user_agent: str = "",
|
||||
options: List[int] = [0, 1, 14],
|
||||
):
|
||||
self.aid = 6383
|
||||
self.pageId = 0 # 1.0.1.19 -> 6241
|
||||
self.salt = "cus" # 1.0.1.19 -> 加密盐 # dhzx
|
||||
self.boe = False
|
||||
self.ddrt = 8.5
|
||||
self.ic = 8.5
|
||||
self.paths = [
|
||||
"^/webcast/",
|
||||
"^/aweme/v1/",
|
||||
"^/aweme/v2/",
|
||||
"/v1/message/send",
|
||||
"^/live/",
|
||||
"^/captcha/",
|
||||
"^/ecom/",
|
||||
]
|
||||
self.array1: list[int] = [] # 加密请求体
|
||||
self.array2: list[int] = [] # 加密请求头 为空
|
||||
self.array3: list[int] = [] # 加密UA
|
||||
self.options = options # GET [0, 1, 8] POST [0, 1, 14]
|
||||
self.ua_key = b"\x00\x01\x0e" # ua加密key
|
||||
|
||||
self.character = (
|
||||
"Dkdpgh2ZmsQB80/MfvV36XI1R45-WUAlEixNLwoqYTOPuzKFjJnry79HbGcaStCe"
|
||||
)
|
||||
self.character2 = (
|
||||
"ckdp1h4ZKsUB80/Mfvw36XIgR25+WQAlEi7NLboqYTOPuzmFjJnryx9HVGDaStCe"
|
||||
)
|
||||
self.character_list = [self.character, self.character2] # 自定义base64字符表
|
||||
|
||||
self.crypto_utility = CryptoUtility(
|
||||
self.salt, self.character_list
|
||||
) # 加密工具类
|
||||
|
||||
self.user_agent = (
|
||||
user_agent
|
||||
if user_agent is not None and user_agent != ""
|
||||
else "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0"
|
||||
) # 自定义ua，为空则设置一个默认ua
|
||||
|
||||
self.browser_fp = (
|
||||
fp
|
||||
if fp is not None and fp != ""
|
||||
else BrowserFingerprintGenerator.generate_fingerprint("Edge")
|
||||
) # 自定义浏览器指纹，为空则生成Edge指纹
|
||||
|
||||
# fmt: off
|
||||
self.sort_index = [
|
||||
18, 20, 52, 26, 30, 34, 58, 38, 40, 53, 42, 21, 27, 54, 55, 31, 35, 57, 39, 41, 43, 22, 28,
|
||||
32, 60, 36, 23, 29, 33, 37, 44, 45, 59, 46, 47, 48, 49, 50, 24, 25, 65, 66, 70, 71
|
||||
]
|
||||
self.sort_index_2 = [
|
||||
18, 20, 26, 30, 34, 38, 40, 42, 21, 27, 31, 35, 39, 41, 43, 22, 28, 32, 36, 23, 29, 33, 37,
|
||||
44, 45, 46, 47, 48, 49, 50, 24, 25, 52, 53, 54, 55, 57, 58, 59, 60, 65, 66, 70, 71
|
||||
]
|
||||
# fmt: on
|
||||
|
||||
def encode_data(self, data: str, alphabet_index: int = 0) -> str:
|
||||
"""
|
||||
使用指定的字符表对数据进行 Base64 编码 (Encode the data using the specified Base64 alphabet).
|
||||
|
||||
Args:
|
||||
data (str): 输入数据 (Input data).
|
||||
alphabet_index (int): 自定义字符表索引 (Custom alphabet index).
|
||||
|
||||
Returns:
|
||||
str: 编码后的数据 (Encoded data).
|
||||
"""
|
||||
return self.crypto_utility.abogus_encode(data, alphabet_index)
|
||||
|
||||
def generate_abogus(self, params: str, body: str = "") -> tuple:
|
||||
"""
|
||||
生成 abogus 参数 (Generate the ABogus parameter).
|
||||
|
||||
Args:
|
||||
params (str): 请求参数 (Request parameters).
|
||||
body (str): 请求体，GET接口则为空 (Request body, empty for GET interfaces).
|
||||
|
||||
Returns:
|
||||
tuple: params 生成的 abogus 参数 和 ua (ABogus parameter generated by params and ua).
|
||||
"""
|
||||
ab_dir = {
|
||||
8: 3, # 固定
|
||||
15: {
|
||||
"aid": self.aid,
|
||||
"pageId": self.pageId,
|
||||
"boe": self.boe,
|
||||
"ddrt": self.ddrt,
|
||||
"paths": self.paths,
|
||||
"track": {"mode": 0, "delay": 300, "paths": []},
|
||||
"dump": True,
|
||||
"rpU": "",
|
||||
},
|
||||
18: 44,
|
||||
19: [1, 0, 1, 0, 1],
|
||||
66: 0, # 固定
|
||||
69: 0, # 固定
|
||||
70: 0, # 固定
|
||||
71: 0, # 固定
|
||||
}
|
||||
|
||||
# 开始加密时间
|
||||
start_encryption = int(time.time() * 1000)
|
||||
|
||||
# params参数加盐加密
|
||||
array1 = self.crypto_utility.params_to_array(
|
||||
self.crypto_utility.params_to_array(params)
|
||||
)
|
||||
array2 = self.crypto_utility.params_to_array(
|
||||
self.crypto_utility.params_to_array(body)
|
||||
)
|
||||
array3 = self.crypto_utility.params_to_array(
|
||||
self.crypto_utility.base64_encode(
|
||||
"".join(
|
||||
[
|
||||
chr(b)
|
||||
for b in self.crypto_utility.rc4_encrypt(
|
||||
self.ua_key, self.user_agent
|
||||
)
|
||||
]
|
||||
),
|
||||
1,
|
||||
),
|
||||
add_salt=False,
|
||||
)
|
||||
|
||||
# 结束加密时间
|
||||
end_encryption = int(time.time() * 1000)
|
||||
|
||||
# 插入加密开始时间
|
||||
ab_dir[20] = (start_encryption >> 24) & 255
|
||||
ab_dir[21] = (start_encryption >> 16) & 255
|
||||
ab_dir[22] = (start_encryption >> 8) & 255
|
||||
ab_dir[23] = start_encryption & 255
|
||||
ab_dir[24] = int(start_encryption / 256 / 256 / 256 / 256) >> 0
|
||||
ab_dir[25] = int(start_encryption / 256 / 256 / 256 / 256 / 256) >> 0
|
||||
|
||||
# 插入请求头配置
|
||||
ab_dir[26] = (self.options[0] >> 24) & 255
|
||||
ab_dir[27] = (self.options[0] >> 16) & 255
|
||||
ab_dir[28] = (self.options[0] >> 8) & 255
|
||||
ab_dir[29] = self.options[0] & 255
|
||||
|
||||
# 插入请求方法
|
||||
ab_dir[30] = int(self.options[1] / 256) & 255
|
||||
ab_dir[31] = (self.options[1] % 256) & 255
|
||||
ab_dir[32] = (self.options[1] >> 24) & 255
|
||||
ab_dir[33] = (self.options[1] >> 16) & 255
|
||||
|
||||
# 插入请求头加密
|
||||
ab_dir[34] = (self.options[2] >> 24) & 255
|
||||
ab_dir[35] = (self.options[2] >> 16) & 255
|
||||
ab_dir[36] = (self.options[2] >> 8) & 255
|
||||
ab_dir[37] = self.options[2] & 255
|
||||
|
||||
# 插入请求体加密
|
||||
ab_dir[38] = array1[21]
|
||||
ab_dir[39] = array1[22]
|
||||
# 插入body加密
|
||||
ab_dir[40] = array2[21]
|
||||
ab_dir[41] = array2[22]
|
||||
# 插入ua加密
|
||||
ab_dir[42] = array3[23]
|
||||
ab_dir[43] = array3[24]
|
||||
|
||||
# 插入加密结束时间
|
||||
ab_dir[44] = (end_encryption >> 24) & 255
|
||||
ab_dir[45] = (end_encryption >> 16) & 255
|
||||
ab_dir[46] = (end_encryption >> 8) & 255
|
||||
ab_dir[47] = end_encryption & 255
|
||||
ab_dir[48] = ab_dir[8]
|
||||
ab_dir[49] = int(end_encryption / 256 / 256 / 256 / 256) >> 0
|
||||
ab_dir[50] = int(end_encryption / 256 / 256 / 256 / 256 / 256) >> 0
|
||||
|
||||
# 插入固定值
|
||||
ab_dir[51] = (self.pageId >> 24) & 255
|
||||
ab_dir[52] = (self.pageId >> 16) & 255
|
||||
ab_dir[53] = (self.pageId >> 8) & 255
|
||||
ab_dir[54] = self.pageId & 255
|
||||
ab_dir[55] = self.pageId
|
||||
ab_dir[56] = self.aid
|
||||
ab_dir[57] = self.aid & 255
|
||||
ab_dir[58] = (self.aid >> 8) & 255
|
||||
ab_dir[59] = (self.aid >> 16) & 255
|
||||
ab_dir[60] = (self.aid >> 24) & 255
|
||||
|
||||
# 插入浏览器指纹
|
||||
ab_dir[64] = len(self.browser_fp)
|
||||
ab_dir[65] = len(self.browser_fp)
|
||||
|
||||
# 获取 ab_dir 中 sort_index 的值
|
||||
sorted_values: List[int] = []
|
||||
for i in self.sort_index:
|
||||
value = ab_dir.get(i, 0)
|
||||
if isinstance(value, (int, float)):
|
||||
sorted_values.append(int(value))
|
||||
elif isinstance(value, str) and value.isdigit():
|
||||
sorted_values.append(int(value))
|
||||
else:
|
||||
sorted_values.append(0) # 对于不能转换为整数的值使用0作为默认值
|
||||
|
||||
# 将浏览器指纹转换为 ASCII 码列表
|
||||
edge_fp_array = StringProcessor.to_char_array(self.browser_fp)
|
||||
|
||||
# 将浏览器指纹长度的低 8 位作为异或值
|
||||
ab_xor = (len(self.browser_fp) & 255) >> 8 & 255
|
||||
|
||||
# 进行异或计算
|
||||
for index in range(len(self.sort_index_2) - 1):
|
||||
if index == 0:
|
||||
# 防御性编程，确保返回的是整数
|
||||
value = ab_dir.get(self.sort_index_2[index], 0)
|
||||
if isinstance(value, (int, float)):
|
||||
ab_xor = int(value)
|
||||
else:
|
||||
ab_xor = 0
|
||||
# 同样防御性地处理第二个值
|
||||
value = ab_dir.get(self.sort_index_2[index + 1], 0)
|
||||
if isinstance(value, (int, float)):
|
||||
ab_xor ^= int(value)
|
||||
|
||||
# 安全转换edge_fp_array中的值为整数
|
||||
clean_edge_fp = []
|
||||
for x in edge_fp_array:
|
||||
if isinstance(x, (int, float, str)) and str(x).isdigit():
|
||||
clean_edge_fp.append(int(x))
|
||||
else:
|
||||
# 对于无法转换的值使用0填充
|
||||
clean_edge_fp.append(0)
|
||||
|
||||
sorted_values.extend(clean_edge_fp)
|
||||
sorted_values.append(ab_xor)
|
||||
|
||||
abogus_bytes_str = (
|
||||
StringProcessor.generate_random_bytes()
|
||||
+ self.crypto_utility.transform_bytes(sorted_values)
|
||||
)
|
||||
|
||||
abogus = self.crypto_utility.abogus_encode(abogus_bytes_str, 0)
|
||||
params = "%s&a_bogus=%s" % (params, abogus)
|
||||
return (params, abogus, self.user_agent, body)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# 24/06/16 晚点开源自定义ua
|
||||
# 24/07/08 支持自定义ua和浏览器指纹
|
||||
# 24/11/15 完成1.0.1.19版本abogus算法，择日开源
|
||||
# 25/03/05 修复POST请求参数加密错误，修补环境
|
||||
# 25/04/27 修复mypy检查错误
|
||||
|
||||
user_agent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.36 Edg/131.0.0.0"
|
||||
chrome_fp = BrowserFingerprintGenerator.generate_fingerprint("Edge")
|
||||
abogus = ABogus(user_agent=user_agent, fp=chrome_fp)
|
||||
|
||||
# GET
|
||||
url = "https://www.douyin.com/aweme/v1/web/aweme/detail/?"
|
||||
params = "device_platform=webapp&aid=6383&channel=channel_pc_web&sec_user_id=MS4wLjABAAAArDVBosPJF3eIWVEFp0szuJ-e1V_-rK0ieJeWwpE77E8&max_cursor=0&locate_query=false&show_live_replay_strategy=1&need_time_list=1&time_list_query=0&whale_cut_token=&cut_version=1&count=18&publish_video_strategy_type=2&from_user_page=1&update_version_code=170400&pc_client_type=1&pc_libra_divert=Windows&support_h265=1&support_dash=0&version_code=290100&version_name=29.1.0&cookie_enabled=true&screen_width=1920&screen_height=1080&browser_language=zh-CN&browser_platform=Win32&browser_name=Edge&browser_version=131.0.0.0&browser_online=true&engine_name=Blink&engine_version=131.0.0.0&os_name=Windows&os_version=10&cpu_core_num=12&device_memory=8&platform=PC&downlink=10&effective_type=4g&round_trip_time=50"
|
||||
body = ""
|
||||
print(url + abogus.generate_abogus(params=params, body=body)[0])
|
||||
|
||||
# POST
|
||||
url = "https://www.douyin.com/aweme/v2/web/aweme/stats/?"
|
||||
params = "device_platform=webapp&aid=6383&channel=channel_pc_web&pc_client_type=1&pc_libra_divert=Windows&update_version_code=170400&support_h265=1&support_dash=0&version_code=170400&version_name=17.4.0&cookie_enabled=true&screen_width=1920&screen_height=1080&browser_language=zh-CN&browser_platform=Win32&browser_name=Edge&browser_version=131.0.0.0&browser_online=true&engine_name=Blink&engine_version=131.0.0.0&os_name=Windows&os_version=10&cpu_core_num=12&device_memory=8&platform=PC&downlink=10&effective_type=4g&round_trip_time=50"
|
||||
body = "aweme_type=0&item_id=7467485482314763572&play_delta=1&source=0"
|
||||
print(url + abogus.generate_abogus(params=params, body=body)[0])
|
||||
|
||||
# # 测试生成100个abogus参数 和 100个指纹所需时间
|
||||
# start = time.time()
|
||||
# for _ in range(100):
|
||||
# abogus.generate_abogus(params=params, body=body)
|
||||
# end = time.time()
|
||||
# print("生成100个abogus参数和指纹所需时间:", end - start) # 生成100个abogus参数和指纹所需时间: 2.203000783920288
|
||||
|
||||
# start = time.time()
|
||||
# for _ in range(100):
|
||||
# BrowserFingerprintGenerator.generate_fingerprint("Chrome")
|
||||
# end = time.time()
|
||||
# print("生成100个指纹所需时间:", end - start) # 生成100个指纹所需时间: 0.00400090217590332
|
||||
@@ -1,153 +0,0 @@
|
||||
"""抖音图文 SSR 静态解析 — 不走浏览器，直接解析服务端渲染页面
|
||||
|
||||
与 douyin_api.py 的 playwright 拦截方案互补：
|
||||
- 视频作品 → fetch_douyin_content（playwright 拦截，可取全部清晰度）
|
||||
- 图文作品 → fetch_douyin_note_ssr（本模块，按 aweme_id 精准渲染，秒级返回）
|
||||
|
||||
SSR 数据（window._ROUTER_DATA / __pace_f RSC）是服务端按作品 ID 渲染的页面数据，
|
||||
不存在 playwright 拦截 aweme/v1/web/aweme/post 时取到作者其他作品的问题。
|
||||
|
||||
注意：必须用移动端 UA 请求 iesdouyin.com/share 端点，桌面 UA 会被 302 回主站
|
||||
并返回 JS 反爬壳（无任何 SSR 数据）。
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from pathlib import Path
|
||||
from typing import List, Optional
|
||||
|
||||
import httpx
|
||||
from nonebot import logger
|
||||
|
||||
from ...models import DouyinFetchError
|
||||
from ...utils import get_temp_root
|
||||
from .douyin_api import _process_animated_note, _process_note_with_parsed
|
||||
from .douyin_parser import parse_douyin_response, parse_note_images, parse_ssr_page
|
||||
|
||||
MOBILE_UA = (
|
||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) "
|
||||
"AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 "
|
||||
"Mobile/15E148 Safari/604.1"
|
||||
)
|
||||
ROUTER_DATA_RE = re.compile(r"window\._ROUTER_DATA\s*=\s*(.*?)</script>", re.S)
|
||||
# 短链重定向后统一为 iesdouyin.com/share/video/{id} 形态（图文作品也如此），
|
||||
# 因此 note / video 两种形态都要能提取 ID
|
||||
AWEME_ID_RE = re.compile(r"/(?:share/)?(?:note|video)/(\d+)")
|
||||
|
||||
|
||||
async def fetch_douyin_note_ssr(
|
||||
note_url: str,
|
||||
cookies: List[dict],
|
||||
) -> tuple[Optional[str], Optional[List[Path]]]:
|
||||
"""
|
||||
解析图文作品（SSR 静态解析，秒级返回）
|
||||
|
||||
接受 note / video 任意形态的抖音链接（重定向后图文也常是 share/video），
|
||||
按 SSR 内容判定：有 images → 图文/动图下载；无 images（真视频）→
|
||||
返回 (None, None) 交由 playwright 链路。
|
||||
|
||||
Returns:
|
||||
(title, file_paths) — 图片/视频本地路径列表；非图文返回 (None, None)；
|
||||
失败抛 DouyinFetchError
|
||||
"""
|
||||
m = AWEME_ID_RE.search(note_url)
|
||||
if not m:
|
||||
raise DouyinFetchError(f"无法从链接提取作品 ID: {note_url}")
|
||||
vid = m.group(1)
|
||||
|
||||
api_response = await _fetch_ssr_api_response(note_url, vid, cookies)
|
||||
|
||||
# 内容判定：SSR 数据无 images → 真视频作品，回落 playwright
|
||||
aweme = (api_response.get("aweme_list") or [None])[0] or {}
|
||||
if not aweme.get("images"):
|
||||
logger.info("SSR 内容为视频作品（无 images），回退 playwright 链路")
|
||||
return None, None
|
||||
|
||||
# 复用原有解析链：标题 / 作者 / 文件名（referer 统一用 note 形态，
|
||||
# 避免输入是 share/video 时被误判为"视频"）
|
||||
parsed = parse_douyin_response(
|
||||
api_response, f"https://www.douyin.com/note/{vid}"
|
||||
)
|
||||
logger.info(f"SSR 标题: {parsed.raw_title}")
|
||||
logger.info(f"SSR 作者: {parsed.raw_nickname}")
|
||||
|
||||
tmp_root = get_temp_root("douyin")
|
||||
headers = {
|
||||
"Referer": "https://www.douyin.com/",
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/122.0.0.0 Safari/537.36"
|
||||
),
|
||||
}
|
||||
|
||||
# 复用原有分派：纯图文 / 图+视频混合 → 并发下载；纯动图 → 逐张下载 mp4
|
||||
images_urls, video_url = parse_note_images(api_response, None, vid)
|
||||
if images_urls:
|
||||
file_paths = await _process_note_with_parsed(
|
||||
images_urls, video_url, tmp_root, parsed.rel_stem, headers
|
||||
)
|
||||
else:
|
||||
file_paths = await _process_animated_note(
|
||||
api_response, tmp_root, parsed.rel_stem, headers
|
||||
)
|
||||
return parsed.raw_title, file_paths
|
||||
|
||||
|
||||
async def _fetch_ssr_api_response(
|
||||
note_url: str, vid: str, cookies: List[dict]
|
||||
) -> dict:
|
||||
"""请求 douyin.com note 页面并提取 SSR 数据，规范化为 aweme_list 格式
|
||||
|
||||
2026-08-13 起 iesdouyin.com/share/note 端点对所有请求（含登录态）只返回
|
||||
33KB 壳页（无任何作品数据），改从 www.douyin.com/note/{id} 提取
|
||||
__pace_f RSC 数据——服务端按作品 id 精确渲染，与 URL 一一对应，
|
||||
不存在 aweme/post 作者列表"取错作品"的问题。
|
||||
必须用桌面 UA + 登录 cookie（whale 风控对未登录请求截断作品数据）。
|
||||
"""
|
||||
url = f"https://www.douyin.com/note/{vid}"
|
||||
cookie_map = {c["name"]: c["value"] for c in cookies} if cookies else None
|
||||
headers = {
|
||||
"User-Agent": (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||||
"Chrome/151.0.0.0 Safari/537.36"
|
||||
),
|
||||
"Referer": "https://www.douyin.com/",
|
||||
}
|
||||
async with httpx.AsyncClient(
|
||||
headers=headers, timeout=20, follow_redirects=True
|
||||
) as client:
|
||||
resp = await client.get(url, cookies=cookie_map)
|
||||
if resp.status_code >= 400:
|
||||
raise DouyinFetchError(f"SSR 页面请求失败: status={resp.status_code}")
|
||||
html = resp.text
|
||||
logger.info(f"SSR 页面: {url} status={resp.status_code} size={len(html)}")
|
||||
|
||||
# 1. 旧版 SSR: window._ROUTER_DATA（snake_case 结构，可直接复用原有解析函数）
|
||||
if m := ROUTER_DATA_RE.search(html):
|
||||
data = _clean_js_json(m.group(1))
|
||||
if data:
|
||||
loader = data.get("loaderData", {})
|
||||
page = loader.get("note_(id)/page") or loader.get("video_(id)/page")
|
||||
item_list = (page.get("videoInfoRes") or {}).get("item_list") or []
|
||||
if item_list:
|
||||
logger.info("SSR 数据来源: _ROUTER_DATA")
|
||||
return {"aweme_list": [item_list[0]]}
|
||||
# 2. 新版 SSR: __pace_f.push RSC 流（parse_ssr_page 已规范化为 aweme_list 格式）
|
||||
if parsed := parse_ssr_page(html):
|
||||
logger.info("SSR 数据来源: __pace_f RSC")
|
||||
return parsed
|
||||
raise DouyinFetchError("SSR 页面无图文数据（可能已被删除或风控）")
|
||||
|
||||
|
||||
def _clean_js_json(raw: str) -> Optional[dict]:
|
||||
"""清洗 JS 对象文本为合法 JSON（$undefined / $Lx 占位符）"""
|
||||
text = raw.strip().rstrip(";").strip()
|
||||
text = re.sub(r"\$undefined", "null", text)
|
||||
text = re.sub(r"\$L\d+", "null", text)
|
||||
try:
|
||||
return json.loads(text)
|
||||
except json.JSONDecodeError:
|
||||
logger.warning("SSR JSON 解析失败")
|
||||
return None
|
||||
@@ -0,0 +1,388 @@
|
||||
"""抖音 web 接口抓取 — 免浏览器方案（2026-10 版）
|
||||
|
||||
链路：作品链接 → 提取 aweme_id → 签名请求 `aweme/v1/web/aweme/detail/`
|
||||
→ 解析 aweme_detail → 下载「最高码率视频 / 最高画质图片（含动图）」。
|
||||
|
||||
网关要求（2026-08 起 ArgusSecurityPlugin）：
|
||||
1. 请求头 `x-tt-argus` 必须存在（实测只做存在性校验，占位值 "1" 即可）；
|
||||
2. 请求头 `uifid` = cookie 里的 UIFID（未登录为 UIFID_TEMP）；
|
||||
3. 查询参数 `a_bogus` 由纯 Python 签名器（douyin_sign.py，源自 f2，Apache-2.0）生成，
|
||||
UA 必须与签名时传入的 UA 一致（签名对 UA 做了二次哈希）。
|
||||
以上三点缺一即被网关 403（报错文案 Uifid Not Found / Signature Not Found）。
|
||||
|
||||
为什么废弃旧链路（SSR + playwright）：
|
||||
- iesdouyin 分享页 SSR 已整体降级为壳页（无 videoInfoRes）；
|
||||
- www.douyin.com 页面 HTML 对非浏览器客户端返回 jsvm 反爬壳，浏览器进页面也会
|
||||
随机触发验证码中间页（headless 同样被拦）；
|
||||
- web 接口是页面播放器同款数据源，含全部清晰度档（bit_rate 梯）与无水印直链。
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import re
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
from typing import Dict, List, Optional, Tuple, Union
|
||||
|
||||
import httpx
|
||||
from nonebot import logger
|
||||
|
||||
from ...models import DouyinFetchError
|
||||
from ...utils import get_temp_root, unique_media_path
|
||||
from .douyin_parser import parse_douyin_response
|
||||
from .douyin_sign import ABogus
|
||||
|
||||
# UA 必须与签名输入一致（ABogus 对 UA 做二次哈希）；browser_name/version 参数同步为 Edge
|
||||
WEB_UA = (
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 "
|
||||
"(KHTML, like Gecko) Chrome/130.0.0.0 Safari/537.36 Edg/130.0.0.0"
|
||||
)
|
||||
MOBILE_UA = (
|
||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_2 like Mac OS X) "
|
||||
"AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.2 "
|
||||
"Mobile/15E148 Safari/604.1"
|
||||
)
|
||||
DETAIL_API = "https://www.douyin.com/aweme/v1/web/aweme/detail/"
|
||||
REFERER = "https://www.douyin.com/"
|
||||
AWEME_ID_RE = re.compile(r"/(?:share/)?(?:note|video)/(\d+)")
|
||||
|
||||
# web 端基础参数（对齐浏览器请求；msToken 实测非必需，不生成）
|
||||
_BASE_PARAMS: Dict[str, Union[str, int]] = {
|
||||
"device_platform": "webapp",
|
||||
"aid": "6383",
|
||||
"channel": "channel_pc_web",
|
||||
"pc_client_type": 1,
|
||||
"publish_video_strategy_type": 2,
|
||||
"pc_libra_divert": "Windows",
|
||||
"version_code": "290100",
|
||||
"version_name": "29.1.0",
|
||||
"cookie_enabled": "true",
|
||||
"screen_width": 1920,
|
||||
"screen_height": 1080,
|
||||
"browser_language": "zh-CN",
|
||||
"browser_platform": "Win32",
|
||||
"browser_name": "Edge",
|
||||
"browser_version": "130.0.0.0",
|
||||
"browser_online": "true",
|
||||
"engine_name": "Blink",
|
||||
"engine_version": "130.0.0.0",
|
||||
"os_name": "Windows",
|
||||
"os_version": "10",
|
||||
"cpu_core_num": 12,
|
||||
"device_memory": 8,
|
||||
"platform": "PC",
|
||||
"downlink": 10,
|
||||
"effective_type": "4g",
|
||||
"round_trip_time": 100,
|
||||
}
|
||||
|
||||
|
||||
def extract_aweme_id(url: str) -> Optional[str]:
|
||||
"""从分享链接 / 全链接里提取作品 id（share/note、share/video、note、video 均支持）"""
|
||||
m = AWEME_ID_RE.search(url)
|
||||
if m:
|
||||
return m.group(1)
|
||||
m = re.search(r"/(\d+)(?:\?|#|$)", url)
|
||||
return m.group(1) if m else None
|
||||
|
||||
|
||||
async def resolve_short_link(url: str) -> Optional[str]:
|
||||
"""短链重定向：取 Location（最多 3 跳）；失败返回 None
|
||||
|
||||
只用于拿真实作品 URL（提取 aweme_id），不依赖 iesdouyin 页面内容。
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
headers={"User-Agent": MOBILE_UA},
|
||||
follow_redirects=False,
|
||||
timeout=10,
|
||||
) as client:
|
||||
current = url
|
||||
for _ in range(3):
|
||||
resp = await client.get(current)
|
||||
if resp.status_code >= 400:
|
||||
return None
|
||||
location = resp.headers.get("Location")
|
||||
if not location:
|
||||
return str(resp.url)
|
||||
current = urllib.parse.urljoin(current, location)
|
||||
return current
|
||||
except Exception:
|
||||
logger.warning(f"抖音短链重定向失败: {url}")
|
||||
return None
|
||||
|
||||
|
||||
def build_signed_query(aweme_id: str) -> str:
|
||||
"""构造带 a_bogus 签名的查询串（纯函数，便于测试）"""
|
||||
params = dict(_BASE_PARAMS)
|
||||
params["aweme_id"] = aweme_id
|
||||
param_str = "&".join(f"{k}={v}" for k, v in params.items())
|
||||
signed, a_bogus, _, _ = ABogus(user_agent=WEB_UA).generate_abogus(param_str)
|
||||
logger.debug(f"抖音签名: a_bogus={a_bogus[:16]}... len={len(a_bogus)}")
|
||||
return signed
|
||||
|
||||
|
||||
def _douyin_cookie_map(cookies: List[dict]) -> Dict[str, str]:
|
||||
"""Netscape cookie 列表 → {name: value}（只保留 douyin 域）"""
|
||||
return {
|
||||
c["name"]: c["value"]
|
||||
for c in cookies or []
|
||||
if "douyin" in (c.get("domain") or "")
|
||||
}
|
||||
|
||||
|
||||
def _build_headers(cookie_map: Dict[str, str]) -> Dict[str, str]:
|
||||
headers = {
|
||||
"User-Agent": WEB_UA,
|
||||
"Referer": REFERER,
|
||||
"Accept": "application/json, text/plain, */*",
|
||||
# 网关只校验存在性，占位值即可（f2 项目 issue #443 实测结论）
|
||||
"x-tt-argus": "1",
|
||||
}
|
||||
uifid = cookie_map.get("UIFID") or cookie_map.get("UIFID_TEMP")
|
||||
if uifid:
|
||||
headers["uifid"] = uifid
|
||||
return headers
|
||||
|
||||
|
||||
async def fetch_douyin_detail(aweme_id: str, cookies: List[dict]) -> dict:
|
||||
"""请求作品 detail 接口，返回 aweme_detail dict（失败抛 DouyinFetchError）"""
|
||||
cookie_map = _douyin_cookie_map(cookies)
|
||||
if not (cookie_map.get("UIFID") or cookie_map.get("UIFID_TEMP")):
|
||||
raise DouyinFetchError("cookies.txt 缺少抖音 UIFID（重新登录抖音网页版导出）")
|
||||
|
||||
url = f"{DETAIL_API}?{build_signed_query(aweme_id)}"
|
||||
headers = _build_headers(cookie_map)
|
||||
last_error: Optional[Exception] = None
|
||||
for attempt in range(2):
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
headers=headers, cookies=cookie_map, timeout=20
|
||||
) as client:
|
||||
resp = await client.get(url)
|
||||
if resp.status_code == 403:
|
||||
raise DouyinFetchError(f"detail 接口被网关拦截: {resp.text[:80]}")
|
||||
if resp.status_code >= 400:
|
||||
raise DouyinFetchError(f"detail 接口请求失败: {resp.status_code}")
|
||||
data = resp.json()
|
||||
detail = data.get("aweme_detail")
|
||||
if detail:
|
||||
return detail
|
||||
# status_code=0 但无数据：反爬空响应（签名偶发不同步），重试一次
|
||||
last_error = DouyinFetchError(
|
||||
f"接口返回空数据 (status_code={data.get('status_code')})"
|
||||
)
|
||||
except DouyinFetchError as e:
|
||||
last_error = e
|
||||
except Exception as e: # 网络异常
|
||||
last_error = e
|
||||
if attempt == 0:
|
||||
logger.warning(f"detail 请求失败，1 秒后重试: {last_error!r}")
|
||||
await asyncio.sleep(1)
|
||||
raise DouyinFetchError(f"获取作品详情失败: {last_error}")
|
||||
|
||||
|
||||
# ============================= 媒体选择 =============================
|
||||
|
||||
|
||||
def _best_urls_from_video(video: dict) -> Tuple[List[str], dict]:
|
||||
"""从 video 对象选最高码率档的 URL 候选列表（含备选 CDN），返回 (urls, meta)"""
|
||||
gears = [
|
||||
g
|
||||
for g in (video.get("bit_rate") or [])
|
||||
if isinstance(g, dict) and (g.get("play_addr") or {}).get("url_list")
|
||||
]
|
||||
if gears:
|
||||
best = max(gears, key=lambda g: g.get("bit_rate") or 0)
|
||||
play_addr = best.get("play_addr") or {}
|
||||
meta = {
|
||||
"gear_name": best.get("gear_name"),
|
||||
"bit_rate": best.get("bit_rate"),
|
||||
"width": play_addr.get("width"),
|
||||
"height": play_addr.get("height"),
|
||||
"is_h265": best.get("is_h265"),
|
||||
"size": play_addr.get("data_size"),
|
||||
}
|
||||
return list(play_addr.get("url_list") or []), meta
|
||||
play_addr = video.get("play_addr") or {}
|
||||
meta = {
|
||||
"gear_name": None,
|
||||
"bit_rate": None,
|
||||
"width": play_addr.get("width"),
|
||||
"height": play_addr.get("height"),
|
||||
"is_h265": video.get("is_h265"),
|
||||
"size": play_addr.get("data_size"),
|
||||
}
|
||||
return list(play_addr.get("url_list") or []), meta
|
||||
|
||||
|
||||
def collect_download_items(
|
||||
detail: dict,
|
||||
) -> Tuple[List[Tuple[List[str], str]], bool]:
|
||||
"""从 aweme_detail 收集下载项。
|
||||
|
||||
返回 (items, is_image_post)：
|
||||
- 视频作品 → ([(视频URL候选, ".mp4")], False)
|
||||
- 图文（含图+视频混合）→ 每项一个 (URL候选列表, 扩展名)：
|
||||
图片项按 无水印 url_list → 带作者水印的 download_url_list 排序；
|
||||
内嵌视频项（动图 / 图集中的视频）走最高码率档。
|
||||
is_image_post=True
|
||||
"""
|
||||
images = detail.get("images") or []
|
||||
if not images:
|
||||
urls, meta = _best_urls_from_video(detail.get("video") or {})
|
||||
if not urls:
|
||||
raise DouyinFetchError("未找到可用视频直链")
|
||||
logger.info(
|
||||
f"抖音视频最高档: {meta.get('gear_name')} "
|
||||
f"{meta.get('width')}x{meta.get('height')} "
|
||||
f"br={meta.get('bit_rate')} size={meta.get('size')} "
|
||||
f"h265={meta.get('is_h265')} 候选={len(urls)}"
|
||||
)
|
||||
return [(urls, ".mp4")], False
|
||||
|
||||
items: List[Tuple[List[str], str]] = []
|
||||
for img in images:
|
||||
if not isinstance(img, dict):
|
||||
continue
|
||||
video = img.get("video")
|
||||
if isinstance(video, dict) and (video.get("play_addr") or {}).get("url_list"):
|
||||
urls, meta = _best_urls_from_video(video)
|
||||
if urls:
|
||||
logger.info(
|
||||
f"图集内视频项: {meta.get('gear_name')} br={meta.get('bit_rate')}"
|
||||
)
|
||||
items.append((urls, ".mp4"))
|
||||
continue
|
||||
# 图片项：url_list 为无水印版（q75 webp/jpeg）；
|
||||
# download_url_list 是带作者水印的"保存到相册"版（tplv-dy-water-v2，
|
||||
# base64 里就是作者昵称），只能当兜底绝不能优先
|
||||
urls = list(img.get("url_list") or []) + list(
|
||||
img.get("download_url_list") or []
|
||||
)
|
||||
if urls:
|
||||
items.append((urls, ""))
|
||||
if not items:
|
||||
raise DouyinFetchError("图文作品未解析到任何图片/视频")
|
||||
logger.info(f"抖音图文作品: {len(items)} 个下载项")
|
||||
return items, True
|
||||
|
||||
|
||||
# ============================= 下载 =============================
|
||||
|
||||
|
||||
def _infer_extension(url: str) -> str:
|
||||
"""从 URL 推断文件扩展名（douyinpic 图片 URL 无扩展名时默认 jpg）"""
|
||||
url_lower = url.lower()
|
||||
for ext in (".webp", ".jpeg", ".jpg", ".png", ".gif", ".avif", ".heic"):
|
||||
if ext in url_lower:
|
||||
return ext
|
||||
return ".jpg"
|
||||
|
||||
|
||||
async def _stream_download(
|
||||
url: str, filepath: Path, headers: dict, timeout: float = 300
|
||||
) -> None:
|
||||
async with httpx.AsyncClient(headers=headers, timeout=timeout) as client:
|
||||
async with client.stream("GET", url) as resp:
|
||||
resp.raise_for_status()
|
||||
with open(filepath, "wb") as f:
|
||||
async for chunk in resp.aiter_bytes(65536):
|
||||
f.write(chunk)
|
||||
|
||||
|
||||
async def _download_with_fallback(
|
||||
candidates: List[str], filepath: Path, headers: dict
|
||||
) -> Optional[Path]:
|
||||
"""按候选 URL 顺序下载（CDN 多备份），全部失败返回 None"""
|
||||
for url in candidates:
|
||||
try:
|
||||
await _stream_download(url, filepath, headers)
|
||||
if filepath.stat().st_size == 0: # 空响应也当失败，换下一个候选
|
||||
raise DouyinFetchError("响应为空（0 字节）")
|
||||
logger.info(f"已保存: {filepath.name} ({filepath.stat().st_size} 字节)")
|
||||
return filepath
|
||||
except Exception as e:
|
||||
logger.warning(f"下载失败，尝试下一个 CDN: {url[:80]}... ({e!r})")
|
||||
return None
|
||||
|
||||
|
||||
async def _download_video_file(
|
||||
candidates: List[str], tmp_root: Path, rel_stem: str, headers: dict
|
||||
) -> Path:
|
||||
output_path = unique_media_path(tmp_root / f"{rel_stem}.mp4")
|
||||
result = await _download_with_fallback(candidates, output_path, headers)
|
||||
if result is None:
|
||||
raise DouyinFetchError("视频下载失败（所有 CDN 均不可用）")
|
||||
logger.info(f"视频下载完成: {result}")
|
||||
return result
|
||||
|
||||
|
||||
async def _download_note_items(
|
||||
items: List[Tuple[List[str], str]],
|
||||
tmp_root: Path,
|
||||
rel_stem: str,
|
||||
headers: dict,
|
||||
) -> List[Path]:
|
||||
"""图集/动图：一个作品一个目录，`001.jpg / 002.mp4 ...` 并发下载"""
|
||||
note_dir = unique_media_path(tmp_root / rel_stem)
|
||||
note_dir.mkdir(parents=True, exist_ok=True)
|
||||
logger.info(f"图集保存目录: {note_dir}")
|
||||
|
||||
async def _one(idx: int, candidates: List[str], ext: str) -> Optional[Path]:
|
||||
ext = ext or _infer_extension(candidates[0])
|
||||
filepath = note_dir / f"{idx:03d}{ext}"
|
||||
return await _download_with_fallback(candidates, filepath, headers)
|
||||
|
||||
results = await asyncio.gather(
|
||||
*(_one(i, urls, ext) for i, (urls, ext) in enumerate(items, 1))
|
||||
)
|
||||
saved = [p for p in results if p is not None]
|
||||
if not saved:
|
||||
raise DouyinFetchError("图集下载失败（所有文件均失败）")
|
||||
logger.info(f"图集下载完成，共 {len(saved)}/{len(items)} 个文件")
|
||||
return saved
|
||||
|
||||
|
||||
# ============================= 主入口 =============================
|
||||
|
||||
|
||||
async def fetch_douyin_web(
|
||||
url: str,
|
||||
cookies: List[dict],
|
||||
) -> Tuple[str, Union[Path, List[Path]], bool]:
|
||||
"""解析抖音作品（视频 / 图文 / 动图），返回 (title, 本地路径, 是否图文)
|
||||
|
||||
失败抛 DouyinFetchError（由调用方兜底）。
|
||||
"""
|
||||
aweme_id = extract_aweme_id(url)
|
||||
if not aweme_id:
|
||||
# 短链（v.douyin.com/xxx）或异常形态 → 先重定向拿全链接再提取
|
||||
resolved = await resolve_short_link(url)
|
||||
if resolved:
|
||||
logger.info(f"抖音短链重定向: {url} -> {resolved}")
|
||||
url = resolved
|
||||
aweme_id = extract_aweme_id(url)
|
||||
if not aweme_id:
|
||||
raise DouyinFetchError(f"无法从链接提取作品 ID: {url}")
|
||||
|
||||
detail = await fetch_douyin_detail(aweme_id, cookies)
|
||||
items, is_img_post = collect_download_items(detail)
|
||||
|
||||
# 内容形态以接口数据为准（分享短链重定向后图文也常是 share/video 形态）
|
||||
referer = f"https://www.douyin.com/{'note' if is_img_post else 'video'}/{aweme_id}"
|
||||
parsed = parse_douyin_response({"aweme_detail": detail}, referer)
|
||||
logger.info(f"抖音标题: {parsed.raw_title}")
|
||||
logger.info(f"抖音作者: {parsed.raw_nickname}")
|
||||
logger.info(f"抖音落盘: {parsed.rel_stem}")
|
||||
|
||||
tmp_root = get_temp_root("douyin")
|
||||
headers = {"Referer": REFERER, "User-Agent": WEB_UA}
|
||||
if is_img_post:
|
||||
paths: Union[Path, List[Path]] = await _download_note_items(
|
||||
items, tmp_root, parsed.rel_stem, headers
|
||||
)
|
||||
else:
|
||||
paths = await _download_video_file(
|
||||
items[0][0], tmp_root, parsed.rel_stem, headers
|
||||
)
|
||||
return parsed.raw_title, paths, is_img_post
|
||||
@@ -0,0 +1,322 @@
|
||||
"""快手作品解析 — 分享页 window.INIT_STATE 静态提取
|
||||
|
||||
借鉴 nonebot-plugin-parser 的 KuaiShouParser：
|
||||
- v.kuaishou.com/{code} 短链 → 取 Location 跳到 v.m.chenzhongtech.com/fw/photo/{id}
|
||||
- www.kuaishou.com/short-video/{id} → 重写为移动端 fw/photo/{id}
|
||||
（桌面站匿名请求返回 `Intercept-Result: risk-control` 的 JSON，拿不到页面）
|
||||
- /fw/long-video/ 与 /fw/photo/ 返回结构不同，统一按 /fw/photo/ 请求
|
||||
- 页面 HTML 提取 window.INIT_STATE → dict 里第一个非空 photo
|
||||
- 视频优先从清晰度梯 manifest.adaptationSet 挑最优档（见 `_manifest_video_url`），
|
||||
没有 manifest 才退回 photo.mainMvUrls（默认档）
|
||||
- 图集取 photo.ext_params.atlas 的 cdnList + list 拼 URL
|
||||
|
||||
必须用移动端（iPhone）UA 请求：桌面 UA 与匿名 www 站都只返回风控壳页。
|
||||
|
||||
返回 (title, 文件)：
|
||||
- 视频作品 → (title, 视频文件 Path)
|
||||
- 图集 / 图+视频混排 → (title, [文件路径列表]，视频排在图片之后)
|
||||
- 纯文字作品 → (title, [])
|
||||
- 解析失败 → (None, None)
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
import urllib.parse
|
||||
from pathlib import Path
|
||||
from typing import Optional, Union
|
||||
|
||||
from nonebot import logger
|
||||
|
||||
from ...models import ContentFetchError
|
||||
from ...utils import (
|
||||
build_author_dir,
|
||||
build_work_stem,
|
||||
get_temp_root,
|
||||
unique_media_path,
|
||||
)
|
||||
|
||||
#: 移动端 UA（桌面 UA 会被风控挡下，见模块文档）
|
||||
KUAISHOU_UA = (
|
||||
"Mozilla/5.0 (iPhone; CPU iPhone OS 16_6 like Mac OS X) "
|
||||
"AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 "
|
||||
"Safari/604.1 Edg/132.0.0.0"
|
||||
)
|
||||
KUAISHOU_REFERER = "https://v.kuaishou.com/"
|
||||
|
||||
SHORT_LINK_RE = re.compile(r"v\.kuaishou\.com/[A-Za-z0-9._?%&+\-=/]+")
|
||||
WEB_SHORT_VIDEO_RE = re.compile(
|
||||
r"(?:www\.)?kuaishou\.com/(?:short-video|fw/[a-z-]+)/([A-Za-z0-9]+)"
|
||||
)
|
||||
INIT_STATE_RE = re.compile(r"window\.INIT_STATE\s*=\s*(.*?)</script>", re.S)
|
||||
|
||||
#: 昵称缺失时的兜底串（utils.build_author_dir 认它，转成"未知作者"规则）
|
||||
FALLBACK_AUTHOR = "快手用户"
|
||||
|
||||
|
||||
async def fetch_kuaishou_content(
|
||||
url: str,
|
||||
) -> tuple[Optional[str], Optional[Union[Path, list[Path]]]]:
|
||||
"""解析快手作品链接（短链 / 分享页 / 网页版短链）"""
|
||||
try:
|
||||
return await _parse(url)
|
||||
except Exception:
|
||||
logger.exception(f"快手解析失败: {url}")
|
||||
return None, None
|
||||
|
||||
|
||||
async def _parse(url: str):
|
||||
# 1. 短链 → 单跳重定向（Location 即 v.m.chenzhongtech.com/fw/photo/{id}）
|
||||
if "v.kuaishou.com" in url:
|
||||
resolved = await _resolve_redirect(url)
|
||||
if resolved:
|
||||
logger.info(f"快手短链重定向: {url} -> {resolved}")
|
||||
url = resolved
|
||||
|
||||
# 2. www.kuaishou.com/short-video/{id} → 移动端分享页（同一套 INIT_STATE）
|
||||
m = WEB_SHORT_VIDEO_RE.search(url)
|
||||
if m and "chenzhongtech.com" not in url:
|
||||
url = f"https://v.m.chenzhongtech.com/fw/photo/{m.group(1)}"
|
||||
logger.info(f"快手链接规范化: {url}")
|
||||
|
||||
# 3. long-video 与 photo 返回结构不同，统一按 photo 请求
|
||||
url = url.replace("/fw/long-video/", "/fw/photo/")
|
||||
|
||||
html = await _fetch_page(url)
|
||||
photo = _extract_photo(html)
|
||||
logger.info(f"快手页面解析: {_describe(photo)}")
|
||||
return await _build_result(photo)
|
||||
|
||||
|
||||
def _describe(photo: dict) -> str:
|
||||
"""日志用的一句话摘要（标题/作者/时长）"""
|
||||
caption = str(photo.get("caption") or "").strip()
|
||||
nickname = str(photo.get("userName") or "").strip()
|
||||
if len(caption) > 30:
|
||||
caption = caption[:30]
|
||||
return f"作者={nickname or '未知'}, 标题={caption or '无'}, 时长={photo.get('duration') or 0}ms"
|
||||
|
||||
|
||||
def _extract_photo(html: str) -> dict:
|
||||
"""从 window.INIT_STATE 提取作品数据（取第一个非空 photo）"""
|
||||
m = INIT_STATE_RE.search(html)
|
||||
if not m:
|
||||
raise ContentFetchError("快手页面无 INIT_STATE（可能已删除或风控）")
|
||||
|
||||
raw = m.group(1).strip()
|
||||
try:
|
||||
data = json.loads(raw)
|
||||
except json.JSONDecodeError:
|
||||
# 页面偶有 undefined 之类的非 JSON 字面量（同小红书预处理）
|
||||
try:
|
||||
data = json.loads(raw.replace("undefined", "null"))
|
||||
except json.JSONDecodeError:
|
||||
raise ContentFetchError("快手 INIT_STATE JSON 解析失败")
|
||||
|
||||
if not isinstance(data, dict):
|
||||
raise ContentFetchError("快手 INIT_STATE 结构异常")
|
||||
for value in data.values():
|
||||
if isinstance(value, dict) and isinstance(value.get("photo"), dict):
|
||||
return value["photo"]
|
||||
raise ContentFetchError("页面数据中未找到作品（可能已删除或风控）")
|
||||
|
||||
|
||||
def _first_url(items) -> Optional[str]:
|
||||
"""[{cdn, url}, ...] / [url, ...] → 第一个可用 URL（快手同一资源给多个 CDN）"""
|
||||
if not isinstance(items, list):
|
||||
return None
|
||||
for item in items:
|
||||
if isinstance(item, dict) and item.get("url"):
|
||||
return str(item["url"])
|
||||
if isinstance(item, str) and item:
|
||||
return item
|
||||
return None
|
||||
|
||||
|
||||
#: 同分辨率/帧率时优先 avc（QQ 客户端播放、群文件解压后二次剪辑都最稳；
|
||||
#: hevc 体积小但兼容性差，只在画质更高时才选）
|
||||
_CODEC_RANK = {"avc": 0, "avc1": 0, "h264": 0}
|
||||
|
||||
|
||||
def _manifest_video_url(photo: dict) -> Optional[str]:
|
||||
"""从清晰度梯 `manifest.adaptationSet[].representation[]` 挑最优档
|
||||
|
||||
排序键 = 分辨率 → 帧率 → 编码（avc 优先，见 `_CODEC_RANK`）→ 码率。
|
||||
参考实现直接用 mainMvUrls（默认档），多档作品拿不到最高清晰度。
|
||||
"""
|
||||
adaptations = (photo.get("manifest") or {}).get("adaptationSet")
|
||||
if not isinstance(adaptations, list):
|
||||
return None
|
||||
|
||||
best_rep = None
|
||||
best_key = None
|
||||
total = 0
|
||||
for adaptation in adaptations:
|
||||
if not isinstance(adaptation, dict):
|
||||
continue
|
||||
for rep in adaptation.get("representation") or []:
|
||||
if not isinstance(rep, dict) or not rep.get("url"):
|
||||
continue
|
||||
total += 1
|
||||
codec = str(rep.get("videoCodec") or "").lower()
|
||||
key = (
|
||||
(rep.get("width") or 0) * (rep.get("height") or 0),
|
||||
rep.get("frameRate") or 0,
|
||||
-_CODEC_RANK.get(codec, 1), # 取 max，故用负值让 avc 排前
|
||||
rep.get("avgBitrate") or 0,
|
||||
)
|
||||
if best_key is None or key > best_key:
|
||||
best_key, best_rep = key, rep
|
||||
|
||||
if best_rep is None:
|
||||
return None
|
||||
logger.info(
|
||||
f"快手清晰度: {best_rep.get('width')}x{best_rep.get('height')} "
|
||||
f"{best_rep.get('frameRate')}fps {best_rep.get('videoCodec')} "
|
||||
f"avg={best_rep.get('avgBitrate')}kbps "
|
||||
f"size={best_rep.get('fileSize')}（{total} 档中选最优）"
|
||||
)
|
||||
return str(best_rep["url"])
|
||||
|
||||
|
||||
def _video_url(photo: dict) -> Optional[str]:
|
||||
"""视频直链：清晰度梯最优档 → mainMvUrls 兜底（老页面可能没 manifest）"""
|
||||
return _manifest_video_url(photo) or _first_url(photo.get("mainMvUrls"))
|
||||
|
||||
|
||||
def _atlas_image_urls(photo: dict) -> list[str]:
|
||||
"""图集图片：ext_params.atlas.cdnList（图床域名）+ list（路径）拼 URL
|
||||
|
||||
atlas 缺失/为空即非图集作品（视频的 ext_params 里只有编码参数）。
|
||||
"""
|
||||
ext = photo.get("ext_params") or photo.get("extParams") or {}
|
||||
atlas = ext.get("atlas") if isinstance(ext, dict) else None
|
||||
if not isinstance(atlas, dict):
|
||||
return []
|
||||
|
||||
routes = atlas.get("list")
|
||||
if not isinstance(routes, list) or not routes:
|
||||
return []
|
||||
|
||||
cdn = ""
|
||||
for item in atlas.get("cdnList") or []:
|
||||
if isinstance(item, dict) and item.get("cdn"):
|
||||
cdn = str(item["cdn"])
|
||||
break
|
||||
if not cdn:
|
||||
return []
|
||||
|
||||
return [
|
||||
f"https://{cdn}/{route}"
|
||||
for route in routes
|
||||
if isinstance(route, str) and route
|
||||
]
|
||||
|
||||
|
||||
def _build_rel_stem(photo: dict) -> str:
|
||||
"""相对平台根的路径词干：`{作者}_{userId}/{作品名}`"""
|
||||
author = str(photo.get("userName") or "").replace("ㅤ", "").strip()
|
||||
author = author or FALLBACK_AUTHOR
|
||||
source = str(photo.get("photoId") or "")
|
||||
return (
|
||||
f"{build_author_dir(author, photo.get('userId'), source=source)}"
|
||||
f"/{build_work_stem(str(photo.get('caption') or ''), source=source)}"
|
||||
)
|
||||
|
||||
|
||||
async def _build_result(photo: dict) -> tuple[str, Union[Path, list[Path]]]:
|
||||
text = str(photo.get("caption") or "").strip() or FALLBACK_AUTHOR
|
||||
rel_stem = _build_rel_stem(photo)
|
||||
|
||||
video_url = _video_url(photo)
|
||||
image_urls = _atlas_image_urls(photo)
|
||||
|
||||
# 图集（含"图 + 视频"混排）：一个作品一个目录，视频排在图片之后
|
||||
if image_urls:
|
||||
files = await _download_images(image_urls, video_url, rel_stem)
|
||||
if not files:
|
||||
raise ContentFetchError("快手媒体下载失败")
|
||||
logger.info(
|
||||
f"快手图集: 图片={len(image_urls)} 张, "
|
||||
f"视频={'有' if video_url else '无'}, 落盘={len(files)} 个"
|
||||
)
|
||||
return text, files
|
||||
|
||||
if video_url:
|
||||
path = await _download_video(video_url, rel_stem)
|
||||
return text, path
|
||||
|
||||
logger.info(f"快手文字作品（无媒体）: {text[:30]}")
|
||||
return text, []
|
||||
|
||||
|
||||
async def _download_images(
|
||||
image_urls: list[str], video_url: Optional[str], rel_stem: str
|
||||
) -> list[Path]:
|
||||
"""并发下载图集（复用抖音图文的下载流程，视频追加在图片之后）"""
|
||||
|
||||
from .douyin_api import _process_note_with_parsed
|
||||
|
||||
tmp_root = get_temp_root("kuaishou")
|
||||
headers = {"Referer": KUAISHOU_REFERER, "User-Agent": KUAISHOU_UA}
|
||||
return await _process_note_with_parsed(
|
||||
[[u] for u in image_urls], video_url, tmp_root, rel_stem, headers
|
||||
)
|
||||
|
||||
|
||||
async def _download_video(video_url: str, rel_stem: str) -> Path:
|
||||
"""流式下载视频到 `{平台根}/{作者目录}/{作品名}.mp4`"""
|
||||
import httpx
|
||||
|
||||
tmp_root = get_temp_root("kuaishou")
|
||||
output_path = unique_media_path(tmp_root / f"{rel_stem}.mp4")
|
||||
headers = {"Referer": KUAISHOU_REFERER, "User-Agent": KUAISHOU_UA}
|
||||
async with httpx.AsyncClient(headers=headers, timeout=300) as client:
|
||||
async with client.stream("GET", video_url) as resp:
|
||||
resp.raise_for_status()
|
||||
with open(output_path, "wb") as f:
|
||||
async for chunk in resp.aiter_bytes(8192):
|
||||
f.write(chunk)
|
||||
logger.info(f"快手视频下载完成: {output_path}")
|
||||
return output_path
|
||||
|
||||
|
||||
async def _fetch_page(url: str) -> str:
|
||||
import httpx
|
||||
|
||||
headers = {"User-Agent": KUAISHOU_UA, "Referer": KUAISHOU_REFERER}
|
||||
async with httpx.AsyncClient(
|
||||
headers=headers, timeout=20, follow_redirects=True
|
||||
) as client:
|
||||
resp = await client.get(url)
|
||||
if resp.status_code >= 400:
|
||||
raise ContentFetchError(f"快手页面请求失败: status={resp.status_code}")
|
||||
return resp.text
|
||||
|
||||
|
||||
async def _resolve_redirect(url: str) -> Optional[str]:
|
||||
"""短链重定向（最多 3 跳，取最终 URL）
|
||||
|
||||
v.kuaishou.com 是 302 + Location，单跳即到 chenzhongtech 分享页；
|
||||
与小红书一样用移动端 headers，避免被当作非 App 来源拒绝。
|
||||
"""
|
||||
import httpx
|
||||
|
||||
headers = {"User-Agent": KUAISHOU_UA, "Referer": KUAISHOU_REFERER}
|
||||
try:
|
||||
async with httpx.AsyncClient(
|
||||
headers=headers, follow_redirects=False, timeout=10
|
||||
) as client:
|
||||
current = url
|
||||
for _ in range(3):
|
||||
resp = await client.get(current)
|
||||
if resp.status_code >= 400:
|
||||
return None
|
||||
location = resp.headers.get("Location")
|
||||
if not location:
|
||||
return str(resp.url)
|
||||
# Location 可能是相对路径，需拼上当前 URL
|
||||
current = urllib.parse.urljoin(current, location)
|
||||
return current
|
||||
except Exception:
|
||||
logger.warning(f"快手短链重定向失败: {url}")
|
||||
return None
|
||||
@@ -6,8 +6,9 @@
|
||||
- xiaohongshu.com/discovery/item/{id}?xsec_token=... → 移动端点（fallback）
|
||||
- 页面 HTML 提取 window.__INITIAL_STATE__ → noteDetailMap[id].note
|
||||
|
||||
请求端点依次尝试 国际站 rednote.com（含无水印原片 originVideoKey）
|
||||
→ 国内站 xiaohongshu.com（国内笔记国际站无数据，需 cookies.txt 登录态）。
|
||||
请求端点按链接来源选 host 顺序：国内链接（xiaohongshu.com/xhslink.com）先试
|
||||
国内站、国际链接（rednote.com）先试国际站（国际站数据含无水印原片 originVideoKey；
|
||||
国内笔记在国际站没有数据，反之亦然），失败再换站。需 cookies.txt 登录态。
|
||||
|
||||
返回 (title, 文件)：
|
||||
- 图文笔记 → (title, [图片路径列表])
|
||||
@@ -44,16 +45,15 @@ REDNOTE_UA = (
|
||||
REDNOTE_REFERER = "https://www.xiaohongshu.com/"
|
||||
|
||||
REDNOTE_SHORT_RE = re.compile(r"xhslink\.(?:com|cn)/[A-Za-z0-9._?%&+=/#@-]+")
|
||||
EXPLORE_RE = re.compile(
|
||||
r"xiaohongshu\.com/explore/([0-9a-zA-Z]+)(\?[^\s\"'<>]+)?"
|
||||
)
|
||||
EXPLORE_RE = re.compile(r"xiaohongshu\.com/explore/([0-9a-zA-Z]+)(\?[^\s\"'<>]+)?")
|
||||
DISCOVERY_RE = re.compile(
|
||||
r"xiaohongshu\.com/discovery/item/([0-9a-zA-Z]+)(\?[^\s\"'<>]+)?"
|
||||
)
|
||||
INITIAL_STATE_RE = re.compile(r"window\.__INITIAL_STATE__=(.*?)</script>", re.S)
|
||||
|
||||
# 请求 host 顺序：国际站优先（含无水印原片），失败回退国内站（国内笔记）
|
||||
REDNOTE_HOSTS = ("www.rednote.com", "www.xiaohongshu.com")
|
||||
# 请求 host 候选：链接来自哪个站就先用哪个站（国内笔记在国际站没有数据，
|
||||
# 国际站笔记（rednote.com）带无水印原片 originVideoKey，互不通用）
|
||||
REDNOTE_HOSTS = ("www.xiaohongshu.com", "www.rednote.com")
|
||||
|
||||
|
||||
async def fetch_rednote_content(
|
||||
@@ -75,14 +75,18 @@ async def _parse(url: str):
|
||||
logger.info(f"xhslink 重定向: {url} -> {resolved}")
|
||||
url = resolved
|
||||
|
||||
# 2. explore 优先（国际站才有 originVideoKey 无水印原片），
|
||||
# 依次尝试 国际站 → 国内站；失败再用同一 id+query 回退 discovery 端点
|
||||
# 链接来自哪个站就先试哪个站（国际站 rednote.com / 国内站 xiaohongshu.com）
|
||||
hosts = REDNOTE_HOSTS
|
||||
if "rednote.com" in url:
|
||||
hosts = tuple(reversed(REDNOTE_HOSTS))
|
||||
|
||||
# 2. explore 优先；失败再用同一 id+query 回退 discovery 端点
|
||||
if m := EXPLORE_RE.search(url):
|
||||
note_id, query = m.group(1), (m.group(2) or "").lstrip("?")
|
||||
return await _fetch_with_retry(note_id, query, ("explore", "discovery"))
|
||||
return await _fetch_with_retry(note_id, query, ("explore", "discovery"), hosts)
|
||||
if m := DISCOVERY_RE.search(url):
|
||||
note_id, query = m.group(1), (m.group(2) or "").lstrip("?")
|
||||
return await _fetch_with_retry(note_id, query, ("discovery",))
|
||||
return await _fetch_with_retry(note_id, query, ("discovery",), hosts)
|
||||
|
||||
# 3. 短链重定向终态: xiaohongshu.com/explore?target_note_id={id}&xsec_token=...
|
||||
# （discovery/item 会 302 到该形态，note id 在 query 里），
|
||||
@@ -91,14 +95,10 @@ async def _parse(url: str):
|
||||
qs = urllib.parse.parse_qs(m.group(1))
|
||||
note_id = (qs.get("target_note_id") or [""])[0]
|
||||
if note_id:
|
||||
keep = {
|
||||
k: v
|
||||
for k, v in qs.items()
|
||||
if k in ("xsec_token", "xsec_source")
|
||||
}
|
||||
keep = {k: v for k, v in qs.items() if k in ("xsec_token", "xsec_source")}
|
||||
query = urllib.parse.urlencode(keep, doseq=True)
|
||||
return await _fetch_with_retry(
|
||||
note_id, query, ("explore", "discovery")
|
||||
note_id, query, ("explore", "discovery"), hosts
|
||||
)
|
||||
|
||||
logger.warning(f"无法识别的小红书链接: {url}")
|
||||
@@ -106,15 +106,18 @@ async def _parse(url: str):
|
||||
|
||||
|
||||
async def _fetch_with_retry(
|
||||
note_id: str, query: str, paths: tuple[str, ...]
|
||||
note_id: str,
|
||||
query: str,
|
||||
paths: tuple[str, ...],
|
||||
hosts: tuple[str, ...] = REDNOTE_HOSTS,
|
||||
) -> tuple[str, Union[Path, list[Path]]] | None:
|
||||
"""依次尝试 国际站 → 国内站 × 端点；全部失败后稍候重试一轮
|
||||
"""依次尝试 host × 端点；全部失败后稍候重试一轮
|
||||
|
||||
小红书对突发请求会节流（页面 200 但 noteDetailMap 为空/无数据），
|
||||
重试一轮可绕过大部分瞬时风控；重试仍失败返回 (None, None)。
|
||||
"""
|
||||
for attempt in range(2):
|
||||
for host in REDNOTE_HOSTS:
|
||||
for host in hosts:
|
||||
for path in paths:
|
||||
fetch = _parse_explore if path == "explore" else _parse_discovery
|
||||
try:
|
||||
@@ -211,9 +214,7 @@ def _extract_note(html: str, note_id: str) -> dict:
|
||||
async def _build_result(note: dict) -> tuple[str, Union[Path, list[Path]]]:
|
||||
# 空壳 note（短链重定向带 undertake_note_error=该内容暂时无法查看）
|
||||
# → 笔记已删除/私密，直接报错而不是误判为纯文字笔记
|
||||
if not any(
|
||||
note.get(k) for k in ("title", "desc", "type", "imageList", "video")
|
||||
):
|
||||
if not any(note.get(k) for k in ("title", "desc", "type", "imageList", "video")):
|
||||
raise ContentFetchError("笔记内容不可见（可能已删除/私密），无法解析")
|
||||
title = note.get("title") or ""
|
||||
desc = note.get("desc") or ""
|
||||
@@ -229,18 +230,19 @@ async def _build_result(note: dict) -> tuple[str, Union[Path, list[Path]]]:
|
||||
# 1. 视频笔记 → 无水印原片优先
|
||||
if note.get("type") == "video" and note.get("video"):
|
||||
# 1a. 无水印原片（国际站数据 video.consumer.originVideoKey）
|
||||
consumer = (note["video"].get("consumer") or {})
|
||||
consumer = note["video"].get("consumer") or {}
|
||||
okey = consumer.get("originVideoKey")
|
||||
if okey:
|
||||
video_url = f"https://sns-video-bd.xhscdn.com/{okey}"
|
||||
logger.info(f"小红书视频: 无水印原片 originVideoKey={okey[:30]}...")
|
||||
video_path = await _download_video(video_url, rel_stem)
|
||||
video_path = await _download_video([video_url], rel_stem)
|
||||
return text, video_path
|
||||
|
||||
# 1b. 无 originVideoKey（国内站数据）→ 从 stream 分组选无水印原片
|
||||
# 国内站 masterUrl 同样是 sns-video-v6 原片 CDN（无水印），
|
||||
# 1b. 无 originVideoKey（国内站数据）→ 从 stream 分组选最高档
|
||||
# 国内站 masterUrl 同样是 sns-video 原片 CDN（无水印），
|
||||
# 但不同抓取批次返回的清晰度集合不同（同组多条/分组顺序不定），
|
||||
# 因此跨全部编码分组收集候选，取 size 最大（质量最高）的流。
|
||||
# 因此跨全部编码分组收集候选，取 size 最大（质量最高）的流；
|
||||
# 每个流附带 backupUrls（备选 CDN），下载时逐个回退。
|
||||
stream = ((note["video"].get("media") or {}).get("stream")) or {}
|
||||
candidates = [
|
||||
it
|
||||
@@ -254,23 +256,34 @@ async def _build_result(note: dict) -> tuple[str, Union[Path, list[Path]]]:
|
||||
candidates,
|
||||
key=lambda it: (it.get("size") or 0, it.get("avgBitrate") or 0),
|
||||
)
|
||||
video_url = best["masterUrl"]
|
||||
urls = [best["masterUrl"]] + [
|
||||
u for u in (best.get("backupUrls") or []) if u
|
||||
]
|
||||
duration = best.get("duration", 0)
|
||||
logger.info(
|
||||
f"小红书视频: 无水印流 {best.get('qualityType')} "
|
||||
f"{best.get('width')}x{best.get('height')} {best.get('fps')}fps "
|
||||
f"size={best.get('size')} duration={duration}ms"
|
||||
f"size={best.get('size')} duration={duration}ms "
|
||||
f"候选={len(urls)}"
|
||||
)
|
||||
video_path = await _download_video(video_url, rel_stem)
|
||||
video_path = await _download_video(urls, rel_stem)
|
||||
return text, video_path
|
||||
raise ContentFetchError("小红书视频流解析失败")
|
||||
|
||||
# 2. 图文笔记
|
||||
images = [
|
||||
img.get("urlDefault") or img.get("url")
|
||||
for img in note.get("imageList") or []
|
||||
]
|
||||
images = [u for u in images if u]
|
||||
# 2. 图文笔记（urlDefault = WB_DFT 默认档；兜底取 infoList 里 WB_DFT 场景）
|
||||
images = []
|
||||
for img in note.get("imageList") or []:
|
||||
if not isinstance(img, dict):
|
||||
continue
|
||||
url = img.get("urlDefault")
|
||||
if not url:
|
||||
for info in img.get("infoList") or []:
|
||||
if isinstance(info, dict) and info.get("imageScene") == "WB_DFT":
|
||||
url = info.get("url")
|
||||
break
|
||||
url = url or img.get("url")
|
||||
if url:
|
||||
images.append(url)
|
||||
if not images:
|
||||
logger.info(f"小红书文字笔记: {text[:30]}")
|
||||
return text, []
|
||||
@@ -290,16 +303,14 @@ def _extract_author_id(note: dict) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def _build_rel_stem(
|
||||
nickname: str, author_id: str, title: str, source: str = ""
|
||||
) -> str:
|
||||
def _build_rel_stem(nickname: str, author_id: str, title: str, source: str = "") -> str:
|
||||
"""相对平台根的路径词干：`{作者}_{userId}/{作品名}`
|
||||
|
||||
昵称/作者 id 都拿不到时用 source（笔记 id）当来源码（见 utils.build_author_dir）。
|
||||
"""
|
||||
return (
|
||||
f"{build_author_dir(nickname, author_id, source=source)}"
|
||||
f"/{build_work_stem(title)}"
|
||||
f"/{build_work_stem(title, source=source)}"
|
||||
)
|
||||
|
||||
|
||||
@@ -318,8 +329,8 @@ async def _download_images(image_urls: list[str], rel_stem: str) -> list[Path]:
|
||||
)
|
||||
|
||||
|
||||
async def _download_video(video_url: str, rel_stem: str) -> Path:
|
||||
"""流式下载视频
|
||||
async def _download_video(video_urls: list[str], rel_stem: str) -> Path:
|
||||
"""流式下载视频（masterUrl + backupUrls 逐个回退）
|
||||
|
||||
注意：sns-video-bd（无水印原片）不带 Referer 或带 xiaohongshu.com
|
||||
均可，但带 rednote.com Referer 会 403，因此不设 Referer。
|
||||
@@ -329,14 +340,23 @@ async def _download_video(video_url: str, rel_stem: str) -> Path:
|
||||
tmp_root = get_temp_root("xiaohongshu")
|
||||
output_path = unique_media_path(tmp_root / f"{rel_stem}.mp4")
|
||||
headers = {"User-Agent": REDNOTE_UA}
|
||||
async with httpx.AsyncClient(headers=headers, timeout=300) as client:
|
||||
async with client.stream("GET", video_url) as resp:
|
||||
resp.raise_for_status()
|
||||
with open(output_path, "wb") as f:
|
||||
async for chunk in resp.aiter_bytes(8192):
|
||||
f.write(chunk)
|
||||
logger.info(f"小红书视频下载完成: {output_path}")
|
||||
return output_path
|
||||
last_error: Exception | None = None
|
||||
for video_url in video_urls:
|
||||
try:
|
||||
async with httpx.AsyncClient(headers=headers, timeout=300) as client:
|
||||
async with client.stream("GET", video_url) as resp:
|
||||
resp.raise_for_status()
|
||||
with open(output_path, "wb") as f:
|
||||
async for chunk in resp.aiter_bytes(65536):
|
||||
f.write(chunk)
|
||||
if output_path.stat().st_size == 0:
|
||||
raise ContentFetchError("响应为空（0 字节）")
|
||||
logger.info(f"小红书视频下载完成: {output_path}")
|
||||
return output_path
|
||||
except Exception as e:
|
||||
last_error = e
|
||||
logger.warning(f"小红书视频下载失败（尝试下一个候选 CDN）: {e}")
|
||||
raise ContentFetchError(f"小红书视频下载失败: {last_error}")
|
||||
|
||||
|
||||
async def _fetch_page(url: str, headers: dict) -> str:
|
||||
|
||||
@@ -19,6 +19,7 @@ from ...utils import (
|
||||
build_work_stem,
|
||||
get_data_dir,
|
||||
get_temp_root,
|
||||
short_source_code,
|
||||
slugify,
|
||||
unique_media_path,
|
||||
)
|
||||
@@ -145,7 +146,8 @@ async def download_video(
|
||||
ext = m.group(1)
|
||||
|
||||
url_stem = Path(url.split("?")[0]).stem or "video"
|
||||
slug_stem = slugify(url_stem, max_length=15) or "视频"
|
||||
# 兜底名同 build_work_stem：用来源短码，别让不同直链挤在同一个 `视频` 下
|
||||
slug_stem = slugify(url_stem, max_length=15) or f"视频_{short_source_code(url)}"
|
||||
# 直链拿不到作者信息 → `未知作者_{来源短码}`（同一链接稳定、不同链接不撞）
|
||||
author_dir = build_author_dir(author, author_id, source=url)
|
||||
final_path = unique_media_path(temp_root / author_dir / f"{slug_stem}.{ext}")
|
||||
@@ -277,8 +279,10 @@ async def download_video(
|
||||
title = ((info or {}).get("title") or "").strip()
|
||||
|
||||
author_dir = build_author_dir(uploader, uploader_id, source=url)
|
||||
# 标题空时的兜底名用来源短码（作品 id 优先，其次作品链接）
|
||||
work_stem = build_work_stem(title, source=(info or {}).get("id") or url)
|
||||
new_path = unique_media_path(
|
||||
temp_root / author_dir / f"{build_work_stem(title)}{original_file.suffix}"
|
||||
temp_root / author_dir / f"{work_stem}{original_file.suffix}"
|
||||
)
|
||||
shutil.move(str(original_file), str(new_path))
|
||||
# 下载用的 scratch 目录已空，顺手收掉（cleanup 不删目录）
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import os
|
||||
import re
|
||||
import zipfile
|
||||
from collections.abc import Iterable
|
||||
@@ -23,6 +24,7 @@ from pathlib import Path
|
||||
from typing import TYPE_CHECKING, Any
|
||||
|
||||
from nonebot import get_bot, logger
|
||||
from nonebot.exception import NetworkError
|
||||
|
||||
if TYPE_CHECKING: # 仅类型检查：运行期不导入，单测可裸加载本模块
|
||||
from ..policy import Policy
|
||||
@@ -154,12 +156,42 @@ def _file_uri(path: Path) -> str:
|
||||
return "file:///" + str(path.resolve()).replace("\\", "/")
|
||||
|
||||
|
||||
#: 群文件上传 per-call 超时（秒）——大文件在协议端「下载 + 上传群文件」会超过
|
||||
#: NoneBot 默认的 30s（同 handlers/sender.py 的 `send_timeout`，就地实现以保持
|
||||
#: 本模块可裸加载；配置键同一个 `video_analysis_send_timeout`）。
|
||||
_DEFAULT_SEND_TIMEOUT = 300.0
|
||||
|
||||
|
||||
def _send_timeout() -> float:
|
||||
raw = (
|
||||
os.environ.get("video_analysis_send_timeout")
|
||||
or os.environ.get("VIDEO_ANALYSIS_SEND_TIMEOUT")
|
||||
or ""
|
||||
).strip()
|
||||
try:
|
||||
value = float(raw)
|
||||
except ValueError:
|
||||
return _DEFAULT_SEND_TIMEOUT
|
||||
return value if value > 0 else _DEFAULT_SEND_TIMEOUT
|
||||
|
||||
|
||||
def _unknown_outcome(exc: BaseException) -> bool:
|
||||
"""超时类错误 = 结果未知：协议端可能仍在处理并最终送达（同 sender 的判定）。
|
||||
|
||||
这种失败**不能**再降级到本地直传重试——那会让群文件里出现两份。
|
||||
"""
|
||||
return isinstance(exc, NetworkError) or isinstance(
|
||||
exc, (asyncio.TimeoutError, TimeoutError)
|
||||
)
|
||||
|
||||
|
||||
async def _call_upload(
|
||||
file_value: str, group_id: int, name: str, folder_id: str | None = None
|
||||
) -> None:
|
||||
params: dict = {"group_id": group_id, "file": file_value, "name": name}
|
||||
if folder_id:
|
||||
params["folder"] = folder_id
|
||||
params["_timeout"] = _send_timeout()
|
||||
await get_bot().call_api("upload_group_file", **params)
|
||||
|
||||
|
||||
@@ -181,7 +213,9 @@ async def upload_group_file(
|
||||
) -> bool:
|
||||
"""上传单个文件到群文件，返回是否成功。
|
||||
|
||||
两级投递，**S3 链接优先、本地直传兜底**：
|
||||
两级投递，**S3 链接优先、本地直传兜底**；只有第 1 级**明确失败**才降级，
|
||||
超时（结果未知）直接收手——协议端可能还在传，再直传一份就重复了
|
||||
（见 `_unknown_outcome`）。
|
||||
|
||||
本环境实测 NapCat 读不到 bot 进程写的本地文件（`realpath ... ENOENT`，
|
||||
媒体消息的本地直发 55 次全失败、换 S3 链接后次次成功），所以直传只留作
|
||||
@@ -206,6 +240,14 @@ async def upload_group_file(
|
||||
try:
|
||||
await _call_upload(url, group_id, display_name, folder_id)
|
||||
except Exception as e:
|
||||
if _unknown_outcome(e):
|
||||
# 超时=结果未知：协议端可能还在传，再走本地直传会出两份群文件
|
||||
logger.warning(
|
||||
f"群文件 S3 链接上传超时，结果未知（协议端可能仍在传输），"
|
||||
f"不再降级直传以免重复（群 {group_id} / {display_name}）: "
|
||||
f"{str(e)[:160]}"
|
||||
)
|
||||
return True
|
||||
logger.warning(
|
||||
f"群文件 S3 链接上传失败（群 {group_id} / {display_name}）: "
|
||||
f"{str(e)[:160]}；改用本地路径重试"
|
||||
@@ -220,6 +262,12 @@ async def upload_group_file(
|
||||
try:
|
||||
await _call_upload(_file_uri(path), group_id, display_name, folder_id)
|
||||
except Exception as e:
|
||||
if _unknown_outcome(e):
|
||||
logger.warning(
|
||||
f"群文件上传超时，结果未知（协议端可能仍在传输）"
|
||||
f"（群 {group_id} / {display_name}）: {str(e)[:160]}"
|
||||
)
|
||||
return True
|
||||
logger.warning(
|
||||
f"群文件上传失败（群 {group_id} / {display_name}）: {str(e)[:160]}"
|
||||
)
|
||||
|
||||
@@ -66,6 +66,10 @@ POSTER_WIDTH = 480
|
||||
POSTER_SUFFIX = "_封面.jpg"
|
||||
POSTER_TIMEOUT = 20
|
||||
|
||||
#: bot 跑在 pythonw.exe（无控制台）下，spawn 控制台程序（ffmpeg）不带这个标志
|
||||
#: 会弹出一个 cmd 黑窗口；只在 Windows 有该常量，其它平台取 0 不影响调用
|
||||
CREATE_NO_WINDOW = getattr(subprocess, "CREATE_NO_WINDOW", 0)
|
||||
|
||||
#: 状态机：queued → running → done / failed
|
||||
ACTIVE_STATUSES = ("queued", "running")
|
||||
|
||||
@@ -136,9 +140,14 @@ async def make_poster(video: Path) -> Optional[Path]:
|
||||
str(out),
|
||||
]
|
||||
try:
|
||||
# 同步子进程必须丢线程池，否则抽帧期间整个事件循环都停摆
|
||||
# 同步子进程必须丢线程池，否则抽帧期间整个事件循环都停摆；
|
||||
# CREATE_NO_WINDOW 防止 pythonw 下弹 cmd 黑窗口
|
||||
await asyncio.to_thread(
|
||||
subprocess.run, cmd, capture_output=True, timeout=POSTER_TIMEOUT
|
||||
subprocess.run,
|
||||
cmd,
|
||||
capture_output=True,
|
||||
timeout=POSTER_TIMEOUT,
|
||||
creationflags=CREATE_NO_WINDOW,
|
||||
)
|
||||
except Exception: # noqa: BLE001 — 封面失败不该拖累任务
|
||||
logger.warning(f"抽关键帧失败：{video.name}")
|
||||
@@ -167,12 +176,14 @@ async def fetch_media(url: str) -> FetchResult:
|
||||
|
||||
分派规则与 `handlers/entry.py::dispatch_url` 一致：抖音 → parse_douyin；
|
||||
B站动态/专栏 → fetch_bilibili_content；其余 B站链接 → download_video（yt-dlp）；
|
||||
小红书 → fetch_rednote_content；其它平台 → download_video。
|
||||
b23 / xhslink / 抖音短链的重定向在各 fetcher 内部自己处理。
|
||||
小红书 → fetch_rednote_content；快手 → fetch_kuaishou_content；
|
||||
其它平台 → download_video。
|
||||
b23 / xhslink / 快手短链的重定向在各 fetcher 内部自己处理。
|
||||
"""
|
||||
from ..handlers.douyin import parse_douyin
|
||||
from ..policy import BILIBILI, DOUYIN, XHS
|
||||
from ..policy import BILIBILI, DOUYIN, KUAISHOU, XHS
|
||||
from .fetchers.bilibili_content import fetch_bilibili_content
|
||||
from .fetchers.kuaishou_content import fetch_kuaishou_content
|
||||
from .fetchers.rednote_content import fetch_rednote_content
|
||||
from .fetchers.video_downloader import download_video
|
||||
|
||||
@@ -183,6 +194,9 @@ async def fetch_media(url: str) -> FetchResult:
|
||||
|
||||
if platform == DOUYIN:
|
||||
title, parsed, image_post = await parse_douyin(url)
|
||||
elif platform == KUAISHOU:
|
||||
title, parsed = await fetch_kuaishou_content(url)
|
||||
image_post = isinstance(parsed, list)
|
||||
elif platform == XHS:
|
||||
title, parsed = await fetch_rednote_content(url)
|
||||
image_post = isinstance(parsed, list)
|
||||
@@ -344,7 +358,7 @@ class JobManager:
|
||||
# ── 清理 ──────────────────────────────────────────────
|
||||
|
||||
def remove(self, job_id: str) -> bool:
|
||||
"""删掉一条任务；还在跑的一并取消（Chrome 那头由 playwright 自己收尾）。"""
|
||||
"""删掉一条任务；还在跑的一并取消（进行中的下载协程由取消信号收尾）。"""
|
||||
job = self._jobs.pop(job_id, None)
|
||||
if job is None:
|
||||
return False
|
||||
|
||||
Reference in new issue
Block a user