From 3a2438f566cb876ac39fa54559c2b4fedc03593c Mon Sep 17 00:00:00 2001 From: yhy Date: Sat, 29 Aug 2026 22:13:55 +0800 Subject: [PATCH] =?UTF-8?q?=E5=A2=9E=E5=8A=A0=E8=AF=AD=E9=9F=B3=E5=AF=B9?= =?UTF-8?q?=E8=AF=9D=E5=8A=9F=E8=83=BD?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- addon/content/chatpapers.css | 212 ++++ addon/content/icons/lecture.svg | 28 + addon/content/preferences.xhtml | 63 ++ addon/locale/en-US/addon.ftl | 35 + addon/locale/en-US/preferences.ftl | 8 + addon/locale/zh-CN/addon.ftl | 34 + addon/locale/zh-CN/preferences.ftl | 8 + addon/manifest.json | 2 +- addon/prefs.js | 7 + docs/newidea/ARCHITECTURE.md | 680 +++++++++++++ docs/newidea/DUAL_MODE_ARCHITECTURE.md | 944 ++++++++++++++++++ docs/newidea/DUAL_MODE_PRD.md | 485 +++++++++ docs/newidea/PLAN.md | 350 +++++++ docs/newidea/PRD.md | 323 ++++++ docs/newidea/README.md | 29 + src/hooks.ts | 2 + .../lecture/application/playLecture.ts | 124 +++ .../lecture/application/prepareLecture.ts | 284 ++++++ src/modules/lecture/domain/types.ts | 97 ++ src/modules/lecture/infrastructure/events.ts | 58 ++ .../lecture/infrastructure/llm/beatLlm.ts | 131 +++ .../lecture/infrastructure/llm/lectureLlm.ts | 67 ++ .../infrastructure/llm/structuredOutput.ts | 95 ++ .../lecture/infrastructure/pdf/chunker.ts | 55 + .../lecture/infrastructure/pdf/fileHash.ts | 25 + .../infrastructure/platform/executable.ts | 62 ++ .../lecture/infrastructure/platform/os.ts | 30 + .../lecture/infrastructure/platform/paths.ts | 33 + .../infrastructure/platform/subprocess.ts | 144 +++ .../infrastructure/storage/audioStorage.ts | 40 + .../infrastructure/storage/lectureStore.ts | 95 ++ .../lecture/infrastructure/storage/paths.ts | 66 ++ .../infrastructure/storage/repository.ts | 85 ++ .../lecture/infrastructure/storage/schema.ts | 98 ++ .../lecture/infrastructure/tts/client.ts | 64 ++ .../infrastructure/tts/local/macSay.ts | 84 ++ .../lecture/infrastructure/tts/local/piper.ts | 84 ++ .../infrastructure/tts/local/windowsSapi.ts | 93 ++ .../lecture/infrastructure/tts/providers.ts | 70 ++ .../lecture/infrastructure/tts/types.ts | 40 + .../lecture/infrastructure/zotero/adapter.ts | 74 ++ src/modules/lecture/ui/lecturePane.ts | 533 ++++++++++ src/modules/lecture/ui/registerLecturePane.ts | 65 ++ src/modules/llm/client.ts | 7 +- src/modules/llm/jsonParse.ts | 124 +++ src/modules/ui/lecturePrefs.ts | 87 ++ src/modules/ui/prefs.ts | 3 + src/modules/ui/readerPane.ts | 6 +- typings/global.d.ts | 31 + typings/i10n.d.ts | 39 + typings/prefs.d.ts | 7 + 51 files changed, 6207 insertions(+), 3 deletions(-) create mode 100644 addon/content/icons/lecture.svg create mode 100644 docs/newidea/ARCHITECTURE.md create mode 100644 docs/newidea/DUAL_MODE_ARCHITECTURE.md create mode 100644 docs/newidea/DUAL_MODE_PRD.md create mode 100644 docs/newidea/PLAN.md create mode 100644 docs/newidea/PRD.md create mode 100644 docs/newidea/README.md create mode 100644 src/modules/lecture/application/playLecture.ts create mode 100644 src/modules/lecture/application/prepareLecture.ts create mode 100644 src/modules/lecture/domain/types.ts create mode 100644 src/modules/lecture/infrastructure/events.ts create mode 100644 src/modules/lecture/infrastructure/llm/beatLlm.ts create mode 100644 src/modules/lecture/infrastructure/llm/lectureLlm.ts create mode 100644 src/modules/lecture/infrastructure/llm/structuredOutput.ts create mode 100644 src/modules/lecture/infrastructure/pdf/chunker.ts create mode 100644 src/modules/lecture/infrastructure/pdf/fileHash.ts create mode 100644 src/modules/lecture/infrastructure/platform/executable.ts create mode 100644 src/modules/lecture/infrastructure/platform/os.ts create mode 100644 src/modules/lecture/infrastructure/platform/paths.ts create mode 100644 src/modules/lecture/infrastructure/platform/subprocess.ts create mode 100644 src/modules/lecture/infrastructure/storage/audioStorage.ts create mode 100644 src/modules/lecture/infrastructure/storage/lectureStore.ts create mode 100644 src/modules/lecture/infrastructure/storage/paths.ts create mode 100644 src/modules/lecture/infrastructure/storage/repository.ts create mode 100644 src/modules/lecture/infrastructure/storage/schema.ts create mode 100644 src/modules/lecture/infrastructure/tts/client.ts create mode 100644 src/modules/lecture/infrastructure/tts/local/macSay.ts create mode 100644 src/modules/lecture/infrastructure/tts/local/piper.ts create mode 100644 src/modules/lecture/infrastructure/tts/local/windowsSapi.ts create mode 100644 src/modules/lecture/infrastructure/tts/providers.ts create mode 100644 src/modules/lecture/infrastructure/tts/types.ts create mode 100644 src/modules/lecture/infrastructure/zotero/adapter.ts create mode 100644 src/modules/lecture/ui/lecturePane.ts create mode 100644 src/modules/lecture/ui/registerLecturePane.ts create mode 100644 src/modules/llm/jsonParse.ts create mode 100644 src/modules/ui/lecturePrefs.ts diff --git a/addon/content/chatpapers.css b/addon/content/chatpapers.css index 57e0b58..7b830c9 100644 --- a/addon/content/chatpapers.css +++ b/addon/content/chatpapers.css @@ -752,3 +752,215 @@ .chatpapers-cite-btn { flex-shrink: 0; } + +/* --- Voice lecture pane --- */ +.chatpapers-lecture-root { + gap: 10px; +} + +.chatpapers-lecture-empty { + padding: 20px 12px; +} + +.chatpapers-lecture-status { + padding: 8px 10px; + border-radius: 8px; + font-size: 12px; + line-height: 1.45; + background: color-mix(in srgb, #1f7a8c 10%, transparent); + border: 1px solid color-mix(in srgb, #1f7a8c 22%, transparent); +} + +.chatpapers-lecture-status[data-status="failed"] { + background: color-mix(in srgb, #dc2626 10%, transparent); + border-color: color-mix(in srgb, #dc2626 25%, transparent); +} + +.chatpapers-lecture-status[data-status="ready"] { + background: color-mix(in srgb, #16a34a 10%, transparent); + border-color: color-mix(in srgb, #16a34a 25%, transparent); +} + +.chatpapers-lecture-actions { + display: flex; + gap: 8px; +} + +.chatpapers-lecture-primary { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 8px 14px; + border: none; + border-radius: 8px; + background: #1f7a8c; + color: #fff; + font-size: 13px; + font-weight: 600; + cursor: pointer; +} + +.chatpapers-lecture-primary:hover:not(:disabled) { + background: #186678; +} + +.chatpapers-lecture-status-text { + white-space: pre-line; +} + +.chatpapers-lecture-primary:disabled { + opacity: 0.55; + cursor: not-allowed; +} + +.chatpapers-lecture-secondary { + display: inline-flex; + align-items: center; + gap: 6px; + padding: 8px 14px; + border: 1px solid color-mix(in srgb, #1f7a8c 35%, transparent); + border-radius: 8px; + background: transparent; + color: inherit; + font-size: 13px; + font-weight: 600; + cursor: pointer; +} + +.chatpapers-lecture-secondary:hover:not(:disabled) { + background: color-mix(in srgb, #1f7a8c 10%, transparent); +} + +.chatpapers-lecture-secondary:disabled { + opacity: 0.55; + cursor: not-allowed; +} + +.chatpapers-lecture-tts-meta { + font-size: 12px; + opacity: 0.75; +} + +.chatpapers-lecture-audio { + width: 100%; + margin-top: 4px; +} + +.chatpapers-lecture-btn-icon { + flex-shrink: 0; +} + +.chatpapers-lecture-hint { + font-size: 11px; + opacity: 0.65; + line-height: 1.4; +} + +.chatpapers-lecture-player { + display: flex; + flex-direction: column; + gap: 8px; + margin-top: 4px; +} + +.chatpapers-lecture-player-title { + font-size: 12px; + font-weight: 600; + opacity: 0.85; +} + +.chatpapers-lecture-beat-list { + display: flex; + flex-direction: column; + gap: 4px; + max-height: 220px; + overflow-y: auto; +} + +.chatpapers-lecture-beat-item { + display: flex; + align-items: center; + justify-content: space-between; + gap: 8px; + padding: 8px 10px; + border: 1px solid color-mix(in srgb, #1f7a8c 18%, transparent); + border-radius: 8px; + background: color-mix(in srgb, #1f7a8c 4%, transparent); + cursor: pointer; + text-align: left; + font: inherit; + color: inherit; +} + +.chatpapers-lecture-beat-item:hover:not(:disabled) { + background: color-mix(in srgb, #1f7a8c 10%, transparent); +} + +.chatpapers-lecture-beat-item.is-active { + border-color: color-mix(in srgb, #16a34a 45%, transparent); + background: color-mix(in srgb, #16a34a 10%, transparent); +} + +.chatpapers-lecture-beat-item.is-pending { + opacity: 0.55; + cursor: default; +} + +.chatpapers-lecture-beat-meta { + display: flex; + flex-direction: column; + gap: 2px; + min-width: 0; +} + +.chatpapers-lecture-beat-type { + font-size: 10px; + font-weight: 600; + opacity: 0.65; + text-transform: uppercase; + letter-spacing: 0.03em; +} + +.chatpapers-lecture-beat-title { + font-size: 12px; + line-height: 1.35; + overflow: hidden; + text-overflow: ellipsis; + white-space: nowrap; +} + +.chatpapers-lecture-beat-status { + flex-shrink: 0; + font-size: 11px; + opacity: 0.7; +} + +.chatpapers-lecture-player-controls { + display: flex; + align-items: center; + justify-content: center; + gap: 8px; +} + +.chatpapers-lecture-player-btn { + display: inline-flex; + align-items: center; + justify-content: center; + width: 36px; + height: 36px; + border: 1px solid color-mix(in srgb, #1f7a8c 30%, transparent); + border-radius: 999px; + background: transparent; + cursor: pointer; + color: inherit; +} + +.chatpapers-lecture-player-btn:hover { + background: color-mix(in srgb, #1f7a8c 12%, transparent); +} + +.chatpapers-lecture-error { + padding: 12px; + color: #dc2626; + font-size: 12px; +} diff --git a/addon/content/icons/lecture.svg b/addon/content/icons/lecture.svg new file mode 100644 index 0000000..7aa8efe --- /dev/null +++ b/addon/content/icons/lecture.svg @@ -0,0 +1,28 @@ + + + + + + + + diff --git a/addon/content/preferences.xhtml b/addon/content/preferences.xhtml index 03a28b1..5e3f456 100644 --- a/addon/content/preferences.xhtml +++ b/addon/content/preferences.xhtml @@ -116,6 +116,69 @@ + + + + + + + + + + + + + + + + + + + + + + + diff --git a/addon/locale/en-US/addon.ftl b/addon/locale/en-US/addon.ftl index c15187c..9809c6a 100644 --- a/addon/locale/en-US/addon.ftl +++ b/addon/locale/en-US/addon.ftl @@ -58,3 +58,38 @@ cite-empty = No styles available cite-copied = Bibliography copied cite-exported = Bibliography exported cite-export-cancel = Export cancelled + +item-section-lecture-head = Voice lecture +item-section-lecture-sidenav = Voice lecture + +lecture-loading = Loading voice lecture… +lecture-mode-label = Voice lecture · Prepare then listen +lecture-empty-title = AI lecture mode +lecture-empty-desc = One-click prep: overview and paragraph-by-paragraph listening, with typed Q&A and spoken answers. +lecture-start = Start preparation +lecture-status-idle = Click Start preparation — AI reads the full paper first +lecture-status-preparing = Preparing lecture… +lecture-status-ready = Lecture ready ✓ +lecture-status-ready-phase1 = Preparation done (Phase 1: summary ready) +lecture-status-ready-step1 = Ready: { $count } segments — tap a segment or press Play +lecture-status-failed = Preparation incomplete +lecture-beat-list-title = Overview lecture +lecture-play-start = Play +lecture-play-pause = Pause +lecture-play-next = Next +lecture-play-prev = Previous +lecture-play-complete = Playback finished +lecture-beat-tts-pending = Synthesizing +lecture-beat-tts-failed = Failed +lecture-reprepare = Re-prepare +lecture-no-pdf = No PDF attachment on this item +lecture-attach-error = Cannot read PDF attachment +lecture-mount-error = Failed to load voice lecture pane +lecture-platform-hint-mac = macOS: set MinerU path under Preferences → ChatPapers → Voice lecture, or leave empty to use mineru on PATH. +lecture-platform-hint-win = Windows: set the full path to mineru.exe under Preferences → ChatPapers → Voice lecture. +lecture-platform-hint-other = Configure MinerU path in Preferences to enable structured PDF parsing. + +lecture-tts-provider = TTS: { $provider } +lecture-tts-test = Test voice +lecture-tts-testing = Synthesizing test speech… +lecture-tts-test-done = Speech ready — playing diff --git a/addon/locale/en-US/preferences.ftl b/addon/locale/en-US/preferences.ftl index 77a5155..4586908 100644 --- a/addon/locale/en-US/preferences.ftl +++ b/addon/locale/en-US/preferences.ftl @@ -27,3 +27,11 @@ prefs-hint-cloud = Cloud provider: paper text is sent to the selected API endpoi prefs-models-empty = No models returned. prefs-models-pick = Enter a model id from the list (showing up to 30): prefs-privacy = Privacy: ChatPapers only sends content when you click Send/Summarize, and only to the endpoint you configured. API keys are stored in local Zotero preferences. + +prefs-section-lecture = Voice lecture +prefs-tts-provider = TTS engine +prefs-tts-voice = Voice (optional) +prefs-piper-path = Piper executable +prefs-piper-model = Piper model path (.onnx) +prefs-mineru-path = MinerU path +prefs-tts-hint-empty = Choose a TTS engine. Local engines are fully offline — no API key needed. diff --git a/addon/locale/zh-CN/addon.ftl b/addon/locale/zh-CN/addon.ftl index 05e9c4e..c1a51bc 100644 --- a/addon/locale/zh-CN/addon.ftl +++ b/addon/locale/zh-CN/addon.ftl @@ -59,3 +59,37 @@ cite-copied = 参考文献已复制到剪贴板 cite-exported = 参考文献已导出 cite-export-cancel = 已取消导出 +item-section-lecture-head = 语音伴读 +item-section-lecture-sidenav = 语音伴读 + +lecture-loading = 正在加载语音伴读… +lecture-mode-label = 语音伴读 · 先备课再听 +lecture-empty-title = AI 老师式伴读 +lecture-empty-desc = 一键备课：整体讲解与逐段精读，随时文字提问并获得语音回答。 +lecture-start = 开始备课 +lecture-status-idle = 点击「开始备课」，AI 将先读完整篇论文 +lecture-status-preparing = 正在备课… +lecture-status-ready = 老师已备课 ✓ +lecture-status-ready-phase1 = 备课完成（Phase 1：摘要已生成） +lecture-status-ready-step1 = 备课完成：{ $count } 段讲解可播放，点击下方段落或播放按钮开始听课 +lecture-status-failed = 备课未完成，请重试或检查 LLM 设置 +lecture-beat-list-title = 整体讲解 +lecture-play-start = 播放 +lecture-play-pause = 暂停 +lecture-play-next = 下一段 +lecture-play-prev = 上一段 +lecture-play-complete = 本轮讲解播放完毕 +lecture-beat-tts-pending = 合成中 +lecture-beat-tts-failed = 失败 +lecture-reprepare = 重新备课 +lecture-no-pdf = 当前条目没有可用的 PDF 附件 +lecture-attach-error = 无法读取 PDF 附件 +lecture-mount-error = 语音伴读面板加载失败 +lecture-platform-hint-mac = macOS：MinerU 路径可在偏好设置 → ChatPapers → 语音伴读 中配置；留空则尝试 PATH 中的 mineru。 +lecture-platform-hint-win = Windows：请在偏好设置中填写 MinerU 可执行文件完整路径（如 mineru.exe）。 +lecture-platform-hint-other = 请在偏好设置中配置 MinerU 路径以启用 PDF 结构化解析。 + +lecture-tts-provider = TTS 引擎：{ $provider } +lecture-tts-test = 试听语音 +lecture-tts-testing = 正在合成测试语音… +lecture-tts-test-done = 语音合成完成，正在播放 diff --git a/addon/locale/zh-CN/preferences.ftl b/addon/locale/zh-CN/preferences.ftl index 4ab3ad2..8d950a3 100644 --- a/addon/locale/zh-CN/preferences.ftl +++ b/addon/locale/zh-CN/preferences.ftl @@ -27,3 +27,11 @@ prefs-hint-cloud = 云端 Provider：论文内容将发送到所选 API 端点 prefs-models-empty = 未返回任何模型。 prefs-models-pick = 从列表中输入模型 ID（最多显示 30 个）： prefs-privacy = 隐私说明：仅在你点击发送/总结时，向你配置的端点发送内容。API Key 仅保存在本地 Zotero 偏好设置中。 + +prefs-section-lecture = 语音伴读 +prefs-tts-provider = TTS 引擎 +prefs-tts-voice = 音色（可选） +prefs-piper-path = Piper 可执行文件 +prefs-piper-model = Piper 模型路径 (.onnx) +prefs-mineru-path = MinerU 路径 +prefs-tts-hint-empty = 请选择 TTS 引擎。本地引擎完全离线，无需 API Key。 diff --git a/addon/manifest.json b/addon/manifest.json index 78d39d7..e0f68d8 100644 --- a/addon/manifest.json +++ b/addon/manifest.json @@ -14,7 +14,7 @@ "id": "__addonID__", "update_url": "__updateURL__", "strict_min_version": "9.0", - "strict_max_version": "9.*" + "strict_max_version": "10.*" } } } diff --git a/addon/prefs.js b/addon/prefs.js index 9cc9ccd..fd136fd 100644 --- a/addon/prefs.js +++ b/addon/prefs.js @@ -13,3 +13,10 @@ pref("systemPrompt", ""); pref("summaryPrompt", ""); pref("sendMetadata", true); pref("verboseLog", false); +pref("ttsProvider", ""); +pref("ttsApiKey", ""); +pref("ttsVoice", ""); +pref("lectureTeacherStyle", "balanced"); +pref("mineruPath", ""); +pref("piperPath", ""); +pref("piperModelPath", ""); diff --git a/docs/newidea/ARCHITECTURE.md b/docs/newidea/ARCHITECTURE.md new file mode 100644 index 0000000..2dc2541 --- /dev/null +++ b/docs/newidea/ARCHITECTURE.md @@ -0,0 +1,680 @@ +# 软件架构设计文档 + +| 字段 | 内容 | +|------|------| +| 产品名称 | BanDu（伴读）— Zotero 插件 | +| 中文名 | 伴读 | +| 英文名 | BanDu | +| 文档版本 | v0.1 | +| 状态 | 草案 | +| 关联文档 | [PLAN.md](./PLAN.md)、[PRD.md](./PRD.md) | + +--- + +## 1. 架构概述 + +### 1.1 架构风格 + +**Local-first 桌面插件架构**：核心计算与缓存均在用户机器完成；仅将段落文本与用户问题发送至用户配置的 LLM / TTS 服务。无自建后端（一期）。 + +### 1.2 设计原则 + +| 原则 | 说明 | +|------|------| +| 备课一次、缓存复用 | Lecture 持久化；attachment ID + 文件哈希为缓存键 | +| 增量交付 | 摘要 → beats → 逐段 → TTS 分阶段就绪，缩短 Time-to-Listen | +| UI / 领域 / 集成 分离 | 降低 Zotero API 变动影响 | +| Grounding 优先 | 生成内容必须可追溯到段落 ID | +| 优雅降级 | refs 无效、无 timestamps、无 BBT 均有 fallback | + +### 1.3 系统上下文 + +``` + ┌─────────────────┐ + │ 用户 │ + └────────┬────────┘ + │ + ┌──────────────▼──────────────┐ + │ Zotero 7 (宿主) │ + │ ┌────────────────────────┐ │ + │ │ BanDu 插件 │ │ + │ │ UI │ Engine │ Store │ │ + │ └─────────┬──────────────┘ │ + └────────────┼────────────────┘ + │ + ┌─────────────────┼─────────────────┐ + │ │ │ + ▼ ▼ ▼ + ┌───────────┐ ┌────────────┐ ┌─────────────┐ + │ MinerU │ │ LLM API │ │ TTS API │ + │ (本地子进程)│ │ / Ollama │ │ ElevenLabs等 │ + └───────────┘ └────────────┘ └─────────────┘ + │ + ▼ + ┌───────────┐ + │ SQLite + │ + │ mp3 文件 │ + └───────────┘ +``` + +--- + +## 2. 逻辑分层 + +``` +┌─────────────────────────────────────────────────────────┐ +│ Presentation Layer（表现层） │ +│ • BanduPane（Item pane 伴读页） │ +│ • LibraryWindow（伴读库） │ +│ • SettingsPane（设置） │ +│ • PlayerController（播放状态机 +