From a0047d5c29396ac066b3f0e2c7b849397f60a4c7 Mon Sep 17 00:00:00 2001 From: bdim404 Date: Wed, 11 Mar 2026 14:08:09 +0800 Subject: [PATCH] feat: Add batch processing for audiobook chapters including parse, generate, and combined process actions. --- qwen3-tts-backend/api/audiobook.py | 60 +++++++++ qwen3-tts-backend/config.py | 3 + qwen3-tts-backend/core/audiobook_service.py | 107 ++++++++++++++- qwen3-tts-frontend/src/lib/api/audiobook.ts | 8 ++ .../src/locales/en-US/audiobook.json | 6 +- .../src/locales/ja-JP/audiobook.json | 6 +- .../src/locales/ko-KR/audiobook.json | 6 +- .../src/locales/zh-CN/audiobook.json | 6 +- .../src/locales/zh-TW/audiobook.json | 6 +- qwen3-tts-frontend/src/pages/Audiobook.tsx | 122 ++++++++++++++---- 10 files changed, 296 insertions(+), 34 deletions(-) diff --git a/qwen3-tts-backend/api/audiobook.py b/qwen3-tts-backend/api/audiobook.py index 3975d9f..f33a6fd 100644 --- a/qwen3-tts-backend/api/audiobook.py +++ b/qwen3-tts-backend/api/audiobook.py @@ -267,6 +267,66 @@ async def parse_chapter( return {"message": "Parsing started", "chapter_id": chapter_id} +@router.post("/projects/{project_id}/parse-all") +async def parse_all_chapters_endpoint( + project_id: int, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db), +): + project = crud.get_audiobook_project(db, project_id, current_user.id) + if not project: + raise HTTPException(status_code=404, detail="Project not found") + if project.status not in ("ready", "done", "error"): + raise HTTPException(status_code=400, detail=f"Project must be in 'ready' state, current: {project.status}") + + if not current_user.llm_api_key or not current_user.llm_base_url or not current_user.llm_model: + raise HTTPException(status_code=400, detail="LLM config not set") + + from core.audiobook_service import parse_all_chapters + from core.database import SessionLocal + + async def run(): + async_db = SessionLocal() + try: + db_user = crud.get_user_by_id(async_db, current_user.id) + await parse_all_chapters(project_id, db_user, async_db) + finally: + async_db.close() + + asyncio.create_task(run()) + return {"message": "Batch parsing started", "project_id": project_id} + + +@router.post("/projects/{project_id}/process-all") +async def process_all_endpoint( + project_id: int, + current_user: User = Depends(get_current_user), + db: Session = Depends(get_db), +): + project = crud.get_audiobook_project(db, project_id, current_user.id) + if not project: + raise HTTPException(status_code=404, detail="Project not found") + if project.status not in ("ready", "generating", "done", "error"): + raise HTTPException(status_code=400, detail=f"Project must be in 'ready' state, current: {project.status}") + + if not current_user.llm_api_key or not current_user.llm_base_url or not current_user.llm_model: + raise HTTPException(status_code=400, detail="LLM config not set") + + from core.audiobook_service import process_all + from core.database import SessionLocal + + async def run(): + async_db = SessionLocal() + try: + db_user = crud.get_user_by_id(async_db, current_user.id) + await process_all(project_id, db_user, async_db) + finally: + async_db.close() + + asyncio.create_task(run()) + return {"message": "Full processing started", "project_id": project_id} + + @router.put("/projects/{project_id}/characters/{char_id}", response_model=AudiobookCharacterResponse) async def update_character( project_id: int, diff --git a/qwen3-tts-backend/config.py b/qwen3-tts-backend/config.py index aeafe90..14582b3 100644 --- a/qwen3-tts-backend/config.py +++ b/qwen3-tts-backend/config.py @@ -44,6 +44,9 @@ class Settings(BaseSettings): DEFAULT_BACKEND: str = Field(default="local") + AUDIOBOOK_PARSE_CONCURRENCY: int = Field(default=3) + AUDIOBOOK_GENERATE_CONCURRENCY: int = Field(default=2) + class Config: env_file = ".env" case_sensitive = True diff --git a/qwen3-tts-backend/core/audiobook_service.py b/qwen3-tts-backend/core/audiobook_service.py index 0f27a25..42bbde6 100644 --- a/qwen3-tts-backend/core/audiobook_service.py +++ b/qwen3-tts-backend/core/audiobook_service.py @@ -281,6 +281,8 @@ async def parse_one_chapter(project_id: int, chapter_id: int, user: User, db) -> ps.append_line(key, f"共 {len(chunks)} 块\n") seg_counter = 0 + failed_chunks = 0 + last_error = "" for i, chunk in enumerate(chunks): ps.append_line(key, f"块 {i + 1}/{len(chunks)} → ") ps.append_line(key, "") @@ -293,6 +295,8 @@ async def parse_one_chapter(project_id: int, chapter_id: int, user: User, db) -> except Exception as e: logger.warning(f"Chapter {chapter_id} chunk {i} failed: {e}") ps.append_line(key, f"\n[回退] {e}") + failed_chunks += 1 + last_error = str(e) narrator = char_map.get("narrator") if narrator: crud.create_audiobook_segment( @@ -319,8 +323,18 @@ async def parse_one_chapter(project_id: int, chapter_id: int, user: User, db) -> ps.append_line(key, f"\n✓ {chunk_count} 段") - ps.append_line(key, f"\n[完成] 共 {seg_counter} 段") - crud.update_audiobook_chapter_status(db, chapter_id, "ready") + if failed_chunks == len(chunks): + # All chunks failed — mark chapter as error, remove fallback segments + crud.delete_audiobook_segments_for_chapter(db, project_id, chapter.chapter_index) + error_msg = f"所有 {len(chunks)} 个块均解析失败: {last_error}" + ps.append_line(key, f"\n[错误] {error_msg}") + crud.update_audiobook_chapter_status(db, chapter_id, "error", error_message=error_msg) + elif failed_chunks > 0: + ps.append_line(key, f"\n[完成] 共 {seg_counter} 段（{failed_chunks}/{len(chunks)} 块回退到旁白）") + crud.update_audiobook_chapter_status(db, chapter_id, "ready") + else: + ps.append_line(key, f"\n[完成] 共 {seg_counter} 段") + crud.update_audiobook_chapter_status(db, chapter_id, "ready") ps.mark_done(key) logger.info(f"Chapter {chapter_id} parsed: {seg_counter} segments") @@ -545,3 +559,92 @@ def merge_audio_files(audio_paths: list[str], output_path: str) -> None: if combined: Path(output_path).parent.mkdir(parents=True, exist_ok=True) combined.export(output_path, format="wav") + + +async def parse_all_chapters(project_id: int, user: User, db: Session) -> None: + """Concurrently parse all pending/error/ready chapters using asyncio.Semaphore.""" + from core.database import SessionLocal + + chapters = crud.list_audiobook_chapters(db, project_id) + pending = [ch for ch in chapters if ch.status in ("pending", "error", "ready")] + if not pending: + return + + max_concurrent = settings.AUDIOBOOK_PARSE_CONCURRENCY + semaphore = asyncio.Semaphore(max_concurrent) + logger.info(f"parse_all_chapters: project={project_id}, {len(pending)} chapters, concurrency={max_concurrent}") + + async def parse_with_limit(chapter): + async with semaphore: + task_db = SessionLocal() + try: + db_user = crud.get_user_by_id(task_db, user.id) + await parse_one_chapter(project_id, chapter.id, db_user, task_db) + except Exception as e: + logger.error(f"parse_all_chapters: chapter {chapter.id} failed: {e}", exc_info=True) + finally: + task_db.close() + + await asyncio.gather(*[parse_with_limit(ch) for ch in pending]) + logger.info(f"parse_all_chapters: project={project_id} complete") + + +async def generate_all_chapters(project_id: int, user: User, db: Session) -> None: + """Concurrently generate audio for all ready chapters using asyncio.Semaphore.""" + from core.database import SessionLocal + + chapters = crud.list_audiobook_chapters(db, project_id) + ready = [ch for ch in chapters if ch.status == "ready"] + if not ready: + return + + crud.update_audiobook_project_status(db, project_id, "generating") + + max_concurrent = settings.AUDIOBOOK_GENERATE_CONCURRENCY + semaphore = asyncio.Semaphore(max_concurrent) + logger.info(f"generate_all_chapters: project={project_id}, {len(ready)} chapters, concurrency={max_concurrent}") + + async def generate_with_limit(chapter): + async with semaphore: + task_db = SessionLocal() + try: + db_user = crud.get_user_by_id(task_db, user.id) + await generate_project(project_id, db_user, task_db, chapter_index=chapter.chapter_index) + except Exception as e: + logger.error(f"generate_all_chapters: chapter {chapter.chapter_index} failed: {e}", exc_info=True) + finally: + task_db.close() + + await asyncio.gather(*[generate_with_limit(ch) for ch in ready]) + + # Check final project status + final_db = SessionLocal() + try: + all_segs = crud.list_audiobook_segments(final_db, project_id) + all_done = all(s.status == "done" for s in all_segs) if all_segs else False + if all_done: + crud.update_audiobook_project_status(final_db, project_id, "done") + else: + crud.update_audiobook_project_status(final_db, project_id, "ready") + finally: + final_db.close() + + logger.info(f"generate_all_chapters: project={project_id} complete") + + +async def process_all(project_id: int, user: User, db: Session) -> None: + """Parse all pending chapters, then generate all ready chapters — both with concurrency.""" + from core.database import SessionLocal + + # Phase 1: parse all pending chapters concurrently + await parse_all_chapters(project_id, user, db) + + # Phase 2: reload chapters and generate all ready ones concurrently + phase2_db = SessionLocal() + try: + await generate_all_chapters(project_id, user, phase2_db) + finally: + phase2_db.close() + + logger.info(f"process_all: project={project_id} complete") + diff --git a/qwen3-tts-frontend/src/lib/api/audiobook.ts b/qwen3-tts-frontend/src/lib/api/audiobook.ts index b7bb1c4..2607b7b 100644 --- a/qwen3-tts-frontend/src/lib/api/audiobook.ts +++ b/qwen3-tts-frontend/src/lib/api/audiobook.ts @@ -139,6 +139,14 @@ export const audiobookApi = { return `/audiobook/projects/${projectId}/segments/${segmentId}/audio` }, + parseAllChapters: async (projectId: number): Promise => { + await apiClient.post(`/audiobook/projects/${projectId}/parse-all`) + }, + + processAll: async (projectId: number): Promise => { + await apiClient.post(`/audiobook/projects/${projectId}/process-all`) + }, + deleteProject: async (id: number): Promise => { await apiClient.delete(`/audiobook/projects/${id}`) }, diff --git a/qwen3-tts-frontend/src/locales/en-US/audiobook.json b/qwen3-tts-frontend/src/locales/en-US/audiobook.json index 59fb5d4..2779f86 100644 --- a/qwen3-tts-frontend/src/locales/en-US/audiobook.json +++ b/qwen3-tts-frontend/src/locales/en-US/audiobook.json @@ -57,6 +57,7 @@ "reanalyzeConfirm": "Re-analyzing will clear all character and chapter data. Continue?", "analyzeStarted": "Analysis started", "generateAll": "Generate Full Book", + "processAll": "⚡ Process All", "downloadAll": "Download Full Book", "deleteConfirm": "Delete project \"{{title}}\" and all its audio?", "deleteSuccess": "Project deleted", @@ -85,7 +86,9 @@ "chapters": { "title": "Chapters ({{count}} total)", - "processAll": "Process All", + "processAll": "⚡ Process All", + "parseAll": "Batch Parse", + "generateAll": "Batch Generate", "defaultTitle": "Chapter {{index}}", "parse": "Parse Chapter", "parsing": "Parsing", @@ -96,6 +99,7 @@ "generateStarted": "Chapter {{index}} generation started", "generateAllStarted": "Full book generation started", "processAllStarted": "All tasks triggered", + "parseAllStarted": "Batch parsing started", "doneBadge": "{{count}} segments done", "segmentProgress": "{{done}}/{{total}} segments" }, diff --git a/qwen3-tts-frontend/src/locales/ja-JP/audiobook.json b/qwen3-tts-frontend/src/locales/ja-JP/audiobook.json index 87799de..6b3f0b1 100644 --- a/qwen3-tts-frontend/src/locales/ja-JP/audiobook.json +++ b/qwen3-tts-frontend/src/locales/ja-JP/audiobook.json @@ -57,6 +57,7 @@ "reanalyzeConfirm": "再分析するとすべてのキャラクターと章のデータが削除されます。続けますか？", "analyzeStarted": "分析を開始しました", "generateAll": "全冊生成", + "processAll": "⚡ 全冊一括処理", "downloadAll": "全冊ダウンロード", "deleteConfirm": "プロジェクト「{{title}}」とすべての音声を削除しますか？", "deleteSuccess": "プロジェクトを削除しました", @@ -85,7 +86,9 @@ "chapters": { "title": "章一覧（全 {{count}} 章）", - "processAll": "すべて処理", + "processAll": "⚡ すべて処理", + "parseAll": "一括解析", + "generateAll": "一括生成", "defaultTitle": "第 {{index}} 章", "parse": "この章を解析", "parsing": "解析中", @@ -96,6 +99,7 @@ "generateStarted": "第 {{index}} 章の生成を開始しました", "generateAllStarted": "全冊生成を開始しました", "processAllStarted": "すべてのタスクを開始しました", + "parseAllStarted": "一括解析を開始しました", "doneBadge": "{{count}} セグメント完了", "segmentProgress": "{{done}}/{{total}} セグメント" }, diff --git a/qwen3-tts-frontend/src/locales/ko-KR/audiobook.json b/qwen3-tts-frontend/src/locales/ko-KR/audiobook.json index 4609154..cec6aac 100644 --- a/qwen3-tts-frontend/src/locales/ko-KR/audiobook.json +++ b/qwen3-tts-frontend/src/locales/ko-KR/audiobook.json @@ -57,6 +57,7 @@ "reanalyzeConfirm": "재분석하면 모든 캐릭터와 챕터 데이터가 삭제됩니다. 계속하시겠습니까?", "analyzeStarted": "분석이 시작되었습니다", "generateAll": "전체 책 생성", + "processAll": "⚡ 전체 일괄 처리", "downloadAll": "전체 책 다운로드", "deleteConfirm": "프로젝트 「{{title}}」와 모든 음성을 삭제하시겠습니까?", "deleteSuccess": "프로젝트가 삭제되었습니다", @@ -85,7 +86,9 @@ "chapters": { "title": "챕터 목록 (총 {{count}}챕터)", - "processAll": "전체 처리", + "processAll": "⚡ 전체 처리", + "parseAll": "일괄 파싱", + "generateAll": "일괄 생성", "defaultTitle": "제 {{index}} 장", "parse": "이 챕터 파싱", "parsing": "파싱 중", @@ -96,6 +99,7 @@ "generateStarted": "제 {{index}} 장 생성이 시작되었습니다", "generateAllStarted": "전체 책 생성이 시작되었습니다", "processAllStarted": "모든 작업이 시작되었습니다", + "parseAllStarted": "일괄 파싱이 시작되었습니다", "doneBadge": "{{count}}개 세그먼트 완료", "segmentProgress": "{{done}}/{{total}} 세그먼트" }, diff --git a/qwen3-tts-frontend/src/locales/zh-CN/audiobook.json b/qwen3-tts-frontend/src/locales/zh-CN/audiobook.json index cc0b990..d698cca 100644 --- a/qwen3-tts-frontend/src/locales/zh-CN/audiobook.json +++ b/qwen3-tts-frontend/src/locales/zh-CN/audiobook.json @@ -57,6 +57,7 @@ "reanalyzeConfirm": "重新分析将清除所有角色和章节数据，确定继续？", "analyzeStarted": "分析已开始", "generateAll": "生成全书", + "processAll": "⚡ 全书一键处理", "downloadAll": "下载全书", "deleteConfirm": "确认删除项目「{{title}}」及所有音频？", "deleteSuccess": "项目已删除", @@ -85,7 +86,9 @@ "chapters": { "title": "章节列表（共 {{count}} 章）", - "processAll": "一键全部处理", + "processAll": "⚡ 全部处理", + "parseAll": "批量解析", + "generateAll": "批量生成", "defaultTitle": "第 {{index}} 章", "parse": "解析此章", "parsing": "解析中", @@ -96,6 +99,7 @@ "generateStarted": "第 {{index}} 章生成已开始", "generateAllStarted": "全书生成已开始", "processAllStarted": "全部任务已触发", + "parseAllStarted": "批量解析已开始", "doneBadge": "已完成 {{count}} 段", "segmentProgress": "{{done}}/{{total}} 段" }, diff --git a/qwen3-tts-frontend/src/locales/zh-TW/audiobook.json b/qwen3-tts-frontend/src/locales/zh-TW/audiobook.json index fe28992..fdee34a 100644 --- a/qwen3-tts-frontend/src/locales/zh-TW/audiobook.json +++ b/qwen3-tts-frontend/src/locales/zh-TW/audiobook.json @@ -57,6 +57,7 @@ "reanalyzeConfirm": "重新分析將清除所有角色和章節資料，確定繼續？", "analyzeStarted": "分析已開始", "generateAll": "生成全書", + "processAll": "⚡ 全書一鍵處理", "downloadAll": "下載全書", "deleteConfirm": "確認刪除專案「{{title}}」及所有音訊？", "deleteSuccess": "專案已刪除", @@ -85,7 +86,9 @@ "chapters": { "title": "章節列表（共 {{count}} 章）", - "processAll": "一鍵全部處理", + "processAll": "⚡ 全部處理", + "parseAll": "批量解析", + "generateAll": "批量生成", "defaultTitle": "第 {{index}} 章", "parse": "解析此章", "parsing": "解析中", @@ -96,6 +99,7 @@ "generateStarted": "第 {{index}} 章生成已開始", "generateAllStarted": "全書生成已開始", "processAllStarted": "全部任務已觸發", + "parseAllStarted": "批量解析已開始", "doneBadge": "已完成 {{count}} 段", "segmentProgress": "{{done}}/{{total}} 段" }, diff --git a/qwen3-tts-frontend/src/pages/Audiobook.tsx b/qwen3-tts-frontend/src/pages/Audiobook.tsx index d25ab50..5fab0a0 100644 --- a/qwen3-tts-frontend/src/pages/Audiobook.tsx +++ b/qwen3-tts-frontend/src/pages/Audiobook.tsx @@ -495,6 +495,43 @@ function ProjectCard({ project, onRefresh }: { project: AudiobookProject; onRefr } } + const handleParseAll = async () => { + setLoadingAction(true) + setIsPolling(true) + try { + await audiobookApi.parseAllChapters(project.id) + toast.success(t('projectCard.chapters.parseAllStarted')) + onRefresh() + fetchDetail() + } catch (e: any) { + setIsPolling(false) + toast.error(formatApiError(e)) + } finally { + setLoadingAction(false) + } + } + + const handleGenerateAll = async () => { + if (!detail) return + setLoadingAction(true) + const ready = detail.chapters.filter(c => c.status === 'ready') + if (ready.length > 0) { + setGeneratingChapterIndices(prev => new Set([...prev, ...ready.map(c => c.chapter_index)])) + } + setIsPolling(true) + try { + await audiobookApi.generate(project.id) + toast.success(t('projectCard.chapters.generateAllStarted')) + onRefresh() + fetchSegments() + } catch (e: any) { + setIsPolling(false) + toast.error(formatApiError(e)) + } finally { + setLoadingAction(false) + } + } + const handleProcessAll = async () => { if (!detail) return setLoadingAction(true) @@ -504,11 +541,7 @@ function ProjectCard({ project, onRefresh }: { project: AudiobookProject; onRefr } setIsPolling(true) try { - const pending = detail.chapters.filter(c => c.status === 'pending' || c.status === 'error') - await Promise.all([ - ...pending.map(c => audiobookApi.parseChapter(project.id, c.id)), - ...ready.map(c => audiobookApi.generate(project.id, c.chapter_index)), - ]) + await audiobookApi.processAll(project.id) toast.success(t('projectCard.chapters.processAllStarted')) onRefresh() fetchDetail() @@ -630,7 +663,7 @@ function ProjectCard({ project, onRefresh }: { project: AudiobookProject; onRefr
- {!isActive && ( + {!isActive && status !== 'characters_ready' && ( )} {status === 'done' && ( @@ -756,16 +790,40 @@ function ProjectCard({ project, onRefresh }: { project: AudiobookProject; onRefr {chaptersCollapsed ? : } {t('projectCard.chapters.title', { count: detail.chapters.length })} - {detail.chapters.some(c => ['pending', 'error', 'ready'].includes(c.status)) && ( - - )} +
+ {detail.chapters.some(c => ['pending', 'error', 'ready'].includes(c.status)) && ( + + )} + {detail.chapters.some(c => c.status === 'ready') && ( + + )} + {detail.chapters.some(c => ['pending', 'error'].includes(c.status)) && detail.chapters.some(c => c.status === 'ready') && ( + + )} +
{!chaptersCollapsed &&
{detail.chapters.map(ch => { @@ -805,12 +863,17 @@ function ProjectCard({ project, onRefresh }: { project: AudiobookProject; onRefr
)} {ch.status === 'ready' && !chGenerating && !chAllDone && !generatingChapterIndices.has(ch.chapter_index) && ( - + <> + + + )} {ch.status === 'ready' && (chGenerating || generatingChapterIndices.has(ch.chapter_index)) && (
@@ -829,9 +892,14 @@ function ProjectCard({ project, onRefresh }: { project: AudiobookProject; onRefr )} {ch.status === 'error' && ( - + <> + + {ch.error_message && ( + {ch.error_message} + )} + )}
{ch.status === 'parsing' && (