+ setCurrentView('upload')}
/>
)}
@@ -40,4 +87,12 @@ function App() {
);
}
+function App() {
+ return (
+
+
+ );
+}
+
export default App;
diff --git a/src/components/EditorScreen.test.tsx b/src/components/EditorScreen.test.tsx
index dc930bd..4b79516 100644
--- a/src/components/EditorScreen.test.tsx
+++ b/src/components/EditorScreen.test.tsx
@@ -4,6 +4,13 @@ import React from 'react';
import { cleanup, fireEvent, render, screen, waitFor } from '@testing-library/react';
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
import EditorScreen from './EditorScreen';
+import { Subtitle } from '../types';
+import { I18nProvider } from '../i18n';
+
+const defaultSubtitleDefaults = {
+ fontSize: 24,
+ bottomOffsetPercent: 10,
+};
const { generateSubtitlePipelineMock, generateTTSMock } = vi.hoisted(() => ({
generateSubtitlePipelineMock: vi.fn(),
@@ -19,6 +26,58 @@ vi.mock('../services/ttsService', () => ({
}));
describe('EditorScreen', () => {
+ const renderEditorScreen = (props: React.ComponentProps) =>
+ render(
+
+ ,
+ );
+
+ const styledSubtitles: Subtitle[] = [
+ {
+ id: 'sub-1',
+ startTime: 0,
+ endTime: 2,
+ originalText: 'Original one',
+ translatedText: 'Styled first subtitle',
+ speaker: 'Young Woman',
+ voiceId: 'voice-1',
+ textStyle: {
+ fontFamily: 'Arial',
+ fontSize: 24,
+ color: '#ffffff',
+ backgroundColor: 'transparent',
+ alignment: 'center',
+ isBold: false,
+ isItalic: false,
+ isUnderline: false,
+ strokeColor: '#000000',
+ strokeWidth: 2,
+ },
+ },
+ {
+ id: 'sub-2',
+ startTime: 2,
+ endTime: 4,
+ originalText: 'Original two',
+ translatedText: 'Styled second subtitle',
+ speaker: 'Young Man',
+ voiceId: 'voice-2',
+ textStyle: {
+ fontFamily: 'Roboto',
+ fontSize: 20,
+ color: '#00ff00',
+ backgroundColor: 'transparent',
+ alignment: 'center',
+ isBold: false,
+ isItalic: false,
+ isUnderline: false,
+ strokeColor: '#000000',
+ strokeWidth: 2,
+ },
+ },
+ ];
+
afterEach(() => {
cleanup();
});
@@ -50,28 +109,26 @@ describe('EditorScreen', () => {
});
it('shows a low-precision notice for fallback subtitle results', async () => {
- render(
- {}}
- />,
- );
+ renderEditorScreen({
+ videoFile: new File(['video'], 'clip.mp4', { type: 'video/mp4' }),
+ targetLanguage: 'en',
+ trimRange: null,
+ initialSubtitleDefaults: defaultSubtitleDefaults,
+ onBack: () => {},
+ });
expect(screen.getAllByLabelText(/llm/i)[0]).toHaveValue('doubao');
expect(await screen.findByText(/low-precision/i)).toBeInTheDocument();
});
it('regenerates subtitles with the selected llm provider', async () => {
- render(
- {}}
- />,
- );
+ renderEditorScreen({
+ videoFile: new File(['video'], 'clip.mp4', { type: 'video/mp4' }),
+ targetLanguage: 'en',
+ trimRange: null,
+ initialSubtitleDefaults: defaultSubtitleDefaults,
+ onBack: () => {},
+ });
await waitFor(() =>
expect(generateSubtitlePipelineMock).toHaveBeenCalledWith(
@@ -79,6 +136,9 @@ describe('EditorScreen', () => {
'en',
'doubao',
null,
+ undefined,
+ expect.any(Function),
+ undefined,
),
);
@@ -92,24 +152,182 @@ describe('EditorScreen', () => {
'en',
'gemini',
null,
+ undefined,
+ expect.any(Function),
+ undefined,
),
);
});
it('only auto-generates subtitles once in StrictMode', async () => {
render(
-
- {}}
- />
- ,
+
+
+ {}}
+ />
+
+ ,
);
await waitFor(() => expect(generateSubtitlePipelineMock).toHaveBeenCalled());
expect(generateSubtitlePipelineMock).toHaveBeenCalledTimes(1);
});
+
+ it('shows subtitle generation progress updates', async () => {
+ generateSubtitlePipelineMock.mockImplementation(
+ (_file, _lang, _provider, _trim, _fetch, onProgress) =>
+ new Promise(() => {
+ onProgress?.({
+ jobId: 'job-1',
+ requestId: 'req-1',
+ status: 'running',
+ stage: 'calling_provider',
+ progress: 70,
+ message: 'Calling provider',
+ });
+ }),
+ );
+
+ renderEditorScreen({
+ videoFile: new File(['video'], 'clip.mp4', { type: 'video/mp4' }),
+ targetLanguage: 'en',
+ trimRange: null,
+ initialSubtitleDefaults: defaultSubtitleDefaults,
+ onBack: () => {},
+ });
+
+ expect(await screen.findByText(/70%/i)).toBeInTheDocument();
+ expect(screen.getByText(/Calling provider/i)).toBeInTheDocument();
+ });
+
+ it('shows the active subtitle speaker and updates only the selected subtitle style by default', async () => {
+ generateSubtitlePipelineMock.mockResolvedValue({
+ subtitles: styledSubtitles,
+ speakers: [],
+ quality: 'full',
+ });
+
+ renderEditorScreen({
+ videoFile: new File(['video'], 'clip.mp4', { type: 'video/mp4' }),
+ targetLanguage: 'en',
+ trimRange: null,
+ initialSubtitleDefaults: defaultSubtitleDefaults,
+ onBack: () => {},
+ });
+
+ expect(await screen.findByText('Young Woman')).toBeInTheDocument();
+
+ fireEvent.click(screen.getByRole('button', { name: /toggle bold/i }));
+
+ const overlay = screen.getByTestId('video-subtitle-overlay');
+ expect(overlay).toHaveStyle({ fontWeight: '700' });
+
+ fireEvent.click(screen.getByDisplayValue('Styled second subtitle'));
+ expect(await screen.findByText('Young Man')).toBeInTheDocument();
+
+ const colorInput = screen.getByLabelText(/subtitle color/i);
+ fireEvent.change(colorInput, { target: { value: '#ff0000' } });
+
+ fireEvent.click(screen.getByDisplayValue('Styled first subtitle'));
+ expect(screen.getByLabelText(/subtitle color/i)).toHaveValue('#ffffff');
+ });
+
+ it('applies style changes to every subtitle when apply to all subtitles is enabled', async () => {
+ generateSubtitlePipelineMock.mockResolvedValue({
+ subtitles: styledSubtitles,
+ speakers: [],
+ quality: 'full',
+ });
+
+ renderEditorScreen({
+ videoFile: new File(['video'], 'clip.mp4', { type: 'video/mp4' }),
+ targetLanguage: 'en',
+ trimRange: null,
+ initialSubtitleDefaults: defaultSubtitleDefaults,
+ onBack: () => {},
+ });
+
+ await screen.findByText('Young Woman');
+
+ fireEvent.click(screen.getByLabelText(/apply style changes to all subtitles/i));
+ fireEvent.change(screen.getByLabelText(/subtitle color/i), {
+ target: { value: '#ff0000' },
+ });
+
+ fireEvent.click(screen.getByDisplayValue('Styled second subtitle'));
+ expect(screen.getByLabelText(/subtitle color/i)).toHaveValue('#ff0000');
+ });
+
+ it('initializes the overlay from upload subtitle defaults', async () => {
+ generateSubtitlePipelineMock.mockResolvedValue({
+ subtitles: [
+ {
+ id: 'sub-default',
+ startTime: 0,
+ endTime: 2,
+ originalText: 'Original',
+ translatedText: 'Uses upload defaults',
+ speaker: 'Narrator',
+ voiceId: 'voice-1',
+ },
+ ],
+ speakers: [],
+ quality: 'full',
+ });
+
+ renderEditorScreen({
+ videoFile: new File(['video'], 'clip.mp4', { type: 'video/mp4' }),
+ targetLanguage: 'en',
+ trimRange: null,
+ initialSubtitleDefaults: { fontSize: 32, bottomOffsetPercent: 18 },
+ onBack: () => {},
+ });
+
+ const overlay = await screen.findByTestId('video-subtitle-overlay');
+ expect(overlay).toHaveStyle({ fontSize: '32px' });
+ expect(overlay.parentElement).toHaveStyle({ bottom: '18%' });
+ });
+
+ it('uses ttsText when generating dubbing audio', async () => {
+ generateSubtitlePipelineMock.mockResolvedValue({
+ subtitles: [
+ {
+ id: 'sub-tts',
+ startTime: 0,
+ endTime: 2,
+ originalText: 'Original',
+ translatedText: 'English subtitle',
+ ttsText: 'Bonjour a tous',
+ ttsLanguage: 'fr',
+ speaker: 'Narrator',
+ voiceId: 'voice-1',
+ },
+ ],
+ speakers: [],
+ quality: 'full',
+ });
+ generateTTSMock.mockResolvedValue('data:audio/mp3;base64,AAA');
+
+ renderEditorScreen({
+ videoFile: new File(['video'], 'clip.mp4', { type: 'video/mp4' }),
+ targetLanguage: 'en',
+ trimRange: null,
+ initialSubtitleDefaults: defaultSubtitleDefaults,
+ onBack: () => {},
+ });
+
+ await screen.findAllByDisplayValue('English subtitle');
+
+ fireEvent.click(screen.getByRole('button', { name: /generate dubbing/i }));
+
+ await waitFor(() =>
+ expect(generateTTSMock).toHaveBeenCalledWith('Bonjour a tous', 'voice-1'),
+ );
+ });
});
diff --git a/src/components/EditorScreen.tsx b/src/components/EditorScreen.tsx
index 2b27917..fbdfa9e 100644
--- a/src/components/EditorScreen.tsx
+++ b/src/components/EditorScreen.tsx
@@ -3,13 +3,61 @@ import axios from 'axios';
import { ChevronLeft, Play, Pause, Volume2, Settings, Download, Save, LayoutTemplate, Type, Image as ImageIcon, Music, Scissors, Plus, Trash2, Maximize2, Loader2 } from 'lucide-react';
import VoiceMarketModal from './VoiceMarketModal';
import ExportModal from './ExportModal';
-import { LlmProvider, PipelineQuality, Subtitle, TextStyles } from '../types';
+import { DEFAULT_SUBTITLE_DEFAULTS, LlmProvider, PipelineQuality, Subtitle, SubtitleDefaults, SubtitleGenerationProgress, TextStyles } from '../types';
import { generateSubtitlePipeline } from '../services/subtitleService';
import { generateTTS } from '../services/ttsService';
import { MINIMAX_VOICES } from '../voices';
import { apiUrl } from '../lib/apiBasePath';
+import { useI18n } from '../i18n';
-export default function EditorScreen({ videoFile, targetLanguage, trimRange, onBack }: { videoFile: File | null; targetLanguage: string; trimRange?: {start: number, end: number} | null; onBack: () => void }) {
+const DEFAULT_TEXT_STYLE: TextStyles = {
+ fontFamily: 'MiSans-Late',
+ fontSize: 24,
+ color: '#FFFFFF',
+ backgroundColor: 'transparent',
+ strokeColor: '#000000',
+ strokeWidth: 2,
+ alignment: 'center',
+ isBold: false,
+ isItalic: false,
+ isUnderline: false,
+};
+
+const FONT_SIZE_PRESETS = {
+ small: 20,
+ normal: 24,
+ large: 30,
+} as const;
+
+const normalizeTextStyle = (textStyle?: Partial): TextStyles => ({
+ ...DEFAULT_TEXT_STYLE,
+ ...textStyle,
+});
+
+const normalizeSubtitle = (subtitle: Subtitle, initialSubtitleDefaults: SubtitleDefaults): Subtitle => ({
+ ...subtitle,
+ textStyle: normalizeTextStyle({
+ fontSize: initialSubtitleDefaults.fontSize,
+ ...subtitle.textStyle,
+ }),
+});
+
+export default function EditorScreen({
+ videoFile,
+ targetLanguage,
+ ttsLanguage,
+ trimRange,
+ initialSubtitleDefaults = DEFAULT_SUBTITLE_DEFAULTS,
+ onBack,
+}: {
+ videoFile: File | null;
+ targetLanguage: string;
+ ttsLanguage?: string;
+ trimRange?: {start: number, end: number} | null;
+ initialSubtitleDefaults?: SubtitleDefaults;
+ onBack: () => void;
+}) {
+ const { m } = useI18n();
const [subtitles, setSubtitles] = useState([]);
const [activeSubtitleId, setActiveSubtitleId] = useState('');
const [showVoiceMarket, setShowVoiceMarket] = useState(false);
@@ -21,6 +69,8 @@ export default function EditorScreen({ videoFile, targetLanguage, trimRange, onB
const [generationError, setGenerationError] = useState(null);
const [subtitleQuality, setSubtitleQuality] = useState('fallback');
const [llmProvider, setLlmProvider] = useState('doubao');
+ const [generationProgress, setGenerationProgress] = useState(null);
+ const [applyToAllSubtitles, setApplyToAllSubtitles] = useState(false);
// Video Player State
const videoRef = useRef(null);
@@ -84,12 +134,25 @@ export default function EditorScreen({ videoFile, targetLanguage, trimRange, onB
if (!videoFile) return;
setIsGenerating(true);
setGenerationError(null);
+ setGenerationProgress({
+ jobId: 'pending',
+ requestId: 'pending',
+ status: 'queued',
+ stage: 'queued',
+ progress: 5,
+ message: m.editor.aiAnalyzing,
+ });
try {
const pipelineResult = await generateSubtitlePipeline(
videoFile,
targetLanguage,
llmProvider,
trimRange,
+ undefined,
+ (progress) => {
+ setGenerationProgress(progress);
+ },
+ ttsLanguage,
);
const generatedSubs = pipelineResult.subtitles;
setSubtitleQuality(pipelineResult.quality);
@@ -105,15 +168,17 @@ export default function EditorScreen({ videoFile, targetLanguage, trimRange, onB
}));
}
- setSubtitles(adjustedSubs);
- if (adjustedSubs.length > 0) {
- setActiveSubtitleId(adjustedSubs[0].id);
+ const normalizedSubs = adjustedSubs.map((subtitle) => normalizeSubtitle(subtitle, initialSubtitleDefaults));
+
+ setSubtitles(normalizedSubs);
+ if (normalizedSubs.length > 0) {
+ setActiveSubtitleId(normalizedSubs[0].id);
}
} catch (err: any) {
console.error("Failed to generate subtitles:", err);
setSubtitleQuality('fallback');
- let errorMessage = "Failed to generate subtitles. Please try again or check your API key.";
+ let errorMessage = m.editor.failedToGenerateSubtitles;
const errString = err instanceof Error ? err.message : JSON.stringify(err);
if (
@@ -123,7 +188,7 @@ export default function EditorScreen({ videoFile, targetLanguage, trimRange, onB
err?.status === 429 ||
err?.error?.code === 429
) {
- errorMessage = "You have exceeded your Volcengine API quota. Please check your plan and billing details.";
+ errorMessage = m.editor.exceededQuota;
} else if (err instanceof Error) {
errorMessage = err.message;
}
@@ -133,8 +198,9 @@ export default function EditorScreen({ videoFile, targetLanguage, trimRange, onB
setActiveSubtitleId('');
} finally {
setIsGenerating(false);
+ setGenerationProgress(null);
}
- }, [videoFile, targetLanguage, trimRange, llmProvider]);
+ }, [videoFile, targetLanguage, ttsLanguage, trimRange, llmProvider, initialSubtitleDefaults, m.editor.aiAnalyzing, m.editor.exceededQuota, m.editor.failedToGenerateSubtitles]);
// Generate subtitles on mount
useEffect(() => {
@@ -142,6 +208,7 @@ export default function EditorScreen({ videoFile, targetLanguage, trimRange, onB
fileName: videoFile?.name || '',
fileSize: videoFile?.size || 0,
targetLanguage,
+ ttsLanguage,
trimRange,
llmProvider,
});
@@ -152,18 +219,7 @@ export default function EditorScreen({ videoFile, targetLanguage, trimRange, onB
autoGenerationKeyRef.current = autoGenerationKey;
fetchSubtitles();
- }, [fetchSubtitles, videoFile, targetLanguage, trimRange, llmProvider]);
-
- const [textStyles, setTextStyles] = useState({
- fontFamily: 'MiSans-Late',
- fontSize: 24,
- color: '#FFFFFF',
- backgroundColor: 'transparent',
- alignment: 'center',
- isBold: false,
- isItalic: false,
- isUnderline: false,
- });
+ }, [fetchSubtitles, videoFile, targetLanguage, ttsLanguage, trimRange, llmProvider]);
const togglePlay = async () => {
if (videoRef.current) {
@@ -228,7 +284,7 @@ export default function EditorScreen({ videoFile, targetLanguage, trimRange, onB
if (sub.audioUrl) continue;
try {
- const textToSpeak = sub.translatedText || sub.text;
+ const textToSpeak = sub.ttsText || sub.translatedText || sub.originalText;
const audioUrl = await generateTTS(textToSpeak, sub.voiceId);
updatedSubtitles[i] = { ...sub, audioUrl };
// Update state incrementally so user sees progress
@@ -415,12 +471,60 @@ export default function EditorScreen({ videoFile, targetLanguage, trimRange, onB
const displayDuration = trimRange ? trimRange.end - trimRange.start : duration;
const displayCurrentTime = trimRange ? Math.max(0, currentTime - trimRange.start) : currentTime;
+ const activeSubtitle = useMemo(
+ () => subtitles.find((subtitle) => subtitle.id === activeSubtitleId) ?? null,
+ [subtitles, activeSubtitleId],
+ );
+ const activeSubtitleStyle = activeSubtitle?.textStyle ?? DEFAULT_TEXT_STYLE;
+ const currentSubtitle = useMemo(
+ () => subtitles.find((subtitle) => displayCurrentTime >= subtitle.startTime && displayCurrentTime <= subtitle.endTime) ?? null,
+ [subtitles, displayCurrentTime],
+ );
+ const currentSubtitleStyle = currentSubtitle?.textStyle ?? DEFAULT_TEXT_STYLE;
+ const currentSubtitleText = currentSubtitle?.translatedText || '';
+ const currentFontSizePreset = (Object.entries(FONT_SIZE_PRESETS).find(
+ ([, fontSize]) => fontSize === activeSubtitleStyle.fontSize,
+ )?.[0] ?? 'normal') as keyof typeof FONT_SIZE_PRESETS;
+ const exportTextStyles = subtitles[0]?.textStyle ?? DEFAULT_TEXT_STYLE;
+ const fontSizePresetLabels: Record = {
+ small: m.editor.small,
+ normal: m.editor.normal,
+ large: m.editor.large,
+ };
+ const alignmentOptions: Array<{ value: TextStyles['alignment']; label: string }> = [
+ { value: 'left', label: m.editor.alignLeft },
+ { value: 'center', label: m.editor.alignCenter },
+ { value: 'right', label: m.editor.alignRight },
+ ];
// Calculate playhead position percentage
const playheadPercent = displayDuration > 0 ? (displayCurrentTime / displayDuration) * 100 : 0;
- // Get current subtitle text
- const currentSubtitleText = subtitles.find(s => displayCurrentTime >= s.startTime && displayCurrentTime <= s.endTime)?.translatedText || '';
+ const updateSubtitleStyles = useCallback((updates: Partial) => {
+ if (!activeSubtitleId && !applyToAllSubtitles) {
+ return;
+ }
+
+ setSubtitles((previousSubtitles) =>
+ previousSubtitles.map((subtitle) => {
+ if (!applyToAllSubtitles && subtitle.id !== activeSubtitleId) {
+ return subtitle;
+ }
+
+ return {
+ ...subtitle,
+ textStyle: {
+ ...normalizeTextStyle(subtitle.textStyle),
+ ...updates,
+ },
+ };
+ }),
+ );
+ }, [activeSubtitleId, applyToAllSubtitles]);
+
+ const activeSpeakerInitial = activeSubtitle?.speaker?.trim().charAt(0).toUpperCase() || 'S';
+ const activeSubtitleContext = activeSubtitle?.translatedText || activeSubtitle?.originalText || '';
+ const subtitleTextShadow = `${currentSubtitleStyle.strokeWidth}px 0 ${currentSubtitleStyle.strokeColor}, -${currentSubtitleStyle.strokeWidth}px 0 ${currentSubtitleStyle.strokeColor}, 0 ${currentSubtitleStyle.strokeWidth}px ${currentSubtitleStyle.strokeColor}, 0 -${currentSubtitleStyle.strokeWidth}px ${currentSubtitleStyle.strokeColor}`;
return (
@@ -471,19 +575,19 @@ export default function EditorScreen({ videoFile, targetLanguage, trimRange, onB
+
{currentView === 'upload' ? (
+
+
+
+
+
+
@@ -432,7 +536,7 @@ export default function EditorScreen({ videoFile, targetLanguage, trimRange, onB
@@ -449,18 +553,18 @@ export default function EditorScreen({ videoFile, targetLanguage, trimRange, onB
className="flex items-center gap-2 px-3 py-1.5 bg-red-50 text-red-600 rounded-md text-sm font-medium hover:bg-red-100"
>
M
- Translate 1.0
+ {m.app.productName}
-
-
+
+
- Tip: After modifying the subtitle text, you need to regenerate the dubbing to take effect! + {m.editor.tip}
AI is analyzing and translating...
-This may take a minute depending on the video length.
++ {generationProgress?.message || m.editor.aiAnalyzing} +
+
+
+
+
+
+
+ {generationProgress?.stage || m.editor.qualityQueued}
+ {generationProgress?.progress ?? 5}%
+
+ {m.editor.thisMayTakeMinutes}
- Low-precision subtitle timing is active for this generation. You can still edit subtitles before dubbing.
+ {m.editor.lowPrecision}
)}
{subtitles.map((sub, index) => (
@@ -589,7 +707,7 @@ export default function EditorScreen({ videoFile, targetLanguage, trimRange, onB
No video loaded
+{m.editor.noVideoLoaded}
{currentSubtitleText} @@ -695,7 +819,7 @@ export default function EditorScreen({ videoFile, targetLanguage, trimRange, onB {bgmUrl && (
-
-
-
- W
+ {activeSubtitle ? (
+ <>
+
-
-
+
- Wife
-
+
+
+ {activeSpeakerInitial}
+
+ {activeSubtitle.speaker}
+
-
@@ -952,7 +1122,7 @@ export default function EditorScreen({ videoFile, targetLanguage, trimRange, onB
subtitles={subtitles}
bgmUrl={bgmUrl}
bgmBase64={bgmBase64}
- textStyles={textStyles}
+ textStyles={exportTextStyles}
trimRange={trimRange}
/>
)}
diff --git a/src/components/ExportModal.tsx b/src/components/ExportModal.tsx
index 69cbb23..9c0212f 100644
--- a/src/components/ExportModal.tsx
+++ b/src/components/ExportModal.tsx
@@ -4,6 +4,7 @@ import axios from 'axios';
import { Subtitle, TextStyles } from '../types';
import { buildExportPayload } from '../lib/exportPayload';
import { apiUrl } from '../lib/apiBasePath';
+import { useI18n } from '../i18n';
interface ExportModalProps {
onClose: () => void;
@@ -16,6 +17,7 @@ interface ExportModalProps {
}
export default function ExportModal({ onClose, videoFile, subtitles, bgmUrl, bgmBase64, textStyles, trimRange }: ExportModalProps) {
+ const { m } = useI18n();
const [isExporting, setIsExporting] = useState(false);
const [progress, setProgress] = useState(0);
const [isDone, setIsDone] = useState(false);
@@ -81,7 +83,7 @@ export default function ExportModal({ onClose, videoFile, subtitles, bgmUrl, bgm
}
} catch (err: any) {
console.error('Export Error:', err);
- setError(err.response?.data?.error || err.message || 'An error occurred during export');
+ setError(err.response?.data?.error || err.message || m.exportModal.errorDefault);
} finally {
setIsExporting(false);
}
@@ -103,7 +105,7 @@ export default function ExportModal({ onClose, videoFile, subtitles, bgmUrl, bgm
Text Styles
- - {/* Style Presets Grid */} -
- {['T', 'T', 'T', 'T', 'T', 'T', 'T', 'T', 'T', 'T', 'T', 'T'].map((item, i) => (
-
- ))}
-
+
+
@@ -822,7 +992,7 @@ export default function EditorScreen({ videoFile, targetLanguage, trimRange, onB
- Stretch the dubbing to control the speed
+ {m.editor.stretchDubbing}
{/* Zoom slider placeholder */}
@@ -845,7 +1015,7 @@ export default function EditorScreen({ videoFile, targetLanguage, trimRange, onB
{/* Video Track */}
{m.editor.textStyles}
+ +
+ {alignmentOptions.map((alignment) => (
+
+ ))}
+
- {/* Font Family */}
-
-
-
+
+
+
- {/* Font Size & Alignment */}
-
-
-
+
-
-
-
-
-
+
+
- {/* Colors */}
-
+
+
+
+
+
-
- Color
-
-
- setTextStyles({...textStyles, color: e.target.value})}
- className="w-6 h-6 rounded border border-gray-300 p-0 cursor-pointer"
- />
- 100%
-
-
- Stroke
-
+ )}
-
- 100%
+
+
+ ) : (
+
+
+ {m.editor.color}
+
+
+ updateSubtitleStyles({ color: e.target.value })}
+ className="w-6 h-6 rounded border border-gray-300 p-0 cursor-pointer"
+ />
+ 100%
+
+
+ {m.editor.stroke}
+
+ updateSubtitleStyles({ strokeColor: e.target.value })}
+ className="w-6 h-6 rounded border border-gray-300 p-0 cursor-pointer"
+ />
+ 100%
+
+
+ {m.editor.selectSubtitleToEdit}
-
- Video Track
+ {m.editor.videoTrack}
{/* Header */}
-
Export Video
+{m.exportModal.title}
@@ -117,7 +119,7 @@ export default function ExportModal({ onClose, videoFile, subtitles, bgmUrl, bgm ) : (
- No Preview
+ {m.exportModal.noPreview}
)}
@@ -128,10 +130,10 @@ export default function ExportModal({ onClose, videoFile, subtitles, bgmUrl, bgm
{/* Form */}
-
+
@@ -139,13 +141,13 @@ export default function ExportModal({ onClose, videoFile, subtitles, bgmUrl, bgm
+
+
+
-
+
-
+
@@ -162,7 +164,7 @@ export default function ExportModal({ onClose, videoFile, subtitles, bgmUrl, bgm
{isExporting && (
)}
- Simple Mode
+ {m.upload.simpleMode}
+
+
- Exporting...
+ {m.exportModal.exporting}
{progress}%
@@ -179,7 +181,7 @@ export default function ExportModal({ onClose, videoFile, subtitles, bgmUrl, bgm
{/* Footer */}
(null);
const timelineRef = useRef(null);
@@ -251,14 +253,14 @@ export default function TrimModal({ file, onClose, onConfirm }: TrimModalProps)
diff --git a/src/components/UploadScreen.test.tsx b/src/components/UploadScreen.test.tsx
new file mode 100644
index 0000000..7c55792
--- /dev/null
+++ b/src/components/UploadScreen.test.tsx
@@ -0,0 +1,128 @@
+// @vitest-environment jsdom
+
+import React from 'react';
+import { cleanup, fireEvent, render, screen } from '@testing-library/react';
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import UploadScreen from './UploadScreen';
+import { I18nProvider } from '../i18n';
+
+vi.mock('./TrimModal', () => ({
+ default: ({
+ file,
+ onClose,
+ onConfirm,
+ }: {
+ file: File;
+ onClose: () => void;
+ onConfirm: (file: File, startTime: number, endTime: number) => void;
+ }) => (
+
+ ,
+ );
+
+ afterEach(() => {
+ cleanup();
+ });
+
+ it('starts from the editor default subtitle size and position and updates the preview', () => {
+ renderUploadScreen();
+
+ const sizeSlider = screen.getByLabelText(/subtitle initial size/i);
+ const positionSlider = screen.getByLabelText(/subtitle initial position/i);
+ const preview = screen.getByTestId('upload-subtitle-preview');
+
+ expect(sizeSlider).toHaveValue('24');
+ expect(positionSlider).toHaveValue('10');
+ expect(preview).toHaveStyle({ fontSize: '24px', bottom: '10%' });
+
+ fireEvent.change(sizeSlider, { target: { value: '32' } });
+ fireEvent.change(positionSlider, { target: { value: '18' } });
+
+ expect(preview).toHaveStyle({ fontSize: '32px', bottom: '18%' });
+ });
+
+ it('shows a fixed English subtitle language and the supported TTS languages', () => {
+ renderUploadScreen();
+
+ expect(screen.getByText('Subtitle Language')).toBeInTheDocument();
+ expect(screen.getAllByText('English')).toHaveLength(2);
+ expect(screen.getByText('TTS Language')).toBeInTheDocument();
+ expect(screen.getByRole('button', { name: 'Chinese' })).toBeInTheDocument();
+ expect(screen.getByRole('button', { name: 'Cantonese' })).toBeInTheDocument();
+ expect(screen.getByRole('button', { name: 'French' })).toBeInTheDocument();
+ expect(screen.getByRole('button', { name: 'Indonesian' })).toBeInTheDocument();
+ expect(screen.getByRole('button', { name: 'German' })).toBeInTheDocument();
+ expect(screen.getByRole('button', { name: 'Filipino' })).toBeInTheDocument();
+
+ expect(screen.queryByRole('button', { name: 'Arabic' })).not.toBeInTheDocument();
+ expect(screen.queryByRole('button', { name: 'Japanese' })).not.toBeInTheDocument();
+ expect(screen.queryByRole('button', { name: 'Korean' })).not.toBeInTheDocument();
+ });
+
+ it('passes subtitle defaults through when upload is confirmed', () => {
+ const onUpload = vi.fn();
+ renderUploadScreen(onUpload);
+
+ fireEvent.change(screen.getByLabelText(/subtitle initial size/i), {
+ target: { value: '30' },
+ });
+ fireEvent.change(screen.getByLabelText(/subtitle initial position/i), {
+ target: { value: '16' },
+ });
+ fireEvent.click(screen.getByRole('button', { name: 'French' }));
+
+ fireEvent.change(screen.getByLabelText(/upload video file/i), {
+ target: {
+ files: [new File(['video'], 'clip.mp4', { type: 'video/mp4' })],
+ },
+ });
+
+ fireEvent.click(screen.getByRole('button', { name: /confirm trim/i }));
+
+ expect(onUpload).toHaveBeenCalledWith(
+ expect.any(File),
+ 'English',
+ 'French',
+ {
+ fontSize: 30,
+ bottomOffsetPercent: 16,
+ },
+ 1,
+ 5,
+ );
+ });
+
+ it('clears the pending upload after closing the trim preview', () => {
+ renderUploadScreen();
+
+ const fileInput = screen.getByLabelText(/upload video file/i);
+ const file = new File(['video'], 'clip.mp4', { type: 'video/mp4' });
+
+ fireEvent.change(fileInput, {
+ target: {
+ files: [file],
+ },
+ });
+
+ expect(screen.getByRole('button', { name: /close trim/i })).toBeInTheDocument();
+
+ fireEvent.click(screen.getByRole('button', { name: /close trim/i }));
+ expect(screen.queryByRole('button', { name: /close trim/i })).not.toBeInTheDocument();
+ expect(screen.getByRole('button', { name: /generate translated video/i })).toBeDisabled();
+ });
+});
diff --git a/src/components/UploadScreen.tsx b/src/components/UploadScreen.tsx
index 94755b2..cd31dc9 100644
--- a/src/components/UploadScreen.tsx
+++ b/src/components/UploadScreen.tsx
@@ -1,30 +1,45 @@
-import React, { useState } from 'react';
+import React, { useRef, useState } from 'react';
import { Upload, CheckCircle2, Circle } from 'lucide-react';
import TrimModal from './TrimModal';
+import { DEFAULT_SUBTITLE_DEFAULTS, SubtitleDefaults } from '../types';
+import { useI18n } from '../i18n';
const LANGUAGES = [
- { code: 'ar', name: 'Arabic', group: 'A' },
- { code: 'zh', name: 'Chinese', group: 'C' },
- { code: 'yue', name: 'Cantonese', group: 'C' },
- { code: 'cs', name: 'Czech', group: 'C' },
- { code: 'zh-TW', name: 'Traditional Chinese', group: 'T' },
- { code: 'th', name: 'Thai', group: 'T' },
- { code: 'tr', name: 'Turkey', group: 'T' },
- { code: 'nl', name: 'Dutch', group: 'D' },
- { code: 'en', name: 'English', group: 'E' },
- { code: 'ru', name: 'Russian', group: 'R' },
- { code: 'ro', name: 'Romanian', group: 'R' },
- { code: 'ja', name: 'Japanese', group: 'J' },
- { code: 'ko', name: 'Korean', group: 'K' },
- { code: 'ms', name: 'Malay', group: 'M' },
- { code: 'fr', name: 'French', group: 'F' },
+ { code: 'zh', apiName: 'Chinese', group: 'C' },
+ { code: 'yue', apiName: 'Cantonese', group: 'C' },
+ { code: 'en', apiName: 'English', group: 'E' },
+ { code: 'id', apiName: 'Indonesian', group: 'I' },
+ { code: 'de', apiName: 'German', group: 'G' },
+ { code: 'fil', apiName: 'Filipino', group: 'F' },
+ { code: 'fr', apiName: 'French', group: 'F' },
];
-export default function UploadScreen({ onUpload }: { onUpload: (file: File, lang: string, startTime?: number, endTime?: number) => void }) {
+export default function UploadScreen({
+ onUpload,
+}: {
+ onUpload: (
+ file: File,
+ targetLanguage: string,
+ ttsLanguage: string,
+ subtitleDefaults: SubtitleDefaults,
+ startTime?: number,
+ endTime?: number,
+ ) => void
+}) {
+ const { m, format } = useI18n();
const [mode, setMode] = useState<'editing' | 'simple'>('editing');
- const [selectedLang, setSelectedLang] = useState('en');
+ const [selectedTtsLanguage, setSelectedTtsLanguage] = useState('en');
const [showTrimModal, setShowTrimModal] = useState(false);
const [tempFile, setTempFile] = useState(null);
+ const [subtitleDefaults, setSubtitleDefaults] = useState(DEFAULT_SUBTITLE_DEFAULTS);
+ const fileInputRef = useRef(null);
+
+ const clearPendingUpload = () => {
+ setTempFile(null);
+ if (fileInputRef.current) {
+ fileInputRef.current.value = '';
+ }
+ };
const handleFileChange = (e: React.ChangeEvent) => {
if (e.target.files && e.target.files[0]) {
@@ -35,8 +50,8 @@ export default function UploadScreen({ onUpload }: { onUpload: (file: File, lang
const handleTrimConfirm = (file: File, startTime: number, endTime: number) => {
setShowTrimModal(false);
- const langName = LANGUAGES.find(l => l.code === selectedLang)?.name || 'English';
- onUpload(file, langName, startTime, endTime);
+ const ttsLanguage = LANGUAGES.find((language) => language.code === selectedTtsLanguage)?.apiName || 'English';
+ onUpload(file, 'English', ttsLanguage, subtitleDefaults, startTime, endTime);
};
return (
@@ -45,20 +60,22 @@ export default function UploadScreen({ onUpload }: { onUpload: (file: File, lang
- - Compositing video with subtitles and AI dubbing. This may take a few minutes depending on the video length. + {m.exportModal.description}
@@ -188,7 +190,7 @@ export default function ExportModal({ onClose, videoFile, subtitles, bgmUrl, bgm
className="px-4 py-2 text-sm text-gray-500 hover:text-gray-700 font-medium"
disabled={isExporting}
>
- {isDone ? 'Close' : 'Cancel'}
+ {isDone ? m.exportModal.close : m.exportModal.cancel}
{isDone ? (
@@ -197,7 +199,7 @@ export default function ExportModal({ onClose, videoFile, subtitles, bgmUrl, bgm
className="bg-[#52c41a] text-white px-6 py-2 rounded-md text-sm font-medium flex items-center gap-2 hover:bg-[#46a616] transition-colors shadow-sm"
>
diff --git a/src/components/TrimModal.tsx b/src/components/TrimModal.tsx
index 36537fd..a29ce02 100644
--- a/src/components/TrimModal.tsx
+++ b/src/components/TrimModal.tsx
@@ -1,5 +1,6 @@
import React, { useState, useRef, useEffect, useMemo } from 'react';
import { X, Play, Pause } from 'lucide-react';
+import { useI18n } from '../i18n';
interface TrimModalProps {
file: File;
@@ -8,6 +9,7 @@ interface TrimModalProps {
}
export default function TrimModal({ file, onClose, onConfirm }: TrimModalProps) {
+ const { m, format } = useI18n();
const videoRef = useRef
- Selected duration: {Math.round(trimDuration)}s
+ {format(m.trim.selectedDuration, { value: Math.round(trimDuration) })}
+
+
+
+ ),
+}));
+
+describe('UploadScreen', () => {
+ const renderUploadScreen = (onUpload = () => {}) =>
+ render(
+ Click to upload or drag files here
+{m.upload.clickToUpload}
- Supported formats: MP4/MOV/WEBM. Maximum file size is 500MB. + {m.upload.supportedFormats}
Supports secondary editing and more precise translation
+{m.upload.editingModeDesc}
{m.upload.simpleModeDesc}
+
+
{/* Language Selection */}
+
+
+
+
+
+ {m.upload.subtitleDefaults}
++ {m.upload.subtitleDefaultsDesc} +
+
+
+
+
+
+
+ + {m.upload.subtitlePreview} +
+
+
+
+
+
+
+ {format(m.upload.fromBottom, { value: subtitleDefaults.bottomOffsetPercent })}
+
+
+ setSubtitleDefaults((previous) => ({
+ ...previous,
+ bottomOffsetPercent: Number(e.target.value),
+ }))
+ }
+ className="w-full accent-blue-500"
+ />
+
+
-
+
+ {format(m.upload.pxValue, { value: subtitleDefaults.fontSize })}
+
+
+ setSubtitleDefaults((previous) => ({
+ ...previous,
+ fontSize: Number(e.target.value),
+ }))
+ }
+ className="w-full accent-blue-500"
+ />
One-click video translation for beginners
- setShowTrimModal(false)}
+ onClose={() => {
+ setShowTrimModal(false);
+ clearPendingUpload();
+ }}
onConfirm={handleTrimConfirm}
/>
)}
diff --git a/src/components/VoiceMarketModal.tsx b/src/components/VoiceMarketModal.tsx
index 141a6f0..6172063 100644
--- a/src/components/VoiceMarketModal.tsx
+++ b/src/components/VoiceMarketModal.tsx
@@ -2,6 +2,7 @@ import React, { useState } from 'react';
import { X, Play, Search, Filter } from 'lucide-react';
import { MINIMAX_VOICES } from '../voices';
import { Voice } from '../types';
+import { useI18n } from '../i18n';
export default function VoiceMarketModal({
onClose,
@@ -12,6 +13,7 @@ export default function VoiceMarketModal({
onSelect: (voiceId: string) => void;
onSelectAll: (voiceId: string) => void;
}) {
+ const { m } = useI18n();
const [searchQuery, setSearchQuery] = useState('');
const [selectedLanguage, setSelectedLanguage] = useState('all');
const [selectedGender, setSelectedGender] = useState('all');
@@ -32,12 +34,12 @@ export default function VoiceMarketModal({
{/* Header */}
) => {
+ if (!values) return template;
+ return Object.entries(values).reduce(
+ (result, [key, value]) => result.replaceAll(`{${key}}`, String(value)),
+ template,
+ );
+};
+
+interface I18nContextValue {
+ locale: Locale;
+ setLocale: (locale: Locale) => void;
+ m: MessageTree;
+ format: (template: string, values?: Record) => string;
+}
+
+const I18nContext = createContext(null);
+
+export const I18nProvider = ({
+ children,
+ initialLocale = 'zh',
+}: {
+ children: React.ReactNode;
+ initialLocale?: Locale;
+}) => {
+ const [locale, setLocale] = useState(initialLocale);
+
+ const value = useMemo(
+ () => ({
+ locale,
+ setLocale,
+ m: messages[locale],
+ format: formatMessage,
+ }),
+ [locale],
+ );
+
+ return {children} ;
+};
+
+export const useI18n = () => {
+ const context = useContext(I18nContext);
+ if (!context) {
+ throw new Error('useI18n must be used within I18nProvider');
+ }
+
+ return context;
+};
diff --git a/src/lib/exportPayload.test.ts b/src/lib/exportPayload.test.ts
index d34e358..2edc3cd 100644
--- a/src/lib/exportPayload.test.ts
+++ b/src/lib/exportPayload.test.ts
@@ -23,6 +23,8 @@ describe('buildExportPayload', () => {
fontSize: 24,
color: '#FFFFFF',
backgroundColor: 'transparent',
+ strokeColor: '#000000',
+ strokeWidth: 2,
alignment: 'center',
isBold: true,
isItalic: false,
@@ -70,6 +72,8 @@ describe('buildExportPayload', () => {
fontSize: 18,
color: '#FF0000',
backgroundColor: 'transparent',
+ strokeColor: '#000000',
+ strokeWidth: 2,
alignment: 'left',
isBold: false,
isItalic: true,
diff --git a/src/server/doubaoTranslation.test.ts b/src/server/doubaoTranslation.test.ts
index fa1e3e4..e7740bc 100644
--- a/src/server/doubaoTranslation.test.ts
+++ b/src/server/doubaoTranslation.test.ts
@@ -51,7 +51,7 @@ describe('doubaoTranslation', () => {
endTime: 1,
},
],
- model: 'doubao-seed-2-0-pro-260215',
+ model: 'doubao-seed-2-0-lite-260215',
requestResponse,
});
@@ -86,7 +86,7 @@ describe('doubaoTranslation', () => {
endTime: 1,
},
],
- model: 'doubao-seed-2-0-pro-260215',
+ model: 'doubao-seed-2-0-lite-260215',
requestResponse,
});
diff --git a/src/server/errorLogging.test.ts b/src/server/errorLogging.test.ts
index af7ebdc..e89e296 100644
--- a/src/server/errorLogging.test.ts
+++ b/src/server/errorLogging.test.ts
@@ -1,5 +1,19 @@
-import { describe, expect, it } from 'vitest';
-import { formatLogContext, serializeError } from './errorLogging';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import { afterEach, describe, expect, it } from 'vitest';
+import { appendLogEntry, formatLogContext, resolveLogFilePath, serializeError } from './errorLogging';
+
+const tempDirs: string[] = [];
+
+afterEach(() => {
+ while (tempDirs.length > 0) {
+ const tempDir = tempDirs.pop();
+ if (tempDir && fs.existsSync(tempDir)) {
+ fs.rmSync(tempDir, { recursive: true, force: true });
+ }
+ }
+});
describe('errorLogging', () => {
it('serializes nested causes and error codes', () => {
@@ -33,4 +47,47 @@ describe('errorLogging', () => {
}),
).toBe('requestId=req-1 provider=doubao durationMs=1234');
});
+
+ it('defaults log file path to logs/server.log under the current working directory', () => {
+ expect(resolveLogFilePath({} as NodeJS.ProcessEnv)).toBe(
+ path.join(process.cwd(), 'logs', 'server.log'),
+ );
+ });
+
+ it('appends structured log entries to a file path from env', () => {
+ const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'subtitle-logs-'));
+ tempDirs.push(tempDir);
+ const logFilePath = path.join(tempDir, 'server.log');
+
+ appendLogEntry({
+ level: 'info',
+ message: 'subtitle request started',
+ context: {
+ requestId: 'req-123',
+ provider: 'doubao',
+ },
+ details: {
+ fileId: 'file-123',
+ },
+ env: {
+ LOG_FILE_PATH: logFilePath,
+ } as NodeJS.ProcessEnv,
+ });
+
+ const written = fs.readFileSync(logFilePath, 'utf8').trim();
+ const parsed = JSON.parse(written);
+
+ expect(parsed).toMatchObject({
+ level: 'info',
+ message: 'subtitle request started',
+ context: {
+ requestId: 'req-123',
+ provider: 'doubao',
+ },
+ details: {
+ fileId: 'file-123',
+ },
+ });
+ expect(typeof parsed.timestamp).toBe('string');
+ });
});
diff --git a/src/server/errorLogging.ts b/src/server/errorLogging.ts
index f9318de..54f6835 100644
--- a/src/server/errorLogging.ts
+++ b/src/server/errorLogging.ts
@@ -1,3 +1,6 @@
+import fs from 'fs';
+import path from 'path';
+
export interface ErrorLogDetails {
message: string;
stack?: string;
@@ -6,6 +9,87 @@ export interface ErrorLogDetails {
name?: string;
}
+export type LogLevel = 'info' | 'warn' | 'error';
+
+export interface LogEntry {
+ timestamp: string;
+ level: LogLevel;
+ message: string;
+ context?: Record;
+ details?: unknown;
+ pid: number;
+}
+
+export const resolveLogFilePath = (env: NodeJS.ProcessEnv = process.env) =>
+ env.LOG_FILE_PATH?.trim() || path.join(process.cwd(), 'logs', 'server.log');
+
+export const appendLogEntry = ({
+ level,
+ message,
+ context,
+ details,
+ env = process.env,
+}: {
+ level: LogLevel;
+ message: string;
+ context?: Record;
+ details?: unknown;
+ env?: NodeJS.ProcessEnv;
+}) => {
+ const logFilePath = resolveLogFilePath(env);
+ const logDir = path.dirname(logFilePath);
+ const entry: LogEntry = {
+ timestamp: new Date().toISOString(),
+ level,
+ message,
+ pid: process.pid,
+ ...(context ? { context } : {}),
+ ...(details !== undefined ? { details } : {}),
+ };
+
+ fs.mkdirSync(logDir, { recursive: true });
+ fs.appendFileSync(logFilePath, `${JSON.stringify(entry)}\n`, 'utf8');
+};
+
+export const logEvent = ({
+ level,
+ message,
+ context,
+ details,
+ env = process.env,
+}: {
+ level: LogLevel;
+ message: string;
+ context?: Record;
+ details?: unknown;
+ env?: NodeJS.ProcessEnv;
+}) => {
+ try {
+ appendLogEntry({ level, message, context, details, env });
+ } catch (error) {
+ console.error(
+ `[log] failed to write file log ${formatLogContext({
+ message,
+ configuredPath: resolveLogFilePath(env),
+ })}`,
+ serializeError(error),
+ );
+ }
+
+ const consoleMethod = level === 'error' ? console.error : level === 'warn' ? console.warn : console.info;
+ if (details !== undefined) {
+ consoleMethod(message, context || {}, details);
+ return;
+ }
+
+ if (context) {
+ consoleMethod(message, context);
+ return;
+ }
+
+ consoleMethod(message);
+};
+
export const serializeError = (error: unknown): ErrorLogDetails => {
if (error instanceof Error) {
const details: ErrorLogDetails = {
diff --git a/src/server/exportVideo.test.ts b/src/server/exportVideo.test.ts
index b6a22a9..fbe1c5a 100644
--- a/src/server/exportVideo.test.ts
+++ b/src/server/exportVideo.test.ts
@@ -11,6 +11,8 @@ const defaultTextStyles: TextStyles = {
fontSize: 24,
color: '#FFFFFF',
backgroundColor: 'transparent',
+ strokeColor: '#000000',
+ strokeWidth: 2,
alignment: 'center',
isBold: true,
isItalic: false,
diff --git a/src/server/exportVideo.ts b/src/server/exportVideo.ts
index dc812ac..37e441c 100644
--- a/src/server/exportVideo.ts
+++ b/src/server/exportVideo.ts
@@ -6,6 +6,8 @@ export const DEFAULT_EXPORT_TEXT_STYLES: TextStyles = {
fontSize: 24,
color: '#FFFFFF',
backgroundColor: 'transparent',
+ strokeColor: '#000000',
+ strokeWidth: 2,
alignment: 'center',
isBold: false,
isItalic: false,
@@ -105,7 +107,7 @@ export const buildAssSubtitleContent = ({
textStyles.fontSize,
toAssColor(textStyles.color),
'&H00000000',
- '&H00000000',
+ toAssColor(textStyles.strokeColor),
'&H64000000',
textStyles.isBold ? -1 : 0,
textStyles.isItalic ? -1 : 0,
@@ -116,7 +118,7 @@ export const buildAssSubtitleContent = ({
0,
0,
1,
- 1,
+ Math.max(1, textStyles.strokeWidth),
2,
mapAlignmentToAss(textStyles.alignment),
48,
diff --git a/src/server/providerTranslation.test.ts b/src/server/providerTranslation.test.ts
index e73b9f4..c0a4836 100644
--- a/src/server/providerTranslation.test.ts
+++ b/src/server/providerTranslation.test.ts
@@ -14,7 +14,7 @@ describe('createSentenceTranslator', () => {
const translator = createSentenceTranslator({
provider: 'doubao',
apiKey: 'ark-key',
- model: 'doubao-seed-2-0-pro-260215',
+ model: 'doubao-seed-2-0-lite-260215',
baseUrl: 'https://ark.cn-beijing.volces.com/api/v3/responses',
timeoutMs: 600000,
});
diff --git a/src/server/subtitleGeneration.test.ts b/src/server/subtitleGeneration.test.ts
index bb0b6c6..3ec4f7a 100644
--- a/src/server/subtitleGeneration.test.ts
+++ b/src/server/subtitleGeneration.test.ts
@@ -132,4 +132,34 @@ describe('generateSubtitlePipeline', () => {
}),
);
});
+
+ it('passes tts language through to video subtitle generation', async () => {
+ const subtitleResult: SubtitlePipelineResult = {
+ subtitles: [],
+ speakers: [],
+ quality: 'fallback',
+ targetLanguage: 'English',
+ };
+ const generateSubtitlesFromVideo = vi.fn(async () => subtitleResult);
+
+ await generateSubtitlePipeline({
+ videoPath: 'clip.mp4',
+ targetLanguage: 'English',
+ ttsLanguage: 'fr',
+ provider: 'doubao',
+ env: {
+ ARK_API_KEY: 'ark-key',
+ },
+ deps: {
+ generateSubtitlesFromVideo,
+ },
+ } as any);
+
+ expect(generateSubtitlesFromVideo).toHaveBeenCalledWith(
+ expect.objectContaining({
+ targetLanguage: 'English',
+ ttsLanguage: 'fr',
+ }),
+ );
+ });
});
diff --git a/src/server/subtitleGeneration.ts b/src/server/subtitleGeneration.ts
index b748f78..d61fa74 100644
--- a/src/server/subtitleGeneration.ts
+++ b/src/server/subtitleGeneration.ts
@@ -1,4 +1,6 @@
+import { SubtitleGenerationProgress } from '../types';
import { resolveAudioPipelineConfig } from './audioPipelineConfig';
+import { logEvent, serializeError } from './errorLogging';
import { resolveLlmProviderConfig, normalizeLlmProvider } from './llmProvider';
import { generateSubtitlesFromVideo as defaultGenerateSubtitlesFromVideo } from './videoSubtitleGeneration';
@@ -6,10 +8,12 @@ export interface GenerateSubtitlePipelineOptions {
videoPath?: string;
fileId?: string;
targetLanguage: string;
+ ttsLanguage?: string;
provider?: string | null;
env: NodeJS.ProcessEnv;
fetchImpl?: typeof fetch;
requestId?: string;
+ onProgress?: (progress: Omit) => void;
deps?: {
generateSubtitlesFromVideo?: typeof defaultGenerateSubtitlesFromVideo;
};
@@ -19,16 +23,19 @@ export const generateSubtitlePipeline = async ({
videoPath,
fileId,
targetLanguage,
+ ttsLanguage,
provider,
env,
fetchImpl,
requestId,
+ onProgress,
deps,
}: GenerateSubtitlePipelineOptions) => {
if (!videoPath && !fileId) {
throw new Error('A video upload or fileId is required.');
}
+ const startedAt = Date.now();
const audioPipelineConfig = resolveAudioPipelineConfig(env);
const selectedProvider = provider
? normalizeLlmProvider(provider)
@@ -37,12 +44,75 @@ export const generateSubtitlePipeline = async ({
const generateSubtitlesFromVideo =
deps?.generateSubtitlesFromVideo || defaultGenerateSubtitlesFromVideo;
- return generateSubtitlesFromVideo({
- providerConfig,
- videoPath,
- fileId,
- targetLanguage,
- requestId,
- ...(fetchImpl ? { fetchImpl } : {}),
+ onProgress?.({
+ status: 'running',
+ stage: 'preparing',
+ progress: 30,
+ message: 'Preparing subtitle generation',
});
+
+ logEvent({
+ level: 'info',
+ message: '[subtitle] pipeline configured',
+ context: {
+ requestId,
+ requestedProvider: provider || undefined,
+ selectedProvider,
+ targetLanguage,
+ hasVideoPath: Boolean(videoPath),
+ hasFileId: Boolean(fileId),
+ },
+ });
+
+ try {
+ onProgress?.({
+ status: 'running',
+ stage: 'calling_provider',
+ progress: 70,
+ message: 'Calling subtitle provider',
+ });
+
+ const result = await generateSubtitlesFromVideo({
+ providerConfig,
+ videoPath,
+ fileId,
+ targetLanguage,
+ ttsLanguage,
+ requestId,
+ ...(fetchImpl ? { fetchImpl } : {}),
+ });
+
+ onProgress?.({
+ status: 'running',
+ stage: 'processing_result',
+ progress: 90,
+ message: 'Processing subtitle result',
+ });
+
+ logEvent({
+ level: 'info',
+ message: '[subtitle] pipeline completed',
+ context: {
+ requestId,
+ provider: providerConfig.provider,
+ durationMs: Date.now() - startedAt,
+ subtitleCount: result.subtitles.length,
+ quality: result.quality,
+ },
+ });
+
+ return result;
+ } catch (error) {
+ logEvent({
+ level: 'error',
+ message: '[subtitle] pipeline failed',
+ context: {
+ requestId,
+ provider: providerConfig.provider,
+ durationMs: Date.now() - startedAt,
+ },
+ details: serializeError(error),
+ });
+ throw error;
+ }
};
diff --git a/src/server/subtitleJobs.test.ts b/src/server/subtitleJobs.test.ts
new file mode 100644
index 0000000..d7ca21a
--- /dev/null
+++ b/src/server/subtitleJobs.test.ts
@@ -0,0 +1,59 @@
+import { describe, expect, it, vi } from 'vitest';
+import {
+ createSubtitleJobStore,
+ createSubtitleJob,
+ pruneExpiredSubtitleJobs,
+ toSubtitleJobResponse,
+} from './subtitleJobs';
+
+describe('subtitleJobs', () => {
+ it('creates jobs with queued defaults', () => {
+ const store = createSubtitleJobStore();
+
+ const job = createSubtitleJob(store, {
+ requestId: 'req-1',
+ provider: 'doubao',
+ targetLanguage: 'English',
+ fileId: 'file-123',
+ });
+
+ expect(job).toMatchObject({
+ requestId: 'req-1',
+ provider: 'doubao',
+ targetLanguage: 'English',
+ fileId: 'file-123',
+ status: 'queued',
+ stage: 'queued',
+ progress: 5,
+ });
+ });
+
+ it('removes expired jobs past the ttl', () => {
+ vi.useFakeTimers();
+ const store = createSubtitleJobStore();
+
+ createSubtitleJob(store, {
+ requestId: 'req-1',
+ provider: 'doubao',
+ targetLanguage: 'English',
+ });
+
+ vi.advanceTimersByTime(60 * 60 * 1000 + 1);
+ pruneExpiredSubtitleJobs(store, 60 * 60 * 1000);
+
+ expect(store.jobs.size).toBe(0);
+ vi.useRealTimers();
+ });
+
+ it('hides file paths from api responses', () => {
+ const store = createSubtitleJobStore();
+ const job = createSubtitleJob(store, {
+ requestId: 'req-1',
+ provider: 'gemini',
+ targetLanguage: 'English',
+ filePath: 'uploads/temp-file.mp4',
+ });
+
+ expect(toSubtitleJobResponse(job)).not.toHaveProperty('filePath');
+ });
+});
diff --git a/src/server/subtitleJobs.ts b/src/server/subtitleJobs.ts
new file mode 100644
index 0000000..7bb1855
--- /dev/null
+++ b/src/server/subtitleJobs.ts
@@ -0,0 +1,168 @@
+import { SubtitleGenerationProgress, SubtitleJobStage, SubtitleJobStatus, SubtitlePipelineResult } from '../types';
+
+const DEFAULT_PROGRESS_BY_STAGE: Record = {
+ queued: 5,
+ upload_received: 15,
+ preparing: 30,
+ calling_provider: 70,
+ processing_result: 90,
+ succeeded: 100,
+ failed: 0,
+};
+
+export interface SubtitleJob {
+ id: string;
+ requestId: string;
+ status: SubtitleJobStatus;
+ stage: SubtitleJobStage;
+ progress: number;
+ message: string;
+ createdAt: number;
+ updatedAt: number;
+ provider?: string | null;
+ targetLanguage: string;
+ ttsLanguage?: string;
+ fileId?: string;
+ filePath?: string;
+ error?: string;
+ result?: SubtitlePipelineResult & {
+ provider?: string | null;
+ requestId: string;
+ };
+}
+
+export interface SubtitleJobStore {
+ jobs: Map;
+ nextId: number;
+}
+
+export const createSubtitleJobStore = (): SubtitleJobStore => ({
+ jobs: new Map(),
+ nextId: 1,
+});
+
+const createSubtitleJobId = (store: SubtitleJobStore) => {
+ const id = `subtitle-job-${Date.now()}-${store.nextId}`;
+ store.nextId += 1;
+ return id;
+};
+
+const defaultMessageForStage = (stage: SubtitleJobStage) => {
+ switch (stage) {
+ case 'queued':
+ return 'Queued';
+ case 'upload_received':
+ return 'Upload received';
+ case 'preparing':
+ return 'Preparing subtitle generation';
+ case 'calling_provider':
+ return 'Calling subtitle provider';
+ case 'processing_result':
+ return 'Processing subtitle result';
+ case 'succeeded':
+ return 'Subtitle generation completed';
+ case 'failed':
+ return 'Subtitle generation failed';
+ }
+};
+
+export const createSubtitleJob = (
+ store: SubtitleJobStore,
+ {
+ requestId,
+ provider,
+ targetLanguage,
+ ttsLanguage,
+ fileId,
+ filePath,
+ }: {
+ requestId: string;
+ provider?: string | null;
+ targetLanguage: string;
+ ttsLanguage?: string;
+ fileId?: string;
+ filePath?: string;
+ },
+): SubtitleJob => {
+ const now = Date.now();
+ const job: SubtitleJob = {
+ id: createSubtitleJobId(store),
+ requestId,
+ provider,
+ targetLanguage,
+ ttsLanguage,
+ fileId,
+ filePath,
+ status: 'queued',
+ stage: 'queued',
+ progress: DEFAULT_PROGRESS_BY_STAGE.queued,
+ message: defaultMessageForStage('queued'),
+ createdAt: now,
+ updatedAt: now,
+ };
+ store.jobs.set(job.id, job);
+ return job;
+};
+
+export const getSubtitleJob = (store: SubtitleJobStore, jobId: string) => store.jobs.get(jobId);
+
+export const updateSubtitleJob = (
+ store: SubtitleJobStore,
+ jobId: string,
+ updates: Partial & {
+ stage?: SubtitleJobStage;
+ status?: SubtitleJobStatus;
+ message?: string;
+ progress?: number;
+ },
+) => {
+ const job = store.jobs.get(jobId);
+ if (!job) {
+ return undefined;
+ }
+
+ const nextStage = updates.stage || job.stage;
+ const nextStatus = updates.status || job.status;
+ const nextProgress =
+ updates.progress !== undefined
+ ? updates.progress
+ : nextStage !== job.stage
+ ? DEFAULT_PROGRESS_BY_STAGE[nextStage]
+ : job.progress;
+
+ const updated: SubtitleJob = {
+ ...job,
+ ...updates,
+ stage: nextStage,
+ status: nextStatus,
+ progress: nextProgress,
+ message: updates.message || (updates.stage ? defaultMessageForStage(nextStage) : job.message),
+ updatedAt: Date.now(),
+ };
+
+ store.jobs.set(jobId, updated);
+ return updated;
+};
+
+export const pruneExpiredSubtitleJobs = (store: SubtitleJobStore, ttlMs: number, now = Date.now()) => {
+ for (const [jobId, job] of store.jobs.entries()) {
+ if (now - job.updatedAt > ttlMs) {
+ store.jobs.delete(jobId);
+ }
+ }
+};
+
+export const toSubtitleGenerationProgress = (job: SubtitleJob): SubtitleGenerationProgress => ({
+ jobId: job.id,
+ requestId: job.requestId,
+ status: job.status,
+ stage: job.stage,
+ progress: job.progress,
+ message: job.message,
+});
+
+export const toSubtitleJobResponse = (job: SubtitleJob) => ({
+ ...toSubtitleGenerationProgress(job),
+ ...(job.error ? { error: job.error } : {}),
+ ...(job.result ? { result: job.result } : {}),
+});
diff --git a/src/server/subtitleRequest.test.ts b/src/server/subtitleRequest.test.ts
index 3db7685..ce5a010 100644
--- a/src/server/subtitleRequest.test.ts
+++ b/src/server/subtitleRequest.test.ts
@@ -6,6 +6,7 @@ describe('parseSubtitleRequest', () => {
expect(parseSubtitleRequest({ targetLanguage: 'English' })).toEqual({
provider: 'doubao',
targetLanguage: 'English',
+ ttsLanguage: 'English',
});
});
@@ -18,6 +19,7 @@ describe('parseSubtitleRequest', () => {
).toEqual({
provider: 'gemini',
targetLanguage: 'English',
+ ttsLanguage: 'English',
});
});
@@ -36,7 +38,21 @@ describe('parseSubtitleRequest', () => {
).toEqual({
provider: 'doubao',
targetLanguage: 'English',
+ ttsLanguage: 'English',
fileId: 'file-123',
});
});
+
+ it('preserves a tts language override when provided', () => {
+ expect(
+ parseSubtitleRequest({
+ targetLanguage: 'English',
+ ttsLanguage: 'fr',
+ } as any),
+ ).toEqual({
+ provider: 'doubao',
+ targetLanguage: 'English',
+ ttsLanguage: 'fr',
+ });
+ });
});
diff --git a/src/server/subtitleRequest.ts b/src/server/subtitleRequest.ts
index 48909eb..a2a1f4c 100644
--- a/src/server/subtitleRequest.ts
+++ b/src/server/subtitleRequest.ts
@@ -3,12 +3,14 @@ import { LlmProvider, normalizeLlmProvider } from './llmProvider';
export interface SubtitleRequestBody {
provider?: string | null;
targetLanguage?: string | null;
+ ttsLanguage?: string | null;
fileId?: string | null;
}
export interface ParsedSubtitleRequest {
provider: LlmProvider;
targetLanguage: string;
+ ttsLanguage: string;
fileId?: string;
}
@@ -20,9 +22,12 @@ export const parseSubtitleRequest = (
throw new Error('Target language is required.');
}
+ const ttsLanguage = body.ttsLanguage?.trim() || targetLanguage;
+
return {
provider: normalizeLlmProvider(body.provider),
targetLanguage,
+ ttsLanguage,
...(body.fileId?.trim() ? { fileId: body.fileId.trim() } : {}),
};
};
diff --git a/src/server/videoSubtitleGeneration.test.ts b/src/server/videoSubtitleGeneration.test.ts
index 2453d53..7dc2b87 100644
--- a/src/server/videoSubtitleGeneration.test.ts
+++ b/src/server/videoSubtitleGeneration.test.ts
@@ -20,8 +20,10 @@ describe('generateSubtitlesFromVideo', () => {
sourceLanguage: 'zh',
subtitles: [
{
- originalText: '你好',
+ originalText: 'hello there',
translatedText: 'Hello',
+ ttsText: 'Bonjour',
+ ttsLanguage: 'fr',
startTime: 0,
endTime: 1,
},
@@ -49,8 +51,9 @@ describe('generateSubtitlesFromVideo', () => {
},
videoPath: 'clip.mp4',
targetLanguage: 'English',
+ ttsLanguage: 'fr',
fetchImpl,
- });
+ } as any);
expect(fetchImpl).toHaveBeenCalledWith(
'https://ark.cn-beijing.volces.com/api/v3/responses',
@@ -73,8 +76,10 @@ describe('generateSubtitlesFromVideo', () => {
sourceLanguage: 'zh',
subtitles: [
{
- originalText: '你好',
+ originalText: 'hello there',
translatedText: 'Hello',
+ ttsText: 'Bonjour',
+ ttsLanguage: 'fr',
startTime: 0,
endTime: 1,
},
@@ -102,8 +107,9 @@ describe('generateSubtitlesFromVideo', () => {
},
fileId: 'file-123',
targetLanguage: 'English',
+ ttsLanguage: 'fr',
fetchImpl,
- });
+ } as any);
const [, request] = fetchImpl.mock.calls[0] as [string, RequestInit];
const payload = JSON.parse(String(request.body));
@@ -112,6 +118,12 @@ describe('generateSubtitlesFromVideo', () => {
expect(payload.input[0].content[0].type).toBe('input_text');
expect(payload.input[0].content[0].text).toContain('# Role');
expect(payload.input[0].content[0].text).toContain('Voice Selection');
+ expect(payload.input[0].content[0].text).toContain('# Output Contract');
+ expect(payload.input[0].content[0].text).toContain('The first character of your response must be {.');
+ expect(payload.input[0].content[0].text).toContain('The last character of your response must be }.');
+ expect(payload.input[0].content[0].text).toContain('"ttsText":');
+ expect(payload.input[0].content[0].text).toContain('"ttsLanguage":');
+ expect(payload.input[0].content[0].text).toContain('translatedText must always be English');
expect(payload.input[1].role).toBe('user');
expect(payload.input[1].content[0]).toEqual({
@@ -119,8 +131,55 @@ describe('generateSubtitlesFromVideo', () => {
file_id: 'file-123',
});
expect(payload.input[1].content[1].type).toBe('input_text');
- expect(payload.input[1].content[1].text).toContain('Target language: English');
- expect(payload.input[1].content[1].text).toContain('Available voices');
- expect(payload.input[1].content[1].text).toContain('Sweet_Girl');
+ expect(payload.input[1].content[1].text).toContain('Subtitle language: English');
+ expect(payload.input[1].content[1].text).toContain('TTS language: fr');
+ expect(payload.input[1].content[1].text).toContain('Available voices for the TTS language');
+ });
+
+ it('extracts tts fields when doubao returns prose around a fenced payload', async () => {
+ const fetchImpl = vi.fn(async () =>
+ new Response(
+ JSON.stringify({
+ output: [
+ {
+ content: [
+ {
+ text: 'Here is the JSON result:\n```json\n{"sourceLanguage":"zh","subtitles":[{"originalText":"hello there","translatedText":"Hello","ttsText":"Bonjour","ttsLanguage":"fr","startTime":0,"endTime":1,"speaker":"Speaker 1","voiceId":"male-qn-qingse"}]}\n```\nUse it directly.',
+ },
+ ],
+ },
+ ],
+ }),
+ {
+ status: 200,
+ headers: { 'Content-Type': 'application/json' },
+ },
+ ),
+ );
+
+ const result = await generateSubtitlesFromVideo({
+ providerConfig: {
+ provider: 'doubao',
+ apiKey: 'ark-key',
+ model: 'doubao-seed-2-0-pro-260215',
+ baseUrl: 'https://ark.cn-beijing.volces.com/api/v3/responses',
+ timeoutMs: 600000,
+ },
+ fileId: 'file-123',
+ targetLanguage: 'English',
+ ttsLanguage: 'fr',
+ fetchImpl,
+ } as any);
+
+ expect(result.sourceLanguage).toBe('zh');
+ expect(result.subtitles).toHaveLength(1);
+ expect(result.subtitles[0]).toMatchObject({
+ originalText: 'hello there',
+ translatedText: 'Hello',
+ ttsText: 'Bonjour',
+ ttsLanguage: 'fr',
+ speaker: 'Speaker 1',
+ voiceId: 'male-qn-qingse',
+ });
});
});
diff --git a/src/server/videoSubtitleGeneration.ts b/src/server/videoSubtitleGeneration.ts
index c9890e7..38ef7fc 100644
--- a/src/server/videoSubtitleGeneration.ts
+++ b/src/server/videoSubtitleGeneration.ts
@@ -2,7 +2,7 @@ import fs from 'fs';
import { GoogleGenAI } from '@google/genai';
import { SubtitlePipelineResult } from '../types';
import { MINIMAX_VOICES } from '../voices';
-import { formatLogContext, serializeError } from './errorLogging';
+import { formatLogContext, logEvent, serializeError } from './errorLogging';
import { DoubaoProviderConfig, GeminiProviderConfig, LlmProviderConfig } from './llmProvider';
interface RawModelSubtitle {
@@ -11,6 +11,8 @@ interface RawModelSubtitle {
endTime?: number | string;
originalText?: string;
translatedText?: string;
+ ttsText?: string;
+ ttsLanguage?: string;
speaker?: string;
voiceId?: string;
}
@@ -25,18 +27,78 @@ const SUPPORTED_VOICE_IDS = new Set(MINIMAX_VOICES.map((voice) => voice.id));
const stripJsonFences = (text: string) => text.replace(/```json\n?|\n?```/g, '').trim();
+const extractBalancedJsonBlock = (text: string) => {
+ const startIndexes = [text.indexOf('{'), text.indexOf('[')].filter((index) => index >= 0);
+ const start = startIndexes.length > 0 ? Math.min(...startIndexes) : -1;
+ if (start < 0) {
+ return text;
+ }
+
+ const opening = text[start];
+ const closing = opening === '{' ? '}' : ']';
+ let depth = 0;
+ let inString = false;
+ let escaped = false;
+
+ for (let index = start; index < text.length; index += 1) {
+ const char = text[index];
+
+ if (inString) {
+ if (escaped) {
+ escaped = false;
+ } else if (char === '\\') {
+ escaped = true;
+ } else if (char === '"') {
+ inString = false;
+ }
+ continue;
+ }
+
+ if (char === '"') {
+ inString = true;
+ continue;
+ }
+
+ if (char === opening) {
+ depth += 1;
+ continue;
+ }
+
+ if (char === closing) {
+ depth -= 1;
+ if (depth === 0) {
+ return text.slice(start, index + 1);
+ }
+ }
+ }
+
+ return text.slice(start);
+};
+
const extractJson = (text: string): RawModelResponse => {
const cleaned = stripJsonFences(text);
if (!cleaned) {
return { subtitles: [] };
}
- const parsed = JSON.parse(cleaned);
- if (Array.isArray(parsed)) {
- return { subtitles: parsed };
- }
+ const directCandidate = cleaned.replace(/^\uFEFF/, '').trim();
- return parsed as RawModelResponse;
+ try {
+ const parsed = JSON.parse(directCandidate);
+ if (Array.isArray(parsed)) {
+ return { subtitles: parsed };
+ }
+
+ return parsed as RawModelResponse;
+ } catch {
+ const extractedCandidate = extractBalancedJsonBlock(directCandidate).trim();
+ const parsed = JSON.parse(extractedCandidate);
+ if (Array.isArray(parsed)) {
+ return { subtitles: parsed };
+ }
+
+ return parsed as RawModelResponse;
+ }
};
const toSeconds = (value: unknown, fallback: number) => {
@@ -93,12 +155,13 @@ const formatVoiceCatalogForPrompt = (targetLanguage: string) => {
const createSystemPrompt = () => `# Role
You are a senior film and TV subtitle expert and an advanced localization translator.
-You deeply understand screen reading experience.
-Subtitles must be short, easy to read, precisely timed to the visuals and speech rhythm, and must never cause viewer reading fatigue.
+You deeply understand subtitle readability, dubbing naturalness, and multilingual localization for short-form video.
# Task
Listen to and watch the user-provided audio or video.
-Transcribe the spoken content and translate it into the target language specified by the user.
+Transcribe the spoken content.
+Translate every subtitle into English for on-screen subtitles.
+Also translate every subtitle into the user-specified TTS language for dubbing playback.
Extract highly accurate start and end timestamps, speaker labels, and speaker gender.
Select the most suitable voiceId for each subtitle item by matching the speaker's gender, tone, style, and delivery to the voice options provided by the user.
Return the result strictly in the required JSON format.
@@ -110,9 +173,8 @@ You must split subtitles according to the speaker's actual breathing, pauses, co
2. Screen-Friendly Length:
Each subtitle item must be short.
-Chinese text should ideally stay within 15 to 20 characters.
-English text should ideally stay within 7 to 10 words.
-If a sentence is too long, you must split it into multiple subtitle objects with consecutive timestamps.
+English subtitle text should ideally stay within 7 to 10 words.
+If a sentence is too long, split it into multiple subtitle objects with consecutive timestamps.
3. Highly Precise Timestamps:
Timestamps must align closely with the actual speech.
@@ -123,18 +185,27 @@ The duration of a single subtitle item should usually not exceed 3 to 5 seconds.
Accurately identify the speaker label and speaker gender.
Gender must be either "male" or "female".
-5. Voice Selection:
-The user will provide the target language and a list of available voices.
+5. Translation Rules:
+- translatedText must always be English subtitle text for on-screen display.
+- ttsText must be translated into the user-provided TTS language.
+- translatedText and ttsText must preserve the same meaning as the original speech.
+- translatedText should prioritize subtitle readability.
+- ttsText should prioritize natural spoken dubbing in the target TTS language.
+
+6. Voice Selection:
+The user will provide a TTS language and a list of available voices.
Each voice includes a voiceId and descriptive metadata.
You must analyze the user-provided voice list and choose the best matching voiceId for each subtitle item.
Only return a voiceId that exists in the user-provided voice list.
Do not invent new voiceId values.
-6. Output Format:
-Return only valid JSON.
-Do not output markdown, code fences, explanations, or any extra text.
+# Output Contract
+You must return exactly one JSON object.
+The first character of your response must be {.
+The last character of your response must be }.
+Do not output markdown, code fences, comments, headings, explanations, or any text before or after the JSON.
-Return an object with this exact structure:
+Return a JSON object with this exact top-level structure:
{
"sourceLanguage": "detected language code",
"subtitles": [
@@ -143,7 +214,9 @@ Return an object with this exact structure:
"startTime": 0.0,
"endTime": 1.2,
"originalText": "source dialogue",
- "translatedText": "translated dialogue in the target language",
+ "translatedText": "english subtitle text",
+ "ttsText": "translated dubbing text in the requested TTS language",
+ "ttsLanguage": "requested tts language code",
"speaker": "short speaker label",
"gender": "male or female",
"voiceId": "one of the user-provided voice ids"
@@ -151,23 +224,39 @@ Return an object with this exact structure:
]
}
-Additional rules:
-1. Use video timeline seconds for startTime and endTime.
-2. Keep subtitles chronological and non-overlapping.
-3. Do not invent dialogue if it is not actually audible.
-4. Preserve meaning naturally while keeping subtitle lines short and readable.
-5. If a long utterance must be split, preserve continuity across consecutive subtitle items.
-6. Output JSON only.`;
+# JSON Rules
+1. sourceLanguage must be a JSON string.
+2. subtitles must be a JSON array.
+3. Every subtitle item must include all of these fields: id, startTime, endTime, originalText, translatedText, ttsText, ttsLanguage, speaker, gender, voiceId.
+4. startTime and endTime must be JSON numbers, not strings.
+5. id, originalText, translatedText, ttsText, ttsLanguage, speaker, gender, and voiceId must be JSON strings.
+6. gender must be exactly "male" or "female".
+7. translatedText must always be English.
+8. ttsLanguage must exactly match the user-requested TTS language code.
+9. voiceId must exactly match one of the user-provided voiceId values.
+10. Do not include any extra fields.
+11. Do not use trailing commas.
+12. Use video timeline seconds for startTime and endTime.
+13. Keep subtitles chronological and non-overlapping.
+14. Do not invent dialogue if it is not actually audible.
+15. Preserve meaning naturally while keeping subtitle lines short and readable.
+16. If a long utterance must be split, preserve continuity across consecutive subtitle items.
+17. Output JSON only.`;
-const createUserPrompt = (targetLanguage: string) => `Target language: ${targetLanguage}
+const createUserPrompt = (ttsLanguage: string) => `Subtitle language: English
+TTS language: ${ttsLanguage}
-Available voices:
-${formatVoiceCatalogForPrompt(targetLanguage)}
+Available voices for the TTS language:
+${formatVoiceCatalogForPrompt(ttsLanguage)}
Please watch and listen to the provided video.
-Transcribe the dialogue, translate it into ${targetLanguage}, and assign the best matching voiceId from the available voices for each subtitle item.`;
+Transcribe the dialogue.
+Generate English subtitle text in translatedText.
+Generate dubbing text in ${ttsLanguage} inside ttsText.
+Set ttsLanguage to ${ttsLanguage} for every subtitle item.
+Assign the best matching voiceId from the available voices for each subtitle item.`;
-const normalizeSubtitles = (raw: RawModelSubtitle[]) => {
+const normalizeSubtitles = (raw: RawModelSubtitle[], fallbackTtsLanguage: string) => {
let lastEnd = 0;
const subtitles = raw
.map((entry, index) => {
@@ -186,6 +275,8 @@ const normalizeSubtitles = (raw: RawModelSubtitle[]) => {
endTime,
originalText,
translatedText: (entry.translatedText || originalText).trim(),
+ ttsText: (entry.ttsText || entry.translatedText || originalText).trim(),
+ ttsLanguage: (entry.ttsLanguage || fallbackTtsLanguage).trim(),
speaker: (entry.speaker || `Speaker ${index + 1}`).trim(),
speakerId: `speaker-${index + 1}`,
voiceId: sanitizeVoiceId(entry.voiceId),
@@ -218,6 +309,7 @@ const generateWithDoubao = async ({
videoDataUrl,
fileId,
targetLanguage,
+ ttsLanguage,
fetchImpl = fetch,
requestId,
}: {
@@ -225,6 +317,7 @@ const generateWithDoubao = async ({
videoDataUrl?: string;
fileId?: string;
targetLanguage: string;
+ ttsLanguage: string;
fetchImpl?: typeof fetch;
requestId?: string;
}) => {
@@ -234,9 +327,21 @@ const generateWithDoubao = async ({
provider: 'doubao',
timeoutMs: config.timeoutMs,
targetLanguage,
+ ttsLanguage,
});
- console.info(`[subtitle] doubao request started ${logContext}`);
+ logEvent({
+ level: 'info',
+ message: `[subtitle] doubao request started ${logContext}`,
+ context: {
+ requestId,
+ provider: 'doubao',
+ timeoutMs: config.timeoutMs,
+ targetLanguage,
+ hasFileId: Boolean(fileId),
+ hasVideoDataUrl: Boolean(videoDataUrl),
+ },
+ });
let response: Response;
try {
@@ -262,45 +367,65 @@ const generateWithDoubao = async ({
fileId
? { type: 'input_video', file_id: fileId }
: { type: 'input_video', video_url: videoDataUrl },
- { type: 'input_text', text: createUserPrompt(targetLanguage) },
+ { type: 'input_text', text: createUserPrompt(ttsLanguage) },
],
},
],
}),
});
} catch (error) {
- console.error(
- `[subtitle] doubao request failed ${formatLogContext({
+ logEvent({
+ level: 'error',
+ message: `[subtitle] doubao request failed ${formatLogContext({
requestId,
provider: 'doubao',
timeoutMs: config.timeoutMs,
durationMs: Date.now() - startedAt,
})}`,
- serializeError(error),
- );
+ context: {
+ requestId,
+ provider: 'doubao',
+ timeoutMs: config.timeoutMs,
+ durationMs: Date.now() - startedAt,
+ },
+ details: serializeError(error),
+ });
throw error;
}
if (!response.ok) {
const payload = await response.text();
- console.error(
- `[subtitle] doubao request returned non-200 ${formatLogContext({
+ logEvent({
+ level: 'error',
+ message: `[subtitle] doubao request returned non-200 ${formatLogContext({
requestId,
status: response.status,
durationMs: Date.now() - startedAt,
})}`,
- payload,
- );
+ context: {
+ requestId,
+ provider: 'doubao',
+ status: response.status,
+ durationMs: Date.now() - startedAt,
+ },
+ details: payload,
+ });
throw new Error(`Doubao subtitle request failed (${response.status}): ${payload}`);
}
const payload = await response.json();
- console.info(
- `[subtitle] doubao request finished ${formatLogContext({
+ logEvent({
+ level: 'info',
+ message: `[subtitle] doubao request finished ${formatLogContext({
requestId,
durationMs: Date.now() - startedAt,
})}`,
- );
+ context: {
+ requestId,
+ provider: 'doubao',
+ durationMs: Date.now() - startedAt,
+ },
+ });
const text = extractDoubaoTextOutput(payload);
return extractJson(text);
};
@@ -309,10 +434,12 @@ const generateWithGemini = async ({
config,
videoBase64,
targetLanguage,
+ ttsLanguage,
}: {
config: GeminiProviderConfig;
videoBase64: string;
targetLanguage: string;
+ ttsLanguage: string;
}) => {
const ai = new GoogleGenAI({ apiKey: config.apiKey });
const response = await ai.models.generateContent({
@@ -327,7 +454,7 @@ const generateWithGemini = async ({
data: videoBase64,
},
},
- { text: `${createSystemPrompt()}\n\n${createUserPrompt(targetLanguage)}` },
+ { text: `${createSystemPrompt()}\n\n${createUserPrompt(ttsLanguage)}` },
],
},
],
@@ -341,6 +468,7 @@ export const generateSubtitlesFromVideo = async ({
videoPath,
fileId,
targetLanguage,
+ ttsLanguage,
fetchImpl = fetch,
requestId,
}: {
@@ -348,6 +476,7 @@ export const generateSubtitlesFromVideo = async ({
videoPath?: string;
fileId?: string;
targetLanguage: string;
+ ttsLanguage?: string;
fetchImpl?: typeof fetch;
requestId?: string;
}): Promise => {
@@ -359,6 +488,8 @@ export const generateSubtitlesFromVideo = async ({
const videoBase64 = videoBuffer?.toString('base64');
const videoDataUrl = videoBase64 ? `data:video/mp4;base64,${videoBase64}` : undefined;
+ const resolvedTtsLanguage = ttsLanguage?.trim() || targetLanguage;
+
const raw =
providerConfig.provider === 'doubao'
? await generateWithDoubao({
@@ -366,6 +497,7 @@ export const generateSubtitlesFromVideo = async ({
videoDataUrl,
fileId,
targetLanguage,
+ ttsLanguage: resolvedTtsLanguage,
fetchImpl,
requestId,
})
@@ -373,9 +505,10 @@ export const generateSubtitlesFromVideo = async ({
config: providerConfig,
videoBase64: videoBase64!,
targetLanguage,
+ ttsLanguage: resolvedTtsLanguage,
});
- const subtitles = normalizeSubtitles(Array.isArray(raw.subtitles) ? raw.subtitles : []);
+ const subtitles = normalizeSubtitles(Array.isArray(raw.subtitles) ? raw.subtitles : [], resolvedTtsLanguage);
return {
subtitles,
@@ -383,6 +516,7 @@ export const generateSubtitlesFromVideo = async ({
quality: subtitles.length > 0 ? 'full' : 'fallback',
sourceLanguage: raw.sourceLanguage,
targetLanguage,
+ ttsLanguage: resolvedTtsLanguage,
duration: subtitles.length > 0 ? subtitles[subtitles.length - 1].endTime : 0,
alignmentEngine: `llm-video-${providerConfig.provider}`,
};
diff --git a/src/services/subtitleService.test.ts b/src/services/subtitleService.test.ts
index 559d3ad..8c1a6db 100644
--- a/src/services/subtitleService.test.ts
+++ b/src/services/subtitleService.test.ts
@@ -6,6 +6,7 @@ import { generateSubtitlePipeline } from './subtitleService';
describe('generateSubtitlePipeline', () => {
beforeEach(() => {
vi.stubEnv('VITE_ARK_API_KEY', 'ark-key');
+ vi.stubEnv('VITE_API_BASE_PATH', '/api');
});
afterEach(() => {
@@ -14,29 +15,59 @@ describe('generateSubtitlePipeline', () => {
});
it('posts the selected provider to the server for gemini', async () => {
- const fetchMock = vi.fn(async () =>
- new Response(
- JSON.stringify({
- subtitles: [],
- speakers: [],
- quality: 'fallback',
- }),
- {
- status: 200,
- headers: {
- 'Content-Type': 'application/json',
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ jobId: 'job-1',
+ requestId: 'req-1',
+ status: 'queued',
+ stage: 'queued',
+ progress: 5,
+ message: 'Queued',
+ }),
+ {
+ status: 202,
+ headers: {
+ 'Content-Type': 'application/json',
+ },
},
- },
- ),
- );
+ ),
+ )
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ jobId: 'job-1',
+ requestId: 'req-1',
+ status: 'succeeded',
+ stage: 'succeeded',
+ progress: 100,
+ result: {
+ subtitles: [],
+ speakers: [],
+ quality: 'fallback',
+ },
+ }),
+ {
+ status: 200,
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ },
+ ),
+ );
- await generateSubtitlePipeline(
+ const promise = generateSubtitlePipeline(
new File(['video'], 'clip.mp4', { type: 'video/mp4' }),
'English',
'gemini',
null,
fetchMock as unknown as typeof fetch,
);
+ await vi.runAllTimersAsync();
+ await promise;
expect(fetchMock).toHaveBeenCalledWith(
'/api/generate-subtitles',
@@ -50,6 +81,68 @@ describe('generateSubtitlePipeline', () => {
const formData = requestInit.body as FormData;
expect(formData.get('targetLanguage')).toBe('English');
expect(formData.get('provider')).toBe('gemini');
+ expect(fetchMock).toHaveBeenNthCalledWith(2, '/api/generate-subtitles/job-1', { method: 'GET' });
+ });
+
+ it('forwards the tts language in subtitle requests', async () => {
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ jobId: 'job-1',
+ requestId: 'req-1',
+ status: 'queued',
+ stage: 'queued',
+ progress: 5,
+ }),
+ {
+ status: 202,
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ },
+ ),
+ )
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ jobId: 'job-1',
+ requestId: 'req-1',
+ status: 'succeeded',
+ stage: 'succeeded',
+ progress: 100,
+ result: {
+ subtitles: [],
+ speakers: [],
+ quality: 'fallback',
+ },
+ }),
+ {
+ status: 200,
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ },
+ ),
+ );
+
+ const promise = generateSubtitlePipeline(
+ new File(['video'], 'clip.mp4', { type: 'video/mp4' }),
+ 'English',
+ 'gemini',
+ null,
+ fetchMock as unknown as typeof fetch,
+ undefined,
+ 'fr',
+ );
+ await vi.runAllTimersAsync();
+ await promise;
+
+ const [, requestInit] = fetchMock.mock.calls[0] as unknown as [string, RequestInit];
+ const formData = requestInit.body as FormData;
+ expect(formData.get('ttsLanguage')).toBe('fr');
});
it('uploads doubao videos to ark files before requesting subtitles', async () => {
@@ -100,9 +193,33 @@ describe('generateSubtitlePipeline', () => {
.mockResolvedValueOnce(
new Response(
JSON.stringify({
- subtitles: [],
- speakers: [],
- quality: 'fallback',
+ jobId: 'job-1',
+ requestId: 'req-1',
+ status: 'queued',
+ stage: 'queued',
+ progress: 5,
+ }),
+ {
+ status: 202,
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ },
+ ),
+ )
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ jobId: 'job-1',
+ requestId: 'req-1',
+ status: 'succeeded',
+ stage: 'succeeded',
+ progress: 100,
+ result: {
+ subtitles: [],
+ speakers: [],
+ quality: 'fallback',
+ },
}),
{
status: 200,
@@ -171,7 +288,9 @@ describe('generateSubtitlePipeline', () => {
fileId: 'file-123',
provider: 'doubao',
targetLanguage: 'English',
+ ttsLanguage: 'English',
});
+ expect(fetchMock).toHaveBeenNthCalledWith(5, '/api/generate-subtitles/job-1', { method: 'GET' });
});
it('stops when ark reports file preprocessing failure', async () => {
@@ -222,32 +341,61 @@ describe('generateSubtitlePipeline', () => {
});
it('keeps multipart uploads for gemini requests', async () => {
- const fetchMock = vi.fn(async () =>
- new Response(
- JSON.stringify({
- subtitles: [],
- speakers: [],
- quality: 'fallback',
- }),
- {
- status: 200,
- headers: {
- 'Content-Type': 'application/json',
+ vi.useFakeTimers();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ jobId: 'job-1',
+ requestId: 'req-1',
+ status: 'queued',
+ stage: 'queued',
+ progress: 5,
+ }),
+ {
+ status: 202,
+ headers: {
+ 'Content-Type': 'application/json',
+ },
},
- },
- ),
- );
+ ),
+ )
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ jobId: 'job-1',
+ requestId: 'req-1',
+ status: 'succeeded',
+ stage: 'succeeded',
+ progress: 100,
+ result: {
+ subtitles: [],
+ speakers: [],
+ quality: 'fallback',
+ },
+ }),
+ {
+ status: 200,
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ },
+ ),
+ );
- await generateSubtitlePipeline(
+ const promise = generateSubtitlePipeline(
new File(['video'], 'clip.mp4', { type: 'video/mp4' }),
'English',
'gemini',
null,
fetchMock as unknown as typeof fetch,
);
+ await vi.runAllTimersAsync();
+ await promise;
- expect(fetchMock).toHaveBeenCalledTimes(1);
- expect(fetchMock).toHaveBeenCalledWith(
+ expect(fetchMock).toHaveBeenNthCalledWith(
+ 1,
'/api/generate-subtitles',
expect.objectContaining({
method: 'POST',
@@ -255,4 +403,98 @@ describe('generateSubtitlePipeline', () => {
}),
);
});
+
+ it('polls every 5 seconds and reports progress updates', async () => {
+ vi.useFakeTimers();
+ const onProgress = vi.fn();
+ const fetchMock = vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ jobId: 'job-1',
+ requestId: 'req-1',
+ status: 'queued',
+ stage: 'queued',
+ progress: 5,
+ message: 'Queued',
+ }),
+ {
+ status: 202,
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ },
+ ),
+ )
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ jobId: 'job-1',
+ requestId: 'req-1',
+ status: 'running',
+ stage: 'calling_provider',
+ progress: 70,
+ message: 'Calling provider',
+ }),
+ {
+ status: 200,
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ },
+ ),
+ )
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ jobId: 'job-1',
+ requestId: 'req-1',
+ status: 'succeeded',
+ stage: 'succeeded',
+ progress: 100,
+ message: 'Done',
+ result: {
+ subtitles: [],
+ speakers: [],
+ quality: 'fallback',
+ },
+ }),
+ {
+ status: 200,
+ headers: {
+ 'Content-Type': 'application/json',
+ },
+ },
+ ),
+ );
+
+ const promise = generateSubtitlePipeline(
+ new File(['video'], 'clip.mp4', { type: 'video/mp4' }),
+ 'English',
+ 'gemini',
+ null,
+ fetchMock as unknown as typeof fetch,
+ onProgress,
+ );
+ await vi.runAllTimersAsync();
+ await promise;
+
+ expect(onProgress).toHaveBeenCalledWith(
+ expect.objectContaining({
+ jobId: 'job-1',
+ status: 'queued',
+ stage: 'queued',
+ progress: 5,
+ }),
+ );
+ expect(onProgress).toHaveBeenCalledWith(
+ expect.objectContaining({
+ jobId: 'job-1',
+ status: 'running',
+ stage: 'calling_provider',
+ progress: 70,
+ }),
+ );
+ });
});
diff --git a/src/services/subtitleService.ts b/src/services/subtitleService.ts
index adeb1ea..57947b3 100644
--- a/src/services/subtitleService.ts
+++ b/src/services/subtitleService.ts
@@ -1,4 +1,4 @@
-import { LlmProvider, PipelineQuality, SubtitlePipelineResult } from '../types';
+import { LlmProvider, PipelineQuality, SubtitleGenerationProgress, SubtitlePipelineResult } from '../types';
import { apiUrl } from '../lib/apiBasePath';
type JsonResponseResult =
@@ -8,6 +8,13 @@ type JsonResponseResult =
const ARK_FILES_URL = 'https://ark.cn-beijing.volces.com/api/v3/files';
const ARK_FILE_STATUS_POLL_INTERVAL_MS = 1000;
const ARK_FILE_STATUS_TIMEOUT_MS = 120000;
+const SUBTITLE_JOB_POLL_INTERVAL_MS = 5000;
+const SUBTITLE_JOB_TIMEOUT_MS = 20 * 60 * 1000;
+
+interface SubtitleJobResponse extends SubtitleGenerationProgress {
+ error?: string;
+ result?: Partial;
+}
const normalizePipelineQuality = (value: unknown): PipelineQuality => {
if (value === 'full' || value === 'partial' || value === 'fallback') {
@@ -82,6 +89,59 @@ const sleep = (durationMs: number) =>
setTimeout(resolve, durationMs);
});
+const toPipelineResult = (
+ parsed: Partial,
+ targetLanguage: string,
+ ttsLanguage: string,
+): SubtitlePipelineResult => ({
+ subtitles: Array.isArray(parsed.subtitles) ? parsed.subtitles : [],
+ speakers: Array.isArray(parsed.speakers) ? parsed.speakers : [],
+ quality: normalizePipelineQuality(parsed.quality),
+ sourceLanguage: parsed.sourceLanguage,
+ targetLanguage: parsed.targetLanguage || targetLanguage,
+ ttsLanguage: parsed.ttsLanguage || ttsLanguage,
+ duration:
+ typeof parsed.duration === 'number' ? parsed.duration : undefined,
+ alignmentEngine: parsed.alignmentEngine,
+});
+
+const pollSubtitleJob = async (
+ jobId: string,
+ targetLanguage: string,
+ ttsLanguage: string,
+ fetchImpl: typeof fetch,
+ onProgress?: (progress: SubtitleGenerationProgress) => void,
+): Promise => {
+ const deadline = Date.now() + SUBTITLE_JOB_TIMEOUT_MS;
+
+ while (true) {
+ const resp = await fetchImpl(apiUrl(`/generate-subtitles/${jobId}`), {
+ method: 'GET',
+ });
+
+ const parsed = await readJsonResponseOnce(resp);
+ if (parsed.ok === false) {
+ throw new Error(parsed.error);
+ }
+
+ onProgress?.(parsed.data);
+
+ if (parsed.data.status === 'succeeded') {
+ return toPipelineResult(parsed.data.result || {}, targetLanguage, ttsLanguage);
+ }
+
+ if (parsed.data.status === 'failed') {
+ throw new Error(parsed.data.error || 'Subtitle generation failed.');
+ }
+
+ if (Date.now() >= deadline) {
+ throw new Error('Timed out while waiting for subtitle generation to complete.');
+ }
+
+ await sleep(SUBTITLE_JOB_POLL_INTERVAL_MS);
+ }
+};
+
const waitForArkFileToBecomeActive = async (
fileId: string,
apiKey: string,
@@ -132,11 +192,15 @@ export const generateSubtitlePipeline = async (
provider: LlmProvider = 'doubao',
trimRange?: { start: number; end: number } | null,
fetchImpl: typeof fetch = fetch,
+ onProgress?: (progress: SubtitleGenerationProgress) => void,
+ ttsLanguage?: string,
): Promise => {
if (!targetLanguage.trim()) {
throw new Error('Target language is required.');
}
+ const resolvedTtsLanguage = ttsLanguage?.trim() || targetLanguage;
+
if (provider === 'doubao') {
const { fileId, apiKey } = await uploadDoubaoVideoFile(videoFile, fetchImpl);
await waitForArkFileToBecomeActive(fileId, apiKey, fetchImpl);
@@ -148,33 +212,27 @@ export const generateSubtitlePipeline = async (
body: JSON.stringify({
fileId,
targetLanguage,
+ ttsLanguage: resolvedTtsLanguage,
provider,
...(trimRange ? { trimRange } : {}),
}),
});
- const parsed = await readJsonResponseOnce>(resp);
+ const parsed = await readJsonResponseOnce(resp);
if (parsed.ok === false) {
const error = new Error(parsed.error);
(error as any).status = resp.status;
throw error;
}
-
- return {
- subtitles: Array.isArray(parsed.data.subtitles) ? parsed.data.subtitles : [],
- speakers: Array.isArray(parsed.data.speakers) ? parsed.data.speakers : [],
- quality: normalizePipelineQuality(parsed.data.quality),
- sourceLanguage: parsed.data.sourceLanguage,
- targetLanguage: parsed.data.targetLanguage || targetLanguage,
- duration:
- typeof parsed.data.duration === 'number' ? parsed.data.duration : undefined,
- alignmentEngine: parsed.data.alignmentEngine,
- };
+ const job = parsed.data as unknown as SubtitleJobResponse;
+ onProgress?.(job);
+ return pollSubtitleJob(job.jobId, targetLanguage, resolvedTtsLanguage, fetchImpl, onProgress);
}
const formData = new FormData();
formData.append('video', videoFile);
formData.append('targetLanguage', targetLanguage);
+ formData.append('ttsLanguage', resolvedTtsLanguage);
formData.append('provider', provider);
if (trimRange) {
formData.append('trimRange', JSON.stringify(trimRange));
@@ -185,21 +243,12 @@ export const generateSubtitlePipeline = async (
body: formData,
});
- const parsed = await readJsonResponseOnce>(resp);
+ const parsed = await readJsonResponseOnce(resp);
if (parsed.ok === false) {
const error = new Error(parsed.error);
(error as any).status = resp.status;
throw error;
}
-
- return {
- subtitles: Array.isArray(parsed.data.subtitles) ? parsed.data.subtitles : [],
- speakers: Array.isArray(parsed.data.speakers) ? parsed.data.speakers : [],
- quality: normalizePipelineQuality(parsed.data.quality),
- sourceLanguage: parsed.data.sourceLanguage,
- targetLanguage: parsed.data.targetLanguage || targetLanguage,
- duration:
- typeof parsed.data.duration === 'number' ? parsed.data.duration : undefined,
- alignmentEngine: parsed.data.alignmentEngine,
- };
+ onProgress?.(parsed.data);
+ return pollSubtitleJob(parsed.data.jobId, targetLanguage, resolvedTtsLanguage, fetchImpl, onProgress);
};
diff --git a/src/types.ts b/src/types.ts
index 63c1a78..03ca427 100644
--- a/src/types.ts
+++ b/src/types.ts
@@ -4,6 +4,8 @@ export interface Subtitle {
endTime: number;
originalText: string;
translatedText: string;
+ ttsText?: string;
+ ttsLanguage?: string;
speaker: string;
speakerId?: string;
words?: WordTiming[];
@@ -14,12 +16,24 @@ export interface Subtitle {
age?: string;
voiceCharacteristics?: string;
emotion?: string;
+ textStyle?: TextStyles;
}
export type LlmProvider = 'doubao' | 'gemini';
export type PipelineQuality = 'full' | 'partial' | 'fallback';
+export type SubtitleJobStatus = 'queued' | 'running' | 'succeeded' | 'failed';
+
+export type SubtitleJobStage =
+ | 'queued'
+ | 'upload_received'
+ | 'preparing'
+ | 'calling_provider'
+ | 'processing_result'
+ | 'succeeded'
+ | 'failed';
+
export interface WordTiming {
text: string;
startTime: number;
@@ -40,21 +54,43 @@ export interface SubtitlePipelineResult {
quality: PipelineQuality;
sourceLanguage?: string;
targetLanguage?: string;
+ ttsLanguage?: string;
duration?: number;
alignmentEngine?: string;
}
+export interface SubtitleGenerationProgress {
+ jobId: string;
+ requestId: string;
+ status: SubtitleJobStatus;
+ stage: SubtitleJobStage;
+ progress: number;
+ message: string;
+}
+
export interface TextStyles {
fontFamily: string;
fontSize: number;
color: string;
backgroundColor: string;
+ strokeColor: string;
+ strokeWidth: number;
alignment: 'left' | 'center' | 'right';
isBold: boolean;
isItalic: boolean;
isUnderline: boolean;
}
+export interface SubtitleDefaults {
+ fontSize: number;
+ bottomOffsetPercent: number;
+}
+
+export const DEFAULT_SUBTITLE_DEFAULTS: SubtitleDefaults = {
+ fontSize: 24,
+ bottomOffsetPercent: 10,
+};
+
export interface Voice {
id: string;
name: string;
Select Translation Language
-AI detect video source language automatically.
+
+
+
+ {m.upload.subtitleLanguage}
+{m.upload.subtitleLanguageHint}
+
+ {m.upload.languages.en}
+
+ {m.upload.ttsLanguage}
+{m.upload.ttsLanguageHint}
{/* Alphabet Tabs */}
-
-
-
-
-
-
-
-
-
-
+
+
+
+
{/* Language List */}
- {['A', 'C', 'T', 'D', 'E', 'R', 'J', 'K', 'M', 'F'].map((letter) => (
+ {['C', 'E', 'F', 'G', 'I'].map((letter) => (
@@ -164,7 +270,10 @@ export default function UploadScreen({ onUpload }: { onUpload: (file: File, lang
{showTrimModal && tempFile && (
{letter}
@@ -129,15 +235,15 @@ export default function UploadScreen({ onUpload }: { onUpload: (file: File, lang
))}
@@ -156,7 +262,7 @@ export default function UploadScreen({ onUpload }: { onUpload: (file: File, lang
}}
disabled={!tempFile}
>
- Generate Translated Video
+ {m.upload.generateTranslatedVideo}
-
diff --git a/src/i18n.tsx b/src/i18n.tsx
new file mode 100644
index 0000000..c61c2ea
--- /dev/null
+++ b/src/i18n.tsx
@@ -0,0 +1,304 @@
+import React, { createContext, useContext, useMemo, useState } from 'react';
+
+export type Locale = 'zh' | 'en';
+
+const messages = {
+ zh: {
+ app: {
+ chinese: '\u4e2d\u6587',
+ english: 'English',
+ productName: 'Translate 1.0',
+ },
+ upload: {
+ uploadVideoFile: '\u4e0a\u4f20\u89c6\u9891\u6587\u4ef6',
+ clickToUpload: '\u70b9\u51fb\u4e0a\u4f20\u6216\u5c06\u89c6\u9891\u62d6\u62fd\u5230\u6b64\u5904',
+ uploadVideo: '\u4e0a\u4f20\u89c6\u9891',
+ supportedFormats: '\u652f\u6301\u683c\u5f0f\uff1aMP4 / MOV / WEBM\uff0c\u6587\u4ef6\u5927\u5c0f\u4e0d\u8d85\u8fc7 500MB\u3002',
+ editingMode: '\u7f16\u8f91\u6a21\u5f0f',
+ editingModeDesc: '\u652f\u6301\u4e8c\u6b21\u7f16\u8f91\u548c\u66f4\u7cbe\u7ec6\u7684\u7ffb\u8bd1\u8c03\u6574',
+ simpleMode: '\u7b80\u5355\u6a21\u5f0f',
+ simpleModeDesc: '\u9002\u5408\u5feb\u901f\u751f\u6210\u7ffb\u8bd1\u89c6\u9891',
+ subtitleDefaults: '\u5b57\u5e55\u521d\u59cb\u6837\u5f0f',
+ subtitleDefaultsDesc: '\u8bbe\u7f6e\u5b57\u5e55\u7684\u521d\u59cb\u4f4d\u7f6e\u548c\u5927\u5c0f\uff0c\u751f\u6210\u540e\u4ecd\u53ef\u5728\u7f16\u8f91\u9875\u7ee7\u7eed\u8c03\u6574\u3002',
+ reset: '\u6062\u590d\u9ed8\u8ba4',
+ subtitlePreview: '\u5b57\u5e55\u9884\u89c8\u793a\u4f8b',
+ subtitleInitialPosition: '\u5b57\u5e55\u521d\u59cb\u4f4d\u7f6e',
+ subtitleInitialSize: '\u5b57\u5e55\u521d\u59cb\u5927\u5c0f',
+ fromBottom: '\u8ddd\u5e95\u90e8 {value}%',
+ pxValue: '{value}px',
+ subtitleLanguage: '\u5b57\u5e55\u8bed\u8a00',
+ subtitleLanguageHint: '\u5b57\u5e55\u7ffb\u8bd1\u56fa\u5b9a\u4e3a\u82f1\u8bed\u3002',
+ ttsLanguage: 'TTS \u8bed\u8a00',
+ ttsLanguageHint: '\u9009\u62e9\u914d\u97f3\u6717\u8bfb\u4f7f\u7528\u7684\u76ee\u6807\u8bed\u8a00\u3002',
+ selectTranslationLanguage: '\u9009\u62e9\u7ffb\u8bd1\u8bed\u8a00',
+ detectSourceLanguage: 'AI \u4f1a\u81ea\u52a8\u8bc6\u522b\u89c6\u9891\u6e90\u8bed\u8a00\u3002',
+ popular: '\u70ed\u95e8',
+ groupABC: 'ABC',
+ groupDEF: 'DEF',
+ groupGHI: 'GHI',
+ generateTranslatedVideo: '\u751f\u6210\u7ffb\u8bd1\u89c6\u9891',
+ languages: {
+ zh: '\u4e2d\u6587\u666e\u901a\u8bdd',
+ yue: '\u4e2d\u6587\u7ca4\u8bed',
+ en: '\u82f1\u8bed',
+ fr: '\u6cd5\u8bed',
+ id: '\u5370\u5c3c\u8bed',
+ de: '\u5fb7\u8bed',
+ fil: '\u83f2\u5f8b\u5bbe\u8bed',
+ },
+ },
+ trim: {
+ confirm: '\u786e\u8ba4',
+ selectedDuration: '\u5df2\u9009\u65f6\u957f\uff1a{value} \u79d2',
+ },
+ editor: {
+ watchVideo: '\u9884\u89c8\u89c6\u9891',
+ saveEditing: '\u4fdd\u5b58\u7f16\u8f91',
+ export: '\u5bfc\u51fa',
+ aiDub: 'AI \u914d\u97f3',
+ voiceClone: '\u58f0\u97f3\u514b\u9686',
+ tip: '\u63d0\u793a\uff1a\u4fee\u6539\u5b57\u5e55\u6587\u672c\u540e\uff0c\u9700\u8981\u91cd\u65b0\u751f\u6210\u914d\u97f3\u624d\u4f1a\u751f\u6548\u3002',
+ llm: 'LLM',
+ llmProvider: '\u6a21\u578b\u63d0\u4f9b\u65b9',
+ generateDubbing: '\u751f\u6210\u914d\u97f3',
+ separatingVocals: '\u6b63\u5728\u5206\u79bb\u4eba\u58f0...',
+ generatingTts: '\u6b63\u5728\u751f\u6210\u914d\u97f3...',
+ lowPrecision: '\u672c\u6b21\u5b57\u5e55\u65f6\u95f4\u8f74\u4e3a\u4f4e\u7cbe\u5ea6\u7ed3\u679c\uff0c\u4f60\u4ecd\u53ef\u5728\u914d\u97f3\u524d\u7ee7\u7eed\u8c03\u6574\u5b57\u5e55\u3002',
+ retryGeneration: '\u91cd\u65b0\u751f\u6210',
+ noVideoLoaded: '\u672a\u52a0\u8f7d\u89c6\u9891',
+ bgm: 'BGM',
+ applyAll: '\u5e94\u7528\u5230\u5168\u90e8\u5b57\u5e55',
+ applyAllAria: '\u5c06\u6837\u5f0f\u53d8\u66f4\u5e94\u7528\u5230\u5168\u90e8\u5b57\u5e55',
+ activeSubtitleContext: '\u5f53\u524d\u5b57\u5e55\u4e0a\u4e0b\u6587',
+ textStyles: '\u5b57\u5e55\u6837\u5f0f',
+ selectSubtitleToEdit: '\u8bf7\u9009\u62e9\u4e00\u6761\u5b57\u5e55\uff0c\u518d\u7f16\u8f91\u5b83\u7684\u8bf4\u8bdd\u4eba\u4fe1\u606f\u548c\u663e\u793a\u6837\u5f0f\u3002',
+ subtitleFontFamily: '\u5b57\u5e55\u5b57\u4f53',
+ subtitleSizePreset: '\u5b57\u5e55\u5927\u5c0f\u9884\u8bbe',
+ toggleBold: '\u5207\u6362\u7c97\u4f53',
+ toggleItalic: '\u5207\u6362\u659c\u4f53',
+ toggleUnderline: '\u5207\u6362\u4e0b\u5212\u7ebf',
+ subtitleColor: '\u5b57\u5e55\u989c\u8272',
+ subtitleStrokeColor: '\u5b57\u5e55\u63cf\u8fb9\u989c\u8272',
+ color: '\u989c\u8272',
+ stroke: '\u63cf\u8fb9',
+ small: '\u5c0f',
+ normal: '\u4e2d',
+ large: '\u5927',
+ alignLeft: '\u5de6\u5bf9\u9f50',
+ alignCenter: '\u5c45\u4e2d\u5bf9\u9f50',
+ alignRight: '\u53f3\u5bf9\u9f50',
+ videoTrack: '\u89c6\u9891\u8f68\u9053',
+ stretchDubbing: '\u62c9\u4f38\u914d\u97f3\u4ee5\u63a7\u5236\u8bed\u901f',
+ qualityQueued: '\u6392\u961f\u4e2d',
+ aiAnalyzing: 'AI \u6b63\u5728\u5206\u6790\u5e76\u7ffb\u8bd1\u89c6\u9891...',
+ thisMayTakeMinutes: '\u8fd9\u53ef\u80fd\u9700\u8981\u51e0\u5206\u949f\uff0c\u5177\u4f53\u53d6\u51b3\u4e8e\u89c6\u9891\u65f6\u957f\u3002',
+ selectVoice: '\u9009\u62e9\u97f3\u8272',
+ failedToGenerateSubtitles: '\u751f\u6210\u5b57\u5e55\u5931\u8d25\uff0c\u8bf7\u91cd\u8bd5\u6216\u68c0\u67e5 API Key\u3002',
+ exceededQuota: 'Volcengine API \u914d\u989d\u5df2\u8d85\u9650\uff0c\u8bf7\u68c0\u67e5\u5957\u9910\u548c\u8d26\u5355\u4fe1\u606f\u3002',
+ },
+ exportModal: {
+ title: '\u5bfc\u51fa\u89c6\u9891',
+ noPreview: '\u6682\u65e0\u9884\u89c8',
+ fileName: '\u6587\u4ef6\u540d',
+ format: '\u683c\u5f0f',
+ resolution: '\u5206\u8fa8\u7387',
+ original: '\u539f\u59cb',
+ errorDefault: '\u5bfc\u51fa\u65f6\u53d1\u751f\u9519\u8bef',
+ exporting: '\u6b63\u5728\u5bfc\u51fa...',
+ description: '\u6b63\u5728\u5408\u6210\u5e26\u5b57\u5e55\u548c AI \u914d\u97f3\u7684\u89c6\u9891\uff0c\u8fd9\u53ef\u80fd\u9700\u8981\u51e0\u5206\u949f\u3002',
+ close: '\u5173\u95ed',
+ cancel: '\u53d6\u6d88',
+ downloadVideo: '\u4e0b\u8f7d\u89c6\u9891',
+ processing: '\u5904\u7406\u4e2d...',
+ startExport: '\u5f00\u59cb\u5bfc\u51fa',
+ translatedSuffix: '_\u7ffb\u8bd1\u7248',
+ },
+ voiceMarket: {
+ title: '\u97f3\u8272\u5e02\u573a',
+ searchVoices: '\u641c\u7d22\u97f3\u8272...',
+ allLanguages: '\u5168\u90e8\u8bed\u8a00',
+ allGenders: '\u5168\u90e8\u6027\u522b',
+ male: '\u7537',
+ female: '\u5973',
+ neutral: '\u4e2d\u6027',
+ tip: '\u63d0\u793a\uff1a\u53ef\u4ee5\u4e3a\u5f53\u524d\u5b57\u5e55\u9009\u62e9\u97f3\u8272\uff0c\u4e5f\u53ef\u4ee5\u4e00\u952e\u5e94\u7528\u5230\u5168\u90e8\u5b57\u5e55\u3002',
+ choose: '\u9009\u62e9',
+ applyAll: '\u5e94\u7528\u5168\u90e8',
+ noVoicesFound: '\u6ca1\u6709\u627e\u5230\u7b26\u5408\u6761\u4ef6\u7684\u97f3\u8272',
+ },
+ },
+ en: {
+ app: {
+ chinese: '\u4e2d\u6587',
+ english: 'English',
+ productName: 'Translate 1.0',
+ },
+ upload: {
+ uploadVideoFile: 'Upload video file',
+ clickToUpload: 'Click to upload or drag files here',
+ uploadVideo: 'Upload Video',
+ supportedFormats: 'Supported formats: MP4 / MOV / WEBM. Maximum file size is 500MB.',
+ editingMode: 'Editing Mode',
+ editingModeDesc: 'Supports secondary editing and more precise translation',
+ simpleMode: 'Simple Mode',
+ simpleModeDesc: 'Best for quickly generating a translated video',
+ subtitleDefaults: 'Subtitle Defaults',
+ subtitleDefaultsDesc: 'Set the initial subtitle position and size. You can still refine them in the editor.',
+ reset: 'Reset',
+ subtitlePreview: 'Subtitle preview',
+ subtitleInitialPosition: 'Subtitle Initial Position',
+ subtitleInitialSize: 'Subtitle Initial Size',
+ fromBottom: '{value}% from bottom',
+ pxValue: '{value}px',
+ subtitleLanguage: 'Subtitle Language',
+ subtitleLanguageHint: 'Subtitle translation is fixed to English.',
+ ttsLanguage: 'TTS Language',
+ ttsLanguageHint: 'Choose the target language used for dubbing playback.',
+ selectTranslationLanguage: 'Select Translation Language',
+ detectSourceLanguage: 'AI detects the video source language automatically.',
+ popular: 'Popular',
+ groupABC: 'ABC',
+ groupDEF: 'DEF',
+ groupGHI: 'GHI',
+ generateTranslatedVideo: 'Generate Translated Video',
+ languages: {
+ zh: 'Chinese',
+ yue: 'Cantonese',
+ en: 'English',
+ fr: 'French',
+ id: 'Indonesian',
+ de: 'German',
+ fil: 'Filipino',
+ },
+ },
+ trim: {
+ confirm: 'Confirm',
+ selectedDuration: 'Selected duration: {value}s',
+ },
+ editor: {
+ watchVideo: 'Watch Video',
+ saveEditing: 'Save Editing',
+ export: 'Export',
+ aiDub: 'AI Dub',
+ voiceClone: 'Voice Clone',
+ tip: 'Tip: After modifying the subtitle text, you need to regenerate the dubbing to take effect.',
+ llm: 'LLM',
+ llmProvider: 'LLM Provider',
+ generateDubbing: 'Generate Dubbing',
+ separatingVocals: 'Separating Vocals...',
+ generatingTts: 'Generating TTS...',
+ lowPrecision: 'Low-precision subtitle timing is active for this generation. You can still edit subtitles before dubbing.',
+ retryGeneration: 'Retry Generation',
+ noVideoLoaded: 'No video loaded',
+ bgm: 'BGM',
+ applyAll: 'Apply to all subtitles',
+ applyAllAria: 'Apply style changes to all subtitles',
+ activeSubtitleContext: 'Active subtitle context',
+ textStyles: 'Text Styles',
+ selectSubtitleToEdit: 'Select a subtitle to edit its speaker details and display style.',
+ subtitleFontFamily: 'Subtitle font family',
+ subtitleSizePreset: 'Subtitle size preset',
+ toggleBold: 'Toggle bold',
+ toggleItalic: 'Toggle italic',
+ toggleUnderline: 'Toggle underline',
+ subtitleColor: 'Subtitle color',
+ subtitleStrokeColor: 'Subtitle stroke color',
+ color: 'Color',
+ stroke: 'Stroke',
+ small: 'Small',
+ normal: 'Normal',
+ large: 'Large',
+ alignLeft: 'Align left',
+ alignCenter: 'Align center',
+ alignRight: 'Align right',
+ videoTrack: 'Video Track',
+ stretchDubbing: 'Stretch the dubbing to control the speed',
+ qualityQueued: 'Queued',
+ aiAnalyzing: 'AI is analyzing and translating...',
+ thisMayTakeMinutes: 'This may take a few minutes depending on the video length.',
+ selectVoice: 'Select Voice',
+ failedToGenerateSubtitles: 'Failed to generate subtitles. Please try again or check your API key.',
+ exceededQuota: 'You have exceeded your Volcengine API quota. Please check your plan and billing details.',
+ },
+ exportModal: {
+ title: 'Export Video',
+ noPreview: 'No Preview',
+ fileName: 'File Name',
+ format: 'Format',
+ resolution: 'Resolution',
+ original: 'Original',
+ errorDefault: 'An error occurred during export',
+ exporting: 'Exporting...',
+ description: 'Compositing video with subtitles and AI dubbing. This may take a few minutes depending on the video length.',
+ close: 'Close',
+ cancel: 'Cancel',
+ downloadVideo: 'Download Video',
+ processing: 'Processing...',
+ startExport: 'Start Export',
+ translatedSuffix: '_translated',
+ },
+ voiceMarket: {
+ title: 'Voice Market',
+ searchVoices: 'Search voices...',
+ allLanguages: 'All Languages',
+ allGenders: 'All Genders',
+ male: 'Male',
+ female: 'Female',
+ neutral: 'Neutral',
+ tip: 'Tip: Select a voice for the current subtitle, or apply it to all subtitles.',
+ choose: 'Choose',
+ applyAll: 'Apply All',
+ noVoicesFound: 'No voices found matching your criteria',
+ },
+ },
+} as const;
+
+type MessageTree = typeof messages.zh;
+
+const formatMessage = (template: string, values?: Record