import React, { useState, useRef, useEffect, useMemo, useCallback } from 'react'; import axios from 'axios'; import { ChevronLeft, Play, Pause, Volume2, Settings, Download, Save, LayoutTemplate, Type, Image as ImageIcon, Music, Scissors, Plus, Trash2, Maximize2, Loader2 } from 'lucide-react'; import VoiceMarketModal from './VoiceMarketModal'; import ExportModal from './ExportModal'; import { LlmProvider, PipelineQuality, Subtitle, TextStyles } from '../types'; import { generateSubtitlePipeline } from '../services/subtitleService'; import { generateTTS } from '../services/ttsService'; import { MINIMAX_VOICES } from '../voices'; import { apiUrl } from '../lib/apiBasePath'; export default function EditorScreen({ videoFile, targetLanguage, trimRange, onBack }: { videoFile: File | null; targetLanguage: string; trimRange?: {start: number, end: number} | null; onBack: () => void }) { const [subtitles, setSubtitles] = useState([]); const [activeSubtitleId, setActiveSubtitleId] = useState(''); const [showVoiceMarket, setShowVoiceMarket] = useState(false); const [voiceMarketTargetId, setVoiceMarketTargetId] = useState(null); const [showExportModal, setShowExportModal] = useState(false); const [isGenerating, setIsGenerating] = useState(false); const [isDubbingGenerating, setIsDubbingGenerating] = useState(false); const [generatingAudioIds, setGeneratingAudioIds] = useState>(new Set()); const [generationError, setGenerationError] = useState(null); const [subtitleQuality, setSubtitleQuality] = useState('fallback'); const [llmProvider, setLlmProvider] = useState('doubao'); // Video Player State const videoRef = useRef(null); const audioRef = useRef(null); const bgmRef = useRef(null); const audioContextRef = useRef(null); const lastPlayedSubId = useRef(null); const [isPlaying, setIsPlaying] = useState(false); const [bgmUrl, setBgmUrl] = useState(null); const [bgmBase64, setBgmBase64] = useState(null); const [isSeparating, setIsSeparating] = useState(false); const [currentTime, setCurrentTime] = useState(0); const [duration, setDuration] = useState(30); // Default 30s, updated on load const [videoUrl, setVideoUrl] = useState(''); const [videoAspectRatio, setVideoAspectRatio] = useState(16/9); const containerRef = useRef(null); const [renderedVideoWidth, setRenderedVideoWidth] = useState('100%'); const autoGenerationKeyRef = useRef(null); // Timeline Dragging State const [draggingId, setDraggingId] = useState(null); const [dragType, setDragType] = useState<'move' | 'resize-left' | 'resize-right' | null>(null); const [dragStartX, setDragStartX] = useState(0); const [initialSubTimes, setInitialSubTimes] = useState<{startTime: number, endTime: number} | null>(null); const [isDraggingPlayhead, setIsDraggingPlayhead] = useState(false); const timelineTrackRef = useRef(null); useEffect(() => { if (!containerRef.current) return; const observer = new ResizeObserver((entries) => { for (let entry of entries) { const { width, height } = entry.contentRect; const containerAspect = width / height; if (containerAspect > videoAspectRatio) { // Container is wider than video. Video height is 100%, width is scaled. setRenderedVideoWidth(height * videoAspectRatio); } else { // Container is taller than video. Video width is 100%. setRenderedVideoWidth(width); } } }); observer.observe(containerRef.current); return () => observer.disconnect(); }, [videoAspectRatio]); useEffect(() => { if (!videoFile) { setVideoUrl(''); return; } const url = URL.createObjectURL(videoFile); setVideoUrl(url); return () => { URL.revokeObjectURL(url); }; }, [videoFile]); const fetchSubtitles = useCallback(async () => { if (!videoFile) return; setIsGenerating(true); setGenerationError(null); try { const pipelineResult = await generateSubtitlePipeline( videoFile, targetLanguage, llmProvider, trimRange, ); const generatedSubs = pipelineResult.subtitles; setSubtitleQuality(pipelineResult.quality); let adjustedSubs = generatedSubs; if (trimRange) { adjustedSubs = generatedSubs .filter(sub => sub.endTime > trimRange.start && sub.startTime < trimRange.end) .map(sub => ({ ...sub, startTime: Math.max(0, sub.startTime - trimRange.start), endTime: Math.min(trimRange.end - trimRange.start, sub.endTime - trimRange.start) })); } setSubtitles(adjustedSubs); if (adjustedSubs.length > 0) { setActiveSubtitleId(adjustedSubs[0].id); } } catch (err: any) { console.error("Failed to generate subtitles:", err); setSubtitleQuality('fallback'); let errorMessage = "Failed to generate subtitles. Please try again or check your API key."; const errString = err instanceof Error ? err.message : JSON.stringify(err); if ( errString.includes("429") || errString.includes("quota") || errString.includes("RESOURCE_EXHAUSTED") || err?.status === 429 || err?.error?.code === 429 ) { errorMessage = "You have exceeded your Volcengine API quota. Please check your plan and billing details."; } else if (err instanceof Error) { errorMessage = err.message; } setGenerationError(errorMessage); setSubtitles([]); setActiveSubtitleId(''); } finally { setIsGenerating(false); } }, [videoFile, targetLanguage, trimRange, llmProvider]); // Generate subtitles on mount useEffect(() => { const autoGenerationKey = JSON.stringify({ fileName: videoFile?.name || '', fileSize: videoFile?.size || 0, targetLanguage, trimRange, llmProvider, }); if (autoGenerationKeyRef.current === autoGenerationKey) { return; } autoGenerationKeyRef.current = autoGenerationKey; fetchSubtitles(); }, [fetchSubtitles, videoFile, targetLanguage, trimRange, llmProvider]); const [textStyles, setTextStyles] = useState({ fontFamily: 'MiSans-Late', fontSize: 24, color: '#FFFFFF', backgroundColor: 'transparent', alignment: 'center', isBold: false, isItalic: false, isUnderline: false, }); const togglePlay = async () => { if (videoRef.current) { if (isPlaying) { videoRef.current.pause(); if (bgmRef.current) bgmRef.current.pause(); if (audioRef.current) audioRef.current.pause(); } else { if (trimRange && (videoRef.current.currentTime < trimRange.start || videoRef.current.currentTime >= trimRange.end)) { videoRef.current.currentTime = trimRange.start; } videoRef.current.play(); if (bgmRef.current) { bgmRef.current.currentTime = videoRef.current.currentTime; bgmRef.current.play(); } } setIsPlaying(!isPlaying); } }; const handleGenerateDubbing = async () => { if (subtitles.length === 0) return; setIsDubbingGenerating(true); // Step 1: Vocal Separation (Scheme A: Server-side AI-like separation) if (!bgmUrl && videoFile) { setIsSeparating(true); try { const formData = new FormData(); formData.append('video', videoFile); const response = await axios.post(apiUrl('/separate-vocal'), formData, { headers: { 'Content-Type': 'multipart/form-data' } }); const { instrumental } = response.data; const blob = await (await fetch(`data:audio/mp3;base64,${instrumental}`)).blob(); const url = URL.createObjectURL(blob); setBgmUrl(url); setBgmBase64(instrumental); console.log("Vocal separation completed (Instrumental extracted via Scheme A)"); } catch (err) { console.error("Vocal separation failed:", err); } finally { setIsSeparating(false); } } // Step 2: TTS Generation const toGenerate = subtitles.filter(s => !s.audioUrl).map(s => s.id); setGeneratingAudioIds(new Set(toGenerate)); try { const updatedSubtitles = [...subtitles]; for (let i = 0; i < updatedSubtitles.length; i++) { const sub = updatedSubtitles[i]; if (sub.audioUrl) continue; try { const textToSpeak = sub.translatedText || sub.text; const audioUrl = await generateTTS(textToSpeak, sub.voiceId); updatedSubtitles[i] = { ...sub, audioUrl }; // Update state incrementally so user sees progress setSubtitles([...updatedSubtitles]); } catch (err) { console.error(`Failed to generate TTS for subtitle ${sub.id}:`, err); } finally { setGeneratingAudioIds(prev => { const next = new Set(prev); next.delete(sub.id); return next; }); } } } catch (err) { console.error("Failed to generate dubbing:", err); } finally { setIsDubbingGenerating(false); setGeneratingAudioIds(new Set()); if (subtitles.some(s => s.audioUrl)) { console.log("Dubbing generation completed."); } } }; const handleTimeUpdate = () => { if (videoRef.current) { let time = videoRef.current.currentTime; if (trimRange) { if (time < trimRange.start) { videoRef.current.currentTime = trimRange.start; time = trimRange.start; } else if (time >= trimRange.end) { videoRef.current.pause(); if (bgmRef.current) bgmRef.current.pause(); if (audioRef.current) audioRef.current.pause(); setIsPlaying(false); videoRef.current.currentTime = trimRange.start; time = trimRange.start; } } setCurrentTime(time); const displayTime = trimRange ? time - trimRange.start : time; // Auto-select active subtitle based on time const activeSub = subtitles.find(s => displayTime >= s.startTime && displayTime <= s.endTime); // Sync BGM with video if (bgmRef.current && videoRef.current && isPlaying) { if (Math.abs(bgmRef.current.currentTime - videoRef.current.currentTime) > 0.3) { bgmRef.current.currentTime = videoRef.current.currentTime; } } if (activeSub) { if (activeSub.id !== activeSubtitleId) { setActiveSubtitleId(activeSub.id); } // Play dubbing if available and not already playing for this sub if (activeSub.audioUrl && lastPlayedSubId.current !== activeSub.id && isPlaying) { if (audioRef.current) { audioRef.current.pause(); } const audio = new Audio(activeSub.audioUrl); audio.volume = activeSub.volume !== undefined ? activeSub.volume : 1.0; audioRef.current = audio; audio.play().catch(e => console.error("Audio playback failed:", e)); lastPlayedSubId.current = activeSub.id; } } else { lastPlayedSubId.current = null; } } }; useEffect(() => { if (videoRef.current) { videoRef.current.volume = bgmUrl ? 0 : 0.3; } }, [bgmUrl]); const handleLoadedMetadata = () => { if (videoRef.current) { setDuration(videoRef.current.duration); videoRef.current.volume = bgmUrl ? 0 : 0.3; // Mute if BGM (instrumental) is present if (bgmRef.current) { bgmRef.current.volume = 0.5; // BGM volume } if (videoRef.current.videoHeight > 0) { setVideoAspectRatio(videoRef.current.videoWidth / videoRef.current.videoHeight); } if (trimRange) { videoRef.current.currentTime = trimRange.start; setCurrentTime(trimRange.start); } } }; const handleTimelineMouseDown = (e: React.MouseEvent, subId: string, type: 'move' | 'resize-left' | 'resize-right') => { e.stopPropagation(); const sub = subtitles.find(s => s.id === subId); if (!sub) return; setDraggingId(subId); setDragType(type); setDragStartX(e.clientX); setInitialSubTimes({ startTime: sub.startTime, endTime: sub.endTime }); setActiveSubtitleId(subId); }; const handleTimelineMouseMove = useCallback((e: MouseEvent) => { if (!timelineTrackRef.current) return; const rect = timelineTrackRef.current.getBoundingClientRect(); const timelineWidth = rect.width - 32; // Subtract padding (1rem each side) const displayDuration = trimRange ? trimRange.end - trimRange.start : duration; if (isDraggingPlayhead) { const deltaX = e.clientX - rect.left - 16; // 1rem padding const percent = Math.max(0, Math.min(1, deltaX / timelineWidth)); const newTime = percent * displayDuration; if (videoRef.current) { videoRef.current.currentTime = newTime + (trimRange?.start || 0); } return; } if (!draggingId || !initialSubTimes) return; const deltaX = e.clientX - dragStartX; const deltaSeconds = (deltaX / timelineWidth) * displayDuration; setSubtitles(prev => prev.map(sub => { if (sub.id !== draggingId) return sub; let newStart = sub.startTime; let newEnd = sub.endTime; if (dragType === 'move') { newStart = Math.max(0, Math.min(displayDuration - (initialSubTimes.endTime - initialSubTimes.startTime), initialSubTimes.startTime + deltaSeconds)); newEnd = newStart + (initialSubTimes.endTime - initialSubTimes.startTime); } else if (dragType === 'resize-left') { newStart = Math.max(0, Math.min(initialSubTimes.endTime - 0.2, initialSubTimes.startTime + deltaSeconds)); } else if (dragType === 'resize-right') { newEnd = Math.max(initialSubTimes.startTime + 0.2, Math.min(displayDuration, initialSubTimes.endTime + deltaSeconds)); } return { ...sub, startTime: newStart, endTime: newEnd }; })); }, [draggingId, dragType, dragStartX, initialSubTimes, duration, trimRange, isDraggingPlayhead]); const handleTimelineMouseUp = useCallback(() => { setDraggingId(null); setDragType(null); setInitialSubTimes(null); setIsDraggingPlayhead(false); }, []); useEffect(() => { if (draggingId || isDraggingPlayhead) { window.addEventListener('mousemove', handleTimelineMouseMove); window.addEventListener('mouseup', handleTimelineMouseUp); } else { window.removeEventListener('mousemove', handleTimelineMouseMove); window.removeEventListener('mouseup', handleTimelineMouseUp); } return () => { window.removeEventListener('mousemove', handleTimelineMouseMove); window.removeEventListener('mouseup', handleTimelineMouseUp); }; }, [draggingId, isDraggingPlayhead, handleTimelineMouseMove, handleTimelineMouseUp]); const formatTime = (seconds: number) => { if (isNaN(seconds) || seconds < 0) seconds = 0; const mins = Math.floor(seconds / 60); const secs = Math.floor(seconds % 60); return `00:${mins.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')}`; }; const displayDuration = trimRange ? trimRange.end - trimRange.start : duration; const displayCurrentTime = trimRange ? Math.max(0, currentTime - trimRange.start) : currentTime; // Calculate playhead position percentage const playheadPercent = displayDuration > 0 ? (displayCurrentTime / displayDuration) * 100 : 0; // Get current subtitle text const currentSubtitleText = subtitles.find(s => displayCurrentTime >= s.startTime && displayCurrentTime <= s.endTime)?.translatedText || ''; return (
{/* Top Header */}
M
Translate 1.0
{/* Main Workspace */}
{/* Left Sidebar - Subtitles */}

Tip: After modifying the subtitle text, you need to regenerate the dubbing to take effect!

{isGenerating ? (

AI is analyzing and translating...

This may take a minute depending on the video length.

) : ( <> {generationError && (

{generationError}

)} {!generationError && subtitleQuality === 'fallback' && (
Low-precision subtitle timing is active for this generation. You can still edit subtitles before dubbing.
)} {subtitles.map((sub, index) => (
{ setActiveSubtitleId(sub.id); if (videoRef.current) { videoRef.current.currentTime = sub.startTime + (trimRange?.start || 0); } }} > {sub.audioUrl && (
)} {generatingAudioIds.has(sub.id) && (
)}
{index + 1}. {formatTime(sub.startTime)} - {formatTime(sub.endTime)}
{sub.audioUrl && (
{ const newSubs = [...subtitles]; newSubs[index].volume = parseFloat(e.target.value); setSubtitles(newSubs); }} className="w-12 h-1 bg-gray-300 rounded-lg appearance-none cursor-pointer accent-blue-500" onClick={(e) => e.stopPropagation()} />
)}