c96c49ff3f
All checks were successful
Gitea Actions Demo / Explore-Gitea-Actions (push) Successful in 2m18s
1276 lines
48 KiB
JavaScript
1276 lines
48 KiB
JavaScript
console.log('视频文件:');
|
||
// WebRTC 音视频通话应用
|
||
// import { chatWithAudioStream } from './chat_with_audio.js';
|
||
import { chatWithAudioStream, initializeHistoryMessage } from './chat_with_audio.js';
|
||
import { AudioProcessor } from './audio_processor.js';
|
||
|
||
// 在应用初始化时调用
|
||
class WebRTCChat {
|
||
constructor() {
|
||
console.log('WebRTCChat 构造函数开始执行');
|
||
|
||
|
||
|
||
// 初始化历史消息（异步）
|
||
this.initializeHistory();
|
||
|
||
this.socket = null;
|
||
this.localStream = null;
|
||
this.peerConnection = null;
|
||
this.isRecording = false;
|
||
this.mediaRecorder = null;
|
||
this.audioChunks = [];
|
||
this.videoMapping = {};
|
||
this.defaultVideo = 'chang.mp4';
|
||
this.interactionVideo = 'chang.mp4';
|
||
this.currentVideoTag = 'default';
|
||
this.currentVideo = null;
|
||
this.videoStreams = new Map(); // 存储不同视频的MediaStream
|
||
this.currentVideoStream = null;
|
||
|
||
// 添加视频相关属性
|
||
this.videoSender = null; // WebRTC视频发送器
|
||
this.currentVideoStream = null; // 当前视频流
|
||
|
||
// 初始化音频处理器
|
||
console.log('开始初始化音频处理器');
|
||
// 初始化音频处理器
|
||
this.audioProcessor = new AudioProcessor({
|
||
onSpeechStart: () => {
|
||
// this.voiceStatus.textContent = '检测到语音，开始录音...';
|
||
this.logMessage('检测到语音，开始录音...', 'info');
|
||
},
|
||
onSpeechEnd: () => {
|
||
// 语音结束回调
|
||
},
|
||
onRecognitionResult: (text) => {
|
||
// ASRTEXT = text;
|
||
// this.voiceStatus.textContent = '识别完成';
|
||
this.logMessage(`语音识别结果: ${text}`, 'success');
|
||
this.handleVoiceInput(text);
|
||
},
|
||
onError: (error) => {
|
||
// this.voiceStatus.textContent = '识别失败';
|
||
this.logMessage(error, 'error');
|
||
},
|
||
onStatusUpdate: (message, status) => {
|
||
// this.voiceStatus.textContent = message;
|
||
}
|
||
});
|
||
|
||
console.log('WebRTC 聊天应用初始化完成');
|
||
|
||
this.initializeElements();
|
||
this.initializeSocket();
|
||
this.loadVideoMapping();
|
||
this.loadVideoList();
|
||
this.loadDefaultVideo();
|
||
this.bindEvents();
|
||
|
||
// 在初始化完成后预加载常用视频
|
||
setTimeout(() => {
|
||
this.logMessage('开始预加载常用视频...', 'info');
|
||
this.preloadCommonVideos().catch(error => {
|
||
this.logMessage(`预加载过程出错: ${error.message}`, 'error');
|
||
});
|
||
}, 500); // 延迟2秒开始预加载，避免影响
|
||
|
||
window.webrtcApp = this;
|
||
}
|
||
|
||
initializeElements() {
|
||
// 视频元素
|
||
this.localVideo = document.getElementById('localVideo');
|
||
this.remoteVideo = document.getElementById('remoteVideo');
|
||
this.recordedVideo = document.getElementById('recordedVideo');
|
||
this.recordedVideoBuffer = document.getElementById('recordedVideoBuffer'); // 新增缓冲视频元素
|
||
this.videoLoading = document.getElementById('videoLoading'); // 加载指示器
|
||
|
||
// 当前活跃的视频元素标识
|
||
this.activeVideoElement = 'main'; // 'main' 或 'buffer'
|
||
|
||
// 音频状态元素
|
||
this.audioStatus = document.getElementById('audioStatus');
|
||
|
||
// 按钮元素
|
||
this.startButton = document.getElementById('startButton');
|
||
this.stopButton = document.getElementById('stopButton');
|
||
this.muteButton = document.getElementById('muteButton');
|
||
this.sendTextButton = document.getElementById('sendTextButton');
|
||
// this.startVoiceButton = document.getElementById('startVoiceButton');
|
||
// this.stopVoiceButton = document.getElementById('stopVoiceButton');
|
||
// this.defaultVideoButton = document.getElementById('defaultVideoButton');
|
||
// this.testVideoButton = document.getElementById('testVideoButton'); // 新增测试按钮
|
||
|
||
// 输入元素
|
||
this.textInput = document.getElementById('textInput');
|
||
// this.voiceStatus = document.getElementById('voiceStatus');
|
||
|
||
// 状态元素
|
||
this.connectionStatus = document.getElementById('connectionStatus');
|
||
this.messageLog = document.getElementById('messageLog');
|
||
this.currentVideoName = document.getElementById('currentVideoName');
|
||
this.videoList = document.getElementById('videoList');
|
||
}
|
||
|
||
initializeSocket() {
|
||
this.socket = io();
|
||
|
||
this.socket.on('connect', () => {
|
||
this.updateStatus('已连接到服务器', 'connected');
|
||
this.logMessage('已连接到服务器', 'success');
|
||
});
|
||
|
||
this.socket.on('disconnect', () => {
|
||
this.updateStatus('与服务器断开连接', 'disconnected');
|
||
this.logMessage('与服务器断开连接', 'error');
|
||
});
|
||
|
||
// WebRTC 信令处理
|
||
this.socket.on('offer', (data) => {
|
||
this.handleOffer(data);
|
||
});
|
||
|
||
this.socket.on('answer', (data) => {
|
||
this.handleAnswer(data);
|
||
});
|
||
|
||
this.socket.on('ice-candidate', (data) => {
|
||
this.handleIceCandidate(data);
|
||
});
|
||
|
||
// 视频流切换处理
|
||
this.socket.on('video-stream-switched', (data) => {
|
||
this.logMessage(`收到视频流切换指令: ${data.videoFile} (${data.type}) 来自用户: ${data.from}`, 'info');
|
||
this.switchVideoStream(data.videoFile, data.type, data.text);
|
||
});
|
||
|
||
// 通话开始处理
|
||
this.socket.on('call-started', (data) => {
|
||
this.logMessage('通话已开始', 'success');
|
||
this.startDefaultVideoStream();
|
||
});
|
||
|
||
// 场景切换处理
|
||
this.socket.on('scene-switched', (data) => {
|
||
this.logMessage(`场景已切换到: ${data.currentScene}`, 'info');
|
||
// 移除自动清除缓存和播放视频的逻辑
|
||
// 现在依赖页面刷新来处理缓存清除
|
||
});
|
||
}
|
||
|
||
async initializeHistory() {
|
||
try {
|
||
await initializeHistoryMessage();
|
||
console.log('历史消息初始化完成');
|
||
} catch (error) {
|
||
console.error('历史消息初始化失败:', error);
|
||
}
|
||
}
|
||
|
||
async loadVideoMapping() {
|
||
try {
|
||
const response = await fetch('/api/video-mapping');
|
||
const data = await response.json();
|
||
this.videoMapping = data.mapping;
|
||
this.interactionVideo = data.mapping['8-4-sh'];
|
||
this.defaultVideo = data.mapping["default"];
|
||
this.logMessage('视频映射加载成功', 'success');
|
||
} catch (error) {
|
||
this.logMessage('加载视频映射失败: ' + error.message, 'error');
|
||
}
|
||
}
|
||
|
||
async loadDefaultVideo() {
|
||
try {
|
||
const response = await fetch('/api/default-video');
|
||
const data = await response.json();
|
||
this.defaultVideo = data.defaultVideo;
|
||
this.logMessage('默认视频配置加载成功', 'success');
|
||
} catch (error) {
|
||
this.logMessage('加载默认视频配置失败: ' + error.message, 'error');
|
||
}
|
||
}
|
||
|
||
async loadVideoList() {
|
||
try {
|
||
const response = await fetch('/api/videos');
|
||
const data = await response.json();
|
||
this.renderVideoList(data.videos);
|
||
this.logMessage('视频列表加载成功', 'success');
|
||
} catch (error) {
|
||
this.logMessage('加载视频列表失败: ' + error.message, 'error');
|
||
}
|
||
}
|
||
|
||
renderVideoList(videos) {
|
||
this.videoList.innerHTML = '';
|
||
videos.forEach(video => {
|
||
const videoItem = document.createElement('div');
|
||
videoItem.className = 'video-item';
|
||
videoItem.textContent = video;
|
||
videoItem.onclick = () => this.selectVideo(video);
|
||
this.videoList.appendChild(videoItem);
|
||
});
|
||
}
|
||
|
||
selectVideo(videoFile) {
|
||
// 移除之前的active类
|
||
document.querySelectorAll('.video-item').forEach(item => {
|
||
item.classList.remove('active');
|
||
});
|
||
|
||
// 添加active类到选中的视频
|
||
event.target.classList.add('active');
|
||
|
||
// 切换到选中的视频流
|
||
this.switchVideoStream(videoFile, 'manual');
|
||
|
||
// 通知服务器切换视频流
|
||
this.socket.emit('switch-video-stream', {
|
||
videoFile,
|
||
type: 'manual'
|
||
});
|
||
}
|
||
|
||
async startDefaultVideoStream() {
|
||
try {
|
||
this.logMessage('开始创建默认视频流', 'info');
|
||
|
||
// 添加加载状态
|
||
this.recordedVideo.classList.add('loading');
|
||
|
||
// 创建默认视频的MediaStream
|
||
const defaultStream = await this.createVideoStream(this.defaultVideo);
|
||
|
||
// 等待流稳定
|
||
await new Promise(resolve => setTimeout(resolve, 500));
|
||
|
||
// 检查流是否有效
|
||
if (!defaultStream || defaultStream.getTracks().length === 0) {
|
||
throw new Error('默认视频流创建失败');
|
||
}
|
||
|
||
// 设置视频流
|
||
this.currentVideoStream = defaultStream;
|
||
this.recordedVideo.srcObject = defaultStream;
|
||
this.currentVideo = this.defaultVideo;
|
||
this.currentVideoName.textContent = `默认视频: ${this.defaultVideo}`;
|
||
|
||
// 等待视频元素准备就绪
|
||
await new Promise(resolve => {
|
||
const checkReady = () => {
|
||
if (this.recordedVideo.readyState >= 2) { // HAVE_CURRENT_DATA
|
||
resolve();
|
||
} else {
|
||
setTimeout(checkReady, 100);
|
||
}
|
||
};
|
||
checkReady();
|
||
});
|
||
|
||
// 确保视频开始播放
|
||
try {
|
||
await this.recordedVideo.play();
|
||
this.logMessage('默认视频开始播放', 'success');
|
||
|
||
// 移除加载状态，添加播放状态
|
||
this.recordedVideo.classList.remove('loading');
|
||
this.recordedVideo.classList.add('playing');
|
||
} catch (playError) {
|
||
this.logMessage(`默认视频播放失败: ${playError.message}`, 'error');
|
||
this.recordedVideo.classList.remove('loading');
|
||
}
|
||
|
||
this.logMessage('默认视频流创建成功', 'success');
|
||
} catch (error) {
|
||
this.logMessage('创建默认视频流失败: ' + error.message, 'error');
|
||
this.recordedVideo.classList.remove('loading');
|
||
}
|
||
}
|
||
|
||
async testVideoFile(videoFile) {
|
||
return new Promise((resolve, reject) => {
|
||
const testVideo = document.createElement('video');
|
||
testVideo.src = `/videos/${videoFile}`;
|
||
testVideo.muted = true;
|
||
|
||
testVideo.onloadedmetadata = () => {
|
||
this.logMessage(`视频文件测试成功: ${videoFile} (${testVideo.videoWidth}x${testVideo.videoHeight})`, 'success');
|
||
resolve(true);
|
||
};
|
||
|
||
testVideo.onerror = () => {
|
||
this.logMessage(`视频文件测试失败: ${videoFile}`, 'error');
|
||
reject(new Error(`视频文件不存在或无法加载: ${videoFile}`));
|
||
};
|
||
|
||
// 设置超时
|
||
setTimeout(() => {
|
||
reject(new Error(`视频文件加载超时: ${videoFile}`));
|
||
}, 10000);
|
||
});
|
||
}
|
||
|
||
async createVideoStream(videoFile) {
|
||
// 检查缓存，但为每个视频创建独立的播放实例
|
||
const cacheKey = `${videoFile}_${Date.now()}`; // 添加时间戳确保唯一性
|
||
|
||
try {
|
||
this.logMessage(`开始创建视频流: ${videoFile}`, 'info');
|
||
|
||
// 先测试视频文件是否存在
|
||
await this.testVideoFile(videoFile);
|
||
|
||
// 创建video元素来加载视频
|
||
const video = document.createElement('video');
|
||
video.src = `/videos/${videoFile}`;
|
||
video.muted = true;
|
||
video.loop = true;
|
||
video.autoplay = false; // 手动控制播放
|
||
video.crossOrigin = 'anonymous';
|
||
video.playsInline = true;
|
||
|
||
// 预加载视频但不播放
|
||
video.preload = 'auto';
|
||
|
||
// 等待视频加载完成
|
||
await new Promise((resolve, reject) => {
|
||
video.onloadeddata = () => {
|
||
this.logMessage(`视频数据加载完成: ${videoFile}`, 'info');
|
||
// 确保从第一帧开始
|
||
video.currentTime = 0;
|
||
resolve();
|
||
};
|
||
video.onerror = (error) => {
|
||
this.logMessage(`视频加载失败: ${videoFile}`, 'error');
|
||
reject(error);
|
||
};
|
||
});
|
||
|
||
// 创建MediaStream
|
||
const canvas = document.createElement('canvas');
|
||
const ctx = canvas.getContext('2d');
|
||
|
||
// 设置canvas尺寸为视频尺寸
|
||
canvas.width = video.videoWidth || 640;
|
||
canvas.height = video.videoHeight || 480;
|
||
|
||
this.logMessage(`Canvas尺寸: ${canvas.width}x${canvas.height}`, 'info');
|
||
|
||
// 确保第一帧立即绘制，避免黑屏
|
||
await new Promise((resolve) => {
|
||
const drawFirstFrame = () => {
|
||
if (video.readyState >= video.HAVE_CURRENT_DATA) {
|
||
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
|
||
this.logMessage('已绘制第一帧到Canvas', 'info');
|
||
resolve();
|
||
} else {
|
||
setTimeout(drawFirstFrame, 10);
|
||
}
|
||
};
|
||
drawFirstFrame();
|
||
});
|
||
|
||
// 开始播放视频
|
||
try {
|
||
await video.play();
|
||
this.logMessage(`视频开始播放: ${videoFile}`, 'info');
|
||
} catch (playError) {
|
||
this.logMessage(`视频播放失败: ${playError.message}`, 'error');
|
||
}
|
||
|
||
// 等待视频真正开始播放
|
||
await new Promise(resolve => {
|
||
const checkPlay = () => {
|
||
if (video.readyState >= video.HAVE_CURRENT_DATA && !video.paused) {
|
||
resolve();
|
||
} else {
|
||
setTimeout(checkPlay, 50);
|
||
}
|
||
};
|
||
checkPlay();
|
||
});
|
||
|
||
// 绘制视频到canvas
|
||
let lastDrawTime = 0;
|
||
let isDrawing = false;
|
||
const drawFrame = () => {
|
||
const now = performance.now();
|
||
if (video.readyState >= video.HAVE_CURRENT_DATA && !isDrawing && (now - lastDrawTime > 16)) {
|
||
isDrawing = true;
|
||
lastDrawTime = now;
|
||
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
|
||
isDrawing = false;
|
||
}
|
||
requestAnimationFrame(drawFrame);
|
||
};
|
||
|
||
// 开始绘制帧
|
||
drawFrame();
|
||
|
||
// 从canvas创建MediaStream
|
||
const stream = canvas.captureStream(30);
|
||
|
||
// 等待流稳定
|
||
await new Promise(resolve => {
|
||
setTimeout(resolve, 200); // 减少等待时间
|
||
});
|
||
|
||
this.logMessage(`视频流创建成功: ${videoFile}`, 'success');
|
||
|
||
// 使用有限缓存策略（最多缓存3个视频流）
|
||
if (this.videoStreams.size >= 3) {
|
||
const firstKey = this.videoStreams.keys().next().value;
|
||
const oldStream = this.videoStreams.get(firstKey);
|
||
if (oldStream) {
|
||
oldStream.getTracks().forEach(track => track.stop());
|
||
}
|
||
this.videoStreams.delete(firstKey);
|
||
}
|
||
|
||
this.videoStreams.set(videoFile, stream);
|
||
|
||
return stream;
|
||
} catch (error) {
|
||
this.logMessage(`创建视频流失败 ${videoFile}: ${error.message}`, 'error');
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
// 在应用初始化时预加载常用视频
|
||
async preloadCommonVideos() {
|
||
// 获取所有可能需要的视频
|
||
console.log("default video, interaction video", [this.defaultVideo, this.interactionVideo])
|
||
const videosToPreload = new Set([this.defaultVideo, this.interactionVideo]);
|
||
|
||
// 添加视频映射中的所有视频
|
||
// Object.values(this.videoMapping).forEach(video => {
|
||
// videosToPreload.add(video);
|
||
// });
|
||
|
||
// 特别确保添加了5.mp4（从日志看这是常用视频）
|
||
// videosToPreload.add('d-0.mp4');
|
||
|
||
// 并行预加载，提高效率
|
||
const preloadPromises = Array.from(videosToPreload).map(async (videoFile) => {
|
||
try {
|
||
this.logMessage(`预加载视频: ${videoFile}`, 'info');
|
||
await this.createVideoStream(videoFile);
|
||
this.logMessage(`预加载完成: ${videoFile}`, 'success');
|
||
} catch (error) {
|
||
this.logMessage(`预加载失败: ${videoFile}: ${error.message}`, 'error');
|
||
}
|
||
});
|
||
|
||
await Promise.allSettled(preloadPromises);
|
||
}
|
||
|
||
// async switchVideoStream(videoFile, type = '', text = '') {
|
||
// try {
|
||
// this.logMessage(`开始切换视频流: ${videoFile} (${type})`, 'info');
|
||
|
||
// // 检查是否已缓存
|
||
// const isCached = this.videoStreams.has(videoFile);
|
||
|
||
// // 如果已缓存，直接使用，避免loading状态
|
||
// if (isCached) {
|
||
// const cachedStream = this.videoStreams.get(videoFile);
|
||
// if (cachedStream && cachedStream.getTracks().length > 0) {
|
||
// // 直接切换到缓存的流
|
||
// this.currentVideoStream = cachedStream;
|
||
// this.recordedVideo.srcObject = cachedStream;
|
||
// this.currentVideo = videoFile;
|
||
|
||
// // 立即播放，无需loading状态
|
||
// await this.recordedVideo.play();
|
||
// this.recordedVideo.classList.add('playing');
|
||
|
||
// this.logMessage(`使用缓存视频流: ${videoFile}`, 'success');
|
||
// return;
|
||
// }
|
||
// }
|
||
|
||
// // 未缓存的视频才显示loading状态
|
||
// this.recordedVideo.classList.add('loading');
|
||
|
||
// // 先创建新的视频流
|
||
// const newStream = await this.createVideoStream(videoFile);
|
||
|
||
// // 减少等待时间
|
||
// await new Promise(resolve => setTimeout(resolve, 100));
|
||
|
||
// // 检查流是否有效
|
||
// if (!newStream || newStream.getTracks().length === 0) {
|
||
// throw new Error('创建的视频流无效');
|
||
// }
|
||
|
||
// // 设置新的视频流
|
||
// this.currentVideoStream = newStream;
|
||
// this.recordedVideo.srcObject = newStream;
|
||
// this.currentVideo = videoFile;
|
||
|
||
// // 确保视频开始播放
|
||
// try {
|
||
// await this.recordedVideo.play();
|
||
// this.logMessage('视频元素开始播放', 'info');
|
||
|
||
// // 移除加载状态，添加播放状态
|
||
// this.recordedVideo.classList.remove('loading');
|
||
// this.recordedVideo.classList.add('playing');
|
||
// } catch (playError) {
|
||
// this.logMessage(`视频播放失败: ${playError.message}`, 'error');
|
||
// this.recordedVideo.classList.remove('loading');
|
||
// }
|
||
|
||
// // 现在停止旧的视频流
|
||
// if (this.currentVideoStream !== newStream) {
|
||
// const oldStream = this.currentVideoStream;
|
||
// setTimeout(() => {
|
||
// if (oldStream) {
|
||
// oldStream.getTracks().forEach(track => {
|
||
// track.stop();
|
||
// this.logMessage(`已停止旧轨道: ${track.kind}`, 'info');
|
||
// });
|
||
// }
|
||
// }, 1000); // 延迟1秒停止旧流，确保新流已经稳定
|
||
// }
|
||
|
||
// if (text) {
|
||
// this.currentVideoName.textContent = `交互视频: ${videoFile} (${type}: ${text})`;
|
||
// this.logMessage(`成功切换到交互视频流: ${videoFile} (${type}: ${text})`, 'success');
|
||
// } else {
|
||
// this.currentVideoName.textContent = `视频流: ${videoFile}`;
|
||
// this.logMessage(`成功切换到视频流: ${videoFile}`, 'success');
|
||
// }
|
||
|
||
// // 检查切换后的状态
|
||
// setTimeout(() => {
|
||
// this.checkVideoStreamStatus();
|
||
// }, 1000);
|
||
|
||
// } catch (error) {
|
||
// this.logMessage(`切换视频流失败: ${error.message}`, 'error');
|
||
// this.recordedVideo.classList.remove('loading');
|
||
|
||
// // 如果切换失败，尝试回到默认视频
|
||
// if (videoFile !== this.defaultVideo) {
|
||
// this.logMessage('尝试回到默认视频', 'info');
|
||
// await this.switchVideoStream(this.defaultVideo, 'fallback');
|
||
// }
|
||
// }
|
||
// }
|
||
|
||
// 修改原有的switchVideoStream方法，使用新的平滑切换
|
||
async switchVideoStream(videoFile, type = '', text = '') {
|
||
// 使用平滑切换方法
|
||
return await this.switchVideoStreamSmooth(videoFile, type, text);
|
||
}
|
||
|
||
// 使用replaceTrack方式切换视频
|
||
async switchVideoWithReplaceTrack(videoFile, type = '', text = '') {
|
||
try {
|
||
this.logMessage(`开始使用replaceTrack切换视频: ${videoFile}`, 'info');
|
||
|
||
// 创建新的视频流
|
||
const newVideoStream = await this.createVideoStream(videoFile);
|
||
const newVideoTrack = newVideoStream.getVideoTracks()[0];
|
||
|
||
if (!newVideoTrack) {
|
||
throw new Error('新视频流中没有视频轨道');
|
||
}
|
||
|
||
// 如果有WebRTC连接且有视频发送器，使用replaceTrack
|
||
if (this.peerConnection && this.videoSender) {
|
||
await this.videoSender.replaceTrack(newVideoTrack);
|
||
this.logMessage('WebRTC视频轨道替换成功', 'success');
|
||
}
|
||
|
||
// 同时更新本地视频显示
|
||
if (this.recordedVideo) {
|
||
// 停止当前视频流
|
||
if (this.currentVideoStream) {
|
||
this.currentVideoStream.getTracks().forEach(track => track.stop());
|
||
}
|
||
|
||
// 设置新的视频流
|
||
this.recordedVideo.srcObject = newVideoStream;
|
||
this.currentVideoStream = newVideoStream;
|
||
|
||
// 确保视频播放
|
||
try {
|
||
await this.recordedVideo.play();
|
||
this.logMessage(`本地视频切换成功: ${videoFile}`, 'success');
|
||
} catch (playError) {
|
||
this.logMessage(`本地视频播放失败: ${playError.message}`, 'error');
|
||
}
|
||
}
|
||
|
||
// 记录切换信息
|
||
if (type && text) {
|
||
this.logMessage(`视频切换完成 - 类型: ${type}, 文本: ${text}`, 'info');
|
||
}
|
||
|
||
return true;
|
||
|
||
} catch (error) {
|
||
this.logMessage(`replaceTrack视频切换失败: ${error.message}`, 'error');
|
||
console.error('switchVideoWithReplaceTrack error:', error);
|
||
|
||
// 回退到原有的切换方式
|
||
try {
|
||
await this.switchVideoStream(videoFile, type, text);
|
||
this.logMessage('已回退到原有视频切换方式', 'info');
|
||
} catch (fallbackError) {
|
||
this.logMessage(`回退切换也失败: ${fallbackError.message}`, 'error');
|
||
}
|
||
|
||
return false;
|
||
}
|
||
}
|
||
|
||
// 新增平滑视频切换方法
|
||
async switchVideoStreamSmooth(videoFile, type = '', text = '') {
|
||
try {
|
||
this.logMessage(`开始平滑切换视频流: ${videoFile} (${type})`, 'info');
|
||
|
||
// 检查是否已缓存，如果已缓存则不显示加载指示器
|
||
const isCached = this.videoStreams.has(videoFile);
|
||
if (!isCached) {
|
||
this.showVideoLoading();
|
||
}
|
||
|
||
// 确定当前活跃的视频元素和缓冲元素
|
||
const currentVideo = this.activeVideoElement === 'main' ? this.recordedVideo : this.recordedVideoBuffer;
|
||
const bufferVideo = this.activeVideoElement === 'main' ? this.recordedVideoBuffer : this.recordedVideo;
|
||
|
||
// 检查是否已缓存
|
||
let newStream;
|
||
|
||
if (isCached) {
|
||
const cachedStream = this.videoStreams.get(videoFile);
|
||
if (cachedStream && cachedStream.getTracks().length > 0) {
|
||
newStream = cachedStream;
|
||
this.logMessage(`使用缓存视频流: ${videoFile}`, 'success');
|
||
} else {
|
||
newStream = await this.createVideoStream(videoFile);
|
||
}
|
||
} else {
|
||
newStream = await this.createVideoStream(videoFile);
|
||
}
|
||
|
||
// 在缓冲视频元素中预加载新视频
|
||
bufferVideo.srcObject = newStream;
|
||
|
||
// 等待缓冲视频准备就绪
|
||
await new Promise((resolve, reject) => {
|
||
const timeout = setTimeout(() => {
|
||
reject(new Error('视频加载超时'));
|
||
}, 5000);
|
||
|
||
const onCanPlay = () => {
|
||
clearTimeout(timeout);
|
||
bufferVideo.removeEventListener('canplay', onCanPlay);
|
||
bufferVideo.removeEventListener('error', onError);
|
||
resolve();
|
||
};
|
||
|
||
const onError = (error) => {
|
||
clearTimeout(timeout);
|
||
bufferVideo.removeEventListener('canplay', onCanPlay);
|
||
bufferVideo.removeEventListener('error', onError);
|
||
reject(error);
|
||
};
|
||
|
||
bufferVideo.addEventListener('canplay', onCanPlay);
|
||
bufferVideo.addEventListener('error', onError);
|
||
|
||
// 开始播放缓冲视频
|
||
bufferVideo.play().catch(onError);
|
||
});
|
||
|
||
// 只有显示了加载指示器才隐藏
|
||
if (!isCached) {
|
||
this.hideVideoLoading();
|
||
}
|
||
|
||
// 执行淡入淡出切换
|
||
// await this.performVideoTransition(currentVideo, bufferVideo);
|
||
|
||
// 更新当前视频流和活跃元素
|
||
this.currentVideoStream = newStream;
|
||
this.currentVideo = videoFile;
|
||
this.activeVideoElement = this.activeVideoElement === 'main' ? 'buffer' : 'main';
|
||
|
||
// 停止旧视频流（延迟停止避免闪烁）
|
||
setTimeout(() => {
|
||
if (currentVideo.srcObject && currentVideo.srcObject !== newStream) {
|
||
currentVideo.srcObject.getTracks().forEach(track => track.stop());
|
||
currentVideo.srcObject = null;
|
||
}
|
||
}, 1000);
|
||
|
||
// 更新WebRTC连接中的视频轨道
|
||
if (this.peerConnection && this.videoSender) {
|
||
const newVideoTrack = newStream.getVideoTracks()[0];
|
||
if (newVideoTrack) {
|
||
await this.videoSender.replaceTrack(newVideoTrack);
|
||
}
|
||
}
|
||
|
||
// 更新显示信息
|
||
if (text) {
|
||
this.currentVideoName.textContent = `交互视频: ${videoFile} (${type}: ${text})`;
|
||
this.logMessage(`成功切换到交互视频流: ${videoFile} (${type}: ${text})`, 'success');
|
||
} else {
|
||
this.currentVideoName.textContent = `视频流: ${videoFile}`;
|
||
this.logMessage(`成功切换到视频流: ${videoFile}`, 'success');
|
||
}
|
||
|
||
} catch (error) {
|
||
this.logMessage(`平滑切换视频流失败: ${error.message}`, 'error');
|
||
// 确保隐藏加载指示器
|
||
this.hideVideoLoading();
|
||
|
||
// 如果切换失败，尝试回到默认视频
|
||
if (videoFile !== this.defaultVideo) {
|
||
this.logMessage('尝试回到默认视频', 'info');
|
||
await this.switchVideoStreamSmooth(this.defaultVideo, 'fallback');
|
||
}
|
||
}
|
||
}
|
||
|
||
// 执行视频过渡动画
|
||
async performVideoTransition(currentVideo, bufferVideo) {
|
||
return new Promise((resolve) => {
|
||
// 添加切换类
|
||
currentVideo.classList.add('switching');
|
||
bufferVideo.classList.add('switching');
|
||
|
||
// 等待CSS过渡完成
|
||
setTimeout(() => {
|
||
// 移除切换类
|
||
currentVideo.classList.remove('switching');
|
||
bufferVideo.classList.remove('switching');
|
||
|
||
// 交换z-index
|
||
if (currentVideo.style.zIndex === '2' || !currentVideo.style.zIndex) {
|
||
currentVideo.style.zIndex = '1';
|
||
bufferVideo.style.zIndex = '2';
|
||
} else {
|
||
currentVideo.style.zIndex = '2';
|
||
bufferVideo.style.zIndex = '1';
|
||
}
|
||
|
||
resolve();
|
||
}, 500); // 与CSS过渡时间一致
|
||
});
|
||
}
|
||
|
||
// 显示加载指示器
|
||
showVideoLoading() {
|
||
if (this.videoLoading) {
|
||
this.videoLoading.classList.add('show');
|
||
}
|
||
}
|
||
|
||
// 隐藏加载指示器
|
||
hideVideoLoading() {
|
||
if (this.videoLoading) {
|
||
this.videoLoading.classList.remove('show');
|
||
}
|
||
}
|
||
|
||
bindEvents() {
|
||
// 开始通话按钮
|
||
this.startButton.onclick = () => this.startCall();
|
||
|
||
// 停止通话按钮
|
||
this.stopButton.onclick = () => this.stopCall();
|
||
|
||
// 静音按钮
|
||
// this.muteButton.onclick = () => this.toggleMute();
|
||
|
||
// 回到默认视频按钮
|
||
// this.defaultVideoButton.onclick = () => this.returnToDefaultVideo();
|
||
|
||
// 测试视频文件按钮
|
||
// this.testVideoButton.onclick = () => this.testAllVideoFiles();
|
||
|
||
// 发送文本按钮
|
||
this.sendTextButton.onclick = () => this.sendText();
|
||
|
||
// 回车键发送文本
|
||
this.textInput.onkeypress = (e) => {
|
||
if (e.key === 'Enter') {
|
||
this.sendText();
|
||
}
|
||
};
|
||
|
||
// 语音输入按钮
|
||
// this.startVoiceButton.onclick = () => this.startVoiceRecording();
|
||
// this.stopVoiceButton.onclick = () => this.stopVoiceRecording();
|
||
}
|
||
|
||
async startCall() {
|
||
try {
|
||
// 切换到通话中图标
|
||
this.switchToCallingIcon();
|
||
|
||
// 添加更详细的错误处理
|
||
console.log('开始请求麦克风权限...');
|
||
this.localStream = await navigator.mediaDevices.getUserMedia({
|
||
video: false,
|
||
audio: true
|
||
});
|
||
console.log('麦克风权限获取成功');
|
||
|
||
await this.createPeerConnection();
|
||
await this.startVoiceRecording();
|
||
|
||
this.startButton.disabled = true;
|
||
this.stopButton.disabled = false;
|
||
|
||
// 显示结束通话按钮
|
||
this.stopButton.classList.add('show');
|
||
|
||
this.updateAudioStatus('已连接', 'connected');
|
||
this.logMessage('音频通话已开始', 'success');
|
||
|
||
// 确保视频映射已加载
|
||
if (Object.keys(this.videoMapping).length === 0) {
|
||
await this.loadVideoMapping();
|
||
}
|
||
|
||
this.logMessage(`视频映射已加载: ${Object.keys(this.videoMapping).length} 个映射`, 'info');
|
||
|
||
// 通知服务器通话开始
|
||
this.socket.emit('call-started');
|
||
|
||
// 开始播放当前场景的默认视频
|
||
await this.startDefaultVideoStream();
|
||
|
||
} catch (error) {
|
||
// 如果出错，恢复到默认图标
|
||
this.switchToDefaultIcon();
|
||
this.logMessage('无法访问麦克风: ' + error.message, 'error');
|
||
}
|
||
}
|
||
|
||
stopCall() {
|
||
// 恢复到默认图标
|
||
this.switchToDefaultIcon();
|
||
|
||
// 发送用户关闭连接事件到后端
|
||
if (this.socket && this.socket.connected) {
|
||
this.socket.emit('user-disconnect');
|
||
}
|
||
|
||
// 停止音频处理器
|
||
if (this.audioProcessor) {
|
||
this.audioProcessor.stopRecording();
|
||
}
|
||
|
||
if (this.localStream) {
|
||
this.localStream.getTracks().forEach(track => {
|
||
track.stop();
|
||
console.log(`停止轨道: ${track.kind}`);
|
||
});
|
||
this.localStream = null;
|
||
}
|
||
|
||
if (this.peerConnection) {
|
||
this.peerConnection.close();
|
||
this.peerConnection = null;
|
||
}
|
||
|
||
// 直接刷新页面清除所有缓存
|
||
console.log('通话已结束，正在刷新页面清除缓存...');
|
||
window.location.reload();
|
||
}
|
||
|
||
// 清除视频缓存的方法
|
||
clearVideoCache() {
|
||
// 清除视频元素缓存
|
||
if (this.recordedVideo) {
|
||
this.recordedVideo.src = '';
|
||
this.recordedVideo.srcObject = null;
|
||
this.recordedVideo.load(); // 重新加载空视频
|
||
// 暂停视频播放
|
||
this.recordedVideo.pause();
|
||
}
|
||
|
||
if (this.recordedVideoBuffer) {
|
||
this.recordedVideoBuffer.src = '';
|
||
this.recordedVideoBuffer.srcObject = null;
|
||
this.recordedVideoBuffer.load();
|
||
this.recordedVideoBuffer.pause();
|
||
}
|
||
|
||
// 清除视频流缓存
|
||
if (this.currentVideoStream) {
|
||
this.currentVideoStream.getTracks().forEach(track => track.stop());
|
||
this.currentVideoStream = null;
|
||
}
|
||
|
||
// 清除视频流映射缓存
|
||
this.videoStreams.clear();
|
||
|
||
// 重置视频相关状态
|
||
this.currentVideo = null;
|
||
this.activeVideoElement = 'main';
|
||
this.videoSender = null;
|
||
|
||
this.logMessage('视频缓存已清除，视频已停止', 'info');
|
||
}
|
||
|
||
// 添加清除API缓存的方法
|
||
async clearApiCache() {
|
||
// 重新加载视频映射
|
||
this.videoMapping = {};
|
||
await this.loadVideoMapping();
|
||
|
||
// 重新加载默认视频
|
||
await this.loadDefaultVideo();
|
||
|
||
// 重新加载视频列表
|
||
await this.loadVideoList();
|
||
|
||
this.logMessage('API缓存已清除并重新加载', 'info');
|
||
}
|
||
|
||
async createPeerConnection() {
|
||
const configuration = {
|
||
iceServers: [
|
||
{ urls: "stun:stun.qq.com:3478" },
|
||
{ urls: "stun:stun.miwifi.com:3478" },
|
||
{ urls: 'stun:stun.l.google.com:19302' },
|
||
{ urls: 'stun:stun1.l.google.com:19302' }
|
||
]
|
||
};
|
||
|
||
this.peerConnection = new RTCPeerConnection(configuration);
|
||
|
||
// 添加本地音频流
|
||
this.localStream.getTracks().forEach(track => {
|
||
this.peerConnection.addTrack(track, this.localStream);
|
||
});
|
||
|
||
// 添加初始视频流（默认视频）
|
||
try {
|
||
const initialVideoStream = await this.createVideoStream(this.defaultVideo);
|
||
const videoTrack = initialVideoStream.getVideoTracks()[0];
|
||
if (videoTrack) {
|
||
this.videoSender = this.peerConnection.addTrack(videoTrack, initialVideoStream);
|
||
this.currentVideoStream = initialVideoStream;
|
||
this.logMessage('初始视频轨道已添加到WebRTC连接', 'success');
|
||
}
|
||
} catch (error) {
|
||
this.logMessage(`添加初始视频轨道失败: ${error.message}`, 'error');
|
||
}
|
||
|
||
// 处理远程流
|
||
this.peerConnection.ontrack = (event) => {
|
||
this.remoteVideo.srcObject = event.streams[0];
|
||
};
|
||
|
||
// 处理ICE候选
|
||
this.peerConnection.onicecandidate = (event) => {
|
||
if (event.candidate) {
|
||
this.socket.emit('ice-candidate', {
|
||
candidate: event.candidate
|
||
});
|
||
}
|
||
};
|
||
|
||
// 创建并发送offer
|
||
this.peerConnection.createOffer()
|
||
.then(offer => this.peerConnection.setLocalDescription(offer))
|
||
.then(() => {
|
||
this.socket.emit('offer', {
|
||
offer: this.peerConnection.localDescription
|
||
});
|
||
})
|
||
.catch(error => {
|
||
this.logMessage('创建offer失败: ' + error.message, 'error');
|
||
});
|
||
}
|
||
|
||
async handleOffer(data) {
|
||
if (!this.peerConnection) {
|
||
await this.startCall();
|
||
}
|
||
|
||
await this.peerConnection.setRemoteDescription(new RTCSessionDescription(data.offer));
|
||
|
||
const answer = await this.peerConnection.createAnswer();
|
||
await this.peerConnection.setLocalDescription(answer);
|
||
|
||
this.socket.emit('answer', {
|
||
answer: this.peerConnection.localDescription
|
||
});
|
||
}
|
||
|
||
async handleAnswer(data) {
|
||
if (this.peerConnection) {
|
||
await this.peerConnection.setRemoteDescription(new RTCSessionDescription(data.answer));
|
||
}
|
||
}
|
||
|
||
async handleIceCandidate(data) {
|
||
if (this.peerConnection) {
|
||
await this.peerConnection.addIceCandidate(new RTCIceCandidate(data.candidate));
|
||
}
|
||
}
|
||
|
||
toggleMute() {
|
||
if (this.localStream) {
|
||
const audioTrack = this.localStream.getAudioTracks()[0];
|
||
if (audioTrack) {
|
||
audioTrack.enabled = !audioTrack.enabled;
|
||
this.muteButton.textContent = audioTrack.enabled ? '静音' : '取消静音';
|
||
this.logMessage(audioTrack.enabled ? '已取消静音' : '已静音', 'info');
|
||
}
|
||
}
|
||
}
|
||
|
||
async sendText() {
|
||
const text = this.textInput.value.trim();
|
||
// const text = 'say-5s-m-sw';
|
||
if (text) {
|
||
this.socket.emit('text-input', { text });
|
||
this.logMessage(`发送文本: ${text}`, 'info');
|
||
this.textInput.value = '';
|
||
|
||
try {
|
||
// 调用chat_with_audio进行大模型回答和音频合成
|
||
this.logMessage('正在处理文本，请稍候...', 'info');
|
||
const result = await chatWithAudioStream(text);
|
||
this.logMessage(`大模型回答: ${result.llmResponse}`, 'success');
|
||
|
||
// // // 根据文本查找对应视频并切换
|
||
// await this.handleTextInput(text);
|
||
|
||
// 并行执行两个操作
|
||
// const [result] = await Promise.all([
|
||
// this.handleTextInput(text),
|
||
// chatWithAudioStream(text)
|
||
// // 视频切换可以立即开始
|
||
// ]);
|
||
|
||
// this.logMessage(`大模型回答: ${result.llmResponse}`, 'success');
|
||
} catch (error) {
|
||
this.logMessage(`处理文本失败: ${error.message}`, 'error');
|
||
console.error('chatWithAudioStream error:', error);
|
||
}
|
||
}
|
||
}
|
||
|
||
async handleTextInput(text) {
|
||
// 根据文本查找对应视频
|
||
let videoFile = this.videoMapping['default'] || this.defaultVideo;
|
||
for (const [key, value] of Object.entries(this.videoMapping)) {
|
||
if (text.toLowerCase().includes(key.toLowerCase())) {
|
||
videoFile = value;
|
||
break;
|
||
}
|
||
}
|
||
|
||
// 检查当前视频播放状态
|
||
const currentVideo = this.recordedVideo;
|
||
const isVideoPlaying = !currentVideo.paused && !currentVideo.ended && currentVideo.currentTime > 0;
|
||
|
||
if (isVideoPlaying && this.currentVideo !== videoFile) {
|
||
// 如果当前视频正在播放且需要切换到不同视频
|
||
// 可以选择立即切换或等待当前视频播放完成
|
||
console.log('当前视频正在播放，准备切换到:', videoFile);
|
||
|
||
// 立即切换（推荐）
|
||
await this.switchVideoStream(videoFile, 'text', text);
|
||
|
||
// 或者等待当前视频播放完成再切换（可选）
|
||
// await this.waitForVideoToFinish();
|
||
// await this.switchVideoStream(videoFile, 'text', text);
|
||
} else {
|
||
// 直接切换
|
||
await this.switchVideoStream(videoFile, 'text', text);
|
||
}
|
||
|
||
// 通知服务器切换视频流
|
||
this.socket.emit('switch-video-stream', {
|
||
videoFile,
|
||
type: 'text',
|
||
text
|
||
});
|
||
}
|
||
|
||
// 修改：使用音频处理器的语音录制功能
|
||
async startVoiceRecording() {
|
||
const success = await this.audioProcessor.startRecording(this.localStream);
|
||
|
||
if (success) {
|
||
this.logMessage('高级语音录制已启动', 'success');
|
||
} else {
|
||
this.logMessage('录音启动失败', 'error');
|
||
}
|
||
}
|
||
|
||
// 修改：停止语音录制
|
||
stopVoiceRecording() {
|
||
this.audioProcessor.stopRecording();
|
||
|
||
// this.startVoiceButton.disabled = false;
|
||
// this.stopVoiceButton.disabled = true;
|
||
// this.startVoiceButton.classList.remove('recording');
|
||
// this.voiceStatus.textContent = '点击开始语音输入';
|
||
|
||
this.logMessage('语音录制已停止', 'info');
|
||
}
|
||
|
||
// 处理语音输入结果
|
||
async handleVoiceInput(text) {
|
||
if(text == ""){
|
||
console.log("识别到用户未说话")
|
||
return
|
||
}
|
||
|
||
// 根据文本查找对应视频
|
||
let videoFile = this.videoMapping['default'] || this.defaultVideo;
|
||
for (const [key, value] of Object.entries(this.videoMapping)) {
|
||
if (text.toLowerCase().includes(key.toLowerCase())) {
|
||
videoFile = value;
|
||
break;
|
||
}
|
||
}
|
||
|
||
// 切换到对应的视频流
|
||
await this.switchVideoStream(videoFile, 'voice', text);
|
||
|
||
// 通知服务器切换视频流
|
||
this.socket.emit('switch-video-stream', {
|
||
videoFile,
|
||
type: 'voice',
|
||
text
|
||
});
|
||
|
||
// 调用大模型处理
|
||
try {
|
||
this.logMessage('正在处理语音输入，请稍候...', 'info', this.currentVideoTag);
|
||
// if (this.currentVideoName === "default"){
|
||
const result = await chatWithAudioStream(text);
|
||
this.logMessage(`大模型回答: ${result.llmResponse}`, 'success');
|
||
// }
|
||
|
||
} catch (error) {
|
||
this.logMessage(`处理语音输入失败: ${error.message}`, 'error');
|
||
console.error('chatWithAudioStream error:', error);
|
||
}
|
||
}
|
||
|
||
// 删除原有的简单音频处理方法
|
||
// processVoiceInput() 和 simulateSpeechRecognition() 方法已被移除
|
||
|
||
simulateSpeechRecognition() {
|
||
// 模拟语音识别，随机返回预设的文本
|
||
const texts = ['你好', '再见', '谢谢', 'hello', 'goodbye', 'thank you'];
|
||
return texts[Math.floor(Math.random() * texts.length)];
|
||
}
|
||
|
||
returnToDefaultVideo() {
|
||
this.switchVideoStream(this.defaultVideo, 'default');
|
||
this.socket.emit('return-to-default');
|
||
this.logMessage(`已返回至默认视频: ${this.defaultVideo}`, 'info');
|
||
}
|
||
|
||
updateStatus(message, type) {
|
||
this.connectionStatus.textContent = message;
|
||
this.connectionStatus.className = `status ${type}`;
|
||
}
|
||
|
||
updateAudioStatus(message, type) {
|
||
this.audioStatus.textContent = message;
|
||
this.audioStatus.className = `status-indicator ${type}`;
|
||
}
|
||
|
||
logMessage(message, type = 'info') {
|
||
const logEntry = document.createElement('div');
|
||
logEntry.className = type;
|
||
logEntry.textContent = `[${new Date().toLocaleTimeString()}] ${message}`;
|
||
|
||
this.messageLog.appendChild(logEntry);
|
||
this.messageLog.scrollTop = this.messageLog.scrollHeight;
|
||
}
|
||
|
||
checkVideoStreamStatus() {
|
||
const status = {
|
||
hasStream: !!this.currentVideoStream,
|
||
streamTracks: this.currentVideoStream ? this.currentVideoStream.getTracks().length : 0,
|
||
videoReadyState: this.recordedVideo.readyState,
|
||
videoPaused: this.recordedVideo.paused,
|
||
videoCurrentTime: this.recordedVideo.currentTime,
|
||
videoDuration: this.recordedVideo.duration,
|
||
currentVideo: this.currentVideo
|
||
};
|
||
|
||
this.logMessage(`视频流状态: ${JSON.stringify(status)}`, 'info');
|
||
return status;
|
||
}
|
||
|
||
// 添加图标切换方法
|
||
switchToCallingIcon() {
|
||
const callIcon = document.getElementById('callIcon');
|
||
const callingIcon = document.getElementById('callingIcon');
|
||
const startButton = this.startButton;
|
||
|
||
if (callIcon && callingIcon && startButton) {
|
||
callIcon.style.display = 'none';
|
||
callingIcon.style.display = 'block';
|
||
startButton.classList.add('calling');
|
||
startButton.title = '通话中...';
|
||
}
|
||
}
|
||
|
||
switchToDefaultIcon() {
|
||
const callIcon = document.getElementById('callIcon');
|
||
const callingIcon = document.getElementById('callingIcon');
|
||
const startButton = this.startButton;
|
||
|
||
if (callIcon && callingIcon && startButton) {
|
||
callIcon.style.display = 'block';
|
||
callingIcon.style.display = 'none';
|
||
startButton.classList.remove('calling');
|
||
startButton.title = '开始通话';
|
||
startButton.disabled = false;
|
||
}
|
||
}
|
||
|
||
async testAllVideoFiles() {
|
||
this.logMessage('开始测试所有视频文件...', 'info');
|
||
|
||
const videoFiles = ['asd.mp4', 'zxc.mp4', 'jkl.mp4'];
|
||
|
||
for (const videoFile of videoFiles) {
|
||
try {
|
||
await this.testVideoFile(videoFile);
|
||
} catch (error) {
|
||
this.logMessage(`视频文件 ${videoFile} 测试失败: ${error.message}`, 'error');
|
||
}
|
||
}
|
||
|
||
this.logMessage('视频文件测试完成', 'info');
|
||
|
||
// 检查当前视频流状态
|
||
this.checkVideoStreamStatus();
|
||
}
|
||
}
|
||
|
||
// 页面加载完成后初始化应用
|
||
document.addEventListener('DOMContentLoaded', () => {
|
||
console.log('DOMContentLoaded 事件触发');
|
||
try {
|
||
new WebRTCChat();
|
||
} catch (error) {
|
||
console.error('WebRTCChat 初始化失败:', error);
|
||
}
|
||
});
|