888 lines
32 KiB
JavaScript
888 lines
32 KiB
JavaScript
console.log('视频文件:');
|
||
// WebRTC 音视频通话应用
|
||
// import { chatWithAudioStream } from './chat_with_audio.js';
|
||
import { chatWithAudioStream, initializeHistoryMessage } from './chat_with_audio.js';
|
||
import { AudioProcessor } from './audio_processor.js';
|
||
|
||
// 在应用初始化时调用
|
||
class WebRTCChat {
|
||
constructor() {
|
||
console.log('WebRTCChat 构造函数开始执行');
|
||
|
||
// 初始化历史消息（异步）
|
||
this.initializeHistory();
|
||
|
||
this.socket = null;
|
||
this.localStream = null;
|
||
this.peerConnection = null;
|
||
this.isRecording = false;
|
||
this.mediaRecorder = null;
|
||
this.audioChunks = [];
|
||
this.videoMapping = {};
|
||
this.defaultVideo = '0.mp4';
|
||
this.currentVideoTag = 'default';
|
||
this.currentVideo = null;
|
||
this.videoStreams = new Map(); // 存储不同视频的MediaStream
|
||
this.currentVideoStream = null;
|
||
|
||
// 初始化音频处理器
|
||
console.log('开始初始化音频处理器');
|
||
// 初始化音频处理器
|
||
this.audioProcessor = new AudioProcessor({
|
||
onSpeechStart: () => {
|
||
this.voiceStatus.textContent = '检测到语音，开始录音...';
|
||
this.logMessage('检测到语音，开始录音...', 'info');
|
||
},
|
||
onSpeechEnd: () => {
|
||
// 语音结束回调
|
||
},
|
||
onRecognitionResult: (text) => {
|
||
// ASRTEXT = text;
|
||
this.voiceStatus.textContent = '识别完成';
|
||
this.logMessage(`语音识别结果: ${text}`, 'success');
|
||
this.handleVoiceInput(text);
|
||
},
|
||
onError: (error) => {
|
||
this.voiceStatus.textContent = '识别失败';
|
||
this.logMessage(error, 'error');
|
||
},
|
||
onStatusUpdate: (message, status) => {
|
||
this.voiceStatus.textContent = message;
|
||
}
|
||
});
|
||
|
||
console.log('WebRTC 聊天应用初始化完成');
|
||
|
||
this.initializeElements();
|
||
this.initializeSocket();
|
||
this.loadVideoMapping();
|
||
this.loadVideoList();
|
||
this.loadDefaultVideo();
|
||
this.bindEvents();
|
||
|
||
// 在初始化完成后预加载常用视频
|
||
setTimeout(() => {
|
||
this.preloadCommonVideos();
|
||
}, 500); // 延迟2秒开始预加载，避免影响
|
||
|
||
window.webrtcApp = this;
|
||
}
|
||
|
||
initializeElements() {
|
||
// 视频元素
|
||
this.localVideo = document.getElementById('localVideo');
|
||
this.remoteVideo = document.getElementById('remoteVideo');
|
||
this.recordedVideo = document.getElementById('recordedVideo');
|
||
|
||
// 音频状态元素
|
||
this.audioStatus = document.getElementById('audioStatus');
|
||
|
||
// 按钮元素
|
||
this.startButton = document.getElementById('startButton');
|
||
this.stopButton = document.getElementById('stopButton');
|
||
this.muteButton = document.getElementById('muteButton');
|
||
this.sendTextButton = document.getElementById('sendTextButton');
|
||
this.startVoiceButton = document.getElementById('startVoiceButton');
|
||
this.stopVoiceButton = document.getElementById('stopVoiceButton');
|
||
this.defaultVideoButton = document.getElementById('defaultVideoButton');
|
||
// this.testVideoButton = document.getElementById('testVideoButton'); // 新增测试按钮
|
||
|
||
// 输入元素
|
||
this.textInput = document.getElementById('textInput');
|
||
this.voiceStatus = document.getElementById('voiceStatus');
|
||
|
||
// 状态元素
|
||
this.connectionStatus = document.getElementById('connectionStatus');
|
||
this.messageLog = document.getElementById('messageLog');
|
||
this.currentVideoName = document.getElementById('currentVideoName');
|
||
this.videoList = document.getElementById('videoList');
|
||
}
|
||
|
||
initializeSocket() {
|
||
this.socket = io();
|
||
|
||
this.socket.on('connect', () => {
|
||
this.updateStatus('已连接到服务器', 'connected');
|
||
this.logMessage('已连接到服务器', 'success');
|
||
});
|
||
|
||
this.socket.on('disconnect', () => {
|
||
this.updateStatus('与服务器断开连接', 'disconnected');
|
||
this.logMessage('与服务器断开连接', 'error');
|
||
});
|
||
|
||
// WebRTC 信令处理
|
||
this.socket.on('offer', (data) => {
|
||
this.handleOffer(data);
|
||
});
|
||
|
||
this.socket.on('answer', (data) => {
|
||
this.handleAnswer(data);
|
||
});
|
||
|
||
this.socket.on('ice-candidate', (data) => {
|
||
this.handleIceCandidate(data);
|
||
});
|
||
|
||
// 视频流切换处理
|
||
this.socket.on('video-stream-switched', (data) => {
|
||
this.logMessage(`收到视频流切换指令: ${data.videoFile} (${data.type}) 来自用户: ${data.from}`, 'info');
|
||
this.switchVideoStream(data.videoFile, data.type, data.text);
|
||
});
|
||
|
||
// 通话开始处理
|
||
this.socket.on('call-started', (data) => {
|
||
this.logMessage('通话已开始', 'success');
|
||
this.startDefaultVideoStream();
|
||
});
|
||
}
|
||
|
||
async initializeHistory() {
|
||
try {
|
||
await initializeHistoryMessage();
|
||
console.log('历史消息初始化完成');
|
||
} catch (error) {
|
||
console.error('历史消息初始化失败:', error);
|
||
}
|
||
}
|
||
|
||
async loadVideoMapping() {
|
||
try {
|
||
const response = await fetch('/api/video-mapping');
|
||
const data = await response.json();
|
||
this.videoMapping = data.mapping;
|
||
this.logMessage('视频映射加载成功', 'success');
|
||
} catch (error) {
|
||
this.logMessage('加载视频映射失败: ' + error.message, 'error');
|
||
}
|
||
}
|
||
|
||
async loadDefaultVideo() {
|
||
try {
|
||
const response = await fetch('/api/default-video');
|
||
const data = await response.json();
|
||
this.defaultVideo = data.defaultVideo;
|
||
this.logMessage('默认视频配置加载成功', 'success');
|
||
} catch (error) {
|
||
this.logMessage('加载默认视频配置失败: ' + error.message, 'error');
|
||
}
|
||
}
|
||
|
||
async loadVideoList() {
|
||
try {
|
||
const response = await fetch('/api/videos');
|
||
const data = await response.json();
|
||
this.renderVideoList(data.videos);
|
||
this.logMessage('视频列表加载成功', 'success');
|
||
} catch (error) {
|
||
this.logMessage('加载视频列表失败: ' + error.message, 'error');
|
||
}
|
||
}
|
||
|
||
renderVideoList(videos) {
|
||
this.videoList.innerHTML = '';
|
||
videos.forEach(video => {
|
||
const videoItem = document.createElement('div');
|
||
videoItem.className = 'video-item';
|
||
videoItem.textContent = video;
|
||
videoItem.onclick = () => this.selectVideo(video);
|
||
this.videoList.appendChild(videoItem);
|
||
});
|
||
}
|
||
|
||
selectVideo(videoFile) {
|
||
// 移除之前的active类
|
||
document.querySelectorAll('.video-item').forEach(item => {
|
||
item.classList.remove('active');
|
||
});
|
||
|
||
// 添加active类到选中的视频
|
||
event.target.classList.add('active');
|
||
|
||
// 切换到选中的视频流
|
||
this.switchVideoStream(videoFile, 'manual');
|
||
|
||
// 通知服务器切换视频流
|
||
this.socket.emit('switch-video-stream', {
|
||
videoFile,
|
||
type: 'manual'
|
||
});
|
||
}
|
||
|
||
async startDefaultVideoStream() {
|
||
try {
|
||
this.logMessage('开始创建默认视频流', 'info');
|
||
|
||
// 添加加载状态
|
||
this.recordedVideo.classList.add('loading');
|
||
|
||
// 创建默认视频的MediaStream
|
||
const defaultStream = await this.createVideoStream(this.defaultVideo);
|
||
|
||
// 等待流稳定
|
||
await new Promise(resolve => setTimeout(resolve, 500));
|
||
|
||
// 检查流是否有效
|
||
if (!defaultStream || defaultStream.getTracks().length === 0) {
|
||
throw new Error('默认视频流创建失败');
|
||
}
|
||
|
||
// 设置视频流
|
||
this.currentVideoStream = defaultStream;
|
||
this.recordedVideo.srcObject = defaultStream;
|
||
this.currentVideo = this.defaultVideo;
|
||
this.currentVideoName.textContent = `默认视频: ${this.defaultVideo}`;
|
||
|
||
// 等待视频元素准备就绪
|
||
await new Promise(resolve => {
|
||
const checkReady = () => {
|
||
if (this.recordedVideo.readyState >= 2) { // HAVE_CURRENT_DATA
|
||
resolve();
|
||
} else {
|
||
setTimeout(checkReady, 100);
|
||
}
|
||
};
|
||
checkReady();
|
||
});
|
||
|
||
// 确保视频开始播放
|
||
try {
|
||
await this.recordedVideo.play();
|
||
this.logMessage('默认视频开始播放', 'success');
|
||
|
||
// 移除加载状态，添加播放状态
|
||
this.recordedVideo.classList.remove('loading');
|
||
this.recordedVideo.classList.add('playing');
|
||
} catch (playError) {
|
||
this.logMessage(`默认视频播放失败: ${playError.message}`, 'error');
|
||
this.recordedVideo.classList.remove('loading');
|
||
}
|
||
|
||
this.logMessage('默认视频流创建成功', 'success');
|
||
} catch (error) {
|
||
this.logMessage('创建默认视频流失败: ' + error.message, 'error');
|
||
this.recordedVideo.classList.remove('loading');
|
||
}
|
||
}
|
||
|
||
async testVideoFile(videoFile) {
|
||
return new Promise((resolve, reject) => {
|
||
const testVideo = document.createElement('video');
|
||
testVideo.src = `/videos/${videoFile}`;
|
||
testVideo.muted = true;
|
||
|
||
testVideo.onloadedmetadata = () => {
|
||
this.logMessage(`视频文件测试成功: ${videoFile} (${testVideo.videoWidth}x${testVideo.videoHeight})`, 'success');
|
||
resolve(true);
|
||
};
|
||
|
||
testVideo.onerror = () => {
|
||
this.logMessage(`视频文件测试失败: ${videoFile}`, 'error');
|
||
reject(new Error(`视频文件不存在或无法加载: ${videoFile}`));
|
||
};
|
||
|
||
// 设置超时
|
||
setTimeout(() => {
|
||
reject(new Error(`视频文件加载超时: ${videoFile}`));
|
||
}, 10000);
|
||
});
|
||
}
|
||
|
||
async createVideoStream(videoFile) {
|
||
// 如果已经缓存了这个视频流，直接返回
|
||
if (this.videoStreams.has(videoFile)) {
|
||
this.logMessage(`使用缓存的视频流: ${videoFile}`, 'info');
|
||
return this.videoStreams.get(videoFile);
|
||
}
|
||
|
||
try {
|
||
this.logMessage(`开始创建视频流: ${videoFile}`, 'info');
|
||
|
||
// 先测试视频文件是否存在
|
||
await this.testVideoFile(videoFile);
|
||
|
||
// 创建video元素来加载视频
|
||
const video = document.createElement('video');
|
||
video.src = `/videos/${videoFile}`;
|
||
video.muted = true;
|
||
video.loop = true;
|
||
video.autoplay = true;
|
||
video.crossOrigin = 'anonymous'; // 添加跨域支持
|
||
video.playsInline = true; // 添加playsInline属性
|
||
|
||
// 等待视频加载完成
|
||
await new Promise((resolve, reject) => {
|
||
video.onloadedmetadata = () => {
|
||
this.logMessage(`视频元数据加载完成: ${videoFile}`, 'info');
|
||
resolve();
|
||
};
|
||
video.onerror = (error) => {
|
||
this.logMessage(`视频加载失败: ${videoFile}`, 'error');
|
||
reject(error);
|
||
};
|
||
video.onloadstart = () => {
|
||
this.logMessage(`开始加载视频: ${videoFile}`, 'info');
|
||
};
|
||
video.oncanplay = () => {
|
||
this.logMessage(`视频可以播放: ${videoFile}`, 'info');
|
||
};
|
||
});
|
||
|
||
// 创建MediaStream
|
||
const canvas = document.createElement('canvas');
|
||
const ctx = canvas.getContext('2d');
|
||
|
||
// 设置canvas尺寸为视频尺寸
|
||
canvas.width = video.videoWidth || 640;
|
||
canvas.height = video.videoHeight || 480;
|
||
|
||
this.logMessage(`Canvas尺寸: ${canvas.width}x${canvas.height}`, 'info');
|
||
|
||
// 开始播放视频
|
||
try {
|
||
await video.play();
|
||
this.logMessage(`视频开始播放: ${videoFile}`, 'info');
|
||
} catch (playError) {
|
||
this.logMessage(`视频播放失败: ${playError.message}`, 'error');
|
||
// 即使播放失败也继续创建流
|
||
}
|
||
|
||
// 等待视频开始播放
|
||
await new Promise(resolve => {
|
||
const checkPlay = () => {
|
||
if (video.readyState >= video.HAVE_CURRENT_DATA) {
|
||
resolve();
|
||
} else {
|
||
setTimeout(checkPlay, 100);
|
||
}
|
||
};
|
||
checkPlay();
|
||
});
|
||
|
||
// 绘制视频到canvas
|
||
let lastDrawTime = 0;
|
||
let isDrawing = false;
|
||
const drawFrame = () => {
|
||
const now = performance.now();
|
||
// 限制绘制频率，确保平滑过渡
|
||
if (video.readyState >= video.HAVE_CURRENT_DATA && !isDrawing && (now - lastDrawTime > 16)) { // 约60fps
|
||
isDrawing = true;
|
||
lastDrawTime = now;
|
||
ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
|
||
isDrawing = false;
|
||
}
|
||
requestAnimationFrame(drawFrame);
|
||
};
|
||
|
||
// 开始绘制帧
|
||
drawFrame();
|
||
|
||
// 从canvas创建MediaStream
|
||
const stream = canvas.captureStream(30); // 30fps
|
||
|
||
// 等待流创建完成并稳定
|
||
await new Promise(resolve => {
|
||
setTimeout(resolve, 500); // 给更多时间让流稳定
|
||
});
|
||
|
||
this.logMessage(`视频流创建成功: ${videoFile}`, 'success');
|
||
|
||
// 缓存这个视频流
|
||
this.videoStreams.set(videoFile, stream);
|
||
|
||
return stream;
|
||
} catch (error) {
|
||
this.logMessage(`创建视频流失败 ${videoFile}: ${error.message}`, 'error');
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
// 在应用初始化时预加载常用视频
|
||
// 在WebRTCChat类中添加预加载方法
|
||
async preloadCommonVideos() {
|
||
// 获取所有可能需要的视频
|
||
const videosToPreload = new Set([]);
|
||
|
||
// // 添加视频映射中的所有视频
|
||
// Object.values(this.videoMapping).forEach(video => {
|
||
// videosToPreload.add(video);
|
||
// });
|
||
|
||
// 特别确保添加了5.mp4（从日志看这是常用视频）
|
||
videosToPreload.add('5.mp4');
|
||
|
||
// 开始预加载
|
||
for (const videoFile of videosToPreload) {
|
||
try {
|
||
this.logMessage(`预加载视频: ${videoFile}`, 'info');
|
||
await this.createVideoStream(videoFile);
|
||
this.logMessage(`预加载完成: ${videoFile}`, 'success');
|
||
} catch (error) {
|
||
this.logMessage(`预加载失败: ${videoFile}: ${error.message}`, 'error');
|
||
}
|
||
}
|
||
}
|
||
|
||
async switchVideoStream(videoFile, type = '', text = '') {
|
||
try {
|
||
this.logMessage(`开始切换视频流: ${videoFile} (${type})`, 'info');
|
||
|
||
// 如果视频已缓存，减少loading时间
|
||
const isCached = this.videoStreams.has(videoFile);
|
||
if (!isCached) {
|
||
this.recordedVideo.classList.add('loading');
|
||
}
|
||
|
||
// // 添加加载状态
|
||
// this.recordedVideo.classList.add('loading');
|
||
|
||
// 先创建新的视频流，不立即停止旧的
|
||
const newStream = await this.createVideoStream(videoFile);
|
||
|
||
// 缓存的视频减少等待时间
|
||
const waitTime = isCached ? 50 : 200;
|
||
await new Promise(resolve => setTimeout(resolve, waitTime));
|
||
|
||
// 检查流是否有效
|
||
if (!newStream || newStream.getTracks().length === 0) {
|
||
throw new Error('创建的视频流无效');
|
||
}
|
||
|
||
// 先设置新的视频流，再停止旧的
|
||
this.currentVideoStream = newStream;
|
||
this.recordedVideo.srcObject = newStream;
|
||
this.currentVideo = videoFile;
|
||
|
||
// 确保视频开始播放
|
||
try {
|
||
await this.recordedVideo.play();
|
||
this.logMessage('视频元素开始播放', 'info');
|
||
|
||
// 移除加载状态，添加播放状态
|
||
this.recordedVideo.classList.remove('loading');
|
||
this.recordedVideo.classList.add('playing');
|
||
} catch (playError) {
|
||
this.logMessage(`视频播放失败: ${playError.message}`, 'error');
|
||
this.recordedVideo.classList.remove('loading');
|
||
}
|
||
|
||
// 现在停止旧的视频流
|
||
if (this.currentVideoStream !== newStream) {
|
||
const oldStream = this.currentVideoStream;
|
||
setTimeout(() => {
|
||
if (oldStream) {
|
||
oldStream.getTracks().forEach(track => {
|
||
track.stop();
|
||
this.logMessage(`已停止旧轨道: ${track.kind}`, 'info');
|
||
});
|
||
}
|
||
}, 1000); // 延迟1秒停止旧流，确保新流已经稳定
|
||
}
|
||
|
||
if (text) {
|
||
this.currentVideoName.textContent = `交互视频: ${videoFile} (${type}: ${text})`;
|
||
this.logMessage(`成功切换到交互视频流: ${videoFile} (${type}: ${text})`, 'success');
|
||
} else {
|
||
this.currentVideoName.textContent = `视频流: ${videoFile}`;
|
||
this.logMessage(`成功切换到视频流: ${videoFile}`, 'success');
|
||
}
|
||
|
||
// 检查切换后的状态
|
||
setTimeout(() => {
|
||
this.checkVideoStreamStatus();
|
||
}, 1000);
|
||
|
||
} catch (error) {
|
||
this.logMessage(`切换视频流失败: ${error.message}`, 'error');
|
||
this.recordedVideo.classList.remove('loading');
|
||
|
||
// 如果切换失败，尝试回到默认视频
|
||
if (videoFile !== this.defaultVideo) {
|
||
this.logMessage('尝试回到默认视频', 'info');
|
||
await this.switchVideoStream(this.defaultVideo, 'fallback');
|
||
}
|
||
}
|
||
}
|
||
|
||
bindEvents() {
|
||
// 开始通话按钮
|
||
this.startButton.onclick = () => this.startCall();
|
||
|
||
// 停止通话按钮
|
||
this.stopButton.onclick = () => this.stopCall();
|
||
|
||
// 静音按钮
|
||
// this.muteButton.onclick = () => this.toggleMute();
|
||
|
||
// 回到默认视频按钮
|
||
// this.defaultVideoButton.onclick = () => this.returnToDefaultVideo();
|
||
|
||
// 测试视频文件按钮
|
||
// this.testVideoButton.onclick = () => this.testAllVideoFiles();
|
||
|
||
// 发送文本按钮
|
||
this.sendTextButton.onclick = () => this.sendText();
|
||
|
||
// 回车键发送文本
|
||
this.textInput.onkeypress = (e) => {
|
||
if (e.key === 'Enter') {
|
||
this.sendText();
|
||
}
|
||
};
|
||
|
||
// 语音输入按钮
|
||
this.startVoiceButton.onclick = () => this.startVoiceRecording();
|
||
this.stopVoiceButton.onclick = () => this.stopVoiceRecording();
|
||
}
|
||
|
||
async startCall() {
|
||
try {
|
||
this.localStream = await navigator.mediaDevices.getUserMedia({
|
||
video: false,
|
||
audio: true
|
||
});
|
||
|
||
this.createPeerConnection();
|
||
|
||
this.startButton.disabled = true;
|
||
this.stopButton.disabled = false;
|
||
|
||
this.updateAudioStatus('已连接', 'connected');
|
||
this.logMessage('音频通话已开始', 'success');
|
||
|
||
// 确保视频映射已加载
|
||
if (Object.keys(this.videoMapping).length === 0) {
|
||
await this.loadVideoMapping();
|
||
}
|
||
|
||
this.logMessage(`视频映射已加载: ${Object.keys(this.videoMapping).length} 个映射`, 'info');
|
||
|
||
// 通知服务器通话开始
|
||
this.socket.emit('call-started');
|
||
|
||
} catch (error) {
|
||
this.logMessage('无法访问麦克风: ' + error.message, 'error');
|
||
}
|
||
}
|
||
|
||
stopCall() {
|
||
if (this.localStream) {
|
||
this.localStream.getTracks().forEach(track => track.stop());
|
||
this.localStream = null;
|
||
}
|
||
|
||
if (this.peerConnection) {
|
||
this.peerConnection.close();
|
||
this.peerConnection = null;
|
||
}
|
||
|
||
// 停止当前视频流
|
||
if (this.currentVideoStream) {
|
||
this.currentVideoStream.getTracks().forEach(track => track.stop());
|
||
this.currentVideoStream = null;
|
||
}
|
||
|
||
this.recordedVideo.srcObject = null;
|
||
this.currentVideo = null;
|
||
this.currentVideoName.textContent = '未选择视频';
|
||
|
||
this.startButton.disabled = false;
|
||
this.stopButton.disabled = true;
|
||
|
||
this.updateAudioStatus('未连接', 'disconnected');
|
||
this.logMessage('音频通话已结束', 'info');
|
||
}
|
||
|
||
createPeerConnection() {
|
||
const configuration = {
|
||
iceServers: [
|
||
{ urls: "stun:stun.qq.com:3478" },
|
||
{ urls: "stun:stun.miwifi.com:3478" },
|
||
{ urls: 'stun:stun.l.google.com:19302' },
|
||
{ urls: 'stun:stun1.l.google.com:19302' }
|
||
]
|
||
};
|
||
|
||
this.peerConnection = new RTCPeerConnection(configuration);
|
||
|
||
// 添加本地音频流
|
||
this.localStream.getTracks().forEach(track => {
|
||
this.peerConnection.addTrack(track, this.localStream);
|
||
});
|
||
|
||
// 处理远程流
|
||
this.peerConnection.ontrack = (event) => {
|
||
this.remoteVideo.srcObject = event.streams[0];
|
||
};
|
||
|
||
// 处理ICE候选
|
||
this.peerConnection.onicecandidate = (event) => {
|
||
if (event.candidate) {
|
||
this.socket.emit('ice-candidate', {
|
||
candidate: event.candidate
|
||
});
|
||
}
|
||
};
|
||
|
||
// 创建并发送offer
|
||
this.peerConnection.createOffer()
|
||
.then(offer => this.peerConnection.setLocalDescription(offer))
|
||
.then(() => {
|
||
this.socket.emit('offer', {
|
||
offer: this.peerConnection.localDescription
|
||
});
|
||
})
|
||
.catch(error => {
|
||
this.logMessage('创建offer失败: ' + error.message, 'error');
|
||
});
|
||
}
|
||
|
||
async handleOffer(data) {
|
||
if (!this.peerConnection) {
|
||
await this.startCall();
|
||
}
|
||
|
||
await this.peerConnection.setRemoteDescription(new RTCSessionDescription(data.offer));
|
||
|
||
const answer = await this.peerConnection.createAnswer();
|
||
await this.peerConnection.setLocalDescription(answer);
|
||
|
||
this.socket.emit('answer', {
|
||
answer: this.peerConnection.localDescription
|
||
});
|
||
}
|
||
|
||
async handleAnswer(data) {
|
||
if (this.peerConnection) {
|
||
await this.peerConnection.setRemoteDescription(new RTCSessionDescription(data.answer));
|
||
}
|
||
}
|
||
|
||
async handleIceCandidate(data) {
|
||
if (this.peerConnection) {
|
||
await this.peerConnection.addIceCandidate(new RTCIceCandidate(data.candidate));
|
||
}
|
||
}
|
||
|
||
toggleMute() {
|
||
if (this.localStream) {
|
||
const audioTrack = this.localStream.getAudioTracks()[0];
|
||
if (audioTrack) {
|
||
audioTrack.enabled = !audioTrack.enabled;
|
||
this.muteButton.textContent = audioTrack.enabled ? '静音' : '取消静音';
|
||
this.logMessage(audioTrack.enabled ? '已取消静音' : '已静音', 'info');
|
||
}
|
||
}
|
||
}
|
||
|
||
async sendText() {
|
||
// const text = this.textInput.value.trim();
|
||
const text = 'say-5s-m-sw';
|
||
if (text) {
|
||
this.socket.emit('text-input', { text });
|
||
this.logMessage(`发送文本: ${text}`, 'info');
|
||
this.textInput.value = '';
|
||
|
||
try {
|
||
// 调用chat_with_audio进行大模型回答和音频合成
|
||
this.logMessage('正在处理文本，请稍候...', 'info');
|
||
const result = await chatWithAudioStream(text);
|
||
this.logMessage(`大模型回答: ${result.llmResponse}`, 'success');
|
||
|
||
// // // 根据文本查找对应视频并切换
|
||
// await this.handleTextInput(text);
|
||
|
||
// 并行执行两个操作
|
||
// const [result] = await Promise.all([
|
||
// this.handleTextInput(text),
|
||
// chatWithAudioStream(text)
|
||
// // 视频切换可以立即开始
|
||
// ]);
|
||
|
||
// this.logMessage(`大模型回答: ${result.llmResponse}`, 'success');
|
||
} catch (error) {
|
||
this.logMessage(`处理文本失败: ${error.message}`, 'error');
|
||
console.error('chatWithAudioStream error:', error);
|
||
}
|
||
}
|
||
}
|
||
|
||
async handleTextInput(text) {
|
||
// 根据文本查找对应视频
|
||
let videoFile = this.videoMapping['default'] || this.defaultVideo;
|
||
for (const [key, value] of Object.entries(this.videoMapping)) {
|
||
if (text.toLowerCase().includes(key.toLowerCase())) {
|
||
videoFile = value;
|
||
break;
|
||
}
|
||
}
|
||
|
||
// 检查当前视频播放状态
|
||
const currentVideo = this.recordedVideo;
|
||
const isVideoPlaying = !currentVideo.paused && !currentVideo.ended && currentVideo.currentTime > 0;
|
||
|
||
if (isVideoPlaying && this.currentVideo !== videoFile) {
|
||
// 如果当前视频正在播放且需要切换到不同视频
|
||
// 可以选择立即切换或等待当前视频播放完成
|
||
console.log('当前视频正在播放，准备切换到:', videoFile);
|
||
|
||
// 立即切换（推荐）
|
||
await this.switchVideoStream(videoFile, 'text', text);
|
||
|
||
// 或者等待当前视频播放完成再切换（可选）
|
||
// await this.waitForVideoToFinish();
|
||
// await this.switchVideoStream(videoFile, 'text', text);
|
||
} else {
|
||
// 直接切换
|
||
await this.switchVideoStream(videoFile, 'text', text);
|
||
}
|
||
|
||
// 通知服务器切换视频流
|
||
this.socket.emit('switch-video-stream', {
|
||
videoFile,
|
||
type: 'text',
|
||
text
|
||
});
|
||
}
|
||
|
||
// 修改：使用音频处理器的语音录制功能
|
||
async startVoiceRecording() {
|
||
const success = await this.audioProcessor.startRecording();
|
||
|
||
if (success) {
|
||
this.startVoiceButton.disabled = true;
|
||
this.stopVoiceButton.disabled = false;
|
||
this.startVoiceButton.classList.add('recording');
|
||
this.voiceStatus.textContent = '等待语音输入...';
|
||
this.logMessage('高级语音录制已启动', 'success');
|
||
} else {
|
||
this.voiceStatus.textContent = '录音启动失败';
|
||
}
|
||
}
|
||
|
||
// 修改：停止语音录制
|
||
stopVoiceRecording() {
|
||
this.audioProcessor.stopRecording();
|
||
|
||
this.startVoiceButton.disabled = false;
|
||
this.stopVoiceButton.disabled = true;
|
||
this.startVoiceButton.classList.remove('recording');
|
||
this.voiceStatus.textContent = '点击开始语音输入';
|
||
|
||
this.logMessage('语音录制已停止', 'info');
|
||
}
|
||
|
||
// 处理语音输入结果
|
||
async handleVoiceInput(text) {
|
||
// 根据文本查找对应视频
|
||
let videoFile = this.videoMapping['default'] || this.defaultVideo;
|
||
for (const [key, value] of Object.entries(this.videoMapping)) {
|
||
if (text.toLowerCase().includes(key.toLowerCase())) {
|
||
videoFile = value;
|
||
break;
|
||
}
|
||
}
|
||
|
||
// 切换到对应的视频流
|
||
await this.switchVideoStream(videoFile, 'voice', text);
|
||
|
||
// 通知服务器切换视频流
|
||
this.socket.emit('switch-video-stream', {
|
||
videoFile,
|
||
type: 'voice',
|
||
text
|
||
});
|
||
|
||
// 调用大模型处理
|
||
try {
|
||
this.logMessage('正在处理语音输入，请稍候...', 'info', this.currentVideoTag);
|
||
// if (this.currentVideoName === "default"){
|
||
const result = await chatWithAudioStream(text);
|
||
this.logMessage(`大模型回答: ${result.llmResponse}`, 'success');
|
||
// }
|
||
|
||
} catch (error) {
|
||
this.logMessage(`处理语音输入失败: ${error.message}`, 'error');
|
||
console.error('chatWithAudioStream error:', error);
|
||
}
|
||
}
|
||
|
||
// 删除原有的简单音频处理方法
|
||
// processVoiceInput() 和 simulateSpeechRecognition() 方法已被移除
|
||
|
||
simulateSpeechRecognition() {
|
||
// 模拟语音识别，随机返回预设的文本
|
||
const texts = ['你好', '再见', '谢谢', 'hello', 'goodbye', 'thank you'];
|
||
return texts[Math.floor(Math.random() * texts.length)];
|
||
}
|
||
|
||
returnToDefaultVideo() {
|
||
this.switchVideoStream(this.defaultVideo, 'default');
|
||
this.socket.emit('return-to-default');
|
||
this.logMessage(`已返回至默认视频: ${this.defaultVideo}`, 'info');
|
||
}
|
||
|
||
updateStatus(message, type) {
|
||
this.connectionStatus.textContent = message;
|
||
this.connectionStatus.className = `status ${type}`;
|
||
}
|
||
|
||
updateAudioStatus(message, type) {
|
||
this.audioStatus.textContent = message;
|
||
this.audioStatus.className = `status-indicator ${type}`;
|
||
}
|
||
|
||
logMessage(message, type = 'info') {
|
||
const logEntry = document.createElement('div');
|
||
logEntry.className = type;
|
||
logEntry.textContent = `[${new Date().toLocaleTimeString()}] ${message}`;
|
||
|
||
this.messageLog.appendChild(logEntry);
|
||
this.messageLog.scrollTop = this.messageLog.scrollHeight;
|
||
}
|
||
|
||
checkVideoStreamStatus() {
|
||
const status = {
|
||
hasStream: !!this.currentVideoStream,
|
||
streamTracks: this.currentVideoStream ? this.currentVideoStream.getTracks().length : 0,
|
||
videoReadyState: this.recordedVideo.readyState,
|
||
videoPaused: this.recordedVideo.paused,
|
||
videoCurrentTime: this.recordedVideo.currentTime,
|
||
videoDuration: this.recordedVideo.duration,
|
||
currentVideo: this.currentVideo
|
||
};
|
||
|
||
this.logMessage(`视频流状态: ${JSON.stringify(status)}`, 'info');
|
||
return status;
|
||
}
|
||
|
||
async testAllVideoFiles() {
|
||
this.logMessage('开始测试所有视频文件...', 'info');
|
||
|
||
const videoFiles = ['asd.mp4', 'zxc.mp4', 'jkl.mp4'];
|
||
|
||
for (const videoFile of videoFiles) {
|
||
try {
|
||
await this.testVideoFile(videoFile);
|
||
} catch (error) {
|
||
this.logMessage(`视频文件 ${videoFile} 测试失败: ${error.message}`, 'error');
|
||
}
|
||
}
|
||
|
||
this.logMessage('视频文件测试完成', 'info');
|
||
|
||
// 检查当前视频流状态
|
||
this.checkVideoStreamStatus();
|
||
}
|
||
}
|
||
|
||
// 页面加载完成后初始化应用
|
||
document.addEventListener('DOMContentLoaded', () => {
|
||
console.log('DOMContentLoaded 事件触发');
|
||
try {
|
||
new WebRTCChat();
|
||
} catch (error) {
|
||
console.error('WebRTCChat 初始化失败:', error);
|
||
}
|
||
});
|