songjvcheng/WebRtc_QingGan
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

切换增加视频过渡效果，避免黑屏。UI大整改

Browse Source
This commit is contained in:
Song367 2025-07-29 18:14:04 +08:00
parent 1ebfd472c4
commit 65f17b4a66
8 changed files with 516 additions and 144 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
server.js
View File

@ -88,15 +88,15 @@ const connectedClients = new Map();
// 视频映射配置 // 视频映射配置
const videoMapping = { const videoMapping = {
// 'say-6s-m-e': '1-m.mp4', // 'say-6s-m-e': '1-m.mp4',
'default': '0-2.mp4', 'default': 'bd-1.mp4',
// 'say-5s-amplitude': '2.mp4', // 'say-5s-amplitude': '2.mp4',
// 'say-5s-m-e': '4.mp4', // 'say-5s-m-e': '4.mp4',
'say-5s-m-sw': '5.mp4', 'say-5s-m-sw': 'd-0.mp4',
// 'say-3s-m-sw': '6.mp4', // 'say-3s-m-sw': '6.mp4',
}; };
// 默认视频流配置 // 默认视频流配置
const DEFAULT_VIDEO = '0-2.mp4'; const DEFAULT_VIDEO = 'bd-1.mp4';
const INTERACTION_TIMEOUT = 10000; // 10秒后回到默认视频 const INTERACTION_TIMEOUT = 10000; // 10秒后回到默认视频
// 获取视频列表 // 获取视频列表

14
src/chat_with_audio.js
View File

@ -69,22 +69,30 @@ function updateHistoryMessage(userInput, assistantResponse) {
} }
} }
// 保存消息到服务端
// 保存消息到服务端 // 保存消息到服务端
async function saveMessage(userInput, assistantResponse) { async function saveMessage(userInput, assistantResponse) {
try { try {
// 验证参数是否有效
if (!userInput || !userInput.trim() || !assistantResponse || !assistantResponse.trim()) {
console.warn('跳过保存消息：用户输入或助手回复为空');
return;
}
const response = await fetch('/api/messages/save', { const response = await fetch('/api/messages/save', {
method: 'POST', method: 'POST',
headers: { headers: {
'Content-Type': 'application/json' 'Content-Type': 'application/json'
}, },
body: JSON.stringify({ body: JSON.stringify({
userInput, userInput: userInput.trim(),
assistantResponse assistantResponse: assistantResponse.trim()
}) })
}); });
if (!response.ok) { if (!response.ok) {
throw new Error('保存消息失败'); const errorData = await response.json().catch(() => ({}));
throw new Error(`保存消息失败: ${response.status} ${errorData.error || response.statusText}`);
} }
console.log('消息已保存到服务端'); console.log('消息已保存到服务端');

257
src/index.html
View File

@ -2,69 +2,260 @@
<html lang="zh-CN"> <html lang="zh-CN">
<head> <head>
<meta charset="UTF-8"> <meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0"> <meta name="viewport" content="width=device-width, initial-scale=1.0, user-scalable=no">
<title>WebRTC 音频通话</title> <title>WebRTC 音频通话</title>
<link rel="stylesheet" href="styles.css"> <link rel="stylesheet" href="styles.css">
<style>
/* 全屏视频样式 */
* {
margin: 0;
padding: 0;
box-sizing: border-box;
}
html, body {
height: 100%;
overflow: hidden;
background: #000;
}
.container {
width: 100vw;
height: 100vh;
margin: 0;
padding: 0;
display: flex;
flex-direction: column;
position: relative;
}
.main-content {
flex: 1;
background: transparent;
border-radius: 0;
padding: 0;
box-shadow: none;
width: 100%;
height: 100%;
display: flex;
flex-direction: column;
}
.recorded-video-section {
flex: 1;
display: flex;
align-items: center;
justify-content: center;
width: 100%;
height: 100%;
position: relative;
}
/* 视频容器样式 - 支持双缓冲 */
.video-container {
position: relative;
width: 100vw;
height: 100vh;
overflow: hidden;
}
#recordedVideo, #recordedVideoBuffer {
position: absolute;
top: 0;
left: 0;
width: 100%;
height: 100%;
object-fit: cover;
border-radius: 0;
box-shadow: none;
transition: opacity 0.5s ease-in-out;
}
/* 主视频默认显示 */
#recordedVideo {
opacity: 1;
z-index: 2;
}
/* 缓冲视频默认隐藏 */
#recordedVideoBuffer {
opacity: 0;
z-index: 1;
}
/* 切换状态 */
#recordedVideo.switching {
opacity: 0;
}
#recordedVideoBuffer.switching {
opacity: 1;
}
/* 加载状态 */
.video-loading {
position: absolute;
top: 50%;
left: 50%;
transform: translate(-50%, -50%);
z-index: 10;
color: white;
font-size: 18px;
opacity: 0;
transition: opacity 0.3s ease;
}
.video-loading.show {
opacity: 1;
}
/* 加载动画 */
.loading-spinner {
width: 40px;
height: 40px;
border: 3px solid rgba(255, 255, 255, 0.3);
border-top: 3px solid white;
border-radius: 50%;
animation: spin 1s linear infinite;
margin: 0 auto 10px;
}
@keyframes spin {
0% { transform: rotate(0deg); }
100% { transform: rotate(360deg); }
}
.controls {
position: absolute;
bottom: 50px;
left: 50%;
transform: translateX(-50%);
z-index: 10;
display: flex;
justify-content: center;
gap: 20px;
}
#startButton {
padding: 15px 30px;
font-size: 1.1rem;
border-radius: 25px;
min-width: 200px;
background: rgba(0, 123, 255, 0.9);
backdrop-filter: blur(10px);
}
#stopButton {
width: 60px;
height: 60px;
border-radius: 50%;
background: rgba(220, 53, 69, 0.9);
backdrop-filter: blur(10px);
border: none;
cursor: pointer;
display: none;
align-items: center;
justify-content: center;
transition: all 0.3s ease;
box-shadow: 0 4px 15px rgba(220, 53, 69, 0.3);
}
#stopButton.show {
display: flex;
}
#stopButton:hover:not(:disabled) {
background: rgba(200, 35, 51, 0.95);
transform: scale(1.1);
box-shadow: 0 6px 20px rgba(220, 53, 69, 0.5);
}
#stopButton svg {
width: 24px;
height: 24px;
fill: white;
}
#stopButton:disabled {
opacity: 0.5;
cursor: not-allowed;
}
</style>
</head> </head>
<body> <body>
<div class="container"> <div class="container">
<header> <!-- 隐藏的header -->
<header style="display: none;">
<h1>WebRTC 音频通话</h1> <h1>WebRTC 音频通话</h1>
<p>实时播放录制视频，支持文本和语音输入</p> <p>实时播放录制视频，支持文本和语音输入</p>
</header> </header>
<div class="main-content"> <div class="main-content">
<!-- 音频状态显示 --> <!-- 隐藏的音频状态显示 -->
<div class="audio-status"> <div class="audio-status" style="display: none;">
<div class="status-indicator"> <div class="status-indicator">
<span id="audioStatus">未连接</span> <span id="audioStatus">未连接</span>
</div> </div>
</div> </div>
<!-- 录制视频播放区域 --> <!-- 录制视频播放区域 - 全屏显示 -->
<div class="recorded-video-section"> <div class="recorded-video-section">
<video id="recordedVideo" autoplay muted> <div class="video-container">
<source src="" type="video/mp4"> <!-- 主视频元素 -->
您的浏览器不支持视频播放 <video id="recordedVideo" autoplay muted>
</video> <source src="" type="video/mp4">
<div class="video-info"> 您的浏览器不支持视频播放
</video>
<!-- 缓冲视频元素 -->
<video id="recordedVideoBuffer" autoplay muted>
<source src="" type="video/mp4">
您的浏览器不支持视频播放
</video>
<!-- 加载指示器 -->
<div class="video-loading" id="videoLoading">
<div class="loading-spinner"></div>
<div>正在切换视频...</div>
</div>
</div>
<div class="video-info" style="display: none;">
<span id="currentVideoName">未选择视频</span> <span id="currentVideoName">未选择视频</span>
</div> </div>
</div> </div>
<!-- 控制按钮 --> <!-- 控制按钮 - 悬浮在视频上方 -->
<div class="controls"> <div class="controls">
<button id="startButton" class="btn btn-primary">开始音频通话</button> <button id="startButton" class="btn btn-primary">开始音频通话</button>
<button id="stopButton" class="btn btn-danger" disabled>停止通话</button> <button id="stopButton" class="btn btn-danger" disabled title="结束通话">
<!-- <button id="muteButton" class="btn btn-secondary">静音</button> <svg viewBox="0 0 24 24">
<button id="defaultVideoButton" class="btn btn-info">回到默认视频</button> <path d="M6.62 10.79c1.44 2.83 3.76 5.14 6.59 6.59l2.2-2.2c.27-.27.67-.36 1.02-.24 1.12.37 2.33.57 3.57.57.55 0 1 .45 1 1V20c0 .55-.45 1-1 1-9.39 0-17-7.61-17-17 0-.55.45-1 1-1h3.5c.55 0 1 .45 1 1 0 1.25.2 2.45.57 3.57.11.35.03.74-.25 1.02l-2.2 2.2z"/>
<button id="testVideoButton" class="btn btn-warning">测试视频文件</button> --> <path d="M19 12h2c0-4.97-4.03-9-9-9v2c3.87 0 7 3.13 7 7z"/>
<path d="M15 12h2c0-2.76-2.24-5-5-5v2c1.66 0 3 1.34 3 3z"/>
<line x1="18" y1="6" x2="6" y2="18" stroke="white" stroke-width="2"/>
</svg>
</button>
</div> </div>
<!-- 输入区域 --> <!-- 隐藏的输入区域 -->
<div class="input-section"> <div class="input-section" style="display: none;">
<div class="text-input-group"> <div class="text-input-group">
<input type="text" id="textInput" placeholder="输入文本内容..." /> <input type="text" id="textInput" placeholder="输入文本内容..." />
<button id="sendTextButton" class="btn btn-primary">发送文本</button> <button id="sendTextButton" class="btn btn-primary">发送文本</button>
</div> </div>
</div>
<div class="voice-input-group">
<button id="startVoiceButton" class="btn btn-success">开始语音输入</button> <!-- 隐藏的视频选择 -->
<button id="stopVoiceButton" class="btn btn-warning" disabled>停止语音输入</button> <div class="video-selection" style="display: none;">
<span id="voiceStatus">点击开始语音输入</span> <h3>选择要播放的视频</h3>
<div id="videoList" class="video-list">
<!-- 视频列表将在这里动态生成 -->
</div> </div>
</div> </div>
<!-- 视频选择 --> <!-- 隐藏的状态显示 -->
<!-- <div class="video-selection"> <div class="status-section" style="display: none;">
<h3>选择要播放的视频</h3>
<div id="videoList" class="video-list">
视频列表将在这里动态生成 -->
<!-- </div>
</div> -->
<!-- 状态显示 -->
<div class="status-section">
<div id="connectionStatus" class="status">未连接</div> <div id="connectionStatus" class="status">未连接</div>
<div id="messageLog" class="message-log"></div> <div id="messageLog" class="message-log"></div>
</div> </div>
@ -78,4 +269,4 @@
<script src="/socket.io/socket.io.js"></script> <script src="/socket.io/socket.io.js"></script>
<script type="module" src="./index.js"></script> <script type="module" src="./index.js"></script>
</body> </body>
</html> </html>

379
src/index.js
View File

@ -19,7 +19,7 @@ class WebRTCChat {
this.mediaRecorder = null; this.mediaRecorder = null;
this.audioChunks = []; this.audioChunks = [];
this.videoMapping = {}; this.videoMapping = {};
this.defaultVideo = '0-2.mp4'; this.defaultVideo = 'bd-1.mp4';
this.currentVideoTag = 'default'; this.currentVideoTag = 'default';
this.currentVideo = null; this.currentVideo = null;
this.videoStreams = new Map(); // 存储不同视频的MediaStream this.videoStreams = new Map(); // 存储不同视频的MediaStream
@ -34,7 +34,7 @@ class WebRTCChat {
// 初始化音频处理器 // 初始化音频处理器
this.audioProcessor = new AudioProcessor({ this.audioProcessor = new AudioProcessor({
onSpeechStart: () => { onSpeechStart: () => {
this.voiceStatus.textContent = '检测到语音，开始录音...'; // this.voiceStatus.textContent = '检测到语音，开始录音...';
this.logMessage('检测到语音，开始录音...', 'info'); this.logMessage('检测到语音，开始录音...', 'info');
}, },
onSpeechEnd: () => { onSpeechEnd: () => {
@ -42,16 +42,16 @@ class WebRTCChat {
}, },
onRecognitionResult: (text) => { onRecognitionResult: (text) => {
// ASRTEXT = text; // ASRTEXT = text;
this.voiceStatus.textContent = '识别完成'; // this.voiceStatus.textContent = '识别完成';
this.logMessage(`语音识别结果: ${text}`, 'success'); this.logMessage(`语音识别结果: ${text}`, 'success');
this.handleVoiceInput(text); this.handleVoiceInput(text);
}, },
onError: (error) => { onError: (error) => {
this.voiceStatus.textContent = '识别失败'; // this.voiceStatus.textContent = '识别失败';
this.logMessage(error, 'error'); this.logMessage(error, 'error');
}, },
onStatusUpdate: (message, status) => { onStatusUpdate: (message, status) => {
this.voiceStatus.textContent = message; // this.voiceStatus.textContent = message;
} }
}); });
@ -80,6 +80,11 @@ class WebRTCChat {
this.localVideo = document.getElementById('localVideo'); this.localVideo = document.getElementById('localVideo');
this.remoteVideo = document.getElementById('remoteVideo'); this.remoteVideo = document.getElementById('remoteVideo');
this.recordedVideo = document.getElementById('recordedVideo'); this.recordedVideo = document.getElementById('recordedVideo');
this.recordedVideoBuffer = document.getElementById('recordedVideoBuffer'); // 新增缓冲视频元素
this.videoLoading = document.getElementById('videoLoading'); // 加载指示器
// 当前活跃的视频元素标识
this.activeVideoElement = 'main'; // 'main' 或 'buffer'
// 音频状态元素 // 音频状态元素
this.audioStatus = document.getElementById('audioStatus'); this.audioStatus = document.getElementById('audioStatus');
@ -89,14 +94,14 @@ class WebRTCChat {
this.stopButton = document.getElementById('stopButton'); this.stopButton = document.getElementById('stopButton');
this.muteButton = document.getElementById('muteButton'); this.muteButton = document.getElementById('muteButton');
this.sendTextButton = document.getElementById('sendTextButton'); this.sendTextButton = document.getElementById('sendTextButton');
this.startVoiceButton = document.getElementById('startVoiceButton'); // this.startVoiceButton = document.getElementById('startVoiceButton');
this.stopVoiceButton = document.getElementById('stopVoiceButton'); // this.stopVoiceButton = document.getElementById('stopVoiceButton');
this.defaultVideoButton = document.getElementById('defaultVideoButton'); // this.defaultVideoButton = document.getElementById('defaultVideoButton');
// this.testVideoButton = document.getElementById('testVideoButton'); // 新增测试按钮 // this.testVideoButton = document.getElementById('testVideoButton'); // 新增测试按钮
// 输入元素 // 输入元素
this.textInput = document.getElementById('textInput'); this.textInput = document.getElementById('textInput');
this.voiceStatus = document.getElementById('voiceStatus'); // this.voiceStatus = document.getElementById('voiceStatus');
// 状态元素 // 状态元素
this.connectionStatus = document.getElementById('connectionStatus'); this.connectionStatus = document.getElementById('connectionStatus');
@ -424,7 +429,7 @@ class WebRTCChat {
// 在应用初始化时预加载常用视频 // 在应用初始化时预加载常用视频
async preloadCommonVideos() { async preloadCommonVideos() {
// 获取所有可能需要的视频 // 获取所有可能需要的视频
const videosToPreload = new Set(['0-2.mp4']); const videosToPreload = new Set(['bd-1.mp4']);
// 添加视频映射中的所有视频 // 添加视频映射中的所有视频
// Object.values(this.videoMapping).forEach(video => { // Object.values(this.videoMapping).forEach(video => {
@ -432,7 +437,7 @@ class WebRTCChat {
// }); // });
// 特别确保添加了5.mp4（从日志看这是常用视频） // 特别确保添加了5.mp4（从日志看这是常用视频）
videosToPreload.add('5.mp4'); videosToPreload.add('d-0.mp4');
// 并行预加载，提高效率 // 并行预加载，提高效率
const preloadPromises = Array.from(videosToPreload).map(async (videoFile) => { const preloadPromises = Array.from(videosToPreload).map(async (videoFile) => {
@ -448,99 +453,105 @@ class WebRTCChat {
await Promise.allSettled(preloadPromises); await Promise.allSettled(preloadPromises);
} }
async switchVideoStream(videoFile, type = '', text = '') { // async switchVideoStream(videoFile, type = '', text = '') {
try { // try {
this.logMessage(`开始切换视频流: ${videoFile} (${type})`, 'info'); // this.logMessage(`开始切换视频流: ${videoFile} (${type})`, 'info');
// 检查是否已缓存 // // 检查是否已缓存
const isCached = this.videoStreams.has(videoFile); // const isCached = this.videoStreams.has(videoFile);
// 如果已缓存，直接使用，避免loading状态 // // 如果已缓存，直接使用，避免loading状态
if (isCached) { // if (isCached) {
const cachedStream = this.videoStreams.get(videoFile); // const cachedStream = this.videoStreams.get(videoFile);
if (cachedStream && cachedStream.getTracks().length > 0) { // if (cachedStream && cachedStream.getTracks().length > 0) {
// 直接切换到缓存的流 // // 直接切换到缓存的流
this.currentVideoStream = cachedStream; // this.currentVideoStream = cachedStream;
this.recordedVideo.srcObject = cachedStream; // this.recordedVideo.srcObject = cachedStream;
this.currentVideo = videoFile; // this.currentVideo = videoFile;
// 立即播放，无需loading状态 // // 立即播放，无需loading状态
await this.recordedVideo.play(); // await this.recordedVideo.play();
this.recordedVideo.classList.add('playing'); // this.recordedVideo.classList.add('playing');
this.logMessage(`使用缓存视频流: ${videoFile}`, 'success'); // this.logMessage(`使用缓存视频流: ${videoFile}`, 'success');
return; // return;
} // }
} // }
// 未缓存的视频才显示loading状态 // // 未缓存的视频才显示loading状态
this.recordedVideo.classList.add('loading'); // this.recordedVideo.classList.add('loading');
// 先创建新的视频流 // // 先创建新的视频流
const newStream = await this.createVideoStream(videoFile); // const newStream = await this.createVideoStream(videoFile);
// 减少等待时间 // // 减少等待时间
await new Promise(resolve => setTimeout(resolve, 100)); // await new Promise(resolve => setTimeout(resolve, 100));
// 检查流是否有效 // // 检查流是否有效
if (!newStream || newStream.getTracks().length === 0) { // if (!newStream || newStream.getTracks().length === 0) {
throw new Error('创建的视频流无效'); // throw new Error('创建的视频流无效');
} // }
// 设置新的视频流 // // 设置新的视频流
this.currentVideoStream = newStream; // this.currentVideoStream = newStream;
this.recordedVideo.srcObject = newStream; // this.recordedVideo.srcObject = newStream;
this.currentVideo = videoFile; // this.currentVideo = videoFile;
// 确保视频开始播放 // // 确保视频开始播放
try { // try {
await this.recordedVideo.play(); // await this.recordedVideo.play();
this.logMessage('视频元素开始播放', 'info'); // this.logMessage('视频元素开始播放', 'info');
// 移除加载状态，添加播放状态 // // 移除加载状态，添加播放状态
this.recordedVideo.classList.remove('loading'); // this.recordedVideo.classList.remove('loading');
this.recordedVideo.classList.add('playing'); // this.recordedVideo.classList.add('playing');
} catch (playError) { // } catch (playError) {
this.logMessage(`视频播放失败: ${playError.message}`, 'error'); // this.logMessage(`视频播放失败: ${playError.message}`, 'error');
this.recordedVideo.classList.remove('loading'); // this.recordedVideo.classList.remove('loading');
} // }
// 现在停止旧的视频流 // // 现在停止旧的视频流
if (this.currentVideoStream !== newStream) { // if (this.currentVideoStream !== newStream) {
const oldStream = this.currentVideoStream; // const oldStream = this.currentVideoStream;
setTimeout(() => { // setTimeout(() => {
if (oldStream) { // if (oldStream) {
oldStream.getTracks().forEach(track => { // oldStream.getTracks().forEach(track => {
track.stop(); // track.stop();
this.logMessage(`已停止旧轨道: ${track.kind}`, 'info'); // this.logMessage(`已停止旧轨道: ${track.kind}`, 'info');
}); // });
} // }
}, 1000); // 延迟1秒停止旧流，确保新流已经稳定 // }, 1000); // 延迟1秒停止旧流，确保新流已经稳定
} // }
if (text) { // if (text) {
this.currentVideoName.textContent = `交互视频: ${videoFile} (${type}: ${text})`; // this.currentVideoName.textContent = `交互视频: ${videoFile} (${type}: ${text})`;
this.logMessage(`成功切换到交互视频流: ${videoFile} (${type}: ${text})`, 'success'); // this.logMessage(`成功切换到交互视频流: ${videoFile} (${type}: ${text})`, 'success');
} else { // } else {
this.currentVideoName.textContent = `视频流: ${videoFile}`; // this.currentVideoName.textContent = `视频流: ${videoFile}`;
this.logMessage(`成功切换到视频流: ${videoFile}`, 'success'); // this.logMessage(`成功切换到视频流: ${videoFile}`, 'success');
} // }
// 检查切换后的状态 // // 检查切换后的状态
setTimeout(() => { // setTimeout(() => {
this.checkVideoStreamStatus(); // this.checkVideoStreamStatus();
}, 1000); // }, 1000);
} catch (error) { // } catch (error) {
this.logMessage(`切换视频流失败: ${error.message}`, 'error'); // this.logMessage(`切换视频流失败: ${error.message}`, 'error');
this.recordedVideo.classList.remove('loading'); // this.recordedVideo.classList.remove('loading');
// 如果切换失败，尝试回到默认视频 // // 如果切换失败，尝试回到默认视频
if (videoFile !== this.defaultVideo) { // if (videoFile !== this.defaultVideo) {
this.logMessage('尝试回到默认视频', 'info'); // this.logMessage('尝试回到默认视频', 'info');
await this.switchVideoStream(this.defaultVideo, 'fallback'); // await this.switchVideoStream(this.defaultVideo, 'fallback');
} // }
} // }
// }
// 修改原有的switchVideoStream方法，使用新的平滑切换
async switchVideoStream(videoFile, type = '', text = '') {
// 使用平滑切换方法
return await this.switchVideoStreamSmooth(videoFile, type, text);
} }
// 使用replaceTrack方式切换视频 // 使用replaceTrack方式切换视频
@ -605,6 +616,153 @@ class WebRTCChat {
} }
} }
// 新增平滑视频切换方法
async switchVideoStreamSmooth(videoFile, type = '', text = '') {
try {
this.logMessage(`开始平滑切换视频流: ${videoFile} (${type})`, 'info');
// 显示加载指示器
this.showVideoLoading();
// 确定当前活跃的视频元素和缓冲元素
const currentVideo = this.activeVideoElement === 'main' ? this.recordedVideo : this.recordedVideoBuffer;
const bufferVideo = this.activeVideoElement === 'main' ? this.recordedVideoBuffer : this.recordedVideo;
// 检查是否已缓存
const isCached = this.videoStreams.has(videoFile);
let newStream;
if (isCached) {
const cachedStream = this.videoStreams.get(videoFile);
if (cachedStream && cachedStream.getTracks().length > 0) {
newStream = cachedStream;
this.logMessage(`使用缓存视频流: ${videoFile}`, 'success');
} else {
newStream = await this.createVideoStream(videoFile);
}
} else {
newStream = await this.createVideoStream(videoFile);
}
// 在缓冲视频元素中预加载新视频
bufferVideo.srcObject = newStream;
// 等待缓冲视频准备就绪
await new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error('视频加载超时'));
}, 5000);
const onCanPlay = () => {
clearTimeout(timeout);
bufferVideo.removeEventListener('canplay', onCanPlay);
bufferVideo.removeEventListener('error', onError);
resolve();
};
const onError = (error) => {
clearTimeout(timeout);
bufferVideo.removeEventListener('canplay', onCanPlay);
bufferVideo.removeEventListener('error', onError);
reject(error);
};
bufferVideo.addEventListener('canplay', onCanPlay);
bufferVideo.addEventListener('error', onError);
// 开始播放缓冲视频
bufferVideo.play().catch(onError);
});
// 隐藏加载指示器
this.hideVideoLoading();
// 执行淡入淡出切换
await this.performVideoTransition(currentVideo, bufferVideo);
// 更新当前视频流和活跃元素
this.currentVideoStream = newStream;
this.currentVideo = videoFile;
this.activeVideoElement = this.activeVideoElement === 'main' ? 'buffer' : 'main';
// 停止旧视频流（延迟停止避免闪烁）
setTimeout(() => {
if (currentVideo.srcObject && currentVideo.srcObject !== newStream) {
currentVideo.srcObject.getTracks().forEach(track => track.stop());
currentVideo.srcObject = null;
}
}, 1000);
// 更新WebRTC连接中的视频轨道
if (this.peerConnection && this.videoSender) {
const newVideoTrack = newStream.getVideoTracks()[0];
if (newVideoTrack) {
await this.videoSender.replaceTrack(newVideoTrack);
}
}
// 更新显示信息
if (text) {
this.currentVideoName.textContent = `交互视频: ${videoFile} (${type}: ${text})`;
this.logMessage(`成功切换到交互视频流: ${videoFile} (${type}: ${text})`, 'success');
} else {
this.currentVideoName.textContent = `视频流: ${videoFile}`;
this.logMessage(`成功切换到视频流: ${videoFile}`, 'success');
}
} catch (error) {
this.logMessage(`平滑切换视频流失败: ${error.message}`, 'error');
this.hideVideoLoading();
// 如果切换失败，尝试回到默认视频
if (videoFile !== this.defaultVideo) {
this.logMessage('尝试回到默认视频', 'info');
await this.switchVideoStreamSmooth(this.defaultVideo, 'fallback');
}
}
}
// 执行视频过渡动画
async performVideoTransition(currentVideo, bufferVideo) {
return new Promise((resolve) => {
// 添加切换类
currentVideo.classList.add('switching');
bufferVideo.classList.add('switching');
// 等待CSS过渡完成
setTimeout(() => {
// 移除切换类
currentVideo.classList.remove('switching');
bufferVideo.classList.remove('switching');
// 交换z-index
if (currentVideo.style.zIndex === '2' || !currentVideo.style.zIndex) {
currentVideo.style.zIndex = '1';
bufferVideo.style.zIndex = '2';
} else {
currentVideo.style.zIndex = '2';
bufferVideo.style.zIndex = '1';
}
resolve();
}, 500); // 与CSS过渡时间一致
});
}
// 显示加载指示器
showVideoLoading() {
if (this.videoLoading) {
this.videoLoading.classList.add('show');
}
}
// 隐藏加载指示器
hideVideoLoading() {
if (this.videoLoading) {
this.videoLoading.classList.remove('show');
}
}
bindEvents() { bindEvents() {
// 开始通话按钮 // 开始通话按钮
this.startButton.onclick = () => this.startCall(); this.startButton.onclick = () => this.startCall();
@ -632,8 +790,8 @@ class WebRTCChat {
}; };
// 语音输入按钮 // 语音输入按钮
this.startVoiceButton.onclick = () => this.startVoiceRecording(); // this.startVoiceButton.onclick = () => this.startVoiceRecording();
this.stopVoiceButton.onclick = () => this.stopVoiceRecording(); // this.stopVoiceButton.onclick = () => this.stopVoiceRecording();
} }
async startCall() { async startCall() {
@ -644,11 +802,15 @@ class WebRTCChat {
}); });
await this.createPeerConnection(); await this.createPeerConnection();
this.startVoiceRecording()
// this.audioProcessor.startRecording()
this.startButton.disabled = true; this.startButton.disabled = true;
this.stopButton.disabled = false; this.stopButton.disabled = false;
this.updateAudioStatus('已连接', 'connected'); // 显示结束通话按钮
this.stopButton.classList.add('show');
this.updateAudioStatus('已连接', 'connected');
this.logMessage('音频通话已开始', 'success'); this.logMessage('音频通话已开始', 'success');
// 确保视频映射已加载 // 确保视频映射已加载
@ -690,8 +852,14 @@ class WebRTCChat {
this.startButton.disabled = false; this.startButton.disabled = false;
this.stopButton.disabled = true; this.stopButton.disabled = true;
// 隐藏结束通话按钮
this.stopButton.classList.remove('show');
this.stopVoiceRecording()
this.updateAudioStatus('未连接', 'disconnected'); this.updateAudioStatus('未连接', 'disconnected');
this.logMessage('音频通话已结束', 'info'); this.logMessage('音频通话已结束', 'info');
} }
async createPeerConnection() { async createPeerConnection() {
@ -864,13 +1032,13 @@ class WebRTCChat {
const success = await this.audioProcessor.startRecording(); const success = await this.audioProcessor.startRecording();
if (success) { if (success) {
this.startVoiceButton.disabled = true; // this.startVoiceButton.disabled = true;
this.stopVoiceButton.disabled = false; // this.stopVoiceButton.disabled = false;
this.startVoiceButton.classList.add('recording'); // this.startVoiceButton.classList.add('recording');
this.voiceStatus.textContent = '等待语音输入...'; // this.voiceStatus.textContent = '等待语音输入...';
this.logMessage('高级语音录制已启动', 'success'); this.logMessage('高级语音录制已启动', 'success');
} else { } else {
this.voiceStatus.textContent = '录音启动失败'; // this.voiceStatus.textContent = '录音启动失败';
} }
} }
@ -878,16 +1046,21 @@ class WebRTCChat {
stopVoiceRecording() { stopVoiceRecording() {
this.audioProcessor.stopRecording(); this.audioProcessor.stopRecording();
this.startVoiceButton.disabled = false; // this.startVoiceButton.disabled = false;
this.stopVoiceButton.disabled = true; // this.stopVoiceButton.disabled = true;
this.startVoiceButton.classList.remove('recording'); // this.startVoiceButton.classList.remove('recording');
this.voiceStatus.textContent = '点击开始语音输入'; // this.voiceStatus.textContent = '点击开始语音输入';
this.logMessage('语音录制已停止', 'info'); this.logMessage('语音录制已停止', 'info');
} }
// 处理语音输入结果 // 处理语音输入结果
async handleVoiceInput(text) { async handleVoiceInput(text) {
if(text == ""){
console.log("识别到用户未说话")
return
}
// 根据文本查找对应视频 // 根据文本查找对应视频
let videoFile = this.videoMapping['default'] || this.defaultVideo; let videoFile = this.videoMapping['default'] || this.defaultVideo;
for (const [key, value] of Object.entries(this.videoMapping)) { for (const [key, value] of Object.entries(this.videoMapping)) {

2
src/llm_stream.js
View File

@ -87,7 +87,7 @@ async function requestLLMStream({ apiKey, model, messages, onSegment }) {
} }
// 如果累积文本长度大于5个字，处理它 // 如果累积文本长度大于5个字，处理它
if (accumulatedText.length > 8 && onSegment) { if (accumulatedText.length > 6 && onSegment) {
console.log('检测到完整段落:', accumulatedText); console.log('检测到完整段落:', accumulatedText);
await onSegment(accumulatedText, false); await onSegment(accumulatedText, false);
hasProcessed = true; hasProcessed = true;

2
src/minimaxi_stream.js
View File

@ -66,7 +66,7 @@ async function processAudioQueue() {
if (!isPlaying && audioQueue.length > 0) { if (!isPlaying && audioQueue.length > 0) {
const audioItem = audioQueue.shift(); const audioItem = audioQueue.shift();
const sayName = 'say-5s-m-sw' const sayName = 'say-5s-m-sw'
const targetVideo = '5.mp4' const targetVideo = 'd-0.mp4'
// 如果是第一个音频片段，触发视频切换 // 如果是第一个音频片段，触发视频切换
if (isFirstChunk && sayName != window.webrtcApp.currentVideoTag && window.webrtcApp && window.webrtcApp.switchVideoWithReplaceTrack) { if (isFirstChunk && sayName != window.webrtcApp.currentVideoTag && window.webrtcApp && window.webrtcApp.switchVideoWithReplaceTrack) {
try { try {

BIN
videos/bd-1.mp4 Normal file
View File

Binary file not shown.

BIN
videos/d-0.mp4 Normal file
View File

Binary file not shown.