微信小程序实现AI语音对话的功能
用户语音检测与沉默判断录音结束，发送消息处理PCM16格式音频文件
记录一下实现AI语音对话功能的过程。
用户语音检测与沉默判断
通过录音管理器的RecorderManager.onFrameRecorded监测已录制完指定帧大小的文件事件，然后对ArrayBuffer文件进行处理，判断用户是否开始说话，说话是否结束等。
onLoad: function (options) {
this.data.recordingManager = wx.getRecorderManager()
const audioCtx = wx.createWebAudioContext()
const analyser = audioCtx.createAnalyser();
this.data.recordingManager.onFrameRecorded(listener => {
if (listener.isLastFrame) {
this.handleSoundIntensityChange(0);
} else {
audioCtx.decodeAudioData(listener.frameBuffer, buffer => {
let source = audioCtx.createBufferSource()
source.buffer = buffer
source.connect(analyser)
source.start()
let n = new Uint8Array(analyser.frequencyBinCount)
analyser.getByteTimeDomainData(n)
let i = 0,
r = 0,
s = 0
r = Math.max.apply(null, n)
s = Math.min.apply(null, n)
i = (r - s) / 128
i = Math.round(i * 100 / 2)
i = i > 100 ? 100 : i
this.handleSoundIntensityChange(i)
}, err => {
console.error('decodeAudioData fail', err)
})
}
})
},
// 处理声音强度变化
handleSoundIntensityChange(i) {
if ((i >= 0 && i <= 10) && this.data.isAllowStart) {
if (!this.data.timer) {
this.data.timer = setTimeout(() => {
if (!this.data.isSpeaking) {
this.data.isSpeaking = true;
this.data.recordingManager.stop();
this.setData({
theDesc: '回答中...'
})
clearTimeout(this.data.timer);
this.data.timer = null;
console.log('说话完毕');
}
}, 2000); // 2秒内没有值变化，认为“说话完毕”
}
} else if (i > 10) {
if (this.data.theNumber <= 2) {
this.data.theNumber++
} else {
this.data.isAllowStart = true
if (this.data.timer) {
clearTimeout(this.data.timer);
this.data.timer = null;
}
}
}
},
录音结束，发送消息
说话结束，发送包含录音文件的消息。然后对websocket的推送数据进行处理，播放音频文件
this.data.recordingManager.onStop((res) => {
// 录音停止后的回调函数
if (this.data.isSpeaking) {
this.sendAudio(res.tempFilePath)
}
});
sendAudio(url) {
let that = this
wx.uploadFile({
url: 'xxx你的地址',
filePath: url,
name: 'file',
success: (uploadRes) => {
const data = JSON.parse(uploadRes.data);
if (data.data && data.data.url) {
that.createChat('', () => {
const ws = that.data.ws;
if (ws && ws.send) {
ws.send({
data: JSON.stringify({
fileUrl: data.data.url,
token: app.globalData.sessionKey,
chatID: that.data.theChatId
}),
});
}
}
}
},
fail: (err) => {
console.error('上传文件失败', err);
}
});
},
处理PCM16格式音频文件
//websocket消息处理
socket.onMessage((event) => {
const message = event.data;
if (message != '{"type":"ping"}') {
that.data.isGoing = true
if (message.includes('[DONE]')) {
that.data.isOver = true
//音频文件格式处理
const wavData = that.convertToWav(wx.base64ToArrayBuffer(that.data.baseUrl), 22000, 1);
const fs = wx.getFileSystemManager();
const filePath = `${wx.env.USER_DATA_PATH}/audio${Date.now()}.wav`;
const uint8ArrayData = new Uint8Array(wavData);
fs.writeFile({
filePath: filePath,
data: uint8ArrayData.buffer,
encoding: 'binary',
success: function () {
if (!that.data.isPlayIng) {
innerAudioContext.src = filePath;
innerAudioContext.play();
} else {
that.data.theAudioList.push(filePath)
}
},
fail: function (err) {
console.error('保存 WAV 文件失败', err);
}
});
return;
}
if (!(message.includes('completion_tokens'))) {
//执行你的音频播放逻辑
}
} else {
if (that.data.isGoing && !that.data.stopTimer) {
that.data.stopTimer = setTimeout(() => {
that.data.isGoing = false
that.data.isOver = true
that.data.stopTimer = null
}, 2000);
}
}
});