纯前端页面实现的智能语音实时听写、大模型答复、语音实时合成功能。
<template>
<div class="Model-container" style="padding: 10px;margin-bottom:50px; ">
<!--聊天窗口开始 -->
<el-row>
<el-col :span="24">
<div
style="width: 1200px;margin: 0 auto;background-color: white;border-radius: 5px;box-shadow: 0 0 10px #cccccc">
<div style="text-align: center;line-height: 50px;">
大模型智能问答
</div>
<!--展示会话窗口-->
<div ref="scrollContainer" style="height: 530px;overflow: auto;border-top:1px solid #ccc"
v-html="content"></div>
<div style="height: 150px;">
<textarea v-model="text" @keydown.enter.prevent="sendAndAsk"
style="height: 160px;width: 100%;padding: 20px; border: none;border-top: 1px solid #ccc;border-bottom: 1px solid #ccc;outline: none">
</textarea>
<div style="text-align: left;padding-right: 10px;">
<el-button type="primary" size="medium" @click="sendAndAsk">发送咨询
</el-button>
<el-button type="success" size="medium" @click="voiceSend"><i class="el-icon-microphone"></i>语音输入
</el-button>
<el-button type="danger" size="medium" @click="stopVoice">停止朗读
</el-button>
<el-button type="danger" size="medium" @click="clearHistory">清空历史
</el-button>
</div>
</div>
</div>
</el-col>
</el-row>
<!--聊天窗口结束 -->
</div>
</template>
<script>
// 初始化录音工具,注意目录
let recorder = new Recorder("../../recorder")
recorder.onStart = () => {
console.log("开始录音了")
}
recorder.onStop = () => {
console.log("结束录音了")
}
// 发送中间帧和最后一帧
recorder.onFrameRecorded = ({isLastFrame, frameBuffer}) => {
if (!isLastFrame && wsFlag) { // 发送中间帧
const params = {
data: {
status: 1,
format: "audio/L16;rate=16000",
encoding: "raw",
audio: toBase64(frameBuffer),
},
};
wsTask.send(JSON.stringify(params)) // 执行发送
} else {
if (wsFlag) {
const params = {
data: {
status: 2,
format: "audio/L16;rate=16000",
encoding: "raw",
audio: "",
},
};
console.log("发送最后一帧", params, wsFlag)
wsTask.send(JSON.stringify(params)) // 执行发送
}
}
}
let wsFlag = false;
let wsTask = {};
let wsFlagModel = false;
let wsTaskModel = {};
const audioPlayer = new AudioPlayer("../../player"); // 播放器
export default {
name: "Model",
data() {
return {
dialogWidth: window.screen.width >= 1920 ? window.screen.width * 0.77 + "px" : window.screen.width * 0.9 + "px",
wangHeight: window.screen.width >= 1920 ? window.screen.height * 0.541 + "px" : window.screen.height * 0.6 + "px",
user: localStorage.getItem("user") ? JSON.parse(localStorage.getItem("user")) : {}, // 获取本地存储用户
text: "",
sendTextForMySql: "",
scrollFlag: false,
URL: 'wss://iat-api.xfyun.cn/v2/iat',
URL_MODEL: 'wss://spark-api.xf-yun.com/v4.0/chat', // 大模型地址
resultText: "",
resultTextTemp: "",
textList: [],// 大模型历史会话记录
messageList: [],
modelRes: "",
content: '', // 现在用到的变量都需要提前定义
needInsertFlag: false,
ttsText: ""
}
},
created() {
// 请求分页数据
this.initUserQuestion()
},
methods: {
stopVoice() {
window.location.reload()
},
doWsWork() {
let bgs = this.bgMusic ? 1 : 0;
const url = this.getWebSocketUrlTts(atob(this.user.apikey), atob(this.user.apisecret));
if ("WebSocket" in window) {
this.ttsWS = new WebSocket(url);
} else if ("MozWebSocket" in window) {
this.ttsWS = new MozWebSocket(url);
} else {
alert("浏览器不支持WebSocket");
return;
}
this.ttsWS.onopen = (e) => {
console.log("链接成功...")
audioPlayer.start({
autoPlay: true,
sampleRate: 16000,
resumePlayDuration: 1000
});
let text = this.ttsText;
let tte = document.getElementById("tte") ? "unicode" : "UTF8";
let params = {
common: {
app_id: atob(this.user.appid),
},
business: {
aue: "raw",
auf: "audio/L16;rate=16000",
vcn: "x4_panting",
bgs: bgs,
tte,
},
data: {
status: 2,
text: this.encodeText(text, tte),
},
};
this.ttsWS.send(JSON.stringify(params));
console.log("发送成功...")
};
this.ttsWS.onmessage = (e) => {
let jsonData = JSON.parse(e.data);
// console.log("合成返回的数据" + JSON.stringify(jsonData));
// 合成失败
if (jsonData.code !== 0) {
console.error(jsonData);
return;
}
audioPlayer.postMessage({
type: "base64",
data: jsonData.data.audio,
isLastData: jsonData.data.status === 2,
});
if (jsonData.code === 0 && jsonData.data.status === 2) {
this.ttsWS.close();
}
};
this.ttsWS.onerror = (e) => {
console.error(e);
};
this.ttsWS.onclose = (e) => {
console.log(e + "链接已关闭");
};
}
,
// 文本编码
encodeText(text, type) {
if (type === "unicode") {
let buf = new ArrayBuffer(text.length * 4);
let bufView = new Uint16Array(buf);
for (let i = 0, strlen = text.length; i < strlen; i++) {
bufView[i] = text.charCodeAt(i);
}
let binary = "";
let bytes = new Uint8Array(buf);
let len = bytes.byteLength;
for (let i = 0; i < len; i++) {
binary += String.fromCharCode(bytes[i]);
}
return window.btoa(binary);
} else {
return base64.encode(text);
}
}
,
// 鉴权方法
getWebSocketUrlTts(apiKey, apiSecret) {
let url = "wss://tts-api.xfyun.cn/v2/tts";
let host = location.host;
let date = new Date().toGMTString();
let algorithm = "hmac-sha256";
let headers = "host date request-line";
let signatureOrigin = `host: ${host}\ndate: ${date}\nGET /v2/tts HTTP/1.1`;
let signatureSha = CryptoJS.HmacSHA256(signatureOrigin, apiSecret);
let signature = CryptoJS.enc.Base64.stringify(signatureSha);
let authorizationOrigin = `api_key="${apiKey}", algorithm="${algorithm}", headers="${headers}", signature="${signature}"`;
let authorization = btoa(authorizationOrigin);
url = `${url}?authorization=${authorization}&date=${date}&host=${host}`;
return url;
}
,
clearHistory() {
this.$http.post("/model/list_delete_by_send_user", {sendUser: this.user.name}).then(res => {
if (res.data.code === "200") {
this.$message.success('清空历史成功')
window.location.reload()
} else {
this.$message.error('清空历史对话失败，' + res.data.message)
}
})
}
,
initUserQuestion() { // 从数据库查询数据，展示用户问答记录
this.needInsertFlag = false;
this.$http.post("/model/list_page", {sendUser: this.user.name}).then(res => {
console.log(res.data)
if (res.data.code === "200") {
// this.$message.success('查询历史对话成功')
this.messageList = res.data.object.data;
// alert("执行")
this.messageList.forEach(item => {
this.createContent(null, item.sendUser, item.sendContent)
let temp = {
"role": "user",
"content": item.sendContent
}
this.textList.push(temp);
this.createContent(null, "大模型", item.modelAnswer)
temp = {
"role": "assistant",
"content": item.modelAnswer
}
this.textList.push(temp);
})
// alert("执行结束")
console.log(JSON.stringify(this.textList))
} else {
this.$message.error('查询历史对话失败，' + res.data.message)
}
})
}
,
voiceSend() { // 开始语音识别要做的动作
// 首先要调用扣费API
this.user.ability = "语音听写能力" // 标记能力
this.$http.post("/big/consume_balance", this.user).then(res => {
if (res.data.code === "200") {
// 触发父级更新user方法
this.$emit("person_fff_user", res.data.object)
this.resultText = "";
this.resultTextTemp = "";
this.wsInit();
} else {
this.$message.error(res.data.message)
return false // 这个必须要做
}
})
// 调用扣费API结束
}
,
stopRecorder() { // 听写可以用到的方法
recorder.stop();
_this.$message.success("实时听写停止！")
}
,
// 建立ws连接
async wsInitModel() {
let _this = this;
if (typeof (WebSocket) == 'undefined') {
console.log('您的浏览器不支持ws...')
} else {
console.log('您的浏览器支持ws！！！')
let reqeustUrl = await _this.getWebSocketUrlModel()
wsTaskModel = new WebSocket(reqeustUrl);
// ws的几个事件，在vue中定义
wsTaskModel.onopen = function () {
_this.modelRes = " " // 每次清空上次结果
console.log('ws已经打开...')
let tempUserInfo = {
role: "user",
content: _this.text
}
_this.textList.push(tempUserInfo) // 添加最新问题，历史问题从数据库查询
wsFlagModel = true
let params = {
"header": {
"app_id": atob(_this.user.appid),
"uid": "fd3f47e4-d"
}, "parameter": {
"chat": {
"domain": "4.0Ultra",
"temperature": 0.01,
"max_tokens": 8192
}
}, "payload": {
"message": {
"text": _this.textList
/* "text": [{
"role": "user", "content": "中国第一个皇帝是谁？"
}, {
"role": "assistant", "content": "秦始皇"
}, {
"role": "user", "content": "秦始皇修的长城吗"
}, {
"role": "assistant", "content": "是的"
}, {
"role": "user", "content": _this.text
}]*/
}
}
};
console.log("发送第一帧数据...")
wsTaskModel.send(JSON.stringify(params)) // 执行发送
_this.sendTextForMySql = _this.text // 记录一份用于存储
_this.text = ""; // 清空文本
}
wsTaskModel.onmessage = function (message) { // 调用第二个API 自动把语音转成文本
// console.log('收到数据===' + message.data)
let jsonData = JSON.parse(message.data);
// console.log(jsonData)
let tempList = jsonData.payload.choices.text;
for (let i = 0; i < tempList.length; i++) {
_this.modelRes = _this.modelRes + tempList[i].content;
// console.log(tempList[i].content)
}
// 检测到结束或异常关闭
if (jsonData.header.code === 0 && jsonData.header.status === 2) { // 拿到最终的听写文本后，我们会调用大模型
wsTaskModel.close();
wsFlagModel = false
_this.createContent(null, "大模型", _this.modelRes);
_this.ttsText = _this.modelRes
_this.doWsWork()
}
if (jsonData.header.code !== 0) {
wsTaskModel.close();
wsFlagModel = false
console.error(jsonData);
}
}
wsTaskModel.onclose = function () {
console.log('ws已关闭...')
}
wsTaskModel.onerror = function () {
console.log('发生错误...')
}
}
}
,
// 获取鉴权地址与参数
getWebSocketUrlModel() {
return new Promise((resolve, reject) => {
// 请求地址根据语种不同变化
var url = this.URL_MODEL;
var host = "spark-api.xf-yun.com";
var apiKeyName = "api_key";
var date = new Date().toGMTString();
var algorithm = "hmac-sha256";
var headers = "host date request-line";
var signatureOrigin = `host: ${host}\ndate: ${date}\nGET /v4.0/chat HTTP/1.1`;
var signatureSha = CryptoJS.HmacSHA256(signatureOrigin, atob(this.user.apisecret));
var signature = CryptoJS.enc.Base64.stringify(signatureSha);
var authorizationOrigin =
`${apiKeyName}="${atob(this.user.apikey)}", algorithm="${algorithm}", headers="${headers}", signature="${signature}"`;
var authorization = base64.encode(authorizationOrigin);
url = `${url}?authorization=${authorization}&date=${encodeURI(date)}&host=${host}`;
console.log(url)
resolve(url); // 主要是返回地址
});
}
,
// 建立ws连接
async wsInit() {
// this.iat = "";
this.$message.success("请您说出提问内容~")
let _this = this;
if (typeof (WebSocket) == 'undefined') {
console.log('您的浏览器不支持ws...')
} else {
console.log('您的浏览器支持ws！！！')
let reqeustUrl = await _this.getWebSocketUrl()
wsTask = new WebSocket(reqeustUrl);
// ws的几个事件，在vue中定义
wsTask.onopen = function () {
console.log('ws已经打开...')
wsFlag = true
let params = { // 第一帧数据
common: {
app_id: atob(_this.user.appid),
},
business: {
language: "zh_cn",
domain: "iat",
accent: "mandarin",
vad_eos: 2000,
dwa: "wpgs",
},
data: {
status: 0,
format: "audio/L16;rate=16000",
encoding: "raw",
},
};
console.log("发送第一帧数据...")
wsTask.send(JSON.stringify(params)) // 执行发送
// 下面就可以循环发送中间帧了
// 开始录音
console.log("开始录音")
recorder.start({
sampleRate: 16000,
frameSize: 1280,
});
}
wsTask.onmessage = function (message) { // 调用第二个API 自动把语音转成文本
// console.log('收到数据===' + message.data)
let jsonData = JSON.parse(message.data);
if (jsonData.data && jsonData.data.result) {
let data = jsonData.data.result;
let str = "";
let ws = data.ws;
for (let i = 0; i < ws.length; i++) {
str = str + ws[i].cw[0].w;
}
if (data.pgs) {
if (data.pgs === "apd") {
// 将resultTextTemp同步给resultText
_this.resultText = _this.resultTextTemp;
}
// 将结果存储在resultTextTemp中
_this.resultTextTemp = _this.resultText + str;
} else {
_this.resultText = _this.resultText + str;
}
_this.text = _this.resultTextTemp || _this.resultText || "";
}
// 检测到结束或异常关闭
if (jsonData.code === 0 && jsonData.data.status === 2) { // 拿到最终的听写文本后，我们会调用大模型
// alert("执行了")
recorder.stop();
_this.$message.success("检测到您2秒没说话，自动结束识别！")
wsTask.close();
wsFlag = false
}
if (jsonData.code !== 0) {
wsTask.close();
wsFlag = false
console.error(jsonData);
}
}
// 关闭事件
wsTask.onclose = function () {
console.log('ws已关闭...')
}
wsTask.onerror = function () {
console.log('发生错误...')
}
}
}
,
// 获取鉴权地址与参数
getWebSocketUrl() {
return new Promise((resolve, reject) => {
// 请求地址根据语种不同变化
var url = this.URL;
var host = "iat-api.xfyun.cn";
var apiKeyName = "api_key";
var date = new Date().toGMTString();
var algorithm = "hmac-sha256";
var headers = "host date request-line";
var signatureOrigin = `host: ${host}\ndate: ${date}\nGET /v2/iat HTTP/1.1`;
var signatureSha = CryptoJS.HmacSHA256(signatureOrigin, atob(this.user.apisecret));
var signature = CryptoJS.enc.Base64.stringify(signatureSha);
var authorizationOrigin =
`${apiKeyName}="${atob(this.user.apikey)}", algorithm="${algorithm}", headers="${headers}", signature="${signature}"`;
var authorization = base64.encode(authorizationOrigin);
url = `${url}?authorization=${authorization}&date=${encodeURI(date)}&host=${host}`;
console.log(url)
resolve(url); // 主要是返回地址
});
}
,
sendAndAsk() { // 用户发送消息
// 首先要调用扣费API
if (this.text == "") {
this.$message.error("发送消息不能为空")
return false
}
this.needInsertFlag = true
this.user.ability = "大模型问答" // 标记能力
this.$http.post("/big/consume_balance", this.user).then(res => {
if (res.data.code === "200") {
// 触发父级更新user方法
this.$emit("person_fff_user", res.data.object)
if (!wsFlag) {
// console.log("我打印的" + this.user.name)
this.createContent(null, this.user.name, this.text);
// 调用大模型
this.wsInitModel();
} else {
this.$message.warning("听写工作中，请稍后再发送...")
}
} else {
this.$message.error(res.data.message)
return false // 这个必须要做
}
})
// 调用扣费API结束
}
,
createContent(remoteUser, nowUser, text) { // 这个方法是用来将 json的聊天消息数据转换成 html的。
if (text == "") {
this.$message.error("发送消息不能为空")
return false
}
let html
// alert("执行了")
// 当前用户消息
if (nowUser == this.user.name) { // nowUser 表示是否显示当前用户发送的聊天消息，绿色气泡
html = "<div class=\"el-row\" style=\"padding: 5px 0;\">\n" +
" <div class=\"el-col el-col-22\" style=\"text-align: right; padding-right: 10px\">\n" +
" <div class=\"tip left myLeft\">" + text + "</div>\n" +
" </div>\n" +
" <div class=\"el-col el-col-2\">\n" +
" <span class=\"el-avatar el-avatar--circle\" style=\"height: 40px; width: 40px; line-height: 40px;\">\n" +
" <img src=\"" + this.user.avatar + "\" style=\"object-fit: cover;\">\n" +
" </span>\n" +
" </div>\n" +
"</div>";
} else { // 其他表示大模型的答复，蓝色的气泡
html = "<div class=\"el-row\" style=\"padding: 5px 0;width: 760px;\">\n" +
" <div class=\"el-col el-col-2\" style=\"text-align: right\">\n" +
" <span class=\"el-avatar el-avatar--circle\" style=\"height: 40px; width: 40px; line-height: 40px;\">\n" +
" <img src=\"" + `https://wdfgdzx.top:3333/document/cd39af3e175b4524890c267e07298f5b.png` + "\" style=\"object-fit: cover;\">\n" +
" </span>\n" +
" </div>\n" +
" <div class=\"el-col el-col-22\" style=\"text-align: left; padding-left: 10px\">\n" +
" <div class=\"tip right myLeft\">" + text + "</div>\n" +
" </div>\n" +
"</div>";
// 大模型答复完毕应该插入数据库记录
let modelEntity = {
sendUser: this.user.name,
sendContent: this.sendTextForMySql,
modelAnswer: text,
type: "大模型文本问答" // 固定值
}
if (this.needInsertFlag) {
this.$http.post("/model/insertOrUpdate", modelEntity).then(res => {
if (res.data.code == "200") {
// 执行成功
} else {
this.$message.error(res.data.message)
}
})
}
}
console.log(html)
this.content += html;
// 滚动到底部
setTimeout(this.scrollToDown, 300) // 延迟滚动才有效果
}
,
scrollToDown() {
this.scrollFlag = true;
if (this.scrollFlag) { // 滚动到底部
let container = this.$refs.scrollContainer;
container.scrollTop = 5000;
}
}
}
}
</script>
<!--scoped 不能加-->
<style>
.tip {
color: white;
border-radius: 10px;
font-family: sans-serif;
padding: 10px;
width: auto;
display: inline-block !important;
display: inline;
}
.right {
background-color: deepskyblue;
}
.myLeft {
text-align: left;
}
.myRight {
text-align: right;
}
.myCenter {
text-align: center;
}
.left {
background-color: forestgreen;
}
</style>
package com.black.controller;
import cn.hutool.poi.excel.ExcelReader;
import cn.hutool.poi.excel.ExcelUtil;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
import com.black.mapper.ModelMapper;
import com.black.mapper.UserMapper;
import com.black.pojo.Model;
import com.black.pojo.User;
import com.black.util.Constants;
import com.black.util.MyUtils;
import com.black.util.Res;
import org.springframework.web.bind.annotation.*;
import org.springframework.web.multipart.MultipartFile;
import javax.annotation.Resource;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
@RestController
@RequestMapping("model")
public class ModelController {
@Resource
ModelMapper modelMapper;
@PostMapping("/insertOrUpdate")
public Res insertOrUpdate(@RequestBody Model model) throws Exception { // @RequestBody很重要
if (model.getId() != null) { // 存在则更新
modelMapper.updateById(model);
} else {
try {
model.setSendTime(new Date());
modelMapper.insert(model);
} catch (Exception e) {
e.printStackTrace();
return Res.error(Constants.CODE_500, "系统错误");
}
}
return Res.success(null);
}
@PostMapping("/delete")
public Res delete(@RequestBody Model model) {
modelMapper.deleteById(model);
return Res.success(null);
}
@PostMapping("/select")
public Res select(@RequestBody Model model) {
Model existModel = modelMapper.selectById(model.getId());
return Res.success(existModel);// 需要返回对象
}
@PostMapping("/list_page")
public Res list_page(@RequestBody Model model) {
// 1、查询条件
QueryWrapper<Model> queryWrapper = new QueryWrapper<>();
if (!MyUtils.blankFlag(model.getSendUser())) { // 如果非空，执行模糊查询
queryWrapper.eq("send_user", model.getSendUser());
}
List<Model> dataList = modelMapper.selectList(queryWrapper); // 进行分页数据查询
// 3、构建分页查询，返回给前端
HashMap<Object, Object> hashMap = new HashMap<>();
hashMap.put("data", dataList);
return Res.success(hashMap);
}
@PostMapping("/list_delete_by_send_user")
public Res list_delete_by_send_user(@RequestBody Model model) {
QueryWrapper<Model> queryWrapper = new QueryWrapper<>();
if (!MyUtils.blankFlag(model.getSendUser())) { // 如果非空，执行模糊查询
queryWrapper.eq("send_user", model.getSendUser());
}
modelMapper.delete(queryWrapper);
return Res.success(null);
}
@PostMapping("/list_delete")
public Res list_delete(@RequestBody Model model) {
modelMapper.deleteBatchIds(model.getRemoveIdList());
return Res.success(null);
}
@PostMapping("/list_model") // 0、查询所有
public Res list_model() {
return Res.success(modelMapper.selectList(null));
}
@RequestMapping("/list_import") // 1、一般用不到导入方法
public Res list_import(@RequestParam("multipartFile") MultipartFile multipartFile, @RequestParam("token") String token) throws Exception {
ExcelReader excelReader12 = ExcelUtil.getReader(multipartFile.getInputStream(), 0);
List<List<Object>> rowList12 = excelReader12.read();
int nameIndex = 0;
for (List<Object> row : rowList12) {
if (nameIndex >= 1 && row.size() == 18 && row.get(3) != null && row.get(3) != "") {
QueryWrapper<Model> modelQueryWrapper = new QueryWrapper<>();
modelQueryWrapper.eq("name", row.get(3).toString());
Model model = modelMapper.selectOne(modelQueryWrapper);
if (model == null) { // 不存在则插入
} else { // 存在则更新
}
}
nameIndex++;
}
return null;
}
}