530 lines
17 KiB
C
530 lines
17 KiB
C
/* =========================================================================
|
||
|
||
DESCRIPTION
|
||
Netease && Xunfei speech processing main program
|
||
Cae voice process
|
||
|
||
Copyright (c) 2017 by Netease, Co,LTD. All Rights Reserved.
|
||
============================================================================ */
|
||
|
||
/* =========================================================================
|
||
|
||
REVISION
|
||
|
||
when who why
|
||
-------- ---------
|
||
-------------------------------------------
|
||
2017/06/28 wangzijiao Created.
|
||
============================================================================ */
|
||
|
||
/* ------------------------------------------------------------------------
|
||
** Includes
|
||
** ------------------------------------------------------------------------ */
|
||
//#define NETEASE_DUILITE_FESPL_SDK 1
|
||
#if NETEASE_DUILITE_FESPL_SDK
|
||
|
||
#include "base.h"
|
||
#include "cae_lib.h"
|
||
#include "error.h"
|
||
#include <assert.h>
|
||
#include <errno.h>
|
||
#include <fcntl.h>
|
||
#include <jansson.h>
|
||
#include <stdio.h>
|
||
#include <stdlib.h>
|
||
#include <string.h>
|
||
|
||
#include "msc.h"
|
||
#include "nduilite.h"
|
||
#include <duilite.h>
|
||
|
||
/* ------------------------------------------------------------------------
|
||
** Macros
|
||
** ------------------------------------------------------------------------ */
|
||
|
||
/* ------------------------------------------------------------------------
|
||
** Defines
|
||
** ------------------------------------------------------------------------ */
|
||
#define DUILITE_DEBUG (0)
|
||
/* ------------------------------------------------------------------------
|
||
** Types
|
||
** ------------------------------------------------------------------------ */
|
||
|
||
/* ------------------------------------------------------------------------
|
||
** Global Variable Definitions
|
||
** ------------------------------------------------------------------------ */
|
||
static struct duilite_fespl *fespa = NULL;
|
||
static NATIVE_MUTEX_HANDLE _duilitelock = NULL;
|
||
static volatile char vadstatus = 0; // 0 stop, 1 start
|
||
static char duiliteok = 0; // 0 not ok, 1 ok
|
||
|
||
#if DUILITE_DEBUG
|
||
static int fd_audio_orig;
|
||
#endif
|
||
|
||
#if USED_NETEASE_DUILITE_VAD
|
||
static struct duilite_vad *duilitevad = NULL;
|
||
static duilite_callback oem_vad_cb = NULL;
|
||
#endif
|
||
|
||
static duilite_callback oem_audio_cb = NULL;
|
||
|
||
/* ------------------------------------------------------------------------
|
||
** Function Definitions
|
||
** ------------------------------------------------------------------------
|
||
*/
|
||
|
||
int Netease_duilite_fespa_set(char *arg) {
|
||
int ret;
|
||
if (NULL == arg) {
|
||
n_error("Arg is null!");
|
||
return -2;
|
||
}
|
||
|
||
if (NULL == fespa) {
|
||
n_error("fespa is null, do not set%s\n", arg);
|
||
return -1;
|
||
}
|
||
|
||
if (1 != duiliteok) {
|
||
n_error("duilite not ok!\n");
|
||
return -3;
|
||
}
|
||
|
||
ret = duilite_fespl_set(fespa, arg);
|
||
n_debug("Set arg to duilite(ret=%d):%s\n", ret, arg);
|
||
}
|
||
|
||
static void duiliteLock(char *who) {
|
||
if (NULL != _duilitelock) {
|
||
// n_debug("Lock! %s\n", who);
|
||
native_mutex_take(_duilitelock, 0);
|
||
}
|
||
}
|
||
|
||
static void duiliteUnlock(char *who) {
|
||
if (NULL != _duilitelock) {
|
||
// n_debug("UnLock!%s\n", who);
|
||
native_mutex_given(_duilitelock);
|
||
}
|
||
}
|
||
|
||
static void vad_audiocb(void *userdata, int type, char *msg, int len) {
|
||
if (NULL != oem_audio_cb) {
|
||
oem_audio_cb(userdata, type, msg, len);
|
||
}
|
||
|
||
#if USED_NETEASE_DUILITE_VAD
|
||
if (type == DUILITE_MSG_TYPE_BINARY && duilitevad != NULL) {
|
||
duiliteLock("write audio");
|
||
if (vadstatus == 1) {
|
||
duilite_vad_feed(duilitevad, msg, len);
|
||
}
|
||
duiliteUnlock("write audio");
|
||
}
|
||
// n_debug("Get beforming data\n");
|
||
#endif
|
||
}
|
||
|
||
#if USED_NETEASE_DUILITE_VAD
|
||
static void vad_status_cb(void *userdata, int type, char *msg, int len) {
|
||
if (oem_vad_cb != NULL) {
|
||
oem_vad_cb(userdata, type, msg, len);
|
||
}
|
||
|
||
if (type == DUILITE_MSG_TYPE_JSON) {
|
||
json_t *js = json_loadb(msg, len, 0, NULL);
|
||
if (js != NULL) {
|
||
json_t *status = json_object_get(js, "status");
|
||
if (status != NULL && json_is_integer(status)) {
|
||
switch (json_integer_value(status)) {
|
||
case 1: // start
|
||
setVadStatus(VadStatus_Begin);
|
||
break;
|
||
case 2: // end
|
||
setVadStatus(VadStatus_Finish);
|
||
Netease_duilite_vad_stop(0);
|
||
break;
|
||
default:
|
||
n_error("Unknow status:%s\n", msg);
|
||
}
|
||
}
|
||
json_decref(js);
|
||
}
|
||
}
|
||
}
|
||
#endif
|
||
|
||
int Netease_duilite_vad_start() {
|
||
#if USED_NETEASE_DUILITE_VAD
|
||
int ret = -1;
|
||
|
||
if (NULL != duilitevad && 1 == duiliteok) {
|
||
duiliteLock("vad_start");
|
||
if (vadstatus == 1) {
|
||
n_debug("Begin to stop, in start!\n");
|
||
duilite_vad_stop(duilitevad);
|
||
}
|
||
ret = duilite_vad_start(duilitevad, NULL);
|
||
n_debug("!!!!VAD start!!%d\n", ret);
|
||
vadstatus = 1;
|
||
|
||
duiliteUnlock("vad_start");
|
||
return ret;
|
||
} else {
|
||
n_error("Duilite check err, duilitevad:0x%p, duiliteok:%d\n",
|
||
duilitevad, duiliteok);
|
||
return -1;
|
||
}
|
||
#else
|
||
return 0;
|
||
#endif
|
||
}
|
||
|
||
int Netease_duilite_vad_stop(char IsLock) {
|
||
#if USED_NETEASE_DUILITE_VAD
|
||
int ret = 0;
|
||
n_debug("Begin to stop vad!\n");
|
||
if (NULL != duilitevad && 1 == duiliteok) {
|
||
if (1 == IsLock) {
|
||
duiliteLock(" vad_stop ");
|
||
}
|
||
if (vadstatus == 1) {
|
||
ret = duilite_vad_stop(duilitevad);
|
||
vadstatus = 0;
|
||
}
|
||
|
||
if (1 == IsLock) {
|
||
duiliteUnlock(" vad_stop ");
|
||
}
|
||
return ret;
|
||
} else {
|
||
n_error("Duilite check err, duilitevad:0x%p, duiliteok:%d\n",
|
||
duilitevad, duiliteok);
|
||
return -1;
|
||
}
|
||
#else
|
||
return 0;
|
||
#endif
|
||
}
|
||
|
||
int Neteaase_duilite_vad_cancel() {
|
||
#if USED_NETEASE_DUILITE_VAD
|
||
int ret = 0;
|
||
if (NULL != duilitevad && 1 == duiliteok) {
|
||
duiliteLock(" vad_cancel ");
|
||
if (1 == vadstatus) {
|
||
ret = duilite_vad_cancel(duilitevad);
|
||
vadstatus = 0;
|
||
}
|
||
duiliteUnlock(" vad_cancel ");
|
||
return ret;
|
||
} else {
|
||
n_error("Duilite check err, duilitevad:0x%p, duiliteok:%d\n",
|
||
duilitevad, duiliteok);
|
||
|
||
return -1;
|
||
}
|
||
#else
|
||
return 0;
|
||
#endif
|
||
}
|
||
|
||
// voice binary data convert by HongChuanRong
|
||
void CheckData(const void *dataIn, int lenIn, void *dataOut, int *lenOut) {
|
||
#define DATA_CHAN_NUM (12)
|
||
static short ckDataBuf[DATA_CHAN_NUM];
|
||
static int ckDataPos = 0;
|
||
static int chanOneFoundFlag = 0;
|
||
if (lenIn % sizeof(int) != 0) {
|
||
fprintf(stderr, "fatal error!\n");
|
||
return;
|
||
}
|
||
const int totalCountsIn = lenIn / sizeof(int);
|
||
const int *pDataIn = (const int *)dataIn;
|
||
short *pDataOut = (short *)dataOut;
|
||
int countsOut = 0;
|
||
|
||
if (!chanOneFoundFlag) {
|
||
// we have to find chan one first (Peng)
|
||
for (auto k = 0; k < totalCountsIn; ++k) {
|
||
const int chanId = (pDataIn[k] >> 8) & 0x0f;
|
||
if (chanId == 1) {
|
||
// chan one found
|
||
chanOneFoundFlag = 1;
|
||
CheckData(pDataIn + k, (totalCountsIn - k) * sizeof(int),
|
||
dataOut, lenOut);
|
||
return;
|
||
}
|
||
}
|
||
return;
|
||
}
|
||
for (auto k = 0; k < totalCountsIn; ++k) {
|
||
const int chanId = (pDataIn[k] >> 8) & 0x0f;
|
||
|
||
if (chanId == (ckDataPos + 1)) {
|
||
ckDataBuf[ckDataPos++] = (short)(pDataIn[k] >> 16);
|
||
} else {
|
||
fprintf(stderr, "expecting chan: %d, input chan: %d\n",
|
||
ckDataPos + 1, chanId);
|
||
ckDataPos = 0;
|
||
}
|
||
|
||
if (ckDataPos == DATA_CHAN_NUM) {
|
||
short finalOut[8];
|
||
finalOut[0] = ckDataBuf[8 - 1];
|
||
finalOut[1] = ckDataBuf[2 - 1];
|
||
finalOut[2] = ckDataBuf[7 - 1];
|
||
finalOut[3] = ckDataBuf[1 - 1];
|
||
finalOut[4] = ckDataBuf[9 - 1];
|
||
finalOut[5] = ckDataBuf[3 - 1];
|
||
finalOut[6] = ckDataBuf[4 - 1];
|
||
finalOut[7] = ckDataBuf[10 - 1];
|
||
|
||
memcpy(pDataOut + countsOut, finalOut, sizeof(finalOut));
|
||
countsOut += 8;
|
||
ckDataPos = 0;
|
||
}
|
||
}
|
||
(*lenOut) = countsOut * sizeof(short);
|
||
}
|
||
|
||
int Netease_nduilite_init(duilite_callback wakeup_callback,
|
||
duilite_callback doa_callback,
|
||
duilite_callback beamforming_callback,
|
||
duilite_callback vad_callback) {
|
||
int ret = 0;
|
||
char *auth_cfg;
|
||
char *cfg;
|
||
char *vad_cfg;
|
||
|
||
switch (getWorkMode()) {
|
||
case WorkMode_R311_C1_EVB_2MIC:
|
||
auth_cfg = "{\"appKey\":\"15205715844583be\",\"secretKey\":"
|
||
"\"e3eee41c3276bd027f2814f6cf3b8991\",\"deviceId\": \"28ede0d08139\","
|
||
"\"provision\":\"/usr/share/netease/duilite/auth/"
|
||
"aiengine-2.9.5-15205715844583be.provision\", "
|
||
"\"serialNumber\":\"/mnt/UDISK/device.number\"}";
|
||
|
||
///*初始化时默认采用夜间唤醒阈值，thresh=0.59;major=0;thresh2=0.36；
|
||
///时间同步成功后再根据实际时间来切换阈值*/
|
||
cfg = "{\"aecBinPath\": "
|
||
"\"/usr/share/netease/duilite/fespa/"
|
||
"AEC_ch4-2-ch2_1ref_common_20180731_v0.9.4.bin\", "
|
||
"\"wakeupBinPath\": "
|
||
"\"/usr/share/netease/duilite/fespa/"
|
||
"wakeup_aifar_comm_20180104.bin\", "
|
||
"\"beamformingBinPath\": "
|
||
"\"/usr/share/netease/duilite/fespa/"
|
||
"UDA_asr_ch2_2_ch2_30mm_20181129_v1.1.0.8_asrpost0.bin\", "
|
||
"\"env\": \"words=di da di da;thresh=0.05;major=0;dcheck=0;thresh2=0.05;\", "
|
||
"\"rollBack\": 0}";
|
||
|
||
#if USED_NETEASE_DUILITE_VAD
|
||
vad_cfg = "{\"resBinPath\": "
|
||
"\"/usr/share/netease/duilite/vad/"
|
||
"vad_aihome_v0.8.bin\",\"pauseTime\": 500}";
|
||
#endif
|
||
break;
|
||
case WorkMode_R311_C1_EVB:
|
||
auth_cfg = "{\"appKey\":\"15205715844583be\",\"secretKey\":"
|
||
"\"e3eee41c3276bd027f2814f6cf3b8991\",\"deviceId\": \"28ede0d08139\","
|
||
"\"provision\":\"/usr/share/netease/duilite/auth/"
|
||
"aiengine-2.9.5-15205715844583be.provision\", "
|
||
"\"serialNumber\":\"/mnt/UDISK/device.number\"}";
|
||
|
||
///*初始化时默认采用夜间唤醒阈值，thresh=0.59;major=0;thresh2=0.36；
|
||
///时间同步成功后再根据实际时间来切换阈值*/
|
||
cfg = "{\"aecBinPath\": "
|
||
"\"/usr/share/netease/duilite/fespa/"
|
||
"AEC_ch6-2-ch4_1ref_common_20180801_v0.9.4.bin\", "
|
||
"\"wakeupBinPath\": "
|
||
"\"/usr/share/netease/duilite/fespa/"
|
||
"wakeup_aifar_comm_20180104.bin\", "
|
||
"\"beamformingBinPath\": "
|
||
"\"/usr/share/netease/duilite/fespa/"
|
||
"ULA_asr_ch4_2_ch4_35mm_20181206_v1.1.0.8_asrpost0.bin\", "
|
||
"\"env\": \"words=di da di da;thresh=0.05;major=0;dcheck=0;thresh2=0.05;\", "
|
||
"\"rollBack\": 0}";
|
||
|
||
#if USED_NETEASE_DUILITE_VAD
|
||
vad_cfg = "{\"resBinPath\": "
|
||
"\"/usr/share/netease/duilite/vad/"
|
||
"vad_aihome_v0.8.bin\",\"pauseTime\": 500}";
|
||
#endif
|
||
break;
|
||
|
||
default:
|
||
auth_cfg = "{\"appKey\":\"15205715844583be\",\"secretKey\":"
|
||
"\"e3eee41c3276bd027f2814f6cf3b8991\", "
|
||
"\"provision\":\"/usr/share/netease/duilite/auth/"
|
||
"aiengine-2.9.5-15205715844583be.provision\", "
|
||
"\"serialNumber\":\"/mnt/UDISK/device.number\"}";
|
||
|
||
///*初始化时默认采用夜间唤醒阈值，thresh=0.59;major=0;thresh2=0.36；
|
||
///时间同步成功后再根据实际时间来切换阈值*/
|
||
cfg = "{\"aecBinPath\": "
|
||
"\"/usr/share/netease/duilite/fespa/"
|
||
"AEC_ch8-2-ch6_2ref_NTES_20180413_v0.9.3.bin\", "
|
||
"\"wakeupBinPath\": "
|
||
"\"/usr/share/netease/duilite/fespa/"
|
||
"wakeup_aihome_ntes_20180914_pre.bin\", "
|
||
"\"beamformingBinPath\": "
|
||
"\"/usr/share/netease/duilite/fespa/"
|
||
"UCA_asr_ch8-2-ch6_70mm_netease_20180425_v1.1.9.bin\", "
|
||
"\"env\": \"words=di da di "
|
||
"da;thresh=0.59;major=0;dcheck=0;thresh2=0.36;\", "
|
||
"\"rollBack\": 0}";
|
||
|
||
#if USED_NETEASE_DUILITE_VAD
|
||
vad_cfg = "{\"resBinPath\": "
|
||
"\"/usr/share/netease/duilite/vad/"
|
||
"vad_aihome_v0.6.bin\",\"pauseTime\": 500}";
|
||
#endif
|
||
}
|
||
|
||
if (1 == duiliteok) {
|
||
return 1;
|
||
}
|
||
n_debug("Begin to init duilite!\n");
|
||
|
||
if (NULL == _duilitelock) {
|
||
_duilitelock = native_mutex_create(NULL, NULL);
|
||
}
|
||
|
||
ret = duilite_library_load(auth_cfg);
|
||
if (0 != ret) {
|
||
n_error("Duilite library load fail!(%d)\n", ret);
|
||
return 0;
|
||
}
|
||
fespa = duilite_fespl_new(cfg);
|
||
if (NULL != fespa) {
|
||
n_debug("fespa new success!\n");
|
||
} else {
|
||
n_error("fespa new fail!\n");
|
||
duilite_library_release();
|
||
return 0;
|
||
}
|
||
|
||
#if USED_NETEASE_DUILITE_VAD
|
||
duilitevad = duilite_vad_new(vad_cfg, vad_status_cb, NULL);
|
||
oem_vad_cb = vad_callback;
|
||
if (NULL == duilitevad) {
|
||
n_debug("Vad new fail!\n");
|
||
duilite_fespl_delete(fespa);
|
||
duilite_library_release();
|
||
|
||
return 0;
|
||
} else {
|
||
n_debug("Vad new success!\n");
|
||
}
|
||
#endif
|
||
duilite_fespl_register(fespa, DUILITE_CALLBACK_FESPL_WAKEUP,
|
||
wakeup_callback, NULL);
|
||
duilite_fespl_register(fespa, DUILITE_CALLBACK_FESPL_DOA, doa_callback,
|
||
NULL);
|
||
duilite_fespl_register(fespa, DUILITE_CALLBACK_FESPL_BEAMFORMING,
|
||
vad_audiocb, NULL);
|
||
oem_audio_cb = beamforming_callback;
|
||
|
||
n_debug("Init duilite finish!\n");
|
||
|
||
#if DUILITE_DEBUG
|
||
fd_audio_orig = open("/tmp/bkconvertdata", O_CREAT | O_APPEND | O_RDWR);
|
||
#endif
|
||
duiliteok = 1;
|
||
return 1;
|
||
}
|
||
|
||
void Dmic_data_convert(const void *audioData, unsigned int audio_len) {
|
||
short *p = audioData;
|
||
short tmp;
|
||
int bitPerChan = 16;
|
||
int chanNum = 8;
|
||
// 16bit 8 channel, 0,1 is null
|
||
if (audio_len % (bitPerChan * chanNum / 8) != 0) {
|
||
n_error("Input data len(%d) error, not full frame!", audio_len);
|
||
}
|
||
|
||
while (p < (audioData + audio_len)) {
|
||
p[0] = p[2];
|
||
p[1] = p[3];
|
||
p[2] = p[4];
|
||
p[3] = p[5];
|
||
p[4] = p[6];
|
||
p[5] = p[7];
|
||
p[6] = 0x0000;
|
||
p[7] = 0x0000;
|
||
p += 8;
|
||
}
|
||
}
|
||
|
||
void Netease_nduilite_writeaudio(const void *audioData,
|
||
unsigned int audio_len) {
|
||
int len = 0;
|
||
char *newbuf = NULL;
|
||
int ret = -1;
|
||
|
||
if (0 == duiliteok) {
|
||
return;
|
||
}
|
||
|
||
if (NULL == fespa) {
|
||
n_debug("nduilite fd is null!\n");
|
||
return;
|
||
}
|
||
|
||
if (audio_len == 0) {
|
||
n_debug("nduilite write audio fail, len = 0\n");
|
||
return;
|
||
}
|
||
|
||
//n_debug("Write nduilite data, len:%d\n", audio_len);
|
||
ret = duilite_fespl_feed(fespa, audioData, audio_len);
|
||
|
||
#if 0
|
||
newbuf = (char *)malloc(audio_len + 60);
|
||
|
||
if (newbuf == NULL) {
|
||
n_error("nduilite malloc fail!\n");
|
||
return;
|
||
}
|
||
|
||
CheckData(audioData, audio_len, (void *)newbuf, &len);
|
||
|
||
ret = duilite_fespl_feed(fespa, newbuf, len);
|
||
|
||
#if DUILITE_DEBUG
|
||
n_debug("orig data len:%d, convert len:%d, ret:%d\n", audio_len, len, ret);
|
||
write(fd_audio_orig, newbuf, len);
|
||
#endif
|
||
free(newbuf);
|
||
#endif
|
||
}
|
||
|
||
int Netease_nduilite_reinit(duilite_callback wakeup_callback,
|
||
duilite_callback doa_callback,
|
||
duilite_callback beamforming_callback,
|
||
duilite_callback vad_callback) {
|
||
if (1 == duiliteok) {
|
||
duiliteok = 0;
|
||
n_debug("Duilite reinit sdk!\n");
|
||
usleep(15000);
|
||
if (NULL != fespa) {
|
||
duilite_fespl_delete(fespa);
|
||
fespa = NULL;
|
||
}
|
||
|
||
if (NULL != duilitevad) {
|
||
duilite_vad_delete(duilitevad);
|
||
duilitevad = NULL;
|
||
}
|
||
|
||
duilite_library_release();
|
||
Netease_nduilite_init(wakeup_callback, doa_callback,
|
||
beamforming_callback, vad_callback);
|
||
}
|
||
|
||
return 1;
|
||
}
|
||
#endif |