/////////////////////////////////////////////////////////////////////////// // Copyright 2019 Amazon.com, Inc. or its affiliates. All Rights Reserved. /////////////////////////////////////////////////////////////////////////// #include #include #include #include "pryon_lite_common_client_properties.h" #include "pryon_lite_PRL2000.h" #define SAMPLES_PER_FRAME (160) #define false 0 #define true 1 // global flag to stop processing, set by application static int quit = 0; // engine handle static PryonLiteV2Handle sHandle = {0}; #define ALIGN(n) __attribute__((aligned(n))) static char* engineBuffer = { 0 }; // should be an array large enough to hold the largest engine //---- Application functions to be implemented by the client ------------------- // ---- Wakeword ---- // binary model buffer, allocated by application // this buffer can be read-only memory as PryonLite will not modify the contents ALIGN(4) static const char * wakewordModelBuffer = { 0 }; // should be an array large enough to hold the largest wakeword model static void loadWakewordModel(const char **model, size_t *sizeofModel) { // In order to detect keywords, the decoder uses a model which defines the parameters, // neural network weights, classifiers, etc that are used at runtime to process the audio // and give detection results. // Each model is packaged in two formats: // 1. A .bin file that can be loaded from disk (via fopen, fread, etc) // 2. A .cpp file that can be hard-coded at compile time *sizeofModel = 1; // example value, will be the size of the binary model byte array *model = wakewordModelBuffer; // pointer to model in memory } // ---- Fingerprinting ---- // binary fingerprint list buffer, allocated by application // this buffer can be read-only memory as PryonLite will not modify the contents ALIGN(4) static const char * fingerprintListBuffer = { 0 }; // should be an array large enough to hold the fingerprint list data. static void loadFingerprintList(const char **fingerprintList, size_t *sizeofFingerprintList) { // In order to suppress wakes from fingerprinted media, PryonLite uses a binary list which // tells its engine which audio to suppress. // Each list is a binary file that can be loaded from disk and should be downloaded // via a DAVS client. *sizeofFingerprintList = 1; // example value, will be the size of the binary fingerprint list byte array *fingerprintList = fingerprintListBuffer; // pointer to fingerprint list in memory } // ---- Watermarking ---- // binary config buffer for watermark based media suppression, allocated by application // this buffer can be read-only memory as PryonLite will not modify the contents ALIGN(4) static const char * watermarkConfigBuffer = { 0 }; // should be an array large enough to hold the largest watermark config static void loadWatermarkConfig(const char **config, size_t *sizeofConfig) { // In order to suppress media generated wakes, the decoder can use a configuration which // defines some pre-embedded watermarks. These watermarks are searched for at runtime to // detect and suppress media generated wakes. // Each configuration is packaged in two formats: // 1. A .bin file that can be loaded from disk (via fopen, fread, etc) // 2. A .cpp file that can be hard-coded at compile time *sizeofConfig = 1; // example value, will be the size of the binary model byte array *config = watermarkConfigBuffer; // pointer to watermark configuration in memory } // client implemented function to read audio samples static void readAudio(short* samples, int sampleCount) { // todo - read samples from file, audio system, etc. } //---- Engine callback functions to be implemented by the client -------------- // ---- Voice Activity Detection ---- // VAD event handler static void vadEventHandler(PryonLiteV2Handle *handle, const PryonLiteVadEvent* vadEvent) { printf("VAD state %d at sample index %lld\n", (int) vadEvent->vadState, vadEvent->beginSampleIndex); } // ---- Fingerprinting ---- // Fingerprint match event handler static void fingerprintMatchEventHandler(PryonLiteV2Handle *handle, const PryonLiteFingerprintMatchEvent* fingerprintMatchEvent) { printf("Detected fingerprint match with keyword '%s'\n", fingerprintMatchEvent->keyword); } // ---- Wakeword ---- // Wakeword event handler static void wakewordEventHandler(PryonLiteV2Handle *handle, const PryonLiteWakewordResult* wwEvent) { printf("Detected wakeword '%s'\n", wwEvent->keyword); } /// /// @brief Callback function triggered by the engine when any event occurs. /// /// @param handle [in] Handle for the engine which created the event /// @param event [in] Event that occurred /// /// @return void /// static void handleEvent(PryonLiteV2Handle *handle, const PryonLiteV2Event* event) { // ---- Voice Activity Detection ---- if (event->vadEvent != NULL) { vadEventHandler(handle, event->vadEvent); } // ---- Fingerprinting ---- if (event->fingerprintMatchEvent != NULL) { fingerprintMatchEventHandler(handle, event->fingerprintMatchEvent); } // ---- Wakeword ---- if (event->wwEvent != NULL) { wakewordEventHandler(handle, event->wwEvent); } } //---- Main processing loop ---------------------------------------------------- // The main loop below shows the full life cycle of the engine. This // life cycle is broken down into three phases. // // Phase 1 - Initialization // STEP 1.1 - Load the models // STEP 1.2 - Configure engine // STEP 1.3 - Enable engine events // STEP 1.4 - Query for configuration specific attributes // STEP 1.5 - Allocate/Check engine buffer // STEP 1.6 - Initialize engine // STEP 1.7 - Post-init functionality setup // STEP 1.8 - Optional : Runtime configuration functions // Phase 2 - Audio Processing // STEP 2.1 - Gather audio // STEP 2.2 - Push audio to engine // STEP 2.3 - Handle engine events // Phase 3 - Cleanup // STEP 3.1 - Functionality specific cleanup // STEP 3.2 - Engine cleanup // // The sample below is for a single locale/model. To change the locale/model // being used, complete Phase 3 - Cleanup for the engine instance and then // create a new instance by going back through Phase 1 - Initialization with // the new model. int main(int argc, char **argv) { PryonLiteV2ConfigAttributes configAttributes = {0}; // Start Phase 1 - Initialization // // The initialization phase begins with nothing and ends with a fully // initialized instance of the engine. // // STEP 1.1 - Load the models // This step covers loading the model data from source. Models are // delivered in two different forms, a C file with an array // containing the model source or a separate bin file. If using // the C file, the model source and size are already defined. If // using the bin file, the model source needs to be read into an // array and the length needs to be retrieved. // ---- Wakeword ---- const char *wakewordModel; size_t wakewordModelSize; loadWakewordModel(&wakewordModel, &wakewordModelSize); // ---- Fingerprinting ---- const char* fingerprintList; size_t sizeofFingerprintList; loadFingerprintList(&fingerprintList, &sizeofFingerprintList); // ---- Watermarking ---- const char *watermarkConfigBlob; size_t sizeofWatermarkConfigBlob; loadWatermarkConfig(&watermarkConfigBlob, &sizeofWatermarkConfigBlob); // STEP 1.2 - Configure engine // PryonLiteV2Config contains initialization-time configuration // parameters. Each feature to be enabled must be configured // individually and then hooked to the top level engine // configuration. For each feature use the _Default macro to set // up the initial values of the configuration structure. There are // required fields which must be modified from their default // values; see the example below. PryonLiteV2Config engineConfig = {0}; // ---- Wakeword ---- PryonLiteWakewordConfig wakewordConfig = PryonLiteWakewordConfig_Default; // Required fields: model, sizeofModel loaded in STEP 1 wakewordConfig.model = wakewordModel; wakewordConfig.sizeofModel = wakewordModelSize; // Required: Link the wakeword configuration to the engine configuration engineConfig.ww = &wakewordConfig; // ---- Fingerprinting ---- PryonLiteFingerprintConfig fingerprintConfig = PryonLiteFingerprintConfig_Default; // Required fields: fingerprintList, sizeofFingerprintList loaded in STEP 1 fingerprintConfig.fingerprintList = fingerprintList; fingerprintConfig.sizeofFingerprintList = sizeofFingerprintList; // Required: Do not configure VAD in the engine config as it is incompatible with fingerprint match suppression // Required: Link the fingerprinting configuration to the engine configuration engineConfig.fingerprinter = &fingerprintConfig; // ---- Voice Activity Detection ---- PryonLiteVadConfig vadConfig = PryonLiteVadConfig_Default; // Example configuration using the EnergyDetection implementation PryonLiteEnergyDetectionConfig energyDetection = PryonLiteEnergyDetectionConfig_Default; energyDetection.enableGate = 1; vadConfig.energyDetection = &energyDetection; // Required: Link the voice activity detection configuration to the engine configuration engineConfig.vad = &vadConfig; // ---- Watermarking ---- PryonLiteWatermarkConfig watermarkConfig = PryonLiteWatermarkConfig_Default; // Required fields: config, sizeofConfig loaded in STEP 1 watermarkConfig.config = watermarkConfigBlob; watermarkConfig.sizeofConfig = sizeofWatermarkConfigBlob; // Required: Link the watermark configuration to the engine configuration engineConfig.watermark = &watermarkConfig; // STEP 1.3 - Enable engine events // PryonLiteV2EventConfig is used to select which events the // engine will pass back to the application layer. Each field in // this structure is a flag that enables or disables the emission // of the event. PryonLiteV2EventConfig engineEventConfig = {0}; // ---- Voice Activity Detection ---- engineEventConfig.enableVadEvent = false; // disable VAD event, set to true to receive VAD events // ---- Fingerprinting ---- engineEventConfig.enableFingerprintMatchEvent = true; // ---- Wakeword ---- engineEventConfig.enableWwEvent = true; // STEP 1.4 - Query for configuration specific attributes // The engine initialization requires a buffer be passed in which // is owned by the application layer. This instance memory buffer // must persist for the life of the engine instance. The size of // this buffer is variable, and dependent on the client-specified // configuration. Use PryonLite_GetConfigAttributes to determine // the size of the buffer and other information about the // configuration. PryonLiteStatus status = PryonLite_GetConfigAttributes(&engineConfig, &engineEventConfig, &configAttributes); if (status.publicCode != PRYON_LITE_ERROR_OK) { // handle error return -1; } // ---- Wakeword ---- // Optional - Sample code showing how to list supported keywords printf("Supported keywords: "); int keyword; for (keyword = 0; keyword < configAttributes.wwConfigAttributes.numKeywords; keyword++) { if (keyword > 0) { printf(", "); } printf("%s", configAttributes.wwConfigAttributes.keywords[keyword]); } printf("\n"); // STEP 1.5 - Allocate/Check engine buffer // Once the size of the engine buffer has been determined, the // application layer must create the buffer. This example uses // a statically-defined buffer. If applicable to the device, // this buffer can be dynamically allocated as well, but must // remain allocated for the duration that the engine is in use. // The requirement is that a buffer that is at least // configAttributes.requiredMem size, in bytes, is created. if (configAttributes.requiredMem > sizeof(engineBuffer)) { // handle error return -1; } // STEP 1.6 - Initialize engine // Pass the engine configuration from STEP 2, event configuration // from STEP 3, and engine buffer from STEP 5 to PryonLite_Initialize // to create an instance of the engine. After this function is // called, the engine instance referenced by sHandle is fully // functional. status = PryonLite_Initialize(&engineConfig, &sHandle, handleEvent, &engineEventConfig, engineBuffer, sizeof(engineBuffer)); if (status.publicCode != PRYON_LITE_ERROR_OK) { // handle error return -1; } // Optional - Check if the engine has been initialized if (!PryonLite_IsInitialized(&sHandle)) { // handle error - PryonLite_Initialize returned success but the engine has not been initialized return -1; } // STEP 1.7 - Post-init functionality setup // Some functionalities require additional setup steps after the // engine is initialized. If such a step is required for a // functionality it will be implemented here. // STEP 1.8 - Optional : Runtime configuration functions // The optional functions below allow for the runtime configuration of // certain aspects of the engine. These functions can be called on // an engine instance any time after a successful PryonLite_Initialize // and before PryonLite_Destroy. // ---- Wakeword ---- // Set detection threshold for all keywords int detectionThreshold = 500; status = PryonLiteWakeword_SetDetectionThreshold(sHandle.ww, NULL, detectionThreshold); if (status.publicCode != PRYON_LITE_ERROR_OK) { // handle error return -1; } // Disable a wakeword keyword // Please note the example uses a random wakeword // When wanting to disable/enable a specific wakeword, substitute "wakewordKeyword" // with the specific keyword string (ex. "ALEXA") in the example below int enableWakewordKeyword = 0; const char* wakewordKeyword = configAttributes.wwConfigAttributes.keywords[0]; status = PryonLiteWakeword_EnableKeyword(sHandle.ww, wakewordKeyword, enableWakewordKeyword); if (status.publicCode != PRYON_LITE_ERROR_OK) { // handle error return -1; } printf("Disabled keyword: %s\n", wakewordKeyword); // Re-enable a wakeword keyword enableWakewordKeyword = 1; status = PryonLiteWakeword_EnableKeyword(sHandle.ww, wakewordKeyword, enableWakewordKeyword); if (status.publicCode != PRYON_LITE_ERROR_OK) { // handle error return -1; } printf("Re-enabled keyword: %s\n", wakewordKeyword); // End Phase 1 - Initialization // Examples - Optional runtime functions // The runtime functions below can be called on an engine // instance any time after a successful PryonLite_Initialize // and before PryonLite_Destroy. // ---- General ---- // Call the set client property API to inform the engine of client state changes status = PryonLite_SetClientProperty(&sHandle, CLIENT_PROP_GROUP_COMMON, CLIENT_PROP_COMMON_AUDIO_PLAYBACK, 1); if (status.publicCode != PRYON_LITE_ERROR_OK) { return -1; } // allocate buffer to hold audio samples short samples[SAMPLES_PER_FRAME]; // run engine while (1) { // Start Phase 2 - Audio Processing // // The audio processing phase is where audio is pushed into an initialized // engine instance, and engine events are emitted for handling by // registered application/client callbacks. // // STEP 1 - Gather audio // Audio must be gathered into frames of length // SAMPLES_PER_FRAME before sending to the engine. readAudio(samples, SAMPLES_PER_FRAME); // STEP 2.2 - Push audio to engine // Once the required amount of audio has been gathered, push // the frame into the engine using PushAudioSamples. This // signals the engine to process the audio and invokes callback // functions to pass any resulting events to the client. status = PryonLite_PushAudioSamples(&sHandle, samples, SAMPLES_PER_FRAME); if (status.publicCode != PRYON_LITE_ERROR_OK) { // handle error return -1; } // STEP 2.3 - Handle engine events // The engine passes back event information to the application // layer through a single callback function passed into // PryonLite_Initialize. The event types emitted depend on the // event configuration setup in Phase 1 Step 3. See the // EventConfig structure definition in the header files for more // information. // // End Phase 2 - Audio Processing // Examples - Optional runtime loop functions // The runtime functions below can be invoked any time // between a successful PryonLite_Initialize and // PryonLite_Destroy. These functions are typically // invoked during the core audio processing loop. if (quit) { // Start Phase 3 - Cleanup // // Cleanup should only occur when the engine is no longer needed // This will flush any unprocessed audio that has been // pushed and destroy the engine instance. // // STEP 3.1 - Functionality-specific cleanup // These functionality-specific cleanup functions should // be called before engine cleanup. If this step is // required it will be implemented below. // STEP 3.2 - Engine cleanup // This will flush any unprocessed audio that has been // pushed and destroy the engine instance. status = PryonLite_Destroy(&sHandle); if (status.publicCode != PRYON_LITE_ERROR_OK) { // handle error return -1; } break; // End Phase 3 - Cleanup } } return 0; }