diff --git a/include/ctrlm_hal_ble.h b/include/ctrlm_hal_ble.h index 0f84fc64..082bdb39 100644 --- a/include/ctrlm_hal_ble.h +++ b/include/ctrlm_hal_ble.h @@ -53,6 +53,14 @@ typedef enum { CTRLM_HAL_BLE_VOICE_STREAM_END_UNKNOWN } ctrlm_hal_ble_VoiceStreamEnd_t; +/// @brief MFV wake word detection types (matches BleRcuAudioService::DetectionType values). +typedef enum { + CTRLM_HAL_BLE_MFV_DETECTION_UNKNOWN = 0x00, + CTRLM_HAL_BLE_MFV_DETECTION_FULL_POWER = 0x01, + CTRLM_HAL_BLE_MFV_DETECTION_AAD = 0x02, + CTRLM_HAL_BLE_MFV_DETECTION_BELOW_THRESHOLD = 0x03, +} ctrlm_hal_ble_MfvDetectionType_t; + /// @brief Supported voice data encoding types. typedef enum { CTRLM_HAL_BLE_PROPERTY_IEEE_ADDDRESS = 0, @@ -85,6 +93,10 @@ typedef enum { CTRLM_HAL_BLE_PROPERTY_WAKEUP_CONFIG, CTRLM_HAL_BLE_PROPERTY_WAKEUP_CUSTOM_LIST, CTRLM_HAL_BLE_PROPERTY_IRDBS_SUPPORTED, + CTRLM_HAL_BLE_PROPERTY_MFV_DETECTION_TYPE, + CTRLM_HAL_BLE_PROPERTY_MFV_DETECTION_DATA, + CTRLM_HAL_BLE_PROPERTY_MFV_PRIVACY, + CTRLM_HAL_BLE_PROPERTY_MFV_CAPABILITIES, CTRLM_HAL_BLE_PROPERTY_UNKNOWN } ctrlm_hal_ble_RcuProperty_t; /// @} @@ -95,34 +107,40 @@ typedef enum { /// @details The Control Manager HAL provides structures that are used in calls to the HAL networks. typedef struct { - int device_minor_id; - uint8_t battery_level; - bool connected; - unsigned long long ieee_address; - char fw_revision[CTRLM_MAX_PARAM_STR_LEN]; - char hw_revision[CTRLM_MAX_PARAM_STR_LEN]; - char sw_revision[CTRLM_MAX_PARAM_STR_LEN]; - char manufacturer[CTRLM_MAX_PARAM_STR_LEN]; - char model[CTRLM_MAX_PARAM_STR_LEN]; - char name[CTRLM_MAX_PARAM_STR_LEN]; - char serial_number[CTRLM_MAX_PARAM_STR_LEN]; - int ir_code; - uint8_t audio_gain_level; - uint32_t audio_codecs; - bool audio_streaming; - unsigned int touch_mode; - bool touch_mode_settable; - bool is_upgrading; - int upgrade_progress; - char upgrade_error[CTRLM_MAX_PARAM_STR_LEN]; - ctrlm_ble_RcuUnpairReason_t unpair_reason; - ctrlm_ble_RcuRebootReason_t reboot_reason; - char assert_report[CTRLM_RCU_ASSERT_REPORT_MAX_SIZE+1]; - uint16_t last_wakeup_key; - uint8_t wakeup_config; - int wakeup_custom_list[CTRLM_WAKEUP_CONFIG_LIST_MAX_SIZE]; - int wakeup_custom_list_size; - uint8_t irdbs_supported; + int device_minor_id; + uint8_t battery_level; + bool connected; + unsigned long long ieee_address; + char fw_revision[CTRLM_MAX_PARAM_STR_LEN]; + char hw_revision[CTRLM_MAX_PARAM_STR_LEN]; + char sw_revision[CTRLM_MAX_PARAM_STR_LEN]; + char manufacturer[CTRLM_MAX_PARAM_STR_LEN]; + char model[CTRLM_MAX_PARAM_STR_LEN]; + char name[CTRLM_MAX_PARAM_STR_LEN]; + char serial_number[CTRLM_MAX_PARAM_STR_LEN]; + int ir_code; + uint8_t audio_gain_level; + uint32_t audio_codecs; + bool audio_streaming; + unsigned int touch_mode; + bool touch_mode_settable; + bool is_upgrading; + int upgrade_progress; + char upgrade_error[CTRLM_MAX_PARAM_STR_LEN]; + ctrlm_ble_RcuUnpairReason_t unpair_reason; + ctrlm_ble_RcuRebootReason_t reboot_reason; + char assert_report[CTRLM_RCU_ASSERT_REPORT_MAX_SIZE+1]; + uint16_t last_wakeup_key; + uint8_t wakeup_config; + int wakeup_custom_list[CTRLM_WAKEUP_CONFIG_LIST_MAX_SIZE]; + int wakeup_custom_list_size; + uint8_t irdbs_supported; + ctrlm_hal_ble_MfvDetectionType_t mfv_detection_type; + uint16_t mfv_ww_start; + uint16_t mfv_ww_end; + uint16_t mfv_confidence; // encoded as percentage * 10 (e.g. 976 = 97.6%) + bool mfv_privacy_enabled; + uint8_t mfv_capabilities; } ctrlm_hal_ble_rcu_data_t; typedef struct { diff --git a/src/ble/ctrlm_ble_controller.cpp b/src/ble/ctrlm_ble_controller.cpp index 388fabb8..01f712e3 100644 --- a/src/ble/ctrlm_ble_controller.cpp +++ b/src/ble/ctrlm_ble_controller.cpp @@ -528,6 +528,76 @@ ctrlm_timestamp_t ctrlm_obj_controller_ble_t::getVoiceStartTimeLocal() const { return(voice_start_time_local_); } +void ctrlm_obj_controller_ble_t::setMfvDetectionType(ctrlm_hal_ble_MfvDetectionType_t type) { + mfv_detection_type_ = type; +} + +ctrlm_hal_ble_MfvDetectionType_t ctrlm_obj_controller_ble_t::getMfvDetectionType() const { + return mfv_detection_type_; +} + +void ctrlm_obj_controller_ble_t::setMfvDetectionData(uint16_t start, uint16_t end, uint16_t confidence) { + mfv_ww_start_ = start; + mfv_ww_end_ = end; + mfv_confidence_ = confidence; +} + +uint16_t ctrlm_obj_controller_ble_t::getMfvWwStart() const { + return mfv_ww_start_; +} + +uint16_t ctrlm_obj_controller_ble_t::getMfvWwEnd() const { + return mfv_ww_end_; +} + +uint16_t ctrlm_obj_controller_ble_t::getMfvConfidence() const { + return mfv_confidence_; +} + +float ctrlm_obj_controller_ble_t::getMfvConfidencePercent() const { + return mfv_confidence_ / 10.0f; +} + +void ctrlm_obj_controller_ble_t::setMfvPrivacy(bool enabled) { + mfv_privacy_enabled_ = enabled; +} + +bool ctrlm_obj_controller_ble_t::getMfvPrivacy() const { + return mfv_privacy_enabled_; +} + +void ctrlm_obj_controller_ble_t::setMfvCapabilities(uint8_t caps) { + mfv_capabilities_ = caps; +} + +uint8_t ctrlm_obj_controller_ble_t::getMfvCapabilities() const { + return mfv_capabilities_; +} + +void ctrlm_obj_controller_ble_t::setMfvDetectionPending(bool pending, int audio_fd) { + mfv_detection_pending_ = pending; + mfv_detection_audio_fd_ = pending ? audio_fd : -1; + if (!pending) { + mfv_detection_data_fresh_ = false; + } +} + +bool ctrlm_obj_controller_ble_t::isMfvDetectionPending() const { + return mfv_detection_pending_; +} + +int ctrlm_obj_controller_ble_t::getMfvDetectionAudioFd() const { + return mfv_detection_audio_fd_; +} + +void ctrlm_obj_controller_ble_t::setMfvDetectionDataFresh(bool fresh) { + mfv_detection_data_fresh_ = fresh; +} + +bool ctrlm_obj_controller_ble_t::isMfvDetectionDataFresh() const { + return mfv_detection_data_fresh_; +} + void ctrlm_obj_controller_ble_t::setSupportedIrdbs(uint8_t vendor_support_bitmask) { this->irdbs_supported_ = vendor_support_bitmask; @@ -689,6 +759,13 @@ void ctrlm_obj_controller_ble_t::print_status() { XLOGD_INFO("Programmed AVR IRDB Code : %s", irdb_entry_id_name_avr_->to_string().c_str()); XLOGD_INFO(""); voice_metrics_->print(__FUNCTION__); + XLOGD_INFO(""); + XLOGD_INFO("MFV Detection Type : 0x%02X", static_cast(mfv_detection_type_)); + XLOGD_INFO("MFV WW Start : %u", mfv_ww_start_); + XLOGD_INFO("MFV WW End : %u", mfv_ww_end_); + XLOGD_INFO("MFV Confidence : %.1f%%", mfv_confidence_ / 10.0); + XLOGD_INFO("MFV Privacy : %s", mfv_privacy_enabled_ ? "enabled" : "disabled"); + XLOGD_INFO("MFV Capabilities : 0x%02X", static_cast(mfv_capabilities_)); XLOGD_WARN("------------------------------------------------------------"); } diff --git a/src/ble/ctrlm_ble_controller.h b/src/ble/ctrlm_ble_controller.h index 432f5c76..60c2d37e 100644 --- a/src/ble/ctrlm_ble_controller.h +++ b/src/ble/ctrlm_ble_controller.h @@ -147,6 +147,27 @@ class ctrlm_obj_controller_ble_t : public ctrlm_obj_controller_t { ctrlm_timestamp_t getVoiceStartTimeKey() const; ctrlm_timestamp_t getVoiceStartTimeLocal() const; + void setMfvDetectionType(ctrlm_hal_ble_MfvDetectionType_t type); + ctrlm_hal_ble_MfvDetectionType_t getMfvDetectionType() const; + void setMfvDetectionData(uint16_t start, uint16_t end, uint16_t confidence); + uint16_t getMfvWwStart() const; + uint16_t getMfvWwEnd() const; + uint16_t getMfvConfidence() const; // returns raw value encoded as integer * 10 (e.g. 953 = 95.3%) + float getMfvConfidencePercent() const; // returns confidence as a percentage (e.g. 95.3) + void setMfvPrivacy(bool enabled); + bool getMfvPrivacy() const; + void setMfvCapabilities(uint8_t caps); + uint8_t getMfvCapabilities() const; + + // MFV wake-word (detection-triggered) session state. Such a session starts audio streaming, waits + // for the detection data (wake word timing/confidence), then opens the voice session. See + // ctrlm_obj_network_ble_t::req_process_detection_voice_session_begin(). + void setMfvDetectionPending(bool pending, int audio_fd = -1); + bool isMfvDetectionPending() const; + int getMfvDetectionAudioFd() const; + void setMfvDetectionDataFresh(bool fresh); // fresh detection data received for the current pending session (stale-data safeguard) + bool isMfvDetectionDataFresh() const; + void update_controller_id_and_db_entry(std::string name, ctrlm_network_id_t network_id, ctrlm_controller_id_t new_id); private: @@ -197,6 +218,16 @@ class ctrlm_obj_controller_ble_t : public ctrlm_obj_controller_t { bool type_z_supported_ = false; + ctrlm_hal_ble_MfvDetectionType_t mfv_detection_type_ = CTRLM_HAL_BLE_MFV_DETECTION_UNKNOWN; + uint16_t mfv_ww_start_ = 0; + uint16_t mfv_ww_end_ = 0; + uint16_t mfv_confidence_ = 0; // encoded as integer * 10 (e.g. 953 = 95.3%) + bool mfv_privacy_enabled_ = false; + uint8_t mfv_capabilities_ = 0; + bool mfv_detection_pending_ = false; // a wake-word session has started audio and is awaiting its detection data + bool mfv_detection_data_fresh_ = false; // fresh detection data arrived for the current pending session + int mfv_detection_audio_fd_ = -1; // fd of the audio stream started for the pending session + bool conn_param_update_before_ota_supported_ = false; ctrlm_sw_version_t conn_param_update_before_ota_version_; diff --git a/src/ble/ctrlm_ble_network.cpp b/src/ble/ctrlm_ble_network.cpp index f5c316b4..ad8a3f1a 100644 --- a/src/ble/ctrlm_ble_network.cpp +++ b/src/ble/ctrlm_ble_network.cpp @@ -64,6 +64,10 @@ using namespace std; #define CTRLM_BLE_PRINT_STATUS_DEFER_MAX (20) #define CTRLM_BLE_EVENT_STATUS_DEFER_MAX (20) +// Maximum time to wait for the MFV detection data (wake word timing/confidence) after audio streaming +// has started before opening the voice session without the wake word stream parameters. +#define CTRLM_BLE_MFV_DETECTION_DATA_TIMEOUT (1500) // 1.5 second + #define CTRLM_VENDOR_BLE_NETWORK_DISABLE_FILE "/etc/vendor/input/ble_network_disable" #define CTRLM_VENDOR_BLE_REMOTE_WHITELIST_FILE "/etc/vendor/input/ble_remote_whitelist.json" #define CTRLM_VENDOR_BLE_NETWORK_OPTIONS_FILE "/etc/vendor/input/ble_network_options.json" @@ -74,6 +78,8 @@ typedef struct { guint upgrade_pause_timer_tag; guint print_status_timer_tag; guint event_status_timer_tag; + guint mfv_detection_timer_tag; + unsigned long long mfv_detection_pending_ieee; } ctrlm_ble_network_t; static ctrlm_ble_network_t g_ctrlm_ble_network; @@ -89,6 +95,7 @@ static const vector ctrlm_ble_ir_key_names { static gboolean ctrlm_ble_print_status_timer_cb(gpointer user_data); static gboolean ctrlm_ble_event_status_timer_cb(gpointer user_data); +static gboolean ctrlm_ble_mfv_detection_timer_cb(gpointer user_data); static gboolean ctrlm_ble_upgrade_controllers_timer_cb(gpointer user_data); static gboolean ctrlm_ble_upgrade_resume_timer_cb(gpointer user_data); @@ -318,6 +325,13 @@ ctrlm_obj_network_ble_t::ctrlm_obj_network_ble_t(ctrlm_network_type_t type, ctrl XLOGD_INFO("BLE voice support is disabled by config"); } } + json_value = json_object_get(obj_options, JSON_BOOL_NAME_NETWORK_BLE_OPTIONS_MFV_BELOW_THRESHOLD_SESSION_ENABLED); + if(json_value != NULL && json_is_boolean(json_value)) { + mfv_below_threshold_session_enabled_ = json_boolean_value(json_value); + if(mfv_below_threshold_session_enabled_) { + XLOGD_INFO("MFV below-threshold detections will start a voice session"); + } + } } } @@ -342,6 +356,7 @@ ctrlm_obj_network_ble_t::~ctrlm_obj_network_ble_t() { ctrlm_timeout_destroy(&g_ctrlm_ble_network.upgrade_pause_timer_tag); ctrlm_timeout_destroy(&g_ctrlm_ble_network.event_status_timer_tag); ctrlm_timeout_destroy(&g_ctrlm_ble_network.print_status_timer_tag); + ctrlm_timeout_destroy(&g_ctrlm_ble_network.mfv_detection_timer_tag); ctrlm_ble_iarm_terminate(); @@ -537,8 +552,10 @@ void ctrlm_obj_network_ble_t::req_process_voice_session_begin(void *data, int si XLOGD_ERROR("Controller doesn't exist!"); dqm->params->result = CTRLM_IARM_CALL_RESULT_ERROR_INVALID_PARAMETER; } else { - // Currently BLE RCUs only support push-to-talk, so hardcoding here for now - ctrlm_voice_device_t device = CTRLM_VOICE_DEVICE_PTT; + // BLE RCUs use PTT for keypress-triggered sessions and FF for MFV wake-word sessions. + // The voice_device field of the message is set by the caller; it defaults to + // CTRLM_VOICE_DEVICE_PTT (0) when the struct is zeroed via memset. + ctrlm_voice_device_t device = dqm->voice_device; ctrlm_voice_session_response_status_t voice_status; // only support ADPCM from ble-rcu component @@ -613,6 +630,40 @@ void ctrlm_obj_network_ble_t::req_process_voice_session_begin(void *data, int si } } +// Safety timeout - the MFV detection data was not received in time, so release the deferred voice +// session connect anyway (without the wake word stream parameters). +static gboolean ctrlm_ble_mfv_detection_timer_cb(gpointer user_data) { + XLOGD_DEBUG("Enter..."); + ctrlm_network_id_t* net_id = (ctrlm_network_id_t*) user_data; + g_ctrlm_ble_network.mfv_detection_timer_tag = 0; + if (net_id != NULL) { + // Process on the ctrlm main thread to serialize with the other network handlers. + ctrlm_main_queue_handler_push(CTRLM_HANDLER_NETWORK, + (ctrlm_msg_handler_network_t)&ctrlm_obj_network_ble_t::req_process_mfv_detection_timeout, + NULL, 0, NULL, *net_id); + } + return false; +} + +void ctrlm_obj_network_ble_t::req_process_mfv_detection_timeout(void *data, int size) { + THREAD_ID_VALIDATE(); + + unsigned long long ieee_address = g_ctrlm_ble_network.mfv_detection_pending_ieee; + ctrlm_controller_id_t controller_id; + if (!getControllerId(ieee_address, &controller_id)) { + XLOGD_WARN("MFV detection data timeout - controller no longer exists"); + return; + } + if (!controllers_[controller_id]->isMfvDetectionPending()) { + // Detection data already arrived (or the session was cancelled) - nothing to do. + return; + } + controllers_[controller_id]->setMfvDetectionPending(false); + XLOGD_WARN("MFV detection data not received within timeout - releasing voice session connect without wake word stream parameters for device: %s", + controllers_[controller_id]->ieee_address_get().to_string().c_str()); + ctrlm_get_voice_obj()->voice_session_stream_params_update(CTRLM_VOICE_DEVICE_MFV, false, 0, 0, 0.0, 0.0); +} + bool ctrlm_obj_network_ble_t::end_voice_session_for_controller(uint64_t ieee_address, ctrlm_voice_session_end_reason_t reason, int32_t audioDuration, int32_t startLag, rdkx_timestamp_t *keyDownTime, rdkx_timestamp_t *keyUpTime) { ctrlm_controller_id_t controller_id; @@ -2069,19 +2120,90 @@ void ctrlm_obj_network_ble_t::ind_process_rcu_status(void *data, int size) { case CTRLM_HAL_BLE_PROPERTY_WAKEUP_CONFIG: controller->setWakeupConfig(dqm->rcu_data.wakeup_config); XLOGD_INFO("Controller <%s> notified wakeup config = <%s>", controller->ieee_address_get().to_string().c_str(), ctrlm_rcu_wakeup_config_str(controller->get_wakeup_config())); - schedule_status_print(); - schedule_status_event(); + //schedule_status_print(); + //schedule_status_event(); break; case CTRLM_HAL_BLE_PROPERTY_WAKEUP_CUSTOM_LIST: controller->setWakeupCustomList(dqm->rcu_data.wakeup_custom_list, dqm->rcu_data.wakeup_custom_list_size); XLOGD_INFO("Controller <%s> notified wakeup custom list = <%s>", controller->ieee_address_get().to_string().c_str(), controller->wakeupCustomListToString().c_str()); - schedule_status_print(); - schedule_status_event(); + //schedule_status_print(); + //schedule_status_event(); break; case CTRLM_HAL_BLE_PROPERTY_IRDBS_SUPPORTED: controller->setSupportedIrdbs(dqm->rcu_data.irdbs_supported); schedule_status_print(); break; + case CTRLM_HAL_BLE_PROPERTY_MFV_DETECTION_TYPE: { + const ctrlm_hal_ble_MfvDetectionType_t detection_type = dqm->rcu_data.mfv_detection_type; + controller->setMfvDetectionType(detection_type); + XLOGD_INFO("Controller <%s> MFV detection type = 0x%02X", controller->ieee_address_get().to_string().c_str(), static_cast(detection_type)); + + // Start a voice session for FullPower and AAD detections. BelowThreshold is a low-confidence + // secondary detection which is controlled by a config (mfv_below_threshold_session_enabled) and + // defaults to false to avoid flooding the server with false triggers from the field. + const bool start_session = (detection_type == CTRLM_HAL_BLE_MFV_DETECTION_FULL_POWER || + detection_type == CTRLM_HAL_BLE_MFV_DETECTION_AAD || + (detection_type == CTRLM_HAL_BLE_MFV_DETECTION_BELOW_THRESHOLD && mfv_below_threshold_session_enabled_)); + if (start_session) { + rdkx_timestamp_t detectionTime; + rdkx_timestamp_get(&detectionTime); + controller->setVoiceStartTime(detectionTime); + + XLOGD_INFO("------------------------------------------------------------------------"); + XLOGD_INFO("MFV wake word detected (%s) for device: %s", + detection_type == CTRLM_HAL_BLE_MFV_DETECTION_FULL_POWER ? "FullPower" : + detection_type == CTRLM_HAL_BLE_MFV_DETECTION_AAD ? "AAD" : "BelowThreshold", + controller->ieee_address_get().to_string().c_str()); + XLOGD_INFO("------------------------------------------------------------------------"); + + ctrlm_voice_iarm_call_voice_session_t v_params; + v_params.ieee_address = dqm->rcu_data.ieee_address; + + ctrlm_main_queue_msg_voice_session_t msg; + errno_t safec_rc = memset_s(&msg, sizeof(msg), 0, sizeof(msg)); + ERR_CHK(safec_rc); + msg.params = &v_params; + msg.voice_device = CTRLM_VOICE_DEVICE_MFV; + + // Open the voice session immediately so audio streams right away (draining the BLE + // pipe). Because this is an MFV wake-word session, the speech router holds its + // connect/init in the buffering state until the wake word stream parameters are + // supplied. Those arrive in a separate detection-data notification; a safety timeout + // releases the connect without them if they never come. + controller->setMfvDetectionPending(true); + req_process_voice_session_begin(&msg, sizeof(msg)); + + g_ctrlm_ble_network.mfv_detection_pending_ieee = dqm->rcu_data.ieee_address; + ctrlm_timeout_destroy(&g_ctrlm_ble_network.mfv_detection_timer_tag); + g_ctrlm_ble_network.mfv_detection_timer_tag = ctrlm_timeout_create(CTRLM_BLE_MFV_DETECTION_DATA_TIMEOUT, ctrlm_ble_mfv_detection_timer_cb, &id_); + } + break; + } + case CTRLM_HAL_BLE_PROPERTY_MFV_DETECTION_DATA: + controller->setMfvDetectionData(dqm->rcu_data.mfv_ww_start, dqm->rcu_data.mfv_ww_end, dqm->rcu_data.mfv_confidence); + XLOGD_INFO("Controller <%s> MFV detection data: start=%u end=%u confidence=%.1f%%", + controller->ieee_address_get().to_string().c_str(), dqm->rcu_data.mfv_ww_start, dqm->rcu_data.mfv_ww_end, dqm->rcu_data.mfv_confidence / 10.0); + + // If a wake-word session is waiting on this detection data, supply its wake word stream + // parameters to the voice engine and release the deferred connect so the vrex init is + // sent with them included. + if (controller->isMfvDetectionPending()) { + controller->setMfvDetectionPending(false); + ctrlm_timeout_destroy(&g_ctrlm_ble_network.mfv_detection_timer_tag); + // Confidence is encoded as percent * 10 (e.g. 991 = 99.1%); normalize to 0.0 - 1.0. + double confidence = dqm->rcu_data.mfv_confidence / 1000.0; + ctrlm_get_voice_obj()->voice_session_stream_params_update(CTRLM_VOICE_DEVICE_MFV, true, + dqm->rcu_data.mfv_ww_start, dqm->rcu_data.mfv_ww_end, confidence, 0.0); + } + break; + case CTRLM_HAL_BLE_PROPERTY_MFV_PRIVACY: + controller->setMfvPrivacy(dqm->rcu_data.mfv_privacy_enabled); + XLOGD_INFO("Controller <%s> MFV privacy = %s", controller->ieee_address_get().to_string().c_str(), dqm->rcu_data.mfv_privacy_enabled ? "enabled" : "disabled"); + break; + case CTRLM_HAL_BLE_PROPERTY_MFV_CAPABILITIES: + controller->setMfvCapabilities(dqm->rcu_data.mfv_capabilities); + XLOGD_INFO("Controller <%s> MFV capabilities = 0x%02X", controller->ieee_address_get().to_string().c_str(), static_cast(dqm->rcu_data.mfv_capabilities)); + break; default: XLOGD_WARN("Unhandled Property: %d !!!!!!!!!!!!!!!!!!!!!!!!", dqm->property_updated); break; @@ -2248,6 +2370,10 @@ ctrlm_controller_id_t ctrlm_obj_network_ble_t::controller_add(ctrlm_hal_ble_rcu_ if (rcu_data.wakeup_config != 0xFF) { controller->setWakeupConfig(rcu_data.wakeup_config); } if (rcu_data.wakeup_custom_list_size != 0) { controller->setWakeupCustomList(rcu_data.wakeup_custom_list, rcu_data.wakeup_custom_list_size); } if (rcu_data.last_wakeup_key != 0xFF) { controller->setLastWakeupKey(rcu_data.last_wakeup_key); } + controller->setMfvDetectionType(rcu_data.mfv_detection_type); + controller->setMfvDetectionData(rcu_data.mfv_ww_start, rcu_data.mfv_ww_end, rcu_data.mfv_confidence); + controller->setMfvPrivacy(rcu_data.mfv_privacy_enabled); + controller->setMfvCapabilities(rcu_data.mfv_capabilities); controller->db_store(); } @@ -2404,6 +2530,16 @@ void ctrlm_obj_network_ble_t::ind_process_keypress(void *data, int size) { if (key_status == CTRLM_KEY_STATUS_DOWN) { if (controller->isVoiceKey(dqm->event.code)) { + // The MFV remote also has a standard push-to-talk mic button. If a wake-word (MFV) session + // is still held waiting on its detection data when the button is pressed, release its deferred + // connect so it is not left stuck in the buffering state. + if (controller->isMfvDetectionPending()) { + XLOGD_INFO("Push-to-talk pressed while an MFV wake-word session was pending - releasing it for device: %s", controller->ieee_address_get().to_string().c_str()); + controller->setMfvDetectionPending(false); + ctrlm_timeout_destroy(&g_ctrlm_ble_network.mfv_detection_timer_tag); + ctrlm_get_voice_obj()->voice_session_stream_params_update(CTRLM_VOICE_DEVICE_MFV, false, 0, 0, 0.0, 0.0); + } + rdkx_timestamp_t keyDownTime; keyDownTime.tv_sec = dqm->event.time.tv_sec; keyDownTime.tv_nsec = dqm->event.time.tv_usec * 1000; diff --git a/src/ble/ctrlm_ble_network.h b/src/ble/ctrlm_ble_network.h index 73f2c8b9..b2a52f0f 100644 --- a/src/ble/ctrlm_ble_network.h +++ b/src/ble/ctrlm_ble_network.h @@ -146,6 +146,7 @@ class ctrlm_obj_network_ble_t : public ctrlm_obj_network_t { virtual void req_process_controller_status(void *data, int size); virtual void req_process_voice_session_begin(void *data, int size); + virtual void req_process_mfv_detection_timeout(void *data, int size); virtual void req_process_voice_session_end(void *data, int size); virtual void req_process_start_pairing(void *data, int size); @@ -224,6 +225,7 @@ class ctrlm_obj_network_ble_t : public ctrlm_obj_network_t { ctrlm_ble_unpair_metrics_t last_rcu_unpair_metrics_; int print_status_defer_count_ = 0; int event_status_defer_count_ = 0; + bool mfv_below_threshold_session_enabled_ = false; std::map controllers_; std::map upgrade_images_; diff --git a/src/ble/ctrlm_ble_rcu_interface.cpp b/src/ble/ctrlm_ble_rcu_interface.cpp index 36c9dbaf..23526e50 100644 --- a/src/ble/ctrlm_ble_rcu_interface.cpp +++ b/src/ble/ctrlm_ble_rcu_interface.cpp @@ -21,6 +21,7 @@ #include "ctrlm_ble_rcu_interface.h" #include "ctrlm_ble_utils.h" #include "ctrlm_voice_obj.h" +#include "blercu/bleservices/blercuaudioservice.h" #define CTRLM_BLE_KEY_MSG_QUEUE_MSG_MAX (10) @@ -490,6 +491,60 @@ bool ctrlm_ble_rcu_interface_t::handleAddedDevice(const BleAddress &address) device->addAdvConfigCustomListChangedSlot(Slot &>(m_isAlive, advConfigCustomListChangedSlot)); + if (auto audioSvc = device->audioService()) { + auto mfvDetectionTypeSlot = [this, address](BleRcuAudioService::DetectionType type) + { + XLOGD_INFO("BLE RCU %s MFV detection type = 0x%02X", address.toString().c_str(), static_cast(type)); + + ctrlm_hal_ble_RcuStatusData_t params = {}; + params.property_updated = CTRLM_HAL_BLE_PROPERTY_MFV_DETECTION_TYPE; + params.rcu_data.ieee_address = address.toUInt64(); + params.rcu_data.mfv_detection_type = static_cast(type); + m_rcuStatusChangedSlots.invoke(¶ms); + }; + audioSvc->addMfvDetectionTypeChangedSlot(Slot(m_isAlive, mfvDetectionTypeSlot)); + + auto mfvDetectionDataSlot = [this, address](const BleRcuAudioService::DetectionData &data) + { + XLOGD_INFO("BLE RCU %s MFV detection data: start=%u end=%u confidence=%.1f%%", + address.toString().c_str(), data.start, data.end, data.confidence / 10.0); + + ctrlm_hal_ble_RcuStatusData_t params = {}; + params.property_updated = CTRLM_HAL_BLE_PROPERTY_MFV_DETECTION_DATA; + params.rcu_data.ieee_address = address.toUInt64(); + params.rcu_data.mfv_ww_start = data.start; + params.rcu_data.mfv_ww_end = data.end; + params.rcu_data.mfv_confidence = data.confidence; + m_rcuStatusChangedSlots.invoke(¶ms); + }; + audioSvc->addMfvDetectionDataChangedSlot(Slot(m_isAlive, mfvDetectionDataSlot)); + + auto mfvPrivacySlot = [this, address](bool enabled) + { + XLOGD_INFO("BLE RCU %s MFV privacy = %s", address.toString().c_str(), enabled ? "enabled" : "disabled"); + + ctrlm_hal_ble_RcuStatusData_t params = {}; + params.property_updated = CTRLM_HAL_BLE_PROPERTY_MFV_PRIVACY; + params.rcu_data.ieee_address = address.toUInt64(); + params.rcu_data.mfv_privacy_enabled = enabled; + m_rcuStatusChangedSlots.invoke(¶ms); + }; + audioSvc->addMfvPrivacyChangedSlot(Slot(m_isAlive, mfvPrivacySlot)); + + auto mfvCapabilitiesSlot = [this, address](uint8_t caps) + { + XLOGD_INFO("BLE RCU %s MFV capabilities = 0x%02X", address.toString().c_str(), caps); + + ctrlm_hal_ble_RcuStatusData_t params = {}; + params.property_updated = CTRLM_HAL_BLE_PROPERTY_MFV_CAPABILITIES; + params.rcu_data.ieee_address = address.toUInt64(); + params.rcu_data.mfv_capabilities = caps; + m_rcuStatusChangedSlots.invoke(¶ms); + }; + audioSvc->addMfvCapabilitiesChangedSlot(Slot(m_isAlive, mfvCapabilitiesSlot)); + } + + auto audioStreamingChangedSlot = [this, address](bool streaming) { XLOGD_DEBUG("BLE RCU audio streaming changed to %s", streaming ? "TRUE" : "FALSE"); @@ -682,6 +737,17 @@ ctrlm_hal_ble_rcu_data_t ctrlm_ble_rcu_interface_t::getAllDeviceProperties(uint6 ret.audio_codecs = device->audioCodecs(); ret.audio_gain_level = device->audioGainLevel(); + // MFV data (part of the RDK Voice / audio service) + if (auto audioSvc = device->audioService()) { + auto detectionData = audioSvc->mfvDetectionData(); + ret.mfv_detection_type = static_cast(audioSvc->mfvDetectionType()); + ret.mfv_ww_start = detectionData.start; + ret.mfv_ww_end = detectionData.end; + ret.mfv_confidence = detectionData.confidence; + ret.mfv_privacy_enabled = audioSvc->mfvPrivacyEnabled(); + ret.mfv_capabilities = audioSvc->mfvCapabilities(); + } + return ret; } diff --git a/src/ble/hal/blercu/bleservices/blercuaudioservice.h b/src/ble/hal/blercu/bleservices/blercuaudioservice.h index bb1b02df..4339bc1d 100644 --- a/src/ble/hal/blercu/bleservices/blercuaudioservice.h +++ b/src/ble/hal/blercu/bleservices/blercuaudioservice.h @@ -36,6 +36,9 @@ #include "utils/audioformat.h" #include "ctrlm_hal.h" +#include +#include + class BleRcuAudioService { protected: @@ -81,6 +84,64 @@ class BleRcuAudioService virtual void status(uint32_t &lastError, uint32_t &expectedPackets, uint32_t &actualPackets, int32_t &voiceKeyHeldMs) = 0; +// MFV types +public: + enum DetectionType { + Unknown = 0x00, + FullPower = 0x01, + Aad = 0x02, + BelowThreshold = 0x03, + }; + + struct DetectionData { + uint16_t start = 0; + uint16_t end = 0; + uint16_t confidence = 0; // encoded as percentage * 10 (e.g. 953 = 95.3%) + }; + + struct ModelVersion { + uint8_t major = 0; + uint8_t minor = 0; + }; + + // TODO: All bytes are currently Reserved/TBD in the firmware spec, check to make sure this is still the case and delete if so + // struct StreamStatsRaw { + // std::vector bytes; + // }; + + enum Capabilities { + MidfieldVoiceCapable = (1 << 0), + SoftwarePrivacyControl = (1 << 1), + SowwEowwTimingAvailable = (1 << 2), + AadSensitivityControlAvailable = (1 << 3), + }; + +// MFV accessors (default no-ops for audio services without MFV support) +public: + virtual DetectionType mfvDetectionType() const { return Unknown; } + virtual DetectionData mfvDetectionData() const { return {}; } + virtual ModelVersion mfvModelVersion() const { return {}; } + virtual bool mfvPrivacyEnabled() const { return false; } + virtual std::vector mfvModelConfiguration() const { return {}; } + virtual uint8_t mfvCapabilities() const { return 0; } + // Not yet implemented: stream stats characteristic is Reserved/TBD in the firmware spec. + // virtual StreamStatsRaw mfvStreamStats() const { return {}; } + + virtual void writeMfvPrivacy(bool enabled, PendingReply<> &&reply) + { + (void)enabled; + reply.setError("MFV not supported"); + reply.finish(); + } + virtual void writeMfvModelConfiguration(uint8_t sensitivity, uint8_t secondary, uint8_t aad, PendingReply<> &&reply) + { + (void)sensitivity; + (void)secondary; + (void)aad; + reply.setError("MFV not supported"); + reply.finish(); + } + // signals: inline void addStreamingChangedSlot(const Slot &func) { @@ -95,10 +156,39 @@ class BleRcuAudioService m_audioCodecsChangedSlots.addSlot(func); } +// MFV signals: + inline void addMfvDetectionTypeChangedSlot(const Slot &func) + { + m_mfvDetectionTypeChangedSlots.addSlot(func); + } + inline void addMfvDetectionDataChangedSlot(const Slot &func) + { + m_mfvDetectionDataChangedSlots.addSlot(func); + } + inline void addMfvPrivacyChangedSlot(const Slot &func) + { + m_mfvPrivacyChangedSlots.addSlot(func); + } + inline void addMfvCapabilitiesChangedSlot(const Slot &func) + { + m_mfvCapabilitiesChangedSlots.addSlot(func); + } + // Not yet implemented: stream stats characteristic is Reserved/TBD in the firmware spec. + // inline void addMfvStreamStatsChangedSlot(const Slot &func) + // { + // m_mfvStreamStatsChangedSlots.addSlot(func); + // } + protected: Slots m_streamingChangedSlots; Slots m_gainLevelChangedSlots; Slots m_audioCodecsChangedSlots; + + Slots m_mfvDetectionTypeChangedSlots; + Slots m_mfvDetectionDataChangedSlots; + Slots m_mfvPrivacyChangedSlots; + Slots m_mfvCapabilitiesChangedSlots; + // Slots m_mfvStreamStatsChangedSlots; }; diff --git a/src/ble/hal/blercu/bleservices/gatt/gatt_audioservice_rdk.cpp b/src/ble/hal/blercu/bleservices/gatt/gatt_audioservice_rdk.cpp index 80897f10..6d6c9f59 100644 --- a/src/ble/hal/blercu/bleservices/gatt/gatt_audioservice_rdk.cpp +++ b/src/ble/hal/blercu/bleservices/gatt/gatt_audioservice_rdk.cpp @@ -35,6 +35,7 @@ #include "ctrlm_log_ble.h" +#include #include #include @@ -45,16 +46,32 @@ #define AUDIO_SEQ_NUM_MAX (0xFF) +// Number of async characteristic reads issued during MFV initialization (capabilities, model version, privacy, model configuration) +#define MFV_TOTAL_INITIAL_READS (4) +#define MFV_NOTIFY_RETRY_DELAY_MS (1000) +#define MFV_NOTIFY_RETRY_MAX_DELAY_MS (32000) + using namespace std; const BleUuid GattAudioServiceRdk::m_serviceUuid(BleUuid::RdkVoice); +const BleUuid GattAudioServiceRdk::m_mfvSessionStartCharUuid(BleUuid::SessionStart); +const BleUuid GattAudioServiceRdk::m_mfvDetectionDataCharUuid(BleUuid::DetectionData); +const BleUuid GattAudioServiceRdk::m_mfvModelVersionCharUuid(BleUuid::WakeWordModelVersion); +const BleUuid GattAudioServiceRdk::m_mfvPrivacyCharUuid(BleUuid::PrivacySettings); +const BleUuid GattAudioServiceRdk::m_mfvModelConfigCharUuid(BleUuid::ModelConfiguration); +const BleUuid GattAudioServiceRdk::m_mfvCapabilitiesCharUuid(BleUuid::MfvCapabilities); + GattAudioServiceRdk::GattAudioServiceRdk(GMainLoop* mainLoop) : GattAudioService(AUDIO_FRAME_SIZE, 5, AUDIO_PACKET_DURATION_USEC, AUDIO_SEQ_NUM_MAX, mainLoop) { } GattAudioServiceRdk::~GattAudioServiceRdk() { + if (m_mfvNotifyRetryTimer > 0) { + g_source_remove(m_mfvNotifyRetryTimer); + m_mfvNotifyRetryTimer = 0; + } } // ----------------------------------------------------------------------------- @@ -75,6 +92,8 @@ BleUuid GattAudioServiceRdk::uuid() */ bool GattAudioServiceRdk::start(const shared_ptr &gattService) { + m_mfvState.reset(); + // sanity check the supplied info is valid if (!gattService->isValid() || (gattService->uuid() != m_serviceUuid)) { XLOGD_WARN("invalid voice gatt service info"); @@ -97,6 +116,25 @@ bool GattAudioServiceRdk::start(const shared_ptr &gattServ requestGainLevel(); requestAudioCodecs(); + // Discover MFV characteristics (optional - not all remotes support MFV) + if (getMfvCharacteristics(gattService)) { + m_mfvState.supported = true; + // Set the total count before issuing any reads so that early or synchronous + // completion of one read cannot prematurely trigger onMfvInitialReadComplete(). + m_mfvState.initialReadsRemaining = MFV_TOTAL_INITIAL_READS; + requestMfvCapabilities(); + requestMfvModelVersion(); + requestMfvPrivacy(); + requestMfvModelConfig(); + + // Enable the wake-word notifications (Session Start + Detection Data) as early as possible - + // right after the characteristics are discovered - rather than waiting for the initial value + // reads above to complete. Subscribing does not depend on those values, and enabling early + // means wake-word detection data can be received while the service is idle/ready, without first + // having to start an audio stream. + maybeEnableMfvNotifications(); + } + return(GattAudioService::start(gattService)); } @@ -111,6 +149,12 @@ void GattAudioServiceRdk::stop() } void GattAudioServiceRdk::onEnteredIdle() { + m_mfvState.reset(); + if (m_mfvNotifyRetryTimer > 0) { + g_source_remove(m_mfvNotifyRetryTimer); + m_mfvNotifyRetryTimer = 0; + } + if (m_audioDataCharacteristic) { XLOGD_INFO("Disabling notifications for m_audioDataCharacteristic"); m_audioDataCharacteristic->disableNotifications(); @@ -124,6 +168,37 @@ void GattAudioServiceRdk::onEnteredIdle() { m_audioCtrlCharacteristic.reset(); m_audioDataCharacteristic.reset(); + // Clean up MFV characteristics + if (m_mfvSessionStartCharacteristic) { + m_mfvSessionStartCharacteristic->disableNotifications(); + m_mfvSessionStartCharacteristic.reset(); + } + if (m_mfvDetectionDataCharacteristic) { + m_mfvDetectionDataCharacteristic->disableNotifications(); + m_mfvDetectionDataCharacteristic.reset(); + } + if (m_mfvPrivacyCharacteristic) { + m_mfvPrivacyCharacteristic->disableNotifications(); + m_mfvPrivacyCharacteristic.reset(); + } + m_mfvModelVersionCharacteristic.reset(); + m_mfvModelConfigCharacteristic.reset(); + m_mfvCapabilitiesCharacteristic.reset(); + if (m_mfvPromiseResults) { + m_mfvPromiseResults->setError("MFV operation cancelled"); + m_mfvPromiseResults->finish(); + m_mfvPromiseResults.reset(); + } + + // Reset cached MFV values so callers don't observe stale data after stop/disconnect. + m_mfvDetectionType = Unknown; + m_mfvDetectionData = {}; + m_mfvModelVersionData = {}; + m_mfvPrivacyEnabled = false; + m_mfvModelConfigurationData.clear(); + m_mfvCapabilitiesValue = 0; + // m_mfvStreamStatsData = {}; + GattAudioService::onEnteredIdle(); } @@ -150,6 +225,8 @@ void GattAudioServiceRdk::onEnteredEnableNotificationsState() stateMachinePostEvent(GattErrorEvent); } else { + maybeEnableMfvNotifications(); + // notifications enabled so post an event to the state machine stateMachinePostEvent(NotificationsEnabledEvent); } @@ -173,6 +250,8 @@ void GattAudioServiceRdk::onEnteredEnableNotificationsState() } if (m_audioDataCharacteristic->notificationsEnabled()) { + maybeEnableMfvNotifications(); + // notifications already enabled so post an event to the state machine stateMachinePostEvent(NotificationsEnabledEvent); } else { @@ -596,3 +675,653 @@ void GattAudioServiceRdk::setGainLevel(uint8_t level) XLOGD_INFO("sending gain level = %u", level); m_audioGainCharacteristic->writeValue(value, PendingReply<>(getIsAlivePtr(), replyHandler)); } + + +// ============================================================================= +// MFV implementation +// ============================================================================= + +// ----------------------------------------------------------------------------- +/*! + \internal + + Attempts to discover all MFV characteristics from the RDK Voice GATT service. + Returns true if all were found, false otherwise. + */ +bool GattAudioServiceRdk::getMfvCharacteristics(const shared_ptr &gattService) +{ + // TODO: Figure out if some characteristics are optional and if so, which ones. For now, require all of them to be present. + + // Reset any previous discovery result so failure paths are unambiguous. + m_mfvSessionStartCharacteristic.reset(); + m_mfvDetectionDataCharacteristic.reset(); + m_mfvModelVersionCharacteristic.reset(); + m_mfvPrivacyCharacteristic.reset(); + m_mfvModelConfigCharacteristic.reset(); + m_mfvCapabilitiesCharacteristic.reset(); + + auto mfvSessionStartCharacteristic = gattService->characteristic(m_mfvSessionStartCharUuid); + if (!mfvSessionStartCharacteristic || !mfvSessionStartCharacteristic->isValid()) { + XLOGD_INFO("MFV Session Start characteristic not found (MFV not supported)"); + return false; + } + + auto mfvDetectionDataCharacteristic = gattService->characteristic(m_mfvDetectionDataCharUuid); + if (!mfvDetectionDataCharacteristic || !mfvDetectionDataCharacteristic->isValid()) { + XLOGD_INFO("MFV Detection Data characteristic not found"); + return false; + } + + auto mfvModelVersionCharacteristic = gattService->characteristic(m_mfvModelVersionCharUuid); + if (!mfvModelVersionCharacteristic || !mfvModelVersionCharacteristic->isValid()) { + XLOGD_INFO("MFV Wake Word Model Version characteristic not found"); + return false; + } + + auto mfvPrivacyCharacteristic = gattService->characteristic(m_mfvPrivacyCharUuid); + if (!mfvPrivacyCharacteristic || !mfvPrivacyCharacteristic->isValid()) { + XLOGD_INFO("MFV Privacy Settings characteristic not found"); + return false; + } + + auto mfvModelConfigCharacteristic = gattService->characteristic(m_mfvModelConfigCharUuid); + if (!mfvModelConfigCharacteristic || !mfvModelConfigCharacteristic->isValid()) { + XLOGD_INFO("MFV Model Configuration characteristic not found"); + return false; + } + + auto mfvCapabilitiesCharacteristic = gattService->characteristic(m_mfvCapabilitiesCharUuid); + if (!mfvCapabilitiesCharacteristic || !mfvCapabilitiesCharacteristic->isValid()) { + XLOGD_INFO("MFV Capabilities characteristic not found"); + return false; + } + + m_mfvSessionStartCharacteristic = std::move(mfvSessionStartCharacteristic); + m_mfvDetectionDataCharacteristic = std::move(mfvDetectionDataCharacteristic); + m_mfvModelVersionCharacteristic = std::move(mfvModelVersionCharacteristic); + m_mfvPrivacyCharacteristic = std::move(mfvPrivacyCharacteristic); + m_mfvModelConfigCharacteristic = std::move(mfvModelConfigCharacteristic); + m_mfvCapabilitiesCharacteristic = std::move(mfvCapabilitiesCharacteristic); + + XLOGD_INFO("All MFV characteristics discovered successfully"); + return true; +} + +// ----------------------------------------------------------------------------- +/*! + \internal + + Called when the last MFV initial read completes. Enables MFV notifications + and fires the changed slots with initial values. + */ +void GattAudioServiceRdk::onMfvInitialReadComplete() +{ + // Publish only values that were successfully read during initialization. + if (m_mfvState.capabilitiesReadValid) { + m_mfvCapabilitiesChangedSlots.invoke(m_mfvCapabilitiesValue); + } + if (m_mfvState.privacyReadValid) { + m_mfvPrivacyChangedSlots.invoke(m_mfvPrivacyEnabled); + } + + // Attempt to enable MFV notifications once initial reads are complete. + // This may run before audio notifications are enabled, which is acceptable. + maybeEnableMfvNotifications(); +} + +void GattAudioServiceRdk::maybeEnableMfvNotifications() +{ + // Notification enabling is intentionally NOT gated on the initial value reads (capabilities / + // model version / privacy / model config). Subscribing to the Session Start and Detection Data + // notifications does not depend on those values, so enabling them is decoupled from the reads and + // can happen as soon as the characteristics are discovered. This lets the remote deliver wake-word + // detection data while the service is idle/ready, without first having to start an audio stream. + if (!m_mfvState.supported) { + return; + } + + // If a backoff retry timer is pending, don't bypass it — let the timer drive the next + // attempt. The timer callback zeroes m_mfvNotifyRetryTimer before calling back here, + // so timer-driven retries are not affected by this guard. + if (m_mfvNotifyRetryTimer > 0) { + return; + } + + // Only request notification enables that are not already enabled. + if (m_mfvSessionStartCharacteristic && + !m_mfvSessionStartCharacteristic->notificationsEnabled() && + !m_mfvState.sessionStartNotifyRequested) { + m_mfvState.sessionStartNotifyRequested = true; + requestStartMfvSessionStartNotify(); + } + if (m_mfvDetectionDataCharacteristic && + !m_mfvDetectionDataCharacteristic->notificationsEnabled() && + !m_mfvState.detectionDataNotifyRequested) { + m_mfvState.detectionDataNotifyRequested = true; + requestStartMfvDetectionDataNotify(); + } + if (m_mfvPrivacyCharacteristic && + !m_mfvPrivacyCharacteristic->notificationsEnabled() && + !m_mfvState.privacyNotifyRequested) { + m_mfvState.privacyNotifyRequested = true; + requestStartMfvPrivacyNotify(); + } + + // Mark enabled once all required notifications are enabled. Privacy is optional here + // if its initial read did not complete successfully. + if (m_mfvSessionStartCharacteristic && m_mfvDetectionDataCharacteristic && + m_mfvSessionStartCharacteristic->notificationsEnabled() && + m_mfvDetectionDataCharacteristic->notificationsEnabled() && + (!m_mfvState.privacyReadValid || (m_mfvPrivacyCharacteristic && m_mfvPrivacyCharacteristic->notificationsEnabled()))) { + m_mfvState.notificationsEnabled = true; + } +} + +void GattAudioServiceRdk::scheduleMfvNotifyRetry() +{ + if (!m_mfvState.supported || (m_mfvNotifyRetryTimer > 0)) { + return; + } + + // Exponential backoff: base * 2^(retryCount-1), capped at MFV_NOTIFY_RETRY_MAX_DELAY_MS. + // Use the max retry count across all MFV characteristics as the backoff index. + unsigned int retryCount = std::max({m_mfvState.sessionStartNotifyRetries, + m_mfvState.detectionDataNotifyRetries, + m_mfvState.privacyNotifyRetries}); + unsigned int shift = (retryCount > 0) ? std::min(retryCount - 1u, 5u) : 0u; + guint delay = std::min((guint)(MFV_NOTIFY_RETRY_DELAY_MS << shift), (guint)MFV_NOTIFY_RETRY_MAX_DELAY_MS); + + XLOGD_DEBUG("scheduling MFV notify retry in %u ms (retry count %u)", delay, retryCount); + + m_mfvNotifyRetryTimer = g_timeout_add(delay, + [](gpointer userData) -> gboolean { + auto *self = static_cast(userData); + self->m_mfvNotifyRetryTimer = 0; + self->maybeEnableMfvNotifications(); + return G_SOURCE_REMOVE; + }, this); +} + +// ----------------------------------------------------------------------------- +/*! + \internal + + Reads the MFV Capabilities characteristic (1 byte bitmask). + */ +void GattAudioServiceRdk::requestMfvCapabilities() +{ + auto replyHandler = [this](PendingReply> *reply) + { + if (!m_mfvState.supported || (m_mfvState.initialReadsRemaining <= 0)) { + XLOGD_DEBUG("ignoring late MFV Capabilities read callback (MFV inactive)"); + return; + } + + if (reply->isError()) { + XLOGD_ERROR("failed to read MFV Capabilities due to <%s>", reply->errorMessage().c_str()); + } else { + std::vector value = reply->result(); + if (value.size() == 1) { + m_mfvCapabilitiesValue = value[0]; + m_mfvState.capabilitiesReadValid = true; + XLOGD_INFO("MFV Capabilities = 0x%02X (midfield=%d privacy_ctrl=%d soww_eoww=%d aad_ctrl=%d)", + m_mfvCapabilitiesValue, + !!(m_mfvCapabilitiesValue & MidfieldVoiceCapable), + !!(m_mfvCapabilitiesValue & SoftwarePrivacyControl), + !!(m_mfvCapabilitiesValue & SowwEowwTimingAvailable), + !!(m_mfvCapabilitiesValue & AadSensitivityControlAvailable)); + } else { + XLOGD_ERROR("MFV Capabilities has invalid length (%zu bytes, expected 1)", value.size()); + } + } + if (--m_mfvState.initialReadsRemaining == 0) { + onMfvInitialReadComplete(); + } + }; + + m_mfvCapabilitiesCharacteristic->readValue(PendingReply>(getIsAlivePtr(), replyHandler)); +} + +// ----------------------------------------------------------------------------- +/*! + \internal + + Reads the Wake Word Model Version characteristic (2 bytes: major, minor). + */ +void GattAudioServiceRdk::requestMfvModelVersion() +{ + auto replyHandler = [this](PendingReply> *reply) + { + if (!m_mfvState.supported || (m_mfvState.initialReadsRemaining <= 0)) { + XLOGD_DEBUG("ignoring late MFV Model Version read callback (MFV inactive)"); + return; + } + + if (reply->isError()) { + XLOGD_ERROR("failed to read MFV Model Version due to <%s>", reply->errorMessage().c_str()); + } else { + std::vector value = reply->result(); + if (value.size() == 2) { + m_mfvModelVersionData.major = value[0]; + m_mfvModelVersionData.minor = value[1]; + m_mfvState.modelVersionReadValid = true; + XLOGD_INFO("MFV Wake Word Model Version = %u.%u", m_mfvModelVersionData.major, m_mfvModelVersionData.minor); + } else { + XLOGD_ERROR("MFV Model Version has invalid length (%zu bytes, expected 2)", value.size()); + } + } + if (--m_mfvState.initialReadsRemaining == 0) { + onMfvInitialReadComplete(); + } + }; + + m_mfvModelVersionCharacteristic->readValue(PendingReply>(getIsAlivePtr(), replyHandler)); +} + +// ----------------------------------------------------------------------------- +/*! + \internal + + Reads the Privacy Settings characteristic (1 byte boolean). + */ +void GattAudioServiceRdk::requestMfvPrivacy() +{ + auto replyHandler = [this](PendingReply> *reply) + { + if (!m_mfvState.supported || (m_mfvState.initialReadsRemaining <= 0)) { + XLOGD_DEBUG("ignoring late MFV Privacy read callback (MFV inactive)"); + return; + } + + if (reply->isError()) { + XLOGD_ERROR("failed to read MFV Privacy due to <%s>", reply->errorMessage().c_str()); + } else { + std::vector value = reply->result(); + if (value.size() == 1) { + m_mfvPrivacyEnabled = (value[0] != 0); + m_mfvState.privacyReadValid = true; + XLOGD_INFO("MFV Privacy = %s", m_mfvPrivacyEnabled ? "enabled" : "disabled"); + } else { + XLOGD_ERROR("MFV Privacy has invalid length (%zu bytes, expected 1)", value.size()); + } + } + if (--m_mfvState.initialReadsRemaining == 0) { + onMfvInitialReadComplete(); + } + }; + + m_mfvPrivacyCharacteristic->readValue(PendingReply>(getIsAlivePtr(), replyHandler)); +} + +// ----------------------------------------------------------------------------- +/*! + \internal + + Reads the Model Configuration characteristic (3 bytes: sensitivity, + secondary sensitivity, AAD sensitivity). + */ +void GattAudioServiceRdk::requestMfvModelConfig() +{ + auto replyHandler = [this](PendingReply> *reply) + { + if (!m_mfvState.supported || (m_mfvState.initialReadsRemaining <= 0)) { + XLOGD_DEBUG("ignoring late MFV Model Config read callback (MFV inactive)"); + return; + } + + if (reply->isError()) { + XLOGD_ERROR("failed to read MFV Model Config due to <%s>", reply->errorMessage().c_str()); + } else { + std::vector value = reply->result(); + if (value.size() == 3) { + m_mfvModelConfigurationData = value; + m_mfvState.modelConfigReadValid = true; + XLOGD_INFO("MFV Model Config: sensitivity=%u secondary=%u aad=%u", + value[0], value[1], value[2]); + } else { + XLOGD_ERROR("MFV Model Config has invalid length (%zu bytes, expected 3)", value.size()); + } + } + if (--m_mfvState.initialReadsRemaining == 0) { + onMfvInitialReadComplete(); + } + }; + + m_mfvModelConfigCharacteristic->readValue(PendingReply>(getIsAlivePtr(), replyHandler)); +} + +// ----------------------------------------------------------------------------- +/*! + \internal + + Enables notifications on the Session Start characteristic (0xEA08). + */ +void GattAudioServiceRdk::requestStartMfvSessionStartNotify() +{ + auto replyHandler = [this](PendingReply<> *reply) + { + if (reply->isError()) { + m_mfvState.sessionStartNotifyRequested = false; + ++m_mfvState.sessionStartNotifyRetries; + + if (reply->errorMessage().find("Notify acquired") != std::string::npos) { + XLOGD_WARN("MFV Session Start notify returned 'Notify acquired' but notification state is still disabled; retrying (attempt %u)", + m_mfvState.sessionStartNotifyRetries); + } else { + XLOGD_ERROR("failed to enable MFV Session Start notifications due to <%s> (retry attempt %u)", + reply->errorMessage().c_str(), m_mfvState.sessionStartNotifyRetries); + } + scheduleMfvNotifyRetry(); + } else { + m_mfvState.sessionStartNotifyRetries = 0; + XLOGD_INFO("MFV Session Start notifications enabled successfully"); + maybeEnableMfvNotifications(); + } + }; + + m_mfvSessionStartCharacteristic->enableNotifications( + Slot &>(getIsAlivePtr(), + std::bind(&GattAudioServiceRdk::onMfvSessionStartChanged, this, std::placeholders::_1)), + PendingReply<>(getIsAlivePtr(), replyHandler)); +} + +// ----------------------------------------------------------------------------- +/*! + \internal + + Enables notifications on the Detection Data characteristic (0xEA09). + */ +void GattAudioServiceRdk::requestStartMfvDetectionDataNotify() +{ + auto replyHandler = [this](PendingReply<> *reply) + { + if (reply->isError()) { + m_mfvState.detectionDataNotifyRequested = false; + ++m_mfvState.detectionDataNotifyRetries; + + if (reply->errorMessage().find("Notify acquired") != std::string::npos) { + XLOGD_WARN("MFV Detection Data notify returned 'Notify acquired' but notification state is still disabled; retrying (attempt %u)", + m_mfvState.detectionDataNotifyRetries); + } else { + XLOGD_ERROR("failed to enable MFV Detection Data notifications due to <%s> (retry attempt %u)", + reply->errorMessage().c_str(), m_mfvState.detectionDataNotifyRetries); + } + scheduleMfvNotifyRetry(); + } else { + m_mfvState.detectionDataNotifyRetries = 0; + XLOGD_INFO("MFV Detection Data notifications enabled successfully"); + maybeEnableMfvNotifications(); + } + }; + + m_mfvDetectionDataCharacteristic->enableNotifications( + Slot &>(getIsAlivePtr(), + std::bind(&GattAudioServiceRdk::onMfvDetectionDataChanged, this, std::placeholders::_1)), + PendingReply<>(getIsAlivePtr(), replyHandler)); +} + +// ----------------------------------------------------------------------------- +/*! + \internal + + Enables notifications on the Privacy Settings characteristic (0xEA0B). + */ +void GattAudioServiceRdk::requestStartMfvPrivacyNotify() +{ + auto replyHandler = [this](PendingReply<> *reply) + { + if (reply->isError()) { + m_mfvState.privacyNotifyRequested = false; + ++m_mfvState.privacyNotifyRetries; + + if (reply->errorMessage().find("Notify acquired") != std::string::npos) { + XLOGD_WARN("MFV Privacy notify returned 'Notify acquired' but notification state is still disabled; retrying (attempt %u)", + m_mfvState.privacyNotifyRetries); + } else { + XLOGD_ERROR("failed to enable MFV Privacy notifications due to <%s> (retry attempt %u)", + reply->errorMessage().c_str(), m_mfvState.privacyNotifyRetries); + } + scheduleMfvNotifyRetry(); + } else { + m_mfvState.privacyNotifyRetries = 0; + XLOGD_DEBUG("MFV Privacy notifications enabled successfully"); + maybeEnableMfvNotifications(); + } + }; + + m_mfvPrivacyCharacteristic->enableNotifications( + Slot &>(getIsAlivePtr(), + std::bind(&GattAudioServiceRdk::onMfvPrivacyChanged, this, std::placeholders::_1)), + PendingReply<>(getIsAlivePtr(), replyHandler)); +} + +// ----------------------------------------------------------------------------- +/*! + \internal + + Called when a Session Start notification is received from the RCU. + Payload: 1 byte detection type (0x01=FullPower, 0x02=AAD, 0x03=BelowThreshold). + */ +void GattAudioServiceRdk::onMfvSessionStartChanged(const std::vector &newValue) +{ + if (newValue.size() != 1) { + XLOGD_ERROR("MFV Session Start notification has invalid length (%zu bytes, expected 1)", newValue.size()); + return; + } + + const uint8_t raw = newValue[0]; + if (raw < 0x01 || raw > 0x03) { + XLOGD_WARN("MFV Session Start notification has unknown detection type 0x%02X", raw); + return; + } + + m_mfvDetectionType = static_cast(raw); + m_mfvDetectionTypeChangedSlots.invoke(m_mfvDetectionType); + XLOGD_INFO("MFV Session Start: detection type = 0x%02X", raw); +} + +// ----------------------------------------------------------------------------- +/*! + \internal + + Called when a Detection Data notification is received from the RCU. + Payload: 6 bytes, little-endian (start, end, confidence). + */ +void GattAudioServiceRdk::onMfvDetectionDataChanged(const std::vector &newValue) +{ + if (newValue.size() != 6) { + XLOGD_ERROR("MFV Detection Data notification has invalid length (%zu bytes, expected 6)", newValue.size()); + return; + } + + m_mfvDetectionData.start = static_cast(newValue[0]) | (static_cast(newValue[1]) << 8); + m_mfvDetectionData.end = static_cast(newValue[2]) | (static_cast(newValue[3]) << 8); + m_mfvDetectionData.confidence = static_cast(newValue[4]) | (static_cast(newValue[5]) << 8); + + XLOGD_INFO("MFV Detection Data: start=%u end=%u confidence=%.1f%%", + m_mfvDetectionData.start, m_mfvDetectionData.end, + m_mfvDetectionData.confidence / 10.0); + + m_mfvDetectionDataChangedSlots.invoke(m_mfvDetectionData); +} + +// ----------------------------------------------------------------------------- +/*! + \internal + + Called when a Privacy Settings notification is received from the RCU. + Payload: 1 byte boolean (1 = enabled, 0 = disabled). + */ +void GattAudioServiceRdk::onMfvPrivacyChanged(const std::vector &newValue) +{ + if (newValue.size() != 1) { + XLOGD_ERROR("MFV Privacy notification has invalid length (%zu bytes, expected 1)", newValue.size()); + return; + } + + m_mfvPrivacyEnabled = (newValue[0] != 0); + XLOGD_INFO("MFV Privacy changed: %s", m_mfvPrivacyEnabled ? "enabled" : "disabled"); + + m_mfvPrivacyChangedSlots.invoke(m_mfvPrivacyEnabled); +} + +// ============================================================================= +// MFV accessors +// ============================================================================= + +BleRcuAudioService::DetectionType GattAudioServiceRdk::mfvDetectionType() const +{ + return m_mfvDetectionType; +} + +BleRcuAudioService::DetectionData GattAudioServiceRdk::mfvDetectionData() const +{ + return m_mfvDetectionData; +} + +BleRcuAudioService::ModelVersion GattAudioServiceRdk::mfvModelVersion() const +{ + return m_mfvModelVersionData; +} + +bool GattAudioServiceRdk::mfvPrivacyEnabled() const +{ + return m_mfvPrivacyEnabled; +} + +std::vector GattAudioServiceRdk::mfvModelConfiguration() const +{ + return m_mfvModelConfigurationData; +} + +uint8_t GattAudioServiceRdk::mfvCapabilities() const +{ + return m_mfvCapabilitiesValue; +} + +// BleRcuAudioService::StreamStatsRaw GattAudioServiceRdk::mfvStreamStats() const +// { +// return m_mfvStreamStatsData; +// } + +// ============================================================================= +// MFV write handlers +// ============================================================================= + +void GattAudioServiceRdk::writeMfvPrivacy(bool enabled, PendingReply<> &&reply) +{ + if (!m_mfvState.supported) { + reply.setError("MFV not supported on this remote"); + reply.finish(); + return; + } + + if (m_mfvPromiseResults) { + reply.setError("MFV write already in progress"); + reply.finish(); + return; + } + + if (!m_mfvPrivacyCharacteristic || !m_mfvPrivacyCharacteristic->isValid()) { + reply.setError("MFV Privacy characteristic is not valid"); + reply.finish(); + return; + } + + m_mfvPromiseResults = make_shared>(std::move(reply)); + + const vector value(1, enabled ? 0x01 : 0x00); + XLOGD_INFO("Writing MFV Privacy = %s", enabled ? "enabled" : "disabled"); + + m_mfvPrivacyCharacteristic->writeValue(value, PendingReply<>(getIsAlivePtr(), + [this, enabled](PendingReply<> *reply) { + if (m_mfvState.supported && m_mfvPromiseResults && !reply->isError()) { + m_mfvPrivacyEnabled = enabled; + m_mfvState.privacyReadValid = true; + } + this->onWriteMfvPrivacyReply(reply); + })); +} + +void GattAudioServiceRdk::onWriteMfvPrivacyReply(PendingReply<> *reply) +{ + if (reply->isError()) { + XLOGD_ERROR("failed to write MFV Privacy due to <%s>", reply->errorMessage().c_str()); + if (m_mfvPromiseResults) { + m_mfvPromiseResults->setError(reply->errorMessage()); + m_mfvPromiseResults->finish(); + m_mfvPromiseResults.reset(); + } + } else { + XLOGD_INFO("MFV Privacy written successfully"); + m_mfvPrivacyChangedSlots.invoke(m_mfvPrivacyEnabled); + if (m_mfvPromiseResults) { + m_mfvPromiseResults->finish(); + m_mfvPromiseResults.reset(); + } + } +} + +void GattAudioServiceRdk::writeMfvModelConfiguration(uint8_t sensitivity, uint8_t secondary, uint8_t aad, PendingReply<> &&reply) +{ + if (!m_mfvState.supported) { + reply.setError("MFV not supported on this remote"); + reply.finish(); + return; + } + + if (m_mfvPromiseResults) { + reply.setError("MFV write already in progress"); + reply.finish(); + return; + } + + if (!m_mfvModelConfigCharacteristic || !m_mfvModelConfigCharacteristic->isValid()) { + reply.setError("MFV Model Configuration characteristic is not valid"); + reply.finish(); + return; + } + + if (aad != 0 && (aad < 60 || aad > 95 || (aad % 5) != 0)) { + XLOGD_ERROR("MFV AAD sensitivity %u is out of valid range [60..95 step 5]", aad); + reply.setError("AAD sensitivity out of valid range [60..95 step 5]"); + reply.finish(); + return; + } + + auto previousConfig = m_mfvModelConfigurationData; + auto previousValid = m_mfvState.modelConfigReadValid; + + m_mfvPromiseResults = make_shared>(std::move(reply)); + + const vector value = { sensitivity, secondary, aad }; + XLOGD_INFO("Writing MFV Model Config: sensitivity=%u secondary=%u aad=%u", + sensitivity, secondary, aad); + + m_mfvModelConfigCharacteristic->writeValue(value, PendingReply<>(getIsAlivePtr(), + [this, value, previousConfig, previousValid](PendingReply<> *writeReply) mutable { + if (writeReply->isError()) { + XLOGD_ERROR("failed to write MFV Model Config due to <%s>", writeReply->errorMessage().c_str()); + + // Restore last-known-good config on failure. + m_mfvModelConfigurationData = previousConfig; + m_mfvState.modelConfigReadValid = previousValid; + + if (m_mfvPromiseResults) { + m_mfvPromiseResults->setError(writeReply->errorMessage()); + m_mfvPromiseResults->finish(); + m_mfvPromiseResults.reset(); + } + } else { + XLOGD_INFO("MFV Model Config written successfully"); + + m_mfvModelConfigurationData = value; + m_mfvState.modelConfigReadValid = true; + + if (m_mfvPromiseResults) { + m_mfvPromiseResults->finish(); + m_mfvPromiseResults.reset(); + } + } + })); +} diff --git a/src/ble/hal/blercu/bleservices/gatt/gatt_audioservice_rdk.h b/src/ble/hal/blercu/bleservices/gatt/gatt_audioservice_rdk.h index f0662604..c74c68c4 100644 --- a/src/ble/hal/blercu/bleservices/gatt/gatt_audioservice_rdk.h +++ b/src/ble/hal/blercu/bleservices/gatt/gatt_audioservice_rdk.h @@ -67,6 +67,18 @@ class GattAudioServiceRdk : public GattAudioService uint8_t gainLevel() const override; void setGainLevel(uint8_t level) override; + // MFV accessors + DetectionType mfvDetectionType() const override; + DetectionData mfvDetectionData() const override; + ModelVersion mfvModelVersion() const override; + bool mfvPrivacyEnabled() const override; + std::vector mfvModelConfiguration() const override; + uint8_t mfvCapabilities() const override; + //StreamStatsRaw mfvStreamStats() const override; + + void writeMfvPrivacy(bool enabled, PendingReply<> &&reply) override; + void writeMfvModelConfiguration(uint8_t sensitivity, uint8_t secondary, uint8_t aad, PendingReply<> &&reply) override; + private: bool getAudioCodecsCharacteristic(const std::shared_ptr &gattService); @@ -75,6 +87,31 @@ class GattAudioServiceRdk : public GattAudioService bool getAudioDataCharacteristic(const std::shared_ptr &gattService); bool getAudioInfoCharacteristic(const std::shared_ptr &gattService); + // MFV characteristic discovery + bool getMfvCharacteristics(const std::shared_ptr &gattService); + + // MFV initial reads + void requestMfvCapabilities(); + void requestMfvModelVersion(); + void requestMfvPrivacy(); + void requestMfvModelConfig(); + void onMfvInitialReadComplete(); + void maybeEnableMfvNotifications(); + void scheduleMfvNotifyRetry(); + + // MFV notification enable + void requestStartMfvSessionStartNotify(); + void requestStartMfvDetectionDataNotify(); + void requestStartMfvPrivacyNotify(); + + // MFV notification handlers + void onMfvSessionStartChanged(const std::vector &newValue); + void onMfvDetectionDataChanged(const std::vector &newValue); + void onMfvPrivacyChanged(const std::vector &newValue); + + // MFV write handlers + void onWriteMfvPrivacyReply(PendingReply<> *reply); + void requestGainLevel(); void requestAudioCodecs(); @@ -89,6 +126,79 @@ class GattAudioServiceRdk : public GattAudioService std::shared_ptr m_audioInfoCharacteristic; std::shared_ptr m_audioCodecsCharacteristic; + struct MfvState { + bool supported = false; + int initialReadsRemaining = 0; + bool notificationsEnabled = false; + + bool capabilitiesReadValid = false; + bool modelVersionReadValid = false; + bool privacyReadValid = false; + bool modelConfigReadValid = false; + + bool sessionStartNotifyRequested = false; + bool detectionDataNotifyRequested = false; + bool privacyNotifyRequested = false; + + unsigned int sessionStartNotifyRetries = 0; + unsigned int detectionDataNotifyRetries = 0; + unsigned int privacyNotifyRetries = 0; + + void reset() + { + supported = false; + initialReadsRemaining = 0; + notificationsEnabled = false; + + capabilitiesReadValid = false; + modelVersionReadValid = false; + privacyReadValid = false; + modelConfigReadValid = false; + + sessionStartNotifyRequested = false; + detectionDataNotifyRequested = false; + privacyNotifyRequested = false; + + sessionStartNotifyRetries = 0; + detectionDataNotifyRetries = 0; + privacyNotifyRetries = 0; + } + + bool areInitialReadsComplete() const + { + return supported && (initialReadsRemaining == 0); + } + }; + + // MFV characteristics and state + MfvState m_mfvState; + + DetectionType m_mfvDetectionType = Unknown; + DetectionData m_mfvDetectionData; + ModelVersion m_mfvModelVersionData; + bool m_mfvPrivacyEnabled = false; + std::vector m_mfvModelConfigurationData; + uint8_t m_mfvCapabilitiesValue = 0; + // StreamStatsRaw m_mfvStreamStatsData; + + std::shared_ptr> m_mfvPromiseResults; + + std::shared_ptr m_mfvSessionStartCharacteristic; + std::shared_ptr m_mfvDetectionDataCharacteristic; + std::shared_ptr m_mfvModelVersionCharacteristic; + std::shared_ptr m_mfvPrivacyCharacteristic; + std::shared_ptr m_mfvModelConfigCharacteristic; + std::shared_ptr m_mfvCapabilitiesCharacteristic; + + unsigned int m_mfvNotifyRetryTimer = 0; + + static const BleUuid m_mfvSessionStartCharUuid; + static const BleUuid m_mfvDetectionDataCharUuid; + static const BleUuid m_mfvModelVersionCharUuid; + static const BleUuid m_mfvPrivacyCharUuid; + static const BleUuid m_mfvModelConfigCharUuid; + static const BleUuid m_mfvCapabilitiesCharUuid; + }; #endif // !defined(GATT_AUDIOSERVICE_RDK_H) diff --git a/src/ble/hal/utils/bleuuid.cpp b/src/ble/hal/utils/bleuuid.cpp index 62d4294c..73b4b7a7 100644 --- a/src/ble/hal/utils/bleuuid.cpp +++ b/src/ble/hal/utils/bleuuid.cpp @@ -113,6 +113,12 @@ bool ble_uuid_names_init(void) BleUuid(BleUuid::PeripheralPreferredConnectionParameters, std::string("Peripheral Preferred Connection Parameters")); BleUuid(BleUuid::ClientCharacteristicConfiguration, std::string("Client Characteristic Configuration")); BleUuid(BleUuid::ReportReference, std::string("Report Reference")); + BleUuid(BleUuid::SessionStart, std::string("Session Start")); + BleUuid(BleUuid::DetectionData, std::string("Detection Data")); + BleUuid(BleUuid::WakeWordModelVersion, std::string("Wake Word Model Version")); + BleUuid(BleUuid::PrivacySettings, std::string("Privacy Settings")); + BleUuid(BleUuid::ModelConfiguration, std::string("Model Configuration")); + BleUuid(BleUuid::MfvCapabilities, std::string("MFV Capabilities")); #ifdef CTRLM_BLE_SERVICES ctrlm_ble_uuid_names_install(); #endif diff --git a/src/ble/hal/utils/bleuuid.h b/src/ble/hal/utils/bleuuid.h index 7b9df4f2..b6413fa4 100644 --- a/src/ble/hal/utils/bleuuid.h +++ b/src/ble/hal/utils/bleuuid.h @@ -105,6 +105,12 @@ class BleUuid AdvertisingConfigCustomList = 0xed06, AssertReport = 0xed07, RawBatteryVoltage = 0xed08, + SessionStart = 0xea08, + DetectionData = 0xea09, + WakeWordModelVersion = 0xea0a, + PrivacySettings = 0xea0b, + ModelConfiguration = 0xea0c, + MfvCapabilities = 0xea0d, }; enum DescriptorType { diff --git a/src/ctrlm_config_default.json b/src/ctrlm_config_default.json index af65c196..78d21e89 100755 --- a/src/ctrlm_config_default.json +++ b/src/ctrlm_config_default.json @@ -217,7 +217,8 @@ }, "network_ble" : { "options": { - "disable_voice" : false + "disable_voice" : false, + "mfv_below_threshold_session_enabled" : false }, "timeouts": { "discovery": 15000, diff --git a/src/ctrlm_network.h b/src/ctrlm_network.h index 592e6e84..0ddd33f9 100644 --- a/src/ctrlm_network.h +++ b/src/ctrlm_network.h @@ -86,6 +86,7 @@ typedef struct { ctrlm_main_queue_msg_header_t header; ctrlm_voice_iarm_call_voice_session_t *params; sem_t * semaphore; + ctrlm_voice_device_t voice_device; // defaults to CTRLM_VOICE_DEVICE_PTT (0) when zeroed } ctrlm_main_queue_msg_voice_session_t; typedef struct { diff --git a/src/voice/ctrlm_voice_obj.cpp b/src/voice/ctrlm_voice_obj.cpp index e6e8636d..5aa53583 100644 --- a/src/voice/ctrlm_voice_obj.cpp +++ b/src/voice/ctrlm_voice_obj.cpp @@ -253,6 +253,8 @@ ctrlm_voice_t::ctrlm_voice_t() { this->device_requires_stb_data[CTRLM_VOICE_DEVICE_MICROPHONE] = true; this->device_status[CTRLM_VOICE_DEVICE_MICROPHONE_TAP] = CTRLM_VOICE_DEVICE_STATUS_NONE; this->device_requires_stb_data[CTRLM_VOICE_DEVICE_MICROPHONE_TAP] = true; + this->device_status[CTRLM_VOICE_DEVICE_MFV] = CTRLM_VOICE_DEVICE_STATUS_NONE; + this->device_requires_stb_data[CTRLM_VOICE_DEVICE_MFV] = true; this->device_status[CTRLM_VOICE_DEVICE_INVALID] = CTRLM_VOICE_DEVICE_STATUS_NOT_SUPPORTED; this->device_requires_stb_data[CTRLM_VOICE_DEVICE_INVALID] = true; @@ -1451,6 +1453,10 @@ ctrlm_voice_session_response_status_t ctrlm_voice_t::voice_session_req(ctrlm_net request_params.value.audio_fd.audio_format = voice_format_to_xrsr(format); request_params.value.audio_fd.callback = (create_pipe) ? NULL : ctrlm_voice_data_post_processing_cb; // RF4CE does not use pipe read callback request_params.value.audio_fd.user_data = (create_pipe) ? NULL : (void *)this; + // MFV wake-word sessions supply their wake word stream parameters (timing/confidence) after the + // session begins. Flag the request so the speech router holds the connect/init until the + // parameters are provided via voice_session_stream_params_update(). + request_params.value.audio_fd.stream_params_required = (device_type == CTRLM_VOICE_DEVICE_MFV); if(false == xrsr_session_request(voice_device_to_xrsr(device_type), dst_index, voice_format_to_xrsr(format), request_params, uuid, false, false)) { XLOGD_TELEMETRY("Failed to acquire voice session"); @@ -1500,9 +1506,21 @@ ctrlm_voice_session_response_status_t ctrlm_voice_t::voice_session_req(ctrlm_net session->has_stream_params = true; session->stream_params = *stream_params; - xrsr_session_keyword_info_set(XRSR_SRC_RCU_FF, stream_params->pre_keyword_sample_qty, stream_params->keyword_sample_qty); + xrsr_session_keyword_info_set(voice_device_to_xrsr(device_type), stream_params->pre_keyword_sample_qty, stream_params->keyword_sample_qty); } + #ifdef VOICE_BUFFER_STATS + // Reset buffer stats for every session type (PTT, FF, MFV, ...) since voice_session_data_post_processing() + // uses voice_packet_interval as a divisor. Leaving it uninitialized/stale for non-PTT sessions can cause a + // divide-by-zero (SIGFPE) in the post processing path. + voice_buffer_warning_triggered = 0; + voice_buffer_high_watermark = 0; + voice_packet_interval = voice_packet_interval_get(session->format, this->opus_samples_per_packet); + #ifdef TIMING_START_TO_FIRST_FRAGMENT + ctrlm_timestamp_get(&voice_session_begin_timestamp); + #endif + #endif + if(CTRLM_VOICE_DEVICE_PTT == session->voice_device) { errno_t safec_rc = memset_s(&session->status, sizeof(session->status), 0, sizeof(session->status)); ERR_CHK(safec_rc); @@ -1516,15 +1534,6 @@ ctrlm_voice_session_response_status_t ctrlm_voice_t::voice_session_req(ctrlm_net memset(&session->stats_session, 0, sizeof(session->stats_session)); session->stats_session.dropped_retry = ULONG_MAX; // Used to indicate whether controller provides stats or not - #ifdef VOICE_BUFFER_STATS - voice_buffer_warning_triggered = 0; - voice_buffer_high_watermark = 0; - voice_packet_interval = voice_packet_interval_get(session->format, this->opus_samples_per_packet); - #ifdef TIMING_START_TO_FIRST_FRAGMENT - ctrlm_timestamp_get(&voice_session_begin_timestamp); - #endif - #endif - // Start packet timeout, but not if this is a voice session by text or file if(!session->is_session_by_text && !session->is_session_by_file) { if(session->network_type == CTRLM_NETWORK_TYPE_IP || session->is_session_by_fifo) { @@ -1553,6 +1562,11 @@ ctrlm_voice_session_response_status_t ctrlm_voice_t::voice_session_req(ctrlm_net return (this->prefs.par_voice_enabled) ? VOICE_SESSION_RESPONSE_AVAILABLE_PAR_VOICE : VOICE_SESSION_RESPONSE_AVAILABLE; } +bool ctrlm_voice_t::voice_session_stream_params_update(ctrlm_voice_device_t device_type, bool valid, uint32_t keyword_sample_begin, uint32_t keyword_sample_end, double confidence, double signal_noise_ratio) { + XLOGD_INFO("device <%s> %s begin <%u> end <%u> confidence <%.3f> snr <%.2f>", ctrlm_voice_device_str(device_type), valid ? "valid" : "release-only", keyword_sample_begin, keyword_sample_end, confidence, signal_noise_ratio); + return xrsr_session_stream_params_set(voice_device_to_xrsr(device_type), valid, keyword_sample_begin, keyword_sample_end, confidence, signal_noise_ratio); +} + bool ctrlm_voice_t::voice_session_term(std::string &session_id) { for(uint32_t group = VOICE_SESSION_GROUP_DEFAULT; group < VOICE_SESSION_GROUP_QTY; group++) { ctrlm_voice_session_t *session = &this->voice_session[group]; @@ -1691,12 +1705,16 @@ void ctrlm_voice_t::voice_session_data_post_processing(int bytes_sent, const cha // The total packets (received + lost) uint32_t packets_total = session->packets_processed + session->packets_lost; long long session_delta = (session_time - ((packets_total - 1) * this->voice_packet_interval)); // in microseconds - unsigned long watermark = (session_delta / this->voice_packet_interval) + 1; - if(session_delta > 0 && watermark > voice_buffer_high_watermark) { - voice_buffer_high_watermark = watermark; - } - if(session_delta > (long long)(VOICE_BUFFER_WARNING_THRESHOLD * this->voice_packet_interval)) { - voice_buffer_warning_triggered = 1; + if(this->voice_packet_interval == 0) { + XLOGD_WARN("voice_packet_interval is 0 for device <%s> format <%s>; skipping buffer stats to avoid divide-by-zero", ctrlm_voice_device_str(session->voice_device), ctrlm_voice_format_str(session->format)); + } else { + unsigned long watermark = (session_delta / this->voice_packet_interval) + 1; + if(session_delta > 0 && watermark > voice_buffer_high_watermark) { + voice_buffer_high_watermark = watermark; + } + if(session_delta > (long long)(VOICE_BUFFER_WARNING_THRESHOLD * this->voice_packet_interval)) { + voice_buffer_warning_triggered = 1; + } } #endif @@ -2749,7 +2767,27 @@ void ctrlm_voice_t::voice_session_end_callback(ctrlm_voice_session_end_cb_t *ses session->session_active_server = false; if(session->state_src == CTRLM_VOICE_STATE_SRC_STREAMING) { - voice_session_end(session, CTRLM_VOICE_SESSION_END_REASON_OTHER_ERROR); + // The source (e.g. an MFV/FF remote) is still streaming when the server session ended. This + // occurs when end of speech is detected on the box side rather than via a controller key + // release. Map the speech router's end reason to the appropriate ctrlm end reason so that a + // normal end of speech is reported as DONE rather than an error. + ctrlm_voice_session_end_reason_t end_reason; + switch(stats->session_end_reason) { + case XRSR_SESSION_END_REASON_EOS: + case XRSR_SESSION_END_REASON_EOT: { + end_reason = CTRLM_VOICE_SESSION_END_REASON_DONE; + break; + } + case XRSR_SESSION_END_REASON_ERROR_SESSION_TIMEOUT: { + end_reason = CTRLM_VOICE_SESSION_END_REASON_TIMEOUT_MAXIMUM; + break; + } + default: { + end_reason = CTRLM_VOICE_SESSION_END_REASON_OTHER_ERROR; + break; + } + } + voice_session_end(session, end_reason); } else if(!session->session_active_controller) { session->state_src = CTRLM_VOICE_STATE_SRC_READY; @@ -3191,6 +3229,7 @@ const char *ctrlm_voice_device_str(ctrlm_voice_device_t device) { case CTRLM_VOICE_DEVICE_FF: return("FF"); case CTRLM_VOICE_DEVICE_MICROPHONE: return("MICROPHONE"); case CTRLM_VOICE_DEVICE_MICROPHONE_TAP: return("MICROPHONE_TAP"); + case CTRLM_VOICE_DEVICE_MFV: return("MFV"); case CTRLM_VOICE_DEVICE_INVALID: return("INVALID"); } return("UNKNOWN"); @@ -3259,6 +3298,10 @@ ctrlm_voice_device_t xrsr_to_voice_device(xrsr_src_t device) { ret = CTRLM_VOICE_DEVICE_MICROPHONE_TAP; break; } + case XRSR_SRC_RCU_MFV: { + ret = CTRLM_VOICE_DEVICE_MFV; + break; + } default: { XLOGD_ERROR("unrecognized device type %d", device); break; @@ -3286,6 +3329,10 @@ xrsr_src_t voice_device_to_xrsr(ctrlm_voice_device_t device) { ret = XRSR_SRC_MICROPHONE_TAP; break; } + case CTRLM_VOICE_DEVICE_MFV: { + ret = XRSR_SRC_RCU_MFV; + break; + } default: { XLOGD_ERROR("unrecognized device type %d", device); break; @@ -3398,6 +3445,7 @@ bool ctrlm_voice_t::is_voice_assistant(ctrlm_voice_device_t device) { bool voice_assistant = false; switch(device) { case CTRLM_VOICE_DEVICE_MICROPHONE: + case CTRLM_VOICE_DEVICE_MFV: case CTRLM_VOICE_DEVICE_FF: { voice_assistant = true; break; diff --git a/src/voice/ctrlm_voice_obj.h b/src/voice/ctrlm_voice_obj.h index 3ec217b2..e7efd86a 100644 --- a/src/voice/ctrlm_voice_obj.h +++ b/src/voice/ctrlm_voice_obj.h @@ -497,6 +497,10 @@ class ctrlm_voice_t { void voice_session_rsp_confirm(bool result, signed long long rsp_time, unsigned int rsp_window, const std::string &err_str, ctrlm_timestamp_t *timestamp); bool voice_session_data(ctrlm_network_id_t network_id, ctrlm_controller_id_t controller_id, const char *buffer, long unsigned int length, ctrlm_timestamp_t *timestamp=NULL, uint8_t *lqi=NULL); bool voice_session_data(ctrlm_network_id_t network_id, ctrlm_controller_id_t controller_id, int fd, const uuid_t *uuid=NULL); + // Supplies wake word stream parameters (timing/confidence) that arrive after the session has begun + // (e.g. BLE MFV detection data) and releases the speech router's deferred connect. Pass valid=false + // to release the connect without wake word parameters (e.g. on a detection-data timeout). + bool voice_session_stream_params_update(ctrlm_voice_device_t device_type, bool valid, uint32_t keyword_sample_begin, uint32_t keyword_sample_end, double confidence, double signal_noise_ratio); void voice_session_data_post_processing(int bytes_sent, const char *action, ctrlm_timestamp_t *timestamp); void voice_session_end(ctrlm_network_id_t network_id, ctrlm_controller_id_t controller_id, ctrlm_voice_session_end_reason_t reason, ctrlm_timestamp_t *timestamp=NULL, ctrlm_voice_session_end_stats_t *stats=NULL); void voice_session_end(ctrlm_voice_session_t *session, ctrlm_voice_session_end_reason_t reason, ctrlm_timestamp_t *timestamp=NULL, ctrlm_voice_session_end_stats_t *stats=NULL); diff --git a/src/voice/ctrlm_voice_obj_generic.cpp b/src/voice/ctrlm_voice_obj_generic.cpp index 4cb35f99..63fbcb89 100644 --- a/src/voice/ctrlm_voice_obj_generic.cpp +++ b/src/voice/ctrlm_voice_obj_generic.cpp @@ -24,6 +24,7 @@ #include #include #include +#include #include #include "ctrlm_voice_obj_generic.h" #include "ctrlm.h" @@ -165,6 +166,21 @@ void ctrlm_voice_generic_t::voice_sdk_update_routes() { errno_t safec_rc = memset_s(&routes, sizeof(routes), 0, sizeof(routes)); ERR_CHK(safec_rc); + // TEMPORARY: hardcode all voice server URLs used by ctrlm, overriding any configured/RFC/settings values. + // ctrlm selects the protocol handler by scheme prefix: "vrngs" -> secure ws_nextgen (vrex), translated + // internally to "wss". A raw "wss" prefix is not recognized and is rejected as "unsupported url". + static const char *CTRLM_VOICE_HARDCODED_URL = "vrngs://eng.voice.xvp.eu-1.xcal.tv/vrex/speech/websocket?stbSwVersion=QSE041.004.00U"; + this->prefs.server_url_src_ptt = CTRLM_VOICE_HARDCODED_URL; + this->prefs.server_url_src_ff = CTRLM_VOICE_HARDCODED_URL; + this->prefs.server_url_src_mic_tap = CTRLM_VOICE_HARDCODED_URL; + + // TEMPORARY: allow the hardcoded URL's hostname by adding its pattern to the accepted hostname list. + // Re-add each call (checking for presence) so it survives any config/RFC reload that clears the list. + const std::string hardcoded_host_regex = "^.*\\.xcal\\.tv$"; + if(std::find(this->prefs.server_hosts.begin(), this->prefs.server_hosts.end(), hardcoded_host_regex) == this->prefs.server_hosts.end()) { + this->url_hostname_pattern_add("*.xcal.tv"); + } + bool networked_standby_supported = ctrlm_is_networked_standby_supported(); // iterate over source to url mapping @@ -181,8 +197,10 @@ void ctrlm_voice_generic_t::voice_sdk_update_routes() { break; } case CTRLM_VOICE_DEVICE_MICROPHONE: + case CTRLM_VOICE_DEVICE_MFV: case CTRLM_VOICE_DEVICE_FF: { - url = &this->prefs.server_url_src_ff; + // If FF URL is not configured, fall back to PTT URL. + url = this->prefs.server_url_src_ff.empty() ? &this->prefs.server_url_src_ptt : &this->prefs.server_url_src_ff; break; } case CTRLM_VOICE_DEVICE_MICROPHONE_TAP: { diff --git a/src/voice/ctrlm_voice_types.h b/src/voice/ctrlm_voice_types.h index e65a49ba..cbecbb0a 100644 --- a/src/voice/ctrlm_voice_types.h +++ b/src/voice/ctrlm_voice_types.h @@ -33,6 +33,7 @@ typedef enum { CTRLM_VOICE_DEVICE_FF, CTRLM_VOICE_DEVICE_MICROPHONE, CTRLM_VOICE_DEVICE_MICROPHONE_TAP, + CTRLM_VOICE_DEVICE_MFV, CTRLM_VOICE_DEVICE_INVALID, } ctrlm_voice_device_t; diff --git a/src/voice/endpoints/ctrlm_voice_endpoint_ws_nextgen.cpp b/src/voice/endpoints/ctrlm_voice_endpoint_ws_nextgen.cpp index 9abc27db..1571537e 100644 --- a/src/voice/endpoints/ctrlm_voice_endpoint_ws_nextgen.cpp +++ b/src/voice/endpoints/ctrlm_voice_endpoint_ws_nextgen.cpp @@ -22,6 +22,8 @@ #define NEXTGEN_AUDIO_MODEL_PTT "ptt" #define NEXTGEN_AUDIO_MODEL_HF "hf" #define NEXTGEN_AUDIO_PROFILE_HF "FFV" +#define NEXTGEN_AUDIO_MODEL_MFV "mfv" +#define NEXTGEN_AUDIO_PROFILE_MFV "MFV" // Structures typedef struct { @@ -289,7 +291,13 @@ void ctrlm_voice_endpoint_ws_nextgen_t::voice_session_begin_callback_ws_nextgen( config_in.ws.cert_revoked_allow = false; config_in.ws.ocsp_expired_allow = false; - if(is_mic) { + if(source == CTRLM_VOICE_DEVICE_MFV) { + // TEMPORARY: hard-code MFV wuw sessions to send the same audio profile/model as the on-device + // HF microphone instead of the MFV-specific values. Revert to NEXTGEN_AUDIO_PROFILE_MFV / + // NEXTGEN_AUDIO_MODEL_MFV to signal to Vrex that this transaction is from mid-field voice. + xrsv_ws_nextgen_update_audio_profile(this->xrsv_obj_ws_nextgen, NEXTGEN_AUDIO_PROFILE_HF); + xrsv_ws_nextgen_update_audio_model(this->xrsv_obj_ws_nextgen, NEXTGEN_AUDIO_MODEL_HF); + } else if(is_mic) { xrsv_ws_nextgen_update_audio_profile(this->xrsv_obj_ws_nextgen, NEXTGEN_AUDIO_PROFILE_HF); xrsv_ws_nextgen_update_audio_model(this->xrsv_obj_ws_nextgen, NEXTGEN_AUDIO_MODEL_HF); } else { @@ -337,7 +345,7 @@ void ctrlm_voice_endpoint_ws_nextgen_t::voice_session_begin_callback_ws_nextgen( stream_params->keyword_sample_end -= delta; } - if(is_mic) { + if(is_mic || source == CTRLM_VOICE_DEVICE_FF || source == CTRLM_VOICE_DEVICE_MFV) { stream_params->push_to_talk = false; } else { stream_params->push_to_talk = true; @@ -358,7 +366,7 @@ void ctrlm_voice_endpoint_ws_nextgen_t::voice_session_begin_callback_ws_nextgen( config_in.ws.app_config = stream_params_out; XLOGD_AUTOMATION_TELEMETRY("session begin - src <%s> ptt <%s> w_SAT <%s> w_MTLS <%s> w_OCSPst <%s> w_OCSPca <%s> keyword begin <%u> end <%u> doa <%u> gain <%4.1f> db", ctrlm_voice_device_str(source), (stream_params->push_to_talk ? "TRUE" : "FALSE"), has_sat ? "YES" : "NO", use_mtls ? "YES" : "NO", ocsp_verify_stapling ? "YES" : "NO", ocsp_verify_ca ? "YES" : "NO", stream_params->keyword_sample_begin, stream_params->keyword_sample_end, stream_params->keyword_doa, stream_params->dynamic_gain); - } else if(!is_mic || dqm->configuration.user_initiated) { + } else if((!is_mic && source != CTRLM_VOICE_DEVICE_FF && source != CTRLM_VOICE_DEVICE_MFV) || dqm->configuration.user_initiated) { xrsv_ws_nextgen_stream_params_t *stream_params = (xrsv_ws_nextgen_stream_params_t *)malloc(sizeof(xrsv_ws_nextgen_stream_params_t)); if(stream_params != NULL) { @@ -371,6 +379,19 @@ void ctrlm_voice_endpoint_ws_nextgen_t::voice_session_begin_callback_ws_nextgen( config_in.ws.app_config = stream_params; XLOGD_AUTOMATION_TELEMETRY("session begin - src <%s> ptt w_SAT <%s> w_MTLS <%s> w_OCSPst <%s> w_OCSPca <%s>", ctrlm_voice_device_str(source), has_sat ? "YES" : "NO", use_mtls ? "YES" : "NO", ocsp_verify_stapling ? "YES" : "NO", ocsp_verify_ca ? "YES" : "NO"); + } else if(source == CTRLM_VOICE_DEVICE_FF || source == CTRLM_VOICE_DEVICE_MFV) { + xrsv_ws_nextgen_stream_params_t *stream_params = (xrsv_ws_nextgen_stream_params_t *)malloc(sizeof(xrsv_ws_nextgen_stream_params_t)); + + if(stream_params != NULL) { + errno_t safec_rc = memset_s(stream_params, sizeof(*stream_params), 0, sizeof(*stream_params)); + ERR_CHK(safec_rc); + + stream_params->push_to_talk = false; + stream_params->par_eos_timeout = params.par_voice_enabled ? params.par_voice_eos_timeout : 0; + } + config_in.ws.app_config = stream_params; + + XLOGD_AUTOMATION_TELEMETRY("session begin - src <%s> ptt w_SAT <%s> w_MTLS <%s> w_OCSPst <%s> w_OCSPca <%s>", ctrlm_voice_device_str(source), has_sat ? "YES" : "NO", use_mtls ? "YES" : "NO", ocsp_verify_stapling ? "YES" : "NO", ocsp_verify_ca ? "YES" : "NO"); } else { XLOGD_ERROR("session begin - invalid params - src <%s>", ctrlm_voice_device_str(source)); } diff --git a/src/voice/ipc/ctrlm_voice_ipc_iarm_thunder.cpp b/src/voice/ipc/ctrlm_voice_ipc_iarm_thunder.cpp index 6b407173..0a9482c8 100644 --- a/src/voice/ipc/ctrlm_voice_ipc_iarm_thunder.cpp +++ b/src/voice/ipc/ctrlm_voice_ipc_iarm_thunder.cpp @@ -941,6 +941,7 @@ const char *voice_device_str(ctrlm_voice_device_t device) { case CTRLM_VOICE_DEVICE_FF: return("ff"); case CTRLM_VOICE_DEVICE_MICROPHONE: return("mic"); case CTRLM_VOICE_DEVICE_MICROPHONE_TAP: return("mic_tap"); + case CTRLM_VOICE_DEVICE_MFV: return("mfv"); default: break; } return("invalid");