xref: /frameworks/av/services/audiopolicy/service/Spatializer.h

/*
 * Copyright (C) 2021 The Android Open Source Project
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

#ifndef ANDROID_MEDIA_SPATIALIZER_H
#define ANDROID_MEDIA_SPATIALIZER_H

#include <android-base/stringprintf.h>
#include <android/media/BnEffect.h>
#include <android/media/BnSpatializer.h>
#include <android/media/audio/common/AudioLatencyMode.h>
#include <android/media/audio/common/HeadTracking.h>
#include <android/media/audio/common/Spatialization.h>
#include <audio_utils/mutex.h>
#include <audio_utils/SimpleLog.h>
#include <math.h>
#include <media/AudioEffect.h>
#include <media/MediaMetricsItem.h>
#include <media/audiohal/EffectsFactoryHalInterface.h>
#include <media/VectorRecorder.h>
#include <media/audiohal/EffectHalInterface.h>
#include <media/stagefright/foundation/ALooper.h>
#include <system/audio_effects/effect_spatializer.h>
#include <string>
#include <unordered_set>

#include "SpatializerPoseController.h"

namespace android {


// ----------------------------------------------------------------------------

/**
 * A callback interface from the Spatializer object or its parent AudioPolicyService.
 * This is implemented by the audio policy service hosting the Spatializer to perform
 * actions needed when a state change inside the Spatializer requires some audio system
 * changes that cannot be performed by the Spatializer. For instance opening or closing a
 * spatializer output stream when the spatializer is enabled or disabled
 */
class SpatializerPolicyCallback {
public:
    /** Called when a stage change occurs that requires the parent audio policy service to take
     * some action.
     */
    virtual void onCheckSpatializer() = 0;

    virtual ~SpatializerPolicyCallback() = default;
};
/**
 * The Spatializer class implements all functional controlling the multichannel spatializer
 * with head tracking implementation in the native audio service: audio policy and audio flinger.
 * It presents an AIDL interface available to the java audio service to discover the availability
 * of the feature and options, control its state and register an active head tracking sensor.
 * It maintains the current state of the platform spatializer and applies the stored parameters
 * when the spatializer engine is created and enabled.
 * Based on the requested spatializer level, it will request the creation of a specialized output
 * mixer to the audio policy service which will in turn notify the Spatializer of the output
 * stream on which a spatializer engine should be created, configured and enabled.
 * The spatializer also hosts the head tracking management logic. This logic receives the
 * desired head tracking mode and selected head tracking sensor, registers a sensor event listener
 * and derives the compounded head pose information to the spatializer engine.
 *
 * Workflow:
 * - Initialization: when the audio policy service starts, it checks if a spatializer effect
 * engine exists and if the audio policy manager reports a dedicated spatializer output profile.
 * If both conditions are met, a Spatializer object is created
 * - Capabilities discovery: AudioService will call AudioSystem::canBeSpatialized() and if true,
 * acquire an ISpatializer interface with AudioSystem::getSpatializer(). This interface
 * will be used to query the implementation capabilities and configure the spatializer.
 * - Enabling: when ISpatializer::setLevel() sets a level different from NONE the spatializer
 * is considered enabled. The audio policy callback onCheckSpatializer() is called. This
 * triggers a request to audio policy manager to open a spatialization output stream and a
 * spatializer mixer is created in audio flinger. When an output is returned by audio policy
 * manager, Spatializer::attachOutput() is called which creates and enables the spatializer
 * stage engine on the specified output.
 * - Disabling: when the spatialization level is set to NONE, the spatializer is considered
 * disabled. The audio policy callback onCheckSpatializer() is called. This triggers a call
 * to Spatializer::detachOutput() and the spatializer engine is released. Then a request is
 * made to audio policy manager to release and close the spatializer output stream and the
 * spatializer mixer thread is destroyed.
 */
class Spatializer : public media::BnSpatializer,
                    public AudioEffect::IAudioEffectCallback,
                    public IBinder::DeathRecipient,
                    private SpatializerPoseController::Listener,
                    public virtual AudioSystem::SupportedLatencyModesCallback {
  public:
    static sp<Spatializer> create(SpatializerPolicyCallback* callback,
                                  const sp<EffectsFactoryHalInterface>& effectsFactoryHal);

           ~Spatializer() override;

    /** RefBase */
    void onFirstRef();

    /** ISpatializer, see ISpatializer.aidl */
    binder::Status release() override;
    binder::Status getSupportedLevels(
            std::vector<media::audio::common::Spatialization::Level>* levels) override;
    binder::Status setLevel(media::audio::common::Spatialization::Level level) override;
    binder::Status getLevel(media::audio::common::Spatialization::Level *level) override;
    binder::Status isHeadTrackingSupported(bool *supports);
    binder::Status getSupportedHeadTrackingModes(
            std::vector<media::audio::common::HeadTracking::Mode>* modes) override;
    binder::Status setDesiredHeadTrackingMode(
            media::audio::common::HeadTracking::Mode mode) override;
    binder::Status getActualHeadTrackingMode(
            media::audio::common::HeadTracking::Mode* mode) override;
    binder::Status recenterHeadTracker() override;
    binder::Status setGlobalTransform(const std::vector<float>& screenToStage) override;
    binder::Status setHeadSensor(int sensorHandle) override;
    binder::Status setScreenSensor(int sensorHandle) override;
    binder::Status setDisplayOrientation(float physicalToLogicalAngle) override;
    binder::Status setHingeAngle(float hingeAngle) override;
    binder::Status setFoldState(bool folded) override;
    binder::Status getSupportedModes(
            std::vector<media::audio::common::Spatialization::Mode>* modes) override;
    binder::Status registerHeadTrackingCallback(
        const sp<media::ISpatializerHeadTrackingCallback>& callback) override;
    binder::Status setParameter(int key, const std::vector<unsigned char>& value) override;
    binder::Status getParameter(int key, std::vector<unsigned char> *value) override;
    binder::Status getOutput(int *output);

    /** IBinder::DeathRecipient. Listen to the death of the INativeSpatializerCallback. */
    virtual void binderDied(const wp<IBinder>& who);

    /** SupportedLatencyModesCallback */
    void onSupportedLatencyModesChanged(
            audio_io_handle_t output, const std::vector<audio_latency_mode_t>& modes) override;

    /** Registers a INativeSpatializerCallback when a client is attached to this Spatializer
     * by audio policy service.
     */
    status_t registerCallback(const sp<media::INativeSpatializerCallback>& callback);

    status_t loadEngineConfiguration(sp<EffectHalInterface> effect);

    /** Level getter for use by local classes. */
    media::audio::common::Spatialization::Level getLevel() const {
        audio_utils::lock_guard lock(mMutex);
        return mLevel;
    }

    /** For test only */
    std::unordered_set<media::audio::common::HeadTracking::ConnectionMode>
            getSupportedHeadtrackingConnectionModes() const {
        return mSupportedHeadtrackingConnectionModes;
    }

    /** For test only */
    media::audio::common::HeadTracking::ConnectionMode getHeadtrackingConnectionMode() const {
        return mHeadtrackingConnectionMode;
    }

    /** For test only */
    std::vector<audio_latency_mode_t> getSupportedLatencyModes() const {
        audio_utils::lock_guard lock(mMutex);
        return mSupportedLatencyModes;
    }

    /** For test only */
    std::vector<audio_latency_mode_t> getOrderedLowLatencyModes() const {
        return mOrderedLowLatencyModes;
    }

    /** For test only */
    audio_latency_mode_t getRequestedLatencyMode() const {
        audio_utils::lock_guard lock(mMutex);
        return mRequestedLatencyMode;
    }

    /** Called by audio policy service when the special output mixer dedicated to spatialization
     * is opened and the spatializer engine must be created.
     */
    status_t attachOutput(audio_io_handle_t output, size_t numActiveTracks);
    /** Called by audio policy service when the special output mixer dedicated to spatialization
     * is closed and the spatializer engine must be release.
     */
    audio_io_handle_t detachOutput();
    /** Returns the output stream the spatializer is attached to. */
    audio_io_handle_t getOutput() const { audio_utils::lock_guard lock(mMutex); return mOutput; }

    /** For test only */
    void setOutput(audio_io_handle_t output) {
        audio_utils::lock_guard lock(mMutex);
        mOutput = output;
    }

    void updateActiveTracks(size_t numActiveTracks);

    /** Gets the channel mask, sampling rate and format set for the spatializer input. */
    audio_config_base_t getAudioInConfig() const;

    void calculateHeadPose();

    /** Convert fields in Spatializer and sub-modules to a string. Disable thread-safety-analysis
     * here because we want to dump mutex guarded members even try_lock failed to provide as much
     * information as possible for debugging purpose. */
    std::string toString(unsigned level) const NO_THREAD_SAFETY_ANALYSIS;

    static std::string toString(audio_latency_mode_t mode) {
        // We convert to the AIDL type to print (eventually the legacy type will be removed).
        const auto result = legacy2aidl_audio_latency_mode_t_AudioLatencyMode(mode);
        return result.has_value() ?
                media::audio::common::toString(*result) : "unknown_latency_mode";
    }

    // If the Spatializer is not created, we send the status for metrics purposes.
    // OK:      Spatializer not expected to be created.
    // NO_INIT: Spatializer creation failed.
    static void sendEmptyCreateSpatializerMetricWithStatus(status_t status);

    /** Made public for test only */
    void onSupportedLatencyModesChangedMsg(
            audio_io_handle_t output, std::vector<audio_latency_mode_t>&& modes);

private:
    Spatializer(effect_descriptor_t engineDescriptor,
                     SpatializerPolicyCallback *callback);

    static void engineCallback(int32_t event, void* user, void *info);

    // From VirtualizerStageController::Listener
    void onHeadToStagePose(const media::Pose3f& headToStage) override;
    void onActualModeChange(media::HeadTrackingMode mode) override;

    void onHeadToStagePoseMsg(const std::vector<float>& headToStage);
    void onActualModeChangeMsg(media::HeadTrackingMode mode);

    static constexpr int kMaxEffectParamValues = 10;
    /**
     * Get a parameter from spatializer engine by calling the effect HAL command method directly.
     * To be used when the engine instance mEngine is not yet created in the effect framework.
     * When MULTI_VALUES is false, the expected reply is only one value of type T.
     * When MULTI_VALUES is true, the expected reply is made of a number (of type T) indicating
     * how many values are returned, followed by this number for values of type T.
     */
    template<bool MULTI_VALUES, typename T>
    status_t getHalParameter(sp<EffectHalInterface> effect, uint32_t type,
                                          std::vector<T> *values) {
        static_assert(sizeof(T) <= sizeof(uint32_t), "The size of T must less than 32 bits");

        uint32_t cmd[sizeof(effect_param_t) / sizeof(uint32_t) + 1];
        uint32_t reply[sizeof(effect_param_t) / sizeof(uint32_t) + 2 + kMaxEffectParamValues];

        effect_param_t *p = (effect_param_t *)cmd;
        p->psize = sizeof(uint32_t);
        if (MULTI_VALUES) {
            p->vsize = (kMaxEffectParamValues + 1) * sizeof(T);
        } else {
            p->vsize = sizeof(T);
        }
        *(uint32_t *)p->data = type;
        uint32_t replySize = sizeof(effect_param_t) + p->psize + p->vsize;

        status_t status = effect->command(EFFECT_CMD_GET_PARAM,
                                          sizeof(effect_param_t) + sizeof(uint32_t), cmd,
                                          &replySize, reply);
        if (status != NO_ERROR) {
            return status;
        }
        if (p->status != NO_ERROR) {
            return p->status;
        }
        if (replySize <
                sizeof(effect_param_t) + sizeof(uint32_t) + (MULTI_VALUES ? 2 : 1) * sizeof(T)) {
            return BAD_VALUE;
        }

        T *params = (T *)((uint8_t *)reply + sizeof(effect_param_t) + sizeof(uint32_t));
        int numParams = 1;
        if (MULTI_VALUES) {
            numParams = (int)*params++;
        }
        if (numParams > kMaxEffectParamValues) {
            return BAD_VALUE;
        }
        (*values).clear();
        std::copy(&params[0], &params[numParams], back_inserter(*values));
        return NO_ERROR;
    }

    /**
     * Set a parameter to spatializer engine by calling setParameter on mEngine AudioEffect object.
     * It is possible to pass more than one value of type T according to the parameter type
     *  according to values vector size.
     */
    template<typename T>
    status_t setEffectParameter_l(uint32_t type, const std::vector<T>& values) REQUIRES(mMutex) {
        static_assert(sizeof(T) <= sizeof(uint32_t), "The size of T must less than 32 bits");

        uint32_t cmd[sizeof(effect_param_t) / sizeof(uint32_t) + 1 + values.size()];
        effect_param_t *p = (effect_param_t *)cmd;
        p->psize = sizeof(uint32_t);
        p->vsize = sizeof(T) * values.size();
        *(uint32_t *)p->data = type;
        memcpy((uint32_t *)p->data + 1, values.data(), sizeof(T) * values.size());

        status_t status = mEngine->setParameter(p);
        if (status != NO_ERROR) {
            return status;
        }
        if (p->status != NO_ERROR) {
            return p->status;
        }
        return NO_ERROR;
    }

    /**
     * Set a parameter to spatializer engine by calling setParameter on mEngine AudioEffect object.
     * The variant is for compound parameters with two values of different base types
     */
    template<typename P1, typename P2>
    status_t setEffectParameter_l(uint32_t type, const P1 val1, const P2 val2) REQUIRES(mMutex) {
        static_assert(sizeof(P1) <= sizeof(uint32_t), "The size of P1 must less than 32 bits");
        static_assert(sizeof(P2) <= sizeof(uint32_t), "The size of P2 must less than 32 bits");

        uint32_t cmd[sizeof(effect_param_t) / sizeof(uint32_t) + 3];
        effect_param_t *p = (effect_param_t *)cmd;
        p->psize = sizeof(uint32_t);
        p->vsize = 2 * sizeof(uint32_t);
        *(uint32_t *)p->data = type;
        *((uint32_t *)p->data + 1) = static_cast<uint32_t>(val1);
        *((uint32_t *)p->data + 2) = static_cast<uint32_t>(val2);

        status_t status = mEngine->setParameter(p);
        if (status != NO_ERROR) {
            return status;
        }
        if (p->status != NO_ERROR) {
            return p->status;
        }
        return NO_ERROR;
    }

    /**
     * Get a parameter from spatializer engine by calling getParameter on AudioEffect object.
     * It is possible to read more than one value of type T according to the parameter type
     * by specifying values vector size.
     */
    template<typename T>
    status_t getEffectParameter_l(uint32_t type, std::vector<T> *values) REQUIRES(mMutex) {
        static_assert(sizeof(T) <= sizeof(uint32_t), "The size of T must less than 32 bits");

        uint32_t cmd[sizeof(effect_param_t) / sizeof(uint32_t) + 1 + values->size()];
        effect_param_t *p = (effect_param_t *)cmd;
        p->psize = sizeof(uint32_t);
        p->vsize = sizeof(T) * values->size();
        *(uint32_t *)p->data = type;

        status_t status = mEngine->getParameter(p);

        if (status != NO_ERROR) {
            return status;
        }
        if (p->status != NO_ERROR) {
            return p->status;
        }

        int numValues = std::min(p->vsize / sizeof(T), values->size());
        (*values).clear();
        T *retValues = (T *)((uint8_t *)p->data + sizeof(uint32_t));
        std::copy(&retValues[0], &retValues[numValues], back_inserter(*values));

        return NO_ERROR;
    }

    /**
     * Get a parameter from spatializer engine by calling getParameter on AudioEffect object.
     * The variant is for compound parameters with two values of different base types
     */
    template<typename P1, typename P2>
    status_t getEffectParameter_l(uint32_t type, P1 *val1, P2 *val2) REQUIRES(mMutex) {
        static_assert(sizeof(P1) <= sizeof(uint32_t), "The size of P1 must less than 32 bits");
        static_assert(sizeof(P2) <= sizeof(uint32_t), "The size of P2 must less than 32 bits");

        uint32_t cmd[sizeof(effect_param_t) / sizeof(uint32_t) + 3];
        effect_param_t *p = (effect_param_t *)cmd;
        p->psize = sizeof(uint32_t);
        p->vsize = 2 * sizeof(uint32_t);
        *(uint32_t *)p->data = type;

        status_t status = mEngine->getParameter(p);

        if (status != NO_ERROR) {
            return status;
        }
        if (p->status != NO_ERROR) {
            return p->status;
        }
        *val1 = static_cast<P1>(*((uint32_t *)p->data + 1));
        *val2 = static_cast<P2>(*((uint32_t *)p->data + 2));
        return NO_ERROR;
    }

    virtual void onFramesProcessed(int32_t framesProcessed) override;

    /**
     * Checks if head and screen sensors must be actively monitored based on
     * spatializer state and playback activity and configures the pose controller
     * accordingly.
     */
    void checkSensorsState_l() REQUIRES(mMutex);

    /**
     * Checks if the head pose controller should be created or destroyed according
     * to desired head tracking mode.
     */
    void checkPoseController_l() REQUIRES(mMutex);

    /**
     * Checks if the spatializer effect should be enabled based on
     * playback activity and requested level.
     */
    void checkEngineState_l() REQUIRES(mMutex);

    /**
     * Reset head tracking mode and recenter pose in engine: Called when the head tracking
     * is disabled.
     */
    void resetEngineHeadPose_l() REQUIRES(mMutex);

    /** Read bluetooth.core.le.dsa_transport_preference property and populate the ordered list of
     * preferred low latency modes in mOrderedLowLatencyModes.
     */
    void loadOrderedLowLatencyModes();

    /**
     * Sort mSupportedLatencyModes list according to the preference order stored in
     * mOrderedLowLatencyModes.
     * Note: Because MODE_FREE is not in mOrderedLowLatencyModes, it will always be at
     * the end of the list.
     */
    void sortSupportedLatencyModes_l() REQUIRES(mMutex);

    /**
     * Called after enabling head tracking in the spatializer engine to indicate which
     * connection mode should be used among those supported. The selection depends on
     * currently supported latency modes reported by the audio HAL.
     * When the connection mode is direct to the sensor, the sensor ID is also communicated
     * to the spatializer engine.
     */
    void setEngineHeadtrackingConnectionMode_l() REQUIRES(mMutex);

    /**
     * Select the desired head tracking connection mode for the spatializer engine among the list
     * stored in mSupportedHeadtrackingConnectionModes at init time.
     * Also returns the desired low latency mode according to selected connection mode.
     */
    audio_latency_mode_t selectHeadtrackingConnectionMode_l() REQUIRES(mMutex);

    /** Effect engine descriptor */
    const effect_descriptor_t mEngineDescriptor;
    /** Callback interface to parent audio policy service */
    SpatializerPolicyCallback* const mPolicyCallback;

    /** Currently there is only one version of the spatializer running */
    static constexpr const char* kDefaultMetricsId =
            AMEDIAMETRICS_KEY_PREFIX_AUDIO_SPATIALIZER "0";
    const std::string mMetricsId = kDefaultMetricsId;

    /** Mutex protecting internal state */
    mutable audio_utils::mutex mMutex{audio_utils::MutexOrder::kSpatializer_Mutex};

    /** Client AudioEffect for the engine */
    sp<AudioEffect> mEngine GUARDED_BY(mMutex);
    /** Output stream the spatializer mixer thread is attached to */
    audio_io_handle_t mOutput GUARDED_BY(mMutex) = AUDIO_IO_HANDLE_NONE;

    /** Callback interface to the client (AudioService) controlling this`Spatializer */
    sp<media::INativeSpatializerCallback> mSpatializerCallback GUARDED_BY(mMutex);

    /** Callback interface for head tracking */
    sp<media::ISpatializerHeadTrackingCallback> mHeadTrackingCallback GUARDED_BY(mMutex);

    /** Requested spatialization level */
    media::audio::common::Spatialization::Level mLevel GUARDED_BY(mMutex) =
            media::audio::common::Spatialization::Level::NONE;

    /** Control logic for head-tracking, etc. */
    std::shared_ptr<SpatializerPoseController> mPoseController GUARDED_BY(mMutex);

    /** Last requested head tracking mode */
    media::HeadTrackingMode mDesiredHeadTrackingMode GUARDED_BY(mMutex)
            = media::HeadTrackingMode::STATIC;

    /** Last-reported actual head-tracking mode. */
    media::audio::common::HeadTracking::Mode mActualHeadTrackingMode GUARDED_BY(mMutex)
            = media::audio::common::HeadTracking::Mode::DISABLED;

    /** Selected Head pose sensor */
    int32_t mHeadSensor GUARDED_BY(mMutex) = SpatializerPoseController::INVALID_SENSOR;

    /** Selected Screen pose sensor */
    int32_t mScreenSensor GUARDED_BY(mMutex) = SpatializerPoseController::INVALID_SENSOR;

    /** Last display orientation received */
    float mDisplayOrientation GUARDED_BY(mMutex) = 0.f;  // aligned to natural up orientation.

    /** Last folded state */
    bool mFoldedState GUARDED_BY(mMutex) = false;  // foldable: true means folded.

    /** Last hinge angle */
    float mHingeAngle GUARDED_BY(mMutex) = 0.f;  // foldable: 0.f is closed, M_PI flat open.

    std::vector<media::audio::common::Spatialization::Level> mLevels;
    std::vector<media::audio::common::HeadTracking::Mode> mHeadTrackingModes;
    std::vector<media::audio::common::Spatialization::Mode> mSpatializationModes;
    std::vector<audio_channel_mask_t> mChannelMasks;
    bool mSupportsHeadTracking;

    /** List of supported head tracking connection modes reported by the spatializer.
     * If the list is empty, the spatializer does not support any optional connection
     * mode and mode HeadTracking::ConnectionMode::FRAMEWORK_PROCESSED is assumed.
     * This is set in the factory constructor and can be accessed without mutex.
     */
    std::unordered_set<media::audio::common::HeadTracking::ConnectionMode>
            mSupportedHeadtrackingConnectionModes;
    /** Selected HT connection mode when several modes are supported by the spatializer */
    media::audio::common::HeadTracking::ConnectionMode mHeadtrackingConnectionMode =
            media::audio::common::HeadTracking::ConnectionMode::FRAMEWORK_PROCESSED;

    // Looper thread for mEngine callbacks
    class EngineCallbackHandler;

    sp<ALooper> mLooper;
    sp<EngineCallbackHandler> mHandler;

    size_t mNumActiveTracks GUARDED_BY(mMutex) = 0;
    std::vector<audio_latency_mode_t> mSupportedLatencyModes GUARDED_BY(mMutex);
    /** preference order for low latency modes according to persist.bluetooth.hid.transport */
    std::vector<audio_latency_mode_t> mOrderedLowLatencyModes;

    audio_latency_mode_t mRequestedLatencyMode GUARDED_BY(mMutex) = AUDIO_LATENCY_MODE_FREE;

    /** string to latency mode map used to parse bluetooth.core.le.dsa_transport_preference */
    static const std::map<std::string, audio_latency_mode_t> sStringToLatencyModeMap;
    static const std::vector<const char*> sHeadPoseKeys;

    // Local log for command messages.
    static constexpr int mMaxLocalLogLine = 10;
    SimpleLog mLocalLog{mMaxLocalLogLine};

    /**
     * @brief Calculate and record sensor data.
     * Dump to local log with max/average pose angle every mPoseRecordThreshold.
     */
    // Record one log line per second (up to mMaxLocalLogLine) to capture most recent sensor data.
    media::VectorRecorder mPoseRecorder GUARDED_BY(mMutex) {
        6 /* vectorSize */, std::chrono::seconds(1), mMaxLocalLogLine, { 3 } /* delimiterIdx */};
    // Record one log line per minute (up to mMaxLocalLogLine) to capture durable sensor data.
    media::VectorRecorder mPoseDurableRecorder GUARDED_BY(mMutex) {
        6 /* vectorSize */, std::chrono::minutes(1), mMaxLocalLogLine, { 3 } /* delimiterIdx */};
};  // Spatializer

}; // namespace android

#endif // ANDROID_MEDIA_SPATIALIZER_H