1 /*
2  * Copyright (C) 2022 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include "VisualizerContext.h"
18 
19 #include <algorithm>
20 #include <math.h>
21 #include <time.h>
22 
23 #include <android/binder_status.h>
24 #include <audio_utils/primitives.h>
25 #include <system/audio.h>
26 #include <Utils.h>
27 
28 #ifndef BUILD_FLOAT
29         #error AIDL Visualizer only support float 32bits, make sure add cflags -DBUILD_FLOAT,
30 #endif
31 
32 using aidl::android::hardware::audio::common::getChannelCount;
33 
34 namespace aidl::android::hardware::audio::effect {
35 
VisualizerContext(int statusDepth,const Parameter::Common & common)36 VisualizerContext::VisualizerContext(int statusDepth, const Parameter::Common& common)
37     : EffectContext(statusDepth, common) {
38 }
39 
~VisualizerContext()40 VisualizerContext::~VisualizerContext() {
41     mState = State::UNINITIALIZED;
42 }
43 
initParams(const Parameter::Common & common)44 RetCode VisualizerContext::initParams(const Parameter::Common& common) {
45     if (common.input != common.output) {
46         LOG(ERROR) << __func__ << " mismatch input: " << common.input.toString()
47                    << " and output: " << common.output.toString();
48         return RetCode::ERROR_ILLEGAL_PARAMETER;
49     }
50 
51     mState = State::INITIALIZED;
52     auto channelCount = getChannelCount(common.input.base.channelMask);
53 #ifdef SUPPORT_MC
54     if (channelCount < 1 || channelCount > FCC_LIMIT) return RetCode::ERROR_ILLEGAL_PARAMETER;
55 #else
56     if (channelCount != FCC_2) return RetCode::ERROR_ILLEGAL_PARAMETER;
57 #endif
58     mChannelCount = channelCount;
59     mCommon = common;
60     std::fill(mCaptureBuf.begin(), mCaptureBuf.end(), 0x80);
61     return RetCode::SUCCESS;
62 }
63 
enable()64 RetCode VisualizerContext::enable() {
65     if (mState != State::INITIALIZED) {
66         return RetCode::ERROR_EFFECT_LIB_ERROR;
67     }
68     mState = State::ACTIVE;
69     return RetCode::SUCCESS;
70 }
71 
disable()72 RetCode VisualizerContext::disable() {
73     if (mState != State::ACTIVE) {
74         return RetCode::ERROR_EFFECT_LIB_ERROR;
75     }
76     mState = State::INITIALIZED;
77     return RetCode::SUCCESS;
78 }
79 
reset()80 void VisualizerContext::reset() {
81     std::fill(mCaptureBuf.begin(), mCaptureBuf.end(), 0x80);
82 }
83 
setCaptureSamples(int samples)84 RetCode VisualizerContext::setCaptureSamples(int samples) {
85     mCaptureSamples = samples;
86     return RetCode::SUCCESS;
87 }
getCaptureSamples()88 int32_t VisualizerContext::getCaptureSamples() {
89     return mCaptureSamples;
90 }
91 
setMeasurementMode(Visualizer::MeasurementMode mode)92 RetCode VisualizerContext::setMeasurementMode(Visualizer::MeasurementMode mode) {
93     mMeasurementMode = mode;
94     return RetCode::SUCCESS;
95 }
getMeasurementMode()96 Visualizer::MeasurementMode VisualizerContext::getMeasurementMode() {
97     return mMeasurementMode;
98 }
99 
setScalingMode(Visualizer::ScalingMode mode)100 RetCode VisualizerContext::setScalingMode(Visualizer::ScalingMode mode) {
101     mScalingMode = mode;
102     return RetCode::SUCCESS;
103 }
getScalingMode()104 Visualizer::ScalingMode VisualizerContext::getScalingMode() {
105     return mScalingMode;
106 }
107 
setDownstreamLatency(int latency)108 RetCode VisualizerContext::setDownstreamLatency(int latency) {
109     mDownstreamLatency = latency;
110     return RetCode::SUCCESS;
111 }
112 
getDownstreamLatency()113 int VisualizerContext::getDownstreamLatency() {
114     return mDownstreamLatency;
115 }
116 
getDeltaTimeMsFromUpdatedTime_l()117 uint32_t VisualizerContext::getDeltaTimeMsFromUpdatedTime_l() {
118     uint32_t deltaMs = 0;
119     if (mBufferUpdateTime.tv_sec != 0) {
120         struct timespec ts;
121         if (clock_gettime(CLOCK_MONOTONIC, &ts) == 0) {
122             time_t secs = ts.tv_sec - mBufferUpdateTime.tv_sec;
123             long nsec = ts.tv_nsec - mBufferUpdateTime.tv_nsec;
124             if (nsec < 0) {
125                 --secs;
126                 nsec += 1000000000;
127             }
128             deltaMs = secs * 1000 + nsec / 1000000;
129         }
130     }
131     return deltaMs;
132 }
133 
getMeasure()134 Visualizer::Measurement VisualizerContext::getMeasure() {
135     uint16_t peakU16 = 0;
136     float sumRmsSquared = 0.0f;
137     uint8_t nbValidMeasurements = 0;
138 
139     {
140         // reset measurements if last measurement was too long ago (which implies stored
141         // measurements aren't relevant anymore and shouldn't bias the new one)
142         const uint32_t delayMs = getDeltaTimeMsFromUpdatedTime_l();
143         if (delayMs > kDiscardMeasurementsTimeMs) {
144             LOG(INFO) << __func__ << " Discarding " << delayMs << " ms old measurements";
145             for (uint32_t i = 0; i < mMeasurementWindowSizeInBuffers; i++) {
146                 mPastMeasurements[i].mIsValid = false;
147                 mPastMeasurements[i].mPeakU16 = 0;
148                 mPastMeasurements[i].mRmsSquared = 0;
149             }
150             mMeasurementBufferIdx = 0;
151         } else {
152             // only use actual measurements, otherwise the first RMS measure happening before
153             // MEASUREMENT_WINDOW_MAX_SIZE_IN_BUFFERS have been played will always be artificially
154             // low
155             for (uint32_t i = 0; i < mMeasurementWindowSizeInBuffers; i++) {
156                 if (mPastMeasurements[i].mIsValid) {
157                     if (mPastMeasurements[i].mPeakU16 > peakU16) {
158                         peakU16 = mPastMeasurements[i].mPeakU16;
159                     }
160                     sumRmsSquared += mPastMeasurements[i].mRmsSquared;
161                     nbValidMeasurements++;
162                 }
163             }
164         }
165     }
166 
167     float rms = nbValidMeasurements == 0 ? 0.0f : sqrtf(sumRmsSquared / nbValidMeasurements);
168     Visualizer::Measurement measure;
169     // convert from I16 sample values to mB and write results
170     measure.rms = (rms < 0.000016f) ? -9600 : (int32_t)(2000 * log10(rms / 32767.0f));
171     measure.peak = (peakU16 == 0) ? -9600 : (int32_t)(2000 * log10(peakU16 / 32767.0f));
172     LOG(VERBOSE) << __func__ << " peak " << peakU16 << " (" << measure.peak << "mB), rms " << rms
173                  << " (" << measure.rms << "mB)";
174     return measure;
175 }
176 
capture()177 std::vector<uint8_t> VisualizerContext::capture() {
178     uint32_t captureSamples = mCaptureSamples;
179     std::vector<uint8_t> result(captureSamples, 0x80);
180     // cts android.media.audio.cts.VisualizerTest expecting silence data when effect not running
181     // RETURN_VALUE_IF(mState != State::ACTIVE, result, "illegalState");
182     if (mState != State::ACTIVE) {
183         return result;
184     }
185 
186     const uint32_t deltaMs = getDeltaTimeMsFromUpdatedTime_l();
187     // if audio framework has stopped playing audio although the effect is still active we must
188     // clear the capture buffer to return silence
189     if ((mLastCaptureIdx == mCaptureIdx) && (mBufferUpdateTime.tv_sec != 0) &&
190         (deltaMs > kMaxStallTimeMs)) {
191         mBufferUpdateTime.tv_sec = 0;
192         return result;
193     }
194     int32_t latencyMs = mDownstreamLatency;
195     latencyMs -= deltaMs;
196     if (latencyMs < 0) {
197         latencyMs = 0;
198     }
199     uint32_t deltaSamples = captureSamples + mCommon.input.base.sampleRate * latencyMs / 1000;
200 
201     // large sample rate, latency, or capture size, could cause overflow.
202     // do not offset more than the size of buffer.
203     if (deltaSamples > kMaxCaptureBufSize) {
204         android_errorWriteLog(0x534e4554, "31781965");
205         deltaSamples = kMaxCaptureBufSize;
206     }
207 
208     int32_t capturePoint;
209     __builtin_sub_overflow((int32_t) mCaptureIdx, deltaSamples, &capturePoint);
210     // a negative capturePoint means we wrap the buffer.
211     if (capturePoint < 0) {
212         uint32_t size = -capturePoint;
213         if (size > captureSamples) {
214             size = captureSamples;
215         }
216         std::copy(std::begin(mCaptureBuf) + kMaxCaptureBufSize - size,
217                   std::begin(mCaptureBuf) + kMaxCaptureBufSize, result.begin());
218         captureSamples -= size;
219         capturePoint = 0;
220     }
221     std::copy(std::begin(mCaptureBuf) + capturePoint,
222               std::begin(mCaptureBuf) + capturePoint + captureSamples,
223               result.begin() + mCaptureSamples - captureSamples);
224     mLastCaptureIdx = mCaptureIdx;
225     return result;
226 }
227 
process(float * in,float * out,int samples)228 IEffect::Status VisualizerContext::process(float* in, float* out, int samples) {
229     IEffect::Status result = {STATUS_NOT_ENOUGH_DATA, 0, 0};
230     RETURN_VALUE_IF(in == nullptr || out == nullptr || samples == 0, result, "dataBufferError");
231 
232     result.status = STATUS_INVALID_OPERATION;
233     RETURN_VALUE_IF(mState != State::ACTIVE, result, "stateNotActive");
234     // perform measurements if needed
235     if (mMeasurementMode == Visualizer::MeasurementMode::PEAK_RMS) {
236         // find the peak and RMS squared for the new buffer
237         float rmsSqAcc = 0;
238         float maxSample = 0.f;
239         for (size_t inIdx = 0; inIdx < (unsigned) samples; ++inIdx) {
240             maxSample = fmax(maxSample, fabs(in[inIdx]));
241             rmsSqAcc += in[inIdx] * in[inIdx];
242         }
243         maxSample *= 1 << 15; // scale to int16_t, with exactly 1 << 15 representing positive num.
244         rmsSqAcc *= 1 << 30; // scale to int16_t * 2
245         mPastMeasurements[mMeasurementBufferIdx] = {.mIsValid = true,
246                                                     .mPeakU16 = (uint16_t)maxSample,
247                                                     .mRmsSquared = rmsSqAcc / samples};
248         if (++mMeasurementBufferIdx >= mMeasurementWindowSizeInBuffers) {
249             mMeasurementBufferIdx = 0;
250         }
251     }
252 
253     float fscale;  // multiplicative scale
254     if (mScalingMode == Visualizer::ScalingMode::NORMALIZED) {
255         // derive capture scaling factor from peak value in current buffer
256         // this gives more interesting captures for display.
257         float maxSample = 0.f;
258         for (size_t inIdx = 0; inIdx < (unsigned)samples; ) {
259             // we reconstruct the actual summed value to ensure proper normalization
260             // for multichannel outputs (channels > 2 may often be 0).
261             float smp = 0.f;
262             for (int i = 0; i < mChannelCount; ++i) {
263                 smp += in[inIdx++];
264             }
265             maxSample = fmax(maxSample, fabs(smp));
266         }
267         if (maxSample > 0.f) {
268             fscale = 0.99f / maxSample;
269             int exp; // unused
270             const float significand = frexp(fscale, &exp);
271             if (significand == 0.5f) {
272                 fscale *= 255.f / 256.f; // avoid returning unaltered PCM signal
273             }
274         } else {
275             // scale doesn't matter, the values are all 0.
276             fscale = 1.f;
277         }
278     } else {
279         assert(mScalingMode == Visualizer::ScalingMode::AS_PLAYED);
280         // Note: if channels are uncorrelated, 1/sqrt(N) could be used at the risk of clipping.
281         fscale = 1.f / mChannelCount;  // account for summing all the channels together.
282     }
283 
284     uint32_t captIdx;
285     uint32_t inIdx;
286     for (inIdx = 0, captIdx = mCaptureIdx; inIdx < (unsigned)samples; captIdx++) {
287         // wrap
288         if (captIdx >= kMaxCaptureBufSize) {
289             captIdx = 0;
290         }
291 
292         float smp = 0.f;
293         for (uint32_t i = 0; i < mChannelCount; ++i) {
294             smp += in[inIdx++];
295         }
296         mCaptureBuf[captIdx] = clamp8_from_float(smp * fscale);
297     }
298 
299     // the following two should really be atomic, though it probably doesn't
300     // matter much for visualization purposes
301     mCaptureIdx = captIdx;
302     // update last buffer update time stamp
303     if (clock_gettime(CLOCK_MONOTONIC, &mBufferUpdateTime) < 0) {
304         mBufferUpdateTime.tv_sec = 0;
305     }
306 
307     // TODO: handle access_mode
308     memcpy(out, in, samples * sizeof(float));
309     return {STATUS_OK, samples, samples};
310 }
311 
312 }  // namespace aidl::android::hardware::audio::effect
313