1 /*
2  * Copyright 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 // TODO(b/129481165): remove the #pragma below and fix conversion issues
18 #pragma clang diagnostic push
19 #pragma clang diagnostic ignored "-Wconversion"
20 #pragma clang diagnostic ignored "-Wextra"
21 
22 //#define LOG_NDEBUG 0
23 #define ATRACE_TAG ATRACE_TAG_GRAPHICS
24 #undef LOG_TAG
25 #define LOG_TAG "RegionSamplingThread"
26 
27 #include "RegionSamplingThread.h"
28 
29 #include <compositionengine/Display.h>
30 #include <compositionengine/impl/OutputCompositionState.h>
31 #include <cutils/properties.h>
32 #include <ftl/future.h>
33 #include <gui/SpHash.h>
34 #include <gui/SyncScreenCaptureListener.h>
35 #include <renderengine/impl/ExternalTexture.h>
36 #include <ui/DisplayStatInfo.h>
37 #include <utils/Trace.h>
38 
39 #include <string>
40 
41 #include "DisplayDevice.h"
42 #include "DisplayRenderArea.h"
43 #include "FrontEnd/LayerCreationArgs.h"
44 #include "Layer.h"
45 #include "RenderAreaBuilder.h"
46 #include "Scheduler/VsyncController.h"
47 #include "SurfaceFlinger.h"
48 
49 namespace android {
50 using namespace std::chrono_literals;
51 
52 using gui::SpHash;
53 
54 constexpr auto lumaSamplingStepTag = "LumaSamplingStep";
55 enum class samplingStep {
56     noWorkNeeded,
57     idleTimerWaiting,
58     waitForQuietFrame,
59     waitForSamplePhase,
60     sample
61 };
62 
63 constexpr auto defaultRegionSamplingWorkDuration = 3ms;
64 constexpr auto defaultRegionSamplingPeriod = 100ms;
65 constexpr auto defaultRegionSamplingTimerTimeout = 100ms;
66 constexpr auto maxRegionSamplingDelay = 100ms;
67 // TODO: (b/127403193) duration to string conversion could probably be constexpr
68 template <typename Rep, typename Per>
toNsString(std::chrono::duration<Rep,Per> t)69 inline std::string toNsString(std::chrono::duration<Rep, Per> t) {
70     return std::to_string(std::chrono::duration_cast<std::chrono::nanoseconds>(t).count());
71 }
72 
EnvironmentTimingTunables()73 RegionSamplingThread::EnvironmentTimingTunables::EnvironmentTimingTunables() {
74     char value[PROPERTY_VALUE_MAX] = {};
75 
76     property_get("debug.sf.region_sampling_duration_ns", value,
77                  toNsString(defaultRegionSamplingWorkDuration).c_str());
78     int const samplingDurationNsRaw = atoi(value);
79 
80     property_get("debug.sf.region_sampling_period_ns", value,
81                  toNsString(defaultRegionSamplingPeriod).c_str());
82     int const samplingPeriodNsRaw = atoi(value);
83 
84     property_get("debug.sf.region_sampling_timer_timeout_ns", value,
85                  toNsString(defaultRegionSamplingTimerTimeout).c_str());
86     int const samplingTimerTimeoutNsRaw = atoi(value);
87 
88     if ((samplingPeriodNsRaw < 0) || (samplingTimerTimeoutNsRaw < 0)) {
89         ALOGW("User-specified sampling tuning options nonsensical. Using defaults");
90         mSamplingDuration = defaultRegionSamplingWorkDuration;
91         mSamplingPeriod = defaultRegionSamplingPeriod;
92         mSamplingTimerTimeout = defaultRegionSamplingTimerTimeout;
93     } else {
94         mSamplingDuration = std::chrono::nanoseconds(samplingDurationNsRaw);
95         mSamplingPeriod = std::chrono::nanoseconds(samplingPeriodNsRaw);
96         mSamplingTimerTimeout = std::chrono::nanoseconds(samplingTimerTimeoutNsRaw);
97     }
98 }
99 
RegionSamplingThread(SurfaceFlinger & flinger,const TimingTunables & tunables)100 RegionSamplingThread::RegionSamplingThread(SurfaceFlinger& flinger, const TimingTunables& tunables)
101       : mFlinger(flinger),
102         mTunables(tunables),
103         mIdleTimer(
104                 "RegSampIdle",
105                 std::chrono::duration_cast<std::chrono::milliseconds>(
106                         mTunables.mSamplingTimerTimeout),
107                 [] {}, [this] { checkForStaleLuma(); }),
108         mLastSampleTime(0ns) {
__anonedeb02ff0302() 109     mThread = std::thread([this]() { threadMain(); });
110     pthread_setname_np(mThread.native_handle(), "RegionSampling");
111     mIdleTimer.start();
112 }
113 
RegionSamplingThread(SurfaceFlinger & flinger)114 RegionSamplingThread::RegionSamplingThread(SurfaceFlinger& flinger)
115       : RegionSamplingThread(flinger,
116                              TimingTunables{defaultRegionSamplingWorkDuration,
117                                             defaultRegionSamplingPeriod,
118                                             defaultRegionSamplingTimerTimeout}) {}
119 
~RegionSamplingThread()120 RegionSamplingThread::~RegionSamplingThread() {
121     mIdleTimer.stop();
122 
123     {
124         std::lock_guard lock(mThreadControlMutex);
125         mRunning = false;
126         mCondition.notify_one();
127     }
128 
129     if (mThread.joinable()) {
130         mThread.join();
131     }
132 }
133 
addListener(const Rect & samplingArea,uint32_t stopLayerId,const sp<IRegionSamplingListener> & listener)134 void RegionSamplingThread::addListener(const Rect& samplingArea, uint32_t stopLayerId,
135                                        const sp<IRegionSamplingListener>& listener) {
136     sp<IBinder> asBinder = IInterface::asBinder(listener);
137     asBinder->linkToDeath(sp<DeathRecipient>::fromExisting(this));
138     std::lock_guard lock(mSamplingMutex);
139     mDescriptors.emplace(wp<IBinder>(asBinder), Descriptor{samplingArea, stopLayerId, listener});
140 }
141 
removeListener(const sp<IRegionSamplingListener> & listener)142 void RegionSamplingThread::removeListener(const sp<IRegionSamplingListener>& listener) {
143     std::lock_guard lock(mSamplingMutex);
144     mDescriptors.erase(wp<IBinder>(IInterface::asBinder(listener)));
145 }
146 
checkForStaleLuma()147 void RegionSamplingThread::checkForStaleLuma() {
148     std::lock_guard lock(mThreadControlMutex);
149 
150     if (mSampleRequestTime.has_value()) {
151         ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::waitForSamplePhase));
152         mSampleRequestTime.reset();
153         mFlinger.scheduleSample();
154     }
155 }
156 
onCompositionComplete(std::optional<std::chrono::steady_clock::time_point> samplingDeadline)157 void RegionSamplingThread::onCompositionComplete(
158         std::optional<std::chrono::steady_clock::time_point> samplingDeadline) {
159     doSample(samplingDeadline);
160 }
161 
doSample(std::optional<std::chrono::steady_clock::time_point> samplingDeadline)162 void RegionSamplingThread::doSample(
163         std::optional<std::chrono::steady_clock::time_point> samplingDeadline) {
164     std::lock_guard lock(mThreadControlMutex);
165     const auto now = std::chrono::steady_clock::now();
166     if (mLastSampleTime + mTunables.mSamplingPeriod > now) {
167         // content changed, but we sampled not too long ago, so we need to sample some time in the
168         // future.
169         ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::idleTimerWaiting));
170         mSampleRequestTime = now;
171         return;
172     }
173     if (!mSampleRequestTime.has_value() || now - *mSampleRequestTime < maxRegionSamplingDelay) {
174         // If there is relatively little time left for surfaceflinger
175         // until the next vsync deadline, defer this sampling work
176         // to a later frame, when hopefully there will be more time.
177         if (samplingDeadline.has_value() && now + mTunables.mSamplingDuration > *samplingDeadline) {
178             ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::waitForQuietFrame));
179             mSampleRequestTime = mSampleRequestTime.value_or(now);
180             return;
181         }
182     }
183 
184     ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::sample));
185 
186     mSampleRequestTime.reset();
187     mLastSampleTime = now;
188 
189     mIdleTimer.reset();
190 
191     mSampleRequested = true;
192     mCondition.notify_one();
193 }
194 
binderDied(const wp<IBinder> & who)195 void RegionSamplingThread::binderDied(const wp<IBinder>& who) {
196     std::lock_guard lock(mSamplingMutex);
197     mDescriptors.erase(who);
198 }
199 
sampleArea(const uint32_t * data,int32_t width,int32_t height,int32_t stride,uint32_t orientation,const Rect & sample_area)200 float sampleArea(const uint32_t* data, int32_t width, int32_t height, int32_t stride,
201                  uint32_t orientation, const Rect& sample_area) {
202     if (!sample_area.isValid() || (sample_area.getWidth() > width) ||
203         (sample_area.getHeight() > height)) {
204         ALOGE("invalid sampling region requested");
205         return 0.0f;
206     }
207 
208     const uint32_t pixelCount =
209             (sample_area.bottom - sample_area.top) * (sample_area.right - sample_area.left);
210     uint32_t accumulatedLuma = 0;
211 
212     // Calculates luma with approximation of Rec. 709 primaries
213     for (int32_t row = sample_area.top; row < sample_area.bottom; ++row) {
214         const uint32_t* rowBase = data + row * stride;
215         for (int32_t column = sample_area.left; column < sample_area.right; ++column) {
216             uint32_t pixel = rowBase[column];
217             const uint32_t r = pixel & 0xFF;
218             const uint32_t g = (pixel >> 8) & 0xFF;
219             const uint32_t b = (pixel >> 16) & 0xFF;
220             const uint32_t luma = (r * 7 + b * 2 + g * 23) >> 5;
221             accumulatedLuma += luma;
222         }
223     }
224 
225     return accumulatedLuma / (255.0f * pixelCount);
226 }
227 
sampleBuffer(const sp<GraphicBuffer> & buffer,const Point & leftTop,const std::vector<RegionSamplingThread::Descriptor> & descriptors,uint32_t orientation)228 std::vector<float> RegionSamplingThread::sampleBuffer(
229         const sp<GraphicBuffer>& buffer, const Point& leftTop,
230         const std::vector<RegionSamplingThread::Descriptor>& descriptors, uint32_t orientation) {
231     void* data_raw = nullptr;
232     buffer->lock(GRALLOC_USAGE_SW_READ_OFTEN, &data_raw);
233     std::shared_ptr<uint32_t> data(reinterpret_cast<uint32_t*>(data_raw),
234                                    [&buffer](auto) { buffer->unlock(); });
235     if (!data) return {};
236 
237     const int32_t width = buffer->getWidth();
238     const int32_t height = buffer->getHeight();
239     const int32_t stride = buffer->getStride();
240     std::vector<float> lumas(descriptors.size());
241     std::transform(descriptors.begin(), descriptors.end(), lumas.begin(),
242                    [&](auto const& descriptor) {
243                        return sampleArea(data.get(), width, height, stride, orientation,
244                                          descriptor.area - leftTop);
245                    });
246     return lumas;
247 }
248 
captureSample()249 void RegionSamplingThread::captureSample() {
250     ATRACE_CALL();
251     std::lock_guard lock(mSamplingMutex);
252 
253     if (mDescriptors.empty()) {
254         return;
255     }
256 
257     wp<const DisplayDevice> displayWeak;
258 
259     ui::LayerStack layerStack;
260     ui::Transform::RotationFlags orientation;
261     ui::Size displaySize;
262 
263     {
264         // TODO(b/159112860): Don't keep sp<DisplayDevice> outside of SF main thread
265         const sp<const DisplayDevice> display = mFlinger.getDefaultDisplayDevice();
266         displayWeak = display;
267         layerStack = display->getLayerStack();
268         orientation = ui::Transform::toRotationFlags(display->getOrientation());
269         displaySize = display->getSize();
270     }
271 
272     std::vector<RegionSamplingThread::Descriptor> descriptors;
273     Region sampleRegion;
274     for (const auto& [listener, descriptor] : mDescriptors) {
275         sampleRegion.orSelf(descriptor.area);
276         descriptors.emplace_back(descriptor);
277     }
278 
279     const Rect sampledBounds = sampleRegion.bounds();
280 
281     std::unordered_set<sp<IRegionSamplingListener>, SpHash<IRegionSamplingListener>> listeners;
282 
283     auto layerFilterFn = [&](const char* layerName, uint32_t layerId, const Rect& bounds,
284                              const ui::Transform transform, bool& outStopTraversal) -> bool {
285         // Likewise if we just found a stop layer, set the flag and abort
286         for (const auto& [area, stopLayerId, listener] : descriptors) {
287             if (stopLayerId != UNASSIGNED_LAYER_ID && layerId == stopLayerId) {
288                 outStopTraversal = true;
289                 return false;
290             }
291         }
292 
293         // Compute the layer's position on the screen
294         constexpr bool roundOutwards = true;
295         Rect transformed = transform.transform(bounds, roundOutwards);
296 
297         // If this layer doesn't intersect with the larger sampledBounds, skip capturing it
298         Rect ignore;
299         if (!transformed.intersect(sampledBounds, &ignore)) return false;
300 
301         // If the layer doesn't intersect a sampling area, skip capturing it
302         bool intersectsAnyArea = false;
303         for (const auto& [area, stopLayer, listener] : descriptors) {
304             if (transformed.intersect(area, &ignore)) {
305                 intersectsAnyArea = true;
306                 listeners.insert(listener);
307             }
308         }
309         if (!intersectsAnyArea) return false;
310 
311         ALOGV("Traversing [%s] [%d, %d, %d, %d]", layerName, bounds.left, bounds.top, bounds.right,
312               bounds.bottom);
313 
314         return true;
315     };
316 
317     auto filterFn = [&](const frontend::LayerSnapshot& snapshot, bool& outStopTraversal) -> bool {
318         const Rect bounds = frontend::RequestedLayerState::reduce(Rect(snapshot.geomLayerBounds),
319                                                                   snapshot.transparentRegionHint);
320         const ui::Transform transform = snapshot.geomLayerTransform;
321         return layerFilterFn(snapshot.name.c_str(), snapshot.path.id, bounds, transform,
322                              outStopTraversal);
323     };
324     auto getLayerSnapshotsFn =
325             mFlinger.getLayerSnapshotsForScreenshots(layerStack, CaptureArgs::UNSET_UID, filterFn);
326 
327     std::shared_ptr<renderengine::ExternalTexture> buffer = nullptr;
328     if (mCachedBuffer && mCachedBuffer->getBuffer()->getWidth() == sampledBounds.getWidth() &&
329         mCachedBuffer->getBuffer()->getHeight() == sampledBounds.getHeight()) {
330         buffer = mCachedBuffer;
331     } else {
332         const uint32_t usage =
333                 GRALLOC_USAGE_SW_READ_OFTEN | GRALLOC_USAGE_HW_RENDER | GRALLOC_USAGE_HW_TEXTURE;
334         sp<GraphicBuffer> graphicBuffer =
335                 sp<GraphicBuffer>::make(sampledBounds.getWidth(), sampledBounds.getHeight(),
336                                         PIXEL_FORMAT_RGBA_8888, 1, usage, "RegionSamplingThread");
337         const status_t bufferStatus = graphicBuffer->initCheck();
338         LOG_ALWAYS_FATAL_IF(bufferStatus != OK, "captureSample: Buffer failed to allocate: %d",
339                             bufferStatus);
340         buffer = std::make_shared<
341                 renderengine::impl::ExternalTexture>(graphicBuffer, mFlinger.getRenderEngine(),
342                                                      renderengine::impl::ExternalTexture::Usage::
343                                                              WRITEABLE);
344     }
345 
346     constexpr bool kRegionSampling = true;
347     constexpr bool kGrayscale = false;
348     constexpr bool kIsProtected = false;
349 
350     SurfaceFlinger::RenderAreaBuilderVariant
351             renderAreaBuilder(std::in_place_type<DisplayRenderAreaBuilder>, sampledBounds,
352                               sampledBounds.getSize(), ui::Dataspace::V0_SRGB, displayWeak,
353                               RenderArea::Options::CAPTURE_SECURE_LAYERS);
354 
355     FenceResult fenceResult;
356     if (FlagManager::getInstance().single_hop_screenshot() &&
357         FlagManager::getInstance().ce_fence_promise() && mFlinger.mRenderEngine->isThreaded()) {
358         std::vector<sp<LayerFE>> layerFEs;
359         auto displayState =
360                 mFlinger.getDisplayAndLayerSnapshotsFromMainThread(renderAreaBuilder,
361                                                                    getLayerSnapshotsFn, layerFEs);
362         fenceResult =
363                 mFlinger.captureScreenshot(renderAreaBuilder, buffer, kRegionSampling, kGrayscale,
364                                            kIsProtected, nullptr, displayState, layerFEs)
365                         .get();
366     } else {
367         fenceResult =
368                 mFlinger.captureScreenshotLegacy(renderAreaBuilder, getLayerSnapshotsFn, buffer,
369                                                  kRegionSampling, kGrayscale, kIsProtected, nullptr)
370                         .get();
371     }
372     if (fenceResult.ok()) {
373         fenceResult.value()->waitForever(LOG_TAG);
374     }
375 
376     std::vector<Descriptor> activeDescriptors;
377     for (const auto& descriptor : descriptors) {
378         if (listeners.count(descriptor.listener) != 0) {
379             activeDescriptors.emplace_back(descriptor);
380         }
381     }
382 
383     ALOGV("Sampling %zu descriptors", activeDescriptors.size());
384     std::vector<float> lumas = sampleBuffer(buffer->getBuffer(), sampledBounds.leftTop(),
385                                             activeDescriptors, orientation);
386     if (lumas.size() != activeDescriptors.size()) {
387         ALOGW("collected %zu median luma values for %zu descriptors", lumas.size(),
388               activeDescriptors.size());
389         return;
390     }
391 
392     for (size_t d = 0; d < activeDescriptors.size(); ++d) {
393         activeDescriptors[d].listener->onSampleCollected(lumas[d]);
394     }
395 
396     mCachedBuffer = buffer;
397     ATRACE_INT(lumaSamplingStepTag, static_cast<int>(samplingStep::noWorkNeeded));
398 }
399 
400 // NO_THREAD_SAFETY_ANALYSIS is because std::unique_lock presently lacks thread safety annotations.
threadMain()401 void RegionSamplingThread::threadMain() NO_THREAD_SAFETY_ANALYSIS {
402     std::unique_lock<std::mutex> lock(mThreadControlMutex);
403     while (mRunning) {
404         if (mSampleRequested) {
405             mSampleRequested = false;
406             lock.unlock();
407             captureSample();
408             lock.lock();
409         }
410         mCondition.wait(lock, [this]() REQUIRES(mThreadControlMutex) {
411             return mSampleRequested || !mRunning;
412         });
413     }
414 }
415 
416 } // namespace android
417 
418 // TODO(b/129481165): remove the #pragma below and fix conversion issues
419 #pragma clang diagnostic pop // ignored "-Wconversion -Wextra"
420