1 /*
2 * Copyright 2018 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "Codec2-InputSurfaceConnection"
19 #include <android-base/logging.h>
20
21 #include <codec2/hidl/1.0/InputSurfaceConnection.h>
22 #include <codec2/hidl/1.0/InputSurfaceConnection.h>
23
24 #include <memory>
25 #include <list>
26 #include <mutex>
27 #include <atomic>
28
29 #include <hidl/HidlSupport.h>
30 #include <media/stagefright/bqhelper/ComponentWrapper.h>
31 #include <system/graphics.h>
32 #include <ui/GraphicBuffer.h>
33 #include <utils/Errors.h>
34
35 #include <C2.h>
36 #include <C2AllocatorGralloc.h>
37 #include <C2BlockInternal.h>
38 #include <C2Buffer.h>
39 #include <C2Component.h>
40 #include <C2Config.h>
41 #include <C2Debug.h>
42 #include <C2PlatformSupport.h>
43 #include <C2Work.h>
44
45 namespace android {
46 namespace hardware {
47 namespace media {
48 namespace c2 {
49 namespace V1_0 {
50 namespace utils {
51
52 constexpr int32_t kBufferCount = 16;
53
54 using namespace ::android;
55 using ::android::hardware::hidl_string;
56 using ::android::hardware::hidl_vec;
57 using ::android::hardware::Return;
58
59 namespace /* unnamed */ {
60
61 class Buffer2D : public C2Buffer {
62 public:
Buffer2D(C2ConstGraphicBlock block)63 explicit Buffer2D(C2ConstGraphicBlock block) : C2Buffer({ block }) {
64 }
65 };
66
67 } // unnamed namespace
68
69 // Derived class of ComponentWrapper for use with
70 // GraphicBufferSource::configure().
71 //
72 struct InputSurfaceConnection::Impl : public ComponentWrapper {
73
Implandroid::hardware::media::c2::V1_0::utils::InputSurfaceConnection::Impl74 Impl(const sp<GraphicBufferSource>& source,
75 const std::shared_ptr<C2Component>& localComp)
76 : mSource{source}, mLocalComp{localComp}, mSink{}, mFrameIndex{0} {
77 std::shared_ptr<C2ComponentInterface> intf = localComp->intf();
78 mSinkName = intf ? intf->getName() : "";
79 }
80
Implandroid::hardware::media::c2::V1_0::utils::InputSurfaceConnection::Impl81 Impl(const sp<GraphicBufferSource>& source,
82 const sp<IInputSink>& sink)
83 : mSource{source}, mLocalComp{}, mSink{sink}, mFrameIndex{0} {
84 Return<sp<IConfigurable>> transResult = sink->getConfigurable();
85 if (!transResult.isOk()) {
86 LOG(ERROR) << "Remote sink is dead.";
87 return;
88 }
89 mSinkConfigurable =
90 static_cast<sp<IConfigurable>>(transResult);
91 if (!mSinkConfigurable) {
92 LOG(ERROR) << "Remote sink is not configurable.";
93 mSinkName = "";
94 return;
95 }
96
97 hidl_string name;
98 Return<void> transStatus = mSinkConfigurable->getName(
__anon0a65086b0202(const hidl_string& n) 99 [&name](const hidl_string& n) {
100 name = n;
101 });
102 if (!transStatus.isOk()) {
103 LOG(ERROR) << "Remote sink's configurable is dead.";
104 mSinkName = "";
105 return;
106 }
107 mSinkName = name.c_str();
108 }
109
~Implandroid::hardware::media::c2::V1_0::utils::InputSurfaceConnection::Impl110 virtual ~Impl() {
111 mSource->stop();
112 mSource->release();
113 }
114
initandroid::hardware::media::c2::V1_0::utils::InputSurfaceConnection::Impl115 bool init() {
116 if (mSource == nullptr) {
117 return false;
118 }
119 status_t err = mSource->initCheck();
120 if (err != OK) {
121 LOG(WARNING) << "Impl::init -- GraphicBufferSource init failed: "
122 << "status = " << err << ".";
123 return false;
124 }
125
126 // TODO: read settings properly from the interface
127 C2StreamPictureSizeInfo::input inputSize;
128 C2StreamUsageTuning::input usage;
129 c2_status_t c2Status = queryFromSink({ &inputSize, &usage },
130 {},
131 C2_MAY_BLOCK,
132 nullptr);
133 if (c2Status != C2_OK) {
134 LOG(WARNING) << "Impl::init -- cannot query information from "
135 "the component interface: "
136 << "status = " << asString(c2Status) << ".";
137 return false;
138 }
139
140 // TODO: proper color aspect & dataspace
141 android_dataspace dataSpace = HAL_DATASPACE_BT709;
142
143 // TODO: use the usage read from intf
144 // uint32_t grallocUsage =
145 // C2AndroidMemoryUsage(C2MemoryUsage(usage.value)).
146 // asGrallocUsage();
147
148 uint32_t grallocUsage =
149 mSinkName.compare(0, 11, "c2.android.") == 0 ?
150 GRALLOC_USAGE_SW_READ_OFTEN :
151 GRALLOC_USAGE_HW_VIDEO_ENCODER;
152
153 err = mSource->configure(
154 this, dataSpace, kBufferCount,
155 inputSize.width, inputSize.height,
156 grallocUsage);
157 if (err != OK) {
158 LOG(WARNING) << "Impl::init -- GBS configure failed: "
159 << "status = " << err << ".";
160 return false;
161 }
162 for (int32_t i = 0; i < kBufferCount; ++i) {
163 if (!mSource->onInputBufferAdded(i).isOk()) {
164 LOG(WARNING) << "Impl::init: failed to populate GBS slots.";
165 return false;
166 }
167 }
168 if (!mSource->start().isOk()) {
169 LOG(WARNING) << "Impl::init -- GBS failed to start.";
170 return false;
171 }
172 mAllocatorMutex.lock();
173 c2_status_t c2err = GetCodec2PlatformAllocatorStore()->fetchAllocator(
174 C2AllocatorStore::PLATFORM_START + 1, // GRALLOC
175 &mAllocator);
176 mAllocatorMutex.unlock();
177 if (c2err != OK) {
178 LOG(WARNING) << "Impl::init -- failed to fetch gralloc allocator: "
179 << "status = " << asString(c2err) << ".";
180 return false;
181 }
182 return true;
183 }
184
185 // From ComponentWrapper
submitBufferandroid::hardware::media::c2::V1_0::utils::InputSurfaceConnection::Impl186 virtual status_t submitBuffer(
187 int32_t bufferId,
188 const sp<GraphicBuffer>& buffer,
189 int64_t timestamp,
190 int fenceFd) override {
191 LOG(VERBOSE) << "Impl::submitBuffer -- bufferId = " << bufferId << ".";
192 // TODO: Use fd to construct fence
193 (void)fenceFd;
194
195 std::shared_ptr<C2GraphicAllocation> alloc;
196 C2Handle* handle = WrapNativeCodec2GrallocHandle(
197 buffer->handle,
198 buffer->width, buffer->height,
199 buffer->format, buffer->usage, buffer->stride);
200 mAllocatorMutex.lock();
201 c2_status_t err = mAllocator->priorGraphicAllocation(handle, &alloc);
202 mAllocatorMutex.unlock();
203 if (err != OK) {
204 return UNKNOWN_ERROR;
205 }
206 std::shared_ptr<C2GraphicBlock> block =
207 _C2BlockFactory::CreateGraphicBlock(alloc);
208
209 std::unique_ptr<C2Work> work(new C2Work);
210 work->input.flags = (C2FrameData::flags_t)0;
211 work->input.ordinal.timestamp = timestamp;
212 work->input.ordinal.frameIndex = mFrameIndex.fetch_add(
213 1, std::memory_order_relaxed);
214 work->input.buffers.clear();
215 std::shared_ptr<C2Buffer> c2Buffer(
216 // TODO: fence
217 new Buffer2D(block->share(
218 C2Rect(block->width(), block->height()), ::C2Fence())),
219 [bufferId, source = mSource](C2Buffer* ptr) {
220 delete ptr;
221 if (source != nullptr) {
222 // TODO: fence
223 (void)source->onInputBufferEmptied(bufferId, -1);
224 }
225 });
226 work->input.buffers.push_back(c2Buffer);
227 work->worklets.clear();
228 work->worklets.emplace_back(new C2Worklet);
229 std::list<std::unique_ptr<C2Work>> items;
230 items.push_back(std::move(work));
231
232 err = queueToSink(&items);
233 return (err == C2_OK) ? OK : UNKNOWN_ERROR;
234 }
235
submitEosandroid::hardware::media::c2::V1_0::utils::InputSurfaceConnection::Impl236 virtual status_t submitEos(int32_t bufferId) override {
237 LOG(VERBOSE) << "Impl::submitEos -- bufferId = " << bufferId << ".";
238 (void)bufferId;
239
240 std::unique_ptr<C2Work> work(new C2Work);
241 work->input.flags = (C2FrameData::flags_t)0;
242 work->input.ordinal.frameIndex = mFrameIndex.fetch_add(
243 1, std::memory_order_relaxed);
244 work->input.buffers.clear();
245 work->worklets.clear();
246 work->worklets.emplace_back(new C2Worklet);
247 std::list<std::unique_ptr<C2Work>> items;
248 items.push_back(std::move(work));
249
250 c2_status_t err = queueToSink(&items);
251 return (err == C2_OK) ? OK : UNKNOWN_ERROR;
252 }
253
dispatchDataSpaceChangedandroid::hardware::media::c2::V1_0::utils::InputSurfaceConnection::Impl254 virtual void dispatchDataSpaceChanged(
255 int32_t dataSpace, int32_t aspects, int32_t pixelFormat) override {
256 // TODO
257 (void)dataSpace;
258 (void)aspects;
259 (void)pixelFormat;
260 }
261
262 // Configurable interface for InputSurfaceConnection::Impl.
263 //
264 // This class is declared as an inner class so that it will have access to
265 // all Impl's members.
266 struct ConfigurableIntf : public ConfigurableC2Intf {
267 sp<Impl> mConnection;
ConfigurableIntfandroid::hardware::media::c2::V1_0::utils::InputSurfaceConnection::Impl::ConfigurableIntf268 ConfigurableIntf(const sp<Impl>& connection)
269 : ConfigurableC2Intf{"input-surface-connection", 0},
270 mConnection{connection} {}
271 virtual c2_status_t config(
272 const std::vector<C2Param*> ¶ms,
273 c2_blocking_t mayBlock,
274 std::vector<std::unique_ptr<C2SettingResult>> *const failures
275 ) override;
276 virtual c2_status_t query(
277 const std::vector<C2Param::Index> &indices,
278 c2_blocking_t mayBlock,
279 std::vector<std::unique_ptr<C2Param>> *const params) const override;
280 virtual c2_status_t querySupportedParams(
281 std::vector<std::shared_ptr<C2ParamDescriptor>> *const params
282 ) const override;
283 virtual c2_status_t querySupportedValues(
284 std::vector<C2FieldSupportedValuesQuery> &fields,
285 c2_blocking_t mayBlock) const override;
286 };
287
288 private:
queryFromSinkandroid::hardware::media::c2::V1_0::utils::InputSurfaceConnection::Impl289 c2_status_t queryFromSink(
290 const std::vector<C2Param*> &stackParams,
291 const std::vector<C2Param::Index> &heapParamIndices,
292 c2_blocking_t mayBlock,
293 std::vector<std::unique_ptr<C2Param>>* const heapParams) {
294 if (mLocalComp) {
295 std::shared_ptr<C2ComponentInterface> intf = mLocalComp->intf();
296 if (intf) {
297 return intf->query_vb(stackParams,
298 heapParamIndices,
299 mayBlock,
300 heapParams);
301 } else {
302 LOG(ERROR) << "queryFromSink -- "
303 << "component does not have an interface.";
304 return C2_BAD_STATE;
305 }
306 }
307
308 CHECK(mSink) << "-- queryFromSink "
309 << "-- connection has no sink.";
310 CHECK(mSinkConfigurable) << "-- queryFromSink "
311 << "-- sink has no configurable.";
312
313 hidl_vec<ParamIndex> indices(
314 stackParams.size() + heapParamIndices.size());
315 size_t numIndices = 0;
316 for (C2Param* const& stackParam : stackParams) {
317 if (!stackParam) {
318 LOG(DEBUG) << "queryFromSink -- null stack param encountered.";
319 continue;
320 }
321 indices[numIndices++] = static_cast<ParamIndex>(stackParam->index());
322 }
323 size_t numStackIndices = numIndices;
324 for (const C2Param::Index& index : heapParamIndices) {
325 indices[numIndices++] =
326 static_cast<ParamIndex>(static_cast<uint32_t>(index));
327 }
328 indices.resize(numIndices);
329 if (heapParams) {
330 heapParams->reserve(heapParams->size() + numIndices);
331 }
332 c2_status_t status;
333 Return<void> transStatus = mSinkConfigurable->query(
334 indices,
335 mayBlock == C2_MAY_BLOCK,
336 [&status, &numStackIndices, &stackParams, heapParams](
337 Status s, const Params& p) {
338 status = static_cast<c2_status_t>(s);
339 if (status != C2_OK && status != C2_BAD_INDEX) {
340 LOG(DEBUG) << "queryFromSink -- call failed: "
341 << "status = " << asString(status) << ".";
342 return;
343 }
344 std::vector<C2Param*> paramPointers;
345 if (!parseParamsBlob(¶mPointers, p)) {
346 LOG(DEBUG) << "queryFromSink -- error while "
347 << "parsing params.";
348 status = C2_CORRUPTED;
349 return;
350 }
351 size_t i = 0;
352 for (auto it = paramPointers.begin();
353 it != paramPointers.end(); ) {
354 C2Param* paramPointer = *it;
355 if (numStackIndices > 0) {
356 --numStackIndices;
357 if (!paramPointer) {
358 LOG(DEBUG) << "queryFromSink -- "
359 "null stack param.";
360 ++it;
361 continue;
362 }
363 for (; i < stackParams.size() &&
364 !stackParams[i]; ) {
365 ++i;
366 }
367 CHECK(i < stackParams.size());
368 if (stackParams[i]->index() !=
369 paramPointer->index()) {
370 LOG(DEBUG) << "queryFromSink -- "
371 "param skipped (index = "
372 << stackParams[i]->index() << ").";
373 stackParams[i++]->invalidate();
374 continue;
375 }
376 if (!stackParams[i++]->updateFrom(*paramPointer)) {
377 LOG(DEBUG) << "queryFromSink -- "
378 "param update failed (index = "
379 << paramPointer->index() << ").";
380 }
381 } else {
382 if (!paramPointer) {
383 LOG(DEBUG) << "queryFromSink -- "
384 "null heap param.";
385 ++it;
386 continue;
387 }
388 if (!heapParams) {
389 LOG(WARNING) << "queryFromSink -- "
390 "too many stack params.";
391 break;
392 }
393 heapParams->emplace_back(C2Param::Copy(*paramPointer));
394 }
395 ++it;
396 }
397 });
398 if (!transStatus.isOk()) {
399 LOG(ERROR) << "queryFromSink -- transaction failed.";
400 return C2_CORRUPTED;
401 }
402 return status;
403 }
404
queueToSinkandroid::hardware::media::c2::V1_0::utils::InputSurfaceConnection::Impl405 c2_status_t queueToSink(std::list<std::unique_ptr<C2Work>>* const items) {
406 if (mLocalComp) {
407 return mLocalComp->queue_nb(items);
408 }
409
410 CHECK(mSink) << "-- queueToSink "
411 << "-- connection has no sink.";
412
413 WorkBundle workBundle;
414 if (!objcpy(&workBundle, *items, nullptr)) {
415 LOG(ERROR) << "queueToSink -- bad input.";
416 return C2_CORRUPTED;
417 }
418 Return<Status> transStatus = mSink->queue(workBundle);
419 if (!transStatus.isOk()) {
420 LOG(ERROR) << "queueToSink -- transaction failed.";
421 return C2_CORRUPTED;
422 }
423 c2_status_t status =
424 static_cast<c2_status_t>(static_cast<Status>(transStatus));
425 if (status != C2_OK) {
426 LOG(DEBUG) << "queueToSink -- call failed: "
427 << asString(status);
428 }
429 return status;
430 }
431
432 sp<GraphicBufferSource> mSource;
433 std::shared_ptr<C2Component> mLocalComp;
434 sp<IInputSink> mSink;
435 sp<IConfigurable> mSinkConfigurable;
436 std::string mSinkName;
437
438 // Needed for ComponentWrapper implementation
439 std::mutex mAllocatorMutex;
440 std::shared_ptr<C2Allocator> mAllocator;
441 std::atomic_uint64_t mFrameIndex;
442
443 };
444
InputSurfaceConnection(const sp<GraphicBufferSource> & source,const std::shared_ptr<C2Component> & comp,const sp<ComponentStore> & store)445 InputSurfaceConnection::InputSurfaceConnection(
446 const sp<GraphicBufferSource>& source,
447 const std::shared_ptr<C2Component>& comp,
448 const sp<ComponentStore>& store)
449 : mImpl{new Impl(source, comp)},
450 mConfigurable{new CachedConfigurable(
451 std::make_unique<Impl::ConfigurableIntf>(mImpl))} {
452 mConfigurable->init(store.get());
453 }
454
InputSurfaceConnection(const sp<GraphicBufferSource> & source,const sp<IInputSink> & sink,const sp<ComponentStore> & store)455 InputSurfaceConnection::InputSurfaceConnection(
456 const sp<GraphicBufferSource>& source,
457 const sp<IInputSink>& sink,
458 const sp<ComponentStore>& store)
459 : mImpl{new Impl(source, sink)},
460 mConfigurable{new CachedConfigurable(
461 std::make_unique<Impl::ConfigurableIntf>(mImpl))} {
462 mConfigurable->init(store.get());
463 }
464
disconnect()465 Return<Status> InputSurfaceConnection::disconnect() {
466 std::lock_guard<std::mutex> lock(mImplMutex);
467 mImpl = nullptr;
468 return Status::OK;
469 }
470
~InputSurfaceConnection()471 InputSurfaceConnection::~InputSurfaceConnection() {
472 mImpl = nullptr;
473 }
474
init()475 bool InputSurfaceConnection::init() {
476 std::lock_guard<std::mutex> lock(mImplMutex);
477 return mImpl->init();
478 }
479
getConfigurable()480 Return<sp<IConfigurable>> InputSurfaceConnection::getConfigurable() {
481 return mConfigurable;
482 }
483
484 // Configurable interface for InputSurfaceConnection::Impl
config(const std::vector<C2Param * > & params,c2_blocking_t mayBlock,std::vector<std::unique_ptr<C2SettingResult>> * const failures)485 c2_status_t InputSurfaceConnection::Impl::ConfigurableIntf::config(
486 const std::vector<C2Param*> ¶ms,
487 c2_blocking_t mayBlock,
488 std::vector<std::unique_ptr<C2SettingResult>> *const failures) {
489 // TODO: implement
490 (void)params;
491 (void)mayBlock;
492 (void)failures;
493 return C2_OK;
494 }
495
query(const std::vector<C2Param::Index> & indices,c2_blocking_t mayBlock,std::vector<std::unique_ptr<C2Param>> * const params) const496 c2_status_t InputSurfaceConnection::Impl::ConfigurableIntf::query(
497 const std::vector<C2Param::Index> &indices,
498 c2_blocking_t mayBlock,
499 std::vector<std::unique_ptr<C2Param>> *const params) const {
500 // TODO: implement
501 (void)indices;
502 (void)mayBlock;
503 (void)params;
504 return C2_OK;
505 }
506
querySupportedParams(std::vector<std::shared_ptr<C2ParamDescriptor>> * const params) const507 c2_status_t InputSurfaceConnection::Impl::ConfigurableIntf::querySupportedParams(
508 std::vector<std::shared_ptr<C2ParamDescriptor>> *const params) const {
509 // TODO: implement
510 (void)params;
511 return C2_OK;
512 }
513
querySupportedValues(std::vector<C2FieldSupportedValuesQuery> & fields,c2_blocking_t mayBlock) const514 c2_status_t InputSurfaceConnection::Impl::ConfigurableIntf::querySupportedValues(
515 std::vector<C2FieldSupportedValuesQuery> &fields,
516 c2_blocking_t mayBlock) const {
517 // TODO: implement
518 (void)fields;
519 (void)mayBlock;
520 return C2_OK;
521 }
522
523 } // namespace utils
524 } // namespace V1_0
525 } // namespace c2
526 } // namespace media
527 } // namespace hardware
528 } // namespace android
529
530