1 /*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #include "hidl/HidlSupport.h"
19 #define LOG_TAG "MediaCodec"
20 #include <utils/Log.h>
21
22 #include <inttypes.h>
23 #include <stdlib.h>
24
25 #include <C2Buffer.h>
26
27 #include "include/SoftwareRenderer.h"
28
29 #include <android/hardware/cas/native/1.0/IDescrambler.h>
30 #include <android/hardware/media/omx/1.0/IGraphicBufferSource.h>
31
32 #include <aidl/android/media/BnResourceManagerClient.h>
33 #include <aidl/android/media/IResourceManagerService.h>
34 #include <android/binder_ibinder.h>
35 #include <android/binder_manager.h>
36 #include <binder/IMemory.h>
37 #include <binder/MemoryDealer.h>
38 #include <cutils/properties.h>
39 #include <gui/BufferQueue.h>
40 #include <gui/Surface.h>
41 #include <hidlmemory/FrameworkUtils.h>
42 #include <mediadrm/ICrypto.h>
43 #include <media/IOMX.h>
44 #include <media/MediaCodecBuffer.h>
45 #include <media/MediaCodecInfo.h>
46 #include <media/MediaMetricsItem.h>
47 #include <media/MediaResource.h>
48 #include <media/stagefright/foundation/ABuffer.h>
49 #include <media/stagefright/foundation/ADebug.h>
50 #include <media/stagefright/foundation/AMessage.h>
51 #include <media/stagefright/foundation/AString.h>
52 #include <media/stagefright/foundation/AUtils.h>
53 #include <media/stagefright/foundation/avc_utils.h>
54 #include <media/stagefright/foundation/hexdump.h>
55 #include <media/stagefright/ACodec.h>
56 #include <media/stagefright/BatteryChecker.h>
57 #include <media/stagefright/BufferProducerWrapper.h>
58 #include <media/stagefright/CCodec.h>
59 #include <media/stagefright/MediaCodec.h>
60 #include <media/stagefright/MediaCodecConstants.h>
61 #include <media/stagefright/MediaCodecList.h>
62 #include <media/stagefright/MediaDefs.h>
63 #include <media/stagefright/MediaErrors.h>
64 #include <media/stagefright/MediaFilter.h>
65 #include <media/stagefright/OMXClient.h>
66 #include <media/stagefright/PersistentSurface.h>
67 #include <media/stagefright/SurfaceUtils.h>
68 #include <private/android_filesystem_config.h>
69 #include <utils/Singleton.h>
70
71 namespace android {
72
73 using Status = ::ndk::ScopedAStatus;
74 using aidl::android::media::BnResourceManagerClient;
75 using aidl::android::media::IResourceManagerClient;
76 using aidl::android::media::IResourceManagerService;
77
78 // key for media statistics
79 static const char *kCodecKeyName = "codec";
80 // attrs for media statistics
81 // NB: these are matched with public Java API constants defined
82 // in frameworks/base/media/java/android/media/MediaCodec.java
83 // These must be kept synchronized with the constants there.
84 static const char *kCodecCodec = "android.media.mediacodec.codec"; /* e.g. OMX.google.aac.decoder */
85 static const char *kCodecMime = "android.media.mediacodec.mime"; /* e.g. audio/mime */
86 static const char *kCodecMode = "android.media.mediacodec.mode"; /* audio, video */
87 static const char *kCodecModeVideo = "video"; /* values returned for kCodecMode */
88 static const char *kCodecModeAudio = "audio";
89 static const char *kCodecEncoder = "android.media.mediacodec.encoder"; /* 0,1 */
90 static const char *kCodecSecure = "android.media.mediacodec.secure"; /* 0, 1 */
91 static const char *kCodecWidth = "android.media.mediacodec.width"; /* 0..n */
92 static const char *kCodecHeight = "android.media.mediacodec.height"; /* 0..n */
93 static const char *kCodecRotation = "android.media.mediacodec.rotation-degrees"; /* 0/90/180/270 */
94
95 // NB: These are not yet exposed as public Java API constants.
96 static const char *kCodecCrypto = "android.media.mediacodec.crypto"; /* 0,1 */
97 static const char *kCodecProfile = "android.media.mediacodec.profile"; /* 0..n */
98 static const char *kCodecLevel = "android.media.mediacodec.level"; /* 0..n */
99 static const char *kCodecBitrateMode = "android.media.mediacodec.bitrate_mode"; /* CQ/VBR/CBR */
100 static const char *kCodecBitrate = "android.media.mediacodec.bitrate"; /* 0..n */
101 static const char *kCodecMaxWidth = "android.media.mediacodec.maxwidth"; /* 0..n */
102 static const char *kCodecMaxHeight = "android.media.mediacodec.maxheight"; /* 0..n */
103 static const char *kCodecError = "android.media.mediacodec.errcode";
104 static const char *kCodecLifetimeMs = "android.media.mediacodec.lifetimeMs"; /* 0..n ms*/
105 static const char *kCodecErrorState = "android.media.mediacodec.errstate";
106 static const char *kCodecLatencyMax = "android.media.mediacodec.latency.max"; /* in us */
107 static const char *kCodecLatencyMin = "android.media.mediacodec.latency.min"; /* in us */
108 static const char *kCodecLatencyAvg = "android.media.mediacodec.latency.avg"; /* in us */
109 static const char *kCodecLatencyCount = "android.media.mediacodec.latency.n";
110 static const char *kCodecLatencyHist = "android.media.mediacodec.latency.hist"; /* in us */
111 static const char *kCodecLatencyUnknown = "android.media.mediacodec.latency.unknown";
112 static const char *kCodecQueueSecureInputBufferError = "android.media.mediacodec.queueSecureInputBufferError";
113 static const char *kCodecQueueInputBufferError = "android.media.mediacodec.queueInputBufferError";
114
115 static const char *kCodecNumLowLatencyModeOn = "android.media.mediacodec.low-latency.on"; /* 0..n */
116 static const char *kCodecNumLowLatencyModeOff = "android.media.mediacodec.low-latency.off"; /* 0..n */
117 static const char *kCodecFirstFrameIndexLowLatencyModeOn = "android.media.mediacodec.low-latency.first-frame"; /* 0..n */
118
119 // the kCodecRecent* fields appear only in getMetrics() results
120 static const char *kCodecRecentLatencyMax = "android.media.mediacodec.recent.max"; /* in us */
121 static const char *kCodecRecentLatencyMin = "android.media.mediacodec.recent.min"; /* in us */
122 static const char *kCodecRecentLatencyAvg = "android.media.mediacodec.recent.avg"; /* in us */
123 static const char *kCodecRecentLatencyCount = "android.media.mediacodec.recent.n";
124 static const char *kCodecRecentLatencyHist = "android.media.mediacodec.recent.hist"; /* in us */
125
126 // XXX suppress until we get our representation right
127 static bool kEmitHistogram = false;
128
129
getId(const std::shared_ptr<IResourceManagerClient> & client)130 static int64_t getId(const std::shared_ptr<IResourceManagerClient> &client) {
131 return (int64_t) client.get();
132 }
133
isResourceError(status_t err)134 static bool isResourceError(status_t err) {
135 return (err == NO_MEMORY);
136 }
137
138 static const int kMaxRetry = 2;
139 static const int kMaxReclaimWaitTimeInUs = 500000; // 0.5s
140 static const int kNumBuffersAlign = 16;
141
142 static const C2MemoryUsage kDefaultReadWriteUsage{
143 C2MemoryUsage::CPU_READ, C2MemoryUsage::CPU_WRITE};
144
145 ////////////////////////////////////////////////////////////////////////////////
146
147 struct ResourceManagerClient : public BnResourceManagerClient {
ResourceManagerClientandroid::ResourceManagerClient148 explicit ResourceManagerClient(MediaCodec* codec) : mMediaCodec(codec) {}
149
reclaimResourceandroid::ResourceManagerClient150 Status reclaimResource(bool* _aidl_return) override {
151 sp<MediaCodec> codec = mMediaCodec.promote();
152 if (codec == NULL) {
153 // codec is already gone.
154 *_aidl_return = true;
155 return Status::ok();
156 }
157 status_t err = codec->reclaim();
158 if (err == WOULD_BLOCK) {
159 ALOGD("Wait for the client to release codec.");
160 usleep(kMaxReclaimWaitTimeInUs);
161 ALOGD("Try to reclaim again.");
162 err = codec->reclaim(true /* force */);
163 }
164 if (err != OK) {
165 ALOGW("ResourceManagerClient failed to release codec with err %d", err);
166 }
167 *_aidl_return = (err == OK);
168 return Status::ok();
169 }
170
getNameandroid::ResourceManagerClient171 Status getName(::std::string* _aidl_return) override {
172 _aidl_return->clear();
173 sp<MediaCodec> codec = mMediaCodec.promote();
174 if (codec == NULL) {
175 // codec is already gone.
176 return Status::ok();
177 }
178
179 AString name;
180 if (codec->getName(&name) == OK) {
181 *_aidl_return = name.c_str();
182 }
183 return Status::ok();
184 }
185
~ResourceManagerClientandroid::ResourceManagerClient186 virtual ~ResourceManagerClient() {}
187
188 private:
189 wp<MediaCodec> mMediaCodec;
190
191 DISALLOW_EVIL_CONSTRUCTORS(ResourceManagerClient);
192 };
193
194 struct MediaCodec::ResourceManagerServiceProxy : public RefBase {
195 ResourceManagerServiceProxy(pid_t pid, uid_t uid,
196 const std::shared_ptr<IResourceManagerClient> &client);
197 virtual ~ResourceManagerServiceProxy();
198
199 void init();
200
201 // implements DeathRecipient
202 static void BinderDiedCallback(void* cookie);
203 void binderDied();
204
205 void addResource(const MediaResourceParcel &resource);
206 void removeResource(const MediaResourceParcel &resource);
207 void removeClient();
208 void markClientForPendingRemoval();
209 bool reclaimResource(const std::vector<MediaResourceParcel> &resources);
210
211 private:
212 Mutex mLock;
213 pid_t mPid;
214 uid_t mUid;
215 std::shared_ptr<IResourceManagerService> mService;
216 std::shared_ptr<IResourceManagerClient> mClient;
217 ::ndk::ScopedAIBinder_DeathRecipient mDeathRecipient;
218 };
219
ResourceManagerServiceProxy(pid_t pid,uid_t uid,const std::shared_ptr<IResourceManagerClient> & client)220 MediaCodec::ResourceManagerServiceProxy::ResourceManagerServiceProxy(
221 pid_t pid, uid_t uid, const std::shared_ptr<IResourceManagerClient> &client)
222 : mPid(pid), mUid(uid), mClient(client),
223 mDeathRecipient(AIBinder_DeathRecipient_new(BinderDiedCallback)) {
224 if (mPid == MediaCodec::kNoPid) {
225 mPid = AIBinder_getCallingPid();
226 }
227 }
228
~ResourceManagerServiceProxy()229 MediaCodec::ResourceManagerServiceProxy::~ResourceManagerServiceProxy() {
230 if (mService != nullptr) {
231 AIBinder_unlinkToDeath(mService->asBinder().get(), mDeathRecipient.get(), this);
232 }
233 }
234
init()235 void MediaCodec::ResourceManagerServiceProxy::init() {
236 ::ndk::SpAIBinder binder(AServiceManager_getService("media.resource_manager"));
237 mService = IResourceManagerService::fromBinder(binder);
238 if (mService == nullptr) {
239 ALOGE("Failed to get ResourceManagerService");
240 return;
241 }
242
243 AIBinder_linkToDeath(mService->asBinder().get(), mDeathRecipient.get(), this);
244 }
245
246 //static
BinderDiedCallback(void * cookie)247 void MediaCodec::ResourceManagerServiceProxy::BinderDiedCallback(void* cookie) {
248 auto thiz = static_cast<ResourceManagerServiceProxy*>(cookie);
249 thiz->binderDied();
250 }
251
binderDied()252 void MediaCodec::ResourceManagerServiceProxy::binderDied() {
253 ALOGW("ResourceManagerService died.");
254 Mutex::Autolock _l(mLock);
255 mService = nullptr;
256 }
257
addResource(const MediaResourceParcel & resource)258 void MediaCodec::ResourceManagerServiceProxy::addResource(
259 const MediaResourceParcel &resource) {
260 std::vector<MediaResourceParcel> resources;
261 resources.push_back(resource);
262
263 Mutex::Autolock _l(mLock);
264 if (mService == nullptr) {
265 return;
266 }
267 mService->addResource(mPid, mUid, getId(mClient), mClient, resources);
268 }
269
removeResource(const MediaResourceParcel & resource)270 void MediaCodec::ResourceManagerServiceProxy::removeResource(
271 const MediaResourceParcel &resource) {
272 std::vector<MediaResourceParcel> resources;
273 resources.push_back(resource);
274
275 Mutex::Autolock _l(mLock);
276 if (mService == nullptr) {
277 return;
278 }
279 mService->removeResource(mPid, getId(mClient), resources);
280 }
281
removeClient()282 void MediaCodec::ResourceManagerServiceProxy::removeClient() {
283 Mutex::Autolock _l(mLock);
284 if (mService == nullptr) {
285 return;
286 }
287 mService->removeClient(mPid, getId(mClient));
288 }
289
markClientForPendingRemoval()290 void MediaCodec::ResourceManagerServiceProxy::markClientForPendingRemoval() {
291 Mutex::Autolock _l(mLock);
292 if (mService == nullptr) {
293 return;
294 }
295 mService->markClientForPendingRemoval(mPid, getId(mClient));
296 }
297
reclaimResource(const std::vector<MediaResourceParcel> & resources)298 bool MediaCodec::ResourceManagerServiceProxy::reclaimResource(
299 const std::vector<MediaResourceParcel> &resources) {
300 Mutex::Autolock _l(mLock);
301 if (mService == NULL) {
302 return false;
303 }
304 bool success;
305 Status status = mService->reclaimResource(mPid, resources, &success);
306 return status.isOk() && success;
307 }
308
309 ////////////////////////////////////////////////////////////////////////////////
310
BufferInfo()311 MediaCodec::BufferInfo::BufferInfo() : mOwnedByClient(false) {}
312
313 ////////////////////////////////////////////////////////////////////////////////
314
315 class MediaCodec::ReleaseSurface {
316 public:
ReleaseSurface()317 ReleaseSurface() {
318 BufferQueue::createBufferQueue(&mProducer, &mConsumer);
319 mSurface = new Surface(mProducer, false /* controlledByApp */);
320 struct ConsumerListener : public BnConsumerListener {
321 void onFrameAvailable(const BufferItem&) override {}
322 void onBuffersReleased() override {}
323 void onSidebandStreamChanged() override {}
324 };
325 sp<ConsumerListener> listener{new ConsumerListener};
326 mConsumer->consumerConnect(listener, false);
327 mConsumer->setConsumerName(String8{"MediaCodec.release"});
328 }
329
getSurface()330 const sp<Surface> &getSurface() {
331 return mSurface;
332 }
333
334 private:
335 sp<IGraphicBufferProducer> mProducer;
336 sp<IGraphicBufferConsumer> mConsumer;
337 sp<Surface> mSurface;
338 };
339
340 ////////////////////////////////////////////////////////////////////////////////
341
342 namespace {
343
344 enum {
345 kWhatFillThisBuffer = 'fill',
346 kWhatDrainThisBuffer = 'drai',
347 kWhatEOS = 'eos ',
348 kWhatStartCompleted = 'Scom',
349 kWhatStopCompleted = 'scom',
350 kWhatReleaseCompleted = 'rcom',
351 kWhatFlushCompleted = 'fcom',
352 kWhatError = 'erro',
353 kWhatComponentAllocated = 'cAll',
354 kWhatComponentConfigured = 'cCon',
355 kWhatInputSurfaceCreated = 'isfc',
356 kWhatInputSurfaceAccepted = 'isfa',
357 kWhatSignaledInputEOS = 'seos',
358 kWhatOutputFramesRendered = 'outR',
359 kWhatOutputBuffersChanged = 'outC',
360 };
361
362 class BufferCallback : public CodecBase::BufferCallback {
363 public:
364 explicit BufferCallback(const sp<AMessage> ¬ify);
365 virtual ~BufferCallback() = default;
366
367 virtual void onInputBufferAvailable(
368 size_t index, const sp<MediaCodecBuffer> &buffer) override;
369 virtual void onOutputBufferAvailable(
370 size_t index, const sp<MediaCodecBuffer> &buffer) override;
371 private:
372 const sp<AMessage> mNotify;
373 };
374
BufferCallback(const sp<AMessage> & notify)375 BufferCallback::BufferCallback(const sp<AMessage> ¬ify)
376 : mNotify(notify) {}
377
onInputBufferAvailable(size_t index,const sp<MediaCodecBuffer> & buffer)378 void BufferCallback::onInputBufferAvailable(
379 size_t index, const sp<MediaCodecBuffer> &buffer) {
380 sp<AMessage> notify(mNotify->dup());
381 notify->setInt32("what", kWhatFillThisBuffer);
382 notify->setSize("index", index);
383 notify->setObject("buffer", buffer);
384 notify->post();
385 }
386
onOutputBufferAvailable(size_t index,const sp<MediaCodecBuffer> & buffer)387 void BufferCallback::onOutputBufferAvailable(
388 size_t index, const sp<MediaCodecBuffer> &buffer) {
389 sp<AMessage> notify(mNotify->dup());
390 notify->setInt32("what", kWhatDrainThisBuffer);
391 notify->setSize("index", index);
392 notify->setObject("buffer", buffer);
393 notify->post();
394 }
395
396 class CodecCallback : public CodecBase::CodecCallback {
397 public:
398 explicit CodecCallback(const sp<AMessage> ¬ify);
399 virtual ~CodecCallback() = default;
400
401 virtual void onEos(status_t err) override;
402 virtual void onStartCompleted() override;
403 virtual void onStopCompleted() override;
404 virtual void onReleaseCompleted() override;
405 virtual void onFlushCompleted() override;
406 virtual void onError(status_t err, enum ActionCode actionCode) override;
407 virtual void onComponentAllocated(const char *componentName) override;
408 virtual void onComponentConfigured(
409 const sp<AMessage> &inputFormat, const sp<AMessage> &outputFormat) override;
410 virtual void onInputSurfaceCreated(
411 const sp<AMessage> &inputFormat,
412 const sp<AMessage> &outputFormat,
413 const sp<BufferProducerWrapper> &inputSurface) override;
414 virtual void onInputSurfaceCreationFailed(status_t err) override;
415 virtual void onInputSurfaceAccepted(
416 const sp<AMessage> &inputFormat,
417 const sp<AMessage> &outputFormat) override;
418 virtual void onInputSurfaceDeclined(status_t err) override;
419 virtual void onSignaledInputEOS(status_t err) override;
420 virtual void onOutputFramesRendered(const std::list<FrameRenderTracker::Info> &done) override;
421 virtual void onOutputBuffersChanged() override;
422 private:
423 const sp<AMessage> mNotify;
424 };
425
CodecCallback(const sp<AMessage> & notify)426 CodecCallback::CodecCallback(const sp<AMessage> ¬ify) : mNotify(notify) {}
427
onEos(status_t err)428 void CodecCallback::onEos(status_t err) {
429 sp<AMessage> notify(mNotify->dup());
430 notify->setInt32("what", kWhatEOS);
431 notify->setInt32("err", err);
432 notify->post();
433 }
434
onStartCompleted()435 void CodecCallback::onStartCompleted() {
436 sp<AMessage> notify(mNotify->dup());
437 notify->setInt32("what", kWhatStartCompleted);
438 notify->post();
439 }
440
onStopCompleted()441 void CodecCallback::onStopCompleted() {
442 sp<AMessage> notify(mNotify->dup());
443 notify->setInt32("what", kWhatStopCompleted);
444 notify->post();
445 }
446
onReleaseCompleted()447 void CodecCallback::onReleaseCompleted() {
448 sp<AMessage> notify(mNotify->dup());
449 notify->setInt32("what", kWhatReleaseCompleted);
450 notify->post();
451 }
452
onFlushCompleted()453 void CodecCallback::onFlushCompleted() {
454 sp<AMessage> notify(mNotify->dup());
455 notify->setInt32("what", kWhatFlushCompleted);
456 notify->post();
457 }
458
onError(status_t err,enum ActionCode actionCode)459 void CodecCallback::onError(status_t err, enum ActionCode actionCode) {
460 sp<AMessage> notify(mNotify->dup());
461 notify->setInt32("what", kWhatError);
462 notify->setInt32("err", err);
463 notify->setInt32("actionCode", actionCode);
464 notify->post();
465 }
466
onComponentAllocated(const char * componentName)467 void CodecCallback::onComponentAllocated(const char *componentName) {
468 sp<AMessage> notify(mNotify->dup());
469 notify->setInt32("what", kWhatComponentAllocated);
470 notify->setString("componentName", componentName);
471 notify->post();
472 }
473
onComponentConfigured(const sp<AMessage> & inputFormat,const sp<AMessage> & outputFormat)474 void CodecCallback::onComponentConfigured(
475 const sp<AMessage> &inputFormat, const sp<AMessage> &outputFormat) {
476 sp<AMessage> notify(mNotify->dup());
477 notify->setInt32("what", kWhatComponentConfigured);
478 notify->setMessage("input-format", inputFormat);
479 notify->setMessage("output-format", outputFormat);
480 notify->post();
481 }
482
onInputSurfaceCreated(const sp<AMessage> & inputFormat,const sp<AMessage> & outputFormat,const sp<BufferProducerWrapper> & inputSurface)483 void CodecCallback::onInputSurfaceCreated(
484 const sp<AMessage> &inputFormat,
485 const sp<AMessage> &outputFormat,
486 const sp<BufferProducerWrapper> &inputSurface) {
487 sp<AMessage> notify(mNotify->dup());
488 notify->setInt32("what", kWhatInputSurfaceCreated);
489 notify->setMessage("input-format", inputFormat);
490 notify->setMessage("output-format", outputFormat);
491 notify->setObject("input-surface", inputSurface);
492 notify->post();
493 }
494
onInputSurfaceCreationFailed(status_t err)495 void CodecCallback::onInputSurfaceCreationFailed(status_t err) {
496 sp<AMessage> notify(mNotify->dup());
497 notify->setInt32("what", kWhatInputSurfaceCreated);
498 notify->setInt32("err", err);
499 notify->post();
500 }
501
onInputSurfaceAccepted(const sp<AMessage> & inputFormat,const sp<AMessage> & outputFormat)502 void CodecCallback::onInputSurfaceAccepted(
503 const sp<AMessage> &inputFormat,
504 const sp<AMessage> &outputFormat) {
505 sp<AMessage> notify(mNotify->dup());
506 notify->setInt32("what", kWhatInputSurfaceAccepted);
507 notify->setMessage("input-format", inputFormat);
508 notify->setMessage("output-format", outputFormat);
509 notify->post();
510 }
511
onInputSurfaceDeclined(status_t err)512 void CodecCallback::onInputSurfaceDeclined(status_t err) {
513 sp<AMessage> notify(mNotify->dup());
514 notify->setInt32("what", kWhatInputSurfaceAccepted);
515 notify->setInt32("err", err);
516 notify->post();
517 }
518
onSignaledInputEOS(status_t err)519 void CodecCallback::onSignaledInputEOS(status_t err) {
520 sp<AMessage> notify(mNotify->dup());
521 notify->setInt32("what", kWhatSignaledInputEOS);
522 if (err != OK) {
523 notify->setInt32("err", err);
524 }
525 notify->post();
526 }
527
onOutputFramesRendered(const std::list<FrameRenderTracker::Info> & done)528 void CodecCallback::onOutputFramesRendered(const std::list<FrameRenderTracker::Info> &done) {
529 sp<AMessage> notify(mNotify->dup());
530 notify->setInt32("what", kWhatOutputFramesRendered);
531 if (MediaCodec::CreateFramesRenderedMessage(done, notify)) {
532 notify->post();
533 }
534 }
535
onOutputBuffersChanged()536 void CodecCallback::onOutputBuffersChanged() {
537 sp<AMessage> notify(mNotify->dup());
538 notify->setInt32("what", kWhatOutputBuffersChanged);
539 notify->post();
540 }
541
542 } // namespace
543
544 ////////////////////////////////////////////////////////////////////////////////
545
546 // static
CreateByType(const sp<ALooper> & looper,const AString & mime,bool encoder,status_t * err,pid_t pid,uid_t uid)547 sp<MediaCodec> MediaCodec::CreateByType(
548 const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err, pid_t pid,
549 uid_t uid) {
550 Vector<AString> matchingCodecs;
551
552 MediaCodecList::findMatchingCodecs(
553 mime.c_str(),
554 encoder,
555 0,
556 &matchingCodecs);
557
558 if (err != NULL) {
559 *err = NAME_NOT_FOUND;
560 }
561 for (size_t i = 0; i < matchingCodecs.size(); ++i) {
562 sp<MediaCodec> codec = new MediaCodec(looper, pid, uid);
563 AString componentName = matchingCodecs[i];
564 status_t ret = codec->init(componentName);
565 if (err != NULL) {
566 *err = ret;
567 }
568 if (ret == OK) {
569 return codec;
570 }
571 ALOGD("Allocating component '%s' failed (%d), try next one.",
572 componentName.c_str(), ret);
573 }
574 return NULL;
575 }
576
577 // static
CreateByComponentName(const sp<ALooper> & looper,const AString & name,status_t * err,pid_t pid,uid_t uid)578 sp<MediaCodec> MediaCodec::CreateByComponentName(
579 const sp<ALooper> &looper, const AString &name, status_t *err, pid_t pid, uid_t uid) {
580 sp<MediaCodec> codec = new MediaCodec(looper, pid, uid);
581
582 const status_t ret = codec->init(name);
583 if (err != NULL) {
584 *err = ret;
585 }
586 return ret == OK ? codec : NULL; // NULL deallocates codec.
587 }
588
589 // static
CreatePersistentInputSurface()590 sp<PersistentSurface> MediaCodec::CreatePersistentInputSurface() {
591 sp<PersistentSurface> pluginSurface = CCodec::CreateInputSurface();
592 if (pluginSurface != nullptr) {
593 return pluginSurface;
594 }
595
596 OMXClient client;
597 if (client.connect() != OK) {
598 ALOGE("Failed to connect to OMX to create persistent input surface.");
599 return NULL;
600 }
601
602 sp<IOMX> omx = client.interface();
603
604 sp<IGraphicBufferProducer> bufferProducer;
605 sp<hardware::media::omx::V1_0::IGraphicBufferSource> bufferSource;
606
607 status_t err = omx->createInputSurface(&bufferProducer, &bufferSource);
608
609 if (err != OK) {
610 ALOGE("Failed to create persistent input surface.");
611 return NULL;
612 }
613
614 return new PersistentSurface(bufferProducer, bufferSource);
615 }
616
MediaCodec(const sp<ALooper> & looper,pid_t pid,uid_t uid)617 MediaCodec::MediaCodec(const sp<ALooper> &looper, pid_t pid, uid_t uid)
618 : mState(UNINITIALIZED),
619 mReleasedByResourceManager(false),
620 mLooper(looper),
621 mCodec(NULL),
622 mReplyID(0),
623 mFlags(0),
624 mStickyError(OK),
625 mSoftRenderer(NULL),
626 mIsVideo(false),
627 mVideoWidth(0),
628 mVideoHeight(0),
629 mRotationDegrees(0),
630 mDequeueInputTimeoutGeneration(0),
631 mDequeueInputReplyID(0),
632 mDequeueOutputTimeoutGeneration(0),
633 mDequeueOutputReplyID(0),
634 mHaveInputSurface(false),
635 mHavePendingInputBuffers(false),
636 mCpuBoostRequested(false),
637 mLatencyUnknown(0),
638 mNumLowLatencyEnables(0),
639 mNumLowLatencyDisables(0),
640 mIsLowLatencyModeOn(false),
641 mIndexOfFirstFrameWhenLowLatencyOn(-1),
642 mInputBufferCounter(0) {
643 if (uid == kNoUid) {
644 mUid = AIBinder_getCallingUid();
645 } else {
646 mUid = uid;
647 }
648 mResourceManagerProxy = new ResourceManagerServiceProxy(pid, mUid,
649 ::ndk::SharedRefBase::make<ResourceManagerClient>(this));
650
651 initMediametrics();
652 }
653
~MediaCodec()654 MediaCodec::~MediaCodec() {
655 CHECK_EQ(mState, UNINITIALIZED);
656 mResourceManagerProxy->removeClient();
657
658 flushMediametrics();
659 }
660
initMediametrics()661 void MediaCodec::initMediametrics() {
662 if (mMetricsHandle == 0) {
663 mMetricsHandle = mediametrics_create(kCodecKeyName);
664 }
665
666 mLatencyHist.setup(kLatencyHistBuckets, kLatencyHistWidth, kLatencyHistFloor);
667
668 {
669 Mutex::Autolock al(mRecentLock);
670 for (int i = 0; i<kRecentLatencyFrames; i++) {
671 mRecentSamples[i] = kRecentSampleInvalid;
672 }
673 mRecentHead = 0;
674 }
675
676 {
677 Mutex::Autolock al(mLatencyLock);
678 mBuffersInFlight.clear();
679 mNumLowLatencyEnables = 0;
680 mNumLowLatencyDisables = 0;
681 mIsLowLatencyModeOn = false;
682 mIndexOfFirstFrameWhenLowLatencyOn = -1;
683 mInputBufferCounter = 0;
684 }
685
686 mLifetimeStartNs = systemTime(SYSTEM_TIME_MONOTONIC);
687 }
688
updateMediametrics()689 void MediaCodec::updateMediametrics() {
690 ALOGV("MediaCodec::updateMediametrics");
691 if (mMetricsHandle == 0) {
692 return;
693 }
694
695 if (mLatencyHist.getCount() != 0 ) {
696 mediametrics_setInt64(mMetricsHandle, kCodecLatencyMax, mLatencyHist.getMax());
697 mediametrics_setInt64(mMetricsHandle, kCodecLatencyMin, mLatencyHist.getMin());
698 mediametrics_setInt64(mMetricsHandle, kCodecLatencyAvg, mLatencyHist.getAvg());
699 mediametrics_setInt64(mMetricsHandle, kCodecLatencyCount, mLatencyHist.getCount());
700
701 if (kEmitHistogram) {
702 // and the histogram itself
703 std::string hist = mLatencyHist.emit();
704 mediametrics_setCString(mMetricsHandle, kCodecLatencyHist, hist.c_str());
705 }
706 }
707 if (mLatencyUnknown > 0) {
708 mediametrics_setInt64(mMetricsHandle, kCodecLatencyUnknown, mLatencyUnknown);
709 }
710 if (mLifetimeStartNs > 0) {
711 nsecs_t lifetime = systemTime(SYSTEM_TIME_MONOTONIC) - mLifetimeStartNs;
712 lifetime = lifetime / (1000 * 1000); // emitted in ms, truncated not rounded
713 mediametrics_setInt64(mMetricsHandle, kCodecLifetimeMs, lifetime);
714 }
715
716 {
717 Mutex::Autolock al(mLatencyLock);
718 mediametrics_setInt64(mMetricsHandle, kCodecNumLowLatencyModeOn, mNumLowLatencyEnables);
719 mediametrics_setInt64(mMetricsHandle, kCodecNumLowLatencyModeOff, mNumLowLatencyDisables);
720 mediametrics_setInt64(mMetricsHandle, kCodecFirstFrameIndexLowLatencyModeOn,
721 mIndexOfFirstFrameWhenLowLatencyOn);
722 }
723 #if 0
724 // enable for short term, only while debugging
725 updateEphemeralMediametrics(mMetricsHandle);
726 #endif
727 }
728
updateEphemeralMediametrics(mediametrics_handle_t item)729 void MediaCodec::updateEphemeralMediametrics(mediametrics_handle_t item) {
730 ALOGD("MediaCodec::updateEphemeralMediametrics()");
731
732 if (item == 0) {
733 return;
734 }
735
736 Histogram recentHist;
737
738 // build an empty histogram
739 recentHist.setup(kLatencyHistBuckets, kLatencyHistWidth, kLatencyHistFloor);
740
741 // stuff it with the samples in the ring buffer
742 {
743 Mutex::Autolock al(mRecentLock);
744
745 for (int i=0; i<kRecentLatencyFrames; i++) {
746 if (mRecentSamples[i] != kRecentSampleInvalid) {
747 recentHist.insert(mRecentSamples[i]);
748 }
749 }
750 }
751
752 // spit the data (if any) into the supplied analytics record
753 if (recentHist.getCount()!= 0 ) {
754 mediametrics_setInt64(item, kCodecRecentLatencyMax, recentHist.getMax());
755 mediametrics_setInt64(item, kCodecRecentLatencyMin, recentHist.getMin());
756 mediametrics_setInt64(item, kCodecRecentLatencyAvg, recentHist.getAvg());
757 mediametrics_setInt64(item, kCodecRecentLatencyCount, recentHist.getCount());
758
759 if (kEmitHistogram) {
760 // and the histogram itself
761 std::string hist = recentHist.emit();
762 mediametrics_setCString(item, kCodecRecentLatencyHist, hist.c_str());
763 }
764 }
765 }
766
flushMediametrics()767 void MediaCodec::flushMediametrics() {
768 updateMediametrics();
769 if (mMetricsHandle != 0) {
770 if (mediametrics_count(mMetricsHandle) > 0) {
771 mediametrics_selfRecord(mMetricsHandle);
772 }
773 mediametrics_delete(mMetricsHandle);
774 mMetricsHandle = 0;
775 }
776 }
777
updateLowLatency(const sp<AMessage> & msg)778 void MediaCodec::updateLowLatency(const sp<AMessage> &msg) {
779 int32_t lowLatency = 0;
780 if (msg->findInt32("low-latency", &lowLatency)) {
781 Mutex::Autolock al(mLatencyLock);
782 if (lowLatency > 0) {
783 ++mNumLowLatencyEnables;
784 // This is just an estimate since low latency mode change happens ONLY at key frame
785 mIsLowLatencyModeOn = true;
786 } else if (lowLatency == 0) {
787 ++mNumLowLatencyDisables;
788 // This is just an estimate since low latency mode change happens ONLY at key frame
789 mIsLowLatencyModeOn = false;
790 }
791 }
792 }
793
setup(int nbuckets,int64_t width,int64_t floor)794 bool MediaCodec::Histogram::setup(int nbuckets, int64_t width, int64_t floor)
795 {
796 if (nbuckets <= 0 || width <= 0) {
797 return false;
798 }
799
800 // get histogram buckets
801 if (nbuckets == mBucketCount && mBuckets != NULL) {
802 // reuse our existing buffer
803 memset(mBuckets, 0, sizeof(*mBuckets) * mBucketCount);
804 } else {
805 // get a new pre-zeroed buffer
806 int64_t *newbuckets = (int64_t *)calloc(nbuckets, sizeof (*mBuckets));
807 if (newbuckets == NULL) {
808 goto bad;
809 }
810 if (mBuckets != NULL)
811 free(mBuckets);
812 mBuckets = newbuckets;
813 }
814
815 mWidth = width;
816 mFloor = floor;
817 mCeiling = floor + nbuckets * width;
818 mBucketCount = nbuckets;
819
820 mMin = INT64_MAX;
821 mMax = INT64_MIN;
822 mSum = 0;
823 mCount = 0;
824 mBelow = mAbove = 0;
825
826 return true;
827
828 bad:
829 if (mBuckets != NULL) {
830 free(mBuckets);
831 mBuckets = NULL;
832 }
833
834 return false;
835 }
836
insert(int64_t sample)837 void MediaCodec::Histogram::insert(int64_t sample)
838 {
839 // histogram is not set up
840 if (mBuckets == NULL) {
841 return;
842 }
843
844 mCount++;
845 mSum += sample;
846 if (mMin > sample) mMin = sample;
847 if (mMax < sample) mMax = sample;
848
849 if (sample < mFloor) {
850 mBelow++;
851 } else if (sample >= mCeiling) {
852 mAbove++;
853 } else {
854 int64_t slot = (sample - mFloor) / mWidth;
855 CHECK(slot < mBucketCount);
856 mBuckets[slot]++;
857 }
858 return;
859 }
860
emit()861 std::string MediaCodec::Histogram::emit()
862 {
863 std::string value;
864 char buffer[64];
865
866 // emits: width,Below{bucket0,bucket1,...., bucketN}above
867 // unconfigured will emit: 0,0{}0
868 // XXX: is this best representation?
869 snprintf(buffer, sizeof(buffer), "%" PRId64 ",%" PRId64 ",%" PRId64 "{",
870 mFloor, mWidth, mBelow);
871 value = buffer;
872 for (int i = 0; i < mBucketCount; i++) {
873 if (i != 0) {
874 value = value + ",";
875 }
876 snprintf(buffer, sizeof(buffer), "%" PRId64, mBuckets[i]);
877 value = value + buffer;
878 }
879 snprintf(buffer, sizeof(buffer), "}%" PRId64 , mAbove);
880 value = value + buffer;
881 return value;
882 }
883
884 // when we send a buffer to the codec;
statsBufferSent(int64_t presentationUs)885 void MediaCodec::statsBufferSent(int64_t presentationUs) {
886
887 // only enqueue if we have a legitimate time
888 if (presentationUs <= 0) {
889 ALOGV("presentation time: %" PRId64, presentationUs);
890 return;
891 }
892
893 if (mBatteryChecker != nullptr) {
894 mBatteryChecker->onCodecActivity([this] () {
895 mResourceManagerProxy->addResource(MediaResource::VideoBatteryResource());
896 });
897 }
898
899 const int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC);
900 BufferFlightTiming_t startdata = { presentationUs, nowNs };
901
902 {
903 // mutex access to mBuffersInFlight and other stats
904 Mutex::Autolock al(mLatencyLock);
905
906
907 // XXX: we *could* make sure that the time is later than the end of queue
908 // as part of a consistency check...
909 mBuffersInFlight.push_back(startdata);
910
911 if (mIsLowLatencyModeOn && mIndexOfFirstFrameWhenLowLatencyOn < 0) {
912 mIndexOfFirstFrameWhenLowLatencyOn = mInputBufferCounter;
913 }
914 ++mInputBufferCounter;
915 }
916 }
917
918 // when we get a buffer back from the codec
statsBufferReceived(int64_t presentationUs)919 void MediaCodec::statsBufferReceived(int64_t presentationUs) {
920
921 CHECK_NE(mState, UNINITIALIZED);
922
923 // mutex access to mBuffersInFlight and other stats
924 Mutex::Autolock al(mLatencyLock);
925
926 // how long this buffer took for the round trip through the codec
927 // NB: pipelining can/will make these times larger. e.g., if each packet
928 // is always 2 msec and we have 3 in flight at any given time, we're going to
929 // see "6 msec" as an answer.
930
931 // ignore stuff with no presentation time
932 if (presentationUs <= 0) {
933 ALOGV("-- returned buffer timestamp %" PRId64 " <= 0, ignore it", presentationUs);
934 mLatencyUnknown++;
935 return;
936 }
937
938 if (mBatteryChecker != nullptr) {
939 mBatteryChecker->onCodecActivity([this] () {
940 mResourceManagerProxy->addResource(MediaResource::VideoBatteryResource());
941 });
942 }
943
944 BufferFlightTiming_t startdata;
945 bool valid = false;
946 while (mBuffersInFlight.size() > 0) {
947 startdata = *mBuffersInFlight.begin();
948 ALOGV("-- Looking at startdata. presentation %" PRId64 ", start %" PRId64,
949 startdata.presentationUs, startdata.startedNs);
950 if (startdata.presentationUs == presentationUs) {
951 // a match
952 ALOGV("-- match entry for %" PRId64 ", hits our frame of %" PRId64,
953 startdata.presentationUs, presentationUs);
954 mBuffersInFlight.pop_front();
955 valid = true;
956 break;
957 } else if (startdata.presentationUs < presentationUs) {
958 // we must have missed the match for this, drop it and keep looking
959 ALOGV("-- drop entry for %" PRId64 ", before our frame of %" PRId64,
960 startdata.presentationUs, presentationUs);
961 mBuffersInFlight.pop_front();
962 continue;
963 } else {
964 // head is after, so we don't have a frame for ourselves
965 ALOGV("-- found entry for %" PRId64 ", AFTER our frame of %" PRId64
966 " we have nothing to pair with",
967 startdata.presentationUs, presentationUs);
968 mLatencyUnknown++;
969 return;
970 }
971 }
972 if (!valid) {
973 ALOGV("-- empty queue, so ignore that.");
974 mLatencyUnknown++;
975 return;
976 }
977
978 // nowNs start our calculations
979 const int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC);
980 int64_t latencyUs = (nowNs - startdata.startedNs + 500) / 1000;
981
982 mLatencyHist.insert(latencyUs);
983
984 // push into the recent samples
985 {
986 Mutex::Autolock al(mRecentLock);
987
988 if (mRecentHead >= kRecentLatencyFrames) {
989 mRecentHead = 0;
990 }
991 mRecentSamples[mRecentHead++] = latencyUs;
992 }
993 }
994
995 // static
PostAndAwaitResponse(const sp<AMessage> & msg,sp<AMessage> * response)996 status_t MediaCodec::PostAndAwaitResponse(
997 const sp<AMessage> &msg, sp<AMessage> *response) {
998 status_t err = msg->postAndAwaitResponse(response);
999
1000 if (err != OK) {
1001 return err;
1002 }
1003
1004 if (!(*response)->findInt32("err", &err)) {
1005 err = OK;
1006 }
1007
1008 return err;
1009 }
1010
PostReplyWithError(const sp<AReplyToken> & replyID,int32_t err)1011 void MediaCodec::PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err) {
1012 int32_t finalErr = err;
1013 if (mReleasedByResourceManager) {
1014 // override the err code if MediaCodec has been released by ResourceManager.
1015 finalErr = DEAD_OBJECT;
1016 }
1017
1018 sp<AMessage> response = new AMessage;
1019 response->setInt32("err", finalErr);
1020 response->postReply(replyID);
1021 }
1022
CreateCCodec()1023 static CodecBase *CreateCCodec() {
1024 return new CCodec;
1025 }
1026
1027 //static
GetCodecBase(const AString & name,const char * owner)1028 sp<CodecBase> MediaCodec::GetCodecBase(const AString &name, const char *owner) {
1029 if (owner) {
1030 if (strcmp(owner, "default") == 0) {
1031 return new ACodec;
1032 } else if (strncmp(owner, "codec2", 6) == 0) {
1033 return CreateCCodec();
1034 }
1035 }
1036
1037 if (name.startsWithIgnoreCase("c2.")) {
1038 return CreateCCodec();
1039 } else if (name.startsWithIgnoreCase("omx.")) {
1040 // at this time only ACodec specifies a mime type.
1041 return new ACodec;
1042 } else if (name.startsWithIgnoreCase("android.filter.")) {
1043 return new MediaFilter;
1044 } else {
1045 return NULL;
1046 }
1047 }
1048
1049 struct CodecListCache {
CodecListCacheandroid::CodecListCache1050 CodecListCache()
1051 : mCodecInfoMap{[] {
1052 const sp<IMediaCodecList> mcl = MediaCodecList::getInstance();
1053 size_t count = mcl->countCodecs();
1054 std::map<std::string, sp<MediaCodecInfo>> codecInfoMap;
1055 for (size_t i = 0; i < count; ++i) {
1056 sp<MediaCodecInfo> info = mcl->getCodecInfo(i);
1057 codecInfoMap.emplace(info->getCodecName(), info);
1058 }
1059 return codecInfoMap;
1060 }()} {
1061 }
1062
1063 const std::map<std::string, sp<MediaCodecInfo>> mCodecInfoMap;
1064 };
1065
GetCodecListCache()1066 static const CodecListCache &GetCodecListCache() {
1067 static CodecListCache sCache{};
1068 return sCache;
1069 }
1070
init(const AString & name)1071 status_t MediaCodec::init(const AString &name) {
1072 mResourceManagerProxy->init();
1073
1074 // save init parameters for reset
1075 mInitName = name;
1076
1077 // Current video decoders do not return from OMX_FillThisBuffer
1078 // quickly, violating the OpenMAX specs, until that is remedied
1079 // we need to invest in an extra looper to free the main event
1080 // queue.
1081
1082 mCodecInfo.clear();
1083
1084 bool secureCodec = false;
1085 const char *owner = "";
1086 if (!name.startsWith("android.filter.")) {
1087 AString tmp = name;
1088 if (tmp.endsWith(".secure")) {
1089 secureCodec = true;
1090 tmp.erase(tmp.size() - 7, 7);
1091 }
1092 const sp<IMediaCodecList> mcl = MediaCodecList::getInstance();
1093 if (mcl == NULL) {
1094 mCodec = NULL; // remove the codec.
1095 return NO_INIT; // if called from Java should raise IOException
1096 }
1097 for (const AString &codecName : { name, tmp }) {
1098 ssize_t codecIdx = mcl->findCodecByName(codecName.c_str());
1099 if (codecIdx < 0) {
1100 continue;
1101 }
1102 mCodecInfo = mcl->getCodecInfo(codecIdx);
1103 Vector<AString> mediaTypes;
1104 mCodecInfo->getSupportedMediaTypes(&mediaTypes);
1105 for (size_t i = 0; i < mediaTypes.size(); i++) {
1106 if (mediaTypes[i].startsWith("video/")) {
1107 mIsVideo = true;
1108 break;
1109 }
1110 }
1111 break;
1112 }
1113 if (mCodecInfo == nullptr) {
1114 return NAME_NOT_FOUND;
1115 }
1116 owner = mCodecInfo->getOwnerName();
1117 }
1118
1119 mCodec = GetCodecBase(name, owner);
1120 if (mCodec == NULL) {
1121 return NAME_NOT_FOUND;
1122 }
1123
1124 if (mIsVideo) {
1125 // video codec needs dedicated looper
1126 if (mCodecLooper == NULL) {
1127 mCodecLooper = new ALooper;
1128 mCodecLooper->setName("CodecLooper");
1129 mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
1130 }
1131
1132 mCodecLooper->registerHandler(mCodec);
1133 } else {
1134 mLooper->registerHandler(mCodec);
1135 }
1136
1137 mLooper->registerHandler(this);
1138
1139 mCodec->setCallback(
1140 std::unique_ptr<CodecBase::CodecCallback>(
1141 new CodecCallback(new AMessage(kWhatCodecNotify, this))));
1142 mBufferChannel = mCodec->getBufferChannel();
1143 mBufferChannel->setCallback(
1144 std::unique_ptr<CodecBase::BufferCallback>(
1145 new BufferCallback(new AMessage(kWhatCodecNotify, this))));
1146
1147 sp<AMessage> msg = new AMessage(kWhatInit, this);
1148 if (mCodecInfo) {
1149 msg->setObject("codecInfo", mCodecInfo);
1150 // name may be different from mCodecInfo->getCodecName() if we stripped
1151 // ".secure"
1152 }
1153 msg->setString("name", name);
1154
1155 if (mMetricsHandle != 0) {
1156 mediametrics_setCString(mMetricsHandle, kCodecCodec, name.c_str());
1157 mediametrics_setCString(mMetricsHandle, kCodecMode,
1158 mIsVideo ? kCodecModeVideo : kCodecModeAudio);
1159 }
1160
1161 if (mIsVideo) {
1162 mBatteryChecker = new BatteryChecker(new AMessage(kWhatCheckBatteryStats, this));
1163 }
1164
1165 status_t err;
1166 std::vector<MediaResourceParcel> resources;
1167 resources.push_back(MediaResource::CodecResource(secureCodec, mIsVideo));
1168 for (int i = 0; i <= kMaxRetry; ++i) {
1169 if (i > 0) {
1170 // Don't try to reclaim resource for the first time.
1171 if (!mResourceManagerProxy->reclaimResource(resources)) {
1172 break;
1173 }
1174 }
1175
1176 sp<AMessage> response;
1177 err = PostAndAwaitResponse(msg, &response);
1178 if (!isResourceError(err)) {
1179 break;
1180 }
1181 }
1182 return err;
1183 }
1184
setCallback(const sp<AMessage> & callback)1185 status_t MediaCodec::setCallback(const sp<AMessage> &callback) {
1186 sp<AMessage> msg = new AMessage(kWhatSetCallback, this);
1187 msg->setMessage("callback", callback);
1188
1189 sp<AMessage> response;
1190 return PostAndAwaitResponse(msg, &response);
1191 }
1192
setOnFrameRenderedNotification(const sp<AMessage> & notify)1193 status_t MediaCodec::setOnFrameRenderedNotification(const sp<AMessage> ¬ify) {
1194 sp<AMessage> msg = new AMessage(kWhatSetNotification, this);
1195 msg->setMessage("on-frame-rendered", notify);
1196 return msg->post();
1197 }
1198
configure(const sp<AMessage> & format,const sp<Surface> & nativeWindow,const sp<ICrypto> & crypto,uint32_t flags)1199 status_t MediaCodec::configure(
1200 const sp<AMessage> &format,
1201 const sp<Surface> &nativeWindow,
1202 const sp<ICrypto> &crypto,
1203 uint32_t flags) {
1204 return configure(format, nativeWindow, crypto, NULL, flags);
1205 }
1206
configure(const sp<AMessage> & format,const sp<Surface> & surface,const sp<ICrypto> & crypto,const sp<IDescrambler> & descrambler,uint32_t flags)1207 status_t MediaCodec::configure(
1208 const sp<AMessage> &format,
1209 const sp<Surface> &surface,
1210 const sp<ICrypto> &crypto,
1211 const sp<IDescrambler> &descrambler,
1212 uint32_t flags) {
1213 sp<AMessage> msg = new AMessage(kWhatConfigure, this);
1214
1215 if (mMetricsHandle != 0) {
1216 int32_t profile = 0;
1217 if (format->findInt32("profile", &profile)) {
1218 mediametrics_setInt32(mMetricsHandle, kCodecProfile, profile);
1219 }
1220 int32_t level = 0;
1221 if (format->findInt32("level", &level)) {
1222 mediametrics_setInt32(mMetricsHandle, kCodecLevel, level);
1223 }
1224 mediametrics_setInt32(mMetricsHandle, kCodecEncoder,
1225 (flags & CONFIGURE_FLAG_ENCODE) ? 1 : 0);
1226 }
1227
1228 if (mIsVideo) {
1229 format->findInt32("width", &mVideoWidth);
1230 format->findInt32("height", &mVideoHeight);
1231 if (!format->findInt32("rotation-degrees", &mRotationDegrees)) {
1232 mRotationDegrees = 0;
1233 }
1234
1235 if (mMetricsHandle != 0) {
1236 mediametrics_setInt32(mMetricsHandle, kCodecWidth, mVideoWidth);
1237 mediametrics_setInt32(mMetricsHandle, kCodecHeight, mVideoHeight);
1238 mediametrics_setInt32(mMetricsHandle, kCodecRotation, mRotationDegrees);
1239 int32_t maxWidth = 0;
1240 if (format->findInt32("max-width", &maxWidth)) {
1241 mediametrics_setInt32(mMetricsHandle, kCodecMaxWidth, maxWidth);
1242 }
1243 int32_t maxHeight = 0;
1244 if (format->findInt32("max-height", &maxHeight)) {
1245 mediametrics_setInt32(mMetricsHandle, kCodecMaxHeight, maxHeight);
1246 }
1247 }
1248
1249 // Prevent possible integer overflow in downstream code.
1250 if (mVideoWidth < 0 || mVideoHeight < 0 ||
1251 (uint64_t)mVideoWidth * mVideoHeight > (uint64_t)INT32_MAX / 4) {
1252 ALOGE("Invalid size(s), width=%d, height=%d", mVideoWidth, mVideoHeight);
1253 return BAD_VALUE;
1254 }
1255 }
1256
1257 updateLowLatency(format);
1258
1259 msg->setMessage("format", format);
1260 msg->setInt32("flags", flags);
1261 msg->setObject("surface", surface);
1262
1263 if (crypto != NULL || descrambler != NULL) {
1264 if (crypto != NULL) {
1265 msg->setPointer("crypto", crypto.get());
1266 } else {
1267 msg->setPointer("descrambler", descrambler.get());
1268 }
1269 if (mMetricsHandle != 0) {
1270 mediametrics_setInt32(mMetricsHandle, kCodecCrypto, 1);
1271 }
1272 } else if (mFlags & kFlagIsSecure) {
1273 ALOGW("Crypto or descrambler should be given for secure codec");
1274 }
1275
1276 // save msg for reset
1277 mConfigureMsg = msg;
1278
1279 status_t err;
1280 std::vector<MediaResourceParcel> resources;
1281 resources.push_back(MediaResource::CodecResource(mFlags & kFlagIsSecure, mIsVideo));
1282 // Don't know the buffer size at this point, but it's fine to use 1 because
1283 // the reclaimResource call doesn't consider the requester's buffer size for now.
1284 resources.push_back(MediaResource::GraphicMemoryResource(1));
1285 for (int i = 0; i <= kMaxRetry; ++i) {
1286 if (i > 0) {
1287 // Don't try to reclaim resource for the first time.
1288 if (!mResourceManagerProxy->reclaimResource(resources)) {
1289 break;
1290 }
1291 }
1292
1293 sp<AMessage> response;
1294 err = PostAndAwaitResponse(msg, &response);
1295 if (err != OK && err != INVALID_OPERATION) {
1296 // MediaCodec now set state to UNINITIALIZED upon any fatal error.
1297 // To maintain backward-compatibility, do a reset() to put codec
1298 // back into INITIALIZED state.
1299 // But don't reset if the err is INVALID_OPERATION, which means
1300 // the configure failure is due to wrong state.
1301
1302 ALOGE("configure failed with err 0x%08x, resetting...", err);
1303 reset();
1304 }
1305 if (!isResourceError(err)) {
1306 break;
1307 }
1308 }
1309
1310 return err;
1311 }
1312
releaseCrypto()1313 status_t MediaCodec::releaseCrypto()
1314 {
1315 ALOGV("releaseCrypto");
1316
1317 sp<AMessage> msg = new AMessage(kWhatDrmReleaseCrypto, this);
1318
1319 sp<AMessage> response;
1320 status_t status = msg->postAndAwaitResponse(&response);
1321
1322 if (status == OK && response != NULL) {
1323 CHECK(response->findInt32("status", &status));
1324 ALOGV("releaseCrypto ret: %d ", status);
1325 }
1326 else {
1327 ALOGE("releaseCrypto err: %d", status);
1328 }
1329
1330 return status;
1331 }
1332
onReleaseCrypto(const sp<AMessage> & msg)1333 void MediaCodec::onReleaseCrypto(const sp<AMessage>& msg)
1334 {
1335 status_t status = INVALID_OPERATION;
1336 if (mCrypto != NULL) {
1337 ALOGV("onReleaseCrypto: mCrypto: %p (%d)", mCrypto.get(), mCrypto->getStrongCount());
1338 mBufferChannel->setCrypto(NULL);
1339 // TODO change to ALOGV
1340 ALOGD("onReleaseCrypto: [before clear] mCrypto: %p (%d)",
1341 mCrypto.get(), mCrypto->getStrongCount());
1342 mCrypto.clear();
1343
1344 status = OK;
1345 }
1346 else {
1347 ALOGW("onReleaseCrypto: No mCrypto. err: %d", status);
1348 }
1349
1350 sp<AMessage> response = new AMessage;
1351 response->setInt32("status", status);
1352
1353 sp<AReplyToken> replyID;
1354 CHECK(msg->senderAwaitsResponse(&replyID));
1355 response->postReply(replyID);
1356 }
1357
setInputSurface(const sp<PersistentSurface> & surface)1358 status_t MediaCodec::setInputSurface(
1359 const sp<PersistentSurface> &surface) {
1360 sp<AMessage> msg = new AMessage(kWhatSetInputSurface, this);
1361 msg->setObject("input-surface", surface.get());
1362
1363 sp<AMessage> response;
1364 return PostAndAwaitResponse(msg, &response);
1365 }
1366
setSurface(const sp<Surface> & surface)1367 status_t MediaCodec::setSurface(const sp<Surface> &surface) {
1368 sp<AMessage> msg = new AMessage(kWhatSetSurface, this);
1369 msg->setObject("surface", surface);
1370
1371 sp<AMessage> response;
1372 return PostAndAwaitResponse(msg, &response);
1373 }
1374
createInputSurface(sp<IGraphicBufferProducer> * bufferProducer)1375 status_t MediaCodec::createInputSurface(
1376 sp<IGraphicBufferProducer>* bufferProducer) {
1377 sp<AMessage> msg = new AMessage(kWhatCreateInputSurface, this);
1378
1379 sp<AMessage> response;
1380 status_t err = PostAndAwaitResponse(msg, &response);
1381 if (err == NO_ERROR) {
1382 // unwrap the sp<IGraphicBufferProducer>
1383 sp<RefBase> obj;
1384 bool found = response->findObject("input-surface", &obj);
1385 CHECK(found);
1386 sp<BufferProducerWrapper> wrapper(
1387 static_cast<BufferProducerWrapper*>(obj.get()));
1388 *bufferProducer = wrapper->getBufferProducer();
1389 } else {
1390 ALOGW("createInputSurface failed, err=%d", err);
1391 }
1392 return err;
1393 }
1394
getGraphicBufferSize()1395 uint64_t MediaCodec::getGraphicBufferSize() {
1396 if (!mIsVideo) {
1397 return 0;
1398 }
1399
1400 uint64_t size = 0;
1401 size_t portNum = sizeof(mPortBuffers) / sizeof((mPortBuffers)[0]);
1402 for (size_t i = 0; i < portNum; ++i) {
1403 // TODO: this is just an estimation, we should get the real buffer size from ACodec.
1404 size += mPortBuffers[i].size() * mVideoWidth * mVideoHeight * 3 / 2;
1405 }
1406 return size;
1407 }
1408
start()1409 status_t MediaCodec::start() {
1410 sp<AMessage> msg = new AMessage(kWhatStart, this);
1411
1412 status_t err;
1413 std::vector<MediaResourceParcel> resources;
1414 resources.push_back(MediaResource::CodecResource(mFlags & kFlagIsSecure, mIsVideo));
1415 // Don't know the buffer size at this point, but it's fine to use 1 because
1416 // the reclaimResource call doesn't consider the requester's buffer size for now.
1417 resources.push_back(MediaResource::GraphicMemoryResource(1));
1418 for (int i = 0; i <= kMaxRetry; ++i) {
1419 if (i > 0) {
1420 // Don't try to reclaim resource for the first time.
1421 if (!mResourceManagerProxy->reclaimResource(resources)) {
1422 break;
1423 }
1424 // Recover codec from previous error before retry start.
1425 err = reset();
1426 if (err != OK) {
1427 ALOGE("retrying start: failed to reset codec");
1428 break;
1429 }
1430 sp<AMessage> response;
1431 err = PostAndAwaitResponse(mConfigureMsg, &response);
1432 if (err != OK) {
1433 ALOGE("retrying start: failed to configure codec");
1434 break;
1435 }
1436 }
1437
1438 sp<AMessage> response;
1439 err = PostAndAwaitResponse(msg, &response);
1440 if (!isResourceError(err)) {
1441 break;
1442 }
1443 }
1444 return err;
1445 }
1446
stop()1447 status_t MediaCodec::stop() {
1448 sp<AMessage> msg = new AMessage(kWhatStop, this);
1449
1450 sp<AMessage> response;
1451 return PostAndAwaitResponse(msg, &response);
1452 }
1453
hasPendingBuffer(int portIndex)1454 bool MediaCodec::hasPendingBuffer(int portIndex) {
1455 return std::any_of(
1456 mPortBuffers[portIndex].begin(), mPortBuffers[portIndex].end(),
1457 [](const BufferInfo &info) { return info.mOwnedByClient; });
1458 }
1459
hasPendingBuffer()1460 bool MediaCodec::hasPendingBuffer() {
1461 return hasPendingBuffer(kPortIndexInput) || hasPendingBuffer(kPortIndexOutput);
1462 }
1463
reclaim(bool force)1464 status_t MediaCodec::reclaim(bool force) {
1465 ALOGD("MediaCodec::reclaim(%p) %s", this, mInitName.c_str());
1466 sp<AMessage> msg = new AMessage(kWhatRelease, this);
1467 msg->setInt32("reclaimed", 1);
1468 msg->setInt32("force", force ? 1 : 0);
1469
1470 sp<AMessage> response;
1471 status_t ret = PostAndAwaitResponse(msg, &response);
1472 if (ret == -ENOENT) {
1473 ALOGD("MediaCodec looper is gone, skip reclaim");
1474 ret = OK;
1475 }
1476 return ret;
1477 }
1478
release()1479 status_t MediaCodec::release() {
1480 sp<AMessage> msg = new AMessage(kWhatRelease, this);
1481 sp<AMessage> response;
1482 return PostAndAwaitResponse(msg, &response);
1483 }
1484
releaseAsync(const sp<AMessage> & notify)1485 status_t MediaCodec::releaseAsync(const sp<AMessage> ¬ify) {
1486 sp<AMessage> msg = new AMessage(kWhatRelease, this);
1487 msg->setMessage("async", notify);
1488 sp<AMessage> response;
1489 return PostAndAwaitResponse(msg, &response);
1490 }
1491
reset()1492 status_t MediaCodec::reset() {
1493 /* When external-facing MediaCodec object is created,
1494 it is already initialized. Thus, reset is essentially
1495 release() followed by init(), plus clearing the state */
1496
1497 status_t err = release();
1498
1499 // unregister handlers
1500 if (mCodec != NULL) {
1501 if (mCodecLooper != NULL) {
1502 mCodecLooper->unregisterHandler(mCodec->id());
1503 } else {
1504 mLooper->unregisterHandler(mCodec->id());
1505 }
1506 mCodec = NULL;
1507 }
1508 mLooper->unregisterHandler(id());
1509
1510 mFlags = 0; // clear all flags
1511 mStickyError = OK;
1512
1513 // reset state not reset by setState(UNINITIALIZED)
1514 mReplyID = 0;
1515 mDequeueInputReplyID = 0;
1516 mDequeueOutputReplyID = 0;
1517 mDequeueInputTimeoutGeneration = 0;
1518 mDequeueOutputTimeoutGeneration = 0;
1519 mHaveInputSurface = false;
1520
1521 if (err == OK) {
1522 err = init(mInitName);
1523 }
1524 return err;
1525 }
1526
queueInputBuffer(size_t index,size_t offset,size_t size,int64_t presentationTimeUs,uint32_t flags,AString * errorDetailMsg)1527 status_t MediaCodec::queueInputBuffer(
1528 size_t index,
1529 size_t offset,
1530 size_t size,
1531 int64_t presentationTimeUs,
1532 uint32_t flags,
1533 AString *errorDetailMsg) {
1534 if (errorDetailMsg != NULL) {
1535 errorDetailMsg->clear();
1536 }
1537
1538 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
1539 msg->setSize("index", index);
1540 msg->setSize("offset", offset);
1541 msg->setSize("size", size);
1542 msg->setInt64("timeUs", presentationTimeUs);
1543 msg->setInt32("flags", flags);
1544 msg->setPointer("errorDetailMsg", errorDetailMsg);
1545
1546 sp<AMessage> response;
1547 return PostAndAwaitResponse(msg, &response);
1548 }
1549
queueSecureInputBuffer(size_t index,size_t offset,const CryptoPlugin::SubSample * subSamples,size_t numSubSamples,const uint8_t key[16],const uint8_t iv[16],CryptoPlugin::Mode mode,const CryptoPlugin::Pattern & pattern,int64_t presentationTimeUs,uint32_t flags,AString * errorDetailMsg)1550 status_t MediaCodec::queueSecureInputBuffer(
1551 size_t index,
1552 size_t offset,
1553 const CryptoPlugin::SubSample *subSamples,
1554 size_t numSubSamples,
1555 const uint8_t key[16],
1556 const uint8_t iv[16],
1557 CryptoPlugin::Mode mode,
1558 const CryptoPlugin::Pattern &pattern,
1559 int64_t presentationTimeUs,
1560 uint32_t flags,
1561 AString *errorDetailMsg) {
1562 if (errorDetailMsg != NULL) {
1563 errorDetailMsg->clear();
1564 }
1565
1566 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
1567 msg->setSize("index", index);
1568 msg->setSize("offset", offset);
1569 msg->setPointer("subSamples", (void *)subSamples);
1570 msg->setSize("numSubSamples", numSubSamples);
1571 msg->setPointer("key", (void *)key);
1572 msg->setPointer("iv", (void *)iv);
1573 msg->setInt32("mode", mode);
1574 msg->setInt32("encryptBlocks", pattern.mEncryptBlocks);
1575 msg->setInt32("skipBlocks", pattern.mSkipBlocks);
1576 msg->setInt64("timeUs", presentationTimeUs);
1577 msg->setInt32("flags", flags);
1578 msg->setPointer("errorDetailMsg", errorDetailMsg);
1579
1580 sp<AMessage> response;
1581 status_t err = PostAndAwaitResponse(msg, &response);
1582
1583 return err;
1584 }
1585
queueBuffer(size_t index,const std::shared_ptr<C2Buffer> & buffer,int64_t presentationTimeUs,uint32_t flags,const sp<AMessage> & tunings,AString * errorDetailMsg)1586 status_t MediaCodec::queueBuffer(
1587 size_t index,
1588 const std::shared_ptr<C2Buffer> &buffer,
1589 int64_t presentationTimeUs,
1590 uint32_t flags,
1591 const sp<AMessage> &tunings,
1592 AString *errorDetailMsg) {
1593 if (errorDetailMsg != NULL) {
1594 errorDetailMsg->clear();
1595 }
1596
1597 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
1598 msg->setSize("index", index);
1599 sp<WrapperObject<std::shared_ptr<C2Buffer>>> obj{
1600 new WrapperObject<std::shared_ptr<C2Buffer>>{buffer}};
1601 msg->setObject("c2buffer", obj);
1602 msg->setInt64("timeUs", presentationTimeUs);
1603 msg->setInt32("flags", flags);
1604 msg->setMessage("tunings", tunings);
1605 msg->setPointer("errorDetailMsg", errorDetailMsg);
1606
1607 sp<AMessage> response;
1608 status_t err = PostAndAwaitResponse(msg, &response);
1609
1610 return err;
1611 }
1612
queueEncryptedBuffer(size_t index,const sp<hardware::HidlMemory> & buffer,size_t offset,const CryptoPlugin::SubSample * subSamples,size_t numSubSamples,const uint8_t key[16],const uint8_t iv[16],CryptoPlugin::Mode mode,const CryptoPlugin::Pattern & pattern,int64_t presentationTimeUs,uint32_t flags,const sp<AMessage> & tunings,AString * errorDetailMsg)1613 status_t MediaCodec::queueEncryptedBuffer(
1614 size_t index,
1615 const sp<hardware::HidlMemory> &buffer,
1616 size_t offset,
1617 const CryptoPlugin::SubSample *subSamples,
1618 size_t numSubSamples,
1619 const uint8_t key[16],
1620 const uint8_t iv[16],
1621 CryptoPlugin::Mode mode,
1622 const CryptoPlugin::Pattern &pattern,
1623 int64_t presentationTimeUs,
1624 uint32_t flags,
1625 const sp<AMessage> &tunings,
1626 AString *errorDetailMsg) {
1627 if (errorDetailMsg != NULL) {
1628 errorDetailMsg->clear();
1629 }
1630
1631 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
1632 msg->setSize("index", index);
1633 sp<WrapperObject<sp<hardware::HidlMemory>>> memory{
1634 new WrapperObject<sp<hardware::HidlMemory>>{buffer}};
1635 msg->setObject("memory", memory);
1636 msg->setSize("offset", offset);
1637 msg->setPointer("subSamples", (void *)subSamples);
1638 msg->setSize("numSubSamples", numSubSamples);
1639 msg->setPointer("key", (void *)key);
1640 msg->setPointer("iv", (void *)iv);
1641 msg->setInt32("mode", mode);
1642 msg->setInt32("encryptBlocks", pattern.mEncryptBlocks);
1643 msg->setInt32("skipBlocks", pattern.mSkipBlocks);
1644 msg->setInt64("timeUs", presentationTimeUs);
1645 msg->setInt32("flags", flags);
1646 msg->setMessage("tunings", tunings);
1647 msg->setPointer("errorDetailMsg", errorDetailMsg);
1648
1649 sp<AMessage> response;
1650 status_t err = PostAndAwaitResponse(msg, &response);
1651
1652 return err;
1653 }
1654
dequeueInputBuffer(size_t * index,int64_t timeoutUs)1655 status_t MediaCodec::dequeueInputBuffer(size_t *index, int64_t timeoutUs) {
1656 sp<AMessage> msg = new AMessage(kWhatDequeueInputBuffer, this);
1657 msg->setInt64("timeoutUs", timeoutUs);
1658
1659 sp<AMessage> response;
1660 status_t err;
1661 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1662 return err;
1663 }
1664
1665 CHECK(response->findSize("index", index));
1666
1667 return OK;
1668 }
1669
dequeueOutputBuffer(size_t * index,size_t * offset,size_t * size,int64_t * presentationTimeUs,uint32_t * flags,int64_t timeoutUs)1670 status_t MediaCodec::dequeueOutputBuffer(
1671 size_t *index,
1672 size_t *offset,
1673 size_t *size,
1674 int64_t *presentationTimeUs,
1675 uint32_t *flags,
1676 int64_t timeoutUs) {
1677 sp<AMessage> msg = new AMessage(kWhatDequeueOutputBuffer, this);
1678 msg->setInt64("timeoutUs", timeoutUs);
1679
1680 sp<AMessage> response;
1681 status_t err;
1682 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1683 return err;
1684 }
1685
1686 CHECK(response->findSize("index", index));
1687 CHECK(response->findSize("offset", offset));
1688 CHECK(response->findSize("size", size));
1689 CHECK(response->findInt64("timeUs", presentationTimeUs));
1690 CHECK(response->findInt32("flags", (int32_t *)flags));
1691
1692 return OK;
1693 }
1694
renderOutputBufferAndRelease(size_t index)1695 status_t MediaCodec::renderOutputBufferAndRelease(size_t index) {
1696 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this);
1697 msg->setSize("index", index);
1698 msg->setInt32("render", true);
1699
1700 sp<AMessage> response;
1701 return PostAndAwaitResponse(msg, &response);
1702 }
1703
renderOutputBufferAndRelease(size_t index,int64_t timestampNs)1704 status_t MediaCodec::renderOutputBufferAndRelease(size_t index, int64_t timestampNs) {
1705 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this);
1706 msg->setSize("index", index);
1707 msg->setInt32("render", true);
1708 msg->setInt64("timestampNs", timestampNs);
1709
1710 sp<AMessage> response;
1711 return PostAndAwaitResponse(msg, &response);
1712 }
1713
releaseOutputBuffer(size_t index)1714 status_t MediaCodec::releaseOutputBuffer(size_t index) {
1715 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this);
1716 msg->setSize("index", index);
1717
1718 sp<AMessage> response;
1719 return PostAndAwaitResponse(msg, &response);
1720 }
1721
signalEndOfInputStream()1722 status_t MediaCodec::signalEndOfInputStream() {
1723 sp<AMessage> msg = new AMessage(kWhatSignalEndOfInputStream, this);
1724
1725 sp<AMessage> response;
1726 return PostAndAwaitResponse(msg, &response);
1727 }
1728
getOutputFormat(sp<AMessage> * format) const1729 status_t MediaCodec::getOutputFormat(sp<AMessage> *format) const {
1730 sp<AMessage> msg = new AMessage(kWhatGetOutputFormat, this);
1731
1732 sp<AMessage> response;
1733 status_t err;
1734 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1735 return err;
1736 }
1737
1738 CHECK(response->findMessage("format", format));
1739
1740 return OK;
1741 }
1742
getInputFormat(sp<AMessage> * format) const1743 status_t MediaCodec::getInputFormat(sp<AMessage> *format) const {
1744 sp<AMessage> msg = new AMessage(kWhatGetInputFormat, this);
1745
1746 sp<AMessage> response;
1747 status_t err;
1748 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1749 return err;
1750 }
1751
1752 CHECK(response->findMessage("format", format));
1753
1754 return OK;
1755 }
1756
getName(AString * name) const1757 status_t MediaCodec::getName(AString *name) const {
1758 sp<AMessage> msg = new AMessage(kWhatGetName, this);
1759
1760 sp<AMessage> response;
1761 status_t err;
1762 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1763 return err;
1764 }
1765
1766 CHECK(response->findString("name", name));
1767
1768 return OK;
1769 }
1770
getCodecInfo(sp<MediaCodecInfo> * codecInfo) const1771 status_t MediaCodec::getCodecInfo(sp<MediaCodecInfo> *codecInfo) const {
1772 sp<AMessage> msg = new AMessage(kWhatGetCodecInfo, this);
1773
1774 sp<AMessage> response;
1775 status_t err;
1776 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1777 return err;
1778 }
1779
1780 sp<RefBase> obj;
1781 CHECK(response->findObject("codecInfo", &obj));
1782 *codecInfo = static_cast<MediaCodecInfo *>(obj.get());
1783
1784 return OK;
1785 }
1786
getMetrics(mediametrics_handle_t & reply)1787 status_t MediaCodec::getMetrics(mediametrics_handle_t &reply) {
1788
1789 reply = 0;
1790
1791 // shouldn't happen, but be safe
1792 if (mMetricsHandle == 0) {
1793 return UNKNOWN_ERROR;
1794 }
1795
1796 // update any in-flight data that's not carried within the record
1797 updateMediametrics();
1798
1799 // send it back to the caller.
1800 reply = mediametrics_dup(mMetricsHandle);
1801
1802 updateEphemeralMediametrics(reply);
1803
1804 return OK;
1805 }
1806
getInputBuffers(Vector<sp<MediaCodecBuffer>> * buffers) const1807 status_t MediaCodec::getInputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const {
1808 sp<AMessage> msg = new AMessage(kWhatGetBuffers, this);
1809 msg->setInt32("portIndex", kPortIndexInput);
1810 msg->setPointer("buffers", buffers);
1811
1812 sp<AMessage> response;
1813 return PostAndAwaitResponse(msg, &response);
1814 }
1815
getOutputBuffers(Vector<sp<MediaCodecBuffer>> * buffers) const1816 status_t MediaCodec::getOutputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const {
1817 sp<AMessage> msg = new AMessage(kWhatGetBuffers, this);
1818 msg->setInt32("portIndex", kPortIndexOutput);
1819 msg->setPointer("buffers", buffers);
1820
1821 sp<AMessage> response;
1822 return PostAndAwaitResponse(msg, &response);
1823 }
1824
getOutputBuffer(size_t index,sp<MediaCodecBuffer> * buffer)1825 status_t MediaCodec::getOutputBuffer(size_t index, sp<MediaCodecBuffer> *buffer) {
1826 sp<AMessage> format;
1827 return getBufferAndFormat(kPortIndexOutput, index, buffer, &format);
1828 }
1829
getOutputFormat(size_t index,sp<AMessage> * format)1830 status_t MediaCodec::getOutputFormat(size_t index, sp<AMessage> *format) {
1831 sp<MediaCodecBuffer> buffer;
1832 return getBufferAndFormat(kPortIndexOutput, index, &buffer, format);
1833 }
1834
getInputBuffer(size_t index,sp<MediaCodecBuffer> * buffer)1835 status_t MediaCodec::getInputBuffer(size_t index, sp<MediaCodecBuffer> *buffer) {
1836 sp<AMessage> format;
1837 return getBufferAndFormat(kPortIndexInput, index, buffer, &format);
1838 }
1839
isExecuting() const1840 bool MediaCodec::isExecuting() const {
1841 return mState == STARTED || mState == FLUSHED;
1842 }
1843
getBufferAndFormat(size_t portIndex,size_t index,sp<MediaCodecBuffer> * buffer,sp<AMessage> * format)1844 status_t MediaCodec::getBufferAndFormat(
1845 size_t portIndex, size_t index,
1846 sp<MediaCodecBuffer> *buffer, sp<AMessage> *format) {
1847 // use mutex instead of a context switch
1848 if (mReleasedByResourceManager) {
1849 ALOGE("getBufferAndFormat - resource already released");
1850 return DEAD_OBJECT;
1851 }
1852
1853 if (buffer == NULL) {
1854 ALOGE("getBufferAndFormat - null MediaCodecBuffer");
1855 return INVALID_OPERATION;
1856 }
1857
1858 if (format == NULL) {
1859 ALOGE("getBufferAndFormat - null AMessage");
1860 return INVALID_OPERATION;
1861 }
1862
1863 buffer->clear();
1864 format->clear();
1865
1866 if (!isExecuting()) {
1867 ALOGE("getBufferAndFormat - not executing");
1868 return INVALID_OPERATION;
1869 }
1870
1871 // we do not want mPortBuffers to change during this section
1872 // we also don't want mOwnedByClient to change during this
1873 Mutex::Autolock al(mBufferLock);
1874
1875 std::vector<BufferInfo> &buffers = mPortBuffers[portIndex];
1876 if (index >= buffers.size()) {
1877 ALOGE("getBufferAndFormat - trying to get buffer with "
1878 "bad index (index=%zu buffer_size=%zu)", index, buffers.size());
1879 return INVALID_OPERATION;
1880 }
1881
1882 const BufferInfo &info = buffers[index];
1883 if (!info.mOwnedByClient) {
1884 ALOGE("getBufferAndFormat - invalid operation "
1885 "(the index %zu is not owned by client)", index);
1886 return INVALID_OPERATION;
1887 }
1888
1889 *buffer = info.mData;
1890 *format = info.mData->format();
1891
1892 return OK;
1893 }
1894
flush()1895 status_t MediaCodec::flush() {
1896 sp<AMessage> msg = new AMessage(kWhatFlush, this);
1897
1898 sp<AMessage> response;
1899 return PostAndAwaitResponse(msg, &response);
1900 }
1901
requestIDRFrame()1902 status_t MediaCodec::requestIDRFrame() {
1903 (new AMessage(kWhatRequestIDRFrame, this))->post();
1904
1905 return OK;
1906 }
1907
requestActivityNotification(const sp<AMessage> & notify)1908 void MediaCodec::requestActivityNotification(const sp<AMessage> ¬ify) {
1909 sp<AMessage> msg = new AMessage(kWhatRequestActivityNotification, this);
1910 msg->setMessage("notify", notify);
1911 msg->post();
1912 }
1913
requestCpuBoostIfNeeded()1914 void MediaCodec::requestCpuBoostIfNeeded() {
1915 if (mCpuBoostRequested) {
1916 return;
1917 }
1918 int32_t colorFormat;
1919 if (mOutputFormat->contains("hdr-static-info")
1920 && mOutputFormat->findInt32("color-format", &colorFormat)
1921 // check format for OMX only, for C2 the format is always opaque since the
1922 // software rendering doesn't go through client
1923 && ((mSoftRenderer != NULL && colorFormat == OMX_COLOR_FormatYUV420Planar16)
1924 || mOwnerName.equalsIgnoreCase("codec2::software"))) {
1925 int32_t left, top, right, bottom, width, height;
1926 int64_t totalPixel = 0;
1927 if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) {
1928 totalPixel = (right - left + 1) * (bottom - top + 1);
1929 } else if (mOutputFormat->findInt32("width", &width)
1930 && mOutputFormat->findInt32("height", &height)) {
1931 totalPixel = width * height;
1932 }
1933 if (totalPixel >= 1920 * 1080) {
1934 mResourceManagerProxy->addResource(MediaResource::CpuBoostResource());
1935 mCpuBoostRequested = true;
1936 }
1937 }
1938 }
1939
BatteryChecker(const sp<AMessage> & msg,int64_t timeoutUs)1940 BatteryChecker::BatteryChecker(const sp<AMessage> &msg, int64_t timeoutUs)
1941 : mTimeoutUs(timeoutUs)
1942 , mLastActivityTimeUs(-1ll)
1943 , mBatteryStatNotified(false)
1944 , mBatteryCheckerGeneration(0)
1945 , mIsExecuting(false)
1946 , mBatteryCheckerMsg(msg) {}
1947
onCodecActivity(std::function<void ()> batteryOnCb)1948 void BatteryChecker::onCodecActivity(std::function<void()> batteryOnCb) {
1949 if (!isExecuting()) {
1950 // ignore if not executing
1951 return;
1952 }
1953 if (!mBatteryStatNotified) {
1954 batteryOnCb();
1955 mBatteryStatNotified = true;
1956 sp<AMessage> msg = mBatteryCheckerMsg->dup();
1957 msg->setInt32("generation", mBatteryCheckerGeneration);
1958
1959 // post checker and clear last activity time
1960 msg->post(mTimeoutUs);
1961 mLastActivityTimeUs = -1ll;
1962 } else {
1963 // update last activity time
1964 mLastActivityTimeUs = ALooper::GetNowUs();
1965 }
1966 }
1967
onCheckBatteryTimer(const sp<AMessage> & msg,std::function<void ()> batteryOffCb)1968 void BatteryChecker::onCheckBatteryTimer(
1969 const sp<AMessage> &msg, std::function<void()> batteryOffCb) {
1970 // ignore if this checker already expired because the client resource was removed
1971 int32_t generation;
1972 if (!msg->findInt32("generation", &generation)
1973 || generation != mBatteryCheckerGeneration) {
1974 return;
1975 }
1976
1977 if (mLastActivityTimeUs < 0ll) {
1978 // timed out inactive, do not repost checker
1979 batteryOffCb();
1980 mBatteryStatNotified = false;
1981 } else {
1982 // repost checker and clear last activity time
1983 msg->post(mTimeoutUs + mLastActivityTimeUs - ALooper::GetNowUs());
1984 mLastActivityTimeUs = -1ll;
1985 }
1986 }
1987
onClientRemoved()1988 void BatteryChecker::onClientRemoved() {
1989 mBatteryStatNotified = false;
1990 mBatteryCheckerGeneration++;
1991 }
1992
1993 ////////////////////////////////////////////////////////////////////////////////
1994
cancelPendingDequeueOperations()1995 void MediaCodec::cancelPendingDequeueOperations() {
1996 if (mFlags & kFlagDequeueInputPending) {
1997 PostReplyWithError(mDequeueInputReplyID, INVALID_OPERATION);
1998
1999 ++mDequeueInputTimeoutGeneration;
2000 mDequeueInputReplyID = 0;
2001 mFlags &= ~kFlagDequeueInputPending;
2002 }
2003
2004 if (mFlags & kFlagDequeueOutputPending) {
2005 PostReplyWithError(mDequeueOutputReplyID, INVALID_OPERATION);
2006
2007 ++mDequeueOutputTimeoutGeneration;
2008 mDequeueOutputReplyID = 0;
2009 mFlags &= ~kFlagDequeueOutputPending;
2010 }
2011 }
2012
handleDequeueInputBuffer(const sp<AReplyToken> & replyID,bool newRequest)2013 bool MediaCodec::handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest) {
2014 if (!isExecuting() || (mFlags & kFlagIsAsync)
2015 || (newRequest && (mFlags & kFlagDequeueInputPending))) {
2016 PostReplyWithError(replyID, INVALID_OPERATION);
2017 return true;
2018 } else if (mFlags & kFlagStickyError) {
2019 PostReplyWithError(replyID, getStickyError());
2020 return true;
2021 }
2022
2023 ssize_t index = dequeuePortBuffer(kPortIndexInput);
2024
2025 if (index < 0) {
2026 CHECK_EQ(index, -EAGAIN);
2027 return false;
2028 }
2029
2030 sp<AMessage> response = new AMessage;
2031 response->setSize("index", index);
2032 response->postReply(replyID);
2033
2034 return true;
2035 }
2036
handleDequeueOutputBuffer(const sp<AReplyToken> & replyID,bool newRequest)2037 bool MediaCodec::handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest) {
2038 if (!isExecuting() || (mFlags & kFlagIsAsync)
2039 || (newRequest && (mFlags & kFlagDequeueOutputPending))) {
2040 PostReplyWithError(replyID, INVALID_OPERATION);
2041 } else if (mFlags & kFlagStickyError) {
2042 PostReplyWithError(replyID, getStickyError());
2043 } else if (mFlags & kFlagOutputBuffersChanged) {
2044 PostReplyWithError(replyID, INFO_OUTPUT_BUFFERS_CHANGED);
2045 mFlags &= ~kFlagOutputBuffersChanged;
2046 } else if (mFlags & kFlagOutputFormatChanged) {
2047 PostReplyWithError(replyID, INFO_FORMAT_CHANGED);
2048 mFlags &= ~kFlagOutputFormatChanged;
2049 } else {
2050 sp<AMessage> response = new AMessage;
2051 ssize_t index = dequeuePortBuffer(kPortIndexOutput);
2052
2053 if (index < 0) {
2054 CHECK_EQ(index, -EAGAIN);
2055 return false;
2056 }
2057
2058 const sp<MediaCodecBuffer> &buffer =
2059 mPortBuffers[kPortIndexOutput][index].mData;
2060
2061 response->setSize("index", index);
2062 response->setSize("offset", buffer->offset());
2063 response->setSize("size", buffer->size());
2064
2065 int64_t timeUs;
2066 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
2067
2068 statsBufferReceived(timeUs);
2069
2070 response->setInt64("timeUs", timeUs);
2071
2072 int32_t flags;
2073 CHECK(buffer->meta()->findInt32("flags", &flags));
2074
2075 response->setInt32("flags", flags);
2076 response->postReply(replyID);
2077 }
2078
2079 return true;
2080 }
2081
onMessageReceived(const sp<AMessage> & msg)2082 void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {
2083 switch (msg->what()) {
2084 case kWhatCodecNotify:
2085 {
2086 int32_t what;
2087 CHECK(msg->findInt32("what", &what));
2088
2089 switch (what) {
2090 case kWhatError:
2091 {
2092 int32_t err, actionCode;
2093 CHECK(msg->findInt32("err", &err));
2094 CHECK(msg->findInt32("actionCode", &actionCode));
2095
2096 ALOGE("Codec reported err %#x, actionCode %d, while in state %d",
2097 err, actionCode, mState);
2098 if (err == DEAD_OBJECT) {
2099 mFlags |= kFlagSawMediaServerDie;
2100 mFlags &= ~kFlagIsComponentAllocated;
2101 }
2102
2103 bool sendErrorResponse = true;
2104
2105 switch (mState) {
2106 case INITIALIZING:
2107 {
2108 setState(UNINITIALIZED);
2109 break;
2110 }
2111
2112 case CONFIGURING:
2113 {
2114 if (actionCode == ACTION_CODE_FATAL) {
2115 mediametrics_setInt32(mMetricsHandle, kCodecError, err);
2116 mediametrics_setCString(mMetricsHandle, kCodecErrorState,
2117 stateString(mState).c_str());
2118 flushMediametrics();
2119 initMediametrics();
2120 }
2121 setState(actionCode == ACTION_CODE_FATAL ?
2122 UNINITIALIZED : INITIALIZED);
2123 break;
2124 }
2125
2126 case STARTING:
2127 {
2128 if (actionCode == ACTION_CODE_FATAL) {
2129 mediametrics_setInt32(mMetricsHandle, kCodecError, err);
2130 mediametrics_setCString(mMetricsHandle, kCodecErrorState,
2131 stateString(mState).c_str());
2132 flushMediametrics();
2133 initMediametrics();
2134 }
2135 setState(actionCode == ACTION_CODE_FATAL ?
2136 UNINITIALIZED : CONFIGURED);
2137 break;
2138 }
2139
2140 case RELEASING:
2141 {
2142 // Ignore the error, assuming we'll still get
2143 // the shutdown complete notification. If we
2144 // don't, we'll timeout and force release.
2145 sendErrorResponse = false;
2146 FALLTHROUGH_INTENDED;
2147 }
2148 case STOPPING:
2149 {
2150 if (mFlags & kFlagSawMediaServerDie) {
2151 // MediaServer died, there definitely won't
2152 // be a shutdown complete notification after
2153 // all.
2154
2155 // note that we're directly going from
2156 // STOPPING->UNINITIALIZED, instead of the
2157 // usual STOPPING->INITIALIZED state.
2158 setState(UNINITIALIZED);
2159 if (mState == RELEASING) {
2160 mComponentName.clear();
2161 }
2162 (new AMessage)->postReply(mReplyID);
2163 sendErrorResponse = false;
2164 }
2165 break;
2166 }
2167
2168 case FLUSHING:
2169 {
2170 if (actionCode == ACTION_CODE_FATAL) {
2171 mediametrics_setInt32(mMetricsHandle, kCodecError, err);
2172 mediametrics_setCString(mMetricsHandle, kCodecErrorState,
2173 stateString(mState).c_str());
2174 flushMediametrics();
2175 initMediametrics();
2176
2177 setState(UNINITIALIZED);
2178 } else {
2179 setState(
2180 (mFlags & kFlagIsAsync) ? FLUSHED : STARTED);
2181 }
2182 break;
2183 }
2184
2185 case FLUSHED:
2186 case STARTED:
2187 {
2188 sendErrorResponse = false;
2189
2190 setStickyError(err);
2191 postActivityNotificationIfPossible();
2192
2193 cancelPendingDequeueOperations();
2194
2195 if (mFlags & kFlagIsAsync) {
2196 onError(err, actionCode);
2197 }
2198 switch (actionCode) {
2199 case ACTION_CODE_TRANSIENT:
2200 break;
2201 case ACTION_CODE_RECOVERABLE:
2202 setState(INITIALIZED);
2203 break;
2204 default:
2205 mediametrics_setInt32(mMetricsHandle, kCodecError, err);
2206 mediametrics_setCString(mMetricsHandle, kCodecErrorState,
2207 stateString(mState).c_str());
2208 flushMediametrics();
2209 initMediametrics();
2210 setState(UNINITIALIZED);
2211 break;
2212 }
2213 break;
2214 }
2215
2216 default:
2217 {
2218 sendErrorResponse = false;
2219
2220 setStickyError(err);
2221 postActivityNotificationIfPossible();
2222
2223 // actionCode in an uninitialized state is always fatal.
2224 if (mState == UNINITIALIZED) {
2225 actionCode = ACTION_CODE_FATAL;
2226 }
2227 if (mFlags & kFlagIsAsync) {
2228 onError(err, actionCode);
2229 }
2230 switch (actionCode) {
2231 case ACTION_CODE_TRANSIENT:
2232 break;
2233 case ACTION_CODE_RECOVERABLE:
2234 setState(INITIALIZED);
2235 break;
2236 default:
2237 setState(UNINITIALIZED);
2238 break;
2239 }
2240 break;
2241 }
2242 }
2243
2244 if (sendErrorResponse) {
2245 PostReplyWithError(mReplyID, err);
2246 }
2247 break;
2248 }
2249
2250 case kWhatComponentAllocated:
2251 {
2252 if (mState == RELEASING || mState == UNINITIALIZED) {
2253 // In case a kWhatError or kWhatRelease message came in and replied,
2254 // we log a warning and ignore.
2255 ALOGW("allocate interrupted by error or release, current state %d",
2256 mState);
2257 break;
2258 }
2259 CHECK_EQ(mState, INITIALIZING);
2260 setState(INITIALIZED);
2261 mFlags |= kFlagIsComponentAllocated;
2262
2263 CHECK(msg->findString("componentName", &mComponentName));
2264
2265 if (mComponentName.c_str()) {
2266 mediametrics_setCString(mMetricsHandle, kCodecCodec,
2267 mComponentName.c_str());
2268 }
2269
2270 const char *owner = mCodecInfo ? mCodecInfo->getOwnerName() : "";
2271 if (mComponentName.startsWith("OMX.google.")
2272 && strncmp(owner, "default", 8) == 0) {
2273 mFlags |= kFlagUsesSoftwareRenderer;
2274 } else {
2275 mFlags &= ~kFlagUsesSoftwareRenderer;
2276 }
2277 mOwnerName = owner;
2278
2279 if (mComponentName.endsWith(".secure")) {
2280 mFlags |= kFlagIsSecure;
2281 mediametrics_setInt32(mMetricsHandle, kCodecSecure, 1);
2282 } else {
2283 mFlags &= ~kFlagIsSecure;
2284 mediametrics_setInt32(mMetricsHandle, kCodecSecure, 0);
2285 }
2286
2287 if (mIsVideo) {
2288 // audio codec is currently ignored.
2289 mResourceManagerProxy->addResource(
2290 MediaResource::CodecResource(mFlags & kFlagIsSecure, mIsVideo));
2291 }
2292
2293 (new AMessage)->postReply(mReplyID);
2294 break;
2295 }
2296
2297 case kWhatComponentConfigured:
2298 {
2299 if (mState == RELEASING || mState == UNINITIALIZED || mState == INITIALIZED) {
2300 // In case a kWhatError or kWhatRelease message came in and replied,
2301 // we log a warning and ignore.
2302 ALOGW("configure interrupted by error or release, current state %d",
2303 mState);
2304 break;
2305 }
2306 CHECK_EQ(mState, CONFIGURING);
2307
2308 // reset input surface flag
2309 mHaveInputSurface = false;
2310
2311 CHECK(msg->findMessage("input-format", &mInputFormat));
2312 CHECK(msg->findMessage("output-format", &mOutputFormat));
2313
2314 // limit to confirming the opt-in behavior to minimize any behavioral change
2315 if (mSurface != nullptr && !mAllowFrameDroppingBySurface) {
2316 // signal frame dropping mode in the input format as this may also be
2317 // meaningful and confusing for an encoder in a transcoder scenario
2318 mInputFormat->setInt32("allow-frame-drop", mAllowFrameDroppingBySurface);
2319 }
2320 sp<AMessage> interestingFormat =
2321 (mFlags & kFlagIsEncoder) ? mOutputFormat : mInputFormat;
2322 ALOGV("[%s] configured as input format: %s, output format: %s",
2323 mComponentName.c_str(),
2324 mInputFormat->debugString(4).c_str(),
2325 mOutputFormat->debugString(4).c_str());
2326 int32_t usingSwRenderer;
2327 if (mOutputFormat->findInt32("using-sw-renderer", &usingSwRenderer)
2328 && usingSwRenderer) {
2329 mFlags |= kFlagUsesSoftwareRenderer;
2330 }
2331 setState(CONFIGURED);
2332 (new AMessage)->postReply(mReplyID);
2333
2334 // augment our media metrics info, now that we know more things
2335 // such as what the codec extracted from any CSD passed in.
2336 if (mMetricsHandle != 0) {
2337 sp<AMessage> format;
2338 if (mConfigureMsg != NULL &&
2339 mConfigureMsg->findMessage("format", &format)) {
2340 // format includes: mime
2341 AString mime;
2342 if (format->findString("mime", &mime)) {
2343 mediametrics_setCString(mMetricsHandle, kCodecMime,
2344 mime.c_str());
2345 }
2346 }
2347 // perhaps video only?
2348 int32_t profile = 0;
2349 if (interestingFormat->findInt32("profile", &profile)) {
2350 mediametrics_setInt32(mMetricsHandle, kCodecProfile, profile);
2351 }
2352 int32_t level = 0;
2353 if (interestingFormat->findInt32("level", &level)) {
2354 mediametrics_setInt32(mMetricsHandle, kCodecLevel, level);
2355 }
2356 // bitrate and bitrate mode, encoder only
2357 if (mFlags & kFlagIsEncoder) {
2358 // encoder specific values
2359 int32_t bitrate_mode = -1;
2360 if (mOutputFormat->findInt32(KEY_BITRATE_MODE, &bitrate_mode)) {
2361 mediametrics_setCString(mMetricsHandle, kCodecBitrateMode,
2362 asString_BitrateMode(bitrate_mode));
2363 }
2364 int32_t bitrate = -1;
2365 if (mOutputFormat->findInt32(KEY_BIT_RATE, &bitrate)) {
2366 mediametrics_setInt32(mMetricsHandle, kCodecBitrate, bitrate);
2367 }
2368 } else {
2369 // decoder specific values
2370 }
2371 }
2372 break;
2373 }
2374
2375 case kWhatInputSurfaceCreated:
2376 {
2377 // response to initiateCreateInputSurface()
2378 status_t err = NO_ERROR;
2379 sp<AMessage> response = new AMessage;
2380 if (!msg->findInt32("err", &err)) {
2381 sp<RefBase> obj;
2382 msg->findObject("input-surface", &obj);
2383 CHECK(msg->findMessage("input-format", &mInputFormat));
2384 CHECK(msg->findMessage("output-format", &mOutputFormat));
2385 ALOGV("[%s] input surface created as input format: %s, output format: %s",
2386 mComponentName.c_str(),
2387 mInputFormat->debugString(4).c_str(),
2388 mOutputFormat->debugString(4).c_str());
2389 CHECK(obj != NULL);
2390 response->setObject("input-surface", obj);
2391 mHaveInputSurface = true;
2392 } else {
2393 response->setInt32("err", err);
2394 }
2395 response->postReply(mReplyID);
2396 break;
2397 }
2398
2399 case kWhatInputSurfaceAccepted:
2400 {
2401 // response to initiateSetInputSurface()
2402 status_t err = NO_ERROR;
2403 sp<AMessage> response = new AMessage();
2404 if (!msg->findInt32("err", &err)) {
2405 CHECK(msg->findMessage("input-format", &mInputFormat));
2406 CHECK(msg->findMessage("output-format", &mOutputFormat));
2407 mHaveInputSurface = true;
2408 } else {
2409 response->setInt32("err", err);
2410 }
2411 response->postReply(mReplyID);
2412 break;
2413 }
2414
2415 case kWhatSignaledInputEOS:
2416 {
2417 // response to signalEndOfInputStream()
2418 sp<AMessage> response = new AMessage;
2419 status_t err;
2420 if (msg->findInt32("err", &err)) {
2421 response->setInt32("err", err);
2422 }
2423 response->postReply(mReplyID);
2424 break;
2425 }
2426
2427 case kWhatStartCompleted:
2428 {
2429 if (mState == RELEASING || mState == UNINITIALIZED) {
2430 // In case a kWhatRelease message came in and replied,
2431 // we log a warning and ignore.
2432 ALOGW("start interrupted by release, current state %d", mState);
2433 break;
2434 }
2435
2436 CHECK_EQ(mState, STARTING);
2437 if (mIsVideo) {
2438 mResourceManagerProxy->addResource(
2439 MediaResource::GraphicMemoryResource(getGraphicBufferSize()));
2440 }
2441 setState(STARTED);
2442 (new AMessage)->postReply(mReplyID);
2443 break;
2444 }
2445
2446 case kWhatOutputBuffersChanged:
2447 {
2448 mFlags |= kFlagOutputBuffersChanged;
2449 postActivityNotificationIfPossible();
2450 break;
2451 }
2452
2453 case kWhatOutputFramesRendered:
2454 {
2455 // ignore these in all states except running, and check that we have a
2456 // notification set
2457 if (mState == STARTED && mOnFrameRenderedNotification != NULL) {
2458 sp<AMessage> notify = mOnFrameRenderedNotification->dup();
2459 notify->setMessage("data", msg);
2460 notify->post();
2461 }
2462 break;
2463 }
2464
2465 case kWhatFillThisBuffer:
2466 {
2467 /* size_t index = */updateBuffers(kPortIndexInput, msg);
2468
2469 if (mState == FLUSHING
2470 || mState == STOPPING
2471 || mState == RELEASING) {
2472 returnBuffersToCodecOnPort(kPortIndexInput);
2473 break;
2474 }
2475
2476 if (!mCSD.empty()) {
2477 ssize_t index = dequeuePortBuffer(kPortIndexInput);
2478 CHECK_GE(index, 0);
2479
2480 // If codec specific data had been specified as
2481 // part of the format in the call to configure and
2482 // if there's more csd left, we submit it here
2483 // clients only get access to input buffers once
2484 // this data has been exhausted.
2485
2486 status_t err = queueCSDInputBuffer(index);
2487
2488 if (err != OK) {
2489 ALOGE("queueCSDInputBuffer failed w/ error %d",
2490 err);
2491
2492 setStickyError(err);
2493 postActivityNotificationIfPossible();
2494
2495 cancelPendingDequeueOperations();
2496 }
2497 break;
2498 }
2499 if (!mLeftover.empty()) {
2500 ssize_t index = dequeuePortBuffer(kPortIndexInput);
2501 CHECK_GE(index, 0);
2502
2503 status_t err = handleLeftover(index);
2504 if (err != OK) {
2505 setStickyError(err);
2506 postActivityNotificationIfPossible();
2507 cancelPendingDequeueOperations();
2508 }
2509 break;
2510 }
2511
2512 if (mFlags & kFlagIsAsync) {
2513 if (!mHaveInputSurface) {
2514 if (mState == FLUSHED) {
2515 mHavePendingInputBuffers = true;
2516 } else {
2517 onInputBufferAvailable();
2518 }
2519 }
2520 } else if (mFlags & kFlagDequeueInputPending) {
2521 CHECK(handleDequeueInputBuffer(mDequeueInputReplyID));
2522
2523 ++mDequeueInputTimeoutGeneration;
2524 mFlags &= ~kFlagDequeueInputPending;
2525 mDequeueInputReplyID = 0;
2526 } else {
2527 postActivityNotificationIfPossible();
2528 }
2529 break;
2530 }
2531
2532 case kWhatDrainThisBuffer:
2533 {
2534 /* size_t index = */updateBuffers(kPortIndexOutput, msg);
2535
2536 if (mState == FLUSHING
2537 || mState == STOPPING
2538 || mState == RELEASING) {
2539 returnBuffersToCodecOnPort(kPortIndexOutput);
2540 break;
2541 }
2542
2543 sp<RefBase> obj;
2544 CHECK(msg->findObject("buffer", &obj));
2545 sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get());
2546
2547 if (mOutputFormat != buffer->format()) {
2548 if (mFlags & kFlagUseBlockModel) {
2549 sp<AMessage> diff1 = mOutputFormat->changesFrom(buffer->format());
2550 sp<AMessage> diff2 = buffer->format()->changesFrom(mOutputFormat);
2551 std::set<std::string> keys;
2552 size_t numEntries = diff1->countEntries();
2553 AMessage::Type type;
2554 for (size_t i = 0; i < numEntries; ++i) {
2555 keys.emplace(diff1->getEntryNameAt(i, &type));
2556 }
2557 numEntries = diff2->countEntries();
2558 for (size_t i = 0; i < numEntries; ++i) {
2559 keys.emplace(diff2->getEntryNameAt(i, &type));
2560 }
2561 sp<WrapperObject<std::set<std::string>>> changedKeys{
2562 new WrapperObject<std::set<std::string>>{std::move(keys)}};
2563 buffer->meta()->setObject("changedKeys", changedKeys);
2564 }
2565 mOutputFormat = buffer->format();
2566 ALOGV("[%s] output format changed to: %s",
2567 mComponentName.c_str(), mOutputFormat->debugString(4).c_str());
2568
2569 if (mSoftRenderer == NULL &&
2570 mSurface != NULL &&
2571 (mFlags & kFlagUsesSoftwareRenderer)) {
2572 AString mime;
2573 CHECK(mOutputFormat->findString("mime", &mime));
2574
2575 // TODO: propagate color aspects to software renderer to allow better
2576 // color conversion to RGB. For now, just mark dataspace for YUV
2577 // rendering.
2578 int32_t dataSpace;
2579 if (mOutputFormat->findInt32("android._dataspace", &dataSpace)) {
2580 ALOGD("[%s] setting dataspace on output surface to #%x",
2581 mComponentName.c_str(), dataSpace);
2582 int err = native_window_set_buffers_data_space(
2583 mSurface.get(), (android_dataspace)dataSpace);
2584 ALOGW_IF(err != 0, "failed to set dataspace on surface (%d)", err);
2585 }
2586 if (mOutputFormat->contains("hdr-static-info")) {
2587 HDRStaticInfo info;
2588 if (ColorUtils::getHDRStaticInfoFromFormat(mOutputFormat, &info)) {
2589 setNativeWindowHdrMetadata(mSurface.get(), &info);
2590 }
2591 }
2592
2593 sp<ABuffer> hdr10PlusInfo;
2594 if (mOutputFormat->findBuffer("hdr10-plus-info", &hdr10PlusInfo)
2595 && hdr10PlusInfo != nullptr && hdr10PlusInfo->size() > 0) {
2596 native_window_set_buffers_hdr10_plus_metadata(mSurface.get(),
2597 hdr10PlusInfo->size(), hdr10PlusInfo->data());
2598 }
2599
2600 if (mime.startsWithIgnoreCase("video/")) {
2601 mSurface->setDequeueTimeout(-1);
2602 mSoftRenderer = new SoftwareRenderer(mSurface, mRotationDegrees);
2603 }
2604 }
2605
2606 requestCpuBoostIfNeeded();
2607
2608 if (mFlags & kFlagIsEncoder) {
2609 // Before we announce the format change we should
2610 // collect codec specific data and amend the output
2611 // format as necessary.
2612 int32_t flags = 0;
2613 (void) buffer->meta()->findInt32("flags", &flags);
2614 if ((flags & BUFFER_FLAG_CODECCONFIG) && !(mFlags & kFlagIsSecure)) {
2615 status_t err =
2616 amendOutputFormatWithCodecSpecificData(buffer);
2617
2618 if (err != OK) {
2619 ALOGE("Codec spit out malformed codec "
2620 "specific data!");
2621 }
2622 }
2623 }
2624 if (mFlags & kFlagIsAsync) {
2625 onOutputFormatChanged();
2626 } else {
2627 mFlags |= kFlagOutputFormatChanged;
2628 postActivityNotificationIfPossible();
2629 }
2630
2631 // Notify mCrypto of video resolution changes
2632 if (mCrypto != NULL) {
2633 int32_t left, top, right, bottom, width, height;
2634 if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) {
2635 mCrypto->notifyResolution(right - left + 1, bottom - top + 1);
2636 } else if (mOutputFormat->findInt32("width", &width)
2637 && mOutputFormat->findInt32("height", &height)) {
2638 mCrypto->notifyResolution(width, height);
2639 }
2640 }
2641 }
2642
2643 if (mFlags & kFlagIsAsync) {
2644 onOutputBufferAvailable();
2645 } else if (mFlags & kFlagDequeueOutputPending) {
2646 CHECK(handleDequeueOutputBuffer(mDequeueOutputReplyID));
2647
2648 ++mDequeueOutputTimeoutGeneration;
2649 mFlags &= ~kFlagDequeueOutputPending;
2650 mDequeueOutputReplyID = 0;
2651 } else {
2652 postActivityNotificationIfPossible();
2653 }
2654
2655 break;
2656 }
2657
2658 case kWhatEOS:
2659 {
2660 // We already notify the client of this by using the
2661 // corresponding flag in "onOutputBufferReady".
2662 break;
2663 }
2664
2665 case kWhatStopCompleted:
2666 {
2667 if (mState != STOPPING) {
2668 ALOGW("Received kWhatStopCompleted in state %d", mState);
2669 break;
2670 }
2671 setState(INITIALIZED);
2672 (new AMessage)->postReply(mReplyID);
2673 break;
2674 }
2675
2676 case kWhatReleaseCompleted:
2677 {
2678 if (mState != RELEASING) {
2679 ALOGW("Received kWhatReleaseCompleted in state %d", mState);
2680 break;
2681 }
2682 setState(UNINITIALIZED);
2683 mComponentName.clear();
2684
2685 mFlags &= ~kFlagIsComponentAllocated;
2686
2687 // off since we're removing all resources including the battery on
2688 if (mBatteryChecker != nullptr) {
2689 mBatteryChecker->onClientRemoved();
2690 }
2691
2692 mResourceManagerProxy->removeClient();
2693 mReleaseSurface.reset();
2694
2695 if (mReplyID != nullptr) {
2696 (new AMessage)->postReply(mReplyID);
2697 }
2698 if (mAsyncReleaseCompleteNotification != nullptr) {
2699 flushMediametrics();
2700 mAsyncReleaseCompleteNotification->post();
2701 mAsyncReleaseCompleteNotification.clear();
2702 }
2703 break;
2704 }
2705
2706 case kWhatFlushCompleted:
2707 {
2708 if (mState != FLUSHING) {
2709 ALOGW("received FlushCompleted message in state %d",
2710 mState);
2711 break;
2712 }
2713
2714 if (mFlags & kFlagIsAsync) {
2715 setState(FLUSHED);
2716 } else {
2717 setState(STARTED);
2718 mCodec->signalResume();
2719 }
2720
2721 (new AMessage)->postReply(mReplyID);
2722 break;
2723 }
2724
2725 default:
2726 TRESPASS();
2727 }
2728 break;
2729 }
2730
2731 case kWhatInit:
2732 {
2733 sp<AReplyToken> replyID;
2734 CHECK(msg->senderAwaitsResponse(&replyID));
2735
2736 if (mState != UNINITIALIZED) {
2737 PostReplyWithError(replyID, INVALID_OPERATION);
2738 break;
2739 }
2740
2741 mReplyID = replyID;
2742 setState(INITIALIZING);
2743
2744 sp<RefBase> codecInfo;
2745 (void)msg->findObject("codecInfo", &codecInfo);
2746 AString name;
2747 CHECK(msg->findString("name", &name));
2748
2749 sp<AMessage> format = new AMessage;
2750 if (codecInfo) {
2751 format->setObject("codecInfo", codecInfo);
2752 }
2753 format->setString("componentName", name);
2754
2755 mCodec->initiateAllocateComponent(format);
2756 break;
2757 }
2758
2759 case kWhatSetNotification:
2760 {
2761 sp<AMessage> notify;
2762 if (msg->findMessage("on-frame-rendered", ¬ify)) {
2763 mOnFrameRenderedNotification = notify;
2764 }
2765 break;
2766 }
2767
2768 case kWhatSetCallback:
2769 {
2770 sp<AReplyToken> replyID;
2771 CHECK(msg->senderAwaitsResponse(&replyID));
2772
2773 if (mState == UNINITIALIZED
2774 || mState == INITIALIZING
2775 || isExecuting()) {
2776 // callback can't be set after codec is executing,
2777 // or before it's initialized (as the callback
2778 // will be cleared when it goes to INITIALIZED)
2779 PostReplyWithError(replyID, INVALID_OPERATION);
2780 break;
2781 }
2782
2783 sp<AMessage> callback;
2784 CHECK(msg->findMessage("callback", &callback));
2785
2786 mCallback = callback;
2787
2788 if (mCallback != NULL) {
2789 ALOGI("MediaCodec will operate in async mode");
2790 mFlags |= kFlagIsAsync;
2791 } else {
2792 mFlags &= ~kFlagIsAsync;
2793 }
2794
2795 sp<AMessage> response = new AMessage;
2796 response->postReply(replyID);
2797 break;
2798 }
2799
2800 case kWhatConfigure:
2801 {
2802 sp<AReplyToken> replyID;
2803 CHECK(msg->senderAwaitsResponse(&replyID));
2804
2805 if (mState != INITIALIZED) {
2806 PostReplyWithError(replyID, INVALID_OPERATION);
2807 break;
2808 }
2809
2810 sp<RefBase> obj;
2811 CHECK(msg->findObject("surface", &obj));
2812
2813 sp<AMessage> format;
2814 CHECK(msg->findMessage("format", &format));
2815
2816 int32_t push;
2817 if (msg->findInt32("push-blank-buffers-on-shutdown", &push) && push != 0) {
2818 mFlags |= kFlagPushBlankBuffersOnShutdown;
2819 }
2820
2821 if (obj != NULL) {
2822 if (!format->findInt32("allow-frame-drop", &mAllowFrameDroppingBySurface)) {
2823 // allow frame dropping by surface by default
2824 mAllowFrameDroppingBySurface = true;
2825 }
2826
2827 format->setObject("native-window", obj);
2828 status_t err = handleSetSurface(static_cast<Surface *>(obj.get()));
2829 if (err != OK) {
2830 PostReplyWithError(replyID, err);
2831 break;
2832 }
2833 } else {
2834 // we are not using surface so this variable is not used, but initialize sensibly anyway
2835 mAllowFrameDroppingBySurface = false;
2836
2837 handleSetSurface(NULL);
2838 }
2839
2840 uint32_t flags;
2841 CHECK(msg->findInt32("flags", (int32_t *)&flags));
2842 if (flags & CONFIGURE_FLAG_USE_BLOCK_MODEL) {
2843 if (!(mFlags & kFlagIsAsync)) {
2844 PostReplyWithError(replyID, INVALID_OPERATION);
2845 break;
2846 }
2847 mFlags |= kFlagUseBlockModel;
2848 }
2849 mReplyID = replyID;
2850 setState(CONFIGURING);
2851
2852 void *crypto;
2853 if (!msg->findPointer("crypto", &crypto)) {
2854 crypto = NULL;
2855 }
2856
2857 ALOGV("kWhatConfigure: Old mCrypto: %p (%d)",
2858 mCrypto.get(), (mCrypto != NULL ? mCrypto->getStrongCount() : 0));
2859
2860 mCrypto = static_cast<ICrypto *>(crypto);
2861 mBufferChannel->setCrypto(mCrypto);
2862
2863 ALOGV("kWhatConfigure: New mCrypto: %p (%d)",
2864 mCrypto.get(), (mCrypto != NULL ? mCrypto->getStrongCount() : 0));
2865
2866 void *descrambler;
2867 if (!msg->findPointer("descrambler", &descrambler)) {
2868 descrambler = NULL;
2869 }
2870
2871 mDescrambler = static_cast<IDescrambler *>(descrambler);
2872 mBufferChannel->setDescrambler(mDescrambler);
2873
2874 format->setInt32("flags", flags);
2875 if (flags & CONFIGURE_FLAG_ENCODE) {
2876 format->setInt32("encoder", true);
2877 mFlags |= kFlagIsEncoder;
2878 }
2879
2880 extractCSD(format);
2881
2882 mCodec->initiateConfigureComponent(format);
2883 break;
2884 }
2885
2886 case kWhatSetSurface:
2887 {
2888 sp<AReplyToken> replyID;
2889 CHECK(msg->senderAwaitsResponse(&replyID));
2890
2891 status_t err = OK;
2892
2893 switch (mState) {
2894 case CONFIGURED:
2895 case STARTED:
2896 case FLUSHED:
2897 {
2898 sp<RefBase> obj;
2899 (void)msg->findObject("surface", &obj);
2900 sp<Surface> surface = static_cast<Surface *>(obj.get());
2901 if (mSurface == NULL) {
2902 // do not support setting surface if it was not set
2903 err = INVALID_OPERATION;
2904 } else if (obj == NULL) {
2905 // do not support unsetting surface
2906 err = BAD_VALUE;
2907 } else {
2908 err = connectToSurface(surface);
2909 if (err == ALREADY_EXISTS) {
2910 // reconnecting to same surface
2911 err = OK;
2912 } else {
2913 if (err == OK) {
2914 if (mFlags & kFlagUsesSoftwareRenderer) {
2915 if (mSoftRenderer != NULL
2916 && (mFlags & kFlagPushBlankBuffersOnShutdown)) {
2917 pushBlankBuffersToNativeWindow(mSurface.get());
2918 }
2919 surface->setDequeueTimeout(-1);
2920 mSoftRenderer = new SoftwareRenderer(surface);
2921 // TODO: check if this was successful
2922 } else {
2923 err = mCodec->setSurface(surface);
2924 }
2925 }
2926 if (err == OK) {
2927 (void)disconnectFromSurface();
2928 mSurface = surface;
2929 }
2930 }
2931 }
2932 break;
2933 }
2934
2935 default:
2936 err = INVALID_OPERATION;
2937 break;
2938 }
2939
2940 PostReplyWithError(replyID, err);
2941 break;
2942 }
2943
2944 case kWhatCreateInputSurface:
2945 case kWhatSetInputSurface:
2946 {
2947 sp<AReplyToken> replyID;
2948 CHECK(msg->senderAwaitsResponse(&replyID));
2949
2950 // Must be configured, but can't have been started yet.
2951 if (mState != CONFIGURED) {
2952 PostReplyWithError(replyID, INVALID_OPERATION);
2953 break;
2954 }
2955
2956 mReplyID = replyID;
2957 if (msg->what() == kWhatCreateInputSurface) {
2958 mCodec->initiateCreateInputSurface();
2959 } else {
2960 sp<RefBase> obj;
2961 CHECK(msg->findObject("input-surface", &obj));
2962
2963 mCodec->initiateSetInputSurface(
2964 static_cast<PersistentSurface *>(obj.get()));
2965 }
2966 break;
2967 }
2968 case kWhatStart:
2969 {
2970 sp<AReplyToken> replyID;
2971 CHECK(msg->senderAwaitsResponse(&replyID));
2972
2973 if (mState == FLUSHED) {
2974 setState(STARTED);
2975 if (mHavePendingInputBuffers) {
2976 onInputBufferAvailable();
2977 mHavePendingInputBuffers = false;
2978 }
2979 mCodec->signalResume();
2980 PostReplyWithError(replyID, OK);
2981 break;
2982 } else if (mState != CONFIGURED) {
2983 PostReplyWithError(replyID, INVALID_OPERATION);
2984 break;
2985 }
2986
2987 mReplyID = replyID;
2988 setState(STARTING);
2989
2990 mCodec->initiateStart();
2991 break;
2992 }
2993
2994 case kWhatStop:
2995 case kWhatRelease:
2996 {
2997 State targetState =
2998 (msg->what() == kWhatStop) ? INITIALIZED : UNINITIALIZED;
2999
3000 sp<AReplyToken> replyID;
3001 CHECK(msg->senderAwaitsResponse(&replyID));
3002
3003 // already stopped/released
3004 if (mState == UNINITIALIZED && mReleasedByResourceManager) {
3005 sp<AMessage> response = new AMessage;
3006 response->setInt32("err", OK);
3007 response->postReply(replyID);
3008 break;
3009 }
3010
3011 int32_t reclaimed = 0;
3012 msg->findInt32("reclaimed", &reclaimed);
3013 if (reclaimed) {
3014 mReleasedByResourceManager = true;
3015
3016 int32_t force = 0;
3017 msg->findInt32("force", &force);
3018 if (!force && hasPendingBuffer()) {
3019 ALOGW("Can't reclaim codec right now due to pending buffers.");
3020
3021 // return WOULD_BLOCK to ask resource manager to retry later.
3022 sp<AMessage> response = new AMessage;
3023 response->setInt32("err", WOULD_BLOCK);
3024 response->postReply(replyID);
3025
3026 // notify the async client
3027 if (mFlags & kFlagIsAsync) {
3028 onError(DEAD_OBJECT, ACTION_CODE_FATAL);
3029 }
3030 break;
3031 }
3032 }
3033
3034 bool isReleasingAllocatedComponent =
3035 (mFlags & kFlagIsComponentAllocated) && targetState == UNINITIALIZED;
3036 if (!isReleasingAllocatedComponent // See 1
3037 && mState != INITIALIZED
3038 && mState != CONFIGURED && !isExecuting()) {
3039 // 1) Permit release to shut down the component if allocated.
3040 //
3041 // 2) We may be in "UNINITIALIZED" state already and
3042 // also shutdown the encoder/decoder without the
3043 // client being aware of this if media server died while
3044 // we were being stopped. The client would assume that
3045 // after stop() returned, it would be safe to call release()
3046 // and it should be in this case, no harm to allow a release()
3047 // if we're already uninitialized.
3048 sp<AMessage> response = new AMessage;
3049 // TODO: we shouldn't throw an exception for stop/release. Change this to wait until
3050 // the previous stop/release completes and then reply with OK.
3051 status_t err = mState == targetState ? OK : INVALID_OPERATION;
3052 response->setInt32("err", err);
3053 if (err == OK && targetState == UNINITIALIZED) {
3054 mComponentName.clear();
3055 }
3056 response->postReply(replyID);
3057 break;
3058 }
3059
3060 // If we're flushing, stopping, configuring or starting but
3061 // received a release request, post the reply for the pending call
3062 // first, and consider it done. The reply token will be replaced
3063 // after this, and we'll no longer be able to reply.
3064 if (mState == FLUSHING || mState == STOPPING
3065 || mState == CONFIGURING || mState == STARTING) {
3066 (new AMessage)->postReply(mReplyID);
3067 }
3068
3069 if (mFlags & kFlagSawMediaServerDie) {
3070 // It's dead, Jim. Don't expect initiateShutdown to yield
3071 // any useful results now...
3072 setState(UNINITIALIZED);
3073 if (targetState == UNINITIALIZED) {
3074 mComponentName.clear();
3075 }
3076 (new AMessage)->postReply(replyID);
3077 break;
3078 }
3079
3080 // If we already have an error, component may not be able to
3081 // complete the shutdown properly. If we're stopping, post the
3082 // reply now with an error to unblock the client, client can
3083 // release after the failure (instead of ANR).
3084 if (msg->what() == kWhatStop && (mFlags & kFlagStickyError)) {
3085 PostReplyWithError(replyID, getStickyError());
3086 break;
3087 }
3088
3089 sp<AMessage> asyncNotify;
3090 if (msg->findMessage("async", &asyncNotify) && asyncNotify != nullptr) {
3091 if (mSurface != NULL) {
3092 if (!mReleaseSurface) {
3093 mReleaseSurface.reset(new ReleaseSurface);
3094 }
3095 if (mSurface != mReleaseSurface->getSurface()) {
3096 status_t err = connectToSurface(mReleaseSurface->getSurface());
3097 ALOGW_IF(err != OK, "error connecting to release surface: err = %d", err);
3098 if (err == OK && !(mFlags & kFlagUsesSoftwareRenderer)) {
3099 err = mCodec->setSurface(mReleaseSurface->getSurface());
3100 ALOGW_IF(err != OK, "error setting release surface: err = %d", err);
3101 }
3102 if (err == OK) {
3103 (void)disconnectFromSurface();
3104 mSurface = mReleaseSurface->getSurface();
3105 }
3106 }
3107 }
3108 }
3109
3110 mReplyID = replyID;
3111 setState(msg->what() == kWhatStop ? STOPPING : RELEASING);
3112
3113 mCodec->initiateShutdown(
3114 msg->what() == kWhatStop /* keepComponentAllocated */);
3115
3116 returnBuffersToCodec(reclaimed);
3117
3118 if (mSoftRenderer != NULL && (mFlags & kFlagPushBlankBuffersOnShutdown)) {
3119 pushBlankBuffersToNativeWindow(mSurface.get());
3120 }
3121
3122 if (asyncNotify != nullptr) {
3123 mResourceManagerProxy->markClientForPendingRemoval();
3124 (new AMessage)->postReply(mReplyID);
3125 mReplyID = 0;
3126 mAsyncReleaseCompleteNotification = asyncNotify;
3127 }
3128
3129 break;
3130 }
3131
3132 case kWhatDequeueInputBuffer:
3133 {
3134 sp<AReplyToken> replyID;
3135 CHECK(msg->senderAwaitsResponse(&replyID));
3136
3137 if (mFlags & kFlagIsAsync) {
3138 ALOGE("dequeueInputBuffer can't be used in async mode");
3139 PostReplyWithError(replyID, INVALID_OPERATION);
3140 break;
3141 }
3142
3143 if (mHaveInputSurface) {
3144 ALOGE("dequeueInputBuffer can't be used with input surface");
3145 PostReplyWithError(replyID, INVALID_OPERATION);
3146 break;
3147 }
3148
3149 if (handleDequeueInputBuffer(replyID, true /* new request */)) {
3150 break;
3151 }
3152
3153 int64_t timeoutUs;
3154 CHECK(msg->findInt64("timeoutUs", &timeoutUs));
3155
3156 if (timeoutUs == 0LL) {
3157 PostReplyWithError(replyID, -EAGAIN);
3158 break;
3159 }
3160
3161 mFlags |= kFlagDequeueInputPending;
3162 mDequeueInputReplyID = replyID;
3163
3164 if (timeoutUs > 0LL) {
3165 sp<AMessage> timeoutMsg =
3166 new AMessage(kWhatDequeueInputTimedOut, this);
3167 timeoutMsg->setInt32(
3168 "generation", ++mDequeueInputTimeoutGeneration);
3169 timeoutMsg->post(timeoutUs);
3170 }
3171 break;
3172 }
3173
3174 case kWhatDequeueInputTimedOut:
3175 {
3176 int32_t generation;
3177 CHECK(msg->findInt32("generation", &generation));
3178
3179 if (generation != mDequeueInputTimeoutGeneration) {
3180 // Obsolete
3181 break;
3182 }
3183
3184 CHECK(mFlags & kFlagDequeueInputPending);
3185
3186 PostReplyWithError(mDequeueInputReplyID, -EAGAIN);
3187
3188 mFlags &= ~kFlagDequeueInputPending;
3189 mDequeueInputReplyID = 0;
3190 break;
3191 }
3192
3193 case kWhatQueueInputBuffer:
3194 {
3195 sp<AReplyToken> replyID;
3196 CHECK(msg->senderAwaitsResponse(&replyID));
3197
3198 if (!isExecuting()) {
3199 PostReplyWithError(replyID, INVALID_OPERATION);
3200 break;
3201 } else if (mFlags & kFlagStickyError) {
3202 PostReplyWithError(replyID, getStickyError());
3203 break;
3204 }
3205
3206 status_t err = UNKNOWN_ERROR;
3207 if (!mLeftover.empty()) {
3208 mLeftover.push_back(msg);
3209 size_t index;
3210 msg->findSize("index", &index);
3211 err = handleLeftover(index);
3212 } else {
3213 err = onQueueInputBuffer(msg);
3214 }
3215
3216 PostReplyWithError(replyID, err);
3217 break;
3218 }
3219
3220 case kWhatDequeueOutputBuffer:
3221 {
3222 sp<AReplyToken> replyID;
3223 CHECK(msg->senderAwaitsResponse(&replyID));
3224
3225 if (mFlags & kFlagIsAsync) {
3226 ALOGE("dequeueOutputBuffer can't be used in async mode");
3227 PostReplyWithError(replyID, INVALID_OPERATION);
3228 break;
3229 }
3230
3231 if (handleDequeueOutputBuffer(replyID, true /* new request */)) {
3232 break;
3233 }
3234
3235 int64_t timeoutUs;
3236 CHECK(msg->findInt64("timeoutUs", &timeoutUs));
3237
3238 if (timeoutUs == 0LL) {
3239 PostReplyWithError(replyID, -EAGAIN);
3240 break;
3241 }
3242
3243 mFlags |= kFlagDequeueOutputPending;
3244 mDequeueOutputReplyID = replyID;
3245
3246 if (timeoutUs > 0LL) {
3247 sp<AMessage> timeoutMsg =
3248 new AMessage(kWhatDequeueOutputTimedOut, this);
3249 timeoutMsg->setInt32(
3250 "generation", ++mDequeueOutputTimeoutGeneration);
3251 timeoutMsg->post(timeoutUs);
3252 }
3253 break;
3254 }
3255
3256 case kWhatDequeueOutputTimedOut:
3257 {
3258 int32_t generation;
3259 CHECK(msg->findInt32("generation", &generation));
3260
3261 if (generation != mDequeueOutputTimeoutGeneration) {
3262 // Obsolete
3263 break;
3264 }
3265
3266 CHECK(mFlags & kFlagDequeueOutputPending);
3267
3268 PostReplyWithError(mDequeueOutputReplyID, -EAGAIN);
3269
3270 mFlags &= ~kFlagDequeueOutputPending;
3271 mDequeueOutputReplyID = 0;
3272 break;
3273 }
3274
3275 case kWhatReleaseOutputBuffer:
3276 {
3277 sp<AReplyToken> replyID;
3278 CHECK(msg->senderAwaitsResponse(&replyID));
3279
3280 if (!isExecuting()) {
3281 PostReplyWithError(replyID, INVALID_OPERATION);
3282 break;
3283 } else if (mFlags & kFlagStickyError) {
3284 PostReplyWithError(replyID, getStickyError());
3285 break;
3286 }
3287
3288 status_t err = onReleaseOutputBuffer(msg);
3289
3290 PostReplyWithError(replyID, err);
3291 break;
3292 }
3293
3294 case kWhatSignalEndOfInputStream:
3295 {
3296 sp<AReplyToken> replyID;
3297 CHECK(msg->senderAwaitsResponse(&replyID));
3298
3299 if (!isExecuting() || !mHaveInputSurface) {
3300 PostReplyWithError(replyID, INVALID_OPERATION);
3301 break;
3302 } else if (mFlags & kFlagStickyError) {
3303 PostReplyWithError(replyID, getStickyError());
3304 break;
3305 }
3306
3307 mReplyID = replyID;
3308 mCodec->signalEndOfInputStream();
3309 break;
3310 }
3311
3312 case kWhatGetBuffers:
3313 {
3314 sp<AReplyToken> replyID;
3315 CHECK(msg->senderAwaitsResponse(&replyID));
3316 if (!isExecuting() || (mFlags & kFlagIsAsync)) {
3317 PostReplyWithError(replyID, INVALID_OPERATION);
3318 break;
3319 } else if (mFlags & kFlagStickyError) {
3320 PostReplyWithError(replyID, getStickyError());
3321 break;
3322 }
3323
3324 int32_t portIndex;
3325 CHECK(msg->findInt32("portIndex", &portIndex));
3326
3327 Vector<sp<MediaCodecBuffer> > *dstBuffers;
3328 CHECK(msg->findPointer("buffers", (void **)&dstBuffers));
3329
3330 dstBuffers->clear();
3331 // If we're using input surface (either non-persistent created by
3332 // createInputSurface(), or persistent set by setInputSurface()),
3333 // give the client an empty input buffers array.
3334 if (portIndex != kPortIndexInput || !mHaveInputSurface) {
3335 if (portIndex == kPortIndexInput) {
3336 mBufferChannel->getInputBufferArray(dstBuffers);
3337 } else {
3338 mBufferChannel->getOutputBufferArray(dstBuffers);
3339 }
3340 }
3341
3342 (new AMessage)->postReply(replyID);
3343 break;
3344 }
3345
3346 case kWhatFlush:
3347 {
3348 sp<AReplyToken> replyID;
3349 CHECK(msg->senderAwaitsResponse(&replyID));
3350
3351 if (!isExecuting()) {
3352 PostReplyWithError(replyID, INVALID_OPERATION);
3353 break;
3354 } else if (mFlags & kFlagStickyError) {
3355 PostReplyWithError(replyID, getStickyError());
3356 break;
3357 }
3358
3359 mReplyID = replyID;
3360 // TODO: skip flushing if already FLUSHED
3361 setState(FLUSHING);
3362
3363 mCodec->signalFlush();
3364 returnBuffersToCodec();
3365 break;
3366 }
3367
3368 case kWhatGetInputFormat:
3369 case kWhatGetOutputFormat:
3370 {
3371 sp<AMessage> format =
3372 (msg->what() == kWhatGetOutputFormat ? mOutputFormat : mInputFormat);
3373
3374 sp<AReplyToken> replyID;
3375 CHECK(msg->senderAwaitsResponse(&replyID));
3376
3377 if ((mState != CONFIGURED && mState != STARTING &&
3378 mState != STARTED && mState != FLUSHING &&
3379 mState != FLUSHED)
3380 || format == NULL) {
3381 PostReplyWithError(replyID, INVALID_OPERATION);
3382 break;
3383 } else if (mFlags & kFlagStickyError) {
3384 PostReplyWithError(replyID, getStickyError());
3385 break;
3386 }
3387
3388 sp<AMessage> response = new AMessage;
3389 response->setMessage("format", format);
3390 response->postReply(replyID);
3391 break;
3392 }
3393
3394 case kWhatRequestIDRFrame:
3395 {
3396 mCodec->signalRequestIDRFrame();
3397 break;
3398 }
3399
3400 case kWhatRequestActivityNotification:
3401 {
3402 CHECK(mActivityNotify == NULL);
3403 CHECK(msg->findMessage("notify", &mActivityNotify));
3404
3405 postActivityNotificationIfPossible();
3406 break;
3407 }
3408
3409 case kWhatGetName:
3410 {
3411 sp<AReplyToken> replyID;
3412 CHECK(msg->senderAwaitsResponse(&replyID));
3413
3414 if (mComponentName.empty()) {
3415 PostReplyWithError(replyID, INVALID_OPERATION);
3416 break;
3417 }
3418
3419 sp<AMessage> response = new AMessage;
3420 response->setString("name", mComponentName.c_str());
3421 response->postReply(replyID);
3422 break;
3423 }
3424
3425 case kWhatGetCodecInfo:
3426 {
3427 sp<AReplyToken> replyID;
3428 CHECK(msg->senderAwaitsResponse(&replyID));
3429
3430 sp<AMessage> response = new AMessage;
3431 response->setObject("codecInfo", mCodecInfo);
3432 response->postReply(replyID);
3433 break;
3434 }
3435
3436 case kWhatSetParameters:
3437 {
3438 sp<AReplyToken> replyID;
3439 CHECK(msg->senderAwaitsResponse(&replyID));
3440
3441 sp<AMessage> params;
3442 CHECK(msg->findMessage("params", ¶ms));
3443
3444 status_t err = onSetParameters(params);
3445
3446 PostReplyWithError(replyID, err);
3447 break;
3448 }
3449
3450 case kWhatDrmReleaseCrypto:
3451 {
3452 onReleaseCrypto(msg);
3453 break;
3454 }
3455
3456 case kWhatCheckBatteryStats:
3457 {
3458 if (mBatteryChecker != nullptr) {
3459 mBatteryChecker->onCheckBatteryTimer(msg, [this] () {
3460 mResourceManagerProxy->removeResource(
3461 MediaResource::VideoBatteryResource());
3462 });
3463 }
3464 break;
3465 }
3466
3467 default:
3468 TRESPASS();
3469 }
3470 }
3471
extractCSD(const sp<AMessage> & format)3472 void MediaCodec::extractCSD(const sp<AMessage> &format) {
3473 mCSD.clear();
3474
3475 size_t i = 0;
3476 for (;;) {
3477 sp<ABuffer> csd;
3478 if (!format->findBuffer(AStringPrintf("csd-%u", i).c_str(), &csd)) {
3479 break;
3480 }
3481 if (csd->size() == 0) {
3482 ALOGW("csd-%zu size is 0", i);
3483 }
3484
3485 mCSD.push_back(csd);
3486 ++i;
3487 }
3488
3489 ALOGV("Found %zu pieces of codec specific data.", mCSD.size());
3490 }
3491
queueCSDInputBuffer(size_t bufferIndex)3492 status_t MediaCodec::queueCSDInputBuffer(size_t bufferIndex) {
3493 CHECK(!mCSD.empty());
3494
3495 sp<ABuffer> csd = *mCSD.begin();
3496 mCSD.erase(mCSD.begin());
3497 std::shared_ptr<C2Buffer> c2Buffer;
3498 sp<hardware::HidlMemory> memory;
3499 size_t offset = 0;
3500
3501 if (mFlags & kFlagUseBlockModel) {
3502 if (hasCryptoOrDescrambler()) {
3503 constexpr size_t kInitialDealerCapacity = 1048576; // 1MB
3504 thread_local sp<MemoryDealer> sDealer = new MemoryDealer(
3505 kInitialDealerCapacity, "CSD(1MB)");
3506 sp<IMemory> mem = sDealer->allocate(csd->size());
3507 if (mem == nullptr) {
3508 size_t newDealerCapacity = sDealer->getMemoryHeap()->getSize() * 2;
3509 while (csd->size() * 2 > newDealerCapacity) {
3510 newDealerCapacity *= 2;
3511 }
3512 sDealer = new MemoryDealer(
3513 newDealerCapacity,
3514 AStringPrintf("CSD(%dMB)", newDealerCapacity / 1048576).c_str());
3515 mem = sDealer->allocate(csd->size());
3516 }
3517 memcpy(mem->unsecurePointer(), csd->data(), csd->size());
3518 ssize_t heapOffset;
3519 memory = hardware::fromHeap(mem->getMemory(&heapOffset, nullptr));
3520 offset += heapOffset;
3521 } else {
3522 std::shared_ptr<C2LinearBlock> block =
3523 FetchLinearBlock(csd->size(), {std::string{mComponentName.c_str()}});
3524 C2WriteView view{block->map().get()};
3525 if (view.error() != C2_OK) {
3526 return -EINVAL;
3527 }
3528 if (csd->size() > view.capacity()) {
3529 return -EINVAL;
3530 }
3531 memcpy(view.base(), csd->data(), csd->size());
3532 c2Buffer = C2Buffer::CreateLinearBuffer(block->share(0, csd->size(), C2Fence{}));
3533 }
3534 } else {
3535 const BufferInfo &info = mPortBuffers[kPortIndexInput][bufferIndex];
3536 const sp<MediaCodecBuffer> &codecInputData = info.mData;
3537
3538 if (csd->size() > codecInputData->capacity()) {
3539 return -EINVAL;
3540 }
3541 if (codecInputData->data() == NULL) {
3542 ALOGV("Input buffer %zu is not properly allocated", bufferIndex);
3543 return -EINVAL;
3544 }
3545
3546 memcpy(codecInputData->data(), csd->data(), csd->size());
3547 }
3548
3549 AString errorDetailMsg;
3550
3551 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
3552 msg->setSize("index", bufferIndex);
3553 msg->setSize("offset", 0);
3554 msg->setSize("size", csd->size());
3555 msg->setInt64("timeUs", 0LL);
3556 msg->setInt32("flags", BUFFER_FLAG_CODECCONFIG);
3557 msg->setPointer("errorDetailMsg", &errorDetailMsg);
3558 if (c2Buffer) {
3559 sp<WrapperObject<std::shared_ptr<C2Buffer>>> obj{
3560 new WrapperObject<std::shared_ptr<C2Buffer>>{c2Buffer}};
3561 msg->setObject("c2buffer", obj);
3562 msg->setMessage("tunings", new AMessage);
3563 } else if (memory) {
3564 sp<WrapperObject<sp<hardware::HidlMemory>>> obj{
3565 new WrapperObject<sp<hardware::HidlMemory>>{memory}};
3566 msg->setObject("memory", obj);
3567 msg->setMessage("tunings", new AMessage);
3568 }
3569
3570 return onQueueInputBuffer(msg);
3571 }
3572
setState(State newState)3573 void MediaCodec::setState(State newState) {
3574 if (newState == INITIALIZED || newState == UNINITIALIZED) {
3575 delete mSoftRenderer;
3576 mSoftRenderer = NULL;
3577
3578 if ( mCrypto != NULL ) {
3579 ALOGV("setState: ~mCrypto: %p (%d)",
3580 mCrypto.get(), (mCrypto != NULL ? mCrypto->getStrongCount() : 0));
3581 }
3582 mCrypto.clear();
3583 mDescrambler.clear();
3584 handleSetSurface(NULL);
3585
3586 mInputFormat.clear();
3587 mOutputFormat.clear();
3588 mFlags &= ~kFlagOutputFormatChanged;
3589 mFlags &= ~kFlagOutputBuffersChanged;
3590 mFlags &= ~kFlagStickyError;
3591 mFlags &= ~kFlagIsEncoder;
3592 mFlags &= ~kFlagIsAsync;
3593 mStickyError = OK;
3594
3595 mActivityNotify.clear();
3596 mCallback.clear();
3597 }
3598
3599 if (newState == UNINITIALIZED) {
3600 // return any straggling buffers, e.g. if we got here on an error
3601 returnBuffersToCodec();
3602
3603 // The component is gone, mediaserver's probably back up already
3604 // but should definitely be back up should we try to instantiate
3605 // another component.. and the cycle continues.
3606 mFlags &= ~kFlagSawMediaServerDie;
3607 }
3608
3609 mState = newState;
3610
3611 if (mBatteryChecker != nullptr) {
3612 mBatteryChecker->setExecuting(isExecuting());
3613 }
3614
3615 cancelPendingDequeueOperations();
3616 }
3617
returnBuffersToCodec(bool isReclaim)3618 void MediaCodec::returnBuffersToCodec(bool isReclaim) {
3619 returnBuffersToCodecOnPort(kPortIndexInput, isReclaim);
3620 returnBuffersToCodecOnPort(kPortIndexOutput, isReclaim);
3621 }
3622
returnBuffersToCodecOnPort(int32_t portIndex,bool isReclaim)3623 void MediaCodec::returnBuffersToCodecOnPort(int32_t portIndex, bool isReclaim) {
3624 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
3625 Mutex::Autolock al(mBufferLock);
3626
3627 if (portIndex == kPortIndexInput) {
3628 mLeftover.clear();
3629 }
3630 for (size_t i = 0; i < mPortBuffers[portIndex].size(); ++i) {
3631 BufferInfo *info = &mPortBuffers[portIndex][i];
3632
3633 if (info->mData != nullptr) {
3634 sp<MediaCodecBuffer> buffer = info->mData;
3635 if (isReclaim && info->mOwnedByClient) {
3636 ALOGD("port %d buffer %zu still owned by client when codec is reclaimed",
3637 portIndex, i);
3638 } else {
3639 info->mOwnedByClient = false;
3640 info->mData.clear();
3641 }
3642 mBufferChannel->discardBuffer(buffer);
3643 }
3644 }
3645
3646 mAvailPortBuffers[portIndex].clear();
3647 }
3648
updateBuffers(int32_t portIndex,const sp<AMessage> & msg)3649 size_t MediaCodec::updateBuffers(
3650 int32_t portIndex, const sp<AMessage> &msg) {
3651 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
3652 size_t index;
3653 CHECK(msg->findSize("index", &index));
3654 sp<RefBase> obj;
3655 CHECK(msg->findObject("buffer", &obj));
3656 sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get());
3657
3658 {
3659 Mutex::Autolock al(mBufferLock);
3660 if (mPortBuffers[portIndex].size() <= index) {
3661 mPortBuffers[portIndex].resize(align(index + 1, kNumBuffersAlign));
3662 }
3663 mPortBuffers[portIndex][index].mData = buffer;
3664 }
3665 mAvailPortBuffers[portIndex].push_back(index);
3666
3667 return index;
3668 }
3669
onQueueInputBuffer(const sp<AMessage> & msg)3670 status_t MediaCodec::onQueueInputBuffer(const sp<AMessage> &msg) {
3671 size_t index;
3672 size_t offset;
3673 size_t size;
3674 int64_t timeUs;
3675 uint32_t flags;
3676 CHECK(msg->findSize("index", &index));
3677 CHECK(msg->findInt64("timeUs", &timeUs));
3678 CHECK(msg->findInt32("flags", (int32_t *)&flags));
3679 std::shared_ptr<C2Buffer> c2Buffer;
3680 sp<hardware::HidlMemory> memory;
3681 sp<RefBase> obj;
3682 if (msg->findObject("c2buffer", &obj)) {
3683 CHECK(obj);
3684 c2Buffer = static_cast<WrapperObject<std::shared_ptr<C2Buffer>> *>(obj.get())->value;
3685 } else if (msg->findObject("memory", &obj)) {
3686 CHECK(obj);
3687 memory = static_cast<WrapperObject<sp<hardware::HidlMemory>> *>(obj.get())->value;
3688 CHECK(msg->findSize("offset", &offset));
3689 } else {
3690 CHECK(msg->findSize("offset", &offset));
3691 }
3692 const CryptoPlugin::SubSample *subSamples;
3693 size_t numSubSamples;
3694 const uint8_t *key;
3695 const uint8_t *iv;
3696 CryptoPlugin::Mode mode = CryptoPlugin::kMode_Unencrypted;
3697
3698 // We allow the simpler queueInputBuffer API to be used even in
3699 // secure mode, by fabricating a single unencrypted subSample.
3700 CryptoPlugin::SubSample ss;
3701 CryptoPlugin::Pattern pattern;
3702
3703 if (msg->findSize("size", &size)) {
3704 if (hasCryptoOrDescrambler()) {
3705 ss.mNumBytesOfClearData = size;
3706 ss.mNumBytesOfEncryptedData = 0;
3707
3708 subSamples = &ss;
3709 numSubSamples = 1;
3710 key = NULL;
3711 iv = NULL;
3712 pattern.mEncryptBlocks = 0;
3713 pattern.mSkipBlocks = 0;
3714 }
3715 } else if (!c2Buffer) {
3716 if (!hasCryptoOrDescrambler()) {
3717 ALOGE("[%s] queuing secure buffer without mCrypto or mDescrambler!",
3718 mComponentName.c_str());
3719 return -EINVAL;
3720 }
3721
3722 CHECK(msg->findPointer("subSamples", (void **)&subSamples));
3723 CHECK(msg->findSize("numSubSamples", &numSubSamples));
3724 CHECK(msg->findPointer("key", (void **)&key));
3725 CHECK(msg->findPointer("iv", (void **)&iv));
3726 CHECK(msg->findInt32("encryptBlocks", (int32_t *)&pattern.mEncryptBlocks));
3727 CHECK(msg->findInt32("skipBlocks", (int32_t *)&pattern.mSkipBlocks));
3728
3729 int32_t tmp;
3730 CHECK(msg->findInt32("mode", &tmp));
3731
3732 mode = (CryptoPlugin::Mode)tmp;
3733
3734 size = 0;
3735 for (size_t i = 0; i < numSubSamples; ++i) {
3736 size += subSamples[i].mNumBytesOfClearData;
3737 size += subSamples[i].mNumBytesOfEncryptedData;
3738 }
3739 }
3740
3741 if (index >= mPortBuffers[kPortIndexInput].size()) {
3742 return -ERANGE;
3743 }
3744
3745 BufferInfo *info = &mPortBuffers[kPortIndexInput][index];
3746 sp<MediaCodecBuffer> buffer = info->mData;
3747
3748 if (c2Buffer || memory) {
3749 sp<AMessage> tunings;
3750 CHECK(msg->findMessage("tunings", &tunings));
3751 onSetParameters(tunings);
3752
3753 status_t err = OK;
3754 if (c2Buffer) {
3755 err = mBufferChannel->attachBuffer(c2Buffer, buffer);
3756 } else if (memory) {
3757 err = mBufferChannel->attachEncryptedBuffer(
3758 memory, (mFlags & kFlagIsSecure), key, iv, mode, pattern,
3759 offset, subSamples, numSubSamples, buffer);
3760 } else {
3761 err = UNKNOWN_ERROR;
3762 }
3763
3764 if (err == OK && !buffer->asC2Buffer()
3765 && c2Buffer && c2Buffer->data().type() == C2BufferData::LINEAR) {
3766 C2ConstLinearBlock block{c2Buffer->data().linearBlocks().front()};
3767 if (block.size() > buffer->size()) {
3768 C2ConstLinearBlock leftover = block.subBlock(
3769 block.offset() + buffer->size(), block.size() - buffer->size());
3770 sp<WrapperObject<std::shared_ptr<C2Buffer>>> obj{
3771 new WrapperObject<std::shared_ptr<C2Buffer>>{
3772 C2Buffer::CreateLinearBuffer(leftover)}};
3773 msg->setObject("c2buffer", obj);
3774 mLeftover.push_front(msg);
3775 // Not sending EOS if we have leftovers
3776 flags &= ~BUFFER_FLAG_EOS;
3777 }
3778 }
3779
3780 offset = buffer->offset();
3781 size = buffer->size();
3782 if (err != OK) {
3783 return err;
3784 }
3785 }
3786
3787 if (buffer == nullptr || !info->mOwnedByClient) {
3788 return -EACCES;
3789 }
3790
3791 if (offset + size > buffer->capacity()) {
3792 return -EINVAL;
3793 }
3794
3795 buffer->setRange(offset, size);
3796 buffer->meta()->setInt64("timeUs", timeUs);
3797 if (flags & BUFFER_FLAG_EOS) {
3798 buffer->meta()->setInt32("eos", true);
3799 }
3800
3801 if (flags & BUFFER_FLAG_CODECCONFIG) {
3802 buffer->meta()->setInt32("csd", true);
3803 }
3804
3805 status_t err = OK;
3806 if (hasCryptoOrDescrambler() && !c2Buffer && !memory) {
3807 AString *errorDetailMsg;
3808 CHECK(msg->findPointer("errorDetailMsg", (void **)&errorDetailMsg));
3809
3810 err = mBufferChannel->queueSecureInputBuffer(
3811 buffer,
3812 (mFlags & kFlagIsSecure),
3813 key,
3814 iv,
3815 mode,
3816 pattern,
3817 subSamples,
3818 numSubSamples,
3819 errorDetailMsg);
3820 if (err != OK) {
3821 mediametrics_setInt32(mMetricsHandle, kCodecQueueSecureInputBufferError, err);
3822 ALOGW("Log queueSecureInputBuffer error: %d", err);
3823 }
3824 } else {
3825 err = mBufferChannel->queueInputBuffer(buffer);
3826 if (err != OK) {
3827 mediametrics_setInt32(mMetricsHandle, kCodecQueueInputBufferError, err);
3828 ALOGW("Log queueInputBuffer error: %d", err);
3829 }
3830 }
3831
3832 if (err == OK) {
3833 // synchronization boundary for getBufferAndFormat
3834 Mutex::Autolock al(mBufferLock);
3835 info->mOwnedByClient = false;
3836 info->mData.clear();
3837
3838 statsBufferSent(timeUs);
3839 }
3840
3841 return err;
3842 }
3843
handleLeftover(size_t index)3844 status_t MediaCodec::handleLeftover(size_t index) {
3845 if (mLeftover.empty()) {
3846 return OK;
3847 }
3848 sp<AMessage> msg = mLeftover.front();
3849 mLeftover.pop_front();
3850 msg->setSize("index", index);
3851 return onQueueInputBuffer(msg);
3852 }
3853
3854 //static
CreateFramesRenderedMessage(const std::list<FrameRenderTracker::Info> & done,sp<AMessage> & msg)3855 size_t MediaCodec::CreateFramesRenderedMessage(
3856 const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg) {
3857 size_t index = 0;
3858
3859 for (std::list<FrameRenderTracker::Info>::const_iterator it = done.cbegin();
3860 it != done.cend(); ++it) {
3861 if (it->getRenderTimeNs() < 0) {
3862 continue; // dropped frame from tracking
3863 }
3864 msg->setInt64(AStringPrintf("%zu-media-time-us", index).c_str(), it->getMediaTimeUs());
3865 msg->setInt64(AStringPrintf("%zu-system-nano", index).c_str(), it->getRenderTimeNs());
3866 ++index;
3867 }
3868 return index;
3869 }
3870
onReleaseOutputBuffer(const sp<AMessage> & msg)3871 status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) {
3872 size_t index;
3873 CHECK(msg->findSize("index", &index));
3874
3875 int32_t render;
3876 if (!msg->findInt32("render", &render)) {
3877 render = 0;
3878 }
3879
3880 if (!isExecuting()) {
3881 return -EINVAL;
3882 }
3883
3884 if (index >= mPortBuffers[kPortIndexOutput].size()) {
3885 return -ERANGE;
3886 }
3887
3888 BufferInfo *info = &mPortBuffers[kPortIndexOutput][index];
3889
3890 if (info->mData == nullptr || !info->mOwnedByClient) {
3891 return -EACCES;
3892 }
3893
3894 // synchronization boundary for getBufferAndFormat
3895 sp<MediaCodecBuffer> buffer;
3896 {
3897 Mutex::Autolock al(mBufferLock);
3898 info->mOwnedByClient = false;
3899 buffer = info->mData;
3900 info->mData.clear();
3901 }
3902
3903 if (render && buffer->size() != 0) {
3904 int64_t mediaTimeUs = -1;
3905 buffer->meta()->findInt64("timeUs", &mediaTimeUs);
3906
3907 int64_t renderTimeNs = 0;
3908 if (!msg->findInt64("timestampNs", &renderTimeNs)) {
3909 // use media timestamp if client did not request a specific render timestamp
3910 ALOGV("using buffer PTS of %lld", (long long)mediaTimeUs);
3911 renderTimeNs = mediaTimeUs * 1000;
3912 }
3913
3914 if (mSoftRenderer != NULL) {
3915 std::list<FrameRenderTracker::Info> doneFrames = mSoftRenderer->render(
3916 buffer->data(), buffer->size(), mediaTimeUs, renderTimeNs,
3917 mPortBuffers[kPortIndexOutput].size(), buffer->format());
3918
3919 // if we are running, notify rendered frames
3920 if (!doneFrames.empty() && mState == STARTED && mOnFrameRenderedNotification != NULL) {
3921 sp<AMessage> notify = mOnFrameRenderedNotification->dup();
3922 sp<AMessage> data = new AMessage;
3923 if (CreateFramesRenderedMessage(doneFrames, data)) {
3924 notify->setMessage("data", data);
3925 notify->post();
3926 }
3927 }
3928 }
3929 mBufferChannel->renderOutputBuffer(buffer, renderTimeNs);
3930 } else {
3931 mBufferChannel->discardBuffer(buffer);
3932 }
3933
3934 return OK;
3935 }
3936
dequeuePortBuffer(int32_t portIndex)3937 ssize_t MediaCodec::dequeuePortBuffer(int32_t portIndex) {
3938 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
3939
3940 List<size_t> *availBuffers = &mAvailPortBuffers[portIndex];
3941
3942 if (availBuffers->empty()) {
3943 return -EAGAIN;
3944 }
3945
3946 size_t index = *availBuffers->begin();
3947 availBuffers->erase(availBuffers->begin());
3948
3949 BufferInfo *info = &mPortBuffers[portIndex][index];
3950 CHECK(!info->mOwnedByClient);
3951 {
3952 Mutex::Autolock al(mBufferLock);
3953 info->mOwnedByClient = true;
3954
3955 // set image-data
3956 if (info->mData->format() != NULL) {
3957 sp<ABuffer> imageData;
3958 if (info->mData->format()->findBuffer("image-data", &imageData)) {
3959 info->mData->meta()->setBuffer("image-data", imageData);
3960 }
3961 int32_t left, top, right, bottom;
3962 if (info->mData->format()->findRect("crop", &left, &top, &right, &bottom)) {
3963 info->mData->meta()->setRect("crop-rect", left, top, right, bottom);
3964 }
3965 }
3966 }
3967
3968 return index;
3969 }
3970
connectToSurface(const sp<Surface> & surface)3971 status_t MediaCodec::connectToSurface(const sp<Surface> &surface) {
3972 status_t err = OK;
3973 if (surface != NULL) {
3974 uint64_t oldId, newId;
3975 if (mSurface != NULL
3976 && surface->getUniqueId(&newId) == NO_ERROR
3977 && mSurface->getUniqueId(&oldId) == NO_ERROR
3978 && newId == oldId) {
3979 ALOGI("[%s] connecting to the same surface. Nothing to do.", mComponentName.c_str());
3980 return ALREADY_EXISTS;
3981 }
3982
3983 err = nativeWindowConnect(surface.get(), "connectToSurface");
3984 if (err == OK) {
3985 // Require a fresh set of buffers after each connect by using a unique generation
3986 // number. Rely on the fact that max supported process id by Linux is 2^22.
3987 // PID is never 0 so we don't have to worry that we use the default generation of 0.
3988 // TODO: come up with a unique scheme if other producers also set the generation number.
3989 static uint32_t mSurfaceGeneration = 0;
3990 uint32_t generation = (getpid() << 10) | (++mSurfaceGeneration & ((1 << 10) - 1));
3991 surface->setGenerationNumber(generation);
3992 ALOGI("[%s] setting surface generation to %u", mComponentName.c_str(), generation);
3993
3994 // HACK: clear any free buffers. Remove when connect will automatically do this.
3995 // This is needed as the consumer may be holding onto stale frames that it can reattach
3996 // to this surface after disconnect/connect, and those free frames would inherit the new
3997 // generation number. Disconnecting after setting a unique generation prevents this.
3998 nativeWindowDisconnect(surface.get(), "connectToSurface(reconnect)");
3999 err = nativeWindowConnect(surface.get(), "connectToSurface(reconnect)");
4000 }
4001
4002 if (err != OK) {
4003 ALOGE("nativeWindowConnect returned an error: %s (%d)", strerror(-err), err);
4004 } else {
4005 if (!mAllowFrameDroppingBySurface) {
4006 disableLegacyBufferDropPostQ(surface);
4007 }
4008 }
4009 }
4010 // do not return ALREADY_EXISTS unless surfaces are the same
4011 return err == ALREADY_EXISTS ? BAD_VALUE : err;
4012 }
4013
disconnectFromSurface()4014 status_t MediaCodec::disconnectFromSurface() {
4015 status_t err = OK;
4016 if (mSurface != NULL) {
4017 // Resetting generation is not technically needed, but there is no need to keep it either
4018 mSurface->setGenerationNumber(0);
4019 err = nativeWindowDisconnect(mSurface.get(), "disconnectFromSurface");
4020 if (err != OK) {
4021 ALOGW("nativeWindowDisconnect returned an error: %s (%d)", strerror(-err), err);
4022 }
4023 // assume disconnected even on error
4024 mSurface.clear();
4025 }
4026 return err;
4027 }
4028
handleSetSurface(const sp<Surface> & surface)4029 status_t MediaCodec::handleSetSurface(const sp<Surface> &surface) {
4030 status_t err = OK;
4031 if (mSurface != NULL) {
4032 (void)disconnectFromSurface();
4033 }
4034 if (surface != NULL) {
4035 err = connectToSurface(surface);
4036 if (err == OK) {
4037 mSurface = surface;
4038 }
4039 }
4040 return err;
4041 }
4042
onInputBufferAvailable()4043 void MediaCodec::onInputBufferAvailable() {
4044 int32_t index;
4045 while ((index = dequeuePortBuffer(kPortIndexInput)) >= 0) {
4046 sp<AMessage> msg = mCallback->dup();
4047 msg->setInt32("callbackID", CB_INPUT_AVAILABLE);
4048 msg->setInt32("index", index);
4049 msg->post();
4050 }
4051 }
4052
onOutputBufferAvailable()4053 void MediaCodec::onOutputBufferAvailable() {
4054 int32_t index;
4055 while ((index = dequeuePortBuffer(kPortIndexOutput)) >= 0) {
4056 const sp<MediaCodecBuffer> &buffer =
4057 mPortBuffers[kPortIndexOutput][index].mData;
4058 sp<AMessage> msg = mCallback->dup();
4059 msg->setInt32("callbackID", CB_OUTPUT_AVAILABLE);
4060 msg->setInt32("index", index);
4061 msg->setSize("offset", buffer->offset());
4062 msg->setSize("size", buffer->size());
4063
4064 int64_t timeUs;
4065 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
4066
4067 msg->setInt64("timeUs", timeUs);
4068
4069 statsBufferReceived(timeUs);
4070
4071 int32_t flags;
4072 CHECK(buffer->meta()->findInt32("flags", &flags));
4073
4074 msg->setInt32("flags", flags);
4075
4076 msg->post();
4077 }
4078 }
4079
onError(status_t err,int32_t actionCode,const char * detail)4080 void MediaCodec::onError(status_t err, int32_t actionCode, const char *detail) {
4081 if (mCallback != NULL) {
4082 sp<AMessage> msg = mCallback->dup();
4083 msg->setInt32("callbackID", CB_ERROR);
4084 msg->setInt32("err", err);
4085 msg->setInt32("actionCode", actionCode);
4086
4087 if (detail != NULL) {
4088 msg->setString("detail", detail);
4089 }
4090
4091 msg->post();
4092 }
4093 }
4094
onOutputFormatChanged()4095 void MediaCodec::onOutputFormatChanged() {
4096 if (mCallback != NULL) {
4097 sp<AMessage> msg = mCallback->dup();
4098 msg->setInt32("callbackID", CB_OUTPUT_FORMAT_CHANGED);
4099 msg->setMessage("format", mOutputFormat);
4100 msg->post();
4101 }
4102 }
4103
postActivityNotificationIfPossible()4104 void MediaCodec::postActivityNotificationIfPossible() {
4105 if (mActivityNotify == NULL) {
4106 return;
4107 }
4108
4109 bool isErrorOrOutputChanged =
4110 (mFlags & (kFlagStickyError
4111 | kFlagOutputBuffersChanged
4112 | kFlagOutputFormatChanged));
4113
4114 if (isErrorOrOutputChanged
4115 || !mAvailPortBuffers[kPortIndexInput].empty()
4116 || !mAvailPortBuffers[kPortIndexOutput].empty()) {
4117 mActivityNotify->setInt32("input-buffers",
4118 mAvailPortBuffers[kPortIndexInput].size());
4119
4120 if (isErrorOrOutputChanged) {
4121 // we want consumer to dequeue as many times as it can
4122 mActivityNotify->setInt32("output-buffers", INT32_MAX);
4123 } else {
4124 mActivityNotify->setInt32("output-buffers",
4125 mAvailPortBuffers[kPortIndexOutput].size());
4126 }
4127 mActivityNotify->post();
4128 mActivityNotify.clear();
4129 }
4130 }
4131
setParameters(const sp<AMessage> & params)4132 status_t MediaCodec::setParameters(const sp<AMessage> ¶ms) {
4133 sp<AMessage> msg = new AMessage(kWhatSetParameters, this);
4134 msg->setMessage("params", params);
4135
4136 sp<AMessage> response;
4137 return PostAndAwaitResponse(msg, &response);
4138 }
4139
onSetParameters(const sp<AMessage> & params)4140 status_t MediaCodec::onSetParameters(const sp<AMessage> ¶ms) {
4141 updateLowLatency(params);
4142 mCodec->signalSetParameters(params);
4143
4144 return OK;
4145 }
4146
amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> & buffer)4147 status_t MediaCodec::amendOutputFormatWithCodecSpecificData(
4148 const sp<MediaCodecBuffer> &buffer) {
4149 AString mime;
4150 CHECK(mOutputFormat->findString("mime", &mime));
4151
4152 if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_AVC)) {
4153 // Codec specific data should be SPS and PPS in a single buffer,
4154 // each prefixed by a startcode (0x00 0x00 0x00 0x01).
4155 // We separate the two and put them into the output format
4156 // under the keys "csd-0" and "csd-1".
4157
4158 unsigned csdIndex = 0;
4159
4160 const uint8_t *data = buffer->data();
4161 size_t size = buffer->size();
4162
4163 const uint8_t *nalStart;
4164 size_t nalSize;
4165 while (getNextNALUnit(&data, &size, &nalStart, &nalSize, true) == OK) {
4166 sp<ABuffer> csd = new ABuffer(nalSize + 4);
4167 memcpy(csd->data(), "\x00\x00\x00\x01", 4);
4168 memcpy(csd->data() + 4, nalStart, nalSize);
4169
4170 mOutputFormat->setBuffer(
4171 AStringPrintf("csd-%u", csdIndex).c_str(), csd);
4172
4173 ++csdIndex;
4174 }
4175
4176 if (csdIndex != 2) {
4177 return ERROR_MALFORMED;
4178 }
4179 } else {
4180 // For everything else we just stash the codec specific data into
4181 // the output format as a single piece of csd under "csd-0".
4182 sp<ABuffer> csd = new ABuffer(buffer->size());
4183 memcpy(csd->data(), buffer->data(), buffer->size());
4184 csd->setRange(0, buffer->size());
4185 mOutputFormat->setBuffer("csd-0", csd);
4186 }
4187
4188 return OK;
4189 }
4190
stateString(State state)4191 std::string MediaCodec::stateString(State state) {
4192 const char *rval = NULL;
4193 char rawbuffer[16]; // room for "%d"
4194
4195 switch (state) {
4196 case UNINITIALIZED: rval = "UNINITIALIZED"; break;
4197 case INITIALIZING: rval = "INITIALIZING"; break;
4198 case INITIALIZED: rval = "INITIALIZED"; break;
4199 case CONFIGURING: rval = "CONFIGURING"; break;
4200 case CONFIGURED: rval = "CONFIGURED"; break;
4201 case STARTING: rval = "STARTING"; break;
4202 case STARTED: rval = "STARTED"; break;
4203 case FLUSHING: rval = "FLUSHING"; break;
4204 case FLUSHED: rval = "FLUSHED"; break;
4205 case STOPPING: rval = "STOPPING"; break;
4206 case RELEASING: rval = "RELEASING"; break;
4207 default:
4208 snprintf(rawbuffer, sizeof(rawbuffer), "%d", state);
4209 rval = rawbuffer;
4210 break;
4211 }
4212 return rval;
4213 }
4214
4215 // static
CanFetchLinearBlock(const std::vector<std::string> & names,bool * isCompatible)4216 status_t MediaCodec::CanFetchLinearBlock(
4217 const std::vector<std::string> &names, bool *isCompatible) {
4218 *isCompatible = false;
4219 if (names.size() == 0) {
4220 *isCompatible = true;
4221 return OK;
4222 }
4223 const CodecListCache &cache = GetCodecListCache();
4224 for (const std::string &name : names) {
4225 auto it = cache.mCodecInfoMap.find(name);
4226 if (it == cache.mCodecInfoMap.end()) {
4227 return NAME_NOT_FOUND;
4228 }
4229 const char *owner = it->second->getOwnerName();
4230 if (owner == nullptr || strncmp(owner, "default", 8) == 0) {
4231 *isCompatible = false;
4232 return OK;
4233 } else if (strncmp(owner, "codec2::", 8) != 0) {
4234 return NAME_NOT_FOUND;
4235 }
4236 }
4237 return CCodec::CanFetchLinearBlock(names, kDefaultReadWriteUsage, isCompatible);
4238 }
4239
4240 // static
FetchLinearBlock(size_t capacity,const std::vector<std::string> & names)4241 std::shared_ptr<C2LinearBlock> MediaCodec::FetchLinearBlock(
4242 size_t capacity, const std::vector<std::string> &names) {
4243 return CCodec::FetchLinearBlock(capacity, kDefaultReadWriteUsage, names);
4244 }
4245
4246 // static
CanFetchGraphicBlock(const std::vector<std::string> & names,bool * isCompatible)4247 status_t MediaCodec::CanFetchGraphicBlock(
4248 const std::vector<std::string> &names, bool *isCompatible) {
4249 *isCompatible = false;
4250 if (names.size() == 0) {
4251 *isCompatible = true;
4252 return OK;
4253 }
4254 const CodecListCache &cache = GetCodecListCache();
4255 for (const std::string &name : names) {
4256 auto it = cache.mCodecInfoMap.find(name);
4257 if (it == cache.mCodecInfoMap.end()) {
4258 return NAME_NOT_FOUND;
4259 }
4260 const char *owner = it->second->getOwnerName();
4261 if (owner == nullptr || strncmp(owner, "default", 8) == 0) {
4262 *isCompatible = false;
4263 return OK;
4264 } else if (strncmp(owner, "codec2.", 7) != 0) {
4265 return NAME_NOT_FOUND;
4266 }
4267 }
4268 return CCodec::CanFetchGraphicBlock(names, isCompatible);
4269 }
4270
4271 // static
FetchGraphicBlock(int32_t width,int32_t height,int32_t format,uint64_t usage,const std::vector<std::string> & names)4272 std::shared_ptr<C2GraphicBlock> MediaCodec::FetchGraphicBlock(
4273 int32_t width,
4274 int32_t height,
4275 int32_t format,
4276 uint64_t usage,
4277 const std::vector<std::string> &names) {
4278 return CCodec::FetchGraphicBlock(width, height, format, usage, names);
4279 }
4280
4281 } // namespace android
4282