1 /*
2 * Copyright 2012, The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "MediaCodec"
19 #include <utils/Log.h>
20
21 #include <inttypes.h>
22 #include <stdlib.h>
23
24 #include "include/SecureBuffer.h"
25 #include "include/SharedMemoryBuffer.h"
26 #include "include/SoftwareRenderer.h"
27 #include "StagefrightPluginLoader.h"
28
29 #include <android/hardware/cas/native/1.0/IDescrambler.h>
30
31 #include <binder/IMemory.h>
32 #include <binder/IPCThreadState.h>
33 #include <binder/IServiceManager.h>
34 #include <binder/MemoryDealer.h>
35 #include <cutils/properties.h>
36 #include <gui/BufferQueue.h>
37 #include <gui/Surface.h>
38 #include <media/ICrypto.h>
39 #include <media/IOMX.h>
40 #include <media/IResourceManagerService.h>
41 #include <media/MediaCodecBuffer.h>
42 #include <media/MediaAnalyticsItem.h>
43 #include <media/stagefright/foundation/ABuffer.h>
44 #include <media/stagefright/foundation/ADebug.h>
45 #include <media/stagefright/foundation/AMessage.h>
46 #include <media/stagefright/foundation/AString.h>
47 #include <media/stagefright/foundation/AUtils.h>
48 #include <media/stagefright/foundation/avc_utils.h>
49 #include <media/stagefright/foundation/hexdump.h>
50 #include <media/stagefright/ACodec.h>
51 #include <media/stagefright/BufferProducerWrapper.h>
52 #include <media/stagefright/MediaCodec.h>
53 #include <media/stagefright/MediaCodecList.h>
54 #include <media/stagefright/MediaDefs.h>
55 #include <media/stagefright/MediaErrors.h>
56 #include <media/stagefright/MediaFilter.h>
57 #include <media/stagefright/OMXClient.h>
58 #include <media/stagefright/PersistentSurface.h>
59 #include <media/stagefright/SurfaceUtils.h>
60 #include <mediautils/BatteryNotifier.h>
61 #include <private/android_filesystem_config.h>
62 #include <utils/Singleton.h>
63
64 namespace android {
65
66 // key for media statistics
67 static const char *kCodecKeyName = "codec";
68 // attrs for media statistics
69 // NB: these are matched with public Java API constants defined
70 // in frameworks/base/media/java/android/media/MediaCodec.java
71 // These must be kept synchronized with the constants there.
72 static const char *kCodecCodec = "android.media.mediacodec.codec"; /* e.g. OMX.google.aac.decoder */
73 static const char *kCodecMime = "android.media.mediacodec.mime"; /* e.g. audio/mime */
74 static const char *kCodecMode = "android.media.mediacodec.mode"; /* audio, video */
75 static const char *kCodecModeVideo = "video"; /* values returned for kCodecMode */
76 static const char *kCodecModeAudio = "audio";
77 static const char *kCodecEncoder = "android.media.mediacodec.encoder"; /* 0,1 */
78 static const char *kCodecSecure = "android.media.mediacodec.secure"; /* 0, 1 */
79 static const char *kCodecWidth = "android.media.mediacodec.width"; /* 0..n */
80 static const char *kCodecHeight = "android.media.mediacodec.height"; /* 0..n */
81 static const char *kCodecRotation = "android.media.mediacodec.rotation-degrees"; /* 0/90/180/270 */
82
83 // NB: These are not yet exposed as public Java API constants.
84 static const char *kCodecCrypto = "android.media.mediacodec.crypto"; /* 0,1 */
85 static const char *kCodecProfile = "android.media.mediacodec.profile"; /* 0..n */
86 static const char *kCodecLevel = "android.media.mediacodec.level"; /* 0..n */
87 static const char *kCodecMaxWidth = "android.media.mediacodec.maxwidth"; /* 0..n */
88 static const char *kCodecMaxHeight = "android.media.mediacodec.maxheight"; /* 0..n */
89 static const char *kCodecError = "android.media.mediacodec.errcode";
90 static const char *kCodecErrorState = "android.media.mediacodec.errstate";
91 static const char *kCodecLatencyMax = "android.media.mediacodec.latency.max"; /* in us */
92 static const char *kCodecLatencyMin = "android.media.mediacodec.latency.min"; /* in us */
93 static const char *kCodecLatencyAvg = "android.media.mediacodec.latency.avg"; /* in us */
94 static const char *kCodecLatencyCount = "android.media.mediacodec.latency.n";
95 static const char *kCodecLatencyHist = "android.media.mediacodec.latency.hist"; /* in us */
96 static const char *kCodecLatencyUnknown = "android.media.mediacodec.latency.unknown";
97
98 // the kCodecRecent* fields appear only in getMetrics() results
99 static const char *kCodecRecentLatencyMax = "android.media.mediacodec.recent.max"; /* in us */
100 static const char *kCodecRecentLatencyMin = "android.media.mediacodec.recent.min"; /* in us */
101 static const char *kCodecRecentLatencyAvg = "android.media.mediacodec.recent.avg"; /* in us */
102 static const char *kCodecRecentLatencyCount = "android.media.mediacodec.recent.n";
103 static const char *kCodecRecentLatencyHist = "android.media.mediacodec.recent.hist"; /* in us */
104
105 // XXX suppress until we get our representation right
106 static bool kEmitHistogram = false;
107
108
getId(const sp<IResourceManagerClient> & client)109 static int64_t getId(const sp<IResourceManagerClient> &client) {
110 return (int64_t) client.get();
111 }
112
isResourceError(status_t err)113 static bool isResourceError(status_t err) {
114 return (err == NO_MEMORY);
115 }
116
117 static const int kMaxRetry = 2;
118 static const int kMaxReclaimWaitTimeInUs = 500000; // 0.5s
119 static const int kNumBuffersAlign = 16;
120
121 ////////////////////////////////////////////////////////////////////////////////
122
123 struct ResourceManagerClient : public BnResourceManagerClient {
ResourceManagerClientandroid::ResourceManagerClient124 explicit ResourceManagerClient(MediaCodec* codec) : mMediaCodec(codec) {}
125
reclaimResourceandroid::ResourceManagerClient126 virtual bool reclaimResource() {
127 sp<MediaCodec> codec = mMediaCodec.promote();
128 if (codec == NULL) {
129 // codec is already gone.
130 return true;
131 }
132 status_t err = codec->reclaim();
133 if (err == WOULD_BLOCK) {
134 ALOGD("Wait for the client to release codec.");
135 usleep(kMaxReclaimWaitTimeInUs);
136 ALOGD("Try to reclaim again.");
137 err = codec->reclaim(true /* force */);
138 }
139 if (err != OK) {
140 ALOGW("ResourceManagerClient failed to release codec with err %d", err);
141 }
142 return (err == OK);
143 }
144
getNameandroid::ResourceManagerClient145 virtual String8 getName() {
146 String8 ret;
147 sp<MediaCodec> codec = mMediaCodec.promote();
148 if (codec == NULL) {
149 // codec is already gone.
150 return ret;
151 }
152
153 AString name;
154 if (codec->getName(&name) == OK) {
155 ret.setTo(name.c_str());
156 }
157 return ret;
158 }
159
160 protected:
~ResourceManagerClientandroid::ResourceManagerClient161 virtual ~ResourceManagerClient() {}
162
163 private:
164 wp<MediaCodec> mMediaCodec;
165
166 DISALLOW_EVIL_CONSTRUCTORS(ResourceManagerClient);
167 };
168
ResourceManagerServiceProxy(pid_t pid)169 MediaCodec::ResourceManagerServiceProxy::ResourceManagerServiceProxy(pid_t pid)
170 : mPid(pid) {
171 if (mPid == MediaCodec::kNoPid) {
172 mPid = IPCThreadState::self()->getCallingPid();
173 }
174 }
175
~ResourceManagerServiceProxy()176 MediaCodec::ResourceManagerServiceProxy::~ResourceManagerServiceProxy() {
177 if (mService != NULL) {
178 IInterface::asBinder(mService)->unlinkToDeath(this);
179 }
180 }
181
init()182 void MediaCodec::ResourceManagerServiceProxy::init() {
183 sp<IServiceManager> sm = defaultServiceManager();
184 sp<IBinder> binder = sm->getService(String16("media.resource_manager"));
185 mService = interface_cast<IResourceManagerService>(binder);
186 if (mService == NULL) {
187 ALOGE("Failed to get ResourceManagerService");
188 return;
189 }
190 IInterface::asBinder(mService)->linkToDeath(this);
191 }
192
binderDied(const wp<IBinder> &)193 void MediaCodec::ResourceManagerServiceProxy::binderDied(const wp<IBinder>& /*who*/) {
194 ALOGW("ResourceManagerService died.");
195 Mutex::Autolock _l(mLock);
196 mService.clear();
197 }
198
addResource(int64_t clientId,const sp<IResourceManagerClient> & client,const Vector<MediaResource> & resources)199 void MediaCodec::ResourceManagerServiceProxy::addResource(
200 int64_t clientId,
201 const sp<IResourceManagerClient> &client,
202 const Vector<MediaResource> &resources) {
203 Mutex::Autolock _l(mLock);
204 if (mService == NULL) {
205 return;
206 }
207 mService->addResource(mPid, clientId, client, resources);
208 }
209
removeResource(int64_t clientId)210 void MediaCodec::ResourceManagerServiceProxy::removeResource(int64_t clientId) {
211 Mutex::Autolock _l(mLock);
212 if (mService == NULL) {
213 return;
214 }
215 mService->removeResource(mPid, clientId);
216 }
217
reclaimResource(const Vector<MediaResource> & resources)218 bool MediaCodec::ResourceManagerServiceProxy::reclaimResource(
219 const Vector<MediaResource> &resources) {
220 Mutex::Autolock _l(mLock);
221 if (mService == NULL) {
222 return false;
223 }
224 return mService->reclaimResource(mPid, resources);
225 }
226
227 ////////////////////////////////////////////////////////////////////////////////
228
BufferInfo()229 MediaCodec::BufferInfo::BufferInfo() : mOwnedByClient(false) {}
230
231 ////////////////////////////////////////////////////////////////////////////////
232
233 namespace {
234
235 enum {
236 kWhatFillThisBuffer = 'fill',
237 kWhatDrainThisBuffer = 'drai',
238 kWhatEOS = 'eos ',
239 kWhatStartCompleted = 'Scom',
240 kWhatStopCompleted = 'scom',
241 kWhatReleaseCompleted = 'rcom',
242 kWhatFlushCompleted = 'fcom',
243 kWhatError = 'erro',
244 kWhatComponentAllocated = 'cAll',
245 kWhatComponentConfigured = 'cCon',
246 kWhatInputSurfaceCreated = 'isfc',
247 kWhatInputSurfaceAccepted = 'isfa',
248 kWhatSignaledInputEOS = 'seos',
249 kWhatOutputFramesRendered = 'outR',
250 kWhatOutputBuffersChanged = 'outC',
251 };
252
253 class BufferCallback : public CodecBase::BufferCallback {
254 public:
255 explicit BufferCallback(const sp<AMessage> ¬ify);
256 virtual ~BufferCallback() = default;
257
258 virtual void onInputBufferAvailable(
259 size_t index, const sp<MediaCodecBuffer> &buffer) override;
260 virtual void onOutputBufferAvailable(
261 size_t index, const sp<MediaCodecBuffer> &buffer) override;
262 private:
263 const sp<AMessage> mNotify;
264 };
265
BufferCallback(const sp<AMessage> & notify)266 BufferCallback::BufferCallback(const sp<AMessage> ¬ify)
267 : mNotify(notify) {}
268
onInputBufferAvailable(size_t index,const sp<MediaCodecBuffer> & buffer)269 void BufferCallback::onInputBufferAvailable(
270 size_t index, const sp<MediaCodecBuffer> &buffer) {
271 sp<AMessage> notify(mNotify->dup());
272 notify->setInt32("what", kWhatFillThisBuffer);
273 notify->setSize("index", index);
274 notify->setObject("buffer", buffer);
275 notify->post();
276 }
277
onOutputBufferAvailable(size_t index,const sp<MediaCodecBuffer> & buffer)278 void BufferCallback::onOutputBufferAvailable(
279 size_t index, const sp<MediaCodecBuffer> &buffer) {
280 sp<AMessage> notify(mNotify->dup());
281 notify->setInt32("what", kWhatDrainThisBuffer);
282 notify->setSize("index", index);
283 notify->setObject("buffer", buffer);
284 notify->post();
285 }
286
287 class CodecCallback : public CodecBase::CodecCallback {
288 public:
289 explicit CodecCallback(const sp<AMessage> ¬ify);
290 virtual ~CodecCallback() = default;
291
292 virtual void onEos(status_t err) override;
293 virtual void onStartCompleted() override;
294 virtual void onStopCompleted() override;
295 virtual void onReleaseCompleted() override;
296 virtual void onFlushCompleted() override;
297 virtual void onError(status_t err, enum ActionCode actionCode) override;
298 virtual void onComponentAllocated(const char *componentName) override;
299 virtual void onComponentConfigured(
300 const sp<AMessage> &inputFormat, const sp<AMessage> &outputFormat) override;
301 virtual void onInputSurfaceCreated(
302 const sp<AMessage> &inputFormat,
303 const sp<AMessage> &outputFormat,
304 const sp<BufferProducerWrapper> &inputSurface) override;
305 virtual void onInputSurfaceCreationFailed(status_t err) override;
306 virtual void onInputSurfaceAccepted(
307 const sp<AMessage> &inputFormat,
308 const sp<AMessage> &outputFormat) override;
309 virtual void onInputSurfaceDeclined(status_t err) override;
310 virtual void onSignaledInputEOS(status_t err) override;
311 virtual void onOutputFramesRendered(const std::list<FrameRenderTracker::Info> &done) override;
312 virtual void onOutputBuffersChanged() override;
313 private:
314 const sp<AMessage> mNotify;
315 };
316
CodecCallback(const sp<AMessage> & notify)317 CodecCallback::CodecCallback(const sp<AMessage> ¬ify) : mNotify(notify) {}
318
onEos(status_t err)319 void CodecCallback::onEos(status_t err) {
320 sp<AMessage> notify(mNotify->dup());
321 notify->setInt32("what", kWhatEOS);
322 notify->setInt32("err", err);
323 notify->post();
324 }
325
onStartCompleted()326 void CodecCallback::onStartCompleted() {
327 sp<AMessage> notify(mNotify->dup());
328 notify->setInt32("what", kWhatStartCompleted);
329 notify->post();
330 }
331
onStopCompleted()332 void CodecCallback::onStopCompleted() {
333 sp<AMessage> notify(mNotify->dup());
334 notify->setInt32("what", kWhatStopCompleted);
335 notify->post();
336 }
337
onReleaseCompleted()338 void CodecCallback::onReleaseCompleted() {
339 sp<AMessage> notify(mNotify->dup());
340 notify->setInt32("what", kWhatReleaseCompleted);
341 notify->post();
342 }
343
onFlushCompleted()344 void CodecCallback::onFlushCompleted() {
345 sp<AMessage> notify(mNotify->dup());
346 notify->setInt32("what", kWhatFlushCompleted);
347 notify->post();
348 }
349
onError(status_t err,enum ActionCode actionCode)350 void CodecCallback::onError(status_t err, enum ActionCode actionCode) {
351 sp<AMessage> notify(mNotify->dup());
352 notify->setInt32("what", kWhatError);
353 notify->setInt32("err", err);
354 notify->setInt32("actionCode", actionCode);
355 notify->post();
356 }
357
onComponentAllocated(const char * componentName)358 void CodecCallback::onComponentAllocated(const char *componentName) {
359 sp<AMessage> notify(mNotify->dup());
360 notify->setInt32("what", kWhatComponentAllocated);
361 notify->setString("componentName", componentName);
362 notify->post();
363 }
364
onComponentConfigured(const sp<AMessage> & inputFormat,const sp<AMessage> & outputFormat)365 void CodecCallback::onComponentConfigured(
366 const sp<AMessage> &inputFormat, const sp<AMessage> &outputFormat) {
367 sp<AMessage> notify(mNotify->dup());
368 notify->setInt32("what", kWhatComponentConfigured);
369 notify->setMessage("input-format", inputFormat);
370 notify->setMessage("output-format", outputFormat);
371 notify->post();
372 }
373
onInputSurfaceCreated(const sp<AMessage> & inputFormat,const sp<AMessage> & outputFormat,const sp<BufferProducerWrapper> & inputSurface)374 void CodecCallback::onInputSurfaceCreated(
375 const sp<AMessage> &inputFormat,
376 const sp<AMessage> &outputFormat,
377 const sp<BufferProducerWrapper> &inputSurface) {
378 sp<AMessage> notify(mNotify->dup());
379 notify->setInt32("what", kWhatInputSurfaceCreated);
380 notify->setMessage("input-format", inputFormat);
381 notify->setMessage("output-format", outputFormat);
382 notify->setObject("input-surface", inputSurface);
383 notify->post();
384 }
385
onInputSurfaceCreationFailed(status_t err)386 void CodecCallback::onInputSurfaceCreationFailed(status_t err) {
387 sp<AMessage> notify(mNotify->dup());
388 notify->setInt32("what", kWhatInputSurfaceCreated);
389 notify->setInt32("err", err);
390 notify->post();
391 }
392
onInputSurfaceAccepted(const sp<AMessage> & inputFormat,const sp<AMessage> & outputFormat)393 void CodecCallback::onInputSurfaceAccepted(
394 const sp<AMessage> &inputFormat,
395 const sp<AMessage> &outputFormat) {
396 sp<AMessage> notify(mNotify->dup());
397 notify->setInt32("what", kWhatInputSurfaceAccepted);
398 notify->setMessage("input-format", inputFormat);
399 notify->setMessage("output-format", outputFormat);
400 notify->post();
401 }
402
onInputSurfaceDeclined(status_t err)403 void CodecCallback::onInputSurfaceDeclined(status_t err) {
404 sp<AMessage> notify(mNotify->dup());
405 notify->setInt32("what", kWhatInputSurfaceAccepted);
406 notify->setInt32("err", err);
407 notify->post();
408 }
409
onSignaledInputEOS(status_t err)410 void CodecCallback::onSignaledInputEOS(status_t err) {
411 sp<AMessage> notify(mNotify->dup());
412 notify->setInt32("what", kWhatSignaledInputEOS);
413 if (err != OK) {
414 notify->setInt32("err", err);
415 }
416 notify->post();
417 }
418
onOutputFramesRendered(const std::list<FrameRenderTracker::Info> & done)419 void CodecCallback::onOutputFramesRendered(const std::list<FrameRenderTracker::Info> &done) {
420 sp<AMessage> notify(mNotify->dup());
421 notify->setInt32("what", kWhatOutputFramesRendered);
422 if (MediaCodec::CreateFramesRenderedMessage(done, notify)) {
423 notify->post();
424 }
425 }
426
onOutputBuffersChanged()427 void CodecCallback::onOutputBuffersChanged() {
428 sp<AMessage> notify(mNotify->dup());
429 notify->setInt32("what", kWhatOutputBuffersChanged);
430 notify->post();
431 }
432
433 } // namespace
434
435 ////////////////////////////////////////////////////////////////////////////////
436
437 // static
CreateByType(const sp<ALooper> & looper,const AString & mime,bool encoder,status_t * err,pid_t pid,uid_t uid)438 sp<MediaCodec> MediaCodec::CreateByType(
439 const sp<ALooper> &looper, const AString &mime, bool encoder, status_t *err, pid_t pid,
440 uid_t uid) {
441 Vector<AString> matchingCodecs;
442
443 MediaCodecList::findMatchingCodecs(
444 mime.c_str(),
445 encoder,
446 0,
447 &matchingCodecs);
448
449 if (err != NULL) {
450 *err = NAME_NOT_FOUND;
451 }
452 for (size_t i = 0; i < matchingCodecs.size(); ++i) {
453 sp<MediaCodec> codec = new MediaCodec(looper, pid, uid);
454 AString componentName = matchingCodecs[i];
455 status_t ret = codec->init(componentName);
456 if (err != NULL) {
457 *err = ret;
458 }
459 if (ret == OK) {
460 return codec;
461 }
462 ALOGD("Allocating component '%s' failed (%d), try next one.",
463 componentName.c_str(), ret);
464 }
465 return NULL;
466 }
467
468 // static
CreateByComponentName(const sp<ALooper> & looper,const AString & name,status_t * err,pid_t pid,uid_t uid)469 sp<MediaCodec> MediaCodec::CreateByComponentName(
470 const sp<ALooper> &looper, const AString &name, status_t *err, pid_t pid, uid_t uid) {
471 sp<MediaCodec> codec = new MediaCodec(looper, pid, uid);
472
473 const status_t ret = codec->init(name);
474 if (err != NULL) {
475 *err = ret;
476 }
477 return ret == OK ? codec : NULL; // NULL deallocates codec.
478 }
479
480 // static
CreatePersistentInputSurface()481 sp<PersistentSurface> MediaCodec::CreatePersistentInputSurface() {
482 // allow plugin to create surface
483 sp<PersistentSurface> pluginSurface =
484 StagefrightPluginLoader::GetCCodecInstance()->createInputSurface();
485 if (pluginSurface != nullptr) {
486 return pluginSurface;
487 }
488
489 OMXClient client;
490 if (client.connect() != OK) {
491 ALOGE("Failed to connect to OMX to create persistent input surface.");
492 return NULL;
493 }
494
495 sp<IOMX> omx = client.interface();
496
497 sp<IGraphicBufferProducer> bufferProducer;
498 sp<IGraphicBufferSource> bufferSource;
499
500 status_t err = omx->createInputSurface(&bufferProducer, &bufferSource);
501
502 if (err != OK) {
503 ALOGE("Failed to create persistent input surface.");
504 return NULL;
505 }
506
507 return new PersistentSurface(bufferProducer, bufferSource);
508 }
509
MediaCodec(const sp<ALooper> & looper,pid_t pid,uid_t uid)510 MediaCodec::MediaCodec(const sp<ALooper> &looper, pid_t pid, uid_t uid)
511 : mState(UNINITIALIZED),
512 mReleasedByResourceManager(false),
513 mLooper(looper),
514 mCodec(NULL),
515 mReplyID(0),
516 mFlags(0),
517 mStickyError(OK),
518 mSoftRenderer(NULL),
519 mAnalyticsItem(NULL),
520 mResourceManagerClient(new ResourceManagerClient(this)),
521 mResourceManagerService(new ResourceManagerServiceProxy(pid)),
522 mBatteryStatNotified(false),
523 mIsVideo(false),
524 mVideoWidth(0),
525 mVideoHeight(0),
526 mRotationDegrees(0),
527 mDequeueInputTimeoutGeneration(0),
528 mDequeueInputReplyID(0),
529 mDequeueOutputTimeoutGeneration(0),
530 mDequeueOutputReplyID(0),
531 mHaveInputSurface(false),
532 mHavePendingInputBuffers(false),
533 mCpuBoostRequested(false),
534 mLatencyUnknown(0) {
535 if (uid == kNoUid) {
536 mUid = IPCThreadState::self()->getCallingUid();
537 } else {
538 mUid = uid;
539 }
540
541 initAnalyticsItem();
542 }
543
~MediaCodec()544 MediaCodec::~MediaCodec() {
545 CHECK_EQ(mState, UNINITIALIZED);
546 mResourceManagerService->removeResource(getId(mResourceManagerClient));
547
548 flushAnalyticsItem();
549 }
550
initAnalyticsItem()551 void MediaCodec::initAnalyticsItem() {
552 if (mAnalyticsItem == NULL) {
553 mAnalyticsItem = new MediaAnalyticsItem(kCodecKeyName);
554 }
555
556 mLatencyHist.setup(kLatencyHistBuckets, kLatencyHistWidth, kLatencyHistFloor);
557
558 {
559 Mutex::Autolock al(mRecentLock);
560 for (int i = 0; i<kRecentLatencyFrames; i++) {
561 mRecentSamples[i] = kRecentSampleInvalid;
562 }
563 mRecentHead = 0;
564 }
565 }
566
updateAnalyticsItem()567 void MediaCodec::updateAnalyticsItem() {
568 ALOGV("MediaCodec::updateAnalyticsItem");
569 if (mAnalyticsItem == NULL) {
570 return;
571 }
572
573 if (mLatencyHist.getCount() != 0 ) {
574 mAnalyticsItem->setInt64(kCodecLatencyMax, mLatencyHist.getMax());
575 mAnalyticsItem->setInt64(kCodecLatencyMin, mLatencyHist.getMin());
576 mAnalyticsItem->setInt64(kCodecLatencyAvg, mLatencyHist.getAvg());
577 mAnalyticsItem->setInt64(kCodecLatencyCount, mLatencyHist.getCount());
578
579 if (kEmitHistogram) {
580 // and the histogram itself
581 std::string hist = mLatencyHist.emit();
582 mAnalyticsItem->setCString(kCodecLatencyHist, hist.c_str());
583 }
584 }
585 if (mLatencyUnknown > 0) {
586 mAnalyticsItem->setInt64(kCodecLatencyUnknown, mLatencyUnknown);
587 }
588
589 #if 0
590 // enable for short term, only while debugging
591 updateEphemeralAnalytics(mAnalyticsItem);
592 #endif
593 }
594
updateEphemeralAnalytics(MediaAnalyticsItem * item)595 void MediaCodec::updateEphemeralAnalytics(MediaAnalyticsItem *item) {
596 ALOGD("MediaCodec::updateEphemeralAnalytics()");
597
598 if (item == NULL) {
599 return;
600 }
601
602 Histogram recentHist;
603
604 // build an empty histogram
605 recentHist.setup(kLatencyHistBuckets, kLatencyHistWidth, kLatencyHistFloor);
606
607 // stuff it with the samples in the ring buffer
608 {
609 Mutex::Autolock al(mRecentLock);
610
611 for (int i=0; i<kRecentLatencyFrames; i++) {
612 if (mRecentSamples[i] != kRecentSampleInvalid) {
613 recentHist.insert(mRecentSamples[i]);
614 }
615 }
616 }
617
618
619 // spit the data (if any) into the supplied analytics record
620 if (recentHist.getCount()!= 0 ) {
621 item->setInt64(kCodecRecentLatencyMax, recentHist.getMax());
622 item->setInt64(kCodecRecentLatencyMin, recentHist.getMin());
623 item->setInt64(kCodecRecentLatencyAvg, recentHist.getAvg());
624 item->setInt64(kCodecRecentLatencyCount, recentHist.getCount());
625
626 if (kEmitHistogram) {
627 // and the histogram itself
628 std::string hist = recentHist.emit();
629 item->setCString(kCodecRecentLatencyHist, hist.c_str());
630 }
631 }
632 }
633
flushAnalyticsItem()634 void MediaCodec::flushAnalyticsItem() {
635 updateAnalyticsItem();
636 if (mAnalyticsItem != NULL) {
637 // don't log empty records
638 if (mAnalyticsItem->count() > 0) {
639 mAnalyticsItem->selfrecord();
640 }
641 delete mAnalyticsItem;
642 mAnalyticsItem = NULL;
643 }
644 }
645
setup(int nbuckets,int64_t width,int64_t floor)646 bool MediaCodec::Histogram::setup(int nbuckets, int64_t width, int64_t floor)
647 {
648 if (nbuckets <= 0 || width <= 0) {
649 return false;
650 }
651
652 // get histogram buckets
653 if (nbuckets == mBucketCount && mBuckets != NULL) {
654 // reuse our existing buffer
655 memset(mBuckets, 0, sizeof(*mBuckets) * mBucketCount);
656 } else {
657 // get a new pre-zeroed buffer
658 int64_t *newbuckets = (int64_t *)calloc(nbuckets, sizeof (*mBuckets));
659 if (newbuckets == NULL) {
660 goto bad;
661 }
662 if (mBuckets != NULL)
663 free(mBuckets);
664 mBuckets = newbuckets;
665 }
666
667 mWidth = width;
668 mFloor = floor;
669 mCeiling = floor + nbuckets * width;
670 mBucketCount = nbuckets;
671
672 mMin = INT64_MAX;
673 mMax = INT64_MIN;
674 mSum = 0;
675 mCount = 0;
676 mBelow = mAbove = 0;
677
678 return true;
679
680 bad:
681 if (mBuckets != NULL) {
682 free(mBuckets);
683 mBuckets = NULL;
684 }
685
686 return false;
687 }
688
insert(int64_t sample)689 void MediaCodec::Histogram::insert(int64_t sample)
690 {
691 // histogram is not set up
692 if (mBuckets == NULL) {
693 return;
694 }
695
696 mCount++;
697 mSum += sample;
698 if (mMin > sample) mMin = sample;
699 if (mMax < sample) mMax = sample;
700
701 if (sample < mFloor) {
702 mBelow++;
703 } else if (sample >= mCeiling) {
704 mAbove++;
705 } else {
706 int64_t slot = (sample - mFloor) / mWidth;
707 CHECK(slot < mBucketCount);
708 mBuckets[slot]++;
709 }
710 return;
711 }
712
emit()713 std::string MediaCodec::Histogram::emit()
714 {
715 std::string value;
716 char buffer[64];
717
718 // emits: width,Below{bucket0,bucket1,...., bucketN}above
719 // unconfigured will emit: 0,0{}0
720 // XXX: is this best representation?
721 snprintf(buffer, sizeof(buffer), "%" PRId64 ",%" PRId64 ",%" PRId64 "{",
722 mFloor, mWidth, mBelow);
723 value = buffer;
724 for (int i = 0; i < mBucketCount; i++) {
725 if (i != 0) {
726 value = value + ",";
727 }
728 snprintf(buffer, sizeof(buffer), "%" PRId64, mBuckets[i]);
729 value = value + buffer;
730 }
731 snprintf(buffer, sizeof(buffer), "}%" PRId64 , mAbove);
732 value = value + buffer;
733 return value;
734 }
735
736 // when we send a buffer to the codec;
statsBufferSent(int64_t presentationUs)737 void MediaCodec::statsBufferSent(int64_t presentationUs) {
738
739 // only enqueue if we have a legitimate time
740 if (presentationUs <= 0) {
741 ALOGV("presentation time: %" PRId64, presentationUs);
742 return;
743 }
744
745 const int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC);
746 BufferFlightTiming_t startdata = { presentationUs, nowNs };
747
748 {
749 // mutex access to mBuffersInFlight and other stats
750 Mutex::Autolock al(mLatencyLock);
751
752
753 // XXX: we *could* make sure that the time is later than the end of queue
754 // as part of a consistency check...
755 mBuffersInFlight.push_back(startdata);
756 }
757 }
758
759 // when we get a buffer back from the codec
statsBufferReceived(int64_t presentationUs)760 void MediaCodec::statsBufferReceived(int64_t presentationUs) {
761
762 CHECK_NE(mState, UNINITIALIZED);
763
764 // mutex access to mBuffersInFlight and other stats
765 Mutex::Autolock al(mLatencyLock);
766
767 // how long this buffer took for the round trip through the codec
768 // NB: pipelining can/will make these times larger. e.g., if each packet
769 // is always 2 msec and we have 3 in flight at any given time, we're going to
770 // see "6 msec" as an answer.
771
772 // ignore stuff with no presentation time
773 if (presentationUs <= 0) {
774 ALOGV("-- returned buffer timestamp %" PRId64 " <= 0, ignore it", presentationUs);
775 mLatencyUnknown++;
776 return;
777 }
778
779 BufferFlightTiming_t startdata;
780 bool valid = false;
781 while (mBuffersInFlight.size() > 0) {
782 startdata = *mBuffersInFlight.begin();
783 ALOGV("-- Looking at startdata. presentation %" PRId64 ", start %" PRId64,
784 startdata.presentationUs, startdata.startedNs);
785 if (startdata.presentationUs == presentationUs) {
786 // a match
787 ALOGV("-- match entry for %" PRId64 ", hits our frame of %" PRId64,
788 startdata.presentationUs, presentationUs);
789 mBuffersInFlight.pop_front();
790 valid = true;
791 break;
792 } else if (startdata.presentationUs < presentationUs) {
793 // we must have missed the match for this, drop it and keep looking
794 ALOGV("-- drop entry for %" PRId64 ", before our frame of %" PRId64,
795 startdata.presentationUs, presentationUs);
796 mBuffersInFlight.pop_front();
797 continue;
798 } else {
799 // head is after, so we don't have a frame for ourselves
800 ALOGV("-- found entry for %" PRId64 ", AFTER our frame of %" PRId64
801 " we have nothing to pair with",
802 startdata.presentationUs, presentationUs);
803 mLatencyUnknown++;
804 return;
805 }
806 }
807 if (!valid) {
808 ALOGV("-- empty queue, so ignore that.");
809 mLatencyUnknown++;
810 return;
811 }
812
813 // nowNs start our calculations
814 const int64_t nowNs = systemTime(SYSTEM_TIME_MONOTONIC);
815 int64_t latencyUs = (nowNs - startdata.startedNs + 500) / 1000;
816
817 mLatencyHist.insert(latencyUs);
818
819 // push into the recent samples
820 {
821 Mutex::Autolock al(mRecentLock);
822
823 if (mRecentHead >= kRecentLatencyFrames) {
824 mRecentHead = 0;
825 }
826 mRecentSamples[mRecentHead++] = latencyUs;
827 }
828 }
829
830 // static
PostAndAwaitResponse(const sp<AMessage> & msg,sp<AMessage> * response)831 status_t MediaCodec::PostAndAwaitResponse(
832 const sp<AMessage> &msg, sp<AMessage> *response) {
833 status_t err = msg->postAndAwaitResponse(response);
834
835 if (err != OK) {
836 return err;
837 }
838
839 if (!(*response)->findInt32("err", &err)) {
840 err = OK;
841 }
842
843 return err;
844 }
845
PostReplyWithError(const sp<AReplyToken> & replyID,int32_t err)846 void MediaCodec::PostReplyWithError(const sp<AReplyToken> &replyID, int32_t err) {
847 int32_t finalErr = err;
848 if (mReleasedByResourceManager) {
849 // override the err code if MediaCodec has been released by ResourceManager.
850 finalErr = DEAD_OBJECT;
851 }
852
853 sp<AMessage> response = new AMessage;
854 response->setInt32("err", finalErr);
855 response->postReply(replyID);
856 }
857
CreateCCodec()858 static CodecBase *CreateCCodec() {
859 return StagefrightPluginLoader::GetCCodecInstance()->createCodec();
860 }
861
862 //static
GetCodecBase(const AString & name)863 sp<CodecBase> MediaCodec::GetCodecBase(const AString &name) {
864 if (name.startsWithIgnoreCase("c2.")) {
865 return CreateCCodec();
866 } else if (name.startsWithIgnoreCase("omx.")) {
867 // at this time only ACodec specifies a mime type.
868 return new ACodec;
869 } else if (name.startsWithIgnoreCase("android.filter.")) {
870 return new MediaFilter;
871 } else {
872 return NULL;
873 }
874 }
875
init(const AString & name)876 status_t MediaCodec::init(const AString &name) {
877 mResourceManagerService->init();
878
879 // save init parameters for reset
880 mInitName = name;
881
882 // Current video decoders do not return from OMX_FillThisBuffer
883 // quickly, violating the OpenMAX specs, until that is remedied
884 // we need to invest in an extra looper to free the main event
885 // queue.
886
887 mCodec = GetCodecBase(name);
888 if (mCodec == NULL) {
889 return NAME_NOT_FOUND;
890 }
891
892 mCodecInfo.clear();
893
894 bool secureCodec = false;
895 AString tmp = name;
896 if (tmp.endsWith(".secure")) {
897 secureCodec = true;
898 tmp.erase(tmp.size() - 7, 7);
899 }
900 const sp<IMediaCodecList> mcl = MediaCodecList::getInstance();
901 if (mcl == NULL) {
902 mCodec = NULL; // remove the codec.
903 return NO_INIT; // if called from Java should raise IOException
904 }
905 for (const AString &codecName : { name, tmp }) {
906 ssize_t codecIdx = mcl->findCodecByName(codecName.c_str());
907 if (codecIdx < 0) {
908 continue;
909 }
910 mCodecInfo = mcl->getCodecInfo(codecIdx);
911 Vector<AString> mimes;
912 mCodecInfo->getSupportedMimes(&mimes);
913 for (size_t i = 0; i < mimes.size(); i++) {
914 if (mimes[i].startsWith("video/")) {
915 mIsVideo = true;
916 break;
917 }
918 }
919 break;
920 }
921 if (mCodecInfo == nullptr) {
922 return NAME_NOT_FOUND;
923 }
924
925 if (mIsVideo) {
926 // video codec needs dedicated looper
927 if (mCodecLooper == NULL) {
928 mCodecLooper = new ALooper;
929 mCodecLooper->setName("CodecLooper");
930 mCodecLooper->start(false, false, ANDROID_PRIORITY_AUDIO);
931 }
932
933 mCodecLooper->registerHandler(mCodec);
934 } else {
935 mLooper->registerHandler(mCodec);
936 }
937
938 mLooper->registerHandler(this);
939
940 mCodec->setCallback(
941 std::unique_ptr<CodecBase::CodecCallback>(
942 new CodecCallback(new AMessage(kWhatCodecNotify, this))));
943 mBufferChannel = mCodec->getBufferChannel();
944 mBufferChannel->setCallback(
945 std::unique_ptr<CodecBase::BufferCallback>(
946 new BufferCallback(new AMessage(kWhatCodecNotify, this))));
947
948 sp<AMessage> msg = new AMessage(kWhatInit, this);
949 msg->setObject("codecInfo", mCodecInfo);
950 // name may be different from mCodecInfo->getCodecName() if we stripped
951 // ".secure"
952 msg->setString("name", name);
953
954 if (mAnalyticsItem != NULL) {
955 mAnalyticsItem->setCString(kCodecCodec, name.c_str());
956 mAnalyticsItem->setCString(kCodecMode, mIsVideo ? kCodecModeVideo : kCodecModeAudio);
957 }
958
959 status_t err;
960 Vector<MediaResource> resources;
961 MediaResource::Type type =
962 secureCodec ? MediaResource::kSecureCodec : MediaResource::kNonSecureCodec;
963 MediaResource::SubType subtype =
964 mIsVideo ? MediaResource::kVideoCodec : MediaResource::kAudioCodec;
965 resources.push_back(MediaResource(type, subtype, 1));
966 for (int i = 0; i <= kMaxRetry; ++i) {
967 if (i > 0) {
968 // Don't try to reclaim resource for the first time.
969 if (!mResourceManagerService->reclaimResource(resources)) {
970 break;
971 }
972 }
973
974 sp<AMessage> response;
975 err = PostAndAwaitResponse(msg, &response);
976 if (!isResourceError(err)) {
977 break;
978 }
979 }
980 return err;
981 }
982
setCallback(const sp<AMessage> & callback)983 status_t MediaCodec::setCallback(const sp<AMessage> &callback) {
984 sp<AMessage> msg = new AMessage(kWhatSetCallback, this);
985 msg->setMessage("callback", callback);
986
987 sp<AMessage> response;
988 return PostAndAwaitResponse(msg, &response);
989 }
990
setOnFrameRenderedNotification(const sp<AMessage> & notify)991 status_t MediaCodec::setOnFrameRenderedNotification(const sp<AMessage> ¬ify) {
992 sp<AMessage> msg = new AMessage(kWhatSetNotification, this);
993 msg->setMessage("on-frame-rendered", notify);
994 return msg->post();
995 }
996
configure(const sp<AMessage> & format,const sp<Surface> & nativeWindow,const sp<ICrypto> & crypto,uint32_t flags)997 status_t MediaCodec::configure(
998 const sp<AMessage> &format,
999 const sp<Surface> &nativeWindow,
1000 const sp<ICrypto> &crypto,
1001 uint32_t flags) {
1002 return configure(format, nativeWindow, crypto, NULL, flags);
1003 }
1004
configure(const sp<AMessage> & format,const sp<Surface> & surface,const sp<ICrypto> & crypto,const sp<IDescrambler> & descrambler,uint32_t flags)1005 status_t MediaCodec::configure(
1006 const sp<AMessage> &format,
1007 const sp<Surface> &surface,
1008 const sp<ICrypto> &crypto,
1009 const sp<IDescrambler> &descrambler,
1010 uint32_t flags) {
1011 sp<AMessage> msg = new AMessage(kWhatConfigure, this);
1012
1013 if (mAnalyticsItem != NULL) {
1014 int32_t profile = 0;
1015 if (format->findInt32("profile", &profile)) {
1016 mAnalyticsItem->setInt32(kCodecProfile, profile);
1017 }
1018 int32_t level = 0;
1019 if (format->findInt32("level", &level)) {
1020 mAnalyticsItem->setInt32(kCodecLevel, level);
1021 }
1022 mAnalyticsItem->setInt32(kCodecEncoder, (flags & CONFIGURE_FLAG_ENCODE) ? 1 : 0);
1023 }
1024
1025 if (mIsVideo) {
1026 format->findInt32("width", &mVideoWidth);
1027 format->findInt32("height", &mVideoHeight);
1028 if (!format->findInt32("rotation-degrees", &mRotationDegrees)) {
1029 mRotationDegrees = 0;
1030 }
1031
1032 if (mAnalyticsItem != NULL) {
1033 mAnalyticsItem->setInt32(kCodecWidth, mVideoWidth);
1034 mAnalyticsItem->setInt32(kCodecHeight, mVideoHeight);
1035 mAnalyticsItem->setInt32(kCodecRotation, mRotationDegrees);
1036 int32_t maxWidth = 0;
1037 if (format->findInt32("max-width", &maxWidth)) {
1038 mAnalyticsItem->setInt32(kCodecMaxWidth, maxWidth);
1039 }
1040 int32_t maxHeight = 0;
1041 if (format->findInt32("max-height", &maxHeight)) {
1042 mAnalyticsItem->setInt32(kCodecMaxHeight, maxHeight);
1043 }
1044 }
1045
1046 // Prevent possible integer overflow in downstream code.
1047 if ((uint64_t)mVideoWidth * mVideoHeight > (uint64_t)INT32_MAX / 4) {
1048 ALOGE("buffer size is too big, width=%d, height=%d", mVideoWidth, mVideoHeight);
1049 return BAD_VALUE;
1050 }
1051 }
1052
1053 msg->setMessage("format", format);
1054 msg->setInt32("flags", flags);
1055 msg->setObject("surface", surface);
1056
1057 if (crypto != NULL || descrambler != NULL) {
1058 if (crypto != NULL) {
1059 msg->setPointer("crypto", crypto.get());
1060 } else {
1061 msg->setPointer("descrambler", descrambler.get());
1062 }
1063 if (mAnalyticsItem != NULL) {
1064 mAnalyticsItem->setInt32(kCodecCrypto, 1);
1065 }
1066 } else if (mFlags & kFlagIsSecure) {
1067 ALOGW("Crypto or descrambler should be given for secure codec");
1068 }
1069
1070 // save msg for reset
1071 mConfigureMsg = msg;
1072
1073 status_t err;
1074 Vector<MediaResource> resources;
1075 MediaResource::Type type = (mFlags & kFlagIsSecure) ?
1076 MediaResource::kSecureCodec : MediaResource::kNonSecureCodec;
1077 MediaResource::SubType subtype =
1078 mIsVideo ? MediaResource::kVideoCodec : MediaResource::kAudioCodec;
1079 resources.push_back(MediaResource(type, subtype, 1));
1080 // Don't know the buffer size at this point, but it's fine to use 1 because
1081 // the reclaimResource call doesn't consider the requester's buffer size for now.
1082 resources.push_back(MediaResource(MediaResource::kGraphicMemory, 1));
1083 for (int i = 0; i <= kMaxRetry; ++i) {
1084 if (i > 0) {
1085 // Don't try to reclaim resource for the first time.
1086 if (!mResourceManagerService->reclaimResource(resources)) {
1087 break;
1088 }
1089 }
1090
1091 sp<AMessage> response;
1092 err = PostAndAwaitResponse(msg, &response);
1093 if (err != OK && err != INVALID_OPERATION) {
1094 // MediaCodec now set state to UNINITIALIZED upon any fatal error.
1095 // To maintain backward-compatibility, do a reset() to put codec
1096 // back into INITIALIZED state.
1097 // But don't reset if the err is INVALID_OPERATION, which means
1098 // the configure failure is due to wrong state.
1099
1100 ALOGE("configure failed with err 0x%08x, resetting...", err);
1101 reset();
1102 }
1103 if (!isResourceError(err)) {
1104 break;
1105 }
1106 }
1107 return err;
1108 }
1109
releaseCrypto()1110 status_t MediaCodec::releaseCrypto()
1111 {
1112 ALOGV("releaseCrypto");
1113
1114 sp<AMessage> msg = new AMessage(kWhatDrmReleaseCrypto, this);
1115
1116 sp<AMessage> response;
1117 status_t status = msg->postAndAwaitResponse(&response);
1118
1119 if (status == OK && response != NULL) {
1120 CHECK(response->findInt32("status", &status));
1121 ALOGV("releaseCrypto ret: %d ", status);
1122 }
1123 else {
1124 ALOGE("releaseCrypto err: %d", status);
1125 }
1126
1127 return status;
1128 }
1129
onReleaseCrypto(const sp<AMessage> & msg)1130 void MediaCodec::onReleaseCrypto(const sp<AMessage>& msg)
1131 {
1132 status_t status = INVALID_OPERATION;
1133 if (mCrypto != NULL) {
1134 ALOGV("onReleaseCrypto: mCrypto: %p (%d)", mCrypto.get(), mCrypto->getStrongCount());
1135 mBufferChannel->setCrypto(NULL);
1136 // TODO change to ALOGV
1137 ALOGD("onReleaseCrypto: [before clear] mCrypto: %p (%d)",
1138 mCrypto.get(), mCrypto->getStrongCount());
1139 mCrypto.clear();
1140
1141 status = OK;
1142 }
1143 else {
1144 ALOGW("onReleaseCrypto: No mCrypto. err: %d", status);
1145 }
1146
1147 sp<AMessage> response = new AMessage;
1148 response->setInt32("status", status);
1149
1150 sp<AReplyToken> replyID;
1151 CHECK(msg->senderAwaitsResponse(&replyID));
1152 response->postReply(replyID);
1153 }
1154
setInputSurface(const sp<PersistentSurface> & surface)1155 status_t MediaCodec::setInputSurface(
1156 const sp<PersistentSurface> &surface) {
1157 sp<AMessage> msg = new AMessage(kWhatSetInputSurface, this);
1158 msg->setObject("input-surface", surface.get());
1159
1160 sp<AMessage> response;
1161 return PostAndAwaitResponse(msg, &response);
1162 }
1163
setSurface(const sp<Surface> & surface)1164 status_t MediaCodec::setSurface(const sp<Surface> &surface) {
1165 sp<AMessage> msg = new AMessage(kWhatSetSurface, this);
1166 msg->setObject("surface", surface);
1167
1168 sp<AMessage> response;
1169 return PostAndAwaitResponse(msg, &response);
1170 }
1171
createInputSurface(sp<IGraphicBufferProducer> * bufferProducer)1172 status_t MediaCodec::createInputSurface(
1173 sp<IGraphicBufferProducer>* bufferProducer) {
1174 sp<AMessage> msg = new AMessage(kWhatCreateInputSurface, this);
1175
1176 sp<AMessage> response;
1177 status_t err = PostAndAwaitResponse(msg, &response);
1178 if (err == NO_ERROR) {
1179 // unwrap the sp<IGraphicBufferProducer>
1180 sp<RefBase> obj;
1181 bool found = response->findObject("input-surface", &obj);
1182 CHECK(found);
1183 sp<BufferProducerWrapper> wrapper(
1184 static_cast<BufferProducerWrapper*>(obj.get()));
1185 *bufferProducer = wrapper->getBufferProducer();
1186 } else {
1187 ALOGW("createInputSurface failed, err=%d", err);
1188 }
1189 return err;
1190 }
1191
getGraphicBufferSize()1192 uint64_t MediaCodec::getGraphicBufferSize() {
1193 if (!mIsVideo) {
1194 return 0;
1195 }
1196
1197 uint64_t size = 0;
1198 size_t portNum = sizeof(mPortBuffers) / sizeof((mPortBuffers)[0]);
1199 for (size_t i = 0; i < portNum; ++i) {
1200 // TODO: this is just an estimation, we should get the real buffer size from ACodec.
1201 size += mPortBuffers[i].size() * mVideoWidth * mVideoHeight * 3 / 2;
1202 }
1203 return size;
1204 }
1205
addResource(MediaResource::Type type,MediaResource::SubType subtype,uint64_t value)1206 void MediaCodec::addResource(
1207 MediaResource::Type type, MediaResource::SubType subtype, uint64_t value) {
1208 Vector<MediaResource> resources;
1209 resources.push_back(MediaResource(type, subtype, value));
1210 mResourceManagerService->addResource(
1211 getId(mResourceManagerClient), mResourceManagerClient, resources);
1212 }
1213
start()1214 status_t MediaCodec::start() {
1215 sp<AMessage> msg = new AMessage(kWhatStart, this);
1216
1217 status_t err;
1218 Vector<MediaResource> resources;
1219 MediaResource::Type type = (mFlags & kFlagIsSecure) ?
1220 MediaResource::kSecureCodec : MediaResource::kNonSecureCodec;
1221 MediaResource::SubType subtype =
1222 mIsVideo ? MediaResource::kVideoCodec : MediaResource::kAudioCodec;
1223 resources.push_back(MediaResource(type, subtype, 1));
1224 // Don't know the buffer size at this point, but it's fine to use 1 because
1225 // the reclaimResource call doesn't consider the requester's buffer size for now.
1226 resources.push_back(MediaResource(MediaResource::kGraphicMemory, 1));
1227 for (int i = 0; i <= kMaxRetry; ++i) {
1228 if (i > 0) {
1229 // Don't try to reclaim resource for the first time.
1230 if (!mResourceManagerService->reclaimResource(resources)) {
1231 break;
1232 }
1233 // Recover codec from previous error before retry start.
1234 err = reset();
1235 if (err != OK) {
1236 ALOGE("retrying start: failed to reset codec");
1237 break;
1238 }
1239 sp<AMessage> response;
1240 err = PostAndAwaitResponse(mConfigureMsg, &response);
1241 if (err != OK) {
1242 ALOGE("retrying start: failed to configure codec");
1243 break;
1244 }
1245 }
1246
1247 sp<AMessage> response;
1248 err = PostAndAwaitResponse(msg, &response);
1249 if (!isResourceError(err)) {
1250 break;
1251 }
1252 }
1253 return err;
1254 }
1255
stop()1256 status_t MediaCodec::stop() {
1257 sp<AMessage> msg = new AMessage(kWhatStop, this);
1258
1259 sp<AMessage> response;
1260 return PostAndAwaitResponse(msg, &response);
1261 }
1262
hasPendingBuffer(int portIndex)1263 bool MediaCodec::hasPendingBuffer(int portIndex) {
1264 return std::any_of(
1265 mPortBuffers[portIndex].begin(), mPortBuffers[portIndex].end(),
1266 [](const BufferInfo &info) { return info.mOwnedByClient; });
1267 }
1268
hasPendingBuffer()1269 bool MediaCodec::hasPendingBuffer() {
1270 return hasPendingBuffer(kPortIndexInput) || hasPendingBuffer(kPortIndexOutput);
1271 }
1272
reclaim(bool force)1273 status_t MediaCodec::reclaim(bool force) {
1274 ALOGD("MediaCodec::reclaim(%p) %s", this, mInitName.c_str());
1275 sp<AMessage> msg = new AMessage(kWhatRelease, this);
1276 msg->setInt32("reclaimed", 1);
1277 msg->setInt32("force", force ? 1 : 0);
1278
1279 sp<AMessage> response;
1280 status_t ret = PostAndAwaitResponse(msg, &response);
1281 if (ret == -ENOENT) {
1282 ALOGD("MediaCodec looper is gone, skip reclaim");
1283 ret = OK;
1284 }
1285 return ret;
1286 }
1287
release()1288 status_t MediaCodec::release() {
1289 sp<AMessage> msg = new AMessage(kWhatRelease, this);
1290
1291 sp<AMessage> response;
1292 return PostAndAwaitResponse(msg, &response);
1293 }
1294
reset()1295 status_t MediaCodec::reset() {
1296 /* When external-facing MediaCodec object is created,
1297 it is already initialized. Thus, reset is essentially
1298 release() followed by init(), plus clearing the state */
1299
1300 status_t err = release();
1301
1302 // unregister handlers
1303 if (mCodec != NULL) {
1304 if (mCodecLooper != NULL) {
1305 mCodecLooper->unregisterHandler(mCodec->id());
1306 } else {
1307 mLooper->unregisterHandler(mCodec->id());
1308 }
1309 mCodec = NULL;
1310 }
1311 mLooper->unregisterHandler(id());
1312
1313 mFlags = 0; // clear all flags
1314 mStickyError = OK;
1315
1316 // reset state not reset by setState(UNINITIALIZED)
1317 mReplyID = 0;
1318 mDequeueInputReplyID = 0;
1319 mDequeueOutputReplyID = 0;
1320 mDequeueInputTimeoutGeneration = 0;
1321 mDequeueOutputTimeoutGeneration = 0;
1322 mHaveInputSurface = false;
1323
1324 if (err == OK) {
1325 err = init(mInitName);
1326 }
1327 return err;
1328 }
1329
queueInputBuffer(size_t index,size_t offset,size_t size,int64_t presentationTimeUs,uint32_t flags,AString * errorDetailMsg)1330 status_t MediaCodec::queueInputBuffer(
1331 size_t index,
1332 size_t offset,
1333 size_t size,
1334 int64_t presentationTimeUs,
1335 uint32_t flags,
1336 AString *errorDetailMsg) {
1337 if (errorDetailMsg != NULL) {
1338 errorDetailMsg->clear();
1339 }
1340
1341 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
1342 msg->setSize("index", index);
1343 msg->setSize("offset", offset);
1344 msg->setSize("size", size);
1345 msg->setInt64("timeUs", presentationTimeUs);
1346 msg->setInt32("flags", flags);
1347 msg->setPointer("errorDetailMsg", errorDetailMsg);
1348
1349 sp<AMessage> response;
1350 return PostAndAwaitResponse(msg, &response);
1351 }
1352
queueSecureInputBuffer(size_t index,size_t offset,const CryptoPlugin::SubSample * subSamples,size_t numSubSamples,const uint8_t key[16],const uint8_t iv[16],CryptoPlugin::Mode mode,const CryptoPlugin::Pattern & pattern,int64_t presentationTimeUs,uint32_t flags,AString * errorDetailMsg)1353 status_t MediaCodec::queueSecureInputBuffer(
1354 size_t index,
1355 size_t offset,
1356 const CryptoPlugin::SubSample *subSamples,
1357 size_t numSubSamples,
1358 const uint8_t key[16],
1359 const uint8_t iv[16],
1360 CryptoPlugin::Mode mode,
1361 const CryptoPlugin::Pattern &pattern,
1362 int64_t presentationTimeUs,
1363 uint32_t flags,
1364 AString *errorDetailMsg) {
1365 if (errorDetailMsg != NULL) {
1366 errorDetailMsg->clear();
1367 }
1368
1369 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
1370 msg->setSize("index", index);
1371 msg->setSize("offset", offset);
1372 msg->setPointer("subSamples", (void *)subSamples);
1373 msg->setSize("numSubSamples", numSubSamples);
1374 msg->setPointer("key", (void *)key);
1375 msg->setPointer("iv", (void *)iv);
1376 msg->setInt32("mode", mode);
1377 msg->setInt32("encryptBlocks", pattern.mEncryptBlocks);
1378 msg->setInt32("skipBlocks", pattern.mSkipBlocks);
1379 msg->setInt64("timeUs", presentationTimeUs);
1380 msg->setInt32("flags", flags);
1381 msg->setPointer("errorDetailMsg", errorDetailMsg);
1382
1383 sp<AMessage> response;
1384 status_t err = PostAndAwaitResponse(msg, &response);
1385
1386 return err;
1387 }
1388
dequeueInputBuffer(size_t * index,int64_t timeoutUs)1389 status_t MediaCodec::dequeueInputBuffer(size_t *index, int64_t timeoutUs) {
1390 sp<AMessage> msg = new AMessage(kWhatDequeueInputBuffer, this);
1391 msg->setInt64("timeoutUs", timeoutUs);
1392
1393 sp<AMessage> response;
1394 status_t err;
1395 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1396 return err;
1397 }
1398
1399 CHECK(response->findSize("index", index));
1400
1401 return OK;
1402 }
1403
dequeueOutputBuffer(size_t * index,size_t * offset,size_t * size,int64_t * presentationTimeUs,uint32_t * flags,int64_t timeoutUs)1404 status_t MediaCodec::dequeueOutputBuffer(
1405 size_t *index,
1406 size_t *offset,
1407 size_t *size,
1408 int64_t *presentationTimeUs,
1409 uint32_t *flags,
1410 int64_t timeoutUs) {
1411 sp<AMessage> msg = new AMessage(kWhatDequeueOutputBuffer, this);
1412 msg->setInt64("timeoutUs", timeoutUs);
1413
1414 sp<AMessage> response;
1415 status_t err;
1416 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1417 return err;
1418 }
1419
1420 CHECK(response->findSize("index", index));
1421 CHECK(response->findSize("offset", offset));
1422 CHECK(response->findSize("size", size));
1423 CHECK(response->findInt64("timeUs", presentationTimeUs));
1424 CHECK(response->findInt32("flags", (int32_t *)flags));
1425
1426 return OK;
1427 }
1428
renderOutputBufferAndRelease(size_t index)1429 status_t MediaCodec::renderOutputBufferAndRelease(size_t index) {
1430 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this);
1431 msg->setSize("index", index);
1432 msg->setInt32("render", true);
1433
1434 sp<AMessage> response;
1435 return PostAndAwaitResponse(msg, &response);
1436 }
1437
renderOutputBufferAndRelease(size_t index,int64_t timestampNs)1438 status_t MediaCodec::renderOutputBufferAndRelease(size_t index, int64_t timestampNs) {
1439 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this);
1440 msg->setSize("index", index);
1441 msg->setInt32("render", true);
1442 msg->setInt64("timestampNs", timestampNs);
1443
1444 sp<AMessage> response;
1445 return PostAndAwaitResponse(msg, &response);
1446 }
1447
releaseOutputBuffer(size_t index)1448 status_t MediaCodec::releaseOutputBuffer(size_t index) {
1449 sp<AMessage> msg = new AMessage(kWhatReleaseOutputBuffer, this);
1450 msg->setSize("index", index);
1451
1452 sp<AMessage> response;
1453 return PostAndAwaitResponse(msg, &response);
1454 }
1455
signalEndOfInputStream()1456 status_t MediaCodec::signalEndOfInputStream() {
1457 sp<AMessage> msg = new AMessage(kWhatSignalEndOfInputStream, this);
1458
1459 sp<AMessage> response;
1460 return PostAndAwaitResponse(msg, &response);
1461 }
1462
getOutputFormat(sp<AMessage> * format) const1463 status_t MediaCodec::getOutputFormat(sp<AMessage> *format) const {
1464 sp<AMessage> msg = new AMessage(kWhatGetOutputFormat, this);
1465
1466 sp<AMessage> response;
1467 status_t err;
1468 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1469 return err;
1470 }
1471
1472 CHECK(response->findMessage("format", format));
1473
1474 return OK;
1475 }
1476
getInputFormat(sp<AMessage> * format) const1477 status_t MediaCodec::getInputFormat(sp<AMessage> *format) const {
1478 sp<AMessage> msg = new AMessage(kWhatGetInputFormat, this);
1479
1480 sp<AMessage> response;
1481 status_t err;
1482 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1483 return err;
1484 }
1485
1486 CHECK(response->findMessage("format", format));
1487
1488 return OK;
1489 }
1490
getName(AString * name) const1491 status_t MediaCodec::getName(AString *name) const {
1492 sp<AMessage> msg = new AMessage(kWhatGetName, this);
1493
1494 sp<AMessage> response;
1495 status_t err;
1496 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1497 return err;
1498 }
1499
1500 CHECK(response->findString("name", name));
1501
1502 return OK;
1503 }
1504
getCodecInfo(sp<MediaCodecInfo> * codecInfo) const1505 status_t MediaCodec::getCodecInfo(sp<MediaCodecInfo> *codecInfo) const {
1506 sp<AMessage> msg = new AMessage(kWhatGetCodecInfo, this);
1507
1508 sp<AMessage> response;
1509 status_t err;
1510 if ((err = PostAndAwaitResponse(msg, &response)) != OK) {
1511 return err;
1512 }
1513
1514 sp<RefBase> obj;
1515 CHECK(response->findObject("codecInfo", &obj));
1516 *codecInfo = static_cast<MediaCodecInfo *>(obj.get());
1517
1518 return OK;
1519 }
1520
getMetrics(MediaAnalyticsItem * & reply)1521 status_t MediaCodec::getMetrics(MediaAnalyticsItem * &reply) {
1522
1523 reply = NULL;
1524
1525 // shouldn't happen, but be safe
1526 if (mAnalyticsItem == NULL) {
1527 return UNKNOWN_ERROR;
1528 }
1529
1530 // update any in-flight data that's not carried within the record
1531 updateAnalyticsItem();
1532
1533 // send it back to the caller.
1534 reply = mAnalyticsItem->dup();
1535
1536 updateEphemeralAnalytics(reply);
1537
1538 return OK;
1539 }
1540
getInputBuffers(Vector<sp<MediaCodecBuffer>> * buffers) const1541 status_t MediaCodec::getInputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const {
1542 sp<AMessage> msg = new AMessage(kWhatGetBuffers, this);
1543 msg->setInt32("portIndex", kPortIndexInput);
1544 msg->setPointer("buffers", buffers);
1545
1546 sp<AMessage> response;
1547 return PostAndAwaitResponse(msg, &response);
1548 }
1549
getOutputBuffers(Vector<sp<MediaCodecBuffer>> * buffers) const1550 status_t MediaCodec::getOutputBuffers(Vector<sp<MediaCodecBuffer> > *buffers) const {
1551 sp<AMessage> msg = new AMessage(kWhatGetBuffers, this);
1552 msg->setInt32("portIndex", kPortIndexOutput);
1553 msg->setPointer("buffers", buffers);
1554
1555 sp<AMessage> response;
1556 return PostAndAwaitResponse(msg, &response);
1557 }
1558
getOutputBuffer(size_t index,sp<MediaCodecBuffer> * buffer)1559 status_t MediaCodec::getOutputBuffer(size_t index, sp<MediaCodecBuffer> *buffer) {
1560 sp<AMessage> format;
1561 return getBufferAndFormat(kPortIndexOutput, index, buffer, &format);
1562 }
1563
getOutputFormat(size_t index,sp<AMessage> * format)1564 status_t MediaCodec::getOutputFormat(size_t index, sp<AMessage> *format) {
1565 sp<MediaCodecBuffer> buffer;
1566 return getBufferAndFormat(kPortIndexOutput, index, &buffer, format);
1567 }
1568
getInputBuffer(size_t index,sp<MediaCodecBuffer> * buffer)1569 status_t MediaCodec::getInputBuffer(size_t index, sp<MediaCodecBuffer> *buffer) {
1570 sp<AMessage> format;
1571 return getBufferAndFormat(kPortIndexInput, index, buffer, &format);
1572 }
1573
isExecuting() const1574 bool MediaCodec::isExecuting() const {
1575 return mState == STARTED || mState == FLUSHED;
1576 }
1577
getBufferAndFormat(size_t portIndex,size_t index,sp<MediaCodecBuffer> * buffer,sp<AMessage> * format)1578 status_t MediaCodec::getBufferAndFormat(
1579 size_t portIndex, size_t index,
1580 sp<MediaCodecBuffer> *buffer, sp<AMessage> *format) {
1581 // use mutex instead of a context switch
1582 if (mReleasedByResourceManager) {
1583 ALOGE("getBufferAndFormat - resource already released");
1584 return DEAD_OBJECT;
1585 }
1586
1587 if (buffer == NULL) {
1588 ALOGE("getBufferAndFormat - null MediaCodecBuffer");
1589 return INVALID_OPERATION;
1590 }
1591
1592 if (format == NULL) {
1593 ALOGE("getBufferAndFormat - null AMessage");
1594 return INVALID_OPERATION;
1595 }
1596
1597 buffer->clear();
1598 format->clear();
1599
1600 if (!isExecuting()) {
1601 ALOGE("getBufferAndFormat - not executing");
1602 return INVALID_OPERATION;
1603 }
1604
1605 // we do not want mPortBuffers to change during this section
1606 // we also don't want mOwnedByClient to change during this
1607 Mutex::Autolock al(mBufferLock);
1608
1609 std::vector<BufferInfo> &buffers = mPortBuffers[portIndex];
1610 if (index >= buffers.size()) {
1611 ALOGE("getBufferAndFormat - trying to get buffer with "
1612 "bad index (index=%zu buffer_size=%zu)", index, buffers.size());
1613 return INVALID_OPERATION;
1614 }
1615
1616 const BufferInfo &info = buffers[index];
1617 if (!info.mOwnedByClient) {
1618 ALOGE("getBufferAndFormat - invalid operation "
1619 "(the index %zu is not owned by client)", index);
1620 return INVALID_OPERATION;
1621 }
1622
1623 *buffer = info.mData;
1624 *format = info.mData->format();
1625
1626 return OK;
1627 }
1628
flush()1629 status_t MediaCodec::flush() {
1630 sp<AMessage> msg = new AMessage(kWhatFlush, this);
1631
1632 sp<AMessage> response;
1633 return PostAndAwaitResponse(msg, &response);
1634 }
1635
requestIDRFrame()1636 status_t MediaCodec::requestIDRFrame() {
1637 (new AMessage(kWhatRequestIDRFrame, this))->post();
1638
1639 return OK;
1640 }
1641
requestActivityNotification(const sp<AMessage> & notify)1642 void MediaCodec::requestActivityNotification(const sp<AMessage> ¬ify) {
1643 sp<AMessage> msg = new AMessage(kWhatRequestActivityNotification, this);
1644 msg->setMessage("notify", notify);
1645 msg->post();
1646 }
1647
requestCpuBoostIfNeeded()1648 void MediaCodec::requestCpuBoostIfNeeded() {
1649 if (mCpuBoostRequested) {
1650 return;
1651 }
1652 int32_t colorFormat;
1653 if (mSoftRenderer != NULL
1654 && mOutputFormat->contains("hdr-static-info")
1655 && mOutputFormat->findInt32("color-format", &colorFormat)
1656 && (colorFormat == OMX_COLOR_FormatYUV420Planar16)) {
1657 int32_t left, top, right, bottom, width, height;
1658 int64_t totalPixel = 0;
1659 if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) {
1660 totalPixel = (right - left + 1) * (bottom - top + 1);
1661 } else if (mOutputFormat->findInt32("width", &width)
1662 && mOutputFormat->findInt32("height", &height)) {
1663 totalPixel = width * height;
1664 }
1665 if (totalPixel >= 1920 * 1080) {
1666 addResource(MediaResource::kCpuBoost,
1667 MediaResource::kUnspecifiedSubType, 1);
1668 mCpuBoostRequested = true;
1669 }
1670 }
1671 }
1672
1673 ////////////////////////////////////////////////////////////////////////////////
1674
cancelPendingDequeueOperations()1675 void MediaCodec::cancelPendingDequeueOperations() {
1676 if (mFlags & kFlagDequeueInputPending) {
1677 PostReplyWithError(mDequeueInputReplyID, INVALID_OPERATION);
1678
1679 ++mDequeueInputTimeoutGeneration;
1680 mDequeueInputReplyID = 0;
1681 mFlags &= ~kFlagDequeueInputPending;
1682 }
1683
1684 if (mFlags & kFlagDequeueOutputPending) {
1685 PostReplyWithError(mDequeueOutputReplyID, INVALID_OPERATION);
1686
1687 ++mDequeueOutputTimeoutGeneration;
1688 mDequeueOutputReplyID = 0;
1689 mFlags &= ~kFlagDequeueOutputPending;
1690 }
1691 }
1692
handleDequeueInputBuffer(const sp<AReplyToken> & replyID,bool newRequest)1693 bool MediaCodec::handleDequeueInputBuffer(const sp<AReplyToken> &replyID, bool newRequest) {
1694 if (!isExecuting() || (mFlags & kFlagIsAsync)
1695 || (newRequest && (mFlags & kFlagDequeueInputPending))) {
1696 PostReplyWithError(replyID, INVALID_OPERATION);
1697 return true;
1698 } else if (mFlags & kFlagStickyError) {
1699 PostReplyWithError(replyID, getStickyError());
1700 return true;
1701 }
1702
1703 ssize_t index = dequeuePortBuffer(kPortIndexInput);
1704
1705 if (index < 0) {
1706 CHECK_EQ(index, -EAGAIN);
1707 return false;
1708 }
1709
1710 sp<AMessage> response = new AMessage;
1711 response->setSize("index", index);
1712 response->postReply(replyID);
1713
1714 return true;
1715 }
1716
handleDequeueOutputBuffer(const sp<AReplyToken> & replyID,bool newRequest)1717 bool MediaCodec::handleDequeueOutputBuffer(const sp<AReplyToken> &replyID, bool newRequest) {
1718 if (!isExecuting() || (mFlags & kFlagIsAsync)
1719 || (newRequest && (mFlags & kFlagDequeueOutputPending))) {
1720 PostReplyWithError(replyID, INVALID_OPERATION);
1721 } else if (mFlags & kFlagStickyError) {
1722 PostReplyWithError(replyID, getStickyError());
1723 } else if (mFlags & kFlagOutputBuffersChanged) {
1724 PostReplyWithError(replyID, INFO_OUTPUT_BUFFERS_CHANGED);
1725 mFlags &= ~kFlagOutputBuffersChanged;
1726 } else if (mFlags & kFlagOutputFormatChanged) {
1727 PostReplyWithError(replyID, INFO_FORMAT_CHANGED);
1728 mFlags &= ~kFlagOutputFormatChanged;
1729 } else {
1730 sp<AMessage> response = new AMessage;
1731 ssize_t index = dequeuePortBuffer(kPortIndexOutput);
1732
1733 if (index < 0) {
1734 CHECK_EQ(index, -EAGAIN);
1735 return false;
1736 }
1737
1738 const sp<MediaCodecBuffer> &buffer =
1739 mPortBuffers[kPortIndexOutput][index].mData;
1740
1741 response->setSize("index", index);
1742 response->setSize("offset", buffer->offset());
1743 response->setSize("size", buffer->size());
1744
1745 int64_t timeUs;
1746 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
1747
1748 statsBufferReceived(timeUs);
1749
1750 response->setInt64("timeUs", timeUs);
1751
1752 int32_t flags;
1753 CHECK(buffer->meta()->findInt32("flags", &flags));
1754
1755 response->setInt32("flags", flags);
1756 response->postReply(replyID);
1757 }
1758
1759 return true;
1760 }
1761
onMessageReceived(const sp<AMessage> & msg)1762 void MediaCodec::onMessageReceived(const sp<AMessage> &msg) {
1763 switch (msg->what()) {
1764 case kWhatCodecNotify:
1765 {
1766 int32_t what;
1767 CHECK(msg->findInt32("what", &what));
1768
1769 switch (what) {
1770 case kWhatError:
1771 {
1772 int32_t err, actionCode;
1773 CHECK(msg->findInt32("err", &err));
1774 CHECK(msg->findInt32("actionCode", &actionCode));
1775
1776 ALOGE("Codec reported err %#x, actionCode %d, while in state %d",
1777 err, actionCode, mState);
1778 if (err == DEAD_OBJECT) {
1779 mFlags |= kFlagSawMediaServerDie;
1780 mFlags &= ~kFlagIsComponentAllocated;
1781 }
1782
1783 bool sendErrorResponse = true;
1784
1785 switch (mState) {
1786 case INITIALIZING:
1787 {
1788 setState(UNINITIALIZED);
1789 break;
1790 }
1791
1792 case CONFIGURING:
1793 {
1794 if (actionCode == ACTION_CODE_FATAL) {
1795 mAnalyticsItem->setInt32(kCodecError, err);
1796 mAnalyticsItem->setCString(kCodecErrorState, stateString(mState).c_str());
1797 flushAnalyticsItem();
1798 initAnalyticsItem();
1799 }
1800 setState(actionCode == ACTION_CODE_FATAL ?
1801 UNINITIALIZED : INITIALIZED);
1802 break;
1803 }
1804
1805 case STARTING:
1806 {
1807 if (actionCode == ACTION_CODE_FATAL) {
1808 mAnalyticsItem->setInt32(kCodecError, err);
1809 mAnalyticsItem->setCString(kCodecErrorState, stateString(mState).c_str());
1810 flushAnalyticsItem();
1811 initAnalyticsItem();
1812 }
1813 setState(actionCode == ACTION_CODE_FATAL ?
1814 UNINITIALIZED : CONFIGURED);
1815 break;
1816 }
1817
1818 case RELEASING:
1819 {
1820 // Ignore the error, assuming we'll still get
1821 // the shutdown complete notification. If we
1822 // don't, we'll timeout and force release.
1823 sendErrorResponse = false;
1824 }
1825 // fall-thru
1826 case STOPPING:
1827 {
1828 if (mFlags & kFlagSawMediaServerDie) {
1829 // MediaServer died, there definitely won't
1830 // be a shutdown complete notification after
1831 // all.
1832
1833 // note that we're directly going from
1834 // STOPPING->UNINITIALIZED, instead of the
1835 // usual STOPPING->INITIALIZED state.
1836 setState(UNINITIALIZED);
1837 if (mState == RELEASING) {
1838 mComponentName.clear();
1839 }
1840 (new AMessage)->postReply(mReplyID);
1841 sendErrorResponse = false;
1842 }
1843 break;
1844 }
1845
1846 case FLUSHING:
1847 {
1848 if (actionCode == ACTION_CODE_FATAL) {
1849 mAnalyticsItem->setInt32(kCodecError, err);
1850 mAnalyticsItem->setCString(kCodecErrorState, stateString(mState).c_str());
1851 flushAnalyticsItem();
1852 initAnalyticsItem();
1853
1854 setState(UNINITIALIZED);
1855 } else {
1856 setState(
1857 (mFlags & kFlagIsAsync) ? FLUSHED : STARTED);
1858 }
1859 break;
1860 }
1861
1862 case FLUSHED:
1863 case STARTED:
1864 {
1865 sendErrorResponse = false;
1866
1867 setStickyError(err);
1868 postActivityNotificationIfPossible();
1869
1870 cancelPendingDequeueOperations();
1871
1872 if (mFlags & kFlagIsAsync) {
1873 onError(err, actionCode);
1874 }
1875 switch (actionCode) {
1876 case ACTION_CODE_TRANSIENT:
1877 break;
1878 case ACTION_CODE_RECOVERABLE:
1879 setState(INITIALIZED);
1880 break;
1881 default:
1882 mAnalyticsItem->setInt32(kCodecError, err);
1883 mAnalyticsItem->setCString(kCodecErrorState, stateString(mState).c_str());
1884 flushAnalyticsItem();
1885 initAnalyticsItem();
1886 setState(UNINITIALIZED);
1887 break;
1888 }
1889 break;
1890 }
1891
1892 default:
1893 {
1894 sendErrorResponse = false;
1895
1896 setStickyError(err);
1897 postActivityNotificationIfPossible();
1898
1899 // actionCode in an uninitialized state is always fatal.
1900 if (mState == UNINITIALIZED) {
1901 actionCode = ACTION_CODE_FATAL;
1902 }
1903 if (mFlags & kFlagIsAsync) {
1904 onError(err, actionCode);
1905 }
1906 switch (actionCode) {
1907 case ACTION_CODE_TRANSIENT:
1908 break;
1909 case ACTION_CODE_RECOVERABLE:
1910 setState(INITIALIZED);
1911 break;
1912 default:
1913 setState(UNINITIALIZED);
1914 break;
1915 }
1916 break;
1917 }
1918 }
1919
1920 if (sendErrorResponse) {
1921 PostReplyWithError(mReplyID, err);
1922 }
1923 break;
1924 }
1925
1926 case kWhatComponentAllocated:
1927 {
1928 CHECK_EQ(mState, INITIALIZING);
1929 setState(INITIALIZED);
1930 mFlags |= kFlagIsComponentAllocated;
1931
1932 CHECK(msg->findString("componentName", &mComponentName));
1933
1934 if (mComponentName.c_str()) {
1935 mAnalyticsItem->setCString(kCodecCodec, mComponentName.c_str());
1936 }
1937
1938 if (mComponentName.startsWith("OMX.google.")) {
1939 mFlags |= kFlagUsesSoftwareRenderer;
1940 } else {
1941 mFlags &= ~kFlagUsesSoftwareRenderer;
1942 }
1943
1944 MediaResource::Type resourceType;
1945 if (mComponentName.endsWith(".secure")) {
1946 mFlags |= kFlagIsSecure;
1947 resourceType = MediaResource::kSecureCodec;
1948 mAnalyticsItem->setInt32(kCodecSecure, 1);
1949 } else {
1950 mFlags &= ~kFlagIsSecure;
1951 resourceType = MediaResource::kNonSecureCodec;
1952 mAnalyticsItem->setInt32(kCodecSecure, 0);
1953 }
1954
1955 if (mIsVideo) {
1956 // audio codec is currently ignored.
1957 addResource(resourceType, MediaResource::kVideoCodec, 1);
1958 }
1959
1960 (new AMessage)->postReply(mReplyID);
1961 break;
1962 }
1963
1964 case kWhatComponentConfigured:
1965 {
1966 if (mState == UNINITIALIZED || mState == INITIALIZED) {
1967 // In case a kWhatError message came in and replied with error,
1968 // we log a warning and ignore.
1969 ALOGW("configure interrupted by error, current state %d", mState);
1970 break;
1971 }
1972 CHECK_EQ(mState, CONFIGURING);
1973
1974 // reset input surface flag
1975 mHaveInputSurface = false;
1976
1977 CHECK(msg->findMessage("input-format", &mInputFormat));
1978 CHECK(msg->findMessage("output-format", &mOutputFormat));
1979 ALOGV("[%s] configured as input format: %s, output format: %s",
1980 mComponentName.c_str(),
1981 mInputFormat->debugString(4).c_str(),
1982 mOutputFormat->debugString(4).c_str());
1983 int32_t usingSwRenderer;
1984 if (mOutputFormat->findInt32("using-sw-renderer", &usingSwRenderer)
1985 && usingSwRenderer) {
1986 mFlags |= kFlagUsesSoftwareRenderer;
1987 }
1988 setState(CONFIGURED);
1989 (new AMessage)->postReply(mReplyID);
1990
1991 // augment our media metrics info, now that we know more things
1992 if (mAnalyticsItem != NULL) {
1993 sp<AMessage> format;
1994 if (mConfigureMsg != NULL &&
1995 mConfigureMsg->findMessage("format", &format)) {
1996 // format includes: mime
1997 AString mime;
1998 if (format->findString("mime", &mime)) {
1999 mAnalyticsItem->setCString(kCodecMime, mime.c_str());
2000 }
2001 }
2002 }
2003 break;
2004 }
2005
2006 case kWhatInputSurfaceCreated:
2007 {
2008 // response to initiateCreateInputSurface()
2009 status_t err = NO_ERROR;
2010 sp<AMessage> response = new AMessage;
2011 if (!msg->findInt32("err", &err)) {
2012 sp<RefBase> obj;
2013 msg->findObject("input-surface", &obj);
2014 CHECK(msg->findMessage("input-format", &mInputFormat));
2015 CHECK(msg->findMessage("output-format", &mOutputFormat));
2016 ALOGV("[%s] input surface created as input format: %s, output format: %s",
2017 mComponentName.c_str(),
2018 mInputFormat->debugString(4).c_str(),
2019 mOutputFormat->debugString(4).c_str());
2020 CHECK(obj != NULL);
2021 response->setObject("input-surface", obj);
2022 mHaveInputSurface = true;
2023 } else {
2024 response->setInt32("err", err);
2025 }
2026 response->postReply(mReplyID);
2027 break;
2028 }
2029
2030 case kWhatInputSurfaceAccepted:
2031 {
2032 // response to initiateSetInputSurface()
2033 status_t err = NO_ERROR;
2034 sp<AMessage> response = new AMessage();
2035 if (!msg->findInt32("err", &err)) {
2036 CHECK(msg->findMessage("input-format", &mInputFormat));
2037 CHECK(msg->findMessage("output-format", &mOutputFormat));
2038 mHaveInputSurface = true;
2039 } else {
2040 response->setInt32("err", err);
2041 }
2042 response->postReply(mReplyID);
2043 break;
2044 }
2045
2046 case kWhatSignaledInputEOS:
2047 {
2048 // response to signalEndOfInputStream()
2049 sp<AMessage> response = new AMessage;
2050 status_t err;
2051 if (msg->findInt32("err", &err)) {
2052 response->setInt32("err", err);
2053 }
2054 response->postReply(mReplyID);
2055 break;
2056 }
2057
2058 case kWhatStartCompleted:
2059 {
2060 CHECK_EQ(mState, STARTING);
2061 if (mIsVideo) {
2062 addResource(
2063 MediaResource::kGraphicMemory,
2064 MediaResource::kUnspecifiedSubType,
2065 getGraphicBufferSize());
2066 }
2067 setState(STARTED);
2068 (new AMessage)->postReply(mReplyID);
2069 break;
2070 }
2071
2072 case kWhatOutputBuffersChanged:
2073 {
2074 mFlags |= kFlagOutputBuffersChanged;
2075 postActivityNotificationIfPossible();
2076 break;
2077 }
2078
2079 case kWhatOutputFramesRendered:
2080 {
2081 // ignore these in all states except running, and check that we have a
2082 // notification set
2083 if (mState == STARTED && mOnFrameRenderedNotification != NULL) {
2084 sp<AMessage> notify = mOnFrameRenderedNotification->dup();
2085 notify->setMessage("data", msg);
2086 notify->post();
2087 }
2088 break;
2089 }
2090
2091 case kWhatFillThisBuffer:
2092 {
2093 /* size_t index = */updateBuffers(kPortIndexInput, msg);
2094
2095 if (mState == FLUSHING
2096 || mState == STOPPING
2097 || mState == RELEASING) {
2098 returnBuffersToCodecOnPort(kPortIndexInput);
2099 break;
2100 }
2101
2102 if (!mCSD.empty()) {
2103 ssize_t index = dequeuePortBuffer(kPortIndexInput);
2104 CHECK_GE(index, 0);
2105
2106 // If codec specific data had been specified as
2107 // part of the format in the call to configure and
2108 // if there's more csd left, we submit it here
2109 // clients only get access to input buffers once
2110 // this data has been exhausted.
2111
2112 status_t err = queueCSDInputBuffer(index);
2113
2114 if (err != OK) {
2115 ALOGE("queueCSDInputBuffer failed w/ error %d",
2116 err);
2117
2118 setStickyError(err);
2119 postActivityNotificationIfPossible();
2120
2121 cancelPendingDequeueOperations();
2122 }
2123 break;
2124 }
2125
2126 if (mFlags & kFlagIsAsync) {
2127 if (!mHaveInputSurface) {
2128 if (mState == FLUSHED) {
2129 mHavePendingInputBuffers = true;
2130 } else {
2131 onInputBufferAvailable();
2132 }
2133 }
2134 } else if (mFlags & kFlagDequeueInputPending) {
2135 CHECK(handleDequeueInputBuffer(mDequeueInputReplyID));
2136
2137 ++mDequeueInputTimeoutGeneration;
2138 mFlags &= ~kFlagDequeueInputPending;
2139 mDequeueInputReplyID = 0;
2140 } else {
2141 postActivityNotificationIfPossible();
2142 }
2143 break;
2144 }
2145
2146 case kWhatDrainThisBuffer:
2147 {
2148 /* size_t index = */updateBuffers(kPortIndexOutput, msg);
2149
2150 if (mState == FLUSHING
2151 || mState == STOPPING
2152 || mState == RELEASING) {
2153 returnBuffersToCodecOnPort(kPortIndexOutput);
2154 break;
2155 }
2156
2157 sp<RefBase> obj;
2158 CHECK(msg->findObject("buffer", &obj));
2159 sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get());
2160
2161 if (mOutputFormat != buffer->format()) {
2162 mOutputFormat = buffer->format();
2163 ALOGV("[%s] output format changed to: %s",
2164 mComponentName.c_str(), mOutputFormat->debugString(4).c_str());
2165
2166 if (mSoftRenderer == NULL &&
2167 mSurface != NULL &&
2168 (mFlags & kFlagUsesSoftwareRenderer)) {
2169 AString mime;
2170 CHECK(mOutputFormat->findString("mime", &mime));
2171
2172 // TODO: propagate color aspects to software renderer to allow better
2173 // color conversion to RGB. For now, just mark dataspace for YUV
2174 // rendering.
2175 int32_t dataSpace;
2176 if (mOutputFormat->findInt32("android._dataspace", &dataSpace)) {
2177 ALOGD("[%s] setting dataspace on output surface to #%x",
2178 mComponentName.c_str(), dataSpace);
2179 int err = native_window_set_buffers_data_space(
2180 mSurface.get(), (android_dataspace)dataSpace);
2181 ALOGW_IF(err != 0, "failed to set dataspace on surface (%d)", err);
2182 }
2183 if (mOutputFormat->contains("hdr-static-info")) {
2184 HDRStaticInfo info;
2185 if (ColorUtils::getHDRStaticInfoFromFormat(mOutputFormat, &info)) {
2186 setNativeWindowHdrMetadata(mSurface.get(), &info);
2187 }
2188 }
2189
2190 if (mime.startsWithIgnoreCase("video/")) {
2191 mSoftRenderer = new SoftwareRenderer(mSurface, mRotationDegrees);
2192 }
2193 }
2194
2195 requestCpuBoostIfNeeded();
2196
2197 if (mFlags & kFlagIsEncoder) {
2198 // Before we announce the format change we should
2199 // collect codec specific data and amend the output
2200 // format as necessary.
2201 int32_t flags = 0;
2202 (void) buffer->meta()->findInt32("flags", &flags);
2203 if (flags & BUFFER_FLAG_CODECCONFIG) {
2204 status_t err =
2205 amendOutputFormatWithCodecSpecificData(buffer);
2206
2207 if (err != OK) {
2208 ALOGE("Codec spit out malformed codec "
2209 "specific data!");
2210 }
2211 }
2212 }
2213 if (mFlags & kFlagIsAsync) {
2214 onOutputFormatChanged();
2215 } else {
2216 mFlags |= kFlagOutputFormatChanged;
2217 postActivityNotificationIfPossible();
2218 }
2219
2220 // Notify mCrypto of video resolution changes
2221 if (mCrypto != NULL) {
2222 int32_t left, top, right, bottom, width, height;
2223 if (mOutputFormat->findRect("crop", &left, &top, &right, &bottom)) {
2224 mCrypto->notifyResolution(right - left + 1, bottom - top + 1);
2225 } else if (mOutputFormat->findInt32("width", &width)
2226 && mOutputFormat->findInt32("height", &height)) {
2227 mCrypto->notifyResolution(width, height);
2228 }
2229 }
2230 }
2231
2232 if (mFlags & kFlagIsAsync) {
2233 onOutputBufferAvailable();
2234 } else if (mFlags & kFlagDequeueOutputPending) {
2235 CHECK(handleDequeueOutputBuffer(mDequeueOutputReplyID));
2236
2237 ++mDequeueOutputTimeoutGeneration;
2238 mFlags &= ~kFlagDequeueOutputPending;
2239 mDequeueOutputReplyID = 0;
2240 } else {
2241 postActivityNotificationIfPossible();
2242 }
2243
2244 break;
2245 }
2246
2247 case kWhatEOS:
2248 {
2249 // We already notify the client of this by using the
2250 // corresponding flag in "onOutputBufferReady".
2251 break;
2252 }
2253
2254 case kWhatStopCompleted:
2255 {
2256 if (mState != STOPPING) {
2257 ALOGW("Received kWhatStopCompleted in state %d", mState);
2258 break;
2259 }
2260 setState(INITIALIZED);
2261 (new AMessage)->postReply(mReplyID);
2262 break;
2263 }
2264
2265 case kWhatReleaseCompleted:
2266 {
2267 if (mState != RELEASING) {
2268 ALOGW("Received kWhatReleaseCompleted in state %d", mState);
2269 break;
2270 }
2271 setState(UNINITIALIZED);
2272 mComponentName.clear();
2273
2274 mFlags &= ~kFlagIsComponentAllocated;
2275
2276 mResourceManagerService->removeResource(getId(mResourceManagerClient));
2277
2278 (new AMessage)->postReply(mReplyID);
2279 break;
2280 }
2281
2282 case kWhatFlushCompleted:
2283 {
2284 if (mState != FLUSHING) {
2285 ALOGW("received FlushCompleted message in state %d",
2286 mState);
2287 break;
2288 }
2289
2290 if (mFlags & kFlagIsAsync) {
2291 setState(FLUSHED);
2292 } else {
2293 setState(STARTED);
2294 mCodec->signalResume();
2295 }
2296
2297 (new AMessage)->postReply(mReplyID);
2298 break;
2299 }
2300
2301 default:
2302 TRESPASS();
2303 }
2304 break;
2305 }
2306
2307 case kWhatInit:
2308 {
2309 sp<AReplyToken> replyID;
2310 CHECK(msg->senderAwaitsResponse(&replyID));
2311
2312 if (mState != UNINITIALIZED) {
2313 PostReplyWithError(replyID, INVALID_OPERATION);
2314 break;
2315 }
2316
2317 mReplyID = replyID;
2318 setState(INITIALIZING);
2319
2320 sp<RefBase> codecInfo;
2321 CHECK(msg->findObject("codecInfo", &codecInfo));
2322 AString name;
2323 CHECK(msg->findString("name", &name));
2324
2325 sp<AMessage> format = new AMessage;
2326 format->setObject("codecInfo", codecInfo);
2327 format->setString("componentName", name);
2328
2329 mCodec->initiateAllocateComponent(format);
2330 break;
2331 }
2332
2333 case kWhatSetNotification:
2334 {
2335 sp<AMessage> notify;
2336 if (msg->findMessage("on-frame-rendered", ¬ify)) {
2337 mOnFrameRenderedNotification = notify;
2338 }
2339 break;
2340 }
2341
2342 case kWhatSetCallback:
2343 {
2344 sp<AReplyToken> replyID;
2345 CHECK(msg->senderAwaitsResponse(&replyID));
2346
2347 if (mState == UNINITIALIZED
2348 || mState == INITIALIZING
2349 || isExecuting()) {
2350 // callback can't be set after codec is executing,
2351 // or before it's initialized (as the callback
2352 // will be cleared when it goes to INITIALIZED)
2353 PostReplyWithError(replyID, INVALID_OPERATION);
2354 break;
2355 }
2356
2357 sp<AMessage> callback;
2358 CHECK(msg->findMessage("callback", &callback));
2359
2360 mCallback = callback;
2361
2362 if (mCallback != NULL) {
2363 ALOGI("MediaCodec will operate in async mode");
2364 mFlags |= kFlagIsAsync;
2365 } else {
2366 mFlags &= ~kFlagIsAsync;
2367 }
2368
2369 sp<AMessage> response = new AMessage;
2370 response->postReply(replyID);
2371 break;
2372 }
2373
2374 case kWhatConfigure:
2375 {
2376 sp<AReplyToken> replyID;
2377 CHECK(msg->senderAwaitsResponse(&replyID));
2378
2379 if (mState != INITIALIZED) {
2380 PostReplyWithError(replyID, INVALID_OPERATION);
2381 break;
2382 }
2383
2384 sp<RefBase> obj;
2385 CHECK(msg->findObject("surface", &obj));
2386
2387 sp<AMessage> format;
2388 CHECK(msg->findMessage("format", &format));
2389
2390 int32_t push;
2391 if (msg->findInt32("push-blank-buffers-on-shutdown", &push) && push != 0) {
2392 mFlags |= kFlagPushBlankBuffersOnShutdown;
2393 }
2394
2395 if (obj != NULL) {
2396 format->setObject("native-window", obj);
2397 status_t err = handleSetSurface(static_cast<Surface *>(obj.get()));
2398 if (err != OK) {
2399 PostReplyWithError(replyID, err);
2400 break;
2401 }
2402 } else {
2403 handleSetSurface(NULL);
2404 }
2405
2406 mReplyID = replyID;
2407 setState(CONFIGURING);
2408
2409 void *crypto;
2410 if (!msg->findPointer("crypto", &crypto)) {
2411 crypto = NULL;
2412 }
2413
2414 ALOGV("kWhatConfigure: Old mCrypto: %p (%d)",
2415 mCrypto.get(), (mCrypto != NULL ? mCrypto->getStrongCount() : 0));
2416
2417 mCrypto = static_cast<ICrypto *>(crypto);
2418 mBufferChannel->setCrypto(mCrypto);
2419
2420 ALOGV("kWhatConfigure: New mCrypto: %p (%d)",
2421 mCrypto.get(), (mCrypto != NULL ? mCrypto->getStrongCount() : 0));
2422
2423 void *descrambler;
2424 if (!msg->findPointer("descrambler", &descrambler)) {
2425 descrambler = NULL;
2426 }
2427
2428 mDescrambler = static_cast<IDescrambler *>(descrambler);
2429 mBufferChannel->setDescrambler(mDescrambler);
2430
2431 uint32_t flags;
2432 CHECK(msg->findInt32("flags", (int32_t *)&flags));
2433
2434 if (flags & CONFIGURE_FLAG_ENCODE) {
2435 format->setInt32("encoder", true);
2436 mFlags |= kFlagIsEncoder;
2437 }
2438
2439 extractCSD(format);
2440
2441 mCodec->initiateConfigureComponent(format);
2442 break;
2443 }
2444
2445 case kWhatSetSurface:
2446 {
2447 sp<AReplyToken> replyID;
2448 CHECK(msg->senderAwaitsResponse(&replyID));
2449
2450 status_t err = OK;
2451
2452 switch (mState) {
2453 case CONFIGURED:
2454 case STARTED:
2455 case FLUSHED:
2456 {
2457 sp<RefBase> obj;
2458 (void)msg->findObject("surface", &obj);
2459 sp<Surface> surface = static_cast<Surface *>(obj.get());
2460 if (mSurface == NULL) {
2461 // do not support setting surface if it was not set
2462 err = INVALID_OPERATION;
2463 } else if (obj == NULL) {
2464 // do not support unsetting surface
2465 err = BAD_VALUE;
2466 } else {
2467 err = connectToSurface(surface);
2468 if (err == ALREADY_EXISTS) {
2469 // reconnecting to same surface
2470 err = OK;
2471 } else {
2472 if (err == OK) {
2473 if (mFlags & kFlagUsesSoftwareRenderer) {
2474 if (mSoftRenderer != NULL
2475 && (mFlags & kFlagPushBlankBuffersOnShutdown)) {
2476 pushBlankBuffersToNativeWindow(mSurface.get());
2477 }
2478 mSoftRenderer = new SoftwareRenderer(surface);
2479 // TODO: check if this was successful
2480 } else {
2481 err = mCodec->setSurface(surface);
2482 }
2483 }
2484 if (err == OK) {
2485 (void)disconnectFromSurface();
2486 mSurface = surface;
2487 }
2488 }
2489 }
2490 break;
2491 }
2492
2493 default:
2494 err = INVALID_OPERATION;
2495 break;
2496 }
2497
2498 PostReplyWithError(replyID, err);
2499 break;
2500 }
2501
2502 case kWhatCreateInputSurface:
2503 case kWhatSetInputSurface:
2504 {
2505 sp<AReplyToken> replyID;
2506 CHECK(msg->senderAwaitsResponse(&replyID));
2507
2508 // Must be configured, but can't have been started yet.
2509 if (mState != CONFIGURED) {
2510 PostReplyWithError(replyID, INVALID_OPERATION);
2511 break;
2512 }
2513
2514 mReplyID = replyID;
2515 if (msg->what() == kWhatCreateInputSurface) {
2516 mCodec->initiateCreateInputSurface();
2517 } else {
2518 sp<RefBase> obj;
2519 CHECK(msg->findObject("input-surface", &obj));
2520
2521 mCodec->initiateSetInputSurface(
2522 static_cast<PersistentSurface *>(obj.get()));
2523 }
2524 break;
2525 }
2526 case kWhatStart:
2527 {
2528 sp<AReplyToken> replyID;
2529 CHECK(msg->senderAwaitsResponse(&replyID));
2530
2531 if (mState == FLUSHED) {
2532 setState(STARTED);
2533 if (mHavePendingInputBuffers) {
2534 onInputBufferAvailable();
2535 mHavePendingInputBuffers = false;
2536 }
2537 mCodec->signalResume();
2538 PostReplyWithError(replyID, OK);
2539 break;
2540 } else if (mState != CONFIGURED) {
2541 PostReplyWithError(replyID, INVALID_OPERATION);
2542 break;
2543 }
2544
2545 mReplyID = replyID;
2546 setState(STARTING);
2547
2548 mCodec->initiateStart();
2549 break;
2550 }
2551
2552 case kWhatStop:
2553 case kWhatRelease:
2554 {
2555 State targetState =
2556 (msg->what() == kWhatStop) ? INITIALIZED : UNINITIALIZED;
2557
2558 sp<AReplyToken> replyID;
2559 CHECK(msg->senderAwaitsResponse(&replyID));
2560
2561 // already stopped/released
2562 if (mState == UNINITIALIZED && mReleasedByResourceManager) {
2563 sp<AMessage> response = new AMessage;
2564 response->setInt32("err", OK);
2565 response->postReply(replyID);
2566 break;
2567 }
2568
2569 int32_t reclaimed = 0;
2570 msg->findInt32("reclaimed", &reclaimed);
2571 if (reclaimed) {
2572 mReleasedByResourceManager = true;
2573
2574 int32_t force = 0;
2575 msg->findInt32("force", &force);
2576 if (!force && hasPendingBuffer()) {
2577 ALOGW("Can't reclaim codec right now due to pending buffers.");
2578
2579 // return WOULD_BLOCK to ask resource manager to retry later.
2580 sp<AMessage> response = new AMessage;
2581 response->setInt32("err", WOULD_BLOCK);
2582 response->postReply(replyID);
2583
2584 // notify the async client
2585 if (mFlags & kFlagIsAsync) {
2586 onError(DEAD_OBJECT, ACTION_CODE_FATAL);
2587 }
2588 break;
2589 }
2590 }
2591
2592 bool isReleasingAllocatedComponent =
2593 (mFlags & kFlagIsComponentAllocated) && targetState == UNINITIALIZED;
2594 if (!isReleasingAllocatedComponent // See 1
2595 && mState != INITIALIZED
2596 && mState != CONFIGURED && !isExecuting()) {
2597 // 1) Permit release to shut down the component if allocated.
2598 //
2599 // 2) We may be in "UNINITIALIZED" state already and
2600 // also shutdown the encoder/decoder without the
2601 // client being aware of this if media server died while
2602 // we were being stopped. The client would assume that
2603 // after stop() returned, it would be safe to call release()
2604 // and it should be in this case, no harm to allow a release()
2605 // if we're already uninitialized.
2606 sp<AMessage> response = new AMessage;
2607 // TODO: we shouldn't throw an exception for stop/release. Change this to wait until
2608 // the previous stop/release completes and then reply with OK.
2609 status_t err = mState == targetState ? OK : INVALID_OPERATION;
2610 response->setInt32("err", err);
2611 if (err == OK && targetState == UNINITIALIZED) {
2612 mComponentName.clear();
2613 }
2614 response->postReply(replyID);
2615 break;
2616 }
2617
2618 // If we're flushing, or we're stopping but received a release
2619 // request, post the reply for the pending call first, and consider
2620 // it done. The reply token will be replaced after this, and we'll
2621 // no longer be able to reply.
2622 if (mState == FLUSHING || mState == STOPPING) {
2623 (new AMessage)->postReply(mReplyID);
2624 }
2625
2626 if (mFlags & kFlagSawMediaServerDie) {
2627 // It's dead, Jim. Don't expect initiateShutdown to yield
2628 // any useful results now...
2629 setState(UNINITIALIZED);
2630 if (targetState == UNINITIALIZED) {
2631 mComponentName.clear();
2632 }
2633 (new AMessage)->postReply(replyID);
2634 break;
2635 }
2636
2637 // If we already have an error, component may not be able to
2638 // complete the shutdown properly. If we're stopping, post the
2639 // reply now with an error to unblock the client, client can
2640 // release after the failure (instead of ANR).
2641 if (msg->what() == kWhatStop && (mFlags & kFlagStickyError)) {
2642 PostReplyWithError(replyID, getStickyError());
2643 break;
2644 }
2645
2646 mReplyID = replyID;
2647 setState(msg->what() == kWhatStop ? STOPPING : RELEASING);
2648
2649 mCodec->initiateShutdown(
2650 msg->what() == kWhatStop /* keepComponentAllocated */);
2651
2652 returnBuffersToCodec(reclaimed);
2653
2654 if (mSoftRenderer != NULL && (mFlags & kFlagPushBlankBuffersOnShutdown)) {
2655 pushBlankBuffersToNativeWindow(mSurface.get());
2656 }
2657
2658 break;
2659 }
2660
2661 case kWhatDequeueInputBuffer:
2662 {
2663 sp<AReplyToken> replyID;
2664 CHECK(msg->senderAwaitsResponse(&replyID));
2665
2666 if (mFlags & kFlagIsAsync) {
2667 ALOGE("dequeueOutputBuffer can't be used in async mode");
2668 PostReplyWithError(replyID, INVALID_OPERATION);
2669 break;
2670 }
2671
2672 if (mHaveInputSurface) {
2673 ALOGE("dequeueInputBuffer can't be used with input surface");
2674 PostReplyWithError(replyID, INVALID_OPERATION);
2675 break;
2676 }
2677
2678 if (handleDequeueInputBuffer(replyID, true /* new request */)) {
2679 break;
2680 }
2681
2682 int64_t timeoutUs;
2683 CHECK(msg->findInt64("timeoutUs", &timeoutUs));
2684
2685 if (timeoutUs == 0ll) {
2686 PostReplyWithError(replyID, -EAGAIN);
2687 break;
2688 }
2689
2690 mFlags |= kFlagDequeueInputPending;
2691 mDequeueInputReplyID = replyID;
2692
2693 if (timeoutUs > 0ll) {
2694 sp<AMessage> timeoutMsg =
2695 new AMessage(kWhatDequeueInputTimedOut, this);
2696 timeoutMsg->setInt32(
2697 "generation", ++mDequeueInputTimeoutGeneration);
2698 timeoutMsg->post(timeoutUs);
2699 }
2700 break;
2701 }
2702
2703 case kWhatDequeueInputTimedOut:
2704 {
2705 int32_t generation;
2706 CHECK(msg->findInt32("generation", &generation));
2707
2708 if (generation != mDequeueInputTimeoutGeneration) {
2709 // Obsolete
2710 break;
2711 }
2712
2713 CHECK(mFlags & kFlagDequeueInputPending);
2714
2715 PostReplyWithError(mDequeueInputReplyID, -EAGAIN);
2716
2717 mFlags &= ~kFlagDequeueInputPending;
2718 mDequeueInputReplyID = 0;
2719 break;
2720 }
2721
2722 case kWhatQueueInputBuffer:
2723 {
2724 sp<AReplyToken> replyID;
2725 CHECK(msg->senderAwaitsResponse(&replyID));
2726
2727 if (!isExecuting()) {
2728 PostReplyWithError(replyID, INVALID_OPERATION);
2729 break;
2730 } else if (mFlags & kFlagStickyError) {
2731 PostReplyWithError(replyID, getStickyError());
2732 break;
2733 }
2734
2735 status_t err = onQueueInputBuffer(msg);
2736
2737 PostReplyWithError(replyID, err);
2738 break;
2739 }
2740
2741 case kWhatDequeueOutputBuffer:
2742 {
2743 sp<AReplyToken> replyID;
2744 CHECK(msg->senderAwaitsResponse(&replyID));
2745
2746 if (mFlags & kFlagIsAsync) {
2747 ALOGE("dequeueOutputBuffer can't be used in async mode");
2748 PostReplyWithError(replyID, INVALID_OPERATION);
2749 break;
2750 }
2751
2752 if (handleDequeueOutputBuffer(replyID, true /* new request */)) {
2753 break;
2754 }
2755
2756 int64_t timeoutUs;
2757 CHECK(msg->findInt64("timeoutUs", &timeoutUs));
2758
2759 if (timeoutUs == 0ll) {
2760 PostReplyWithError(replyID, -EAGAIN);
2761 break;
2762 }
2763
2764 mFlags |= kFlagDequeueOutputPending;
2765 mDequeueOutputReplyID = replyID;
2766
2767 if (timeoutUs > 0ll) {
2768 sp<AMessage> timeoutMsg =
2769 new AMessage(kWhatDequeueOutputTimedOut, this);
2770 timeoutMsg->setInt32(
2771 "generation", ++mDequeueOutputTimeoutGeneration);
2772 timeoutMsg->post(timeoutUs);
2773 }
2774 break;
2775 }
2776
2777 case kWhatDequeueOutputTimedOut:
2778 {
2779 int32_t generation;
2780 CHECK(msg->findInt32("generation", &generation));
2781
2782 if (generation != mDequeueOutputTimeoutGeneration) {
2783 // Obsolete
2784 break;
2785 }
2786
2787 CHECK(mFlags & kFlagDequeueOutputPending);
2788
2789 PostReplyWithError(mDequeueOutputReplyID, -EAGAIN);
2790
2791 mFlags &= ~kFlagDequeueOutputPending;
2792 mDequeueOutputReplyID = 0;
2793 break;
2794 }
2795
2796 case kWhatReleaseOutputBuffer:
2797 {
2798 sp<AReplyToken> replyID;
2799 CHECK(msg->senderAwaitsResponse(&replyID));
2800
2801 if (!isExecuting()) {
2802 PostReplyWithError(replyID, INVALID_OPERATION);
2803 break;
2804 } else if (mFlags & kFlagStickyError) {
2805 PostReplyWithError(replyID, getStickyError());
2806 break;
2807 }
2808
2809 status_t err = onReleaseOutputBuffer(msg);
2810
2811 PostReplyWithError(replyID, err);
2812 break;
2813 }
2814
2815 case kWhatSignalEndOfInputStream:
2816 {
2817 sp<AReplyToken> replyID;
2818 CHECK(msg->senderAwaitsResponse(&replyID));
2819
2820 if (!isExecuting() || !mHaveInputSurface) {
2821 PostReplyWithError(replyID, INVALID_OPERATION);
2822 break;
2823 } else if (mFlags & kFlagStickyError) {
2824 PostReplyWithError(replyID, getStickyError());
2825 break;
2826 }
2827
2828 mReplyID = replyID;
2829 mCodec->signalEndOfInputStream();
2830 break;
2831 }
2832
2833 case kWhatGetBuffers:
2834 {
2835 sp<AReplyToken> replyID;
2836 CHECK(msg->senderAwaitsResponse(&replyID));
2837 if (!isExecuting() || (mFlags & kFlagIsAsync)) {
2838 PostReplyWithError(replyID, INVALID_OPERATION);
2839 break;
2840 } else if (mFlags & kFlagStickyError) {
2841 PostReplyWithError(replyID, getStickyError());
2842 break;
2843 }
2844
2845 int32_t portIndex;
2846 CHECK(msg->findInt32("portIndex", &portIndex));
2847
2848 Vector<sp<MediaCodecBuffer> > *dstBuffers;
2849 CHECK(msg->findPointer("buffers", (void **)&dstBuffers));
2850
2851 dstBuffers->clear();
2852 // If we're using input surface (either non-persistent created by
2853 // createInputSurface(), or persistent set by setInputSurface()),
2854 // give the client an empty input buffers array.
2855 if (portIndex != kPortIndexInput || !mHaveInputSurface) {
2856 if (portIndex == kPortIndexInput) {
2857 mBufferChannel->getInputBufferArray(dstBuffers);
2858 } else {
2859 mBufferChannel->getOutputBufferArray(dstBuffers);
2860 }
2861 }
2862
2863 (new AMessage)->postReply(replyID);
2864 break;
2865 }
2866
2867 case kWhatFlush:
2868 {
2869 sp<AReplyToken> replyID;
2870 CHECK(msg->senderAwaitsResponse(&replyID));
2871
2872 if (!isExecuting()) {
2873 PostReplyWithError(replyID, INVALID_OPERATION);
2874 break;
2875 } else if (mFlags & kFlagStickyError) {
2876 PostReplyWithError(replyID, getStickyError());
2877 break;
2878 }
2879
2880 mReplyID = replyID;
2881 // TODO: skip flushing if already FLUSHED
2882 setState(FLUSHING);
2883
2884 mCodec->signalFlush();
2885 returnBuffersToCodec();
2886 break;
2887 }
2888
2889 case kWhatGetInputFormat:
2890 case kWhatGetOutputFormat:
2891 {
2892 sp<AMessage> format =
2893 (msg->what() == kWhatGetOutputFormat ? mOutputFormat : mInputFormat);
2894
2895 sp<AReplyToken> replyID;
2896 CHECK(msg->senderAwaitsResponse(&replyID));
2897
2898 if ((mState != CONFIGURED && mState != STARTING &&
2899 mState != STARTED && mState != FLUSHING &&
2900 mState != FLUSHED)
2901 || format == NULL) {
2902 PostReplyWithError(replyID, INVALID_OPERATION);
2903 break;
2904 } else if (mFlags & kFlagStickyError) {
2905 PostReplyWithError(replyID, getStickyError());
2906 break;
2907 }
2908
2909 sp<AMessage> response = new AMessage;
2910 response->setMessage("format", format);
2911 response->postReply(replyID);
2912 break;
2913 }
2914
2915 case kWhatRequestIDRFrame:
2916 {
2917 mCodec->signalRequestIDRFrame();
2918 break;
2919 }
2920
2921 case kWhatRequestActivityNotification:
2922 {
2923 CHECK(mActivityNotify == NULL);
2924 CHECK(msg->findMessage("notify", &mActivityNotify));
2925
2926 postActivityNotificationIfPossible();
2927 break;
2928 }
2929
2930 case kWhatGetName:
2931 {
2932 sp<AReplyToken> replyID;
2933 CHECK(msg->senderAwaitsResponse(&replyID));
2934
2935 if (mComponentName.empty()) {
2936 PostReplyWithError(replyID, INVALID_OPERATION);
2937 break;
2938 }
2939
2940 sp<AMessage> response = new AMessage;
2941 response->setString("name", mComponentName.c_str());
2942 response->postReply(replyID);
2943 break;
2944 }
2945
2946 case kWhatGetCodecInfo:
2947 {
2948 sp<AReplyToken> replyID;
2949 CHECK(msg->senderAwaitsResponse(&replyID));
2950
2951 sp<AMessage> response = new AMessage;
2952 response->setObject("codecInfo", mCodecInfo);
2953 response->postReply(replyID);
2954 break;
2955 }
2956
2957 case kWhatSetParameters:
2958 {
2959 sp<AReplyToken> replyID;
2960 CHECK(msg->senderAwaitsResponse(&replyID));
2961
2962 sp<AMessage> params;
2963 CHECK(msg->findMessage("params", ¶ms));
2964
2965 status_t err = onSetParameters(params);
2966
2967 PostReplyWithError(replyID, err);
2968 break;
2969 }
2970
2971 case kWhatDrmReleaseCrypto:
2972 {
2973 onReleaseCrypto(msg);
2974 break;
2975 }
2976
2977 default:
2978 TRESPASS();
2979 }
2980 }
2981
extractCSD(const sp<AMessage> & format)2982 void MediaCodec::extractCSD(const sp<AMessage> &format) {
2983 mCSD.clear();
2984
2985 size_t i = 0;
2986 for (;;) {
2987 sp<ABuffer> csd;
2988 if (!format->findBuffer(AStringPrintf("csd-%u", i).c_str(), &csd)) {
2989 break;
2990 }
2991 if (csd->size() == 0) {
2992 ALOGW("csd-%zu size is 0", i);
2993 }
2994
2995 mCSD.push_back(csd);
2996 ++i;
2997 }
2998
2999 ALOGV("Found %zu pieces of codec specific data.", mCSD.size());
3000 }
3001
queueCSDInputBuffer(size_t bufferIndex)3002 status_t MediaCodec::queueCSDInputBuffer(size_t bufferIndex) {
3003 CHECK(!mCSD.empty());
3004
3005 const BufferInfo &info = mPortBuffers[kPortIndexInput][bufferIndex];
3006
3007 sp<ABuffer> csd = *mCSD.begin();
3008 mCSD.erase(mCSD.begin());
3009
3010 const sp<MediaCodecBuffer> &codecInputData = info.mData;
3011
3012 if (csd->size() > codecInputData->capacity()) {
3013 return -EINVAL;
3014 }
3015 if (codecInputData->data() == NULL) {
3016 ALOGV("Input buffer %zu is not properly allocated", bufferIndex);
3017 return -EINVAL;
3018 }
3019
3020 memcpy(codecInputData->data(), csd->data(), csd->size());
3021
3022 AString errorDetailMsg;
3023
3024 sp<AMessage> msg = new AMessage(kWhatQueueInputBuffer, this);
3025 msg->setSize("index", bufferIndex);
3026 msg->setSize("offset", 0);
3027 msg->setSize("size", csd->size());
3028 msg->setInt64("timeUs", 0ll);
3029 msg->setInt32("flags", BUFFER_FLAG_CODECCONFIG);
3030 msg->setPointer("errorDetailMsg", &errorDetailMsg);
3031
3032 return onQueueInputBuffer(msg);
3033 }
3034
setState(State newState)3035 void MediaCodec::setState(State newState) {
3036 if (newState == INITIALIZED || newState == UNINITIALIZED) {
3037 delete mSoftRenderer;
3038 mSoftRenderer = NULL;
3039
3040 if ( mCrypto != NULL ) {
3041 ALOGV("setState: ~mCrypto: %p (%d)",
3042 mCrypto.get(), (mCrypto != NULL ? mCrypto->getStrongCount() : 0));
3043 }
3044 mCrypto.clear();
3045 mDescrambler.clear();
3046 handleSetSurface(NULL);
3047
3048 mInputFormat.clear();
3049 mOutputFormat.clear();
3050 mFlags &= ~kFlagOutputFormatChanged;
3051 mFlags &= ~kFlagOutputBuffersChanged;
3052 mFlags &= ~kFlagStickyError;
3053 mFlags &= ~kFlagIsEncoder;
3054 mFlags &= ~kFlagIsAsync;
3055 mStickyError = OK;
3056
3057 mActivityNotify.clear();
3058 mCallback.clear();
3059 }
3060
3061 if (newState == UNINITIALIZED) {
3062 // return any straggling buffers, e.g. if we got here on an error
3063 returnBuffersToCodec();
3064
3065 // The component is gone, mediaserver's probably back up already
3066 // but should definitely be back up should we try to instantiate
3067 // another component.. and the cycle continues.
3068 mFlags &= ~kFlagSawMediaServerDie;
3069 }
3070
3071 mState = newState;
3072
3073 cancelPendingDequeueOperations();
3074
3075 updateBatteryStat();
3076 }
3077
returnBuffersToCodec(bool isReclaim)3078 void MediaCodec::returnBuffersToCodec(bool isReclaim) {
3079 returnBuffersToCodecOnPort(kPortIndexInput, isReclaim);
3080 returnBuffersToCodecOnPort(kPortIndexOutput, isReclaim);
3081 }
3082
returnBuffersToCodecOnPort(int32_t portIndex,bool isReclaim)3083 void MediaCodec::returnBuffersToCodecOnPort(int32_t portIndex, bool isReclaim) {
3084 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
3085 Mutex::Autolock al(mBufferLock);
3086
3087 for (size_t i = 0; i < mPortBuffers[portIndex].size(); ++i) {
3088 BufferInfo *info = &mPortBuffers[portIndex][i];
3089
3090 if (info->mData != nullptr) {
3091 sp<MediaCodecBuffer> buffer = info->mData;
3092 if (isReclaim && info->mOwnedByClient) {
3093 ALOGD("port %d buffer %zu still owned by client when codec is reclaimed",
3094 portIndex, i);
3095 } else {
3096 info->mOwnedByClient = false;
3097 info->mData.clear();
3098 }
3099 mBufferChannel->discardBuffer(buffer);
3100 }
3101 }
3102
3103 mAvailPortBuffers[portIndex].clear();
3104 }
3105
updateBuffers(int32_t portIndex,const sp<AMessage> & msg)3106 size_t MediaCodec::updateBuffers(
3107 int32_t portIndex, const sp<AMessage> &msg) {
3108 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
3109 size_t index;
3110 CHECK(msg->findSize("index", &index));
3111 sp<RefBase> obj;
3112 CHECK(msg->findObject("buffer", &obj));
3113 sp<MediaCodecBuffer> buffer = static_cast<MediaCodecBuffer *>(obj.get());
3114
3115 {
3116 Mutex::Autolock al(mBufferLock);
3117 if (mPortBuffers[portIndex].size() <= index) {
3118 mPortBuffers[portIndex].resize(align(index + 1, kNumBuffersAlign));
3119 }
3120 mPortBuffers[portIndex][index].mData = buffer;
3121 }
3122 mAvailPortBuffers[portIndex].push_back(index);
3123
3124 return index;
3125 }
3126
onQueueInputBuffer(const sp<AMessage> & msg)3127 status_t MediaCodec::onQueueInputBuffer(const sp<AMessage> &msg) {
3128 size_t index;
3129 size_t offset;
3130 size_t size;
3131 int64_t timeUs;
3132 uint32_t flags;
3133 CHECK(msg->findSize("index", &index));
3134 CHECK(msg->findSize("offset", &offset));
3135 CHECK(msg->findInt64("timeUs", &timeUs));
3136 CHECK(msg->findInt32("flags", (int32_t *)&flags));
3137
3138 const CryptoPlugin::SubSample *subSamples;
3139 size_t numSubSamples;
3140 const uint8_t *key;
3141 const uint8_t *iv;
3142 CryptoPlugin::Mode mode = CryptoPlugin::kMode_Unencrypted;
3143
3144 // We allow the simpler queueInputBuffer API to be used even in
3145 // secure mode, by fabricating a single unencrypted subSample.
3146 CryptoPlugin::SubSample ss;
3147 CryptoPlugin::Pattern pattern;
3148
3149 if (msg->findSize("size", &size)) {
3150 if (hasCryptoOrDescrambler()) {
3151 ss.mNumBytesOfClearData = size;
3152 ss.mNumBytesOfEncryptedData = 0;
3153
3154 subSamples = &ss;
3155 numSubSamples = 1;
3156 key = NULL;
3157 iv = NULL;
3158 pattern.mEncryptBlocks = 0;
3159 pattern.mSkipBlocks = 0;
3160 }
3161 } else {
3162 if (!hasCryptoOrDescrambler()) {
3163 ALOGE("[%s] queuing secure buffer without mCrypto or mDescrambler!",
3164 mComponentName.c_str());
3165 return -EINVAL;
3166 }
3167
3168 CHECK(msg->findPointer("subSamples", (void **)&subSamples));
3169 CHECK(msg->findSize("numSubSamples", &numSubSamples));
3170 CHECK(msg->findPointer("key", (void **)&key));
3171 CHECK(msg->findPointer("iv", (void **)&iv));
3172 CHECK(msg->findInt32("encryptBlocks", (int32_t *)&pattern.mEncryptBlocks));
3173 CHECK(msg->findInt32("skipBlocks", (int32_t *)&pattern.mSkipBlocks));
3174
3175 int32_t tmp;
3176 CHECK(msg->findInt32("mode", &tmp));
3177
3178 mode = (CryptoPlugin::Mode)tmp;
3179
3180 size = 0;
3181 for (size_t i = 0; i < numSubSamples; ++i) {
3182 size += subSamples[i].mNumBytesOfClearData;
3183 size += subSamples[i].mNumBytesOfEncryptedData;
3184 }
3185 }
3186
3187 if (index >= mPortBuffers[kPortIndexInput].size()) {
3188 return -ERANGE;
3189 }
3190
3191 BufferInfo *info = &mPortBuffers[kPortIndexInput][index];
3192
3193 if (info->mData == nullptr || !info->mOwnedByClient) {
3194 return -EACCES;
3195 }
3196
3197 if (offset + size > info->mData->capacity()) {
3198 return -EINVAL;
3199 }
3200
3201 info->mData->setRange(offset, size);
3202 info->mData->meta()->setInt64("timeUs", timeUs);
3203 if (flags & BUFFER_FLAG_EOS) {
3204 info->mData->meta()->setInt32("eos", true);
3205 }
3206
3207 if (flags & BUFFER_FLAG_CODECCONFIG) {
3208 info->mData->meta()->setInt32("csd", true);
3209 }
3210
3211 sp<MediaCodecBuffer> buffer = info->mData;
3212 status_t err = OK;
3213 if (hasCryptoOrDescrambler()) {
3214 AString *errorDetailMsg;
3215 CHECK(msg->findPointer("errorDetailMsg", (void **)&errorDetailMsg));
3216
3217 err = mBufferChannel->queueSecureInputBuffer(
3218 buffer,
3219 (mFlags & kFlagIsSecure),
3220 key,
3221 iv,
3222 mode,
3223 pattern,
3224 subSamples,
3225 numSubSamples,
3226 errorDetailMsg);
3227 } else {
3228 err = mBufferChannel->queueInputBuffer(buffer);
3229 }
3230
3231 if (err == OK) {
3232 // synchronization boundary for getBufferAndFormat
3233 Mutex::Autolock al(mBufferLock);
3234 info->mOwnedByClient = false;
3235 info->mData.clear();
3236
3237 statsBufferSent(timeUs);
3238 }
3239
3240 return err;
3241 }
3242
3243 //static
CreateFramesRenderedMessage(const std::list<FrameRenderTracker::Info> & done,sp<AMessage> & msg)3244 size_t MediaCodec::CreateFramesRenderedMessage(
3245 const std::list<FrameRenderTracker::Info> &done, sp<AMessage> &msg) {
3246 size_t index = 0;
3247
3248 for (std::list<FrameRenderTracker::Info>::const_iterator it = done.cbegin();
3249 it != done.cend(); ++it) {
3250 if (it->getRenderTimeNs() < 0) {
3251 continue; // dropped frame from tracking
3252 }
3253 msg->setInt64(AStringPrintf("%zu-media-time-us", index).c_str(), it->getMediaTimeUs());
3254 msg->setInt64(AStringPrintf("%zu-system-nano", index).c_str(), it->getRenderTimeNs());
3255 ++index;
3256 }
3257 return index;
3258 }
3259
onReleaseOutputBuffer(const sp<AMessage> & msg)3260 status_t MediaCodec::onReleaseOutputBuffer(const sp<AMessage> &msg) {
3261 size_t index;
3262 CHECK(msg->findSize("index", &index));
3263
3264 int32_t render;
3265 if (!msg->findInt32("render", &render)) {
3266 render = 0;
3267 }
3268
3269 if (!isExecuting()) {
3270 return -EINVAL;
3271 }
3272
3273 if (index >= mPortBuffers[kPortIndexOutput].size()) {
3274 return -ERANGE;
3275 }
3276
3277 BufferInfo *info = &mPortBuffers[kPortIndexOutput][index];
3278
3279 if (info->mData == nullptr || !info->mOwnedByClient) {
3280 return -EACCES;
3281 }
3282
3283 // synchronization boundary for getBufferAndFormat
3284 sp<MediaCodecBuffer> buffer;
3285 {
3286 Mutex::Autolock al(mBufferLock);
3287 info->mOwnedByClient = false;
3288 buffer = info->mData;
3289 info->mData.clear();
3290 }
3291
3292 if (render && buffer->size() != 0) {
3293 int64_t mediaTimeUs = -1;
3294 buffer->meta()->findInt64("timeUs", &mediaTimeUs);
3295
3296 int64_t renderTimeNs = 0;
3297 if (!msg->findInt64("timestampNs", &renderTimeNs)) {
3298 // use media timestamp if client did not request a specific render timestamp
3299 ALOGV("using buffer PTS of %lld", (long long)mediaTimeUs);
3300 renderTimeNs = mediaTimeUs * 1000;
3301 }
3302
3303 if (mSoftRenderer != NULL) {
3304 std::list<FrameRenderTracker::Info> doneFrames = mSoftRenderer->render(
3305 buffer->data(), buffer->size(), mediaTimeUs, renderTimeNs,
3306 mPortBuffers[kPortIndexOutput].size(), buffer->format());
3307
3308 // if we are running, notify rendered frames
3309 if (!doneFrames.empty() && mState == STARTED && mOnFrameRenderedNotification != NULL) {
3310 sp<AMessage> notify = mOnFrameRenderedNotification->dup();
3311 sp<AMessage> data = new AMessage;
3312 if (CreateFramesRenderedMessage(doneFrames, data)) {
3313 notify->setMessage("data", data);
3314 notify->post();
3315 }
3316 }
3317 }
3318 mBufferChannel->renderOutputBuffer(buffer, renderTimeNs);
3319 } else {
3320 mBufferChannel->discardBuffer(buffer);
3321 }
3322
3323 return OK;
3324 }
3325
dequeuePortBuffer(int32_t portIndex)3326 ssize_t MediaCodec::dequeuePortBuffer(int32_t portIndex) {
3327 CHECK(portIndex == kPortIndexInput || portIndex == kPortIndexOutput);
3328
3329 List<size_t> *availBuffers = &mAvailPortBuffers[portIndex];
3330
3331 if (availBuffers->empty()) {
3332 return -EAGAIN;
3333 }
3334
3335 size_t index = *availBuffers->begin();
3336 availBuffers->erase(availBuffers->begin());
3337
3338 BufferInfo *info = &mPortBuffers[portIndex][index];
3339 CHECK(!info->mOwnedByClient);
3340 {
3341 Mutex::Autolock al(mBufferLock);
3342 info->mOwnedByClient = true;
3343
3344 // set image-data
3345 if (info->mData->format() != NULL) {
3346 sp<ABuffer> imageData;
3347 if (info->mData->format()->findBuffer("image-data", &imageData)) {
3348 info->mData->meta()->setBuffer("image-data", imageData);
3349 }
3350 int32_t left, top, right, bottom;
3351 if (info->mData->format()->findRect("crop", &left, &top, &right, &bottom)) {
3352 info->mData->meta()->setRect("crop-rect", left, top, right, bottom);
3353 }
3354 }
3355 }
3356
3357 return index;
3358 }
3359
connectToSurface(const sp<Surface> & surface)3360 status_t MediaCodec::connectToSurface(const sp<Surface> &surface) {
3361 status_t err = OK;
3362 if (surface != NULL) {
3363 uint64_t oldId, newId;
3364 if (mSurface != NULL
3365 && surface->getUniqueId(&newId) == NO_ERROR
3366 && mSurface->getUniqueId(&oldId) == NO_ERROR
3367 && newId == oldId) {
3368 ALOGI("[%s] connecting to the same surface. Nothing to do.", mComponentName.c_str());
3369 return ALREADY_EXISTS;
3370 }
3371
3372 err = nativeWindowConnect(surface.get(), "connectToSurface");
3373 if (err == OK) {
3374 // Require a fresh set of buffers after each connect by using a unique generation
3375 // number. Rely on the fact that max supported process id by Linux is 2^22.
3376 // PID is never 0 so we don't have to worry that we use the default generation of 0.
3377 // TODO: come up with a unique scheme if other producers also set the generation number.
3378 static uint32_t mSurfaceGeneration = 0;
3379 uint32_t generation = (getpid() << 10) | (++mSurfaceGeneration & ((1 << 10) - 1));
3380 surface->setGenerationNumber(generation);
3381 ALOGI("[%s] setting surface generation to %u", mComponentName.c_str(), generation);
3382
3383 // HACK: clear any free buffers. Remove when connect will automatically do this.
3384 // This is needed as the consumer may be holding onto stale frames that it can reattach
3385 // to this surface after disconnect/connect, and those free frames would inherit the new
3386 // generation number. Disconnecting after setting a unique generation prevents this.
3387 nativeWindowDisconnect(surface.get(), "connectToSurface(reconnect)");
3388 err = nativeWindowConnect(surface.get(), "connectToSurface(reconnect)");
3389 }
3390
3391 if (err != OK) {
3392 ALOGE("nativeWindowConnect returned an error: %s (%d)", strerror(-err), err);
3393 }
3394 }
3395 // do not return ALREADY_EXISTS unless surfaces are the same
3396 return err == ALREADY_EXISTS ? BAD_VALUE : err;
3397 }
3398
disconnectFromSurface()3399 status_t MediaCodec::disconnectFromSurface() {
3400 status_t err = OK;
3401 if (mSurface != NULL) {
3402 // Resetting generation is not technically needed, but there is no need to keep it either
3403 mSurface->setGenerationNumber(0);
3404 err = nativeWindowDisconnect(mSurface.get(), "disconnectFromSurface");
3405 if (err != OK) {
3406 ALOGW("nativeWindowDisconnect returned an error: %s (%d)", strerror(-err), err);
3407 }
3408 // assume disconnected even on error
3409 mSurface.clear();
3410 }
3411 return err;
3412 }
3413
handleSetSurface(const sp<Surface> & surface)3414 status_t MediaCodec::handleSetSurface(const sp<Surface> &surface) {
3415 status_t err = OK;
3416 if (mSurface != NULL) {
3417 (void)disconnectFromSurface();
3418 }
3419 if (surface != NULL) {
3420 err = connectToSurface(surface);
3421 if (err == OK) {
3422 mSurface = surface;
3423 }
3424 }
3425 return err;
3426 }
3427
onInputBufferAvailable()3428 void MediaCodec::onInputBufferAvailable() {
3429 int32_t index;
3430 while ((index = dequeuePortBuffer(kPortIndexInput)) >= 0) {
3431 sp<AMessage> msg = mCallback->dup();
3432 msg->setInt32("callbackID", CB_INPUT_AVAILABLE);
3433 msg->setInt32("index", index);
3434 msg->post();
3435 }
3436 }
3437
onOutputBufferAvailable()3438 void MediaCodec::onOutputBufferAvailable() {
3439 int32_t index;
3440 while ((index = dequeuePortBuffer(kPortIndexOutput)) >= 0) {
3441 const sp<MediaCodecBuffer> &buffer =
3442 mPortBuffers[kPortIndexOutput][index].mData;
3443 sp<AMessage> msg = mCallback->dup();
3444 msg->setInt32("callbackID", CB_OUTPUT_AVAILABLE);
3445 msg->setInt32("index", index);
3446 msg->setSize("offset", buffer->offset());
3447 msg->setSize("size", buffer->size());
3448
3449 int64_t timeUs;
3450 CHECK(buffer->meta()->findInt64("timeUs", &timeUs));
3451
3452 msg->setInt64("timeUs", timeUs);
3453
3454 statsBufferReceived(timeUs);
3455
3456 int32_t flags;
3457 CHECK(buffer->meta()->findInt32("flags", &flags));
3458
3459 msg->setInt32("flags", flags);
3460
3461 msg->post();
3462 }
3463 }
3464
onError(status_t err,int32_t actionCode,const char * detail)3465 void MediaCodec::onError(status_t err, int32_t actionCode, const char *detail) {
3466 if (mCallback != NULL) {
3467 sp<AMessage> msg = mCallback->dup();
3468 msg->setInt32("callbackID", CB_ERROR);
3469 msg->setInt32("err", err);
3470 msg->setInt32("actionCode", actionCode);
3471
3472 if (detail != NULL) {
3473 msg->setString("detail", detail);
3474 }
3475
3476 msg->post();
3477 }
3478 }
3479
onOutputFormatChanged()3480 void MediaCodec::onOutputFormatChanged() {
3481 if (mCallback != NULL) {
3482 sp<AMessage> msg = mCallback->dup();
3483 msg->setInt32("callbackID", CB_OUTPUT_FORMAT_CHANGED);
3484 msg->setMessage("format", mOutputFormat);
3485 msg->post();
3486 }
3487 }
3488
postActivityNotificationIfPossible()3489 void MediaCodec::postActivityNotificationIfPossible() {
3490 if (mActivityNotify == NULL) {
3491 return;
3492 }
3493
3494 bool isErrorOrOutputChanged =
3495 (mFlags & (kFlagStickyError
3496 | kFlagOutputBuffersChanged
3497 | kFlagOutputFormatChanged));
3498
3499 if (isErrorOrOutputChanged
3500 || !mAvailPortBuffers[kPortIndexInput].empty()
3501 || !mAvailPortBuffers[kPortIndexOutput].empty()) {
3502 mActivityNotify->setInt32("input-buffers",
3503 mAvailPortBuffers[kPortIndexInput].size());
3504
3505 if (isErrorOrOutputChanged) {
3506 // we want consumer to dequeue as many times as it can
3507 mActivityNotify->setInt32("output-buffers", INT32_MAX);
3508 } else {
3509 mActivityNotify->setInt32("output-buffers",
3510 mAvailPortBuffers[kPortIndexOutput].size());
3511 }
3512 mActivityNotify->post();
3513 mActivityNotify.clear();
3514 }
3515 }
3516
setParameters(const sp<AMessage> & params)3517 status_t MediaCodec::setParameters(const sp<AMessage> ¶ms) {
3518 sp<AMessage> msg = new AMessage(kWhatSetParameters, this);
3519 msg->setMessage("params", params);
3520
3521 sp<AMessage> response;
3522 return PostAndAwaitResponse(msg, &response);
3523 }
3524
onSetParameters(const sp<AMessage> & params)3525 status_t MediaCodec::onSetParameters(const sp<AMessage> ¶ms) {
3526 mCodec->signalSetParameters(params);
3527
3528 return OK;
3529 }
3530
amendOutputFormatWithCodecSpecificData(const sp<MediaCodecBuffer> & buffer)3531 status_t MediaCodec::amendOutputFormatWithCodecSpecificData(
3532 const sp<MediaCodecBuffer> &buffer) {
3533 AString mime;
3534 CHECK(mOutputFormat->findString("mime", &mime));
3535
3536 if (!strcasecmp(mime.c_str(), MEDIA_MIMETYPE_VIDEO_AVC)) {
3537 // Codec specific data should be SPS and PPS in a single buffer,
3538 // each prefixed by a startcode (0x00 0x00 0x00 0x01).
3539 // We separate the two and put them into the output format
3540 // under the keys "csd-0" and "csd-1".
3541
3542 unsigned csdIndex = 0;
3543
3544 const uint8_t *data = buffer->data();
3545 size_t size = buffer->size();
3546
3547 const uint8_t *nalStart;
3548 size_t nalSize;
3549 while (getNextNALUnit(&data, &size, &nalStart, &nalSize, true) == OK) {
3550 sp<ABuffer> csd = new ABuffer(nalSize + 4);
3551 memcpy(csd->data(), "\x00\x00\x00\x01", 4);
3552 memcpy(csd->data() + 4, nalStart, nalSize);
3553
3554 mOutputFormat->setBuffer(
3555 AStringPrintf("csd-%u", csdIndex).c_str(), csd);
3556
3557 ++csdIndex;
3558 }
3559
3560 if (csdIndex != 2) {
3561 return ERROR_MALFORMED;
3562 }
3563 } else {
3564 // For everything else we just stash the codec specific data into
3565 // the output format as a single piece of csd under "csd-0".
3566 sp<ABuffer> csd = new ABuffer(buffer->size());
3567 memcpy(csd->data(), buffer->data(), buffer->size());
3568 csd->setRange(0, buffer->size());
3569 mOutputFormat->setBuffer("csd-0", csd);
3570 }
3571
3572 return OK;
3573 }
3574
updateBatteryStat()3575 void MediaCodec::updateBatteryStat() {
3576 if (!mIsVideo) {
3577 return;
3578 }
3579
3580 if (mState == CONFIGURED && !mBatteryStatNotified) {
3581 BatteryNotifier::getInstance().noteStartVideo(mUid);
3582 mBatteryStatNotified = true;
3583 } else if (mState == UNINITIALIZED && mBatteryStatNotified) {
3584 BatteryNotifier::getInstance().noteStopVideo(mUid);
3585 mBatteryStatNotified = false;
3586 }
3587 }
3588
stateString(State state)3589 std::string MediaCodec::stateString(State state) {
3590 const char *rval = NULL;
3591 char rawbuffer[16]; // room for "%d"
3592
3593 switch (state) {
3594 case UNINITIALIZED: rval = "UNINITIALIZED"; break;
3595 case INITIALIZING: rval = "INITIALIZING"; break;
3596 case INITIALIZED: rval = "INITIALIZED"; break;
3597 case CONFIGURING: rval = "CONFIGURING"; break;
3598 case CONFIGURED: rval = "CONFIGURED"; break;
3599 case STARTING: rval = "STARTING"; break;
3600 case STARTED: rval = "STARTED"; break;
3601 case FLUSHING: rval = "FLUSHING"; break;
3602 case FLUSHED: rval = "FLUSHED"; break;
3603 case STOPPING: rval = "STOPPING"; break;
3604 case RELEASING: rval = "RELEASING"; break;
3605 default:
3606 snprintf(rawbuffer, sizeof(rawbuffer), "%d", state);
3607 rval = rawbuffer;
3608 break;
3609 }
3610 return rval;
3611 }
3612
3613 } // namespace android
3614