1 // Copyright (C) 2019 The Android Open Source Project
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14
15 #include "host-common/MediaH264DecoderVideoToolBox.h"
16
17 #include "host-common/H264NaluParser.h"
18
19 #include <VideoToolbox/VideoToolbox.h>
20
21 #include <cstdint>
22 #include <string>
23 #include <vector>
24
25 #include <stdio.h>
26 #include <string.h>
27
28 #ifndef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder
29 #define kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder CFSTR("RequireHardwareAcceleratedVideoDecoder")
30 #endif
31
32 #define MEDIA_H264_DEBUG 0
33
34 #if MEDIA_H264_DEBUG
35 #define H264_DPRINT(fmt,...) fprintf(stderr, "h264-videotoolbox-dec: %s:%d " fmt "\n", __func__, __LINE__, ##__VA_ARGS__);
36 #else
37 #define H264_DPRINT(fmt,...)
38 #endif
39
40
41 namespace android {
42 namespace emulation {
43
44 using InitContextParam = H264PingInfoParser::InitContextParam;
45 using DecodeFrameParam = H264PingInfoParser::DecodeFrameParam;
46 using ResetParam = H264PingInfoParser::ResetParam;
47 using GetImageParam = H264PingInfoParser::GetImageParam;
48 using H264NaluType = H264NaluParser::H264NaluType;
49
MediaH264DecoderVideoToolBox(uint64_t id,H264PingInfoParser parser)50 MediaH264DecoderVideoToolBox::MediaH264DecoderVideoToolBox(
51 uint64_t id,
52 H264PingInfoParser parser)
53 : mId(id), mParser(parser) {
54 H264_DPRINT("created MediaH264DecoderVideoToolBox %p", this);
55 }
56
clone()57 MediaH264DecoderPlugin* MediaH264DecoderVideoToolBox::clone() {
58 return new MediaH264DecoderVideoToolBox(mId, mParser);
59 }
60
~MediaH264DecoderVideoToolBox()61 MediaH264DecoderVideoToolBox::~MediaH264DecoderVideoToolBox() {
62 destroyH264Context();
63 }
64
65 // static
videoToolboxDecompressCallback(void * opaque,void * sourceFrameRefCon,OSStatus status,VTDecodeInfoFlags flags,CVImageBufferRef image_buffer,CMTime pts,CMTime duration)66 void MediaH264DecoderVideoToolBox::videoToolboxDecompressCallback(void* opaque,
67 void* sourceFrameRefCon,
68 OSStatus status,
69 VTDecodeInfoFlags flags,
70 CVImageBufferRef image_buffer,
71 CMTime pts,
72 CMTime duration) {
73 H264_DPRINT("%s", __func__);
74 auto ptr = static_cast<MediaH264DecoderVideoToolBox*>(opaque);
75
76 if (ptr->mDecodedFrame) {
77 CVPixelBufferRelease(ptr->mDecodedFrame);
78 ptr->mDecodedFrame = nullptr;
79 }
80
81 if (!image_buffer) {
82 H264_DPRINT("%s: output image buffer is null", __func__);
83 return;
84 }
85
86 ptr->mOutputPts = pts.value;
87 ptr->mDecodedFrame = CVPixelBufferRetain(image_buffer);
88 // Image is ready to be comsumed
89 ptr->copyFrame();
90 ptr->mImageReady = true;
91 H264_DPRINT("Got decoded frame");
92 }
93
94 // static
createOutputBufferAttributes(int width,int height,OSType pix_fmt)95 CFDictionaryRef MediaH264DecoderVideoToolBox::createOutputBufferAttributes(int width,
96 int height,
97 OSType pix_fmt) {
98 CFMutableDictionaryRef buffer_attributes;
99 CFMutableDictionaryRef io_surface_properties;
100 CFNumberRef cv_pix_fmt;
101 CFNumberRef w;
102 CFNumberRef h;
103
104 w = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &width);
105 h = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &height);
106 cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pix_fmt);
107
108 buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
109 4,
110 &kCFTypeDictionaryKeyCallBacks,
111 &kCFTypeDictionaryValueCallBacks);
112 io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault,
113 0,
114 &kCFTypeDictionaryKeyCallBacks,
115 &kCFTypeDictionaryValueCallBacks);
116
117 if (pix_fmt) {
118 CFDictionarySetValue(buffer_attributes, kCVPixelBufferPixelFormatTypeKey, cv_pix_fmt);
119 }
120 CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfacePropertiesKey, io_surface_properties);
121 CFDictionarySetValue(buffer_attributes, kCVPixelBufferWidthKey, w);
122 CFDictionarySetValue(buffer_attributes, kCVPixelBufferHeightKey, h);
123 // Not sure if this will work becuase we are passing the pixel buffer back into the guest
124 CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey, kCFBooleanTrue);
125
126 CFRelease(io_surface_properties);
127 CFRelease(cv_pix_fmt);
128 CFRelease(w);
129 CFRelease(h);
130
131 return buffer_attributes;
132 }
133
134 // static
createSampleBuffer(CMFormatDescriptionRef fmtDesc,void * buffer,size_t sz)135 CMSampleBufferRef MediaH264DecoderVideoToolBox::createSampleBuffer(CMFormatDescriptionRef fmtDesc,
136 void* buffer,
137 size_t sz) {
138 OSStatus status;
139 CMBlockBufferRef blockBuf = nullptr;
140 CMSampleBufferRef sampleBuf = nullptr;
141
142 status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, // structureAllocator
143 buffer, // memoryBlock
144 sz, // blockLength
145 kCFAllocatorNull, // blockAllocator
146 NULL, // customBlockSource
147 0, // offsetToData
148 sz, // dataLength
149 0, // flags
150 &blockBuf);
151
152 if (!status) {
153 status = CMSampleBufferCreate(kCFAllocatorDefault, // allocator
154 blockBuf, // dataBuffer
155 TRUE, // dataReady
156 0, // makeDataReadyCallback
157 0, // makeDataReadyRefCon
158 fmtDesc, // formatDescription
159 1, // numSamples
160 0, // numSampleTimingEntries
161 NULL, // sampleTimingArray
162 0, // numSampleSizeEntries
163 NULL, // sampleSizeArray
164 &sampleBuf);
165 }
166
167 if (blockBuf) {
168 CFRelease(blockBuf);
169 }
170
171 return sampleBuf;
172 }
173
174 // static
toNativePixelFormat(PixelFormat pixFmt)175 OSType MediaH264DecoderVideoToolBox::toNativePixelFormat(PixelFormat pixFmt) {
176 switch (pixFmt) {
177 case PixelFormat::YUV420P:
178 return kCVPixelFormatType_420YpCbCr8Planar;
179 case PixelFormat::UYVY422:
180 return kCVPixelFormatType_422YpCbCr8;
181 case PixelFormat::BGRA8888:
182 return kCVPixelFormatType_32BGRA;
183 default:
184 H264_DPRINT("Unsupported VideoToolbox pixel format");
185 return '0000';
186 }
187 }
188
189 // static
getReturnAddress(void * ptr)190 void* MediaH264DecoderVideoToolBox::getReturnAddress(void* ptr) {
191 uint8_t* xptr = (uint8_t*)ptr;
192 void* pint = (void*)(xptr + 256);
193 return pint;
194 }
195
createCMFormatDescription()196 void MediaH264DecoderVideoToolBox::createCMFormatDescription() {
197 uint8_t* parameterSets[2] = {mSPS.data(), mPPS.data()};
198 size_t parameterSetSizes[2] = {mSPS.size(), mPPS.size()};
199
200 if (mCmFmtDesc) {
201 CFRelease(mCmFmtDesc);
202 mCmFmtDesc = nullptr;
203 }
204
205 OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets(
206 kCFAllocatorDefault,
207 2,
208 (const uint8_t *const*)parameterSets,
209 parameterSetSizes,
210 4,
211 &mCmFmtDesc);
212
213 if (status == noErr) {
214 H264_DPRINT("Created CMFormatDescription from SPS/PPS sets");
215 } else {
216 H264_DPRINT("Unable to create CMFormatDescription (%d)\n", (int)status);
217 }
218 }
219
createVTDecoderConfig()220 CFDataRef MediaH264DecoderVideoToolBox::createVTDecoderConfig() {
221 CFDataRef data = nullptr;
222 return data;
223 }
224
initH264Context(void * ptr)225 void MediaH264DecoderVideoToolBox::initH264Context(void* ptr) {
226 InitContextParam param{};
227 mParser.parseInitContextParams(ptr, param);
228 initH264ContextInternal(param.width, param.height, param.outputWidth,
229 param.outputHeight, param.outputPixelFormat);
230 }
231
initH264ContextInternal(unsigned int width,unsigned int height,unsigned int outWidth,unsigned int outHeight,PixelFormat outPixFmt)232 void MediaH264DecoderVideoToolBox::initH264ContextInternal(
233 unsigned int width,
234 unsigned int height,
235 unsigned int outWidth,
236 unsigned int outHeight,
237 PixelFormat outPixFmt) {
238 H264_DPRINT("%s(w=%u h=%u out_w=%u out_h=%u pixfmt=%u)",
239 __func__, width, height, outWidth, outHeight, (uint8_t)outPixFmt);
240 mWidth = width;
241 mHeight = height;
242 mOutputWidth = outWidth;
243 mOutputHeight = outHeight;
244 mOutPixFmt = outPixFmt;
245 mOutBufferSize = outWidth * outHeight * 3 / 2;
246 }
247
reset(void * ptr)248 void MediaH264DecoderVideoToolBox::reset(void* ptr) {
249 destroyH264Context();
250 ResetParam param{};
251 mParser.parseResetParams(ptr, param);
252 initH264ContextInternal(param.width, param.height, param.outputWidth,
253 param.outputHeight, param.outputPixelFormat);
254 }
255
destroyH264Context()256 void MediaH264DecoderVideoToolBox::destroyH264Context() {
257 H264_DPRINT("%s", __func__);
258 if (mDecoderSession) {
259 VTDecompressionSessionInvalidate(mDecoderSession);
260 CFRelease(mDecoderSession);
261 mDecoderSession = nullptr;
262 }
263 if (mCmFmtDesc) {
264 CFRelease(mCmFmtDesc);
265 mCmFmtDesc = nullptr;
266 }
267 if (mDecodedFrame) {
268 CVPixelBufferRelease(mDecodedFrame);
269 mDecodedFrame = nullptr;
270 }
271 }
272
dumpBytes(const uint8_t * img,size_t szBytes,bool all=false)273 static void dumpBytes(const uint8_t* img, size_t szBytes, bool all = false) {
274 #if MEDIA_H264_DEBUG
275 printf("data=");
276 size_t numBytes = szBytes;
277 if (!all) {
278 numBytes = 32;
279 }
280
281 for (size_t i = 0; i < (numBytes > szBytes ? szBytes : numBytes); ++i) {
282 if (i % 8 == 0) {
283 printf("\n");
284 }
285 printf("0x%02x ", img[i]);
286 }
287 printf("\n");
288 #endif
289 }
290
decodeFrame(void * ptr)291 void MediaH264DecoderVideoToolBox::decodeFrame(void* ptr) {
292 DecodeFrameParam param{};
293 mParser.parseDecodeFrameParams(ptr, param);
294
295 const uint8_t* frame = param.pData;
296 size_t szBytes = param.size;
297 uint64_t pts = param.pts;
298
299 decodeFrameInternal(param.pConsumedBytes, param.pDecoderErrorCode, frame, szBytes, pts, 0);
300 }
301
oneShotDecode(std::vector<uint8_t> & data,uint64_t pts)302 void MediaH264DecoderVideoToolBox::oneShotDecode(std::vector<uint8_t> & data, uint64_t pts) {
303 //TODO: check if data has more than one Nalu
304 decodeFrameInternal(nullptr, nullptr, data.data(), data.size(), pts, 0);
305 }
306
decodeFrameInternal(size_t * pRetSzBytes,int32_t * pRetErr,const uint8_t * frame,size_t szBytes,uint64_t pts,size_t consumedSzBytes)307 void MediaH264DecoderVideoToolBox::decodeFrameInternal(size_t* pRetSzBytes, int32_t* pRetErr, const uint8_t* frame, size_t szBytes, uint64_t pts, size_t consumedSzBytes) {
308 // IMPORTANT: There is an assumption that each |frame| we get from the guest are contain
309 // complete NALUs. Usually, an H.264 bitstream would be continuously fed to us, but in this case,
310 // it seems that the Android frameworks passes us complete NALUs, and may be more than one NALU at
311 // a time.
312 //
313 // Let's only process one NALU per decodeFrame() call because, as soon as we receive a PPS NALU,
314 // we need to let the OMX plugin know to reset it's state because we are also recreating our
315 // decoder context.
316 H264_DPRINT("%s(frame=%p, sz=%zu)", __func__, frame, szBytes);
317 Err h264Err = Err::NoErr;
318
319 const uint8_t* currentNalu = H264NaluParser::getNextStartCodeHeader(frame, szBytes);
320 if (currentNalu == nullptr) {
321 H264_DPRINT("No start code header found in this frame");
322 h264Err = Err::NoDecodedFrame;
323 // TODO: return the error code and num bytes processed, szBytes.
324 if (pRetSzBytes) *pRetSzBytes = szBytes;
325 if (pRetErr) *pRetErr = (int32_t)h264Err;
326 return;
327 }
328 const uint8_t* nextNalu = nullptr;
329 size_t remaining = szBytes - (currentNalu - frame);
330
331 // Figure out the size of |currentNalu|.
332 size_t currentNaluSize = remaining;
333 // 3 is the minimum size of the start code header (3 or 4 bytes).
334 dumpBytes(currentNalu, currentNaluSize);
335 nextNalu = H264NaluParser::getNextStartCodeHeader(currentNalu + 3, remaining - 3);
336 if (nextNalu != nullptr) {
337 currentNaluSize = nextNalu - currentNalu;
338 }
339
340 // |data| is currentNalu, but with the start code header discarded.
341 uint8_t* data = nullptr;
342 H264NaluType naluType = H264NaluParser::getFrameNaluType(currentNalu, currentNaluSize, &data);
343 size_t dataSize = currentNaluSize - (data - currentNalu);
344 const std::string naluTypeStr = H264NaluParser::naluTypeToString(naluType);
345 H264_DPRINT("Got frame type=%u (%s)", (uint8_t)naluType, naluTypeStr.c_str());
346
347 // We can't do anything until we set up a CMFormatDescription from a set of SPS and PPS NALUs.
348 // So just discard the NALU.
349 if (naluType != H264NaluType::SPS && naluType != H264NaluType::PPS &&
350 mCmFmtDesc == nullptr) {
351 H264_DPRINT("CMFormatDescription not set up yet. Need SPS/PPS frames.");
352 h264Err = Err::NALUIgnored;
353 if (pRetSzBytes) *pRetSzBytes = currentNaluSize;
354 if (pRetErr) *pRetErr = (int32_t)h264Err;
355 return;
356 }
357
358 switch (naluType) {
359 case H264NaluType::SPS:
360 // We should be getting a PPS frame on the next decodeFrame(). Once we have
361 // both sps and pps, we can create/recreate the decoder session.
362 // Don't include the start code header when we copy the sps/pps.
363 mSPS.assign(data, data + dataSize);
364 if (!mIsLoadingFromSnapshot) {
365 mSnapshotState = SnapshotState{};
366 std::vector<uint8_t> vec;
367 vec.assign(currentNalu, currentNalu + currentNaluSize);
368 mSnapshotState.saveSps(vec);
369 }
370 break;
371 case H264NaluType::PPS:
372 mPPS.assign(data, data + dataSize);
373 createCMFormatDescription();
374 // TODO: We will need to recreate the decompression session whenever we get a
375 // resolution change.
376 if (mDecoderSession != nullptr) {
377 H264_DPRINT("Decoder session is restarting");
378 //h264Err = Err::DecoderRestarted;
379 }
380 if (!mIsLoadingFromSnapshot) {
381 std::vector<uint8_t> vec;
382 vec.assign(currentNalu, currentNalu + currentNaluSize);
383 mSnapshotState.savePps(vec);
384 mSnapshotState.savedPackets.clear();
385 }
386 recreateDecompressionSession();
387 break;
388 case H264NaluType::SEI:
389 // dumpBytes(nextNalu, remaining, true);
390 // In some cases, after the SPS and PPS NALUs are emitted, we'll get a frame that
391 // contains both an SEI NALU and a CodedSliceIDR NALU.
392 handleSEIFrame(currentNalu, currentNaluSize);
393 break;
394 case H264NaluType::CodedSliceIDR:
395 handleIDRFrame(currentNalu, currentNaluSize, pts);
396 if (!mIsLoadingFromSnapshot) {
397 H264_DPRINT("disacarding previously saved frames %d", (int)mSnapshotState.savedPackets.size());
398 mSnapshotState.savedPackets.clear();
399 mSnapshotState.savePacket(currentNalu, currentNaluSize, pts);
400 }
401 break;
402 case H264NaluType::CodedSliceNonIDR:
403 handleNonIDRFrame(currentNalu, currentNaluSize, pts);
404 if (!mIsLoadingFromSnapshot) {
405 mSnapshotState.savePacket(currentNalu, currentNaluSize, pts);
406 }
407 break;
408 default:
409 H264_DPRINT("Support for nalu_type=%u not implemented", (uint8_t)naluType);
410 break;
411 }
412
413 remaining -= currentNaluSize;
414 currentNalu = nextNalu;
415
416 // return two things: the error code and the number of bytes we processed.
417 if (pRetSzBytes) *pRetSzBytes = currentNaluSize + consumedSzBytes;
418 if (pRetErr) *pRetErr = (int32_t)h264Err;
419
420 // disable recursive decoding due to the possibility of session creation failure
421 // keep it simple for now
422 //if (currentNalu) {
423 // decodeFrameInternal(ptr, currentNalu, remaining, pts, consumedSzBytes + currentNaluSize);
424 //}
425 }
426
handleIDRFrame(const uint8_t * ptr,size_t szBytes,uint64_t pts)427 void MediaH264DecoderVideoToolBox::handleIDRFrame(const uint8_t* ptr, size_t szBytes, uint64_t pts) {
428 H264_DPRINT("Got IDR frame (sz=%zu)", szBytes);
429 uint8_t* fptr = const_cast<uint8_t*>(ptr);
430
431 // We can assume fptr has a valid start code header because it has already
432 // gone through validation in H264NaluParser.
433 uint8_t startHeaderSz = fptr[2] == 1 ? 3 : 4;
434 uint32_t dataSz = szBytes - startHeaderSz;
435 std::unique_ptr<uint8_t> idr(new uint8_t[dataSz + 4]);
436 uint32_t dataSzNl = htonl(dataSz);
437
438 // AVCC format requires us to replace the start code header on this NALU
439 // with the size of the data. Start code is either 0x000001 or 0x00000001.
440 // The size needs to be the first four bytes in network byte order.
441 memcpy(idr.get(), &dataSzNl, 4);
442 memcpy(idr.get() + 4, ptr + startHeaderSz, dataSz);
443
444 CMSampleBufferRef sampleBuf = nullptr;
445 sampleBuf = createSampleBuffer(mCmFmtDesc, (void*)idr.get(), dataSz + 4);
446 if (!sampleBuf) {
447 H264_DPRINT("%s: Failed to create CMSampleBufferRef", __func__);
448 return;
449 }
450
451 CMSampleBufferSetOutputPresentationTimeStamp(sampleBuf, CMTimeMake(pts, 1));
452
453 OSStatus status;
454 status = VTDecompressionSessionDecodeFrame(mDecoderSession,
455 sampleBuf,
456 0, // decodeFlags
457 NULL, // sourceFrameRefCon
458 0); // infoFlagsOut
459
460 if (status == noErr) {
461 // TODO: this call blocks until the frame has been decoded. Perhaps it will be
462 // more efficient to signal the guest when the frame is ready to be read instead.
463 status = VTDecompressionSessionWaitForAsynchronousFrames(mDecoderSession);
464 mIsInFlush = false;
465 } else {
466 H264_DPRINT("%s: Failed to decompress frame (err=%d)", __func__, status);
467 }
468
469 CFRelease(sampleBuf);
470 H264_DPRINT("Success decoding IDR frame");
471 }
472
handleNonIDRFrame(const uint8_t * ptr,size_t szBytes,uint64_t pts)473 void MediaH264DecoderVideoToolBox::handleNonIDRFrame(const uint8_t* ptr, size_t szBytes, uint64_t pts) {
474 // Same as handling an IDR frame
475 handleIDRFrame(ptr, szBytes, pts);
476 }
477
handleSEIFrame(const uint8_t * ptr,size_t szBytes)478 void MediaH264DecoderVideoToolBox::handleSEIFrame(const uint8_t* ptr, size_t szBytes) {
479 H264_DPRINT("NOT IMPLEMENTED");
480 }
481
flush(void * ptr)482 void MediaH264DecoderVideoToolBox::flush(void* ptr) {
483 H264_DPRINT("%s: NOT IMPLEMENTED", __func__);
484 mIsInFlush = true;
485 }
486
copyFrame()487 void MediaH264DecoderVideoToolBox::copyFrame() {
488 if (mIsLoadingFromSnapshot) return;
489
490 int imgWidth = CVPixelBufferGetWidth(mDecodedFrame);
491 int imgHeight = CVPixelBufferGetHeight(mDecodedFrame);
492 int imageSize = CVPixelBufferGetDataSize(mDecodedFrame);
493 int stride = CVPixelBufferGetBytesPerRow(mDecodedFrame);
494
495 mOutputWidth = imgWidth;
496 mOutputHeight = imgHeight;
497 mOutBufferSize = mOutputWidth * mOutputHeight * 3 / 2;
498
499 H264_DPRINT("copying size=%d dimension=[%dx%d] stride=%d", imageSize, imgWidth, imgHeight, stride);
500
501 // Copies the image data to the guest.
502 mSavedDecodedFrame.resize(imgWidth * imgHeight * 3 / 2);
503 uint8_t* dst = mSavedDecodedFrame.data();
504
505 CVPixelBufferLockBaseAddress(mDecodedFrame, kCVPixelBufferLock_ReadOnly);
506 if (CVPixelBufferIsPlanar(mDecodedFrame)) {
507 imageSize = 0; // add up the size from the planes
508 int planes = CVPixelBufferGetPlaneCount(mDecodedFrame);
509 for (int i = 0; i < planes; ++i) {
510 void* planeData = CVPixelBufferGetBaseAddressOfPlane(mDecodedFrame, i);
511 int linesize = CVPixelBufferGetBytesPerRowOfPlane(mDecodedFrame, i);
512 int planeWidth = CVPixelBufferGetWidthOfPlane(mDecodedFrame, i);
513 int planeHeight = CVPixelBufferGetHeightOfPlane(mDecodedFrame, i);
514 H264_DPRINT("plane=%d data=%p linesize=%d pwidth=%d pheight=%d", i, planeData, linesize, planeWidth, planeHeight);
515 // For kCVPixelFormatType_420YpCbCr8Planar, plane 0 is Y, UV planes are 1 and 2
516 if (planeWidth != imgWidth && planeWidth != imgWidth / 2) {
517 H264_DPRINT("ERROR: Unable to determine YUV420 plane type");
518 continue;
519 }
520
521 // Sometimes the buffer stride can be longer than the actual data width. This means that
522 // the extra bytes are just padding and need to be discarded.
523 if (linesize <= planeWidth) {
524 int sz = planeHeight * planeWidth;
525 imageSize += sz;
526 memcpy(dst, planeData, sz);
527 dst += sz;
528 } else {
529 // Need to copy line by line
530 int sz = planeWidth;
531 for (int j = 0; j < planeHeight; ++j) {
532 uint8_t* ptr = (uint8_t*)planeData;
533 ptr += linesize * j;
534 memcpy(dst, ptr, sz);
535 imageSize += sz;
536 dst += sz;
537 }
538 }
539 }
540 if (imageSize != mOutBufferSize) {
541 H264_DPRINT("ERROR: Total size of planes not same as guestSz (guestSz=%u, imageSize=%d)", mOutBufferSize, imageSize);
542 }
543 } else {
544 if (imageSize > mOutBufferSize) {
545 H264_DPRINT("Buffer size mismatch (guestSz=%u, imageSize=%d). Using guestSz instead.", mOutBufferSize, imageSize);
546 imageSize = mOutBufferSize;
547 }
548
549 // IMPORTANT: mDecodedFrame must be locked before accessing the contents with CPU
550 void* data = CVPixelBufferGetBaseAddress(mDecodedFrame);
551 memcpy(dst, data, imageSize);
552 }
553 CVPixelBufferUnlockBaseAddress(mDecodedFrame, kCVPixelBufferLock_ReadOnly);
554 }
555
getImage(void * ptr)556 void MediaH264DecoderVideoToolBox::getImage(void* ptr) {
557 // return parameters:
558 // 1) either size of the image (> 0) or error code (<= 0).
559 // 2) image width
560 // 3) image height
561 GetImageParam param{};
562 mParser.parseGetImageParams(ptr, param);
563
564 int* retErr = param.pDecoderErrorCode;
565 uint32_t* retWidth = param.pRetWidth;
566 uint32_t* retHeight = param.pRetHeight;
567 uint64_t* retPts = param.pRetPts;
568 uint32_t* retColorPrimaries = param.pRetColorPrimaries;
569 uint32_t* retColorRange = param.pRetColorRange;
570 uint32_t* retColorTransfer = param.pRetColorTransfer;
571 uint32_t* retColorSpace = param.pRetColorSpace;
572
573 if (!mDecodedFrame) {
574 H264_DPRINT("%s: frame is null", __func__);
575 *retErr = static_cast<int>(Err::NoDecodedFrame);
576 return;
577 }
578 if (!mImageReady) {
579 bool hasMoreFrames = false;
580 if (mIsInFlush) {
581 OSStatus status = noErr;
582 status = VTDecompressionSessionWaitForAsynchronousFrames(mDecoderSession);
583 if (status == noErr) {
584 hasMoreFrames = mImageReady;
585 if (hasMoreFrames) {
586 H264_DPRINT("%s: got frame in flush mode", __func__);
587 }
588 }
589 }
590
591 if (!hasMoreFrames) {
592 H264_DPRINT("%s: no new frame yet", __func__);
593 *retErr = static_cast<int>(Err::NoDecodedFrame);
594 return;
595 }
596 }
597
598 *retWidth = mOutputWidth;
599 *retHeight = mOutputHeight;
600
601 if (mParser.version() == 200 && param.hostColorBufferId >= 0) {
602 mRenderer.renderToHostColorBuffer(param.hostColorBufferId,
603 mOutputWidth, mOutputHeight,
604 mSavedDecodedFrame.data());
605 } else {
606 memcpy(param.pDecodedFrame, mSavedDecodedFrame.data(), mSavedDecodedFrame.size());;
607 }
608
609 *retErr = mSavedDecodedFrame.size();
610
611 *retPts = mOutputPts;
612
613 H264_DPRINT("Copying completed pts %lld", (long long)mOutputPts);
614 mImageReady = false;
615 }
616
recreateDecompressionSession()617 void MediaH264DecoderVideoToolBox::recreateDecompressionSession() {
618 if (mCmFmtDesc == nullptr) {
619 H264_DPRINT("CMFormatDescription not created. Need sps and pps NALUs.");
620 return;
621 }
622
623 // Create a new VideoToolbox decoder session if one already exists
624 if (mDecoderSession != nullptr) {
625 // TODO: Once we implement async frame readback, we'll need to flush all of the frames here and
626 // store them somewhere for the guest to read later.
627 VTDecompressionSessionInvalidate(mDecoderSession);
628 CFRelease(mDecoderSession);
629 mDecoderSession = nullptr;
630 if (mDecodedFrame) {
631 CVPixelBufferRelease(mDecodedFrame);
632 mDecodedFrame = nullptr;
633 }
634 }
635
636 CMVideoCodecType codecType = kCMVideoCodecType_H264;
637 CFMutableDictionaryRef decoder_spec = CFDictionaryCreateMutable(kCFAllocatorDefault,
638 0,
639 &kCFTypeDictionaryKeyCallBacks,
640 &kCFTypeDictionaryValueCallBacks);
641 CFDictionarySetValue(decoder_spec,
642 kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder,
643 kCFBooleanTrue);
644
645 CFDictionaryRef bufAttr = createOutputBufferAttributes(mOutputWidth,
646 mOutputHeight,
647 toNativePixelFormat(mOutPixFmt));
648
649 VTDecompressionOutputCallbackRecord decoderCb;
650 decoderCb.decompressionOutputCallback = videoToolboxDecompressCallback;
651 decoderCb.decompressionOutputRefCon = this;
652
653 OSStatus status;
654 status = VTDecompressionSessionCreate(NULL, // allocator
655 mCmFmtDesc, // videoFormatDescription
656 decoder_spec, // videoDecoderSpecification
657 bufAttr, // destinationImageBufferAttributes
658 &decoderCb, // outputCallback
659 &mDecoderSession); // decompressionSessionOut
660
661 if (decoder_spec) {
662 CFRelease(decoder_spec);
663 }
664 if (bufAttr) {
665 CFRelease(bufAttr);
666 }
667
668 mIsInFlush = false;
669 mState = DecoderState::BAD_STATE;
670 switch (status) {
671 case kVTVideoDecoderNotAvailableNowErr:
672 H264_DPRINT("VideoToolbox session not available");
673 return;
674 case kVTVideoDecoderUnsupportedDataFormatErr:
675 H264_DPRINT("VideoToolbox does not support this format");
676 return;
677 case kVTVideoDecoderMalfunctionErr:
678 H264_DPRINT("VideoToolbox malfunction");
679 return;
680 case kVTVideoDecoderBadDataErr:
681 H264_DPRINT("VideoToolbox reported invalid data");
682 return;
683 case 0:
684 H264_DPRINT("VideoToolbox session created");
685 mState = DecoderState::GOOD_STATE;
686 return;
687 default:
688 H264_DPRINT("Unknown VideoToolbox session creation error %d", status);
689 return;
690 }
691 }
692
save(base::Stream * stream) const693 void MediaH264DecoderVideoToolBox::save(base::Stream* stream) const {
694 stream->putBe32(mParser.version());
695 stream->putBe32(mWidth);
696 stream->putBe32(mHeight);
697 stream->putBe32((int)mOutPixFmt);
698
699 const int hasContext = mDecoderSession != nullptr ? 1 : 0;
700 stream->putBe32(hasContext);
701
702 if (mImageReady) {
703 mSnapshotState.saveDecodedFrame(
704 mSavedDecodedFrame, mOutputWidth, mOutputHeight,
705 ColorAspects{}, mOutputPts);
706 } else {
707 mSnapshotState.savedDecodedFrame.data.clear();
708 }
709 H264_DPRINT("saving packets now %d",
710 (int)(mSnapshotState.savedPackets.size()));
711 mSnapshotState.save(stream);
712 }
713
load(base::Stream * stream)714 bool MediaH264DecoderVideoToolBox::load(base::Stream* stream) {
715 mIsLoadingFromSnapshot = true;
716 uint32_t version = stream->getBe32();
717 mParser = H264PingInfoParser{version};
718
719 mWidth = stream->getBe32();
720 mHeight = stream->getBe32();
721 mOutPixFmt = (PixelFormat)stream->getBe32();
722
723 const int hasContext = stream->getBe32();
724 if (hasContext) {
725 initH264ContextInternal(mWidth, mHeight, mWidth, mHeight, mOutPixFmt);
726 }
727 mSnapshotState.load(stream);
728 if (hasContext && mSnapshotState.sps.size() > 0) {
729 oneShotDecode(mSnapshotState.sps, 0);
730 if (mSnapshotState.pps.size() > 0) {
731 oneShotDecode(mSnapshotState.pps, 0);
732 if (mSnapshotState.savedPackets.size() > 0) {
733 for (int i = 0; i < mSnapshotState.savedPackets.size(); ++i) {
734 PacketInfo& pkt = mSnapshotState.savedPackets[i];
735 oneShotDecode(pkt.data, pkt.pts);
736 }
737 }
738 }
739 }
740
741 if (mSnapshotState.savedDecodedFrame.data.size() > 0) {
742 mSavedDecodedFrame = std::move(mSnapshotState.savedDecodedFrame.data);
743 mOutBufferSize = mSnapshotState.savedDecodedFrame.data.size();
744 mOutputWidth = mSnapshotState.savedDecodedFrame.width;
745 mOutputHeight = mSnapshotState.savedDecodedFrame.height;
746 mOutputPts = mSnapshotState.savedDecodedFrame.pts;
747 mImageReady = true;
748 } else {
749 mOutputWidth = mWidth;
750 mOutputHeight = mHeight;
751 mOutBufferSize = mOutputWidth * mOutputHeight * 3 / 2;
752 mImageReady = false;
753 }
754 mIsLoadingFromSnapshot = false;
755 return true;
756 }
757
758 } // namespace emulation
759 } // namespace android
760