1 // Copyright (C) 2019 The Android Open Source Project
2 //
3 // Licensed under the Apache License, Version 2.0 (the "License");
4 // you may not use this file except in compliance with the License.
5 // You may obtain a copy of the License at
6 //
7 // http://www.apache.org/licenses/LICENSE-2.0
8 //
9 // Unless required by applicable law or agreed to in writing, software
10 // distributed under the License is distributed on an "AS IS" BASIS,
11 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 // See the License for the specific language governing permissions and
13 // limitations under the License.
14 
15 #include "host-common/MediaH264DecoderVideoToolBox.h"
16 
17 #include "host-common/H264NaluParser.h"
18 
19 #include <VideoToolbox/VideoToolbox.h>
20 
21 #include <cstdint>
22 #include <string>
23 #include <vector>
24 
25 #include <stdio.h>
26 #include <string.h>
27 
28 #ifndef kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder
29 #define kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder CFSTR("RequireHardwareAcceleratedVideoDecoder")
30 #endif
31 
32 #define MEDIA_H264_DEBUG 0
33 
34 #if MEDIA_H264_DEBUG
35 #define H264_DPRINT(fmt,...) fprintf(stderr, "h264-videotoolbox-dec: %s:%d " fmt "\n", __func__, __LINE__, ##__VA_ARGS__);
36 #else
37 #define H264_DPRINT(fmt,...)
38 #endif
39 
40 
41 namespace android {
42 namespace emulation {
43 
44 using InitContextParam = H264PingInfoParser::InitContextParam;
45 using DecodeFrameParam = H264PingInfoParser::DecodeFrameParam;
46 using ResetParam = H264PingInfoParser::ResetParam;
47 using GetImageParam = H264PingInfoParser::GetImageParam;
48 using H264NaluType = H264NaluParser::H264NaluType;
49 
MediaH264DecoderVideoToolBox(uint64_t id,H264PingInfoParser parser)50 MediaH264DecoderVideoToolBox::MediaH264DecoderVideoToolBox(
51         uint64_t id,
52         H264PingInfoParser parser)
53     : mId(id), mParser(parser) {
54     H264_DPRINT("created MediaH264DecoderVideoToolBox %p", this);
55 }
56 
clone()57 MediaH264DecoderPlugin* MediaH264DecoderVideoToolBox::clone() {
58     return new MediaH264DecoderVideoToolBox(mId, mParser);
59 }
60 
~MediaH264DecoderVideoToolBox()61 MediaH264DecoderVideoToolBox::~MediaH264DecoderVideoToolBox() {
62     destroyH264Context();
63 }
64 
65 // static
videoToolboxDecompressCallback(void * opaque,void * sourceFrameRefCon,OSStatus status,VTDecodeInfoFlags flags,CVImageBufferRef image_buffer,CMTime pts,CMTime duration)66 void MediaH264DecoderVideoToolBox::videoToolboxDecompressCallback(void* opaque,
67                                                           void* sourceFrameRefCon,
68                                                           OSStatus status,
69                                                           VTDecodeInfoFlags flags,
70                                                           CVImageBufferRef image_buffer,
71                                                           CMTime pts,
72                                                           CMTime duration) {
73     H264_DPRINT("%s", __func__);
74     auto ptr = static_cast<MediaH264DecoderVideoToolBox*>(opaque);
75 
76     if (ptr->mDecodedFrame) {
77         CVPixelBufferRelease(ptr->mDecodedFrame);
78         ptr->mDecodedFrame = nullptr;
79     }
80 
81     if (!image_buffer) {
82         H264_DPRINT("%s: output image buffer is null", __func__);
83         return;
84     }
85 
86     ptr->mOutputPts = pts.value;
87     ptr->mDecodedFrame = CVPixelBufferRetain(image_buffer);
88     // Image is ready to be comsumed
89     ptr->copyFrame();
90     ptr->mImageReady = true;
91     H264_DPRINT("Got decoded frame");
92 }
93 
94 // static
createOutputBufferAttributes(int width,int height,OSType pix_fmt)95 CFDictionaryRef MediaH264DecoderVideoToolBox::createOutputBufferAttributes(int width,
96                                                                    int height,
97                                                                    OSType pix_fmt) {
98     CFMutableDictionaryRef buffer_attributes;
99     CFMutableDictionaryRef io_surface_properties;
100     CFNumberRef cv_pix_fmt;
101     CFNumberRef w;
102     CFNumberRef h;
103 
104     w = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &width);
105     h = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &height);
106     cv_pix_fmt = CFNumberCreate(kCFAllocatorDefault, kCFNumberSInt32Type, &pix_fmt);
107 
108     buffer_attributes = CFDictionaryCreateMutable(kCFAllocatorDefault,
109                                                   4,
110                                                   &kCFTypeDictionaryKeyCallBacks,
111                                                   &kCFTypeDictionaryValueCallBacks);
112     io_surface_properties = CFDictionaryCreateMutable(kCFAllocatorDefault,
113                                                       0,
114                                                       &kCFTypeDictionaryKeyCallBacks,
115                                                       &kCFTypeDictionaryValueCallBacks);
116 
117     if (pix_fmt) {
118         CFDictionarySetValue(buffer_attributes, kCVPixelBufferPixelFormatTypeKey, cv_pix_fmt);
119     }
120     CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfacePropertiesKey, io_surface_properties);
121     CFDictionarySetValue(buffer_attributes, kCVPixelBufferWidthKey, w);
122     CFDictionarySetValue(buffer_attributes, kCVPixelBufferHeightKey, h);
123     // Not sure if this will work becuase we are passing the pixel buffer back into the guest
124     CFDictionarySetValue(buffer_attributes, kCVPixelBufferIOSurfaceOpenGLTextureCompatibilityKey, kCFBooleanTrue);
125 
126     CFRelease(io_surface_properties);
127     CFRelease(cv_pix_fmt);
128     CFRelease(w);
129     CFRelease(h);
130 
131     return buffer_attributes;
132 }
133 
134 // static
createSampleBuffer(CMFormatDescriptionRef fmtDesc,void * buffer,size_t sz)135 CMSampleBufferRef MediaH264DecoderVideoToolBox::createSampleBuffer(CMFormatDescriptionRef fmtDesc,
136                                                            void* buffer,
137                                                            size_t sz) {
138     OSStatus status;
139     CMBlockBufferRef blockBuf = nullptr;
140     CMSampleBufferRef sampleBuf = nullptr;
141 
142     status = CMBlockBufferCreateWithMemoryBlock(kCFAllocatorDefault, // structureAllocator
143                                                 buffer,              // memoryBlock
144                                                 sz,                  // blockLength
145                                                 kCFAllocatorNull,    // blockAllocator
146                                                 NULL,                // customBlockSource
147                                                 0,                   // offsetToData
148                                                 sz,                  // dataLength
149                                                 0,                   // flags
150                                                 &blockBuf);
151 
152     if (!status) {
153         status = CMSampleBufferCreate(kCFAllocatorDefault, // allocator
154                                       blockBuf,            // dataBuffer
155                                       TRUE,                // dataReady
156                                       0,                   // makeDataReadyCallback
157                                       0,                   // makeDataReadyRefCon
158                                       fmtDesc,             // formatDescription
159                                       1,                   // numSamples
160                                       0,                   // numSampleTimingEntries
161                                       NULL,                // sampleTimingArray
162                                       0,                   // numSampleSizeEntries
163                                       NULL,                // sampleSizeArray
164                                       &sampleBuf);
165     }
166 
167     if (blockBuf) {
168         CFRelease(blockBuf);
169     }
170 
171     return sampleBuf;
172 }
173 
174 // static
toNativePixelFormat(PixelFormat pixFmt)175 OSType MediaH264DecoderVideoToolBox::toNativePixelFormat(PixelFormat pixFmt) {
176     switch (pixFmt) {
177         case PixelFormat::YUV420P:
178             return kCVPixelFormatType_420YpCbCr8Planar;
179         case PixelFormat::UYVY422:
180             return kCVPixelFormatType_422YpCbCr8;
181         case PixelFormat::BGRA8888:
182             return kCVPixelFormatType_32BGRA;
183         default:
184             H264_DPRINT("Unsupported VideoToolbox pixel format");
185             return '0000';
186     }
187 }
188 
189 // static
getReturnAddress(void * ptr)190 void* MediaH264DecoderVideoToolBox::getReturnAddress(void* ptr) {
191     uint8_t* xptr = (uint8_t*)ptr;
192     void* pint = (void*)(xptr + 256);
193     return pint;
194 }
195 
createCMFormatDescription()196 void MediaH264DecoderVideoToolBox::createCMFormatDescription() {
197     uint8_t*  parameterSets[2] = {mSPS.data(), mPPS.data()};
198     size_t parameterSetSizes[2] = {mSPS.size(), mPPS.size()};
199 
200     if (mCmFmtDesc) {
201         CFRelease(mCmFmtDesc);
202         mCmFmtDesc = nullptr;
203     }
204 
205     OSStatus status = CMVideoFormatDescriptionCreateFromH264ParameterSets(
206                               kCFAllocatorDefault,
207                               2,
208                               (const uint8_t *const*)parameterSets,
209                               parameterSetSizes,
210                               4,
211                               &mCmFmtDesc);
212 
213     if (status == noErr) {
214         H264_DPRINT("Created CMFormatDescription from SPS/PPS sets");
215     } else {
216         H264_DPRINT("Unable to create CMFormatDescription (%d)\n", (int)status);
217     }
218 }
219 
createVTDecoderConfig()220 CFDataRef MediaH264DecoderVideoToolBox::createVTDecoderConfig() {
221     CFDataRef data = nullptr;
222     return data;
223 }
224 
initH264Context(void * ptr)225 void MediaH264DecoderVideoToolBox::initH264Context(void* ptr) {
226     InitContextParam param{};
227     mParser.parseInitContextParams(ptr, param);
228     initH264ContextInternal(param.width, param.height, param.outputWidth,
229                             param.outputHeight, param.outputPixelFormat);
230 }
231 
initH264ContextInternal(unsigned int width,unsigned int height,unsigned int outWidth,unsigned int outHeight,PixelFormat outPixFmt)232 void MediaH264DecoderVideoToolBox::initH264ContextInternal(
233         unsigned int width,
234         unsigned int height,
235         unsigned int outWidth,
236         unsigned int outHeight,
237         PixelFormat outPixFmt) {
238     H264_DPRINT("%s(w=%u h=%u out_w=%u out_h=%u pixfmt=%u)",
239                 __func__, width, height, outWidth, outHeight, (uint8_t)outPixFmt);
240     mWidth = width;
241     mHeight = height;
242     mOutputWidth = outWidth;
243     mOutputHeight = outHeight;
244     mOutPixFmt = outPixFmt;
245     mOutBufferSize = outWidth * outHeight * 3 / 2;
246 }
247 
reset(void * ptr)248 void MediaH264DecoderVideoToolBox::reset(void* ptr) {
249     destroyH264Context();
250     ResetParam param{};
251     mParser.parseResetParams(ptr, param);
252     initH264ContextInternal(param.width, param.height, param.outputWidth,
253                             param.outputHeight, param.outputPixelFormat);
254 }
255 
destroyH264Context()256 void MediaH264DecoderVideoToolBox::destroyH264Context() {
257     H264_DPRINT("%s", __func__);
258     if (mDecoderSession) {
259         VTDecompressionSessionInvalidate(mDecoderSession);
260         CFRelease(mDecoderSession);
261         mDecoderSession = nullptr;
262     }
263     if (mCmFmtDesc) {
264         CFRelease(mCmFmtDesc);
265         mCmFmtDesc = nullptr;
266     }
267     if (mDecodedFrame) {
268         CVPixelBufferRelease(mDecodedFrame);
269         mDecodedFrame = nullptr;
270     }
271 }
272 
dumpBytes(const uint8_t * img,size_t szBytes,bool all=false)273 static void dumpBytes(const uint8_t* img, size_t szBytes, bool all = false) {
274 #if MEDIA_H264_DEBUG
275     printf("data=");
276     size_t numBytes = szBytes;
277     if (!all) {
278         numBytes = 32;
279     }
280 
281     for (size_t i = 0; i < (numBytes > szBytes ? szBytes : numBytes); ++i) {
282         if (i % 8 == 0) {
283            printf("\n");
284         }
285         printf("0x%02x ", img[i]);
286     }
287     printf("\n");
288 #endif
289 }
290 
decodeFrame(void * ptr)291 void MediaH264DecoderVideoToolBox::decodeFrame(void* ptr) {
292     DecodeFrameParam param{};
293     mParser.parseDecodeFrameParams(ptr, param);
294 
295     const uint8_t* frame = param.pData;
296     size_t szBytes = param.size;
297     uint64_t pts = param.pts;
298 
299     decodeFrameInternal(param.pConsumedBytes, param.pDecoderErrorCode, frame, szBytes, pts, 0);
300 }
301 
oneShotDecode(std::vector<uint8_t> & data,uint64_t pts)302 void MediaH264DecoderVideoToolBox::oneShotDecode(std::vector<uint8_t> & data, uint64_t pts) {
303     //TODO: check if data has more than one Nalu
304     decodeFrameInternal(nullptr, nullptr, data.data(), data.size(), pts, 0);
305 }
306 
decodeFrameInternal(size_t * pRetSzBytes,int32_t * pRetErr,const uint8_t * frame,size_t szBytes,uint64_t pts,size_t consumedSzBytes)307 void MediaH264DecoderVideoToolBox::decodeFrameInternal(size_t* pRetSzBytes, int32_t* pRetErr, const uint8_t* frame, size_t szBytes, uint64_t pts, size_t consumedSzBytes) {
308     // IMPORTANT: There is an assumption that each |frame| we get from the guest are contain
309     // complete NALUs. Usually, an H.264 bitstream would be continuously fed to us, but in this case,
310     // it seems that the Android frameworks passes us complete NALUs, and may be more than one NALU at
311     // a time.
312     //
313     // Let's only process one NALU per decodeFrame() call because, as soon as we receive a PPS NALU,
314     // we need to let the OMX plugin know to reset it's state because we are also recreating our
315     // decoder context.
316     H264_DPRINT("%s(frame=%p, sz=%zu)", __func__, frame, szBytes);
317     Err h264Err = Err::NoErr;
318 
319     const uint8_t* currentNalu = H264NaluParser::getNextStartCodeHeader(frame, szBytes);
320     if (currentNalu == nullptr) {
321         H264_DPRINT("No start code header found in this frame");
322         h264Err = Err::NoDecodedFrame;
323         // TODO: return the error code and num bytes processed, szBytes.
324         if (pRetSzBytes) *pRetSzBytes = szBytes;
325         if (pRetErr) *pRetErr = (int32_t)h264Err;
326         return;
327     }
328     const uint8_t* nextNalu = nullptr;
329     size_t remaining = szBytes - (currentNalu - frame);
330 
331     // Figure out the size of |currentNalu|.
332     size_t currentNaluSize = remaining;
333     // 3 is the minimum size of the start code header (3 or 4 bytes).
334     dumpBytes(currentNalu, currentNaluSize);
335     nextNalu = H264NaluParser::getNextStartCodeHeader(currentNalu + 3, remaining - 3);
336     if (nextNalu != nullptr) {
337         currentNaluSize = nextNalu - currentNalu;
338     }
339 
340     // |data| is currentNalu, but with the start code header discarded.
341     uint8_t* data = nullptr;
342     H264NaluType naluType = H264NaluParser::getFrameNaluType(currentNalu, currentNaluSize, &data);
343     size_t dataSize = currentNaluSize - (data - currentNalu);
344     const std::string naluTypeStr = H264NaluParser::naluTypeToString(naluType);
345     H264_DPRINT("Got frame type=%u (%s)", (uint8_t)naluType, naluTypeStr.c_str());
346 
347     // We can't do anything until we set up a CMFormatDescription from a set of SPS and PPS NALUs.
348     // So just discard the NALU.
349     if (naluType != H264NaluType::SPS && naluType != H264NaluType::PPS &&
350         mCmFmtDesc == nullptr) {
351         H264_DPRINT("CMFormatDescription not set up yet. Need SPS/PPS frames.");
352         h264Err = Err::NALUIgnored;
353         if (pRetSzBytes) *pRetSzBytes = currentNaluSize;
354         if (pRetErr) *pRetErr = (int32_t)h264Err;
355         return;
356     }
357 
358     switch (naluType) {
359         case H264NaluType::SPS:
360             // We should be getting a PPS frame on the next decodeFrame(). Once we have
361             // both sps and pps, we can create/recreate the decoder session.
362             // Don't include the start code header when we copy the sps/pps.
363             mSPS.assign(data, data + dataSize);
364             if (!mIsLoadingFromSnapshot) {
365                 mSnapshotState = SnapshotState{};
366                 std::vector<uint8_t> vec;
367                 vec.assign(currentNalu, currentNalu + currentNaluSize);
368                 mSnapshotState.saveSps(vec);
369             }
370             break;
371         case H264NaluType::PPS:
372             mPPS.assign(data, data + dataSize);
373             createCMFormatDescription();
374             // TODO: We will need to recreate the decompression session whenever we get a
375             // resolution change.
376             if (mDecoderSession != nullptr) {
377                 H264_DPRINT("Decoder session is restarting");
378                 //h264Err = Err::DecoderRestarted;
379             }
380             if (!mIsLoadingFromSnapshot) {
381                 std::vector<uint8_t> vec;
382                 vec.assign(currentNalu, currentNalu + currentNaluSize);
383                 mSnapshotState.savePps(vec);
384                 mSnapshotState.savedPackets.clear();
385             }
386             recreateDecompressionSession();
387             break;
388         case H264NaluType::SEI:
389 //                dumpBytes(nextNalu, remaining, true);
390             // In some cases, after the SPS and PPS NALUs are emitted, we'll get a frame that
391             // contains both an SEI NALU and a CodedSliceIDR NALU.
392             handleSEIFrame(currentNalu, currentNaluSize);
393             break;
394         case H264NaluType::CodedSliceIDR:
395             handleIDRFrame(currentNalu, currentNaluSize, pts);
396             if (!mIsLoadingFromSnapshot) {
397                 H264_DPRINT("disacarding previously saved frames %d", (int)mSnapshotState.savedPackets.size());
398                 mSnapshotState.savedPackets.clear();
399                 mSnapshotState.savePacket(currentNalu, currentNaluSize, pts);
400             }
401             break;
402         case H264NaluType::CodedSliceNonIDR:
403             handleNonIDRFrame(currentNalu, currentNaluSize, pts);
404             if (!mIsLoadingFromSnapshot) {
405                 mSnapshotState.savePacket(currentNalu, currentNaluSize, pts);
406             }
407             break;
408         default:
409             H264_DPRINT("Support for nalu_type=%u not implemented", (uint8_t)naluType);
410             break;
411     }
412 
413     remaining -= currentNaluSize;
414     currentNalu = nextNalu;
415 
416     // return two things: the error code and the number of bytes we processed.
417     if (pRetSzBytes) *pRetSzBytes = currentNaluSize + consumedSzBytes;
418     if (pRetErr) *pRetErr = (int32_t)h264Err;
419 
420     // disable recursive decoding due to the possibility of session creation failure
421     // keep it simple for now
422     //if (currentNalu) {
423     //    decodeFrameInternal(ptr, currentNalu, remaining, pts, consumedSzBytes + currentNaluSize);
424     //}
425 }
426 
handleIDRFrame(const uint8_t * ptr,size_t szBytes,uint64_t pts)427 void MediaH264DecoderVideoToolBox::handleIDRFrame(const uint8_t* ptr, size_t szBytes, uint64_t pts) {
428     H264_DPRINT("Got IDR frame (sz=%zu)", szBytes);
429     uint8_t* fptr = const_cast<uint8_t*>(ptr);
430 
431     // We can assume fptr has a valid start code header because it has already
432     // gone through validation in H264NaluParser.
433     uint8_t startHeaderSz = fptr[2] == 1 ? 3 : 4;
434     uint32_t dataSz = szBytes - startHeaderSz;
435     std::unique_ptr<uint8_t> idr(new uint8_t[dataSz + 4]);
436     uint32_t dataSzNl = htonl(dataSz);
437 
438     // AVCC format requires us to replace the start code header on this NALU
439     // with the size of the data. Start code is either 0x000001 or 0x00000001.
440     // The size needs to be the first four bytes in network byte order.
441     memcpy(idr.get(), &dataSzNl, 4);
442     memcpy(idr.get() + 4, ptr + startHeaderSz, dataSz);
443 
444     CMSampleBufferRef sampleBuf = nullptr;
445     sampleBuf = createSampleBuffer(mCmFmtDesc, (void*)idr.get(), dataSz + 4);
446     if (!sampleBuf) {
447         H264_DPRINT("%s: Failed to create CMSampleBufferRef", __func__);
448         return;
449     }
450 
451     CMSampleBufferSetOutputPresentationTimeStamp(sampleBuf, CMTimeMake(pts, 1));
452 
453     OSStatus status;
454     status = VTDecompressionSessionDecodeFrame(mDecoderSession,
455                                                sampleBuf,
456                                                0,       // decodeFlags
457                                                NULL,    // sourceFrameRefCon
458                                                0);      // infoFlagsOut
459 
460     if (status == noErr) {
461         // TODO: this call blocks until the frame has been decoded. Perhaps it will be
462         // more efficient to signal the guest when the frame is ready to be read instead.
463         status = VTDecompressionSessionWaitForAsynchronousFrames(mDecoderSession);
464         mIsInFlush = false;
465     } else {
466         H264_DPRINT("%s: Failed to decompress frame (err=%d)", __func__, status);
467     }
468 
469     CFRelease(sampleBuf);
470     H264_DPRINT("Success decoding IDR frame");
471 }
472 
handleNonIDRFrame(const uint8_t * ptr,size_t szBytes,uint64_t pts)473 void MediaH264DecoderVideoToolBox::handleNonIDRFrame(const uint8_t* ptr, size_t szBytes, uint64_t pts) {
474     // Same as handling an IDR frame
475     handleIDRFrame(ptr, szBytes, pts);
476 }
477 
handleSEIFrame(const uint8_t * ptr,size_t szBytes)478 void MediaH264DecoderVideoToolBox::handleSEIFrame(const uint8_t* ptr, size_t szBytes) {
479     H264_DPRINT("NOT IMPLEMENTED");
480 }
481 
flush(void * ptr)482 void MediaH264DecoderVideoToolBox::flush(void* ptr) {
483     H264_DPRINT("%s: NOT IMPLEMENTED", __func__);
484     mIsInFlush = true;
485 }
486 
copyFrame()487 void MediaH264DecoderVideoToolBox::copyFrame() {
488     if (mIsLoadingFromSnapshot) return;
489 
490     int imgWidth = CVPixelBufferGetWidth(mDecodedFrame);
491     int imgHeight = CVPixelBufferGetHeight(mDecodedFrame);
492     int imageSize = CVPixelBufferGetDataSize(mDecodedFrame);
493     int stride = CVPixelBufferGetBytesPerRow(mDecodedFrame);
494 
495     mOutputWidth = imgWidth;
496     mOutputHeight = imgHeight;
497     mOutBufferSize = mOutputWidth * mOutputHeight * 3 / 2;
498 
499     H264_DPRINT("copying size=%d dimension=[%dx%d] stride=%d", imageSize, imgWidth, imgHeight, stride);
500 
501     // Copies the image data to the guest.
502     mSavedDecodedFrame.resize(imgWidth * imgHeight * 3 / 2);
503     uint8_t* dst = mSavedDecodedFrame.data();
504 
505     CVPixelBufferLockBaseAddress(mDecodedFrame, kCVPixelBufferLock_ReadOnly);
506     if (CVPixelBufferIsPlanar(mDecodedFrame)) {
507         imageSize = 0; // add up the size from the planes
508         int planes = CVPixelBufferGetPlaneCount(mDecodedFrame);
509         for (int i = 0; i < planes; ++i) {
510             void* planeData = CVPixelBufferGetBaseAddressOfPlane(mDecodedFrame, i);
511             int linesize = CVPixelBufferGetBytesPerRowOfPlane(mDecodedFrame, i);
512             int planeWidth = CVPixelBufferGetWidthOfPlane(mDecodedFrame, i);
513             int planeHeight = CVPixelBufferGetHeightOfPlane(mDecodedFrame, i);
514             H264_DPRINT("plane=%d data=%p linesize=%d pwidth=%d pheight=%d", i, planeData, linesize, planeWidth, planeHeight);
515             // For kCVPixelFormatType_420YpCbCr8Planar, plane 0 is Y, UV planes are 1 and 2
516             if (planeWidth != imgWidth && planeWidth != imgWidth / 2) {
517                 H264_DPRINT("ERROR: Unable to determine YUV420 plane type");
518                 continue;
519             }
520 
521             // Sometimes the buffer stride can be longer than the actual data width. This means that
522             // the extra bytes are just padding and need to be discarded.
523             if (linesize <= planeWidth) {
524                 int sz = planeHeight * planeWidth;
525                 imageSize += sz;
526                 memcpy(dst, planeData, sz);
527                 dst += sz;
528             } else {
529                 // Need to copy line by line
530                 int sz = planeWidth;
531                 for (int j = 0; j < planeHeight; ++j) {
532                     uint8_t* ptr = (uint8_t*)planeData;
533                     ptr += linesize * j;
534                     memcpy(dst, ptr, sz);
535                     imageSize += sz;
536                     dst += sz;
537                 }
538             }
539         }
540         if (imageSize != mOutBufferSize) {
541             H264_DPRINT("ERROR: Total size of planes not same as guestSz (guestSz=%u, imageSize=%d)", mOutBufferSize, imageSize);
542         }
543     } else {
544         if (imageSize > mOutBufferSize) {
545             H264_DPRINT("Buffer size mismatch (guestSz=%u, imageSize=%d). Using guestSz instead.", mOutBufferSize, imageSize);
546             imageSize = mOutBufferSize;
547         }
548 
549         // IMPORTANT: mDecodedFrame must be locked before accessing the contents with CPU
550         void* data = CVPixelBufferGetBaseAddress(mDecodedFrame);
551         memcpy(dst, data, imageSize);
552     }
553     CVPixelBufferUnlockBaseAddress(mDecodedFrame, kCVPixelBufferLock_ReadOnly);
554 }
555 
getImage(void * ptr)556 void MediaH264DecoderVideoToolBox::getImage(void* ptr) {
557     // return parameters:
558     // 1) either size of the image (> 0) or error code (<= 0).
559     // 2) image width
560     // 3) image height
561     GetImageParam param{};
562     mParser.parseGetImageParams(ptr, param);
563 
564     int* retErr = param.pDecoderErrorCode;
565     uint32_t* retWidth = param.pRetWidth;
566     uint32_t* retHeight = param.pRetHeight;
567     uint64_t* retPts = param.pRetPts;
568     uint32_t* retColorPrimaries = param.pRetColorPrimaries;
569     uint32_t* retColorRange = param.pRetColorRange;
570     uint32_t* retColorTransfer = param.pRetColorTransfer;
571     uint32_t* retColorSpace = param.pRetColorSpace;
572 
573     if (!mDecodedFrame) {
574         H264_DPRINT("%s: frame is null", __func__);
575         *retErr = static_cast<int>(Err::NoDecodedFrame);
576         return;
577     }
578     if (!mImageReady) {
579         bool hasMoreFrames = false;
580         if (mIsInFlush) {
581             OSStatus status = noErr;
582             status = VTDecompressionSessionWaitForAsynchronousFrames(mDecoderSession);
583             if (status == noErr) {
584                 hasMoreFrames = mImageReady;
585                 if (hasMoreFrames) {
586                     H264_DPRINT("%s: got frame in flush mode", __func__);
587                 }
588             }
589         }
590 
591         if (!hasMoreFrames) {
592             H264_DPRINT("%s: no new frame yet", __func__);
593             *retErr = static_cast<int>(Err::NoDecodedFrame);
594             return;
595         }
596     }
597 
598     *retWidth = mOutputWidth;
599     *retHeight = mOutputHeight;
600 
601     if (mParser.version() == 200 && param.hostColorBufferId >= 0) {
602         mRenderer.renderToHostColorBuffer(param.hostColorBufferId,
603                                           mOutputWidth, mOutputHeight,
604                                           mSavedDecodedFrame.data());
605     } else {
606         memcpy(param.pDecodedFrame, mSavedDecodedFrame.data(), mSavedDecodedFrame.size());;
607     }
608 
609     *retErr = mSavedDecodedFrame.size();
610 
611     *retPts = mOutputPts;
612 
613     H264_DPRINT("Copying completed pts %lld", (long long)mOutputPts);
614     mImageReady = false;
615 }
616 
recreateDecompressionSession()617 void MediaH264DecoderVideoToolBox::recreateDecompressionSession() {
618     if (mCmFmtDesc == nullptr) {
619         H264_DPRINT("CMFormatDescription not created. Need sps and pps NALUs.");
620         return;
621     }
622 
623     // Create a new VideoToolbox decoder session if one already exists
624     if (mDecoderSession != nullptr) {
625         // TODO: Once we implement async frame readback, we'll need to flush all of the frames here and
626         // store them somewhere for the guest to read later.
627         VTDecompressionSessionInvalidate(mDecoderSession);
628         CFRelease(mDecoderSession);
629         mDecoderSession = nullptr;
630         if (mDecodedFrame) {
631             CVPixelBufferRelease(mDecodedFrame);
632             mDecodedFrame = nullptr;
633         }
634     }
635 
636     CMVideoCodecType codecType = kCMVideoCodecType_H264;
637     CFMutableDictionaryRef decoder_spec = CFDictionaryCreateMutable(kCFAllocatorDefault,
638                                                                     0,
639                                                                     &kCFTypeDictionaryKeyCallBacks,
640                                                                     &kCFTypeDictionaryValueCallBacks);
641     CFDictionarySetValue(decoder_spec,
642                          kVTVideoDecoderSpecification_RequireHardwareAcceleratedVideoDecoder,
643                          kCFBooleanTrue);
644 
645     CFDictionaryRef bufAttr = createOutputBufferAttributes(mOutputWidth,
646                                                            mOutputHeight,
647                                                            toNativePixelFormat(mOutPixFmt));
648 
649     VTDecompressionOutputCallbackRecord decoderCb;
650     decoderCb.decompressionOutputCallback = videoToolboxDecompressCallback;
651     decoderCb.decompressionOutputRefCon = this;
652 
653     OSStatus status;
654     status = VTDecompressionSessionCreate(NULL,              // allocator
655                                           mCmFmtDesc,        // videoFormatDescription
656                                           decoder_spec,      // videoDecoderSpecification
657                                           bufAttr,           // destinationImageBufferAttributes
658                                           &decoderCb,        // outputCallback
659                                           &mDecoderSession); // decompressionSessionOut
660 
661     if (decoder_spec) {
662         CFRelease(decoder_spec);
663     }
664     if (bufAttr) {
665         CFRelease(bufAttr);
666     }
667 
668     mIsInFlush = false;
669     mState = DecoderState::BAD_STATE;
670     switch (status) {
671         case kVTVideoDecoderNotAvailableNowErr:
672             H264_DPRINT("VideoToolbox session not available");
673             return;
674         case kVTVideoDecoderUnsupportedDataFormatErr:
675             H264_DPRINT("VideoToolbox does not support this format");
676             return;
677         case kVTVideoDecoderMalfunctionErr:
678             H264_DPRINT("VideoToolbox malfunction");
679             return;
680         case kVTVideoDecoderBadDataErr:
681             H264_DPRINT("VideoToolbox reported invalid data");
682             return;
683         case 0:
684             H264_DPRINT("VideoToolbox session created");
685             mState = DecoderState::GOOD_STATE;
686             return;
687         default:
688             H264_DPRINT("Unknown VideoToolbox session creation error %d", status);
689             return;
690     }
691 }
692 
save(base::Stream * stream) const693 void MediaH264DecoderVideoToolBox::save(base::Stream* stream) const {
694     stream->putBe32(mParser.version());
695     stream->putBe32(mWidth);
696     stream->putBe32(mHeight);
697     stream->putBe32((int)mOutPixFmt);
698 
699     const int hasContext = mDecoderSession != nullptr ? 1 : 0;
700     stream->putBe32(hasContext);
701 
702     if (mImageReady) {
703         mSnapshotState.saveDecodedFrame(
704                 mSavedDecodedFrame, mOutputWidth, mOutputHeight,
705                 ColorAspects{}, mOutputPts);
706     } else {
707         mSnapshotState.savedDecodedFrame.data.clear();
708     }
709     H264_DPRINT("saving packets now %d",
710                 (int)(mSnapshotState.savedPackets.size()));
711     mSnapshotState.save(stream);
712 }
713 
load(base::Stream * stream)714 bool MediaH264DecoderVideoToolBox::load(base::Stream* stream) {
715     mIsLoadingFromSnapshot = true;
716     uint32_t version = stream->getBe32();
717     mParser = H264PingInfoParser{version};
718 
719     mWidth = stream->getBe32();
720     mHeight = stream->getBe32();
721     mOutPixFmt = (PixelFormat)stream->getBe32();
722 
723     const int hasContext = stream->getBe32();
724     if (hasContext) {
725         initH264ContextInternal(mWidth, mHeight, mWidth, mHeight, mOutPixFmt);
726     }
727     mSnapshotState.load(stream);
728     if (hasContext && mSnapshotState.sps.size() > 0) {
729         oneShotDecode(mSnapshotState.sps, 0);
730         if (mSnapshotState.pps.size() > 0) {
731             oneShotDecode(mSnapshotState.pps, 0);
732             if (mSnapshotState.savedPackets.size() > 0) {
733                 for (int i = 0; i < mSnapshotState.savedPackets.size(); ++i) {
734                     PacketInfo& pkt = mSnapshotState.savedPackets[i];
735                     oneShotDecode(pkt.data, pkt.pts);
736                 }
737             }
738         }
739     }
740 
741     if (mSnapshotState.savedDecodedFrame.data.size() > 0) {
742         mSavedDecodedFrame = std::move(mSnapshotState.savedDecodedFrame.data);
743         mOutBufferSize = mSnapshotState.savedDecodedFrame.data.size();
744         mOutputWidth = mSnapshotState.savedDecodedFrame.width;
745         mOutputHeight = mSnapshotState.savedDecodedFrame.height;
746         mOutputPts = mSnapshotState.savedDecodedFrame.pts;
747         mImageReady = true;
748     } else {
749         mOutputWidth = mWidth;
750         mOutputHeight = mHeight;
751         mOutBufferSize = mOutputWidth * mOutputHeight * 3 / 2;
752         mImageReady = false;
753     }
754     mIsLoadingFromSnapshot = false;
755     return true;
756 }
757 
758 }  // namespace emulation
759 }  // namespace android
760