1 /*
2  * Copyright (C) 2016 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 #include <inttypes.h>
18 
19 //#define LOG_NDEBUG 0
20 #define LOG_TAG "NdkImageReader"
21 
22 #include "NdkImagePriv.h"
23 #include "NdkImageReaderPriv.h"
24 
25 #include <utils/Log.h>
26 #include <android_runtime/android_view_Surface.h>
27 
28 using namespace android;
29 
30 namespace {
31     // Get an ID that's unique within this process.
createProcessUniqueId()32     static int32_t createProcessUniqueId() {
33         static volatile int32_t globalCounter = 0;
34         return android_atomic_inc(&globalCounter);
35     }
36 }
37 
38 const char* AImageReader::kCallbackFpKey = "Callback";
39 const char* AImageReader::kContextKey    = "Context";
40 
41 bool
isSupportedFormat(int32_t format)42 AImageReader::isSupportedFormat(int32_t format) {
43     switch (format) {
44         case AIMAGE_FORMAT_YUV_420_888:
45         case AIMAGE_FORMAT_JPEG:
46         case AIMAGE_FORMAT_RAW16:
47         case AIMAGE_FORMAT_RAW_PRIVATE:
48         case AIMAGE_FORMAT_RAW10:
49         case AIMAGE_FORMAT_RAW12:
50         case AIMAGE_FORMAT_DEPTH16:
51         case AIMAGE_FORMAT_DEPTH_POINT_CLOUD:
52             return true;
53         default:
54             return false;
55     }
56 }
57 
58 int
getNumPlanesForFormat(int32_t format)59 AImageReader::getNumPlanesForFormat(int32_t format) {
60     switch (format) {
61         case AIMAGE_FORMAT_YUV_420_888:
62             return 3;
63         case AIMAGE_FORMAT_JPEG:
64         case AIMAGE_FORMAT_RAW16:
65         case AIMAGE_FORMAT_RAW_PRIVATE:
66         case AIMAGE_FORMAT_RAW10:
67         case AIMAGE_FORMAT_RAW12:
68         case AIMAGE_FORMAT_DEPTH16:
69         case AIMAGE_FORMAT_DEPTH_POINT_CLOUD:
70             return 1;
71         default:
72             return -1;
73     }
74 }
75 
76 void
onFrameAvailable(const BufferItem &)77 AImageReader::FrameListener::onFrameAvailable(const BufferItem& /*item*/) {
78     Mutex::Autolock _l(mLock);
79     sp<AImageReader> reader = mReader.promote();
80     if (reader == nullptr) {
81         ALOGW("A frame is available after AImageReader closed!");
82         return; // reader has been closed
83     }
84     if (mListener.onImageAvailable == nullptr) {
85         return; // No callback registered
86     }
87 
88     sp<AMessage> msg = new AMessage(AImageReader::kWhatImageAvailable, reader->mHandler);
89     msg->setPointer(AImageReader::kCallbackFpKey, (void *) mListener.onImageAvailable);
90     msg->setPointer(AImageReader::kContextKey, mListener.context);
91     msg->post();
92 }
93 
94 media_status_t
setImageListener(AImageReader_ImageListener * listener)95 AImageReader::FrameListener::setImageListener(AImageReader_ImageListener* listener) {
96     Mutex::Autolock _l(mLock);
97     if (listener == nullptr) {
98         mListener.context = nullptr;
99         mListener.onImageAvailable = nullptr;
100     } else {
101         mListener = *listener;
102     }
103     return AMEDIA_OK;
104 }
105 
106 media_status_t
setImageListenerLocked(AImageReader_ImageListener * listener)107 AImageReader::setImageListenerLocked(AImageReader_ImageListener* listener) {
108     return mFrameListener->setImageListener(listener);
109 }
110 
111 media_status_t
setImageListener(AImageReader_ImageListener * listener)112 AImageReader::setImageListener(AImageReader_ImageListener* listener) {
113     Mutex::Autolock _l(mLock);
114     return setImageListenerLocked(listener);
115 }
116 
onMessageReceived(const sp<AMessage> & msg)117 void AImageReader::CallbackHandler::onMessageReceived(
118         const sp<AMessage> &msg) {
119     switch (msg->what()) {
120         case kWhatImageAvailable:
121         {
122             AImageReader_ImageCallback onImageAvailable;
123             void* context;
124             bool found = msg->findPointer(kCallbackFpKey, (void**) &onImageAvailable);
125             if (!found || onImageAvailable == nullptr) {
126                 ALOGE("%s: Cannot find onImageAvailable callback fp!", __FUNCTION__);
127                 return;
128             }
129             found = msg->findPointer(kContextKey, &context);
130             if (!found) {
131                 ALOGE("%s: Cannot find callback context!", __FUNCTION__);
132                 return;
133             }
134             (*onImageAvailable)(context, mReader);
135             break;
136         }
137         default:
138             ALOGE("%s: unknown message type %d", __FUNCTION__, msg->what());
139             break;
140     }
141 }
142 
AImageReader(int32_t width,int32_t height,int32_t format,int32_t maxImages)143 AImageReader::AImageReader(int32_t width, int32_t height, int32_t format, int32_t maxImages) :
144         mWidth(width), mHeight(height), mFormat(format), mMaxImages(maxImages),
145         mNumPlanes(getNumPlanesForFormat(format)),
146         mFrameListener(new FrameListener(this)) {}
147 
148 media_status_t
init()149 AImageReader::init() {
150     PublicFormat publicFormat = static_cast<PublicFormat>(mFormat);
151     mHalFormat = android_view_Surface_mapPublicFormatToHalFormat(publicFormat);
152     mHalDataSpace = android_view_Surface_mapPublicFormatToHalDataspace(publicFormat);
153 
154     sp<IGraphicBufferProducer> gbProducer;
155     sp<IGraphicBufferConsumer> gbConsumer;
156     BufferQueue::createBufferQueue(&gbProducer, &gbConsumer);
157 
158     sp<CpuConsumer> cpuConsumer;
159     String8 consumerName = String8::format("ImageReader-%dx%df%xm%d-%d-%d",
160             mWidth, mHeight, mFormat, mMaxImages, getpid(),
161             createProcessUniqueId());
162 
163     cpuConsumer = new CpuConsumer(gbConsumer, mMaxImages, /*controlledByApp*/true);
164     if (cpuConsumer == nullptr) {
165         ALOGE("Failed to allocate CpuConsumer");
166         return AMEDIA_ERROR_UNKNOWN;
167     }
168 
169     mCpuConsumer = cpuConsumer;
170     mCpuConsumer->setName(consumerName);
171     mProducer = gbProducer;
172 
173     sp<ConsumerBase> consumer = cpuConsumer;
174     consumer->setFrameAvailableListener(mFrameListener);
175 
176     status_t res;
177     res = cpuConsumer->setDefaultBufferSize(mWidth, mHeight);
178     if (res != OK) {
179         ALOGE("Failed to set CpuConsumer buffer size");
180         return AMEDIA_ERROR_UNKNOWN;
181     }
182     res = cpuConsumer->setDefaultBufferFormat(mHalFormat);
183     if (res != OK) {
184         ALOGE("Failed to set CpuConsumer buffer format");
185         return AMEDIA_ERROR_UNKNOWN;
186     }
187     res = cpuConsumer->setDefaultBufferDataSpace(mHalDataSpace);
188     if (res != OK) {
189         ALOGE("Failed to set CpuConsumer buffer dataSpace");
190         return AMEDIA_ERROR_UNKNOWN;
191     }
192 
193     mSurface = new Surface(mProducer, /*controlledByApp*/true);
194     if (mSurface == nullptr) {
195         ALOGE("Failed to create surface");
196         return AMEDIA_ERROR_UNKNOWN;
197     }
198     mWindow = static_cast<ANativeWindow*>(mSurface.get());
199 
200     for (int i = 0; i < mMaxImages; i++) {
201         CpuConsumer::LockedBuffer* buffer = new CpuConsumer::LockedBuffer;
202         mBuffers.push_back(buffer);
203     }
204 
205     mCbLooper = new ALooper;
206     mCbLooper->setName(consumerName.string());
207     res = mCbLooper->start(
208             /*runOnCallingThread*/false,
209             /*canCallJava*/       true,
210             PRIORITY_DEFAULT);
211     if (res != OK) {
212         ALOGE("Failed to start the looper");
213         return AMEDIA_ERROR_UNKNOWN;
214     }
215     mHandler = new CallbackHandler(this);
216     mCbLooper->registerHandler(mHandler);
217 
218     return AMEDIA_OK;
219 }
220 
~AImageReader()221 AImageReader::~AImageReader() {
222     Mutex::Autolock _l(mLock);
223     AImageReader_ImageListener nullListener = {nullptr, nullptr};
224     setImageListenerLocked(&nullListener);
225 
226     if (mCbLooper != nullptr) {
227         mCbLooper->unregisterHandler(mHandler->id());
228         mCbLooper->stop();
229     }
230     mCbLooper.clear();
231     mHandler.clear();
232 
233     // Close all previously acquired images
234     for (auto it = mAcquiredImages.begin();
235               it != mAcquiredImages.end(); it++) {
236         AImage* image = *it;
237         image->close();
238     }
239 
240     // Delete LockedBuffers
241     for (auto it = mBuffers.begin();
242               it != mBuffers.end(); it++) {
243         delete *it;
244     }
245 
246     if (mCpuConsumer != nullptr) {
247         mCpuConsumer->abandon();
248         mCpuConsumer->setFrameAvailableListener(nullptr);
249     }
250 }
251 
252 media_status_t
acquireCpuConsumerImageLocked(AImage ** image)253 AImageReader::acquireCpuConsumerImageLocked(/*out*/AImage** image) {
254     *image = nullptr;
255     CpuConsumer::LockedBuffer* buffer = getLockedBufferLocked();
256     if (buffer == nullptr) {
257         ALOGW("Unable to acquire a lockedBuffer, very likely client tries to lock more than"
258             " maxImages buffers");
259         return AMEDIA_IMGREADER_MAX_IMAGES_ACQUIRED;
260     }
261 
262     status_t res = mCpuConsumer->lockNextBuffer(buffer);
263     if (res != NO_ERROR) {
264         returnLockedBufferLocked(buffer);
265         if (res != BAD_VALUE /*no buffers*/) {
266             if (res == NOT_ENOUGH_DATA) {
267                 return AMEDIA_IMGREADER_MAX_IMAGES_ACQUIRED;
268             } else {
269                 ALOGE("%s Fail to lockNextBuffer with error: %d ",
270                       __FUNCTION__, res);
271                 return AMEDIA_ERROR_UNKNOWN;
272             }
273         }
274         return AMEDIA_IMGREADER_NO_BUFFER_AVAILABLE;
275     }
276 
277     if (buffer->flexFormat == HAL_PIXEL_FORMAT_YCrCb_420_SP) {
278         ALOGE("NV21 format is not supported by AImageReader");
279         return AMEDIA_ERROR_UNSUPPORTED;
280     }
281 
282     // Check if the left-top corner of the crop rect is origin, we currently assume this point is
283     // zero, will revist this once this assumption turns out problematic.
284     Point lt = buffer->crop.leftTop();
285     if (lt.x != 0 || lt.y != 0) {
286         ALOGE("crop left top corner [%d, %d] need to be at origin", lt.x, lt.y);
287         return AMEDIA_ERROR_UNKNOWN;
288     }
289 
290     // Check if the producer buffer configurations match what ImageReader configured.
291     int outputWidth = getBufferWidth(buffer);
292     int outputHeight = getBufferHeight(buffer);
293 
294     int readerFmt = mHalFormat;
295     int readerWidth = mWidth;
296     int readerHeight = mHeight;
297 
298     if ((buffer->format != HAL_PIXEL_FORMAT_BLOB) && (readerFmt != HAL_PIXEL_FORMAT_BLOB) &&
299             (readerWidth != outputWidth || readerHeight != outputHeight)) {
300         ALOGW("%s: Producer buffer size: %dx%d, doesn't match AImageReader configured size: %dx%d",
301                 __FUNCTION__, outputWidth, outputHeight, readerWidth, readerHeight);
302     }
303 
304     int bufFmt = buffer->format;
305     if (readerFmt == HAL_PIXEL_FORMAT_YCbCr_420_888) {
306         bufFmt = buffer->flexFormat;
307     }
308 
309     if (readerFmt != bufFmt) {
310         if (readerFmt == HAL_PIXEL_FORMAT_YCbCr_420_888 && (bufFmt ==
311                 HAL_PIXEL_FORMAT_YCrCb_420_SP || bufFmt == HAL_PIXEL_FORMAT_YV12)) {
312             // Special casing for when producer switches to a format compatible with flexible YUV
313             // (HAL_PIXEL_FORMAT_YCbCr_420_888).
314             mHalFormat = bufFmt;
315             ALOGD("%s: Overriding buffer format YUV_420_888 to %x.", __FUNCTION__, bufFmt);
316         } else {
317             // Return the buffer to the queue.
318             mCpuConsumer->unlockBuffer(*buffer);
319             returnLockedBufferLocked(buffer);
320 
321             ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x",
322                     buffer->format, readerFmt);
323 
324             return AMEDIA_ERROR_UNKNOWN;
325         }
326     }
327 
328     if (mHalFormat == HAL_PIXEL_FORMAT_BLOB) {
329         *image = new AImage(this, mFormat, buffer, buffer->timestamp,
330                             readerWidth, readerHeight, mNumPlanes);
331     } else {
332         *image = new AImage(this, mFormat, buffer, buffer->timestamp,
333                             outputWidth, outputHeight, mNumPlanes);
334     }
335     mAcquiredImages.push_back(*image);
336     return AMEDIA_OK;
337 }
338 
339 CpuConsumer::LockedBuffer*
getLockedBufferLocked()340 AImageReader::getLockedBufferLocked() {
341     if (mBuffers.empty()) {
342         return nullptr;
343     }
344     // Return a LockedBuffer pointer and remove it from the list
345     auto it = mBuffers.begin();
346     CpuConsumer::LockedBuffer* buffer = *it;
347     mBuffers.erase(it);
348     return buffer;
349 }
350 
351 void
returnLockedBufferLocked(CpuConsumer::LockedBuffer * buffer)352 AImageReader::returnLockedBufferLocked(CpuConsumer::LockedBuffer* buffer) {
353     mBuffers.push_back(buffer);
354 }
355 
356 void
releaseImageLocked(AImage * image)357 AImageReader::releaseImageLocked(AImage* image) {
358     CpuConsumer::LockedBuffer* buffer = image->mBuffer;
359     if (buffer == nullptr) {
360         // This should not happen, but is not fatal
361         ALOGW("AImage %p has no buffer!", image);
362         return;
363     }
364 
365     mCpuConsumer->unlockBuffer(*buffer);
366     returnLockedBufferLocked(buffer);
367     image->mBuffer = nullptr;
368 
369     bool found = false;
370     // cleanup acquired image list
371     for (auto it = mAcquiredImages.begin();
372               it != mAcquiredImages.end(); it++) {
373         AImage* readerCopy = *it;
374         if (readerCopy == image) {
375             found = true;
376             mAcquiredImages.erase(it);
377             break;
378         }
379     }
380     if (!found) {
381         ALOGE("Error: AImage %p is not generated by AImageReader %p",
382                 image, this);
383     }
384 }
385 
386 int
getBufferWidth(CpuConsumer::LockedBuffer * buffer)387 AImageReader::getBufferWidth(CpuConsumer::LockedBuffer* buffer) {
388     if (buffer == nullptr) return -1;
389 
390     if (!buffer->crop.isEmpty()) {
391         return buffer->crop.getWidth();
392     }
393     return buffer->width;
394 }
395 
396 int
getBufferHeight(CpuConsumer::LockedBuffer * buffer)397 AImageReader::getBufferHeight(CpuConsumer::LockedBuffer* buffer) {
398     if (buffer == nullptr) return -1;
399 
400     if (!buffer->crop.isEmpty()) {
401         return buffer->crop.getHeight();
402     }
403     return buffer->height;
404 }
405 
406 media_status_t
acquireNextImage(AImage ** image)407 AImageReader::acquireNextImage(/*out*/AImage** image) {
408     Mutex::Autolock _l(mLock);
409     return acquireCpuConsumerImageLocked(image);
410 }
411 
412 media_status_t
acquireLatestImage(AImage ** image)413 AImageReader::acquireLatestImage(/*out*/AImage** image) {
414     if (image == nullptr) {
415         return AMEDIA_ERROR_INVALID_PARAMETER;
416     }
417     Mutex::Autolock _l(mLock);
418     *image = nullptr;
419     AImage* prevImage = nullptr;
420     AImage* nextImage = nullptr;
421     media_status_t ret = acquireCpuConsumerImageLocked(&prevImage);
422     if (prevImage == nullptr) {
423         return ret;
424     }
425     for (;;) {
426         ret = acquireCpuConsumerImageLocked(&nextImage);
427         if (nextImage == nullptr) {
428             *image = prevImage;
429             return AMEDIA_OK;
430         }
431         prevImage->close();
432         prevImage->free();
433         prevImage = nextImage;
434         nextImage = nullptr;
435     }
436 }
437 
438 EXPORT
AImageReader_new(int32_t width,int32_t height,int32_t format,int32_t maxImages,AImageReader ** reader)439 media_status_t AImageReader_new(
440         int32_t width, int32_t height, int32_t format, int32_t maxImages,
441         /*out*/AImageReader** reader) {
442     ALOGV("%s", __FUNCTION__);
443 
444     if (width < 1 || height < 1) {
445         ALOGE("%s: image dimension must be positive: w:%d h:%d",
446                 __FUNCTION__, width, height);
447         return AMEDIA_ERROR_INVALID_PARAMETER;
448     }
449 
450     if (maxImages < 1) {
451         ALOGE("%s: max outstanding image count must be at least 1 (%d)",
452                 __FUNCTION__, maxImages);
453         return AMEDIA_ERROR_INVALID_PARAMETER;
454     }
455 
456     if (!AImageReader::isSupportedFormat(format)) {
457         ALOGE("%s: format %d is not supported by AImageReader",
458                 __FUNCTION__, format);
459         return AMEDIA_ERROR_INVALID_PARAMETER;
460     }
461 
462     if (reader == nullptr) {
463         ALOGE("%s: reader argument is null", __FUNCTION__);
464         return AMEDIA_ERROR_INVALID_PARAMETER;
465     }
466 
467     //*reader = new AImageReader(width, height, format, maxImages);
468     AImageReader* tmpReader = new AImageReader(width, height, format, maxImages);
469     if (tmpReader == nullptr) {
470         ALOGE("%s: AImageReader allocation failed", __FUNCTION__);
471         return AMEDIA_ERROR_UNKNOWN;
472     }
473     media_status_t ret = tmpReader->init();
474     if (ret != AMEDIA_OK) {
475         ALOGE("%s: AImageReader initialization failed!", __FUNCTION__);
476         delete tmpReader;
477         return ret;
478     }
479     *reader = tmpReader;
480     (*reader)->incStrong((void*) AImageReader_new);
481     return AMEDIA_OK;
482 }
483 
484 EXPORT
AImageReader_delete(AImageReader * reader)485 void AImageReader_delete(AImageReader* reader) {
486     ALOGV("%s", __FUNCTION__);
487     if (reader != nullptr) {
488         reader->decStrong((void*) AImageReader_delete);
489     }
490     return;
491 }
492 
493 EXPORT
AImageReader_getWindow(AImageReader * reader,ANativeWindow ** window)494 media_status_t AImageReader_getWindow(AImageReader* reader, /*out*/ANativeWindow** window) {
495     ALOGE("%s", __FUNCTION__);
496     if (reader == nullptr || window == nullptr) {
497         ALOGE("%s: invalid argument. reader %p, window %p",
498                 __FUNCTION__, reader, window);
499         return AMEDIA_ERROR_INVALID_PARAMETER;
500     }
501     *window = reader->getWindow();
502     return AMEDIA_OK;
503 }
504 
505 EXPORT
AImageReader_getWidth(const AImageReader * reader,int32_t * width)506 media_status_t AImageReader_getWidth(const AImageReader* reader, /*out*/int32_t* width) {
507     ALOGV("%s", __FUNCTION__);
508     if (reader == nullptr || width == nullptr) {
509         ALOGE("%s: invalid argument. reader %p, width %p",
510                 __FUNCTION__, reader, width);
511         return AMEDIA_ERROR_INVALID_PARAMETER;
512     }
513     *width = reader->getWidth();
514     return AMEDIA_OK;
515 }
516 
517 EXPORT
AImageReader_getHeight(const AImageReader * reader,int32_t * height)518 media_status_t AImageReader_getHeight(const AImageReader* reader, /*out*/int32_t* height) {
519     ALOGV("%s", __FUNCTION__);
520     if (reader == nullptr || height == nullptr) {
521         ALOGE("%s: invalid argument. reader %p, height %p",
522                 __FUNCTION__, reader, height);
523         return AMEDIA_ERROR_INVALID_PARAMETER;
524     }
525     *height = reader->getHeight();
526     return AMEDIA_OK;
527 }
528 
529 EXPORT
AImageReader_getFormat(const AImageReader * reader,int32_t * format)530 media_status_t AImageReader_getFormat(const AImageReader* reader, /*out*/int32_t* format) {
531     ALOGV("%s", __FUNCTION__);
532     if (reader == nullptr || format == nullptr) {
533         ALOGE("%s: invalid argument. reader %p, format %p",
534                 __FUNCTION__, reader, format);
535         return AMEDIA_ERROR_INVALID_PARAMETER;
536     }
537     *format = reader->getFormat();
538     return AMEDIA_OK;
539 }
540 
541 EXPORT
AImageReader_getMaxImages(const AImageReader * reader,int32_t * maxImages)542 media_status_t AImageReader_getMaxImages(const AImageReader* reader, /*out*/int32_t* maxImages) {
543     ALOGV("%s", __FUNCTION__);
544     if (reader == nullptr || maxImages == nullptr) {
545         ALOGE("%s: invalid argument. reader %p, maxImages %p",
546                 __FUNCTION__, reader, maxImages);
547         return AMEDIA_ERROR_INVALID_PARAMETER;
548     }
549     *maxImages = reader->getMaxImages();
550     return AMEDIA_OK;
551 }
552 
553 EXPORT
AImageReader_acquireNextImage(AImageReader * reader,AImage ** image)554 media_status_t AImageReader_acquireNextImage(AImageReader* reader, /*out*/AImage** image) {
555     ALOGV("%s", __FUNCTION__);
556     if (reader == nullptr || image == nullptr) {
557         ALOGE("%s: invalid argument. reader %p, maxImages %p",
558                 __FUNCTION__, reader, image);
559         return AMEDIA_ERROR_INVALID_PARAMETER;
560     }
561     return reader->acquireNextImage(image);
562 }
563 
564 EXPORT
AImageReader_acquireLatestImage(AImageReader * reader,AImage ** image)565 media_status_t AImageReader_acquireLatestImage(AImageReader* reader, /*out*/AImage** image) {
566     ALOGV("%s", __FUNCTION__);
567     if (reader == nullptr || image == nullptr) {
568         ALOGE("%s: invalid argument. reader %p, maxImages %p",
569                 __FUNCTION__, reader, image);
570         return AMEDIA_ERROR_INVALID_PARAMETER;
571     }
572     return reader->acquireLatestImage(image);
573 }
574 
575 EXPORT
AImageReader_setImageListener(AImageReader * reader,AImageReader_ImageListener * listener)576 media_status_t AImageReader_setImageListener(
577         AImageReader* reader, AImageReader_ImageListener* listener) {
578     ALOGV("%s", __FUNCTION__);
579     if (reader == nullptr) {
580         ALOGE("%s: invalid argument! reader %p", __FUNCTION__, reader);
581         return AMEDIA_ERROR_INVALID_PARAMETER;
582     }
583 
584     reader->setImageListener(listener);
585     return AMEDIA_OK;
586 }
587