1 /*
2  * Copyright 2013 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "ImageReader_JNI"
19 #include "android_media_Utils.h"
20 #include <utils/Log.h>
21 #include <utils/misc.h>
22 #include <utils/List.h>
23 #include <utils/String8.h>
24 
25 #include <cstdio>
26 
27 #include <gui/BufferItemConsumer.h>
28 #include <gui/Surface.h>
29 #include <camera3.h>
30 
31 #include <android_runtime/AndroidRuntime.h>
32 #include <android_runtime/android_view_Surface.h>
33 
34 #include <jni.h>
35 #include <JNIHelp.h>
36 
37 #include <stdint.h>
38 #include <inttypes.h>
39 
40 #define ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID       "mNativeContext"
41 #define ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID   "mNativeBuffer"
42 #define ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID       "mTimestamp"
43 
44 // ----------------------------------------------------------------------------
45 
46 using namespace android;
47 
48 
49 enum {
50     ACQUIRE_SUCCESS = 0,
51     ACQUIRE_NO_BUFFERS = 1,
52     ACQUIRE_MAX_IMAGES = 2,
53 };
54 
55 static struct {
56     jfieldID mNativeContext;
57     jmethodID postEventFromNative;
58 } gImageReaderClassInfo;
59 
60 static struct {
61     jfieldID mNativeBuffer;
62     jfieldID mTimestamp;
63     jfieldID mPlanes;
64 } gSurfaceImageClassInfo;
65 
66 static struct {
67     jclass clazz;
68     jmethodID ctor;
69 } gSurfacePlaneClassInfo;
70 
71 // Get an ID that's unique within this process.
createProcessUniqueId()72 static int32_t createProcessUniqueId() {
73     static volatile int32_t globalCounter = 0;
74     return android_atomic_inc(&globalCounter);
75 }
76 
77 // ----------------------------------------------------------------------------
78 
79 class JNIImageReaderContext : public ConsumerBase::FrameAvailableListener
80 {
81 public:
82     JNIImageReaderContext(JNIEnv* env, jobject weakThiz, jclass clazz, int maxImages);
83 
84     virtual ~JNIImageReaderContext();
85 
86     virtual void onFrameAvailable(const BufferItem& item);
87 
88     BufferItem* getBufferItem();
89     void returnBufferItem(BufferItem* buffer);
90 
91 
setBufferConsumer(const sp<BufferItemConsumer> & consumer)92     void setBufferConsumer(const sp<BufferItemConsumer>& consumer) { mConsumer = consumer; }
getBufferConsumer()93     BufferItemConsumer* getBufferConsumer() { return mConsumer.get(); }
94 
setProducer(const sp<IGraphicBufferProducer> & producer)95     void setProducer(const sp<IGraphicBufferProducer>& producer) { mProducer = producer; }
getProducer()96     IGraphicBufferProducer* getProducer() { return mProducer.get(); }
97 
setBufferFormat(int format)98     void setBufferFormat(int format) { mFormat = format; }
getBufferFormat()99     int getBufferFormat() { return mFormat; }
100 
setBufferDataspace(android_dataspace dataSpace)101     void setBufferDataspace(android_dataspace dataSpace) { mDataSpace = dataSpace; }
getBufferDataspace()102     android_dataspace getBufferDataspace() { return mDataSpace; }
103 
setBufferWidth(int width)104     void setBufferWidth(int width) { mWidth = width; }
getBufferWidth()105     int getBufferWidth() { return mWidth; }
106 
setBufferHeight(int height)107     void setBufferHeight(int height) { mHeight = height; }
getBufferHeight()108     int getBufferHeight() { return mHeight; }
109 
110 private:
111     static JNIEnv* getJNIEnv(bool* needsDetach);
112     static void detachJNI();
113 
114     List<BufferItem*> mBuffers;
115     sp<BufferItemConsumer> mConsumer;
116     sp<IGraphicBufferProducer> mProducer;
117     jobject mWeakThiz;
118     jclass mClazz;
119     int mFormat;
120     android_dataspace mDataSpace;
121     int mWidth;
122     int mHeight;
123 };
124 
JNIImageReaderContext(JNIEnv * env,jobject weakThiz,jclass clazz,int maxImages)125 JNIImageReaderContext::JNIImageReaderContext(JNIEnv* env,
126         jobject weakThiz, jclass clazz, int maxImages) :
127     mWeakThiz(env->NewGlobalRef(weakThiz)),
128     mClazz((jclass)env->NewGlobalRef(clazz)),
129     mFormat(0),
130     mDataSpace(HAL_DATASPACE_UNKNOWN),
131     mWidth(-1),
132     mHeight(-1) {
133     for (int i = 0; i < maxImages; i++) {
134         BufferItem* buffer = new BufferItem;
135         mBuffers.push_back(buffer);
136     }
137 }
138 
getJNIEnv(bool * needsDetach)139 JNIEnv* JNIImageReaderContext::getJNIEnv(bool* needsDetach) {
140     LOG_ALWAYS_FATAL_IF(needsDetach == NULL, "needsDetach is null!!!");
141     *needsDetach = false;
142     JNIEnv* env = AndroidRuntime::getJNIEnv();
143     if (env == NULL) {
144         JavaVMAttachArgs args = {JNI_VERSION_1_4, NULL, NULL};
145         JavaVM* vm = AndroidRuntime::getJavaVM();
146         int result = vm->AttachCurrentThread(&env, (void*) &args);
147         if (result != JNI_OK) {
148             ALOGE("thread attach failed: %#x", result);
149             return NULL;
150         }
151         *needsDetach = true;
152     }
153     return env;
154 }
155 
detachJNI()156 void JNIImageReaderContext::detachJNI() {
157     JavaVM* vm = AndroidRuntime::getJavaVM();
158     int result = vm->DetachCurrentThread();
159     if (result != JNI_OK) {
160         ALOGE("thread detach failed: %#x", result);
161     }
162 }
163 
getBufferItem()164 BufferItem* JNIImageReaderContext::getBufferItem() {
165     if (mBuffers.empty()) {
166         return NULL;
167     }
168     // Return a BufferItem pointer and remove it from the list
169     List<BufferItem*>::iterator it = mBuffers.begin();
170     BufferItem* buffer = *it;
171     mBuffers.erase(it);
172     return buffer;
173 }
174 
returnBufferItem(BufferItem * buffer)175 void JNIImageReaderContext::returnBufferItem(BufferItem* buffer) {
176     mBuffers.push_back(buffer);
177 }
178 
~JNIImageReaderContext()179 JNIImageReaderContext::~JNIImageReaderContext() {
180     bool needsDetach = false;
181     JNIEnv* env = getJNIEnv(&needsDetach);
182     if (env != NULL) {
183         env->DeleteGlobalRef(mWeakThiz);
184         env->DeleteGlobalRef(mClazz);
185     } else {
186         ALOGW("leaking JNI object references");
187     }
188     if (needsDetach) {
189         detachJNI();
190     }
191 
192     // Delete buffer items.
193     for (List<BufferItem *>::iterator it = mBuffers.begin();
194             it != mBuffers.end(); it++) {
195         delete *it;
196     }
197 
198     if (mConsumer != 0) {
199         mConsumer.clear();
200     }
201 }
202 
onFrameAvailable(const BufferItem &)203 void JNIImageReaderContext::onFrameAvailable(const BufferItem& /*item*/)
204 {
205     ALOGV("%s: frame available", __FUNCTION__);
206     bool needsDetach = false;
207     JNIEnv* env = getJNIEnv(&needsDetach);
208     if (env != NULL) {
209         env->CallStaticVoidMethod(mClazz, gImageReaderClassInfo.postEventFromNative, mWeakThiz);
210     } else {
211         ALOGW("onFrameAvailable event will not posted");
212     }
213     if (needsDetach) {
214         detachJNI();
215     }
216 }
217 
218 // ----------------------------------------------------------------------------
219 
220 extern "C" {
221 
ImageReader_getContext(JNIEnv * env,jobject thiz)222 static JNIImageReaderContext* ImageReader_getContext(JNIEnv* env, jobject thiz)
223 {
224     JNIImageReaderContext *ctx;
225     ctx = reinterpret_cast<JNIImageReaderContext *>
226               (env->GetLongField(thiz, gImageReaderClassInfo.mNativeContext));
227     return ctx;
228 }
229 
ImageReader_getProducer(JNIEnv * env,jobject thiz)230 static IGraphicBufferProducer* ImageReader_getProducer(JNIEnv* env, jobject thiz)
231 {
232     ALOGV("%s:", __FUNCTION__);
233     JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
234     if (ctx == NULL) {
235         jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
236         return NULL;
237     }
238 
239     return ctx->getProducer();
240 }
241 
ImageReader_setNativeContext(JNIEnv * env,jobject thiz,sp<JNIImageReaderContext> ctx)242 static void ImageReader_setNativeContext(JNIEnv* env,
243         jobject thiz, sp<JNIImageReaderContext> ctx)
244 {
245     ALOGV("%s:", __FUNCTION__);
246     JNIImageReaderContext* const p = ImageReader_getContext(env, thiz);
247     if (ctx != 0) {
248         ctx->incStrong((void*)ImageReader_setNativeContext);
249     }
250     if (p) {
251         p->decStrong((void*)ImageReader_setNativeContext);
252     }
253     env->SetLongField(thiz, gImageReaderClassInfo.mNativeContext,
254             reinterpret_cast<jlong>(ctx.get()));
255 }
256 
ImageReader_getBufferConsumer(JNIEnv * env,jobject thiz)257 static BufferItemConsumer* ImageReader_getBufferConsumer(JNIEnv* env, jobject thiz)
258 {
259     ALOGV("%s:", __FUNCTION__);
260     JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
261     if (ctx == NULL) {
262         jniThrowRuntimeException(env, "ImageReaderContext is not initialized");
263         return NULL;
264     }
265 
266     return ctx->getBufferConsumer();
267 }
268 
Image_setBufferItem(JNIEnv * env,jobject thiz,const BufferItem * buffer)269 static void Image_setBufferItem(JNIEnv* env, jobject thiz,
270         const BufferItem* buffer)
271 {
272     env->SetLongField(thiz, gSurfaceImageClassInfo.mNativeBuffer, reinterpret_cast<jlong>(buffer));
273 }
274 
Image_getBufferItem(JNIEnv * env,jobject image)275 static BufferItem* Image_getBufferItem(JNIEnv* env, jobject image)
276 {
277     return reinterpret_cast<BufferItem*>(
278             env->GetLongField(image, gSurfaceImageClassInfo.mNativeBuffer));
279 }
280 
281 
282 // ----------------------------------------------------------------------------
283 
ImageReader_classInit(JNIEnv * env,jclass clazz)284 static void ImageReader_classInit(JNIEnv* env, jclass clazz)
285 {
286     ALOGV("%s:", __FUNCTION__);
287 
288     jclass imageClazz = env->FindClass("android/media/ImageReader$SurfaceImage");
289     LOG_ALWAYS_FATAL_IF(imageClazz == NULL,
290                         "can't find android/graphics/ImageReader$SurfaceImage");
291     gSurfaceImageClassInfo.mNativeBuffer = env->GetFieldID(
292             imageClazz, ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID, "J");
293     LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mNativeBuffer == NULL,
294                         "can't find android/graphics/ImageReader.%s",
295                         ANDROID_MEDIA_SURFACEIMAGE_BUFFER_JNI_ID);
296 
297     gSurfaceImageClassInfo.mTimestamp = env->GetFieldID(
298             imageClazz, ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID, "J");
299     LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mTimestamp == NULL,
300                         "can't find android/graphics/ImageReader.%s",
301                         ANDROID_MEDIA_SURFACEIMAGE_TS_JNI_ID);
302 
303     gSurfaceImageClassInfo.mPlanes = env->GetFieldID(
304             imageClazz, "mPlanes", "[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;");
305     LOG_ALWAYS_FATAL_IF(gSurfaceImageClassInfo.mPlanes == NULL,
306             "can't find android/media/ImageReader$ReaderSurfaceImage.mPlanes");
307 
308     gImageReaderClassInfo.mNativeContext = env->GetFieldID(
309             clazz, ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID, "J");
310     LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.mNativeContext == NULL,
311                         "can't find android/graphics/ImageReader.%s",
312                           ANDROID_MEDIA_IMAGEREADER_CTX_JNI_ID);
313 
314     gImageReaderClassInfo.postEventFromNative = env->GetStaticMethodID(
315             clazz, "postEventFromNative", "(Ljava/lang/Object;)V");
316     LOG_ALWAYS_FATAL_IF(gImageReaderClassInfo.postEventFromNative == NULL,
317                         "can't find android/graphics/ImageReader.postEventFromNative");
318 
319     jclass planeClazz = env->FindClass("android/media/ImageReader$SurfaceImage$SurfacePlane");
320     LOG_ALWAYS_FATAL_IF(planeClazz == NULL, "Can not find SurfacePlane class");
321     // FindClass only gives a local reference of jclass object.
322     gSurfacePlaneClassInfo.clazz = (jclass) env->NewGlobalRef(planeClazz);
323     gSurfacePlaneClassInfo.ctor = env->GetMethodID(gSurfacePlaneClassInfo.clazz, "<init>",
324             "(Landroid/media/ImageReader$SurfaceImage;IILjava/nio/ByteBuffer;)V");
325     LOG_ALWAYS_FATAL_IF(gSurfacePlaneClassInfo.ctor == NULL,
326             "Can not find SurfacePlane constructor");
327 }
328 
ImageReader_init(JNIEnv * env,jobject thiz,jobject weakThiz,jint width,jint height,jint format,jint maxImages)329 static void ImageReader_init(JNIEnv* env, jobject thiz, jobject weakThiz,
330                              jint width, jint height, jint format, jint maxImages)
331 {
332     status_t res;
333     int nativeFormat;
334     android_dataspace nativeDataspace;
335 
336     ALOGV("%s: width:%d, height: %d, format: 0x%x, maxImages:%d",
337           __FUNCTION__, width, height, format, maxImages);
338 
339     PublicFormat publicFormat = static_cast<PublicFormat>(format);
340     nativeFormat = android_view_Surface_mapPublicFormatToHalFormat(
341         publicFormat);
342     nativeDataspace = android_view_Surface_mapPublicFormatToHalDataspace(
343         publicFormat);
344 
345     jclass clazz = env->GetObjectClass(thiz);
346     if (clazz == NULL) {
347         jniThrowRuntimeException(env, "Can't find android/graphics/ImageReader");
348         return;
349     }
350     sp<JNIImageReaderContext> ctx(new JNIImageReaderContext(env, weakThiz, clazz, maxImages));
351 
352     sp<IGraphicBufferProducer> gbProducer;
353     sp<IGraphicBufferConsumer> gbConsumer;
354     BufferQueue::createBufferQueue(&gbProducer, &gbConsumer);
355     sp<BufferItemConsumer> bufferConsumer;
356     String8 consumerName = String8::format("ImageReader-%dx%df%xm%d-%d-%d",
357             width, height, format, maxImages, getpid(),
358             createProcessUniqueId());
359     uint32_t consumerUsage = GRALLOC_USAGE_SW_READ_OFTEN;
360 
361     if (isFormatOpaque(nativeFormat)) {
362         // Use the SW_READ_NEVER usage to tell producer that this format is not for preview or video
363         // encoding. The only possibility will be ZSL output.
364         consumerUsage = GRALLOC_USAGE_SW_READ_NEVER;
365     }
366     bufferConsumer = new BufferItemConsumer(gbConsumer, consumerUsage, maxImages,
367             /*controlledByApp*/true);
368     if (bufferConsumer == nullptr) {
369         jniThrowExceptionFmt(env, "java/lang/RuntimeException",
370                 "Failed to allocate native buffer consumer for format 0x%x", nativeFormat);
371         return;
372     }
373     ctx->setBufferConsumer(bufferConsumer);
374     bufferConsumer->setName(consumerName);
375 
376     ctx->setProducer(gbProducer);
377     bufferConsumer->setFrameAvailableListener(ctx);
378     ImageReader_setNativeContext(env, thiz, ctx);
379     ctx->setBufferFormat(nativeFormat);
380     ctx->setBufferDataspace(nativeDataspace);
381     ctx->setBufferWidth(width);
382     ctx->setBufferHeight(height);
383 
384     // Set the width/height/format/dataspace to the bufferConsumer.
385     res = bufferConsumer->setDefaultBufferSize(width, height);
386     if (res != OK) {
387         jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
388                           "Failed to set buffer consumer default size (%dx%d) for format 0x%x",
389                           width, height, nativeFormat);
390         return;
391     }
392     res = bufferConsumer->setDefaultBufferFormat(nativeFormat);
393     if (res != OK) {
394         jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
395                           "Failed to set buffer consumer default format 0x%x", nativeFormat);
396     }
397     res = bufferConsumer->setDefaultBufferDataSpace(nativeDataspace);
398     if (res != OK) {
399         jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
400                           "Failed to set buffer consumer default dataSpace 0x%x", nativeDataspace);
401     }
402 }
403 
ImageReader_close(JNIEnv * env,jobject thiz)404 static void ImageReader_close(JNIEnv* env, jobject thiz)
405 {
406     ALOGV("%s:", __FUNCTION__);
407 
408     JNIImageReaderContext* const ctx = ImageReader_getContext(env, thiz);
409     if (ctx == NULL) {
410         // ImageReader is already closed.
411         return;
412     }
413 
414     BufferItemConsumer* consumer = NULL;
415     consumer = ImageReader_getBufferConsumer(env, thiz);
416 
417     if (consumer != NULL) {
418         consumer->abandon();
419         consumer->setFrameAvailableListener(NULL);
420     }
421     ImageReader_setNativeContext(env, thiz, NULL);
422 }
423 
Image_unlockIfLocked(JNIEnv * env,jobject image)424 static sp<Fence> Image_unlockIfLocked(JNIEnv* env, jobject image) {
425     ALOGV("%s", __FUNCTION__);
426     BufferItem* buffer = Image_getBufferItem(env, image);
427     if (buffer == NULL) {
428         jniThrowException(env, "java/lang/IllegalStateException",
429                 "Image is not initialized");
430         return Fence::NO_FENCE;
431     }
432 
433     // Is locked?
434     bool wasBufferLocked = false;
435     jobject planes = NULL;
436     if (!isFormatOpaque(buffer->mGraphicBuffer->getPixelFormat())) {
437         planes = env->GetObjectField(image, gSurfaceImageClassInfo.mPlanes);
438     }
439     wasBufferLocked = (planes != NULL);
440     if (wasBufferLocked) {
441         status_t res = OK;
442         int fenceFd = -1;
443         if (wasBufferLocked) {
444             res = buffer->mGraphicBuffer->unlockAsync(&fenceFd);
445             if (res != OK) {
446                 jniThrowRuntimeException(env, "unlock buffer failed");
447                 return Fence::NO_FENCE;
448             }
449         }
450         sp<Fence> releaseFence = new Fence(fenceFd);
451         return releaseFence;
452         ALOGV("Successfully unlocked the image");
453     }
454     return Fence::NO_FENCE;
455 }
456 
ImageReader_imageRelease(JNIEnv * env,jobject thiz,jobject image)457 static void ImageReader_imageRelease(JNIEnv* env, jobject thiz, jobject image)
458 {
459     ALOGV("%s:", __FUNCTION__);
460     JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
461     if (ctx == NULL) {
462         ALOGW("ImageReader#close called before Image#close, consider calling Image#close first");
463         return;
464     }
465 
466     BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
467     BufferItem* buffer = Image_getBufferItem(env, image);
468     if (buffer == nullptr) {
469         // Release an already closed image is harmless.
470         return;
471     }
472 
473     sp<Fence> releaseFence = Image_unlockIfLocked(env, image);
474     bufferConsumer->releaseBuffer(*buffer, releaseFence);
475     Image_setBufferItem(env, image, NULL);
476     ctx->returnBufferItem(buffer);
477     ALOGV("%s: Image (format: 0x%x) has been released", __FUNCTION__, ctx->getBufferFormat());
478 }
479 
ImageReader_imageSetup(JNIEnv * env,jobject thiz,jobject image)480 static jint ImageReader_imageSetup(JNIEnv* env, jobject thiz, jobject image) {
481     ALOGV("%s:", __FUNCTION__);
482     JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
483     if (ctx == NULL) {
484         jniThrowException(env, "java/lang/IllegalStateException",
485                 "ImageReader is not initialized or was already closed");
486         return -1;
487     }
488 
489     BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
490     BufferItem* buffer = ctx->getBufferItem();
491     if (buffer == NULL) {
492         ALOGW("Unable to acquire a buffer item, very likely client tried to acquire more than"
493             " maxImages buffers");
494         return ACQUIRE_MAX_IMAGES;
495     }
496 
497     status_t res = bufferConsumer->acquireBuffer(buffer, 0);
498     if (res != OK) {
499         ctx->returnBufferItem(buffer);
500         if (res != BufferQueue::NO_BUFFER_AVAILABLE) {
501             if (res == INVALID_OPERATION) {
502                 // Max number of images were already acquired.
503                 ALOGE("%s: Max number of buffers allowed are already acquired : %s (%d)",
504                         __FUNCTION__, strerror(-res), res);
505                 return ACQUIRE_MAX_IMAGES;
506             } else {
507                 ALOGE("%s: Acquire image failed with some unknown error: %s (%d)",
508                         __FUNCTION__, strerror(-res), res);
509                 jniThrowExceptionFmt(env, "java/lang/IllegalStateException",
510                         "Unknown error (%d) when we tried to acquire an image.",
511                                           res);
512                 return ACQUIRE_NO_BUFFERS;
513             }
514         }
515         // This isn't really an error case, as the application may acquire buffer at any time.
516         return ACQUIRE_NO_BUFFERS;
517     }
518 
519     // Add some extra checks for non-opaque formats.
520     if (!isFormatOpaque(ctx->getBufferFormat())) {
521         // Check if the left-top corner of the crop rect is origin, we currently assume this point is
522         // zero, will revisit this once this assumption turns out problematic.
523         Point lt = buffer->mCrop.leftTop();
524         if (lt.x != 0 || lt.y != 0) {
525             jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
526                     "crop left top corner [%d, %d] need to be at origin", lt.x, lt.y);
527             return -1;
528         }
529 
530         // Check if the producer buffer configurations match what ImageReader configured.
531         int outputWidth = getBufferWidth(buffer);
532         int outputHeight = getBufferHeight(buffer);
533 
534         int imgReaderFmt = ctx->getBufferFormat();
535         int imageReaderWidth = ctx->getBufferWidth();
536         int imageReaderHeight = ctx->getBufferHeight();
537         int bufferFormat = buffer->mGraphicBuffer->getPixelFormat();
538         if ((bufferFormat != HAL_PIXEL_FORMAT_BLOB) && (imgReaderFmt != HAL_PIXEL_FORMAT_BLOB) &&
539                 (imageReaderWidth != outputWidth || imageReaderHeight != outputHeight)) {
540             ALOGV("%s: Producer buffer size: %dx%d, doesn't match ImageReader configured size: %dx%d",
541                     __FUNCTION__, outputWidth, outputHeight, imageReaderWidth, imageReaderHeight);
542         }
543         if (imgReaderFmt != bufferFormat) {
544             if (imgReaderFmt == HAL_PIXEL_FORMAT_YCbCr_420_888 &&
545                     isPossiblyYUV(bufferFormat)) {
546                 // Treat formats that are compatible with flexible YUV
547                 // (HAL_PIXEL_FORMAT_YCbCr_420_888) as HAL_PIXEL_FORMAT_YCbCr_420_888.
548                 ALOGV("%s: Treat buffer format to 0x%x as HAL_PIXEL_FORMAT_YCbCr_420_888",
549                         __FUNCTION__, bufferFormat);
550             } else if (imgReaderFmt == HAL_PIXEL_FORMAT_BLOB &&
551                     bufferFormat == HAL_PIXEL_FORMAT_RGBA_8888) {
552                 // Using HAL_PIXEL_FORMAT_RGBA_8888 Gralloc buffers containing JPEGs to get around
553                 // SW write limitations for (b/17379185).
554                 ALOGV("%s: Receiving JPEG in HAL_PIXEL_FORMAT_RGBA_8888 buffer.", __FUNCTION__);
555             } else {
556                 // Return the buffer to the queue. No need to provide fence, as this buffer wasn't
557                 // used anywhere yet.
558                 bufferConsumer->releaseBuffer(*buffer);
559                 ctx->returnBufferItem(buffer);
560 
561                 // Throw exception
562                 ALOGE("Producer output buffer format: 0x%x, ImageReader configured format: 0x%x",
563                         bufferFormat, ctx->getBufferFormat());
564                 String8 msg;
565                 msg.appendFormat("The producer output buffer format 0x%x doesn't "
566                         "match the ImageReader's configured buffer format 0x%x.",
567                         bufferFormat, ctx->getBufferFormat());
568                 jniThrowException(env, "java/lang/UnsupportedOperationException",
569                         msg.string());
570                 return -1;
571             }
572         }
573 
574     }
575 
576     // Set SurfaceImage instance member variables
577     Image_setBufferItem(env, image, buffer);
578     env->SetLongField(image, gSurfaceImageClassInfo.mTimestamp,
579             static_cast<jlong>(buffer->mTimestamp));
580 
581     return ACQUIRE_SUCCESS;
582 }
583 
ImageReader_detachImage(JNIEnv * env,jobject thiz,jobject image)584 static jint ImageReader_detachImage(JNIEnv* env, jobject thiz, jobject image) {
585     ALOGV("%s:", __FUNCTION__);
586     JNIImageReaderContext* ctx = ImageReader_getContext(env, thiz);
587     if (ctx == NULL) {
588         jniThrowException(env, "java/lang/IllegalStateException", "ImageReader was already closed");
589         return -1;
590     }
591 
592     BufferItemConsumer* bufferConsumer = ctx->getBufferConsumer();
593     BufferItem* buffer = Image_getBufferItem(env, image);
594     if (!buffer) {
595         ALOGE(
596                 "Image already released and can not be detached from ImageReader!!!");
597         jniThrowException(env, "java/lang/IllegalStateException",
598                 "Image detach from ImageReader failed: buffer was already released");
599         return -1;
600     }
601 
602     status_t res = OK;
603     Image_unlockIfLocked(env, image);
604     res = bufferConsumer->detachBuffer(buffer->mSlot);
605     if (res != OK) {
606         ALOGE("Image detach failed: %s (%d)!!!", strerror(-res), res);
607         jniThrowRuntimeException(env,
608                 "nativeDetachImage failed for image!!!");
609         return res;
610     }
611     return OK;
612 }
613 
ImageReader_getSurface(JNIEnv * env,jobject thiz)614 static jobject ImageReader_getSurface(JNIEnv* env, jobject thiz)
615 {
616     ALOGV("%s: ", __FUNCTION__);
617 
618     IGraphicBufferProducer* gbp = ImageReader_getProducer(env, thiz);
619     if (gbp == NULL) {
620         jniThrowRuntimeException(env, "Buffer consumer is uninitialized");
621         return NULL;
622     }
623 
624     // Wrap the IGBP in a Java-language Surface.
625     return android_view_Surface_createFromIGraphicBufferProducer(env, gbp);
626 }
627 
Image_getLockedImage(JNIEnv * env,jobject thiz,LockedImage * image)628 static void Image_getLockedImage(JNIEnv* env, jobject thiz, LockedImage *image) {
629     ALOGV("%s", __FUNCTION__);
630     BufferItem* buffer = Image_getBufferItem(env, thiz);
631     if (buffer == NULL) {
632         jniThrowException(env, "java/lang/IllegalStateException",
633                 "Image is not initialized");
634         return;
635     }
636 
637     status_t res = lockImageFromBuffer(buffer,
638             GRALLOC_USAGE_SW_READ_OFTEN, buffer->mFence->dup(), image);
639     if (res != OK) {
640         jniThrowExceptionFmt(env, "java/lang/RuntimeException",
641                 "lock buffer failed for format 0x%x",
642                 buffer->mGraphicBuffer->getPixelFormat());
643         return;
644     }
645 
646     // Carry over some fields from BufferItem.
647     image->crop        = buffer->mCrop;
648     image->transform   = buffer->mTransform;
649     image->scalingMode = buffer->mScalingMode;
650     image->timestamp   = buffer->mTimestamp;
651     image->dataSpace   = buffer->mDataSpace;
652     image->frameNumber = buffer->mFrameNumber;
653 
654     ALOGV("%s: Successfully locked the image", __FUNCTION__);
655     // crop, transform, scalingMode, timestamp, and frameNumber should be set by producer,
656     // and we don't set them here.
657 }
658 
Image_getLockedImageInfo(JNIEnv * env,LockedImage * buffer,int idx,int32_t writerFormat,uint8_t ** base,uint32_t * size,int * pixelStride,int * rowStride)659 static void Image_getLockedImageInfo(JNIEnv* env, LockedImage* buffer, int idx,
660         int32_t writerFormat, uint8_t **base, uint32_t *size, int *pixelStride, int *rowStride) {
661     ALOGV("%s", __FUNCTION__);
662 
663     status_t res = getLockedImageInfo(buffer, idx, writerFormat, base, size,
664             pixelStride, rowStride);
665     if (res != OK) {
666         jniThrowExceptionFmt(env, "java/lang/UnsupportedOperationException",
667                              "Pixel format: 0x%x is unsupported", buffer->flexFormat);
668     }
669 }
670 
Image_createSurfacePlanes(JNIEnv * env,jobject thiz,int numPlanes,int readerFormat)671 static jobjectArray Image_createSurfacePlanes(JNIEnv* env, jobject thiz,
672         int numPlanes, int readerFormat)
673 {
674     ALOGV("%s: create SurfacePlane array with size %d", __FUNCTION__, numPlanes);
675     int rowStride = 0;
676     int pixelStride = 0;
677     uint8_t *pData = NULL;
678     uint32_t dataSize = 0;
679     jobject byteBuffer = NULL;
680 
681     PublicFormat publicReaderFormat = static_cast<PublicFormat>(readerFormat);
682     int halReaderFormat = android_view_Surface_mapPublicFormatToHalFormat(
683         publicReaderFormat);
684 
685     if (isFormatOpaque(halReaderFormat) && numPlanes > 0) {
686         String8 msg;
687         msg.appendFormat("Format 0x%x is opaque, thus not writable, the number of planes (%d)"
688                 " must be 0", halReaderFormat, numPlanes);
689         jniThrowException(env, "java/lang/IllegalArgumentException", msg.string());
690         return NULL;
691     }
692 
693     jobjectArray surfacePlanes = env->NewObjectArray(numPlanes, gSurfacePlaneClassInfo.clazz,
694             /*initial_element*/NULL);
695     if (surfacePlanes == NULL) {
696         jniThrowRuntimeException(env, "Failed to create SurfacePlane arrays,"
697                 " probably out of memory");
698         return NULL;
699     }
700     if (isFormatOpaque(halReaderFormat)) {
701         // Return 0 element surface array.
702         return surfacePlanes;
703     }
704 
705     LockedImage lockedImg = LockedImage();
706     Image_getLockedImage(env, thiz, &lockedImg);
707     // Create all SurfacePlanes
708     for (int i = 0; i < numPlanes; i++) {
709         Image_getLockedImageInfo(env, &lockedImg, i, halReaderFormat,
710                 &pData, &dataSize, &pixelStride, &rowStride);
711         byteBuffer = env->NewDirectByteBuffer(pData, dataSize);
712         if ((byteBuffer == NULL) && (env->ExceptionCheck() == false)) {
713             jniThrowException(env, "java/lang/IllegalStateException",
714                     "Failed to allocate ByteBuffer");
715             return NULL;
716         }
717 
718         // Finally, create this SurfacePlane.
719         jobject surfacePlane = env->NewObject(gSurfacePlaneClassInfo.clazz,
720                     gSurfacePlaneClassInfo.ctor, thiz, rowStride, pixelStride, byteBuffer);
721         env->SetObjectArrayElement(surfacePlanes, i, surfacePlane);
722     }
723 
724     return surfacePlanes;
725 }
726 
Image_getWidth(JNIEnv * env,jobject thiz)727 static jint Image_getWidth(JNIEnv* env, jobject thiz)
728 {
729     BufferItem* buffer = Image_getBufferItem(env, thiz);
730     return getBufferWidth(buffer);
731 }
732 
Image_getHeight(JNIEnv * env,jobject thiz)733 static jint Image_getHeight(JNIEnv* env, jobject thiz)
734 {
735     BufferItem* buffer = Image_getBufferItem(env, thiz);
736     return getBufferHeight(buffer);
737 }
738 
Image_getFormat(JNIEnv * env,jobject thiz,jint readerFormat)739 static jint Image_getFormat(JNIEnv* env, jobject thiz, jint readerFormat)
740 {
741     if (isFormatOpaque(readerFormat)) {
742         // Assuming opaque reader produce opaque images.
743         return static_cast<jint>(PublicFormat::PRIVATE);
744     } else {
745         BufferItem* buffer = Image_getBufferItem(env, thiz);
746         int readerHalFormat = android_view_Surface_mapPublicFormatToHalFormat(
747                 static_cast<PublicFormat>(readerFormat));
748         int32_t fmt = applyFormatOverrides(
749                 buffer->mGraphicBuffer->getPixelFormat(), readerHalFormat);
750         // Override the image format to HAL_PIXEL_FORMAT_YCbCr_420_888 if the actual format is
751         // NV21 or YV12. This could only happen when the Gralloc HAL version is v0.1 thus doesn't
752         // support lockycbcr(), the CpuConsumer need to use the lock() method in the
753         // lockNextBuffer() call. For Gralloc HAL v0.2 or newer, this format should already be
754         // overridden to HAL_PIXEL_FORMAT_YCbCr_420_888 for the flexible YUV compatible formats.
755         if (isPossiblyYUV(fmt)) {
756             fmt = HAL_PIXEL_FORMAT_YCbCr_420_888;
757         }
758         PublicFormat publicFmt = android_view_Surface_mapHalFormatDataspaceToPublicFormat(
759                 fmt, buffer->mDataSpace);
760         return static_cast<jint>(publicFmt);
761     }
762 }
763 
764 } // extern "C"
765 
766 // ----------------------------------------------------------------------------
767 
768 static const JNINativeMethod gImageReaderMethods[] = {
769     {"nativeClassInit",        "()V",                        (void*)ImageReader_classInit },
770     {"nativeInit",             "(Ljava/lang/Object;IIII)V",  (void*)ImageReader_init },
771     {"nativeClose",            "()V",                        (void*)ImageReader_close },
772     {"nativeReleaseImage",     "(Landroid/media/Image;)V",   (void*)ImageReader_imageRelease },
773     {"nativeImageSetup",       "(Landroid/media/Image;)I",   (void*)ImageReader_imageSetup },
774     {"nativeGetSurface",       "()Landroid/view/Surface;",   (void*)ImageReader_getSurface },
775     {"nativeDetachImage",      "(Landroid/media/Image;)I",   (void*)ImageReader_detachImage },
776 };
777 
778 static const JNINativeMethod gImageMethods[] = {
779     {"nativeCreatePlanes",      "(II)[Landroid/media/ImageReader$SurfaceImage$SurfacePlane;",
780                                                               (void*)Image_createSurfacePlanes },
781     {"nativeGetWidth",         "()I",                        (void*)Image_getWidth },
782     {"nativeGetHeight",        "()I",                        (void*)Image_getHeight },
783     {"nativeGetFormat",        "(I)I",                        (void*)Image_getFormat },
784 };
785 
register_android_media_ImageReader(JNIEnv * env)786 int register_android_media_ImageReader(JNIEnv *env) {
787 
788     int ret1 = AndroidRuntime::registerNativeMethods(env,
789                    "android/media/ImageReader", gImageReaderMethods, NELEM(gImageReaderMethods));
790 
791     int ret2 = AndroidRuntime::registerNativeMethods(env,
792                    "android/media/ImageReader$SurfaceImage", gImageMethods, NELEM(gImageMethods));
793 
794     return (ret1 || ret2);
795 }
796