1 /*
2  * Copyright (C) 2019 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 
17 //#define LOG_NDEBUG 0
18 #define LOG_TAG "JpegCompressor"
19 
20 #include "JpegCompressor.h"
21 
22 #include <cutils/properties.h>
23 #include <hardware/camera3.h>
24 #include <libyuv.h>
25 #include <utils/Log.h>
26 #include <utils/Trace.h>
27 
28 namespace android {
29 
30 using google_camera_hal::ErrorCode;
31 using google_camera_hal::MessageType;
32 using google_camera_hal::NotifyMessage;
33 
JpegCompressor()34 JpegCompressor::JpegCompressor() {
35   ATRACE_CALL();
36   char value[PROPERTY_VALUE_MAX];
37   if (property_get("ro.product.manufacturer", value, "unknown") <= 0) {
38     ALOGW("%s: No Exif make data!", __FUNCTION__);
39   }
40   exif_make_ = std::string(value);
41 
42   if (property_get("ro.product.model", value, "unknown") <= 0) {
43     ALOGW("%s: No Exif model data!", __FUNCTION__);
44   }
45   exif_model_ = std::string(value);
46 
47   jpeg_processing_thread_ = std::thread([this] { this->ThreadLoop(); });
48 }
49 
~JpegCompressor()50 JpegCompressor::~JpegCompressor() {
51   ATRACE_CALL();
52 
53   // Abort the ongoing compression and flush any pending jobs
54   jpeg_done_ = true;
55   condition_.notify_one();
56   jpeg_processing_thread_.join();
57   while (!pending_yuv_jobs_.empty()) {
58     auto job = std::move(pending_yuv_jobs_.front());
59     job->output->stream_buffer.status = BufferStatus::kError;
60     pending_yuv_jobs_.pop();
61   }
62 }
63 
QueueYUV420(std::unique_ptr<JpegYUV420Job> job)64 status_t JpegCompressor::QueueYUV420(std::unique_ptr<JpegYUV420Job> job) {
65   ATRACE_CALL();
66 
67   if ((job->input.get() == nullptr) || (job->output.get() == nullptr) ||
68       (job->output->format != PixelFormat::BLOB) ||
69       (job->output->dataSpace != HAL_DATASPACE_V0_JFIF)) {
70     ALOGE("%s: Unable to find buffers for JPEG source/destination",
71           __FUNCTION__);
72     return BAD_VALUE;
73   }
74 
75   std::unique_lock<std::mutex> lock(mutex_);
76   pending_yuv_jobs_.push(std::move(job));
77   condition_.notify_one();
78 
79   return OK;
80 }
81 
ThreadLoop()82 void JpegCompressor::ThreadLoop() {
83   ATRACE_CALL();
84 
85   while (!jpeg_done_) {
86     std::unique_ptr<JpegYUV420Job> current_yuv_job = nullptr;
87     {
88       std::lock_guard<std::mutex> lock(mutex_);
89       if (!pending_yuv_jobs_.empty()) {
90         current_yuv_job = std::move(pending_yuv_jobs_.front());
91         pending_yuv_jobs_.pop();
92       }
93     }
94 
95     if (current_yuv_job.get() != nullptr) {
96       CompressYUV420(std::move(current_yuv_job));
97     }
98 
99     std::unique_lock<std::mutex> lock(mutex_);
100     auto ret = condition_.wait_for(lock, std::chrono::milliseconds(10));
101     if (ret == std::cv_status::timeout) {
102       ALOGV("%s: Jpeg thread timeout", __FUNCTION__);
103     }
104   }
105 }
106 
CompressYUV420(std::unique_ptr<JpegYUV420Job> job)107 void JpegCompressor::CompressYUV420(std::unique_ptr<JpegYUV420Job> job) {
108   const uint8_t* app1_buffer = nullptr;
109   size_t app1_buffer_size = 0;
110   std::vector<uint8_t> thumbnail_jpeg_buffer;
111   size_t encoded_thumbnail_size = 0;
112   if ((job->exif_utils.get() != nullptr) &&
113       (job->result_metadata.get() != nullptr)) {
114     if (job->exif_utils->Initialize()) {
115       camera_metadata_ro_entry_t entry;
116       size_t thumbnail_width = 0;
117       size_t thumbnail_height = 0;
118       std::vector<uint8_t> thumb_yuv420_frame;
119       YCbCrPlanes thumb_planes;
120       auto ret = job->result_metadata->Get(ANDROID_JPEG_THUMBNAIL_SIZE, &entry);
121       if ((ret == OK) && (entry.count == 2)) {
122         thumbnail_width = entry.data.i32[0];
123         thumbnail_height = entry.data.i32[1];
124         if ((thumbnail_width > 0) && (thumbnail_height > 0)) {
125           thumb_yuv420_frame.resize((thumbnail_width * thumbnail_height * 3) /
126                                     2);
127           thumb_planes = {
128               .img_y = thumb_yuv420_frame.data(),
129               .img_cb = thumb_yuv420_frame.data() +
130                         thumbnail_width * thumbnail_height,
131               .img_cr = thumb_yuv420_frame.data() +
132                         (thumbnail_width * thumbnail_height * 5) / 4,
133               .y_stride = static_cast<uint32_t>(thumbnail_width),
134               .cbcr_stride = static_cast<uint32_t>(thumbnail_width) / 2};
135           // TODO: Crop thumbnail according to documentation
136           auto stat = I420Scale(
137               job->input->yuv_planes.img_y, job->input->yuv_planes.y_stride,
138               job->input->yuv_planes.img_cb, job->input->yuv_planes.cbcr_stride,
139               job->input->yuv_planes.img_cr, job->input->yuv_planes.cbcr_stride,
140               job->input->width, job->input->height, thumb_planes.img_y,
141               thumb_planes.y_stride, thumb_planes.img_cb,
142               thumb_planes.cbcr_stride, thumb_planes.img_cr,
143               thumb_planes.cbcr_stride, thumbnail_width, thumbnail_height,
144               libyuv::kFilterNone);
145           if (stat != 0) {
146             ALOGE("%s: Failed during thumbnail scaling: %d", __FUNCTION__, stat);
147             thumb_yuv420_frame.clear();
148           }
149         }
150       }
151 
152       if (job->exif_utils->SetFromMetadata(
153               *job->result_metadata, job->input->width, job->input->height)) {
154         if (!thumb_yuv420_frame.empty()) {
155           thumbnail_jpeg_buffer.resize(64 * 1024);  // APP1 is limited by 64k
156           encoded_thumbnail_size = CompressYUV420Frame(
157               {.output_buffer = thumbnail_jpeg_buffer.data(),
158                .output_buffer_size = thumbnail_jpeg_buffer.size(),
159                .yuv_planes = thumb_planes,
160                .width = thumbnail_width,
161                .height = thumbnail_height,
162                .app1_buffer = nullptr,
163                .app1_buffer_size = 0});
164           if (encoded_thumbnail_size > 0) {
165             job->output->stream_buffer.status = BufferStatus::kOk;
166           } else {
167             ALOGE("%s: Failed encoding thumbail!", __FUNCTION__);
168             thumbnail_jpeg_buffer.clear();
169           }
170         }
171 
172         job->exif_utils->SetMake(exif_make_);
173         job->exif_utils->SetModel(exif_model_);
174         if (job->exif_utils->GenerateApp1(thumbnail_jpeg_buffer.empty()
175                                               ? nullptr
176                                               : thumbnail_jpeg_buffer.data(),
177                                           encoded_thumbnail_size)) {
178           app1_buffer = job->exif_utils->GetApp1Buffer();
179           app1_buffer_size = job->exif_utils->GetApp1Length();
180         } else {
181           ALOGE("%s: Unable to generate App1 buffer", __FUNCTION__);
182         }
183       } else {
184         ALOGE("%s: Unable to generate EXIF section!", __FUNCTION__);
185       }
186     } else {
187       ALOGE("%s: Unable to initialize Exif generator!", __FUNCTION__);
188     }
189   }
190 
191   auto encoded_size = CompressYUV420Frame(
192       {.output_buffer = job->output->plane.img.img,
193        .output_buffer_size = job->output->plane.img.buffer_size,
194        .yuv_planes = job->input->yuv_planes,
195        .width = job->input->width,
196        .height = job->input->height,
197        .app1_buffer = app1_buffer,
198        .app1_buffer_size = app1_buffer_size});
199   if (encoded_size > 0) {
200     job->output->stream_buffer.status = BufferStatus::kOk;
201   } else {
202     job->output->stream_buffer.status = BufferStatus::kError;
203     return;
204   }
205 
206   auto jpeg_header_offset =
207       job->output->plane.img.buffer_size - sizeof(struct camera3_jpeg_blob);
208   if (jpeg_header_offset > encoded_size) {
209     struct camera3_jpeg_blob* blob =
210         reinterpret_cast<struct camera3_jpeg_blob*>(job->output->plane.img.img +
211                                                     jpeg_header_offset);
212     blob->jpeg_blob_id = CAMERA3_JPEG_BLOB_ID;
213     blob->jpeg_size = encoded_size;
214   } else {
215     ALOGW("%s: No space for jpeg header at offset: %u and jpeg size: %u",
216           __FUNCTION__, static_cast<unsigned>(jpeg_header_offset),
217           static_cast<unsigned>(encoded_size));
218   }
219 }
220 
CompressYUV420Frame(YUV420Frame frame)221 size_t JpegCompressor::CompressYUV420Frame(YUV420Frame frame) {
222   ATRACE_CALL();
223 
224   struct CustomJpegDestMgr : public jpeg_destination_mgr {
225     JOCTET* buffer;
226     size_t buffer_size;
227     size_t encoded_size;
228     bool success;
229   } dmgr;
230 
231   // Set up error management
232   jpeg_error_info_ = NULL;
233   jpeg_error_mgr jerr;
234 
235   auto cinfo = std::make_unique<jpeg_compress_struct>();
236   cinfo->err = jpeg_std_error(&jerr);
237   cinfo->err->error_exit = [](j_common_ptr cinfo) {
238     (*cinfo->err->output_message)(cinfo);
239     if (cinfo->client_data) {
240       auto& dmgr = *static_cast<CustomJpegDestMgr*>(cinfo->client_data);
241       dmgr.success = false;
242     }
243   };
244 
245   jpeg_create_compress(cinfo.get());
246   if (CheckError("Error initializing compression")) {
247     return 0;
248   }
249 
250   dmgr.buffer = static_cast<JOCTET*>(frame.output_buffer);
251   dmgr.buffer_size = frame.output_buffer_size;
252   dmgr.encoded_size = 0;
253   dmgr.success = true;
254   cinfo->client_data = static_cast<void*>(&dmgr);
255   dmgr.init_destination = [](j_compress_ptr cinfo) {
256     auto& dmgr = static_cast<CustomJpegDestMgr&>(*cinfo->dest);
257     dmgr.next_output_byte = dmgr.buffer;
258     dmgr.free_in_buffer = dmgr.buffer_size;
259     ALOGV("%s:%d jpeg start: %p [%zu]", __FUNCTION__, __LINE__, dmgr.buffer,
260           dmgr.buffer_size);
261   };
262 
263   dmgr.empty_output_buffer = [](j_compress_ptr cinfo __unused) {
264     ALOGE("%s:%d Out of buffer", __FUNCTION__, __LINE__);
265     return 0;
266   };
267 
268   dmgr.term_destination = [](j_compress_ptr cinfo) {
269     auto& dmgr = static_cast<CustomJpegDestMgr&>(*cinfo->dest);
270     dmgr.encoded_size = dmgr.buffer_size - dmgr.free_in_buffer;
271     ALOGV("%s:%d Done with jpeg: %zu", __FUNCTION__, __LINE__,
272           dmgr.encoded_size);
273   };
274 
275   cinfo->dest = reinterpret_cast<struct jpeg_destination_mgr*>(&dmgr);
276 
277   // Set up compression parameters
278   cinfo->image_width = frame.width;
279   cinfo->image_height = frame.height;
280   cinfo->input_components = 3;
281   cinfo->in_color_space = JCS_YCbCr;
282 
283   jpeg_set_defaults(cinfo.get());
284   if (CheckError("Error configuring defaults")) {
285     return 0;
286   }
287 
288   jpeg_set_colorspace(cinfo.get(), JCS_YCbCr);
289   if (CheckError("Error configuring color space")) {
290     return 0;
291   }
292 
293   cinfo->raw_data_in = 1;
294   // YUV420 planar with chroma subsampling
295   cinfo->comp_info[0].h_samp_factor = 2;
296   cinfo->comp_info[0].v_samp_factor = 2;
297   cinfo->comp_info[1].h_samp_factor = 1;
298   cinfo->comp_info[1].v_samp_factor = 1;
299   cinfo->comp_info[2].h_samp_factor = 1;
300   cinfo->comp_info[2].v_samp_factor = 1;
301 
302   int max_vsamp_factor = std::max({cinfo->comp_info[0].v_samp_factor,
303                                    cinfo->comp_info[1].v_samp_factor,
304                                    cinfo->comp_info[2].v_samp_factor});
305   int c_vsub_sampling =
306       cinfo->comp_info[0].v_samp_factor / cinfo->comp_info[1].v_samp_factor;
307 
308   // Start compression
309   jpeg_start_compress(cinfo.get(), TRUE);
310   if (CheckError("Error starting compression")) {
311     return 0;
312   }
313 
314   if ((frame.app1_buffer != nullptr) && (frame.app1_buffer_size > 0)) {
315     jpeg_write_marker(cinfo.get(), JPEG_APP0 + 1,
316                       static_cast<const JOCTET*>(frame.app1_buffer),
317                       frame.app1_buffer_size);
318   }
319 
320   // Compute our macroblock height, so we can pad our input to be vertically
321   // macroblock aligned.
322 
323   size_t mcu_v = DCTSIZE * max_vsamp_factor;
324   size_t padded_height = mcu_v * ((cinfo->image_height + mcu_v - 1) / mcu_v);
325 
326   std::vector<JSAMPROW> y_lines(padded_height);
327   std::vector<JSAMPROW> cb_lines(padded_height / c_vsub_sampling);
328   std::vector<JSAMPROW> cr_lines(padded_height / c_vsub_sampling);
329 
330   uint8_t* py = static_cast<uint8_t*>(frame.yuv_planes.img_y);
331   uint8_t* pcr = static_cast<uint8_t*>(frame.yuv_planes.img_cr);
332   uint8_t* pcb = static_cast<uint8_t*>(frame.yuv_planes.img_cb);
333 
334   for (uint32_t i = 0; i < padded_height; i++) {
335     /* Once we are in the padding territory we still point to the last line
336      * effectively replicating it several times ~ CLAMP_TO_EDGE */
337     int li = std::min(i, cinfo->image_height - 1);
338     y_lines[i] = static_cast<JSAMPROW>(py + li * frame.yuv_planes.y_stride);
339     if (i < padded_height / c_vsub_sampling) {
340       li = std::min(i, (cinfo->image_height - 1) / c_vsub_sampling);
341       cr_lines[i] =
342           static_cast<JSAMPROW>(pcr + li * frame.yuv_planes.cbcr_stride);
343       cb_lines[i] =
344           static_cast<JSAMPROW>(pcb + li * frame.yuv_planes.cbcr_stride);
345     }
346   }
347 
348   const uint32_t batch_size = DCTSIZE * max_vsamp_factor;
349   while (cinfo->next_scanline < cinfo->image_height) {
350     JSAMPARRAY planes[3]{&y_lines[cinfo->next_scanline],
351                          &cb_lines[cinfo->next_scanline / c_vsub_sampling],
352                          &cr_lines[cinfo->next_scanline / c_vsub_sampling]};
353 
354     jpeg_write_raw_data(cinfo.get(), planes, batch_size);
355     if (CheckError("Error while compressing")) {
356       return 0;
357     }
358 
359     if (jpeg_done_) {
360       ALOGV("%s: Cancel called, exiting early", __FUNCTION__);
361       jpeg_finish_compress(cinfo.get());
362       return 0;
363     }
364   }
365 
366   jpeg_finish_compress(cinfo.get());
367   if (CheckError("Error while finishing compression")) {
368     return 0;
369   }
370 
371   return dmgr.encoded_size;
372 }
373 
CheckError(const char * msg)374 bool JpegCompressor::CheckError(const char* msg) {
375   if (jpeg_error_info_) {
376     char err_buffer[JMSG_LENGTH_MAX];
377     jpeg_error_info_->err->format_message(jpeg_error_info_, err_buffer);
378     ALOGE("%s: %s: %s", __FUNCTION__, msg, err_buffer);
379     jpeg_error_info_ = NULL;
380     return true;
381   }
382 
383   return false;
384 }
385 
386 }  // namespace android
387