1 /*
2 * Copyright (C) 2009 The Android Open Source Project
3 *
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
7 *
8 * http://www.apache.org/licenses/LICENSE-2.0
9 *
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
15 */
16
17 #define LOG_TAG "SoftwareRenderer"
18 #include <utils/Log.h>
19
20 #include "../include/SoftwareRenderer.h"
21 #include <cutils/properties.h> // for property_get
22 #include <media/stagefright/foundation/ADebug.h>
23 #include <media/stagefright/foundation/AMessage.h>
24 #include <media/stagefright/foundation/ColorUtils.h>
25 #include <media/stagefright/SurfaceUtils.h>
26 #include <system/window.h>
27 #include <ui/Fence.h>
28 #include <ui/GraphicBufferMapper.h>
29 #include <ui/GraphicBuffer.h>
30 #include <ui/Rect.h>
31
32 namespace android {
33
ALIGN(int x,int y)34 static int ALIGN(int x, int y) {
35 // y must be a power of 2.
36 return (x + y - 1) & ~(y - 1);
37 }
38
SoftwareRenderer(const sp<ANativeWindow> & nativeWindow,int32_t rotation)39 SoftwareRenderer::SoftwareRenderer(
40 const sp<ANativeWindow> &nativeWindow, int32_t rotation)
41 : mColorFormat(OMX_COLOR_FormatUnused),
42 mConverter(NULL),
43 mYUVMode(None),
44 mNativeWindow(nativeWindow),
45 mWidth(0),
46 mHeight(0),
47 mCropLeft(0),
48 mCropTop(0),
49 mCropRight(0),
50 mCropBottom(0),
51 mCropWidth(0),
52 mCropHeight(0),
53 mRotationDegrees(rotation),
54 mDataSpace(HAL_DATASPACE_UNKNOWN) {
55 memset(&mHDRStaticInfo, 0, sizeof(mHDRStaticInfo));
56 }
57
~SoftwareRenderer()58 SoftwareRenderer::~SoftwareRenderer() {
59 delete mConverter;
60 mConverter = NULL;
61 }
62
resetFormatIfChanged(const sp<AMessage> & format,size_t numOutputBuffers)63 void SoftwareRenderer::resetFormatIfChanged(
64 const sp<AMessage> &format, size_t numOutputBuffers) {
65 CHECK(format != NULL);
66
67 int32_t colorFormatNew;
68 CHECK(format->findInt32("color-format", &colorFormatNew));
69
70 int32_t widthNew, heightNew;
71 CHECK(format->findInt32("stride", &widthNew));
72 CHECK(format->findInt32("slice-height", &heightNew));
73
74 int32_t cropLeftNew, cropTopNew, cropRightNew, cropBottomNew;
75 if (!format->findRect(
76 "crop", &cropLeftNew, &cropTopNew, &cropRightNew, &cropBottomNew)) {
77 cropLeftNew = cropTopNew = 0;
78 cropRightNew = widthNew - 1;
79 cropBottomNew = heightNew - 1;
80 }
81
82 // The native window buffer format for high-bitdepth content could
83 // depend on the dataspace also.
84 android_dataspace dataSpace;
85 bool dataSpaceChangedForPlanar16 = false;
86 if (colorFormatNew == OMX_COLOR_FormatYUV420Planar16
87 && format->findInt32("android._dataspace", (int32_t *)&dataSpace)
88 && dataSpace != mDataSpace) {
89 // Do not modify mDataSpace here, it's only modified at last
90 // when we do native_window_set_buffers_data_space().
91 dataSpaceChangedForPlanar16 = true;
92 }
93
94 if (static_cast<int32_t>(mColorFormat) == colorFormatNew &&
95 mWidth == widthNew &&
96 mHeight == heightNew &&
97 mCropLeft == cropLeftNew &&
98 mCropTop == cropTopNew &&
99 mCropRight == cropRightNew &&
100 mCropBottom == cropBottomNew &&
101 !dataSpaceChangedForPlanar16) {
102 // Nothing changed, no need to reset renderer.
103 return;
104 }
105
106 mColorFormat = static_cast<OMX_COLOR_FORMATTYPE>(colorFormatNew);
107 mWidth = widthNew;
108 mHeight = heightNew;
109 mCropLeft = cropLeftNew;
110 mCropTop = cropTopNew;
111 mCropRight = cropRightNew;
112 mCropBottom = cropBottomNew;
113
114 mCropWidth = mCropRight - mCropLeft + 1;
115 mCropHeight = mCropBottom - mCropTop + 1;
116
117 // by default convert everything to RGB565
118 int halFormat = HAL_PIXEL_FORMAT_RGB_565;
119 size_t bufWidth = mCropWidth;
120 size_t bufHeight = mCropHeight;
121
122 // hardware has YUV12 and RGBA8888 support, so convert known formats
123 {
124 switch (mColorFormat) {
125 case OMX_COLOR_FormatYUV420Planar:
126 case OMX_COLOR_FormatYUV420SemiPlanar:
127 case OMX_TI_COLOR_FormatYUV420PackedSemiPlanar:
128 {
129 halFormat = HAL_PIXEL_FORMAT_YV12;
130 bufWidth = (mCropWidth + 1) & ~1;
131 bufHeight = (mCropHeight + 1) & ~1;
132 break;
133 }
134 case OMX_COLOR_Format24bitRGB888:
135 {
136 halFormat = HAL_PIXEL_FORMAT_RGB_888;
137 bufWidth = (mCropWidth + 1) & ~1;
138 bufHeight = (mCropHeight + 1) & ~1;
139 break;
140 }
141 case OMX_COLOR_Format32bitARGB8888:
142 case OMX_COLOR_Format32BitRGBA8888:
143 {
144 halFormat = HAL_PIXEL_FORMAT_RGBA_8888;
145 bufWidth = (mCropWidth + 1) & ~1;
146 bufHeight = (mCropHeight + 1) & ~1;
147 break;
148 }
149 case OMX_COLOR_FormatYUV420Planar16:
150 {
151 if (((dataSpace & HAL_DATASPACE_STANDARD_MASK) == HAL_DATASPACE_STANDARD_BT2020)
152 && ((dataSpace & HAL_DATASPACE_TRANSFER_MASK) == HAL_DATASPACE_TRANSFER_ST2084)) {
153 // Here we would convert OMX_COLOR_FormatYUV420Planar16 into
154 // OMX_COLOR_FormatYUV444Y410, and put it inside a buffer with
155 // format HAL_PIXEL_FORMAT_RGBA_1010102. Surfaceflinger will
156 // use render engine to convert it to RGB if needed.
157 halFormat = HAL_PIXEL_FORMAT_RGBA_1010102;
158 } else {
159 halFormat = HAL_PIXEL_FORMAT_YV12;
160 }
161 bufWidth = (mCropWidth + 1) & ~1;
162 bufHeight = (mCropHeight + 1) & ~1;
163 break;
164 }
165 default:
166 {
167 break;
168 }
169 }
170 }
171
172 if (halFormat == HAL_PIXEL_FORMAT_RGB_565) {
173 mConverter = new ColorConverter(
174 mColorFormat, OMX_COLOR_Format16bitRGB565);
175 CHECK(mConverter->isValid());
176 } else if (halFormat == HAL_PIXEL_FORMAT_RGBA_1010102) {
177 mConverter = new ColorConverter(
178 mColorFormat, OMX_COLOR_FormatYUV444Y410);
179 CHECK(mConverter->isValid());
180 }
181
182 CHECK(mNativeWindow != NULL);
183 CHECK(mCropWidth > 0);
184 CHECK(mCropHeight > 0);
185 CHECK(mConverter == NULL || mConverter->isValid());
186
187 CHECK_EQ(0,
188 native_window_set_usage(
189 mNativeWindow.get(),
190 GRALLOC_USAGE_SW_READ_NEVER | GRALLOC_USAGE_SW_WRITE_RARELY
191 | GRALLOC_USAGE_HW_TEXTURE | GRALLOC_USAGE_EXTERNAL_DISP));
192
193 CHECK_EQ(0,
194 native_window_set_scaling_mode(
195 mNativeWindow.get(),
196 NATIVE_WINDOW_SCALING_MODE_SCALE_TO_WINDOW));
197
198 // Width must be multiple of 32???
199 CHECK_EQ(0, native_window_set_buffers_dimensions(
200 mNativeWindow.get(),
201 bufWidth,
202 bufHeight));
203 CHECK_EQ(0, native_window_set_buffers_format(
204 mNativeWindow.get(),
205 halFormat));
206 if (OK != native_window_set_buffer_count(
207 mNativeWindow.get(), numOutputBuffers + 4)) {
208 ALOGE("Failed to set native window buffer count to (%zu + 4)",
209 numOutputBuffers);
210 }
211
212 // NOTE: native window uses extended right-bottom coordinate
213 android_native_rect_t crop;
214 crop.left = mCropLeft;
215 crop.top = mCropTop;
216 crop.right = mCropRight + 1;
217 crop.bottom = mCropBottom + 1;
218 ALOGV("setting crop: [%d, %d, %d, %d] for size [%zu, %zu]",
219 crop.left, crop.top, crop.right, crop.bottom, bufWidth, bufHeight);
220
221 CHECK_EQ(0, native_window_set_crop(mNativeWindow.get(), &crop));
222
223 int32_t rotationDegrees;
224 if (!format->findInt32("rotation-degrees", &rotationDegrees)) {
225 rotationDegrees = mRotationDegrees;
226 }
227 uint32_t transform;
228 switch (rotationDegrees) {
229 case 0: transform = 0; break;
230 case 90: transform = HAL_TRANSFORM_ROT_90; break;
231 case 180: transform = HAL_TRANSFORM_ROT_180; break;
232 case 270: transform = HAL_TRANSFORM_ROT_270; break;
233 default: transform = 0; break;
234 }
235
236 CHECK_EQ(0, native_window_set_buffers_transform(
237 mNativeWindow.get(), transform));
238 }
239
clearTracker()240 void SoftwareRenderer::clearTracker() {
241 mRenderTracker.clear(-1 /* lastRenderTimeNs */);
242 }
243
render(const void * data,size_t,int64_t mediaTimeUs,nsecs_t renderTimeNs,size_t numOutputBuffers,const sp<AMessage> & format)244 std::list<FrameRenderTracker::Info> SoftwareRenderer::render(
245 const void *data, size_t , int64_t mediaTimeUs, nsecs_t renderTimeNs,
246 size_t numOutputBuffers, const sp<AMessage>& format) {
247 resetFormatIfChanged(format, numOutputBuffers);
248 FrameRenderTracker::Info *info = NULL;
249
250 ANativeWindowBuffer *buf;
251 int fenceFd = -1;
252 int err = mNativeWindow->dequeueBuffer(mNativeWindow.get(), &buf, &fenceFd);
253 if (err == 0 && fenceFd >= 0) {
254 info = mRenderTracker.updateInfoForDequeuedBuffer(buf, fenceFd, 0);
255 sp<Fence> fence = new Fence(fenceFd);
256 err = fence->waitForever("SoftwareRenderer::render");
257 }
258 if (err != 0) {
259 ALOGW("Surface::dequeueBuffer returned error %d", err);
260 // complete (drop) dequeued frame if fence wait failed; otherwise,
261 // this returns an empty list as no frames should have rendered and not yet returned.
262 return mRenderTracker.checkFencesAndGetRenderedFrames(info, false /* dropIncomplete */);
263 }
264
265 GraphicBufferMapper &mapper = GraphicBufferMapper::get();
266
267 Rect bounds(mCropWidth, mCropHeight);
268
269 void *dst;
270 CHECK_EQ(0, mapper.lock(buf->handle,
271 GRALLOC_USAGE_SW_READ_NEVER | GRALLOC_USAGE_SW_WRITE_RARELY,
272 bounds, &dst));
273
274 // TODO move the other conversions also into ColorConverter, and
275 // fix cropping issues (when mCropLeft/Top != 0 or mWidth != mCropWidth)
276 if (mConverter) {
277 mConverter->convert(
278 data,
279 mWidth, mHeight,
280 mCropLeft, mCropTop, mCropRight, mCropBottom,
281 dst,
282 buf->stride, buf->height,
283 0, 0, mCropWidth - 1, mCropHeight - 1);
284 } else if (mColorFormat == OMX_COLOR_FormatYUV420Planar) {
285 const uint8_t *src_y = (const uint8_t *)data;
286 const uint8_t *src_u =
287 (const uint8_t *)data + mWidth * mHeight;
288 const uint8_t *src_v = src_u + (mWidth / 2 * mHeight / 2);
289
290 src_y +=mCropLeft + mCropTop * mWidth;
291 src_u +=(mCropLeft + mCropTop * mWidth / 2)/2;
292 src_v +=(mCropLeft + mCropTop * mWidth / 2)/2;
293
294 uint8_t *dst_y = (uint8_t *)dst;
295 size_t dst_y_size = buf->stride * buf->height;
296 size_t dst_c_stride = ALIGN(buf->stride / 2, 16);
297 size_t dst_c_size = dst_c_stride * buf->height / 2;
298 uint8_t *dst_v = dst_y + dst_y_size;
299 uint8_t *dst_u = dst_v + dst_c_size;
300
301 dst_y += mCropTop * buf->stride + mCropLeft;
302 dst_v += (mCropTop/2) * dst_c_stride + mCropLeft/2;
303 dst_u += (mCropTop/2) * dst_c_stride + mCropLeft/2;
304
305 for (int y = 0; y < mCropHeight; ++y) {
306 memcpy(dst_y, src_y, mCropWidth);
307
308 src_y += mWidth;
309 dst_y += buf->stride;
310 }
311
312 for (int y = 0; y < (mCropHeight + 1) / 2; ++y) {
313 memcpy(dst_u, src_u, (mCropWidth + 1) / 2);
314 memcpy(dst_v, src_v, (mCropWidth + 1) / 2);
315
316 src_u += mWidth / 2;
317 src_v += mWidth / 2;
318 dst_u += dst_c_stride;
319 dst_v += dst_c_stride;
320 }
321 } else if (mColorFormat == OMX_COLOR_FormatYUV420Planar16) {
322 const uint16_t *src_y = (const uint16_t *)data;
323 const uint16_t *src_u = (const uint16_t *)data + mWidth * mHeight;
324 const uint16_t *src_v = src_u + (mWidth / 2 * mHeight / 2);
325
326 src_y += mCropLeft + mCropTop * mWidth;
327 src_u += (mCropLeft + mCropTop * mWidth / 2) / 2;
328 src_v += (mCropLeft + mCropTop * mWidth / 2) / 2;
329
330 uint8_t *dst_y = (uint8_t *)dst;
331 size_t dst_y_size = buf->stride * buf->height;
332 size_t dst_c_stride = ALIGN(buf->stride / 2, 16);
333 size_t dst_c_size = dst_c_stride * buf->height / 2;
334 uint8_t *dst_v = dst_y + dst_y_size;
335 uint8_t *dst_u = dst_v + dst_c_size;
336
337 dst_y += mCropTop * buf->stride + mCropLeft;
338 dst_v += (mCropTop / 2) * dst_c_stride + mCropLeft / 2;
339 dst_u += (mCropTop / 2) * dst_c_stride + mCropLeft / 2;
340
341 for (int y = 0; y < mCropHeight; ++y) {
342 for (int x = 0; x < mCropWidth; ++x) {
343 dst_y[x] = (uint8_t)(src_y[x] >> 2);
344 }
345
346 src_y += mWidth;
347 dst_y += buf->stride;
348 }
349
350 for (int y = 0; y < (mCropHeight + 1) / 2; ++y) {
351 for (int x = 0; x < (mCropWidth + 1) / 2; ++x) {
352 dst_u[x] = (uint8_t)(src_u[x] >> 2);
353 dst_v[x] = (uint8_t)(src_v[x] >> 2);
354 }
355
356 src_u += mWidth / 2;
357 src_v += mWidth / 2;
358 dst_u += dst_c_stride;
359 dst_v += dst_c_stride;
360 }
361 } else if (mColorFormat == OMX_TI_COLOR_FormatYUV420PackedSemiPlanar
362 || mColorFormat == OMX_COLOR_FormatYUV420SemiPlanar) {
363 const uint8_t *src_y = (const uint8_t *)data;
364 const uint8_t *src_uv = (const uint8_t *)data
365 + mWidth * mHeight;
366
367 src_y += mCropLeft + mCropTop * mWidth;
368 src_uv += (mCropLeft + mCropTop * mWidth) / 2;
369
370 uint8_t *dst_y = (uint8_t *)dst;
371
372 size_t dst_y_size = buf->stride * buf->height;
373 size_t dst_c_stride = ALIGN(buf->stride / 2, 16);
374 size_t dst_c_size = dst_c_stride * buf->height / 2;
375 uint8_t *dst_v = dst_y + dst_y_size;
376 uint8_t *dst_u = dst_v + dst_c_size;
377
378 dst_y += mCropTop * buf->stride + mCropLeft;
379 dst_v += (mCropTop/2) * dst_c_stride + mCropLeft/2;
380 dst_u += (mCropTop/2) * dst_c_stride + mCropLeft/2;
381
382 for (int y = 0; y < mCropHeight; ++y) {
383 memcpy(dst_y, src_y, mCropWidth);
384
385 src_y += mWidth;
386 dst_y += buf->stride;
387 }
388
389 for (int y = 0; y < (mCropHeight + 1) / 2; ++y) {
390 size_t tmp = (mCropWidth + 1) / 2;
391 for (size_t x = 0; x < tmp; ++x) {
392 dst_u[x] = src_uv[2 * x];
393 dst_v[x] = src_uv[2 * x + 1];
394 }
395
396 src_uv += mWidth;
397 dst_u += dst_c_stride;
398 dst_v += dst_c_stride;
399 }
400 } else if (mColorFormat == OMX_COLOR_Format24bitRGB888) {
401 uint8_t* srcPtr = (uint8_t*)data + mWidth * mCropTop * 3 + mCropLeft * 3;
402 uint8_t* dstPtr = (uint8_t*)dst + buf->stride * mCropTop * 3 + mCropLeft * 3;
403
404 for (size_t y = 0; y < (size_t)mCropHeight; ++y) {
405 memcpy(dstPtr, srcPtr, mCropWidth * 3);
406 srcPtr += mWidth * 3;
407 dstPtr += buf->stride * 3;
408 }
409 } else if (mColorFormat == OMX_COLOR_Format32bitARGB8888) {
410 uint8_t *srcPtr, *dstPtr;
411
412 for (size_t y = 0; y < (size_t)mCropHeight; ++y) {
413 srcPtr = (uint8_t*)data + mWidth * 4 * (y + mCropTop) + mCropLeft * 4;
414 dstPtr = (uint8_t*)dst + buf->stride * 4 * (y + mCropTop) + mCropLeft * 4;
415 for (size_t x = 0; x < (size_t)mCropWidth; ++x) {
416 uint8_t a = *srcPtr++;
417 for (size_t i = 0; i < 3; ++i) { // copy RGB
418 *dstPtr++ = *srcPtr++;
419 }
420 *dstPtr++ = a; // alpha last (ARGB to RGBA)
421 }
422 }
423 } else if (mColorFormat == OMX_COLOR_Format32BitRGBA8888) {
424 uint8_t* srcPtr = (uint8_t*)data + mWidth * mCropTop * 4 + mCropLeft * 4;
425 uint8_t* dstPtr = (uint8_t*)dst + buf->stride * mCropTop * 4 + mCropLeft * 4;
426
427 for (size_t y = 0; y < (size_t)mCropHeight; ++y) {
428 memcpy(dstPtr, srcPtr, mCropWidth * 4);
429 srcPtr += mWidth * 4;
430 dstPtr += buf->stride * 4;
431 }
432 } else {
433 LOG_ALWAYS_FATAL("bad color format %#x", mColorFormat);
434 }
435
436 skip_copying:
437 CHECK_EQ(0, mapper.unlock(buf->handle));
438
439 if (renderTimeNs >= 0) {
440 if ((err = native_window_set_buffers_timestamp(mNativeWindow.get(),
441 renderTimeNs)) != 0) {
442 ALOGW("Surface::set_buffers_timestamp returned error %d", err);
443 }
444 }
445
446 // TODO: propagate color aspects to software renderer to allow better
447 // color conversion to RGB. For now, just mark dataspace for YUV rendering.
448 android_dataspace dataSpace;
449 if (format->findInt32("android._dataspace", (int32_t *)&dataSpace) && dataSpace != mDataSpace) {
450 mDataSpace = dataSpace;
451
452 if (mConverter != NULL && mConverter->isDstRGB()) {
453 // graphics only supports full range RGB. ColorConverter should have
454 // converted any YUV to full range.
455 dataSpace = (android_dataspace)
456 ((dataSpace & ~HAL_DATASPACE_RANGE_MASK) | HAL_DATASPACE_RANGE_FULL);
457 }
458
459 ALOGD("setting dataspace on output surface to #%x", dataSpace);
460 if ((err = native_window_set_buffers_data_space(mNativeWindow.get(), dataSpace))) {
461 ALOGW("failed to set dataspace on surface (%d)", err);
462 }
463 }
464 if (format->contains("hdr-static-info")) {
465 HDRStaticInfo info;
466 if (ColorUtils::getHDRStaticInfoFromFormat(format, &info)
467 && memcmp(&mHDRStaticInfo, &info, sizeof(info))) {
468 setNativeWindowHdrMetadata(mNativeWindow.get(), &info);
469 mHDRStaticInfo = info;
470 }
471 }
472
473 if ((err = mNativeWindow->queueBuffer(mNativeWindow.get(), buf, -1)) != 0) {
474 ALOGW("Surface::queueBuffer returned error %d", err);
475 } else {
476 mRenderTracker.onFrameQueued(mediaTimeUs, (GraphicBuffer *)buf, Fence::NO_FENCE);
477 }
478
479 buf = NULL;
480 return mRenderTracker.checkFencesAndGetRenderedFrames(info, info != NULL /* dropIncomplete */);
481 }
482
483 } // namespace android
484