1 /* 2 * Copyright (C) 2012 The Android Open Source Project 3 * 4 * Licensed under the Apache License, Version 2.0 (the "License"); 5 * you may not use this file except in compliance with the License. 6 * You may obtain a copy of the License at 7 * 8 * http://www.apache.org/licenses/LICENSE-2.0 9 * 10 * Unless required by applicable law or agreed to in writing, software 11 * distributed under the License is distributed on an "AS IS" BASIS, 12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13 * See the License for the specific language governing permissions and 14 * limitations under the License. 15 */ 16 17 package android.media; 18 19 import static android.media.codec.Flags.FLAG_NULL_OUTPUT_SURFACE; 20 import static android.media.codec.Flags.FLAG_REGION_OF_INTEREST; 21 22 import static com.android.media.codec.flags.Flags.FLAG_LARGE_AUDIO_FRAME; 23 24 import android.Manifest; 25 import android.annotation.FlaggedApi; 26 import android.annotation.IntDef; 27 import android.annotation.NonNull; 28 import android.annotation.Nullable; 29 import android.annotation.RequiresPermission; 30 import android.annotation.SystemApi; 31 import android.compat.annotation.UnsupportedAppUsage; 32 import android.graphics.ImageFormat; 33 import android.graphics.Rect; 34 import android.graphics.SurfaceTexture; 35 import android.hardware.HardwareBuffer; 36 import android.media.MediaCodecInfo.CodecCapabilities; 37 import android.os.Build; 38 import android.os.Bundle; 39 import android.os.Handler; 40 import android.os.IHwBinder; 41 import android.os.Looper; 42 import android.os.Message; 43 import android.os.PersistableBundle; 44 import android.view.Surface; 45 46 import java.io.IOException; 47 import java.lang.annotation.Retention; 48 import java.lang.annotation.RetentionPolicy; 49 import java.nio.ByteBuffer; 50 import java.nio.ByteOrder; 51 import java.nio.ReadOnlyBufferException; 52 import java.util.ArrayDeque; 53 import java.util.ArrayList; 54 import java.util.Arrays; 55 import java.util.BitSet; 56 import java.util.Collections; 57 import java.util.HashMap; 58 import java.util.HashSet; 59 import java.util.List; 60 import java.util.Map; 61 import java.util.Objects; 62 import java.util.Optional; 63 import java.util.Set; 64 import java.util.concurrent.BlockingQueue; 65 import java.util.concurrent.LinkedBlockingQueue; 66 import java.util.concurrent.locks.Lock; 67 import java.util.concurrent.locks.ReentrantLock; 68 import java.util.function.Supplier; 69 70 /** 71 MediaCodec class can be used to access low-level media codecs, i.e. encoder/decoder components. 72 It is part of the Android low-level multimedia support infrastructure (normally used together 73 with {@link MediaExtractor}, {@link MediaSync}, {@link MediaMuxer}, {@link MediaCrypto}, 74 {@link MediaDrm}, {@link Image}, {@link Surface}, and {@link AudioTrack}.) 75 <p> 76 <center> 77 <img src="../../../images/media/mediacodec_buffers.svg" style="width: 540px; height: 205px" 78 alt="MediaCodec buffer flow diagram"> 79 </center> 80 <p> 81 In broad terms, a codec processes input data to generate output data. It processes data 82 asynchronously and uses a set of input and output buffers. At a simplistic level, you request 83 (or receive) an empty input buffer, fill it up with data and send it to the codec for 84 processing. The codec uses up the data and transforms it into one of its empty output buffers. 85 Finally, you request (or receive) a filled output buffer, consume its contents and release it 86 back to the codec. 87 88 <h3 id=qualityFloor><a name="qualityFloor">Minimum Quality Floor for Video Encoding</h3> 89 <p> 90 Beginning with {@link android.os.Build.VERSION_CODES#S}, Android's Video MediaCodecs enforce a 91 minimum quality floor. The intent is to eliminate poor quality video encodings. This quality 92 floor is applied when the codec is in Variable Bitrate (VBR) mode; it is not applied when 93 the codec is in Constant Bitrate (CBR) mode. The quality floor enforcement is also restricted 94 to a particular size range; this size range is currently for video resolutions 95 larger than 320x240 up through 1920x1080. 96 97 <p> 98 When this quality floor is in effect, the codec and supporting framework code will work to 99 ensure that the generated video is of at least a "fair" or "good" quality. The metric 100 used to choose these targets is the VMAF (Video Multi-method Assessment Function) with a 101 target score of 70 for selected test sequences. 102 103 <p> 104 The typical effect is that 105 some videos will generate a higher bitrate than originally configured. This will be most 106 notable for videos which were configured with very low bitrates; the codec will use a bitrate 107 that is determined to be more likely to generate an "fair" or "good" quality video. Another 108 situation is where a video includes very complicated content (lots of motion and detail); 109 in such configurations, the codec will use extra bitrate as needed to avoid losing all of 110 the content's finer detail. 111 112 <p> 113 This quality floor will not impact content captured at high bitrates (a high bitrate should 114 already provide the codec with sufficient capacity to encode all of the detail). 115 The quality floor does not operate on CBR encodings. 116 The quality floor currently does not operate on resolutions of 320x240 or lower, nor on 117 videos with resolution above 1920x1080. 118 119 <h3>Data Types</h3> 120 <p> 121 Codecs operate on three kinds of data: compressed data, raw audio data and raw video data. 122 All three kinds of data can be processed using {@link ByteBuffer ByteBuffers}, but you should use 123 a {@link Surface} for raw video data to improve codec performance. Surface uses native video 124 buffers without mapping or copying them to ByteBuffers; thus, it is much more efficient. 125 You normally cannot access the raw video data when using a Surface, but you can use the 126 {@link ImageReader} class to access unsecured decoded (raw) video frames. This may still be more 127 efficient than using ByteBuffers, as some native buffers may be mapped into {@linkplain 128 ByteBuffer#isDirect direct} ByteBuffers. When using ByteBuffer mode, you can access raw video 129 frames using the {@link Image} class and {@link #getInputImage getInput}/{@link #getOutputImage 130 OutputImage(int)}. 131 132 <h4>Compressed Buffers</h4> 133 <p> 134 Input buffers (for decoders) and output buffers (for encoders) contain compressed data according 135 to the {@linkplain MediaFormat#KEY_MIME format's type}. For video types this is normally a single 136 compressed video frame. For audio data this is normally a single access unit (an encoded audio 137 segment typically containing a few milliseconds of audio as dictated by the format type), but 138 this requirement is slightly relaxed in that a buffer may contain multiple encoded access units 139 of audio. In either case, buffers do not start or end on arbitrary byte boundaries, but rather on 140 frame/access unit boundaries unless they are flagged with {@link #BUFFER_FLAG_PARTIAL_FRAME}. 141 142 <h4>Raw Audio Buffers</h4> 143 <p> 144 Raw audio buffers contain entire frames of PCM audio data, which is one sample for each channel 145 in channel order. Each PCM audio sample is either a 16 bit signed integer or a float, 146 in native byte order. 147 Raw audio buffers in the float PCM encoding are only possible 148 if the MediaFormat's {@linkplain MediaFormat#KEY_PCM_ENCODING} 149 is set to {@linkplain AudioFormat#ENCODING_PCM_FLOAT} during MediaCodec 150 {@link #configure configure(…)} 151 and confirmed by {@link #getOutputFormat} for decoders 152 or {@link #getInputFormat} for encoders. 153 A sample method to check for float PCM in the MediaFormat is as follows: 154 155 <pre class=prettyprint> 156 static boolean isPcmFloat(MediaFormat format) { 157 return format.getInteger(MediaFormat.KEY_PCM_ENCODING, AudioFormat.ENCODING_PCM_16BIT) 158 == AudioFormat.ENCODING_PCM_FLOAT; 159 }</pre> 160 161 In order to extract, in a short array, 162 one channel of a buffer containing 16 bit signed integer audio data, 163 the following code may be used: 164 165 <pre class=prettyprint> 166 // Assumes the buffer PCM encoding is 16 bit. 167 short[] getSamplesForChannel(MediaCodec codec, int bufferId, int channelIx) { 168 ByteBuffer outputBuffer = codec.getOutputBuffer(bufferId); 169 MediaFormat format = codec.getOutputFormat(bufferId); 170 ShortBuffer samples = outputBuffer.order(ByteOrder.nativeOrder()).asShortBuffer(); 171 int numChannels = format.getInteger(MediaFormat.KEY_CHANNEL_COUNT); 172 if (channelIx < 0 || channelIx >= numChannels) { 173 return null; 174 } 175 short[] res = new short[samples.remaining() / numChannels]; 176 for (int i = 0; i < res.length; ++i) { 177 res[i] = samples.get(i * numChannels + channelIx); 178 } 179 return res; 180 }</pre> 181 182 <h4>Raw Video Buffers</h4> 183 <p> 184 In ByteBuffer mode video buffers are laid out according to their {@linkplain 185 MediaFormat#KEY_COLOR_FORMAT color format}. You can get the supported color formats as an array 186 from {@link #getCodecInfo}{@code .}{@link MediaCodecInfo#getCapabilitiesForType 187 getCapabilitiesForType(…)}{@code .}{@link CodecCapabilities#colorFormats colorFormats}. 188 Video codecs may support three kinds of color formats: 189 <ul> 190 <li><strong>native raw video format:</strong> This is marked by {@link 191 CodecCapabilities#COLOR_FormatSurface} and it can be used with an input or output Surface.</li> 192 <li><strong>flexible YUV buffers</strong> (such as {@link 193 CodecCapabilities#COLOR_FormatYUV420Flexible}): These can be used with an input/output Surface, 194 as well as in ByteBuffer mode, by using {@link #getInputImage getInput}/{@link #getOutputImage 195 OutputImage(int)}.</li> 196 <li><strong>other, specific formats:</strong> These are normally only supported in ByteBuffer 197 mode. Some color formats are vendor specific. Others are defined in {@link CodecCapabilities}. 198 For color formats that are equivalent to a flexible format, you can still use {@link 199 #getInputImage getInput}/{@link #getOutputImage OutputImage(int)}.</li> 200 </ul> 201 <p> 202 All video codecs support flexible YUV 4:2:0 buffers since {@link 203 android.os.Build.VERSION_CODES#LOLLIPOP_MR1}. 204 205 <h4>Accessing Raw Video ByteBuffers on Older Devices</h4> 206 <p> 207 Prior to {@link android.os.Build.VERSION_CODES#LOLLIPOP} and {@link Image} support, you need to 208 use the {@link MediaFormat#KEY_STRIDE} and {@link MediaFormat#KEY_SLICE_HEIGHT} output format 209 values to understand the layout of the raw output buffers. 210 <p class=note> 211 Note that on some devices the slice-height is advertised as 0. This could mean either that the 212 slice-height is the same as the frame height, or that the slice-height is the frame height 213 aligned to some value (usually a power of 2). Unfortunately, there is no standard and simple way 214 to tell the actual slice height in this case. Furthermore, the vertical stride of the {@code U} 215 plane in planar formats is also not specified or defined, though usually it is half of the slice 216 height. 217 <p> 218 The {@link MediaFormat#KEY_WIDTH} and {@link MediaFormat#KEY_HEIGHT} keys specify the size of the 219 video frames; however, for most encondings the video (picture) only occupies a portion of the 220 video frame. This is represented by the 'crop rectangle'. 221 <p> 222 You need to use the following keys to get the crop rectangle of raw output images from the 223 {@linkplain #getOutputFormat output format}. If these keys are not present, the video occupies the 224 entire video frame.The crop rectangle is understood in the context of the output frame 225 <em>before</em> applying any {@linkplain MediaFormat#KEY_ROTATION rotation}. 226 <table style="width: 0%"> 227 <thead> 228 <tr> 229 <th>Format Key</th> 230 <th>Type</th> 231 <th>Description</th> 232 </tr> 233 </thead> 234 <tbody> 235 <tr> 236 <td>{@link MediaFormat#KEY_CROP_LEFT}</td> 237 <td>Integer</td> 238 <td>The left-coordinate (x) of the crop rectangle</td> 239 </tr><tr> 240 <td>{@link MediaFormat#KEY_CROP_TOP}</td> 241 <td>Integer</td> 242 <td>The top-coordinate (y) of the crop rectangle</td> 243 </tr><tr> 244 <td>{@link MediaFormat#KEY_CROP_RIGHT}</td> 245 <td>Integer</td> 246 <td>The right-coordinate (x) <strong>MINUS 1</strong> of the crop rectangle</td> 247 </tr><tr> 248 <td>{@link MediaFormat#KEY_CROP_BOTTOM}</td> 249 <td>Integer</td> 250 <td>The bottom-coordinate (y) <strong>MINUS 1</strong> of the crop rectangle</td> 251 </tr><tr> 252 <td colspan=3> 253 The right and bottom coordinates can be understood as the coordinates of the right-most 254 valid column/bottom-most valid row of the cropped output image. 255 </td> 256 </tr> 257 </tbody> 258 </table> 259 <p> 260 The size of the video frame (before rotation) can be calculated as such: 261 <pre class=prettyprint> 262 MediaFormat format = decoder.getOutputFormat(…); 263 int width = format.getInteger(MediaFormat.KEY_WIDTH); 264 if (format.containsKey(MediaFormat.KEY_CROP_LEFT) 265 && format.containsKey(MediaFormat.KEY_CROP_RIGHT)) { 266 width = format.getInteger(MediaFormat.KEY_CROP_RIGHT) + 1 267 - format.getInteger(MediaFormat.KEY_CROP_LEFT); 268 } 269 int height = format.getInteger(MediaFormat.KEY_HEIGHT); 270 if (format.containsKey(MediaFormat.KEY_CROP_TOP) 271 && format.containsKey(MediaFormat.KEY_CROP_BOTTOM)) { 272 height = format.getInteger(MediaFormat.KEY_CROP_BOTTOM) + 1 273 - format.getInteger(MediaFormat.KEY_CROP_TOP); 274 } 275 </pre> 276 <p class=note> 277 Also note that the meaning of {@link BufferInfo#offset BufferInfo.offset} was not consistent across 278 devices. On some devices the offset pointed to the top-left pixel of the crop rectangle, while on 279 most devices it pointed to the top-left pixel of the entire frame. 280 281 <h3>States</h3> 282 <p> 283 During its life a codec conceptually exists in one of three states: Stopped, Executing or 284 Released. The Stopped collective state is actually the conglomeration of three states: 285 Uninitialized, Configured and Error, whereas the Executing state conceptually progresses through 286 three sub-states: Flushed, Running and End-of-Stream. 287 <p> 288 <center> 289 <img src="../../../images/media/mediacodec_states.svg" style="width: 519px; height: 356px" 290 alt="MediaCodec state diagram"> 291 </center> 292 <p> 293 When you create a codec using one of the factory methods, the codec is in the Uninitialized 294 state. First, you need to configure it via {@link #configure configure(…)}, which brings 295 it to the Configured state, then call {@link #start} to move it to the Executing state. In this 296 state you can process data through the buffer queue manipulation described above. 297 <p> 298 The Executing state has three sub-states: Flushed, Running and End-of-Stream. Immediately after 299 {@link #start} the codec is in the Flushed sub-state, where it holds all the buffers. As soon 300 as the first input buffer is dequeued, the codec moves to the Running sub-state, where it spends 301 most of its life. When you queue an input buffer with the {@linkplain #BUFFER_FLAG_END_OF_STREAM 302 end-of-stream marker}, the codec transitions to the End-of-Stream sub-state. In this state the 303 codec no longer accepts further input buffers, but still generates output buffers until the 304 end-of-stream is reached on the output. For decoders, you can move back to the Flushed sub-state 305 at any time while in the Executing state using {@link #flush}. 306 <p class=note> 307 <strong>Note:</strong> Going back to Flushed state is only supported for decoders, and may not 308 work for encoders (the behavior is undefined). 309 <p> 310 Call {@link #stop} to return the codec to the Uninitialized state, whereupon it may be configured 311 again. When you are done using a codec, you must release it by calling {@link #release}. 312 <p> 313 On rare occasions the codec may encounter an error and move to the Error state. This is 314 communicated using an invalid return value from a queuing operation, or sometimes via an 315 exception. Call {@link #reset} to make the codec usable again. You can call it from any state to 316 move the codec back to the Uninitialized state. Otherwise, call {@link #release} to move to the 317 terminal Released state. 318 319 <h3>Creation</h3> 320 <p> 321 Use {@link MediaCodecList} to create a MediaCodec for a specific {@link MediaFormat}. When 322 decoding a file or a stream, you can get the desired format from {@link 323 MediaExtractor#getTrackFormat MediaExtractor.getTrackFormat}. Inject any specific features that 324 you want to add using {@link MediaFormat#setFeatureEnabled MediaFormat.setFeatureEnabled}, then 325 call {@link MediaCodecList#findDecoderForFormat MediaCodecList.findDecoderForFormat} to get the 326 name of a codec that can handle that specific media format. Finally, create the codec using 327 {@link #createByCodecName}. 328 <p class=note> 329 <strong>Note:</strong> On {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the format to 330 {@code MediaCodecList.findDecoder}/{@code EncoderForFormat} must not contain a {@linkplain 331 MediaFormat#KEY_FRAME_RATE frame rate}. Use 332 <code class=prettyprint>format.setString(MediaFormat.KEY_FRAME_RATE, null)</code> 333 to clear any existing frame rate setting in the format. 334 <p> 335 You can also create the preferred codec for a specific MIME type using {@link 336 #createDecoderByType createDecoder}/{@link #createEncoderByType EncoderByType(String)}. 337 This, however, cannot be used to inject features, and may create a codec that cannot handle the 338 specific desired media format. 339 340 <h4>Creating secure decoders</h4> 341 <p> 342 On versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and earlier, secure codecs might 343 not be listed in {@link MediaCodecList}, but may still be available on the system. Secure codecs 344 that exist can be instantiated by name only, by appending {@code ".secure"} to the name of a 345 regular codec (the name of all secure codecs must end in {@code ".secure"}.) {@link 346 #createByCodecName} will throw an {@code IOException} if the codec is not present on the system. 347 <p> 348 From {@link android.os.Build.VERSION_CODES#LOLLIPOP} onwards, you should use the {@link 349 CodecCapabilities#FEATURE_SecurePlayback} feature in the media format to create a secure decoder. 350 351 <h3>Initialization</h3> 352 <p> 353 After creating the codec, you can set a callback using {@link #setCallback setCallback} if you 354 want to process data asynchronously. Then, {@linkplain #configure configure} the codec using the 355 specific media format. This is when you can specify the output {@link Surface} for video 356 producers – codecs that generate raw video data (e.g. video decoders). This is also when 357 you can set the decryption parameters for secure codecs (see {@link MediaCrypto}). Finally, since 358 some codecs can operate in multiple modes, you must specify whether you want it to work as a 359 decoder or an encoder. 360 <p> 361 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you can query the resulting input and 362 output format in the Configured state. You can use this to verify the resulting configuration, 363 e.g. color formats, before starting the codec. 364 <p> 365 If you want to process raw input video buffers natively with a video consumer – a codec 366 that processes raw video input, such as a video encoder – create a destination Surface for 367 your input data using {@link #createInputSurface} after configuration. Alternately, set up the 368 codec to use a previously created {@linkplain #createPersistentInputSurface persistent input 369 surface} by calling {@link #setInputSurface}. 370 371 <h4 id=EncoderProfiles><a name="EncoderProfiles"></a>Encoder Profiles</h4> 372 <p> 373 When using an encoder, it is recommended to set the desired codec {@link MediaFormat#KEY_PROFILE 374 profile} during {@link #configure configure()}. (This is only meaningful for 375 {@link MediaFormat#KEY_MIME media formats} for which profiles are defined.) 376 <p> 377 If a profile is not specified during {@code configure}, the encoder will choose a profile for the 378 session based on the available information. We will call this value the <i>default profile</i>. 379 The selection of the default profile is device specific and may not be deterministic 380 (could be ad hoc or even experimental). The encoder may choose a default profile that is not 381 suitable for the intended encoding session, which may result in the encoder ultimately rejecting 382 the session. 383 <p> 384 The encoder may reject the encoding session if the configured (or default if unspecified) profile 385 does not support the codec input (mainly the {@link MediaFormat#KEY_COLOR_FORMAT color format} for 386 video/image codecs, or the {@link MediaFormat#KEY_PCM_ENCODING sample encoding} and the {@link 387 MediaFormat#KEY_CHANNEL_COUNT number of channels} for audio codecs, but also possibly 388 {@link MediaFormat#KEY_WIDTH width}, {@link MediaFormat#KEY_HEIGHT height}, 389 {@link MediaFormat#KEY_FRAME_RATE frame rate}, {@link MediaFormat#KEY_BIT_RATE bitrate} or 390 {@link MediaFormat#KEY_SAMPLE_RATE sample rate}.) 391 Alternatively, the encoder may choose to (but is not required to) convert the input to support the 392 selected (or default) profile - or adjust the chosen profile based on the presumed or detected 393 input format - to ensure a successful encoding session. <b>Note</b>: Converting the input to match 394 an incompatible profile will in most cases result in decreased codec performance. 395 <p> 396 To ensure backward compatibility, the following guarantees are provided by Android: 397 <ul> 398 <li>The default video encoder profile always supports 8-bit YUV 4:2:0 color format ({@link 399 CodecCapabilities#COLOR_FormatYUV420Flexible COLOR_FormatYUV420Flexible} and equivalent 400 {@link CodecCapabilities#colorFormats supported formats}) for both Surface and ByteBuffer modes. 401 <li>The default video encoder profile always supports the default 8-bit RGBA color format in 402 Surface mode even if no such formats are enumerated in the {@link CodecCapabilities#colorFormats 403 supported formats}. 404 </ul> 405 <p class=note> 406 <b>Note</b>: the accepted profile can be queried through the {@link #getOutputFormat output 407 format} of the encoder after {@code configure} to allow applications to set up their 408 codec input to a format supported by the encoder profile. 409 <p> 410 <b>Implication:</b> 411 <ul> 412 <li>Applications that want to encode 4:2:2, 4:4:4, 10+ bit or HDR video input <b>MUST</b> configure 413 a suitable profile for encoders. 414 </ul> 415 416 <h4 id=CSD><a name="CSD"></a>Codec-specific Data</h4> 417 <p> 418 Some formats, notably AAC audio and MPEG4, H.264 and H.265 video formats require the actual data 419 to be prefixed by a number of buffers containing setup data, or codec specific data. When 420 processing such compressed formats, this data must be submitted to the codec after {@link 421 #start} and before any frame data. Such data must be marked using the flag {@link 422 #BUFFER_FLAG_CODEC_CONFIG} in a call to {@link #queueInputBuffer queueInputBuffer}. 423 <p> 424 Codec-specific data can also be included in the format passed to {@link #configure configure} in 425 ByteBuffer entries with keys "csd-0", "csd-1", etc. These keys are always included in the track 426 {@link MediaFormat} obtained from the {@link MediaExtractor#getTrackFormat MediaExtractor}. 427 Codec-specific data in the format is automatically submitted to the codec upon {@link #start}; 428 you <strong>MUST NOT</strong> submit this data explicitly. If the format did not contain codec 429 specific data, you can choose to submit it using the specified number of buffers in the correct 430 order, according to the format requirements. In case of H.264 AVC, you can also concatenate all 431 codec-specific data and submit it as a single codec-config buffer. 432 <p> 433 Android uses the following codec-specific data buffers. These are also required to be set in 434 the track format for proper {@link MediaMuxer} track configuration. Each parameter set and the 435 codec-specific-data sections marked with (<sup>*</sup>) must start with a start code of 436 {@code "\x00\x00\x00\x01"}. 437 <p> 438 <style>td.NA { background: #ccc; } .mid > tr > td { vertical-align: middle; }</style> 439 <table> 440 <thead> 441 <th>Format</th> 442 <th>CSD buffer #0</th> 443 <th>CSD buffer #1</th> 444 <th>CSD buffer #2</th> 445 </thead> 446 <tbody class=mid> 447 <tr> 448 <td>AAC</td> 449 <td>Decoder-specific information from ESDS<sup>*</sup></td> 450 <td class=NA>Not Used</td> 451 <td class=NA>Not Used</td> 452 </tr> 453 <tr> 454 <td>VORBIS</td> 455 <td>Identification header</td> 456 <td>Setup header</td> 457 <td class=NA>Not Used</td> 458 </tr> 459 <tr> 460 <td>OPUS</td> 461 <td>Identification header</td> 462 <td>Pre-skip in nanosecs<br> 463 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)<br> 464 This overrides the pre-skip value in the identification header.</td> 465 <td>Seek Pre-roll in nanosecs<br> 466 (unsigned 64-bit {@linkplain ByteOrder#nativeOrder native-order} integer.)</td> 467 </tr> 468 <tr> 469 <td>FLAC</td> 470 <td>"fLaC", the FLAC stream marker in ASCII,<br> 471 followed by the STREAMINFO block (the mandatory metadata block),<br> 472 optionally followed by any number of other metadata blocks</td> 473 <td class=NA>Not Used</td> 474 <td class=NA>Not Used</td> 475 </tr> 476 <tr> 477 <td>MPEG-4</td> 478 <td>Decoder-specific information from ESDS<sup>*</sup></td> 479 <td class=NA>Not Used</td> 480 <td class=NA>Not Used</td> 481 </tr> 482 <tr> 483 <td>H.264 AVC</td> 484 <td>SPS (Sequence Parameter Sets<sup>*</sup>)</td> 485 <td>PPS (Picture Parameter Sets<sup>*</sup>)</td> 486 <td class=NA>Not Used</td> 487 </tr> 488 <tr> 489 <td>H.265 HEVC</td> 490 <td>VPS (Video Parameter Sets<sup>*</sup>) +<br> 491 SPS (Sequence Parameter Sets<sup>*</sup>) +<br> 492 PPS (Picture Parameter Sets<sup>*</sup>)</td> 493 <td class=NA>Not Used</td> 494 <td class=NA>Not Used</td> 495 </tr> 496 <tr> 497 <td>VP9</td> 498 <td>VP9 <a href="http://wiki.webmproject.org/vp9-codecprivate">CodecPrivate</a> Data 499 (optional)</td> 500 <td class=NA>Not Used</td> 501 <td class=NA>Not Used</td> 502 </tr> 503 <tr> 504 <td>AV1</td> 505 <td>AV1 <a href="https://aomediacodec.github.io/av1-isobmff/#av1codecconfigurationbox-syntax"> 506 AV1CodecConfigurationRecord</a> Data (optional) 507 </td> 508 <td class=NA>Not Used</td> 509 <td class=NA>Not Used</td> 510 </tr> 511 </tbody> 512 </table> 513 514 <p class=note> 515 <strong>Note:</strong> care must be taken if the codec is flushed immediately or shortly 516 after start, before any output buffer or output format change has been returned, as the codec 517 specific data may be lost during the flush. You must resubmit the data using buffers marked with 518 {@link #BUFFER_FLAG_CODEC_CONFIG} after such flush to ensure proper codec operation. 519 <p> 520 Encoders (or codecs that generate compressed data) will create and return the codec specific data 521 before any valid output buffer in output buffers marked with the {@linkplain 522 #BUFFER_FLAG_CODEC_CONFIG codec-config flag}. Buffers containing codec-specific-data have no 523 meaningful timestamps. 524 525 <h3>Data Processing</h3> 526 <p> 527 Each codec maintains a set of input and output buffers that are referred to by a buffer-ID in 528 API calls. After a successful call to {@link #start} the client "owns" neither input nor output 529 buffers. In synchronous mode, call {@link #dequeueInputBuffer dequeueInput}/{@link 530 #dequeueOutputBuffer OutputBuffer(…)} to obtain (get ownership of) an input or output 531 buffer from the codec. In asynchronous mode, you will automatically receive available buffers via 532 the {@link Callback#onInputBufferAvailable MediaCodec.Callback.onInput}/{@link 533 Callback#onOutputBufferAvailable OutputBufferAvailable(…)} callbacks. 534 <p> 535 Upon obtaining an input buffer, fill it with data and submit it to the codec using {@link 536 #queueInputBuffer queueInputBuffer} – or {@link #queueSecureInputBuffer 537 queueSecureInputBuffer} if using decryption. Do not submit multiple input buffers with the same 538 timestamp (unless it is <a href="#CSD">codec-specific data</a> marked as such). 539 <p> 540 The codec in turn will return a read-only output buffer via the {@link 541 Callback#onOutputBufferAvailable onOutputBufferAvailable} callback in asynchronous mode, or in 542 response to a {@link #dequeueOutputBuffer dequeueOutputBuffer} call in synchronous mode. After the 543 output buffer has been processed, call one of the {@link #releaseOutputBuffer 544 releaseOutputBuffer} methods to return the buffer to the codec. 545 <p> 546 While you are not required to resubmit/release buffers immediately to the codec, holding onto 547 input and/or output buffers may stall the codec, and this behavior is device dependent. 548 <strong>Specifically, it is possible that a codec may hold off on generating output buffers until 549 <em>all</em> outstanding buffers have been released/resubmitted.</strong> Therefore, try to 550 hold onto to available buffers as little as possible. 551 <p> 552 Depending on the API version, you can process data in three ways: 553 <table> 554 <thead> 555 <tr> 556 <th>Processing Mode</th> 557 <th>API version <= 20<br>Jelly Bean/KitKat</th> 558 <th>API version >= 21<br>Lollipop and later</th> 559 </tr> 560 </thead> 561 <tbody> 562 <tr> 563 <td>Synchronous API using buffer arrays</td> 564 <td>Supported</td> 565 <td>Deprecated</td> 566 </tr> 567 <tr> 568 <td>Synchronous API using buffers</td> 569 <td class=NA>Not Available</td> 570 <td>Supported</td> 571 </tr> 572 <tr> 573 <td>Asynchronous API using buffers</td> 574 <td class=NA>Not Available</td> 575 <td>Supported</td> 576 </tr> 577 </tbody> 578 </table> 579 580 <h4>Asynchronous Processing using Buffers</h4> 581 <p> 582 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, the preferred method is to process data 583 asynchronously by setting a callback before calling {@link #configure configure}. Asynchronous 584 mode changes the state transitions slightly, because you must call {@link #start} after {@link 585 #flush} to transition the codec to the Running sub-state and start receiving input buffers. 586 Similarly, upon an initial call to {@code start} the codec will move directly to the Running 587 sub-state and start passing available input buffers via the callback. 588 <p> 589 <center> 590 <img src="../../../images/media/mediacodec_async_states.svg" style="width: 516px; height: 353px" 591 alt="MediaCodec state diagram for asynchronous operation"> 592 </center> 593 <p> 594 MediaCodec is typically used like this in asynchronous mode: 595 <pre class=prettyprint> 596 MediaCodec codec = MediaCodec.createByCodecName(name); 597 MediaFormat mOutputFormat; // member variable 598 codec.setCallback(new MediaCodec.Callback() { 599 {@literal @Override} 600 void onInputBufferAvailable(MediaCodec mc, int inputBufferId) { 601 ByteBuffer inputBuffer = codec.getInputBuffer(inputBufferId); 602 // fill inputBuffer with valid data 603 … 604 codec.queueInputBuffer(inputBufferId, …); 605 } 606 607 {@literal @Override} 608 void onOutputBufferAvailable(MediaCodec mc, int outputBufferId, …) { 609 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 610 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 611 // bufferFormat is equivalent to mOutputFormat 612 // outputBuffer is ready to be processed or rendered. 613 … 614 codec.releaseOutputBuffer(outputBufferId, …); 615 } 616 617 {@literal @Override} 618 void onOutputFormatChanged(MediaCodec mc, MediaFormat format) { 619 // Subsequent data will conform to new format. 620 // Can ignore if using getOutputFormat(outputBufferId) 621 mOutputFormat = format; // option B 622 } 623 624 {@literal @Override} 625 void onError(…) { 626 … 627 } 628 {@literal @Override} 629 void onCryptoError(…) { 630 … 631 } 632 }); 633 codec.configure(format, …); 634 mOutputFormat = codec.getOutputFormat(); // option B 635 codec.start(); 636 // wait for processing to complete 637 codec.stop(); 638 codec.release();</pre> 639 640 <h4>Synchronous Processing using Buffers</h4> 641 <p> 642 Since {@link android.os.Build.VERSION_CODES#LOLLIPOP}, you should retrieve input and output 643 buffers using {@link #getInputBuffer getInput}/{@link #getOutputBuffer OutputBuffer(int)} and/or 644 {@link #getInputImage getInput}/{@link #getOutputImage OutputImage(int)} even when using the 645 codec in synchronous mode. This allows certain optimizations by the framework, e.g. when 646 processing dynamic content. This optimization is disabled if you call {@link #getInputBuffers 647 getInput}/{@link #getOutputBuffers OutputBuffers()}. 648 649 <p class=note> 650 <strong>Note:</strong> do not mix the methods of using buffers and buffer arrays at the same 651 time. Specifically, only call {@code getInput}/{@code OutputBuffers} directly after {@link 652 #start} or after having dequeued an output buffer ID with the value of {@link 653 #INFO_OUTPUT_FORMAT_CHANGED}. 654 <p> 655 MediaCodec is typically used like this in synchronous mode: 656 <pre> 657 MediaCodec codec = MediaCodec.createByCodecName(name); 658 codec.configure(format, …); 659 MediaFormat outputFormat = codec.getOutputFormat(); // option B 660 codec.start(); 661 for (;;) { 662 int inputBufferId = codec.dequeueInputBuffer(timeoutUs); 663 if (inputBufferId >= 0) { 664 ByteBuffer inputBuffer = codec.getInputBuffer(…); 665 // fill inputBuffer with valid data 666 … 667 codec.queueInputBuffer(inputBufferId, …); 668 } 669 int outputBufferId = codec.dequeueOutputBuffer(…); 670 if (outputBufferId >= 0) { 671 ByteBuffer outputBuffer = codec.getOutputBuffer(outputBufferId); 672 MediaFormat bufferFormat = codec.getOutputFormat(outputBufferId); // option A 673 // bufferFormat is identical to outputFormat 674 // outputBuffer is ready to be processed or rendered. 675 … 676 codec.releaseOutputBuffer(outputBufferId, …); 677 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 678 // Subsequent data will conform to new format. 679 // Can ignore if using getOutputFormat(outputBufferId) 680 outputFormat = codec.getOutputFormat(); // option B 681 } 682 } 683 codec.stop(); 684 codec.release();</pre> 685 686 <h4>Synchronous Processing using Buffer Arrays (deprecated)</h4> 687 <p> 688 In versions {@link android.os.Build.VERSION_CODES#KITKAT_WATCH} and before, the set of input and 689 output buffers are represented by the {@code ByteBuffer[]} arrays. After a successful call to 690 {@link #start}, retrieve the buffer arrays using {@link #getInputBuffers getInput}/{@link 691 #getOutputBuffers OutputBuffers()}. Use the buffer ID-s as indices into these arrays (when 692 non-negative), as demonstrated in the sample below. Note that there is no inherent correlation 693 between the size of the arrays and the number of input and output buffers used by the system, 694 although the array size provides an upper bound. 695 <pre> 696 MediaCodec codec = MediaCodec.createByCodecName(name); 697 codec.configure(format, …); 698 codec.start(); 699 ByteBuffer[] inputBuffers = codec.getInputBuffers(); 700 ByteBuffer[] outputBuffers = codec.getOutputBuffers(); 701 for (;;) { 702 int inputBufferId = codec.dequeueInputBuffer(…); 703 if (inputBufferId >= 0) { 704 // fill inputBuffers[inputBufferId] with valid data 705 … 706 codec.queueInputBuffer(inputBufferId, …); 707 } 708 int outputBufferId = codec.dequeueOutputBuffer(…); 709 if (outputBufferId >= 0) { 710 // outputBuffers[outputBufferId] is ready to be processed or rendered. 711 … 712 codec.releaseOutputBuffer(outputBufferId, …); 713 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { 714 outputBuffers = codec.getOutputBuffers(); 715 } else if (outputBufferId == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { 716 // Subsequent data will conform to new format. 717 MediaFormat format = codec.getOutputFormat(); 718 } 719 } 720 codec.stop(); 721 codec.release();</pre> 722 723 <h4>End-of-stream Handling</h4> 724 <p> 725 When you reach the end of the input data, you must signal it to the codec by specifying the 726 {@link #BUFFER_FLAG_END_OF_STREAM} flag in the call to {@link #queueInputBuffer 727 queueInputBuffer}. You can do this on the last valid input buffer, or by submitting an additional 728 empty input buffer with the end-of-stream flag set. If using an empty buffer, the timestamp will 729 be ignored. 730 <p> 731 The codec will continue to return output buffers until it eventually signals the end of the 732 output stream by specifying the same end-of-stream flag in the {@link BufferInfo} set in {@link 733 #dequeueOutputBuffer dequeueOutputBuffer} or returned via {@link Callback#onOutputBufferAvailable 734 onOutputBufferAvailable}. This can be set on the last valid output buffer, or on an empty buffer 735 after the last valid output buffer. The timestamp of such empty buffer should be ignored. 736 <p> 737 Do not submit additional input buffers after signaling the end of the input stream, unless the 738 codec has been flushed, or stopped and restarted. 739 740 <h4>Using an Output Surface</h4> 741 <p> 742 The data processing is nearly identical to the ByteBuffer mode when using an output {@link 743 Surface}; however, the output buffers will not be accessible, and are represented as {@code null} 744 values. E.g. {@link #getOutputBuffer getOutputBuffer}/{@link #getOutputImage Image(int)} will 745 return {@code null} and {@link #getOutputBuffers} will return an array containing only {@code 746 null}-s. 747 <p> 748 When using an output Surface, you can select whether or not to render each output buffer on the 749 surface. You have three choices: 750 <ul> 751 <li><strong>Do not render the buffer:</strong> Call {@link #releaseOutputBuffer(int, boolean) 752 releaseOutputBuffer(bufferId, false)}.</li> 753 <li><strong>Render the buffer with the default timestamp:</strong> Call {@link 754 #releaseOutputBuffer(int, boolean) releaseOutputBuffer(bufferId, true)}.</li> 755 <li><strong>Render the buffer with a specific timestamp:</strong> Call {@link 756 #releaseOutputBuffer(int, long) releaseOutputBuffer(bufferId, timestamp)}.</li> 757 </ul> 758 <p> 759 Since {@link android.os.Build.VERSION_CODES#M}, the default timestamp is the {@linkplain 760 BufferInfo#presentationTimeUs presentation timestamp} of the buffer (converted to nanoseconds). 761 It was not defined prior to that. 762 <p> 763 Also since {@link android.os.Build.VERSION_CODES#M}, you can change the output Surface 764 dynamically using {@link #setOutputSurface setOutputSurface}. 765 <p> 766 When rendering output to a Surface, the Surface may be configured to drop excessive frames (that 767 are not consumed by the Surface in a timely manner). Or it may be configured to not drop excessive 768 frames. In the latter mode if the Surface is not consuming output frames fast enough, it will 769 eventually block the decoder. Prior to {@link android.os.Build.VERSION_CODES#Q} the exact behavior 770 was undefined, with the exception that View surfaces (SurfaceView or TextureView) always dropped 771 excessive frames. Since {@link android.os.Build.VERSION_CODES#Q} the default behavior is to drop 772 excessive frames. Applications can opt out of this behavior for non-View surfaces (such as 773 ImageReader or SurfaceTexture) by targeting SDK {@link android.os.Build.VERSION_CODES#Q} and 774 setting the key {@link MediaFormat#KEY_ALLOW_FRAME_DROP} to {@code 0} 775 in their configure format. 776 777 <h4>Transformations When Rendering onto Surface</h4> 778 779 If the codec is configured into Surface mode, any crop rectangle, {@linkplain 780 MediaFormat#KEY_ROTATION rotation} and {@linkplain #setVideoScalingMode video scaling 781 mode} will be automatically applied with one exception: 782 <p class=note> 783 Prior to the {@link android.os.Build.VERSION_CODES#M} release, software decoders may not 784 have applied the rotation when being rendered onto a Surface. Unfortunately, there is no standard 785 and simple way to identify software decoders, or if they apply the rotation other than by trying 786 it out. 787 <p> 788 There are also some caveats. 789 <p class=note> 790 Note that the pixel aspect ratio is not considered when displaying the output onto the 791 Surface. This means that if you are using {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT} mode, you 792 must position the output Surface so that it has the proper final display aspect ratio. Conversely, 793 you can only use {@link #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode for content with 794 square pixels (pixel aspect ratio or 1:1). 795 <p class=note> 796 Note also that as of {@link android.os.Build.VERSION_CODES#N} release, {@link 797 #VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING} mode may not work correctly for videos rotated 798 by 90 or 270 degrees. 799 <p class=note> 800 When setting the video scaling mode, note that it must be reset after each time the output 801 buffers change. Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, you can 802 do this after each time the output format changes. 803 804 <h4>Using an Input Surface</h4> 805 <p> 806 When using an input Surface, there are no accessible input buffers, as buffers are automatically 807 passed from the input surface to the codec. Calling {@link #dequeueInputBuffer 808 dequeueInputBuffer} will throw an {@code IllegalStateException}, and {@link #getInputBuffers} 809 returns a bogus {@code ByteBuffer[]} array that <strong>MUST NOT</strong> be written into. 810 <p> 811 Call {@link #signalEndOfInputStream} to signal end-of-stream. The input surface will stop 812 submitting data to the codec immediately after this call. 813 <p> 814 815 <h3>Seeking & Adaptive Playback Support</h3> 816 <p> 817 Video decoders (and in general codecs that consume compressed video data) behave differently 818 regarding seek and format change whether or not they support and are configured for adaptive 819 playback. You can check if a decoder supports {@linkplain 820 CodecCapabilities#FEATURE_AdaptivePlayback adaptive playback} via {@link 821 CodecCapabilities#isFeatureSupported CodecCapabilities.isFeatureSupported(String)}. Adaptive 822 playback support for video decoders is only activated if you configure the codec to decode onto a 823 {@link Surface}. 824 825 <h4 id=KeyFrames><a name="KeyFrames"></a>Stream Boundary and Key Frames</h4> 826 <p> 827 It is important that the input data after {@link #start} or {@link #flush} starts at a suitable 828 stream boundary: the first frame must be a key frame. A <em>key frame</em> can be decoded 829 completely on its own (for most codecs this means an I-frame), and no frames that are to be 830 displayed after a key frame refer to frames before the key frame. 831 <p> 832 The following table summarizes suitable key frames for various video formats. 833 <table> 834 <thead> 835 <tr> 836 <th>Format</th> 837 <th>Suitable key frame</th> 838 </tr> 839 </thead> 840 <tbody class=mid> 841 <tr> 842 <td>VP9/VP8</td> 843 <td>a suitable intraframe where no subsequent frames refer to frames prior to this frame.<br> 844 <i>(There is no specific name for such key frame.)</i></td> 845 </tr> 846 <tr> 847 <td>H.265 HEVC</td> 848 <td>IDR or CRA</td> 849 </tr> 850 <tr> 851 <td>H.264 AVC</td> 852 <td>IDR</td> 853 </tr> 854 <tr> 855 <td>MPEG-4<br>H.263<br>MPEG-2</td> 856 <td>a suitable I-frame where no subsequent frames refer to frames prior to this frame.<br> 857 <i>(There is no specific name for such key frame.)</td> 858 </tr> 859 </tbody> 860 </table> 861 862 <h4>For decoders that do not support adaptive playback (including when not decoding onto a 863 Surface)</h4> 864 <p> 865 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 866 seek) you <strong>MUST</strong> flush the decoder. Since all output buffers are immediately 867 revoked at the point of the flush, you may want to first signal then wait for the end-of-stream 868 before you call {@code flush}. It is important that the input data after a flush starts at a 869 suitable stream boundary/key frame. 870 <p class=note> 871 <strong>Note:</strong> the format of the data submitted after a flush must not change; {@link 872 #flush} does not support format discontinuities; for that, a full {@link #stop} - {@link 873 #configure configure(…)} - {@link #start} cycle is necessary. 874 875 <p class=note> 876 <strong>Also note:</strong> if you flush the codec too soon after {@link #start} – 877 generally, before the first output buffer or output format change is received – you 878 will need to resubmit the codec-specific-data to the codec. See the <a 879 href="#CSD">codec-specific-data section</a> for more info. 880 881 <h4>For decoders that support and are configured for adaptive playback</h4> 882 <p> 883 In order to start decoding data that is not adjacent to previously submitted data (i.e. after a 884 seek) it is <em>not necessary</em> to flush the decoder; however, input data after the 885 discontinuity must start at a suitable stream boundary/key frame. 886 <p> 887 For some video formats - namely H.264, H.265, VP8 and VP9 - it is also possible to change the 888 picture size or configuration mid-stream. To do this you must package the entire new 889 codec-specific configuration data together with the key frame into a single buffer (including 890 any start codes), and submit it as a <strong>regular</strong> input buffer. 891 <p> 892 You will receive an {@link #INFO_OUTPUT_FORMAT_CHANGED} return value from {@link 893 #dequeueOutputBuffer dequeueOutputBuffer} or a {@link Callback#onOutputBufferAvailable 894 onOutputFormatChanged} callback just after the picture-size change takes place and before any 895 frames with the new size have been returned. 896 <p class=note> 897 <strong>Note:</strong> just as the case for codec-specific data, be careful when calling 898 {@link #flush} shortly after you have changed the picture size. If you have not received 899 confirmation of the picture size change, you will need to repeat the request for the new picture 900 size. 901 902 <h3>Error handling</h3> 903 <p> 904 The factory methods {@link #createByCodecName createByCodecName} and {@link #createDecoderByType 905 createDecoder}/{@link #createEncoderByType EncoderByType} throw {@code IOException} on failure 906 which you must catch or declare to pass up. MediaCodec methods throw {@code 907 IllegalStateException} when the method is called from a codec state that does not allow it; this 908 is typically due to incorrect application API usage. Methods involving secure buffers may throw 909 {@link CryptoException}, which has further error information obtainable from {@link 910 CryptoException#getErrorCode}. 911 <p> 912 Internal codec errors result in a {@link CodecException}, which may be due to media content 913 corruption, hardware failure, resource exhaustion, and so forth, even when the application is 914 correctly using the API. The recommended action when receiving a {@code CodecException} 915 can be determined by calling {@link CodecException#isRecoverable} and {@link 916 CodecException#isTransient}: 917 <ul> 918 <li><strong>recoverable errors:</strong> If {@code isRecoverable()} returns true, then call 919 {@link #stop}, {@link #configure configure(…)}, and {@link #start} to recover.</li> 920 <li><strong>transient errors:</strong> If {@code isTransient()} returns true, then resources are 921 temporarily unavailable and the method may be retried at a later time.</li> 922 <li><strong>fatal errors:</strong> If both {@code isRecoverable()} and {@code isTransient()} 923 return false, then the {@code CodecException} is fatal and the codec must be {@linkplain #reset 924 reset} or {@linkplain #release released}.</li> 925 </ul> 926 <p> 927 Both {@code isRecoverable()} and {@code isTransient()} do not return true at the same time. 928 929 <h2 id=History><a name="History"></a>Valid API Calls and API History</h2> 930 <p> 931 This sections summarizes the valid API calls in each state and the API history of the MediaCodec 932 class. For API version numbers, see {@link android.os.Build.VERSION_CODES}. 933 934 <style> 935 .api > tr > th, .api > tr > td { text-align: center; padding: 4px 4px; } 936 .api > tr > th { vertical-align: bottom; } 937 .api > tr > td { vertical-align: middle; } 938 .sml > tr > th, .sml > tr > td { text-align: center; padding: 2px 4px; } 939 .fn { text-align: left; } 940 .fn > code > a { font: 14px/19px Roboto Condensed, sans-serif; } 941 .deg45 { 942 white-space: nowrap; background: none; border: none; vertical-align: bottom; 943 width: 30px; height: 83px; 944 } 945 .deg45 > div { 946 transform: skew(-45deg, 0deg) translate(1px, -67px); 947 transform-origin: bottom left 0; 948 width: 30px; height: 20px; 949 } 950 .deg45 > div > div { border: 1px solid #ddd; background: #999; height: 90px; width: 42px; } 951 .deg45 > div > div > div { transform: skew(45deg, 0deg) translate(-55px, 55px) rotate(-45deg); } 952 </style> 953 954 <table align="right" style="width: 0%"> 955 <thead> 956 <tr><th>Symbol</th><th>Meaning</th></tr> 957 </thead> 958 <tbody class=sml> 959 <tr><td>●</td><td>Supported</td></tr> 960 <tr><td>⁕</td><td>Semantics changed</td></tr> 961 <tr><td>○</td><td>Experimental support</td></tr> 962 <tr><td>[ ]</td><td>Deprecated</td></tr> 963 <tr><td>⎋</td><td>Restricted to surface input mode</td></tr> 964 <tr><td>⎆</td><td>Restricted to surface output mode</td></tr> 965 <tr><td>▧</td><td>Restricted to ByteBuffer input mode</td></tr> 966 <tr><td>↩</td><td>Restricted to synchronous mode</td></tr> 967 <tr><td>⇄</td><td>Restricted to asynchronous mode</td></tr> 968 <tr><td>( )</td><td>Can be called, but shouldn't</td></tr> 969 </tbody> 970 </table> 971 972 <table style="width: 100%;"> 973 <thead class=api> 974 <tr> 975 <th class=deg45><div><div style="background:#4285f4"><div>Uninitialized</div></div></div></th> 976 <th class=deg45><div><div style="background:#f4b400"><div>Configured</div></div></div></th> 977 <th class=deg45><div><div style="background:#e67c73"><div>Flushed</div></div></div></th> 978 <th class=deg45><div><div style="background:#0f9d58"><div>Running</div></div></div></th> 979 <th class=deg45><div><div style="background:#f7cb4d"><div>End of Stream</div></div></div></th> 980 <th class=deg45><div><div style="background:#db4437"><div>Error</div></div></div></th> 981 <th class=deg45><div><div style="background:#666"><div>Released</div></div></div></th> 982 <th></th> 983 <th colspan="8">SDK Version</th> 984 </tr> 985 <tr> 986 <th colspan="7">State</th> 987 <th>Method</th> 988 <th>16</th> 989 <th>17</th> 990 <th>18</th> 991 <th>19</th> 992 <th>20</th> 993 <th>21</th> 994 <th>22</th> 995 <th>23</th> 996 </tr> 997 </thead> 998 <tbody class=api> 999 <tr> 1000 <td></td> 1001 <td></td> 1002 <td></td> 1003 <td></td> 1004 <td></td> 1005 <td></td> 1006 <td></td> 1007 <td class=fn>{@link #createByCodecName createByCodecName}</td> 1008 <td>●</td> 1009 <td>●</td> 1010 <td>●</td> 1011 <td>●</td> 1012 <td>●</td> 1013 <td>●</td> 1014 <td>●</td> 1015 <td>●</td> 1016 </tr> 1017 <tr> 1018 <td></td> 1019 <td></td> 1020 <td></td> 1021 <td></td> 1022 <td></td> 1023 <td></td> 1024 <td></td> 1025 <td class=fn>{@link #createDecoderByType createDecoderByType}</td> 1026 <td>●</td> 1027 <td>●</td> 1028 <td>●</td> 1029 <td>●</td> 1030 <td>●</td> 1031 <td>●</td> 1032 <td>●</td> 1033 <td>●</td> 1034 </tr> 1035 <tr> 1036 <td></td> 1037 <td></td> 1038 <td></td> 1039 <td></td> 1040 <td></td> 1041 <td></td> 1042 <td></td> 1043 <td class=fn>{@link #createEncoderByType createEncoderByType}</td> 1044 <td>●</td> 1045 <td>●</td> 1046 <td>●</td> 1047 <td>●</td> 1048 <td>●</td> 1049 <td>●</td> 1050 <td>●</td> 1051 <td>●</td> 1052 </tr> 1053 <tr> 1054 <td></td> 1055 <td></td> 1056 <td></td> 1057 <td></td> 1058 <td></td> 1059 <td></td> 1060 <td></td> 1061 <td class=fn>{@link #createPersistentInputSurface createPersistentInputSurface}</td> 1062 <td></td> 1063 <td></td> 1064 <td></td> 1065 <td></td> 1066 <td></td> 1067 <td></td> 1068 <td></td> 1069 <td>●</td> 1070 </tr> 1071 <tr> 1072 <td>16+</td> 1073 <td>-</td> 1074 <td>-</td> 1075 <td>-</td> 1076 <td>-</td> 1077 <td>-</td> 1078 <td>-</td> 1079 <td class=fn>{@link #configure configure}</td> 1080 <td>●</td> 1081 <td>●</td> 1082 <td>●</td> 1083 <td>●</td> 1084 <td>●</td> 1085 <td>⁕</td> 1086 <td>●</td> 1087 <td>●</td> 1088 </tr> 1089 <tr> 1090 <td>-</td> 1091 <td>18+</td> 1092 <td>-</td> 1093 <td>-</td> 1094 <td>-</td> 1095 <td>-</td> 1096 <td>-</td> 1097 <td class=fn>{@link #createInputSurface createInputSurface}</td> 1098 <td></td> 1099 <td></td> 1100 <td>⎋</td> 1101 <td>⎋</td> 1102 <td>⎋</td> 1103 <td>⎋</td> 1104 <td>⎋</td> 1105 <td>⎋</td> 1106 </tr> 1107 <tr> 1108 <td>-</td> 1109 <td>-</td> 1110 <td>16+</td> 1111 <td>16+</td> 1112 <td>(16+)</td> 1113 <td>-</td> 1114 <td>-</td> 1115 <td class=fn>{@link #dequeueInputBuffer dequeueInputBuffer}</td> 1116 <td>●</td> 1117 <td>●</td> 1118 <td>▧</td> 1119 <td>▧</td> 1120 <td>▧</td> 1121 <td>⁕▧↩</td> 1122 <td>▧↩</td> 1123 <td>▧↩</td> 1124 </tr> 1125 <tr> 1126 <td>-</td> 1127 <td>-</td> 1128 <td>16+</td> 1129 <td>16+</td> 1130 <td>16+</td> 1131 <td>-</td> 1132 <td>-</td> 1133 <td class=fn>{@link #dequeueOutputBuffer dequeueOutputBuffer}</td> 1134 <td>●</td> 1135 <td>●</td> 1136 <td>●</td> 1137 <td>●</td> 1138 <td>●</td> 1139 <td>⁕↩</td> 1140 <td>↩</td> 1141 <td>↩</td> 1142 </tr> 1143 <tr> 1144 <td>-</td> 1145 <td>-</td> 1146 <td>16+</td> 1147 <td>16+</td> 1148 <td>16+</td> 1149 <td>-</td> 1150 <td>-</td> 1151 <td class=fn>{@link #flush flush}</td> 1152 <td>●</td> 1153 <td>●</td> 1154 <td>●</td> 1155 <td>●</td> 1156 <td>●</td> 1157 <td>●</td> 1158 <td>●</td> 1159 <td>●</td> 1160 </tr> 1161 <tr> 1162 <td>18+</td> 1163 <td>18+</td> 1164 <td>18+</td> 1165 <td>18+</td> 1166 <td>18+</td> 1167 <td>18+</td> 1168 <td>-</td> 1169 <td class=fn>{@link #getCodecInfo getCodecInfo}</td> 1170 <td></td> 1171 <td></td> 1172 <td>●</td> 1173 <td>●</td> 1174 <td>●</td> 1175 <td>●</td> 1176 <td>●</td> 1177 <td>●</td> 1178 </tr> 1179 <tr> 1180 <td>-</td> 1181 <td>-</td> 1182 <td>(21+)</td> 1183 <td>21+</td> 1184 <td>(21+)</td> 1185 <td>-</td> 1186 <td>-</td> 1187 <td class=fn>{@link #getInputBuffer getInputBuffer}</td> 1188 <td></td> 1189 <td></td> 1190 <td></td> 1191 <td></td> 1192 <td></td> 1193 <td>●</td> 1194 <td>●</td> 1195 <td>●</td> 1196 </tr> 1197 <tr> 1198 <td>-</td> 1199 <td>-</td> 1200 <td>16+</td> 1201 <td>(16+)</td> 1202 <td>(16+)</td> 1203 <td>-</td> 1204 <td>-</td> 1205 <td class=fn>{@link #getInputBuffers getInputBuffers}</td> 1206 <td>●</td> 1207 <td>●</td> 1208 <td>●</td> 1209 <td>●</td> 1210 <td>●</td> 1211 <td>[⁕↩]</td> 1212 <td>[↩]</td> 1213 <td>[↩]</td> 1214 </tr> 1215 <tr> 1216 <td>-</td> 1217 <td>21+</td> 1218 <td>(21+)</td> 1219 <td>(21+)</td> 1220 <td>(21+)</td> 1221 <td>-</td> 1222 <td>-</td> 1223 <td class=fn>{@link #getInputFormat getInputFormat}</td> 1224 <td></td> 1225 <td></td> 1226 <td></td> 1227 <td></td> 1228 <td></td> 1229 <td>●</td> 1230 <td>●</td> 1231 <td>●</td> 1232 </tr> 1233 <tr> 1234 <td>-</td> 1235 <td>-</td> 1236 <td>(21+)</td> 1237 <td>21+</td> 1238 <td>(21+)</td> 1239 <td>-</td> 1240 <td>-</td> 1241 <td class=fn>{@link #getInputImage getInputImage}</td> 1242 <td></td> 1243 <td></td> 1244 <td></td> 1245 <td></td> 1246 <td></td> 1247 <td>○</td> 1248 <td>●</td> 1249 <td>●</td> 1250 </tr> 1251 <tr> 1252 <td>18+</td> 1253 <td>18+</td> 1254 <td>18+</td> 1255 <td>18+</td> 1256 <td>18+</td> 1257 <td>18+</td> 1258 <td>-</td> 1259 <td class=fn>{@link #getName getName}</td> 1260 <td></td> 1261 <td></td> 1262 <td>●</td> 1263 <td>●</td> 1264 <td>●</td> 1265 <td>●</td> 1266 <td>●</td> 1267 <td>●</td> 1268 </tr> 1269 <tr> 1270 <td>-</td> 1271 <td>-</td> 1272 <td>(21+)</td> 1273 <td>21+</td> 1274 <td>21+</td> 1275 <td>-</td> 1276 <td>-</td> 1277 <td class=fn>{@link #getOutputBuffer getOutputBuffer}</td> 1278 <td></td> 1279 <td></td> 1280 <td></td> 1281 <td></td> 1282 <td></td> 1283 <td>●</td> 1284 <td>●</td> 1285 <td>●</td> 1286 </tr> 1287 <tr> 1288 <td>-</td> 1289 <td>-</td> 1290 <td>16+</td> 1291 <td>16+</td> 1292 <td>16+</td> 1293 <td>-</td> 1294 <td>-</td> 1295 <td class=fn>{@link #getOutputBuffers getOutputBuffers}</td> 1296 <td>●</td> 1297 <td>●</td> 1298 <td>●</td> 1299 <td>●</td> 1300 <td>●</td> 1301 <td>[⁕↩]</td> 1302 <td>[↩]</td> 1303 <td>[↩]</td> 1304 </tr> 1305 <tr> 1306 <td>-</td> 1307 <td>21+</td> 1308 <td>16+</td> 1309 <td>16+</td> 1310 <td>16+</td> 1311 <td>-</td> 1312 <td>-</td> 1313 <td class=fn>{@link #getOutputFormat()}</td> 1314 <td>●</td> 1315 <td>●</td> 1316 <td>●</td> 1317 <td>●</td> 1318 <td>●</td> 1319 <td>●</td> 1320 <td>●</td> 1321 <td>●</td> 1322 </tr> 1323 <tr> 1324 <td>-</td> 1325 <td>-</td> 1326 <td>(21+)</td> 1327 <td>21+</td> 1328 <td>21+</td> 1329 <td>-</td> 1330 <td>-</td> 1331 <td class=fn>{@link #getOutputFormat(int)}</td> 1332 <td></td> 1333 <td></td> 1334 <td></td> 1335 <td></td> 1336 <td></td> 1337 <td>●</td> 1338 <td>●</td> 1339 <td>●</td> 1340 </tr> 1341 <tr> 1342 <td>-</td> 1343 <td>-</td> 1344 <td>(21+)</td> 1345 <td>21+</td> 1346 <td>21+</td> 1347 <td>-</td> 1348 <td>-</td> 1349 <td class=fn>{@link #getOutputImage getOutputImage}</td> 1350 <td></td> 1351 <td></td> 1352 <td></td> 1353 <td></td> 1354 <td></td> 1355 <td>○</td> 1356 <td>●</td> 1357 <td>●</td> 1358 </tr> 1359 <tr> 1360 <td>-</td> 1361 <td>-</td> 1362 <td>-</td> 1363 <td>16+</td> 1364 <td>(16+)</td> 1365 <td>-</td> 1366 <td>-</td> 1367 <td class=fn>{@link #queueInputBuffer queueInputBuffer}</td> 1368 <td>●</td> 1369 <td>●</td> 1370 <td>●</td> 1371 <td>●</td> 1372 <td>●</td> 1373 <td>⁕</td> 1374 <td>●</td> 1375 <td>●</td> 1376 </tr> 1377 <tr> 1378 <td>-</td> 1379 <td>-</td> 1380 <td>-</td> 1381 <td>16+</td> 1382 <td>(16+)</td> 1383 <td>-</td> 1384 <td>-</td> 1385 <td class=fn>{@link #queueSecureInputBuffer queueSecureInputBuffer}</td> 1386 <td>●</td> 1387 <td>●</td> 1388 <td>●</td> 1389 <td>●</td> 1390 <td>●</td> 1391 <td>⁕</td> 1392 <td>●</td> 1393 <td>●</td> 1394 </tr> 1395 <tr> 1396 <td>16+</td> 1397 <td>16+</td> 1398 <td>16+</td> 1399 <td>16+</td> 1400 <td>16+</td> 1401 <td>16+</td> 1402 <td>16+</td> 1403 <td class=fn>{@link #release release}</td> 1404 <td>●</td> 1405 <td>●</td> 1406 <td>●</td> 1407 <td>●</td> 1408 <td>●</td> 1409 <td>●</td> 1410 <td>●</td> 1411 <td>●</td> 1412 </tr> 1413 <tr> 1414 <td>-</td> 1415 <td>-</td> 1416 <td>-</td> 1417 <td>16+</td> 1418 <td>16+</td> 1419 <td>-</td> 1420 <td>-</td> 1421 <td class=fn>{@link #releaseOutputBuffer(int, boolean)}</td> 1422 <td>●</td> 1423 <td>●</td> 1424 <td>●</td> 1425 <td>●</td> 1426 <td>●</td> 1427 <td>⁕</td> 1428 <td>●</td> 1429 <td>⁕</td> 1430 </tr> 1431 <tr> 1432 <td>-</td> 1433 <td>-</td> 1434 <td>-</td> 1435 <td>21+</td> 1436 <td>21+</td> 1437 <td>-</td> 1438 <td>-</td> 1439 <td class=fn>{@link #releaseOutputBuffer(int, long)}</td> 1440 <td></td> 1441 <td></td> 1442 <td></td> 1443 <td></td> 1444 <td></td> 1445 <td>⎆</td> 1446 <td>⎆</td> 1447 <td>⎆</td> 1448 </tr> 1449 <tr> 1450 <td>21+</td> 1451 <td>21+</td> 1452 <td>21+</td> 1453 <td>21+</td> 1454 <td>21+</td> 1455 <td>21+</td> 1456 <td>-</td> 1457 <td class=fn>{@link #reset reset}</td> 1458 <td></td> 1459 <td></td> 1460 <td></td> 1461 <td></td> 1462 <td></td> 1463 <td>●</td> 1464 <td>●</td> 1465 <td>●</td> 1466 </tr> 1467 <tr> 1468 <td>21+</td> 1469 <td>-</td> 1470 <td>-</td> 1471 <td>-</td> 1472 <td>-</td> 1473 <td>-</td> 1474 <td>-</td> 1475 <td class=fn>{@link #setCallback(Callback) setCallback}</td> 1476 <td></td> 1477 <td></td> 1478 <td></td> 1479 <td></td> 1480 <td></td> 1481 <td>●</td> 1482 <td>●</td> 1483 <td>{@link #setCallback(Callback, Handler) ⁕}</td> 1484 </tr> 1485 <tr> 1486 <td>-</td> 1487 <td>23+</td> 1488 <td>-</td> 1489 <td>-</td> 1490 <td>-</td> 1491 <td>-</td> 1492 <td>-</td> 1493 <td class=fn>{@link #setInputSurface setInputSurface}</td> 1494 <td></td> 1495 <td></td> 1496 <td></td> 1497 <td></td> 1498 <td></td> 1499 <td></td> 1500 <td></td> 1501 <td>⎋</td> 1502 </tr> 1503 <tr> 1504 <td>23+</td> 1505 <td>23+</td> 1506 <td>23+</td> 1507 <td>23+</td> 1508 <td>23+</td> 1509 <td>(23+)</td> 1510 <td>(23+)</td> 1511 <td class=fn>{@link #setOnFrameRenderedListener setOnFrameRenderedListener}</td> 1512 <td></td> 1513 <td></td> 1514 <td></td> 1515 <td></td> 1516 <td></td> 1517 <td></td> 1518 <td></td> 1519 <td>○ ⎆</td> 1520 </tr> 1521 <tr> 1522 <td>-</td> 1523 <td>23+</td> 1524 <td>23+</td> 1525 <td>23+</td> 1526 <td>23+</td> 1527 <td>-</td> 1528 <td>-</td> 1529 <td class=fn>{@link #setOutputSurface setOutputSurface}</td> 1530 <td></td> 1531 <td></td> 1532 <td></td> 1533 <td></td> 1534 <td></td> 1535 <td></td> 1536 <td></td> 1537 <td>⎆</td> 1538 </tr> 1539 <tr> 1540 <td>19+</td> 1541 <td>19+</td> 1542 <td>19+</td> 1543 <td>19+</td> 1544 <td>19+</td> 1545 <td>(19+)</td> 1546 <td>-</td> 1547 <td class=fn>{@link #setParameters setParameters}</td> 1548 <td></td> 1549 <td></td> 1550 <td></td> 1551 <td>●</td> 1552 <td>●</td> 1553 <td>●</td> 1554 <td>●</td> 1555 <td>●</td> 1556 </tr> 1557 <tr> 1558 <td>-</td> 1559 <td>(16+)</td> 1560 <td>(16+)</td> 1561 <td>16+</td> 1562 <td>(16+)</td> 1563 <td>(16+)</td> 1564 <td>-</td> 1565 <td class=fn>{@link #setVideoScalingMode setVideoScalingMode}</td> 1566 <td>⎆</td> 1567 <td>⎆</td> 1568 <td>⎆</td> 1569 <td>⎆</td> 1570 <td>⎆</td> 1571 <td>⎆</td> 1572 <td>⎆</td> 1573 <td>⎆</td> 1574 </tr> 1575 <tr> 1576 <td>(29+)</td> 1577 <td>29+</td> 1578 <td>29+</td> 1579 <td>29+</td> 1580 <td>(29+)</td> 1581 <td>(29+)</td> 1582 <td>-</td> 1583 <td class=fn>{@link #setAudioPresentation setAudioPresentation}</td> 1584 <td></td> 1585 <td></td> 1586 <td></td> 1587 <td></td> 1588 <td></td> 1589 <td></td> 1590 <td></td> 1591 <td></td> 1592 </tr> 1593 <tr> 1594 <td>-</td> 1595 <td>-</td> 1596 <td>18+</td> 1597 <td>18+</td> 1598 <td>-</td> 1599 <td>-</td> 1600 <td>-</td> 1601 <td class=fn>{@link #signalEndOfInputStream signalEndOfInputStream}</td> 1602 <td></td> 1603 <td></td> 1604 <td>⎋</td> 1605 <td>⎋</td> 1606 <td>⎋</td> 1607 <td>⎋</td> 1608 <td>⎋</td> 1609 <td>⎋</td> 1610 </tr> 1611 <tr> 1612 <td>-</td> 1613 <td>16+</td> 1614 <td>21+(⇄)</td> 1615 <td>-</td> 1616 <td>-</td> 1617 <td>-</td> 1618 <td>-</td> 1619 <td class=fn>{@link #start start}</td> 1620 <td>●</td> 1621 <td>●</td> 1622 <td>●</td> 1623 <td>●</td> 1624 <td>●</td> 1625 <td>⁕</td> 1626 <td>●</td> 1627 <td>●</td> 1628 </tr> 1629 <tr> 1630 <td>-</td> 1631 <td>-</td> 1632 <td>16+</td> 1633 <td>16+</td> 1634 <td>16+</td> 1635 <td>-</td> 1636 <td>-</td> 1637 <td class=fn>{@link #stop stop}</td> 1638 <td>●</td> 1639 <td>●</td> 1640 <td>●</td> 1641 <td>●</td> 1642 <td>●</td> 1643 <td>●</td> 1644 <td>●</td> 1645 <td>●</td> 1646 </tr> 1647 </tbody> 1648 </table> 1649 */ 1650 final public class MediaCodec { 1651 1652 /** 1653 * Per buffer metadata includes an offset and size specifying 1654 * the range of valid data in the associated codec (output) buffer. 1655 */ 1656 public final static class BufferInfo { 1657 /** 1658 * Update the buffer metadata information. 1659 * 1660 * @param newOffset the start-offset of the data in the buffer. 1661 * @param newSize the amount of data (in bytes) in the buffer. 1662 * @param newTimeUs the presentation timestamp in microseconds. 1663 * @param newFlags buffer flags associated with the buffer. This 1664 * should be a combination of {@link #BUFFER_FLAG_KEY_FRAME} and 1665 * {@link #BUFFER_FLAG_END_OF_STREAM}. 1666 */ set( int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags)1667 public void set( 1668 int newOffset, int newSize, long newTimeUs, @BufferFlag int newFlags) { 1669 offset = newOffset; 1670 size = newSize; 1671 presentationTimeUs = newTimeUs; 1672 flags = newFlags; 1673 } 1674 1675 /** 1676 * The start-offset of the data in the buffer. 1677 */ 1678 public int offset; 1679 1680 /** 1681 * The amount of data (in bytes) in the buffer. If this is {@code 0}, 1682 * the buffer has no data in it and can be discarded. The only 1683 * use of a 0-size buffer is to carry the end-of-stream marker. 1684 */ 1685 public int size; 1686 1687 /** 1688 * The presentation timestamp in microseconds for the buffer. 1689 * This is derived from the presentation timestamp passed in 1690 * with the corresponding input buffer. This should be ignored for 1691 * a 0-sized buffer. 1692 */ 1693 public long presentationTimeUs; 1694 1695 /** 1696 * Buffer flags associated with the buffer. A combination of 1697 * {@link #BUFFER_FLAG_KEY_FRAME} and {@link #BUFFER_FLAG_END_OF_STREAM}. 1698 * 1699 * <p>Encoded buffers that are key frames are marked with 1700 * {@link #BUFFER_FLAG_KEY_FRAME}. 1701 * 1702 * <p>The last output buffer corresponding to the input buffer 1703 * marked with {@link #BUFFER_FLAG_END_OF_STREAM} will also be marked 1704 * with {@link #BUFFER_FLAG_END_OF_STREAM}. In some cases this could 1705 * be an empty buffer, whose sole purpose is to carry the end-of-stream 1706 * marker. 1707 */ 1708 @BufferFlag 1709 public int flags; 1710 1711 /** @hide */ 1712 @NonNull dup()1713 public BufferInfo dup() { 1714 BufferInfo copy = new BufferInfo(); 1715 copy.set(offset, size, presentationTimeUs, flags); 1716 return copy; 1717 } 1718 }; 1719 1720 // The follow flag constants MUST stay in sync with their equivalents 1721 // in MediaCodec.h ! 1722 1723 /** 1724 * This indicates that the (encoded) buffer marked as such contains 1725 * the data for a key frame. 1726 * 1727 * @deprecated Use {@link #BUFFER_FLAG_KEY_FRAME} instead. 1728 */ 1729 public static final int BUFFER_FLAG_SYNC_FRAME = 1; 1730 1731 /** 1732 * This indicates that the (encoded) buffer marked as such contains 1733 * the data for a key frame. 1734 */ 1735 public static final int BUFFER_FLAG_KEY_FRAME = 1; 1736 1737 /** 1738 * This indicated that the buffer marked as such contains codec 1739 * initialization / codec specific data instead of media data. 1740 */ 1741 public static final int BUFFER_FLAG_CODEC_CONFIG = 2; 1742 1743 /** 1744 * This signals the end of stream, i.e. no buffers will be available 1745 * after this, unless of course, {@link #flush} follows. 1746 */ 1747 public static final int BUFFER_FLAG_END_OF_STREAM = 4; 1748 1749 /** 1750 * This indicates that the buffer only contains part of a frame, 1751 * and the decoder should batch the data until a buffer without 1752 * this flag appears before decoding the frame. 1753 */ 1754 public static final int BUFFER_FLAG_PARTIAL_FRAME = 8; 1755 1756 /** 1757 * This indicates that the buffer contains non-media data for the 1758 * muxer to process. 1759 * 1760 * All muxer data should start with a FOURCC header that determines the type of data. 1761 * 1762 * For example, when it contains Exif data sent to a MediaMuxer track of 1763 * {@link MediaFormat#MIMETYPE_IMAGE_ANDROID_HEIC} type, the data must start with 1764 * Exif header ("Exif\0\0"), followed by the TIFF header (See JEITA CP-3451C Section 4.5.2.) 1765 * 1766 * @hide 1767 */ 1768 public static final int BUFFER_FLAG_MUXER_DATA = 16; 1769 1770 /** 1771 * This indicates that the buffer is decoded and updates the internal state of the decoder, 1772 * but does not produce any output buffer. 1773 * 1774 * When a buffer has this flag set, 1775 * {@link OnFrameRenderedListener#onFrameRendered(MediaCodec, long, long)} and 1776 * {@link Callback#onOutputBufferAvailable(MediaCodec, int, BufferInfo)} will not be called for 1777 * that given buffer. 1778 * 1779 * For example, when seeking to a certain frame, that frame may need to reference previous 1780 * frames in order for it to produce output. The preceding frames can be marked with this flag 1781 * so that they are only decoded and their data is used when decoding the latter frame that 1782 * should be initially displayed post-seek. 1783 * Another example would be trick play, trick play is when a video is fast-forwarded and only a 1784 * subset of the frames is to be rendered on the screen. The frames not to be rendered can be 1785 * marked with this flag for the same reason as the above one. 1786 * Marking frames with this flag improves the overall performance of playing a video stream as 1787 * fewer frames need to be passed back to the app. 1788 * 1789 * In {@link CodecCapabilities#FEATURE_TunneledPlayback}, buffers marked with this flag 1790 * are not rendered on the output surface. 1791 * 1792 * A frame should not be marked with this flag and {@link #BUFFER_FLAG_END_OF_STREAM} 1793 * simultaneously, doing so will produce a {@link InvalidBufferFlagsException} 1794 */ 1795 public static final int BUFFER_FLAG_DECODE_ONLY = 32; 1796 1797 /** @hide */ 1798 @IntDef( 1799 flag = true, 1800 value = { 1801 BUFFER_FLAG_SYNC_FRAME, 1802 BUFFER_FLAG_KEY_FRAME, 1803 BUFFER_FLAG_CODEC_CONFIG, 1804 BUFFER_FLAG_END_OF_STREAM, 1805 BUFFER_FLAG_PARTIAL_FRAME, 1806 BUFFER_FLAG_MUXER_DATA, 1807 BUFFER_FLAG_DECODE_ONLY, 1808 }) 1809 @Retention(RetentionPolicy.SOURCE) 1810 public @interface BufferFlag {} 1811 1812 private EventHandler mEventHandler; 1813 private EventHandler mOnFirstTunnelFrameReadyHandler; 1814 private EventHandler mOnFrameRenderedHandler; 1815 private EventHandler mCallbackHandler; 1816 private Callback mCallback; 1817 private OnFirstTunnelFrameReadyListener mOnFirstTunnelFrameReadyListener; 1818 private OnFrameRenderedListener mOnFrameRenderedListener; 1819 private final Object mListenerLock = new Object(); 1820 private MediaCodecInfo mCodecInfo; 1821 private final Object mCodecInfoLock = new Object(); 1822 private MediaCrypto mCrypto; 1823 1824 private static final int EVENT_CALLBACK = 1; 1825 private static final int EVENT_SET_CALLBACK = 2; 1826 private static final int EVENT_FRAME_RENDERED = 3; 1827 private static final int EVENT_FIRST_TUNNEL_FRAME_READY = 4; 1828 1829 private static final int CB_INPUT_AVAILABLE = 1; 1830 private static final int CB_OUTPUT_AVAILABLE = 2; 1831 private static final int CB_ERROR = 3; 1832 private static final int CB_OUTPUT_FORMAT_CHANGE = 4; 1833 private static final String EOS_AND_DECODE_ONLY_ERROR_MESSAGE = "An input buffer cannot have " 1834 + "both BUFFER_FLAG_END_OF_STREAM and BUFFER_FLAG_DECODE_ONLY flags"; 1835 private static final int CB_CRYPTO_ERROR = 6; 1836 private static final int CB_LARGE_FRAME_OUTPUT_AVAILABLE = 7; 1837 1838 private class EventHandler extends Handler { 1839 private MediaCodec mCodec; 1840 EventHandler(@onNull MediaCodec codec, @NonNull Looper looper)1841 public EventHandler(@NonNull MediaCodec codec, @NonNull Looper looper) { 1842 super(looper); 1843 mCodec = codec; 1844 } 1845 1846 @Override handleMessage(@onNull Message msg)1847 public void handleMessage(@NonNull Message msg) { 1848 switch (msg.what) { 1849 case EVENT_CALLBACK: 1850 { 1851 handleCallback(msg); 1852 break; 1853 } 1854 case EVENT_SET_CALLBACK: 1855 { 1856 mCallback = (MediaCodec.Callback) msg.obj; 1857 break; 1858 } 1859 case EVENT_FRAME_RENDERED: 1860 Map<String, Object> map = (Map<String, Object>)msg.obj; 1861 for (int i = 0; ; ++i) { 1862 Object mediaTimeUs = map.get(i + "-media-time-us"); 1863 Object systemNano = map.get(i + "-system-nano"); 1864 OnFrameRenderedListener onFrameRenderedListener; 1865 synchronized (mListenerLock) { 1866 onFrameRenderedListener = mOnFrameRenderedListener; 1867 } 1868 if (mediaTimeUs == null || systemNano == null 1869 || onFrameRenderedListener == null) { 1870 break; 1871 } 1872 onFrameRenderedListener.onFrameRendered( 1873 mCodec, (long)mediaTimeUs, (long)systemNano); 1874 } 1875 break; 1876 case EVENT_FIRST_TUNNEL_FRAME_READY: 1877 OnFirstTunnelFrameReadyListener onFirstTunnelFrameReadyListener; 1878 synchronized (mListenerLock) { 1879 onFirstTunnelFrameReadyListener = mOnFirstTunnelFrameReadyListener; 1880 } 1881 if (onFirstTunnelFrameReadyListener == null) { 1882 break; 1883 } 1884 onFirstTunnelFrameReadyListener.onFirstTunnelFrameReady(mCodec); 1885 break; 1886 default: 1887 { 1888 break; 1889 } 1890 } 1891 } 1892 handleCallback(@onNull Message msg)1893 private void handleCallback(@NonNull Message msg) { 1894 if (mCallback == null) { 1895 return; 1896 } 1897 1898 switch (msg.arg1) { 1899 case CB_INPUT_AVAILABLE: 1900 { 1901 int index = msg.arg2; 1902 synchronized(mBufferLock) { 1903 switch (mBufferMode) { 1904 case BUFFER_MODE_LEGACY: 1905 validateInputByteBufferLocked(mCachedInputBuffers, index); 1906 break; 1907 case BUFFER_MODE_BLOCK: 1908 while (mQueueRequests.size() <= index) { 1909 mQueueRequests.add(null); 1910 } 1911 QueueRequest request = mQueueRequests.get(index); 1912 if (request == null) { 1913 request = new QueueRequest(mCodec, index); 1914 mQueueRequests.set(index, request); 1915 } 1916 request.setAccessible(true); 1917 break; 1918 default: 1919 throw new IllegalStateException( 1920 "Unrecognized buffer mode: " + mBufferMode); 1921 } 1922 } 1923 mCallback.onInputBufferAvailable(mCodec, index); 1924 break; 1925 } 1926 1927 case CB_OUTPUT_AVAILABLE: 1928 { 1929 int index = msg.arg2; 1930 BufferInfo info = (MediaCodec.BufferInfo) msg.obj; 1931 synchronized(mBufferLock) { 1932 switch (mBufferMode) { 1933 case BUFFER_MODE_LEGACY: 1934 validateOutputByteBufferLocked(mCachedOutputBuffers, index, info); 1935 break; 1936 case BUFFER_MODE_BLOCK: 1937 while (mOutputFrames.size() <= index) { 1938 mOutputFrames.add(null); 1939 } 1940 OutputFrame frame = mOutputFrames.get(index); 1941 if (frame == null) { 1942 frame = new OutputFrame(index); 1943 mOutputFrames.set(index, frame); 1944 } 1945 frame.setBufferInfo(info); 1946 frame.setAccessible(true); 1947 break; 1948 default: 1949 throw new IllegalStateException( 1950 "Unrecognized buffer mode: " + mBufferMode); 1951 } 1952 } 1953 mCallback.onOutputBufferAvailable( 1954 mCodec, index, info); 1955 break; 1956 } 1957 1958 case CB_LARGE_FRAME_OUTPUT_AVAILABLE: 1959 { 1960 int index = msg.arg2; 1961 ArrayDeque<BufferInfo> infos = (ArrayDeque<BufferInfo>)msg.obj; 1962 synchronized(mBufferLock) { 1963 switch (mBufferMode) { 1964 case BUFFER_MODE_LEGACY: 1965 validateOutputByteBuffersLocked(mCachedOutputBuffers, 1966 index, infos); 1967 break; 1968 case BUFFER_MODE_BLOCK: 1969 while (mOutputFrames.size() <= index) { 1970 mOutputFrames.add(null); 1971 } 1972 OutputFrame frame = mOutputFrames.get(index); 1973 if (frame == null) { 1974 frame = new OutputFrame(index); 1975 mOutputFrames.set(index, frame); 1976 } 1977 frame.setBufferInfos(infos); 1978 frame.setAccessible(true); 1979 break; 1980 default: 1981 throw new IllegalArgumentException( 1982 "Unrecognized buffer mode: for large frame output"); 1983 } 1984 } 1985 mCallback.onOutputBuffersAvailable( 1986 mCodec, index, infos); 1987 1988 break; 1989 } 1990 1991 case CB_ERROR: 1992 { 1993 mCallback.onError(mCodec, (MediaCodec.CodecException) msg.obj); 1994 break; 1995 } 1996 1997 case CB_CRYPTO_ERROR: 1998 { 1999 mCallback.onCryptoError(mCodec, (MediaCodec.CryptoException) msg.obj); 2000 break; 2001 } 2002 2003 case CB_OUTPUT_FORMAT_CHANGE: 2004 { 2005 mCallback.onOutputFormatChanged(mCodec, 2006 new MediaFormat((Map<String, Object>) msg.obj)); 2007 break; 2008 } 2009 2010 default: 2011 { 2012 break; 2013 } 2014 } 2015 } 2016 } 2017 2018 // HACKY(b/325389296): aconfig flag accessors may not work in all contexts where MediaCodec API 2019 // is used, so allow accessors to fail. In those contexts use a default value, normally false. 2020 2021 /* package private */ GetFlag(Supplier<Boolean> flagValueSupplier)2022 static boolean GetFlag(Supplier<Boolean> flagValueSupplier) { 2023 return GetFlag(flagValueSupplier, false /* defaultValue */); 2024 } 2025 2026 /* package private */ GetFlag(Supplier<Boolean> flagValueSupplier, boolean defaultValue)2027 static boolean GetFlag(Supplier<Boolean> flagValueSupplier, boolean defaultValue) { 2028 try { 2029 return flagValueSupplier.get(); 2030 } catch (java.lang.RuntimeException e) { 2031 return defaultValue; 2032 } 2033 } 2034 2035 private boolean mHasSurface = false; 2036 2037 /** 2038 * Instantiate the preferred decoder supporting input data of the given mime type. 2039 * 2040 * The following is a partial list of defined mime types and their semantics: 2041 * <ul> 2042 * <li>"video/x-vnd.on2.vp8" - VP8 video (i.e. video in .webm) 2043 * <li>"video/x-vnd.on2.vp9" - VP9 video (i.e. video in .webm) 2044 * <li>"video/avc" - H.264/AVC video 2045 * <li>"video/hevc" - H.265/HEVC video 2046 * <li>"video/mp4v-es" - MPEG4 video 2047 * <li>"video/3gpp" - H.263 video 2048 * <li>"audio/3gpp" - AMR narrowband audio 2049 * <li>"audio/amr-wb" - AMR wideband audio 2050 * <li>"audio/mpeg" - MPEG1/2 audio layer III 2051 * <li>"audio/mp4a-latm" - AAC audio (note, this is raw AAC packets, not packaged in LATM!) 2052 * <li>"audio/vorbis" - vorbis audio 2053 * <li>"audio/g711-alaw" - G.711 alaw audio 2054 * <li>"audio/g711-mlaw" - G.711 ulaw audio 2055 * </ul> 2056 * 2057 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findDecoderForFormat} 2058 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 2059 * given format. 2060 * 2061 * @param type The mime type of the input data. 2062 * @throws IOException if the codec cannot be created. 2063 * @throws IllegalArgumentException if type is not a valid mime type. 2064 * @throws NullPointerException if type is null. 2065 */ 2066 @NonNull createDecoderByType(@onNull String type)2067 public static MediaCodec createDecoderByType(@NonNull String type) 2068 throws IOException { 2069 return new MediaCodec(type, true /* nameIsType */, false /* encoder */); 2070 } 2071 2072 /** 2073 * Instantiate the preferred encoder supporting output data of the given mime type. 2074 * 2075 * <strong>Note:</strong> It is preferred to use {@link MediaCodecList#findEncoderForFormat} 2076 * and {@link #createByCodecName} to ensure that the resulting codec can handle a 2077 * given format. 2078 * 2079 * @param type The desired mime type of the output data. 2080 * @throws IOException if the codec cannot be created. 2081 * @throws IllegalArgumentException if type is not a valid mime type. 2082 * @throws NullPointerException if type is null. 2083 */ 2084 @NonNull createEncoderByType(@onNull String type)2085 public static MediaCodec createEncoderByType(@NonNull String type) 2086 throws IOException { 2087 return new MediaCodec(type, true /* nameIsType */, true /* encoder */); 2088 } 2089 2090 /** 2091 * If you know the exact name of the component you want to instantiate 2092 * use this method to instantiate it. Use with caution. 2093 * Likely to be used with information obtained from {@link android.media.MediaCodecList} 2094 * @param name The name of the codec to be instantiated. 2095 * @throws IOException if the codec cannot be created. 2096 * @throws IllegalArgumentException if name is not valid. 2097 * @throws NullPointerException if name is null. 2098 */ 2099 @NonNull createByCodecName(@onNull String name)2100 public static MediaCodec createByCodecName(@NonNull String name) 2101 throws IOException { 2102 return new MediaCodec(name, false /* nameIsType */, false /* encoder */); 2103 } 2104 2105 /** 2106 * This is the same as createByCodecName, but allows for instantiating a codec on behalf of a 2107 * client process. This is used for system apps or system services that create MediaCodecs on 2108 * behalf of other processes and will reclaim resources as necessary from processes with lower 2109 * priority than the client process, rather than processes with lower priority than the system 2110 * app or system service. Likely to be used with information obtained from 2111 * {@link android.media.MediaCodecList}. 2112 * @param name 2113 * @param clientPid 2114 * @param clientUid 2115 * @throws IOException if the codec cannot be created. 2116 * @throws IllegalArgumentException if name is not valid. 2117 * @throws NullPointerException if name is null. 2118 * @throws SecurityException if the MEDIA_RESOURCE_OVERRIDE_PID permission is not granted. 2119 * 2120 * @hide 2121 */ 2122 @NonNull 2123 @SystemApi 2124 @RequiresPermission(Manifest.permission.MEDIA_RESOURCE_OVERRIDE_PID) createByCodecNameForClient(@onNull String name, int clientPid, int clientUid)2125 public static MediaCodec createByCodecNameForClient(@NonNull String name, int clientPid, 2126 int clientUid) throws IOException { 2127 return new MediaCodec(name, false /* nameIsType */, false /* encoder */, clientPid, 2128 clientUid); 2129 } 2130 MediaCodec(@onNull String name, boolean nameIsType, boolean encoder)2131 private MediaCodec(@NonNull String name, boolean nameIsType, boolean encoder) { 2132 this(name, nameIsType, encoder, -1 /* pid */, -1 /* uid */); 2133 } 2134 MediaCodec(@onNull String name, boolean nameIsType, boolean encoder, int pid, int uid)2135 private MediaCodec(@NonNull String name, boolean nameIsType, boolean encoder, int pid, 2136 int uid) { 2137 Looper looper; 2138 if ((looper = Looper.myLooper()) != null) { 2139 mEventHandler = new EventHandler(this, looper); 2140 } else if ((looper = Looper.getMainLooper()) != null) { 2141 mEventHandler = new EventHandler(this, looper); 2142 } else { 2143 mEventHandler = null; 2144 } 2145 mCallbackHandler = mEventHandler; 2146 mOnFirstTunnelFrameReadyHandler = mEventHandler; 2147 mOnFrameRenderedHandler = mEventHandler; 2148 2149 mBufferLock = new Object(); 2150 2151 // save name used at creation 2152 mNameAtCreation = nameIsType ? null : name; 2153 2154 native_setup(name, nameIsType, encoder, pid, uid); 2155 } 2156 2157 private String mNameAtCreation; 2158 2159 @Override finalize()2160 protected void finalize() { 2161 native_finalize(); 2162 mCrypto = null; 2163 } 2164 2165 /** 2166 * Returns the codec to its initial (Uninitialized) state. 2167 * 2168 * Call this if an {@link MediaCodec.CodecException#isRecoverable unrecoverable} 2169 * error has occured to reset the codec to its initial state after creation. 2170 * 2171 * @throws CodecException if an unrecoverable error has occured and the codec 2172 * could not be reset. 2173 * @throws IllegalStateException if in the Released state. 2174 */ reset()2175 public final void reset() { 2176 freeAllTrackedBuffers(); // free buffers first 2177 native_reset(); 2178 mCrypto = null; 2179 } 2180 native_reset()2181 private native final void native_reset(); 2182 2183 /** 2184 * Free up resources used by the codec instance. 2185 * 2186 * Make sure you call this when you're done to free up any opened 2187 * component instance instead of relying on the garbage collector 2188 * to do this for you at some point in the future. 2189 */ release()2190 public final void release() { 2191 freeAllTrackedBuffers(); // free buffers first 2192 native_release(); 2193 mCrypto = null; 2194 } 2195 native_release()2196 private native final void native_release(); 2197 2198 /** 2199 * If this codec is to be used as an encoder, pass this flag. 2200 */ 2201 public static final int CONFIGURE_FLAG_ENCODE = 1; 2202 2203 /** 2204 * If this codec is to be used with {@link LinearBlock} and/or {@link 2205 * HardwareBuffer}, pass this flag. 2206 * <p> 2207 * When this flag is set, the following APIs throw {@link IncompatibleWithBlockModelException}. 2208 * <ul> 2209 * <li>{@link #getInputBuffer} 2210 * <li>{@link #getInputImage} 2211 * <li>{@link #getInputBuffers} 2212 * <li>{@link #getOutputBuffer} 2213 * <li>{@link #getOutputImage} 2214 * <li>{@link #getOutputBuffers} 2215 * <li>{@link #queueInputBuffer} 2216 * <li>{@link #queueSecureInputBuffer} 2217 * <li>{@link #dequeueInputBuffer} 2218 * <li>{@link #dequeueOutputBuffer} 2219 * </ul> 2220 */ 2221 public static final int CONFIGURE_FLAG_USE_BLOCK_MODEL = 2; 2222 2223 /** 2224 * This flag should be used on a secure decoder only. MediaCodec configured with this 2225 * flag does decryption in a separate thread. The flag requires MediaCodec to operate 2226 * asynchronously and will throw CryptoException if any, in the onCryptoError() 2227 * callback. Applications should override the default implementation of 2228 * onCryptoError() and access the associated CryptoException. 2229 * 2230 * CryptoException thrown will contain {@link MediaCodec.CryptoInfo} 2231 * This can be accessed using getCryptoInfo() 2232 */ 2233 public static final int CONFIGURE_FLAG_USE_CRYPTO_ASYNC = 4; 2234 2235 /** 2236 * Configure the codec with a detached output surface. 2237 * <p> 2238 * This flag is only defined for a video decoder. MediaCodec 2239 * configured with this flag will be in Surface mode even though 2240 * the surface parameter is null. 2241 * 2242 * @see detachOutputSurface 2243 */ 2244 @FlaggedApi(FLAG_NULL_OUTPUT_SURFACE) 2245 public static final int CONFIGURE_FLAG_DETACHED_SURFACE = 8; 2246 2247 /** @hide */ 2248 @IntDef( 2249 flag = true, 2250 value = { 2251 CONFIGURE_FLAG_ENCODE, 2252 CONFIGURE_FLAG_USE_BLOCK_MODEL, 2253 CONFIGURE_FLAG_USE_CRYPTO_ASYNC, 2254 }) 2255 @Retention(RetentionPolicy.SOURCE) 2256 public @interface ConfigureFlag {} 2257 2258 /** 2259 * Thrown when the codec is configured for block model and an incompatible API is called. 2260 */ 2261 public class IncompatibleWithBlockModelException extends RuntimeException { IncompatibleWithBlockModelException()2262 IncompatibleWithBlockModelException() { } 2263 IncompatibleWithBlockModelException(String message)2264 IncompatibleWithBlockModelException(String message) { 2265 super(message); 2266 } 2267 IncompatibleWithBlockModelException(String message, Throwable cause)2268 IncompatibleWithBlockModelException(String message, Throwable cause) { 2269 super(message, cause); 2270 } 2271 IncompatibleWithBlockModelException(Throwable cause)2272 IncompatibleWithBlockModelException(Throwable cause) { 2273 super(cause); 2274 } 2275 } 2276 2277 /** 2278 * Thrown when a buffer is marked with an invalid combination of flags 2279 * (e.g. both {@link #BUFFER_FLAG_END_OF_STREAM} and {@link #BUFFER_FLAG_DECODE_ONLY}) 2280 */ 2281 public class InvalidBufferFlagsException extends RuntimeException { InvalidBufferFlagsException(String message)2282 InvalidBufferFlagsException(String message) { 2283 super(message); 2284 } 2285 } 2286 2287 /** 2288 * Configures a component. 2289 * 2290 * @param format The format of the input data (decoder) or the desired 2291 * format of the output data (encoder). Passing {@code null} 2292 * as {@code format} is equivalent to passing an 2293 * {@link MediaFormat#MediaFormat an empty mediaformat}. 2294 * @param surface Specify a surface on which to render the output of this 2295 * decoder. Pass {@code null} as {@code surface} if the 2296 * codec does not generate raw video output (e.g. not a video 2297 * decoder) and/or if you want to configure the codec for 2298 * {@link ByteBuffer} output. 2299 * @param crypto Specify a crypto object to facilitate secure decryption 2300 * of the media data. Pass {@code null} as {@code crypto} for 2301 * non-secure codecs. 2302 * Please note that {@link MediaCodec} does NOT take ownership 2303 * of the {@link MediaCrypto} object; it is the application's 2304 * responsibility to properly cleanup the {@link MediaCrypto} object 2305 * when not in use. 2306 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 2307 * component as an encoder. 2308 * @throws IllegalArgumentException if the surface has been released (or is invalid), 2309 * or the format is unacceptable (e.g. missing a mandatory key), 2310 * or the flags are not set properly 2311 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 2312 * @throws IllegalStateException if not in the Uninitialized state. 2313 * @throws CryptoException upon DRM error. 2314 * @throws CodecException upon codec error. 2315 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @ConfigureFlag int flags)2316 public void configure( 2317 @Nullable MediaFormat format, 2318 @Nullable Surface surface, @Nullable MediaCrypto crypto, 2319 @ConfigureFlag int flags) { 2320 configure(format, surface, crypto, null, flags); 2321 } 2322 2323 /** 2324 * Configure a component to be used with a descrambler. 2325 * @param format The format of the input data (decoder) or the desired 2326 * format of the output data (encoder). Passing {@code null} 2327 * as {@code format} is equivalent to passing an 2328 * {@link MediaFormat#MediaFormat an empty mediaformat}. 2329 * @param surface Specify a surface on which to render the output of this 2330 * decoder. Pass {@code null} as {@code surface} if the 2331 * codec does not generate raw video output (e.g. not a video 2332 * decoder) and/or if you want to configure the codec for 2333 * {@link ByteBuffer} output. 2334 * @param flags Specify {@link #CONFIGURE_FLAG_ENCODE} to configure the 2335 * component as an encoder. 2336 * @param descrambler Specify a descrambler object to facilitate secure 2337 * descrambling of the media data, or null for non-secure codecs. 2338 * @throws IllegalArgumentException if the surface has been released (or is invalid), 2339 * or the format is unacceptable (e.g. missing a mandatory key), 2340 * or the flags are not set properly 2341 * (e.g. missing {@link #CONFIGURE_FLAG_ENCODE} for an encoder). 2342 * @throws IllegalStateException if not in the Uninitialized state. 2343 * @throws CryptoException upon DRM error. 2344 * @throws CodecException upon codec error. 2345 */ configure( @ullable MediaFormat format, @Nullable Surface surface, @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler)2346 public void configure( 2347 @Nullable MediaFormat format, @Nullable Surface surface, 2348 @ConfigureFlag int flags, @Nullable MediaDescrambler descrambler) { 2349 configure(format, surface, null, 2350 descrambler != null ? descrambler.getBinder() : null, flags); 2351 } 2352 2353 private static final int BUFFER_MODE_INVALID = -1; 2354 private static final int BUFFER_MODE_LEGACY = 0; 2355 private static final int BUFFER_MODE_BLOCK = 1; 2356 private int mBufferMode = BUFFER_MODE_INVALID; 2357 configure( @ullable MediaFormat format, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags)2358 private void configure( 2359 @Nullable MediaFormat format, @Nullable Surface surface, 2360 @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, 2361 @ConfigureFlag int flags) { 2362 if (crypto != null && descramblerBinder != null) { 2363 throw new IllegalArgumentException("Can't use crypto and descrambler together!"); 2364 } 2365 2366 // at the moment no codecs support detachable surface 2367 boolean canDetach = GetFlag(() -> android.media.codec.Flags.nullOutputSurfaceSupport()); 2368 if (GetFlag(() -> android.media.codec.Flags.nullOutputSurface())) { 2369 // Detached surface flag is only meaningful if surface is null. Otherwise, it is 2370 // ignored. 2371 if (surface == null && (flags & CONFIGURE_FLAG_DETACHED_SURFACE) != 0 && !canDetach) { 2372 throw new IllegalArgumentException("Codec does not support detached surface"); 2373 } 2374 } else { 2375 // don't allow detaching if API is disabled 2376 canDetach = false; 2377 } 2378 2379 String[] keys = null; 2380 Object[] values = null; 2381 2382 if (format != null) { 2383 Map<String, Object> formatMap = format.getMap(); 2384 keys = new String[formatMap.size()]; 2385 values = new Object[formatMap.size()]; 2386 2387 int i = 0; 2388 for (Map.Entry<String, Object> entry: formatMap.entrySet()) { 2389 if (entry.getKey().equals(MediaFormat.KEY_AUDIO_SESSION_ID)) { 2390 int sessionId = 0; 2391 try { 2392 sessionId = (Integer)entry.getValue(); 2393 } 2394 catch (Exception e) { 2395 throw new IllegalArgumentException("Wrong Session ID Parameter!"); 2396 } 2397 keys[i] = "audio-hw-sync"; 2398 values[i] = AudioSystem.getAudioHwSyncForSession(sessionId); 2399 } else { 2400 keys[i] = entry.getKey(); 2401 values[i] = entry.getValue(); 2402 } 2403 ++i; 2404 } 2405 } 2406 2407 mHasSurface = surface != null; 2408 mCrypto = crypto; 2409 synchronized (mBufferLock) { 2410 if ((flags & CONFIGURE_FLAG_USE_BLOCK_MODEL) != 0) { 2411 mBufferMode = BUFFER_MODE_BLOCK; 2412 } else { 2413 mBufferMode = BUFFER_MODE_LEGACY; 2414 } 2415 } 2416 2417 native_configure(keys, values, surface, crypto, descramblerBinder, flags); 2418 2419 if (canDetach) { 2420 // If we were able to configure native codec with a detached surface 2421 // we now know that we have a surface. 2422 if (surface == null && (flags & CONFIGURE_FLAG_DETACHED_SURFACE) != 0) { 2423 mHasSurface = true; 2424 } 2425 } 2426 } 2427 2428 /** 2429 * Dynamically sets the output surface of a codec. 2430 * <p> 2431 * This can only be used if the codec was configured with an output surface. The 2432 * new output surface should have a compatible usage type to the original output surface. 2433 * E.g. codecs may not support switching from a SurfaceTexture (GPU readable) output 2434 * to ImageReader (software readable) output. 2435 * @param surface the output surface to use. It must not be {@code null}. 2436 * @throws IllegalStateException if the codec does not support setting the output 2437 * surface in the current state. 2438 * @throws IllegalArgumentException if the new surface is not of a suitable type for the codec. 2439 */ setOutputSurface(@onNull Surface surface)2440 public void setOutputSurface(@NonNull Surface surface) { 2441 if (!mHasSurface) { 2442 throw new IllegalStateException("codec was not configured for an output surface"); 2443 } 2444 native_setSurface(surface); 2445 } 2446 native_setSurface(@onNull Surface surface)2447 private native void native_setSurface(@NonNull Surface surface); 2448 2449 /** 2450 * Detach the current output surface of a codec. 2451 * <p> 2452 * Detaches the currently associated output Surface from the 2453 * MediaCodec decoder. This allows the SurfaceView or other 2454 * component holding the Surface to be safely destroyed or 2455 * modified without affecting the decoder's operation. After 2456 * calling this method (and after it returns), the decoder will 2457 * enter detached-Surface mode and will no longer render 2458 * output. 2459 * 2460 * @throws IllegalStateException if the codec was not 2461 * configured in surface mode or if the codec does not support 2462 * detaching the output surface. 2463 * @see CONFIGURE_FLAG_DETACHED_SURFACE 2464 */ 2465 @FlaggedApi(FLAG_NULL_OUTPUT_SURFACE) detachOutputSurface()2466 public void detachOutputSurface() { 2467 if (!mHasSurface) { 2468 throw new IllegalStateException("codec was not configured for an output surface"); 2469 } 2470 2471 // note: we still have a surface in detached mode, so keep mHasSurface 2472 // we also technically allow calling detachOutputSurface multiple times in a row 2473 2474 if (GetFlag(() -> android.media.codec.Flags.nullOutputSurfaceSupport())) { 2475 native_detachOutputSurface(); 2476 } else { 2477 throw new IllegalStateException("codec does not support detaching output surface"); 2478 } 2479 } 2480 native_detachOutputSurface()2481 private native void native_detachOutputSurface(); 2482 2483 /** 2484 * Create a persistent input surface that can be used with codecs that normally have an input 2485 * surface, such as video encoders. A persistent input can be reused by subsequent 2486 * {@link MediaCodec} or {@link MediaRecorder} instances, but can only be used by at 2487 * most one codec or recorder instance concurrently. 2488 * <p> 2489 * The application is responsible for calling release() on the Surface when done. 2490 * 2491 * @return an input surface that can be used with {@link #setInputSurface}. 2492 */ 2493 @NonNull createPersistentInputSurface()2494 public static Surface createPersistentInputSurface() { 2495 return native_createPersistentInputSurface(); 2496 } 2497 2498 static class PersistentSurface extends Surface { 2499 @SuppressWarnings("unused") PersistentSurface()2500 PersistentSurface() {} // used by native 2501 2502 @Override release()2503 public void release() { 2504 native_releasePersistentInputSurface(this); 2505 super.release(); 2506 } 2507 2508 private long mPersistentObject; 2509 }; 2510 2511 /** 2512 * Configures the codec (e.g. encoder) to use a persistent input surface in place of input 2513 * buffers. This may only be called after {@link #configure} and before {@link #start}, in 2514 * lieu of {@link #createInputSurface}. 2515 * @param surface a persistent input surface created by {@link #createPersistentInputSurface} 2516 * @throws IllegalStateException if not in the Configured state or does not require an input 2517 * surface. 2518 * @throws IllegalArgumentException if the surface was not created by 2519 * {@link #createPersistentInputSurface}. 2520 */ setInputSurface(@onNull Surface surface)2521 public void setInputSurface(@NonNull Surface surface) { 2522 if (!(surface instanceof PersistentSurface)) { 2523 throw new IllegalArgumentException("not a PersistentSurface"); 2524 } 2525 native_setInputSurface(surface); 2526 } 2527 2528 @NonNull native_createPersistentInputSurface()2529 private static native final PersistentSurface native_createPersistentInputSurface(); native_releasePersistentInputSurface(@onNull Surface surface)2530 private static native final void native_releasePersistentInputSurface(@NonNull Surface surface); native_setInputSurface(@onNull Surface surface)2531 private native final void native_setInputSurface(@NonNull Surface surface); 2532 native_setCallback(@ullable Callback cb)2533 private native final void native_setCallback(@Nullable Callback cb); 2534 native_configure( @ullable String[] keys, @Nullable Object[] values, @Nullable Surface surface, @Nullable MediaCrypto crypto, @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags)2535 private native final void native_configure( 2536 @Nullable String[] keys, @Nullable Object[] values, 2537 @Nullable Surface surface, @Nullable MediaCrypto crypto, 2538 @Nullable IHwBinder descramblerBinder, @ConfigureFlag int flags); 2539 2540 /** 2541 * Requests a Surface to use as the input to an encoder, in place of input buffers. This 2542 * may only be called after {@link #configure} and before {@link #start}. 2543 * <p> 2544 * The application is responsible for calling release() on the Surface when 2545 * done. 2546 * <p> 2547 * The Surface must be rendered with a hardware-accelerated API, such as OpenGL ES. 2548 * {@link android.view.Surface#lockCanvas(android.graphics.Rect)} may fail or produce 2549 * unexpected results. 2550 * @throws IllegalStateException if not in the Configured state. 2551 */ 2552 @NonNull createInputSurface()2553 public native final Surface createInputSurface(); 2554 2555 /** 2556 * After successfully configuring the component, call {@code start}. 2557 * <p> 2558 * Call {@code start} also if the codec is configured in asynchronous mode, 2559 * and it has just been flushed, to resume requesting input buffers. 2560 * @throws IllegalStateException if not in the Configured state 2561 * or just after {@link #flush} for a codec that is configured 2562 * in asynchronous mode. 2563 * @throws MediaCodec.CodecException upon codec error. Note that some codec errors 2564 * for start may be attributed to future method calls. 2565 */ start()2566 public final void start() { 2567 native_start(); 2568 } native_start()2569 private native final void native_start(); 2570 2571 /** 2572 * Finish the decode/encode session, note that the codec instance 2573 * remains active and ready to be {@link #start}ed again. 2574 * To ensure that it is available to other client call {@link #release} 2575 * and don't just rely on garbage collection to eventually do this for you. 2576 * @throws IllegalStateException if in the Released state. 2577 */ stop()2578 public final void stop() { 2579 native_stop(); 2580 freeAllTrackedBuffers(); 2581 2582 synchronized (mListenerLock) { 2583 if (mCallbackHandler != null) { 2584 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 2585 mCallbackHandler.removeMessages(EVENT_CALLBACK); 2586 } 2587 if (mOnFirstTunnelFrameReadyHandler != null) { 2588 mOnFirstTunnelFrameReadyHandler.removeMessages(EVENT_FIRST_TUNNEL_FRAME_READY); 2589 } 2590 if (mOnFrameRenderedHandler != null) { 2591 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 2592 } 2593 } 2594 } 2595 native_stop()2596 private native final void native_stop(); 2597 2598 /** 2599 * Flush both input and output ports of the component. 2600 * <p> 2601 * Upon return, all indices previously returned in calls to {@link #dequeueInputBuffer 2602 * dequeueInputBuffer} and {@link #dequeueOutputBuffer dequeueOutputBuffer} — or obtained 2603 * via {@link Callback#onInputBufferAvailable onInputBufferAvailable} or 2604 * {@link Callback#onOutputBufferAvailable onOutputBufferAvailable} callbacks — become 2605 * invalid, and all buffers are owned by the codec. 2606 * <p> 2607 * If the codec is configured in asynchronous mode, call {@link #start} 2608 * after {@code flush} has returned to resume codec operations. The codec 2609 * will not request input buffers until this has happened. 2610 * <strong>Note, however, that there may still be outstanding {@code onOutputBufferAvailable} 2611 * callbacks that were not handled prior to calling {@code flush}. 2612 * The indices returned via these callbacks also become invalid upon calling {@code flush} and 2613 * should be discarded.</strong> 2614 * <p> 2615 * If the codec is configured in synchronous mode, codec will resume 2616 * automatically if it is configured with an input surface. Otherwise, it 2617 * will resume when {@link #dequeueInputBuffer dequeueInputBuffer} is called. 2618 * 2619 * @throws IllegalStateException if not in the Executing state. 2620 * @throws MediaCodec.CodecException upon codec error. 2621 */ flush()2622 public final void flush() { 2623 synchronized(mBufferLock) { 2624 invalidateByteBuffersLocked(mCachedInputBuffers); 2625 invalidateByteBuffersLocked(mCachedOutputBuffers); 2626 mValidInputIndices.clear(); 2627 mValidOutputIndices.clear(); 2628 mDequeuedInputBuffers.clear(); 2629 mDequeuedOutputBuffers.clear(); 2630 } 2631 native_flush(); 2632 } 2633 native_flush()2634 private native final void native_flush(); 2635 2636 /** 2637 * Thrown when an internal codec error occurs. 2638 */ 2639 public final static class CodecException extends IllegalStateException { 2640 @UnsupportedAppUsage CodecException(int errorCode, int actionCode, @Nullable String detailMessage)2641 CodecException(int errorCode, int actionCode, @Nullable String detailMessage) { 2642 super(detailMessage); 2643 mErrorCode = errorCode; 2644 mActionCode = actionCode; 2645 2646 // TODO get this from codec 2647 final String sign = errorCode < 0 ? "neg_" : ""; 2648 mDiagnosticInfo = 2649 "android.media.MediaCodec.error_" + sign + Math.abs(errorCode); 2650 } 2651 2652 /** 2653 * Returns true if the codec exception is a transient issue, 2654 * perhaps due to resource constraints, and that the method 2655 * (or encoding/decoding) may be retried at a later time. 2656 */ 2657 public boolean isTransient() { 2658 return mActionCode == ACTION_TRANSIENT; 2659 } 2660 2661 /** 2662 * Returns true if the codec cannot proceed further, 2663 * but can be recovered by stopping, configuring, 2664 * and starting again. 2665 */ 2666 public boolean isRecoverable() { 2667 return mActionCode == ACTION_RECOVERABLE; 2668 } 2669 2670 /** 2671 * Retrieve the error code associated with a CodecException 2672 */ 2673 public int getErrorCode() { 2674 return mErrorCode; 2675 } 2676 2677 /** 2678 * Retrieve a developer-readable diagnostic information string 2679 * associated with the exception. Do not show this to end-users, 2680 * since this string will not be localized or generally 2681 * comprehensible to end-users. 2682 */ 2683 public @NonNull String getDiagnosticInfo() { 2684 return mDiagnosticInfo; 2685 } 2686 2687 /** 2688 * This indicates required resource was not able to be allocated. 2689 */ 2690 public static final int ERROR_INSUFFICIENT_RESOURCE = 1100; 2691 2692 /** 2693 * This indicates the resource manager reclaimed the media resource used by the codec. 2694 * <p> 2695 * With this exception, the codec must be released, as it has moved to terminal state. 2696 */ 2697 public static final int ERROR_RECLAIMED = 1101; 2698 2699 /** @hide */ 2700 @IntDef({ 2701 ERROR_INSUFFICIENT_RESOURCE, 2702 ERROR_RECLAIMED, 2703 }) 2704 @Retention(RetentionPolicy.SOURCE) 2705 public @interface ReasonCode {} 2706 2707 /* Must be in sync with android_media_MediaCodec.cpp */ 2708 private final static int ACTION_TRANSIENT = 1; 2709 private final static int ACTION_RECOVERABLE = 2; 2710 2711 private final String mDiagnosticInfo; 2712 private final int mErrorCode; 2713 private final int mActionCode; 2714 } 2715 2716 /** 2717 * Thrown when a crypto error occurs while queueing a secure input buffer. 2718 */ 2719 public final static class CryptoException extends RuntimeException 2720 implements MediaDrmThrowable { 2721 public CryptoException(int errorCode, @Nullable String detailMessage) { 2722 this(detailMessage, errorCode, 0, 0, 0, null); 2723 } 2724 2725 /** 2726 * @hide 2727 */ 2728 public CryptoException(String message, int errorCode, int vendorError, int oemError, 2729 int errorContext, @Nullable CryptoInfo cryptoInfo) { 2730 super(message); 2731 mErrorCode = errorCode; 2732 mVendorError = vendorError; 2733 mOemError = oemError; 2734 mErrorContext = errorContext; 2735 mCryptoInfo = cryptoInfo; 2736 } 2737 2738 /** 2739 * This indicates that the requested key was not found when trying to 2740 * perform a decrypt operation. The operation can be retried after adding 2741 * the correct decryption key. 2742 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_NO_KEY}. 2743 */ 2744 public static final int ERROR_NO_KEY = MediaDrm.ErrorCodes.ERROR_NO_KEY; 2745 2746 /** 2747 * This indicates that the key used for decryption is no longer 2748 * valid due to license term expiration. The operation can be retried 2749 * after updating the expired keys. 2750 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_KEY_EXPIRED}. 2751 */ 2752 public static final int ERROR_KEY_EXPIRED = MediaDrm.ErrorCodes.ERROR_KEY_EXPIRED; 2753 2754 /** 2755 * This indicates that a required crypto resource was not able to be 2756 * allocated while attempting the requested operation. The operation 2757 * can be retried if the app is able to release resources. 2758 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_RESOURCE_BUSY} 2759 */ 2760 public static final int ERROR_RESOURCE_BUSY = MediaDrm.ErrorCodes.ERROR_RESOURCE_BUSY; 2761 2762 /** 2763 * This indicates that the output protection levels supported by the 2764 * device are not sufficient to meet the requirements set by the 2765 * content owner in the license policy. 2766 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_INSUFFICIENT_OUTPUT_PROTECTION} 2767 */ 2768 public static final int ERROR_INSUFFICIENT_OUTPUT_PROTECTION = 2769 MediaDrm.ErrorCodes.ERROR_INSUFFICIENT_OUTPUT_PROTECTION; 2770 2771 /** 2772 * This indicates that decryption was attempted on a session that is 2773 * not opened, which could be due to a failure to open the session, 2774 * closing the session prematurely, or the session being reclaimed 2775 * by the resource manager. 2776 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_SESSION_NOT_OPENED} 2777 */ 2778 public static final int ERROR_SESSION_NOT_OPENED = 2779 MediaDrm.ErrorCodes.ERROR_SESSION_NOT_OPENED; 2780 2781 /** 2782 * This indicates that an operation was attempted that could not be 2783 * supported by the crypto system of the device in its current 2784 * configuration. It may occur when the license policy requires 2785 * device security features that aren't supported by the device, 2786 * or due to an internal error in the crypto system that prevents 2787 * the specified security policy from being met. 2788 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_UNSUPPORTED_OPERATION} 2789 */ 2790 public static final int ERROR_UNSUPPORTED_OPERATION = 2791 MediaDrm.ErrorCodes.ERROR_UNSUPPORTED_OPERATION; 2792 2793 /** 2794 * This indicates that the security level of the device is not 2795 * sufficient to meet the requirements set by the content owner 2796 * in the license policy. 2797 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_INSUFFICIENT_SECURITY} 2798 */ 2799 public static final int ERROR_INSUFFICIENT_SECURITY = 2800 MediaDrm.ErrorCodes.ERROR_INSUFFICIENT_SECURITY; 2801 2802 /** 2803 * This indicates that the video frame being decrypted exceeds 2804 * the size of the device's protected output buffers. When 2805 * encountering this error the app should try playing content 2806 * of a lower resolution. 2807 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_FRAME_TOO_LARGE} 2808 */ 2809 public static final int ERROR_FRAME_TOO_LARGE = MediaDrm.ErrorCodes.ERROR_FRAME_TOO_LARGE; 2810 2811 /** 2812 * This error indicates that session state has been 2813 * invalidated. It can occur on devices that are not capable 2814 * of retaining crypto session state across device 2815 * suspend/resume. The session must be closed and a new 2816 * session opened to resume operation. 2817 * @deprecated Please use {@link MediaDrm.ErrorCodes#ERROR_LOST_STATE} 2818 */ 2819 public static final int ERROR_LOST_STATE = MediaDrm.ErrorCodes.ERROR_LOST_STATE; 2820 2821 /** @hide */ 2822 @IntDef({ 2823 MediaDrm.ErrorCodes.ERROR_NO_KEY, 2824 MediaDrm.ErrorCodes.ERROR_KEY_EXPIRED, 2825 MediaDrm.ErrorCodes.ERROR_RESOURCE_BUSY, 2826 MediaDrm.ErrorCodes.ERROR_INSUFFICIENT_OUTPUT_PROTECTION, 2827 MediaDrm.ErrorCodes.ERROR_SESSION_NOT_OPENED, 2828 MediaDrm.ErrorCodes.ERROR_UNSUPPORTED_OPERATION, 2829 MediaDrm.ErrorCodes.ERROR_INSUFFICIENT_SECURITY, 2830 MediaDrm.ErrorCodes.ERROR_FRAME_TOO_LARGE, 2831 MediaDrm.ErrorCodes.ERROR_LOST_STATE, 2832 MediaDrm.ErrorCodes.ERROR_GENERIC_OEM, 2833 MediaDrm.ErrorCodes.ERROR_GENERIC_PLUGIN, 2834 MediaDrm.ErrorCodes.ERROR_LICENSE_PARSE, 2835 MediaDrm.ErrorCodes.ERROR_MEDIA_FRAMEWORK, 2836 MediaDrm.ErrorCodes.ERROR_ZERO_SUBSAMPLES 2837 }) 2838 @Retention(RetentionPolicy.SOURCE) 2839 public @interface CryptoErrorCode {} 2840 2841 /** 2842 * Returns error code associated with this {@link CryptoException}. 2843 * <p> 2844 * Please refer to {@link MediaDrm.ErrorCodes} for the general error 2845 * handling strategy and details about each possible return value. 2846 * 2847 * @return an error code defined in {@link MediaDrm.ErrorCodes}. 2848 */ 2849 @CryptoErrorCode 2850 public int getErrorCode() { 2851 return mErrorCode; 2852 } 2853 2854 /** 2855 * Returns CryptoInfo associated with this {@link CryptoException} 2856 * if any 2857 * 2858 * @return CryptoInfo object if any. {@link MediaCodec.CryptoException} 2859 */ 2860 public @Nullable CryptoInfo getCryptoInfo() { 2861 return mCryptoInfo; 2862 } 2863 2864 @Override 2865 public int getVendorError() { 2866 return mVendorError; 2867 } 2868 2869 @Override 2870 public int getOemError() { 2871 return mOemError; 2872 } 2873 2874 @Override 2875 public int getErrorContext() { 2876 return mErrorContext; 2877 } 2878 2879 private final int mErrorCode, mVendorError, mOemError, mErrorContext; 2880 private CryptoInfo mCryptoInfo; 2881 } 2882 2883 /** 2884 * After filling a range of the input buffer at the specified index 2885 * submit it to the component. Once an input buffer is queued to 2886 * the codec, it MUST NOT be used until it is later retrieved by 2887 * {@link #getInputBuffer} in response to a {@link #dequeueInputBuffer} 2888 * return value or a {@link Callback#onInputBufferAvailable} 2889 * callback. 2890 * <p> 2891 * Many decoders require the actual compressed data stream to be 2892 * preceded by "codec specific data", i.e. setup data used to initialize 2893 * the codec such as PPS/SPS in the case of AVC video or code tables 2894 * in the case of vorbis audio. 2895 * The class {@link android.media.MediaExtractor} provides codec 2896 * specific data as part of 2897 * the returned track format in entries named "csd-0", "csd-1" ... 2898 * <p> 2899 * These buffers can be submitted directly after {@link #start} or 2900 * {@link #flush} by specifying the flag {@link 2901 * #BUFFER_FLAG_CODEC_CONFIG}. However, if you configure the 2902 * codec with a {@link MediaFormat} containing these keys, they 2903 * will be automatically submitted by MediaCodec directly after 2904 * start. Therefore, the use of {@link 2905 * #BUFFER_FLAG_CODEC_CONFIG} flag is discouraged and is 2906 * recommended only for advanced users. 2907 * <p> 2908 * To indicate that this is the final piece of input data (or rather that 2909 * no more input data follows unless the decoder is subsequently flushed) 2910 * specify the flag {@link #BUFFER_FLAG_END_OF_STREAM}. 2911 * <p class=note> 2912 * <strong>Note:</strong> Prior to {@link android.os.Build.VERSION_CODES#M}, 2913 * {@code presentationTimeUs} was not propagated to the frame timestamp of (rendered) 2914 * Surface output buffers, and the resulting frame timestamp was undefined. 2915 * Use {@link #releaseOutputBuffer(int, long)} to ensure a specific frame timestamp is set. 2916 * Similarly, since frame timestamps can be used by the destination surface for rendering 2917 * synchronization, <strong>care must be taken to normalize presentationTimeUs so as to not be 2918 * mistaken for a system time. (See {@linkplain #releaseOutputBuffer(int, long) 2919 * SurfaceView specifics}).</strong> 2920 * 2921 * @param index The index of a client-owned input buffer previously returned 2922 * in a call to {@link #dequeueInputBuffer}. 2923 * @param offset The byte offset into the input buffer at which the data starts. 2924 * @param size The number of bytes of valid input data. 2925 * @param presentationTimeUs The presentation timestamp in microseconds for this 2926 * buffer. This is normally the media time at which this 2927 * buffer should be presented (rendered). When using an output 2928 * surface, this will be propagated as the {@link 2929 * SurfaceTexture#getTimestamp timestamp} for the frame (after 2930 * conversion to nanoseconds). 2931 * @param flags A bitmask of flags 2932 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 2933 * While not prohibited, most codecs do not use the 2934 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 2935 * @throws IllegalStateException if not in the Executing state. 2936 * @throws MediaCodec.CodecException upon codec error. 2937 * @throws CryptoException if a crypto object has been specified in 2938 * {@link #configure} 2939 */ 2940 public final void queueInputBuffer( 2941 int index, 2942 int offset, int size, long presentationTimeUs, int flags) 2943 throws CryptoException { 2944 if ((flags & BUFFER_FLAG_DECODE_ONLY) != 0 2945 && (flags & BUFFER_FLAG_END_OF_STREAM) != 0) { 2946 throw new InvalidBufferFlagsException(EOS_AND_DECODE_ONLY_ERROR_MESSAGE); 2947 } 2948 synchronized(mBufferLock) { 2949 if (mBufferMode == BUFFER_MODE_BLOCK) { 2950 throw new IncompatibleWithBlockModelException("queueInputBuffer() " 2951 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 2952 + "Please use getQueueRequest() to queue buffers"); 2953 } 2954 invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); 2955 mDequeuedInputBuffers.remove(index); 2956 } 2957 try { 2958 native_queueInputBuffer( 2959 index, offset, size, presentationTimeUs, flags); 2960 } catch (CryptoException | IllegalStateException e) { 2961 revalidateByteBuffer(mCachedInputBuffers, index, true /* input */); 2962 throw e; 2963 } 2964 } 2965 2966 /** 2967 * Submit multiple access units to the codec along with multiple 2968 * {@link MediaCodec.BufferInfo} describing the contents of the buffer. This method 2969 * is supported only in asynchronous mode. While this method can be used for all codecs, 2970 * it is meant for buffer batching, which is only supported by codecs that advertise 2971 * FEATURE_MultipleFrames. Other codecs will not output large output buffers via 2972 * onOutputBuffersAvailable, and instead will output single-access-unit output via 2973 * onOutputBufferAvailable. 2974 * <p> 2975 * Output buffer size can be configured using the following MediaFormat keys. 2976 * {@link MediaFormat#KEY_BUFFER_BATCH_MAX_OUTPUT_SIZE} and 2977 * {@link MediaFormat#KEY_BUFFER_BATCH_THRESHOLD_OUTPUT_SIZE}. 2978 * Details for each access unit present in the buffer should be described using 2979 * {@link MediaCodec.BufferInfo}. Access units must be laid out contiguously (without any gaps) 2980 * and in order. Multiple access units in the output if present, will be available in 2981 * {@link Callback#onOutputBuffersAvailable} or {@link Callback#onOutputBufferAvailable} 2982 * in case of single-access-unit output or when output does not contain any buffers, 2983 * such as flags. 2984 * <p> 2985 * All other details for populating {@link MediaCodec.BufferInfo} is the same as described in 2986 * {@link #queueInputBuffer}. 2987 * 2988 * @param index The index of a client-owned input buffer previously returned 2989 * in a call to {@link #dequeueInputBuffer}. 2990 * @param bufferInfos ArrayDeque of {@link MediaCodec.BufferInfo} that describes the 2991 * contents in the buffer. The ArrayDeque and the BufferInfo objects provided 2992 * can be recycled by the caller for re-use. 2993 * @throws IllegalStateException if not in the Executing state or not in asynchronous mode. 2994 * @throws MediaCodec.CodecException upon codec error. 2995 * @throws IllegalArgumentException upon if bufferInfos is empty, contains null, or if the 2996 * access units are not contiguous. 2997 * @throws CryptoException if a crypto object has been specified in 2998 * {@link #configure} 2999 */ 3000 @FlaggedApi(FLAG_LARGE_AUDIO_FRAME) 3001 public final void queueInputBuffers( 3002 int index, 3003 @NonNull ArrayDeque<BufferInfo> bufferInfos) { 3004 synchronized(mBufferLock) { 3005 if (mBufferMode == BUFFER_MODE_BLOCK) { 3006 throw new IncompatibleWithBlockModelException("queueInputBuffers() " 3007 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3008 + "Please use getQueueRequest() to queue buffers"); 3009 } 3010 invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); 3011 mDequeuedInputBuffers.remove(index); 3012 } 3013 try { 3014 native_queueInputBuffers( 3015 index, bufferInfos.toArray()); 3016 } catch (CryptoException | IllegalStateException | IllegalArgumentException e) { 3017 revalidateByteBuffer(mCachedInputBuffers, index, true /* input */); 3018 throw e; 3019 } 3020 } 3021 3022 private native final void native_queueInputBuffer( 3023 int index, 3024 int offset, int size, long presentationTimeUs, int flags) 3025 throws CryptoException; 3026 3027 private native final void native_queueInputBuffers( 3028 int index, 3029 @NonNull Object[] infos) 3030 throws CryptoException, CodecException; 3031 3032 public static final int CRYPTO_MODE_UNENCRYPTED = 0; 3033 public static final int CRYPTO_MODE_AES_CTR = 1; 3034 public static final int CRYPTO_MODE_AES_CBC = 2; 3035 3036 /** 3037 * Metadata describing the structure of an encrypted input sample. 3038 * <p> 3039 * A buffer's data is considered to be partitioned into "subSamples". Each subSample starts with 3040 * a run of plain, unencrypted bytes followed by a run of encrypted bytes. Either of these runs 3041 * may be empty. If pattern encryption applies, each of the encrypted runs is encrypted only 3042 * partly, according to a repeating pattern of "encrypt" and "skip" blocks. 3043 * {@link #numBytesOfClearData} can be null to indicate that all data is encrypted, and 3044 * {@link #numBytesOfEncryptedData} can be null to indicate that all data is clear. At least one 3045 * of {@link #numBytesOfClearData} and {@link #numBytesOfEncryptedData} must be non-null. 3046 * <p> 3047 * This information encapsulates per-sample metadata as outlined in ISO/IEC FDIS 23001-7:2016 3048 * "Common encryption in ISO base media file format files". 3049 * <p> 3050 * <h3>ISO-CENC Schemes</h3> 3051 * ISO/IEC FDIS 23001-7:2016 defines four possible schemes by which media may be encrypted, 3052 * corresponding to each possible combination of an AES mode with the presence or absence of 3053 * patterned encryption. 3054 * 3055 * <table style="width: 0%"> 3056 * <thead> 3057 * <tr> 3058 * <th> </th> 3059 * <th>AES-CTR</th> 3060 * <th>AES-CBC</th> 3061 * </tr> 3062 * </thead> 3063 * <tbody> 3064 * <tr> 3065 * <th>Without Patterns</th> 3066 * <td>cenc</td> 3067 * <td>cbc1</td> 3068 * </tr><tr> 3069 * <th>With Patterns</th> 3070 * <td>cens</td> 3071 * <td>cbcs</td> 3072 * </tr> 3073 * </tbody> 3074 * </table> 3075 * 3076 * For {@code CryptoInfo}, the scheme is selected implicitly by the combination of the 3077 * {@link #mode} field and the value set with {@link #setPattern}. For the pattern, setting the 3078 * pattern to all zeroes (that is, both {@code blocksToEncrypt} and {@code blocksToSkip} are 3079 * zero) is interpreted as turning patterns off completely. A scheme that does not use patterns 3080 * will be selected, either cenc or cbc1. Setting the pattern to any nonzero value will choose 3081 * one of the pattern-supporting schemes, cens or cbcs. The default pattern if 3082 * {@link #setPattern} is never called is all zeroes. 3083 * <p> 3084 * <h4>HLS SAMPLE-AES Audio</h4> 3085 * HLS SAMPLE-AES audio is encrypted in a manner compatible with the cbcs scheme, except that it 3086 * does not use patterned encryption. However, if {@link #setPattern} is used to set the pattern 3087 * to all zeroes, this will be interpreted as selecting the cbc1 scheme. The cbc1 scheme cannot 3088 * successfully decrypt HLS SAMPLE-AES audio because of differences in how the IVs are handled. 3089 * For this reason, it is recommended that a pattern of {@code 1} encrypted block and {@code 0} 3090 * skip blocks be used with HLS SAMPLE-AES audio. This will trigger decryption to use cbcs mode 3091 * while still decrypting every block. 3092 */ 3093 public final static class CryptoInfo { 3094 /** 3095 * The number of subSamples that make up the buffer's contents. 3096 */ 3097 public int numSubSamples; 3098 /** 3099 * The number of leading unencrypted bytes in each subSample. If null, all bytes are treated 3100 * as encrypted and {@link #numBytesOfEncryptedData} must be specified. 3101 */ 3102 public int[] numBytesOfClearData; 3103 /** 3104 * The number of trailing encrypted bytes in each subSample. If null, all bytes are treated 3105 * as clear and {@link #numBytesOfClearData} must be specified. 3106 */ 3107 public int[] numBytesOfEncryptedData; 3108 /** 3109 * A 16-byte key id 3110 */ 3111 public byte[] key; 3112 /** 3113 * A 16-byte initialization vector 3114 */ 3115 public byte[] iv; 3116 /** 3117 * The type of encryption that has been applied, 3118 * see {@link #CRYPTO_MODE_UNENCRYPTED}, {@link #CRYPTO_MODE_AES_CTR} 3119 * and {@link #CRYPTO_MODE_AES_CBC} 3120 */ 3121 public int mode; 3122 3123 /** 3124 * Metadata describing an encryption pattern for the protected bytes in a subsample. An 3125 * encryption pattern consists of a repeating sequence of crypto blocks comprised of a 3126 * number of encrypted blocks followed by a number of unencrypted, or skipped, blocks. 3127 */ 3128 public final static class Pattern { 3129 /** 3130 * Number of blocks to be encrypted in the pattern. If both this and 3131 * {@link #mSkipBlocks} are zero, pattern encryption is inoperative. 3132 */ 3133 private int mEncryptBlocks; 3134 3135 /** 3136 * Number of blocks to be skipped (left clear) in the pattern. If both this and 3137 * {@link #mEncryptBlocks} are zero, pattern encryption is inoperative. 3138 */ 3139 private int mSkipBlocks; 3140 3141 /** 3142 * Construct a sample encryption pattern given the number of blocks to encrypt and skip 3143 * in the pattern. If both parameters are zero, pattern encryption is inoperative. 3144 */ 3145 public Pattern(int blocksToEncrypt, int blocksToSkip) { 3146 set(blocksToEncrypt, blocksToSkip); 3147 } 3148 3149 /** 3150 * Set the number of blocks to encrypt and skip in a sample encryption pattern. If both 3151 * parameters are zero, pattern encryption is inoperative. 3152 */ 3153 public void set(int blocksToEncrypt, int blocksToSkip) { 3154 mEncryptBlocks = blocksToEncrypt; 3155 mSkipBlocks = blocksToSkip; 3156 } 3157 3158 /** 3159 * Return the number of blocks to skip in a sample encryption pattern. 3160 */ 3161 public int getSkipBlocks() { 3162 return mSkipBlocks; 3163 } 3164 3165 /** 3166 * Return the number of blocks to encrypt in a sample encryption pattern. 3167 */ 3168 public int getEncryptBlocks() { 3169 return mEncryptBlocks; 3170 } 3171 }; 3172 3173 private static final Pattern ZERO_PATTERN = new Pattern(0, 0); 3174 3175 /** 3176 * The pattern applicable to the protected data in each subsample. 3177 */ 3178 private Pattern mPattern = ZERO_PATTERN; 3179 3180 /** 3181 * Set the subsample count, clear/encrypted sizes, key, IV and mode fields of 3182 * a {@link MediaCodec.CryptoInfo} instance. 3183 */ 3184 public void set( 3185 int newNumSubSamples, 3186 @NonNull int[] newNumBytesOfClearData, 3187 @NonNull int[] newNumBytesOfEncryptedData, 3188 @NonNull byte[] newKey, 3189 @NonNull byte[] newIV, 3190 int newMode) { 3191 numSubSamples = newNumSubSamples; 3192 numBytesOfClearData = newNumBytesOfClearData; 3193 numBytesOfEncryptedData = newNumBytesOfEncryptedData; 3194 key = newKey; 3195 iv = newIV; 3196 mode = newMode; 3197 mPattern = ZERO_PATTERN; 3198 } 3199 3200 /** 3201 * Returns the {@link Pattern encryption pattern}. 3202 */ 3203 public @NonNull Pattern getPattern() { 3204 return new Pattern(mPattern.getEncryptBlocks(), mPattern.getSkipBlocks()); 3205 } 3206 3207 /** 3208 * Set the encryption pattern on a {@link MediaCodec.CryptoInfo} instance. 3209 * See {@link Pattern}. 3210 */ 3211 public void setPattern(Pattern newPattern) { 3212 if (newPattern == null) { 3213 newPattern = ZERO_PATTERN; 3214 } 3215 setPattern(newPattern.getEncryptBlocks(), newPattern.getSkipBlocks()); 3216 } 3217 3218 // Accessed from android_media_MediaExtractor.cpp. 3219 private void setPattern(int blocksToEncrypt, int blocksToSkip) { 3220 mPattern = new Pattern(blocksToEncrypt, blocksToSkip); 3221 } 3222 3223 @Override 3224 public String toString() { 3225 StringBuilder builder = new StringBuilder(); 3226 builder.append(numSubSamples + " subsamples, key ["); 3227 String hexdigits = "0123456789abcdef"; 3228 for (int i = 0; i < key.length; i++) { 3229 builder.append(hexdigits.charAt((key[i] & 0xf0) >> 4)); 3230 builder.append(hexdigits.charAt(key[i] & 0x0f)); 3231 } 3232 builder.append("], iv ["); 3233 for (int i = 0; i < iv.length; i++) { 3234 builder.append(hexdigits.charAt((iv[i] & 0xf0) >> 4)); 3235 builder.append(hexdigits.charAt(iv[i] & 0x0f)); 3236 } 3237 builder.append("], clear "); Arrays.toString(numBytesOfClearData)3238 builder.append(Arrays.toString(numBytesOfClearData)); 3239 builder.append(", encrypted "); Arrays.toString(numBytesOfEncryptedData)3240 builder.append(Arrays.toString(numBytesOfEncryptedData)); 3241 builder.append(", pattern (encrypt: "); builder.append(mPattern.mEncryptBlocks)3242 builder.append(mPattern.mEncryptBlocks); 3243 builder.append(", skip: "); builder.append(mPattern.mSkipBlocks)3244 builder.append(mPattern.mSkipBlocks); 3245 builder.append(")"); 3246 return builder.toString(); 3247 } 3248 }; 3249 3250 /** 3251 * Similar to {@link #queueInputBuffer queueInputBuffer} but submits a buffer that is 3252 * potentially encrypted. 3253 * <strong>Check out further notes at {@link #queueInputBuffer queueInputBuffer}.</strong> 3254 * 3255 * @param index The index of a client-owned input buffer previously returned 3256 * in a call to {@link #dequeueInputBuffer}. 3257 * @param offset The byte offset into the input buffer at which the data starts. 3258 * @param info Metadata required to facilitate decryption, the object can be 3259 * reused immediately after this call returns. 3260 * @param presentationTimeUs The presentation timestamp in microseconds for this 3261 * buffer. This is normally the media time at which this 3262 * buffer should be presented (rendered). 3263 * @param flags A bitmask of flags 3264 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 3265 * While not prohibited, most codecs do not use the 3266 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 3267 * @throws IllegalStateException if not in the Executing state. 3268 * @throws MediaCodec.CodecException upon codec error. 3269 * @throws CryptoException if an error occurs while attempting to decrypt the buffer. 3270 * An error code associated with the exception helps identify the 3271 * reason for the failure. 3272 */ queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)3273 public final void queueSecureInputBuffer( 3274 int index, 3275 int offset, 3276 @NonNull CryptoInfo info, 3277 long presentationTimeUs, 3278 int flags) throws CryptoException { 3279 if ((flags & BUFFER_FLAG_DECODE_ONLY) != 0 3280 && (flags & BUFFER_FLAG_END_OF_STREAM) != 0) { 3281 throw new InvalidBufferFlagsException(EOS_AND_DECODE_ONLY_ERROR_MESSAGE); 3282 } 3283 synchronized(mBufferLock) { 3284 if (mBufferMode == BUFFER_MODE_BLOCK) { 3285 throw new IncompatibleWithBlockModelException("queueSecureInputBuffer() " 3286 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3287 + "Please use getQueueRequest() to queue buffers"); 3288 } 3289 invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); 3290 mDequeuedInputBuffers.remove(index); 3291 } 3292 try { 3293 native_queueSecureInputBuffer( 3294 index, offset, info, presentationTimeUs, flags); 3295 } catch (CryptoException | IllegalStateException e) { 3296 revalidateByteBuffer(mCachedInputBuffers, index, true /* input */); 3297 throw e; 3298 } 3299 } 3300 3301 /** 3302 * Similar to {@link #queueInputBuffers queueInputBuffers} but submits multiple access units 3303 * in a buffer that is potentially encrypted. 3304 * <strong>Check out further notes at {@link #queueInputBuffers queueInputBuffers}.</strong> 3305 * 3306 * @param index The index of a client-owned input buffer previously returned 3307 * in a call to {@link #dequeueInputBuffer}. 3308 * @param bufferInfos ArrayDeque of {@link MediaCodec.BufferInfo} that describes the 3309 * contents in the buffer. The ArrayDeque and the BufferInfo objects provided 3310 * can be recycled by the caller for re-use. 3311 * @param cryptoInfos ArrayDeque of {@link MediaCodec.CryptoInfo} objects to facilitate the 3312 * decryption of the contents. The ArrayDeque and the CryptoInfo objects 3313 * provided can be reused immediately after the call returns. These objects 3314 * should correspond to bufferInfo objects to ensure correct decryption. 3315 * @throws IllegalStateException if not in the Executing state or not in asynchronous mode. 3316 * @throws MediaCodec.CodecException upon codec error. 3317 * @throws IllegalArgumentException upon if bufferInfos is empty, contains null, or if the 3318 * access units are not contiguous. 3319 * @throws CryptoException if an error occurs while attempting to decrypt the buffer. 3320 * An error code associated with the exception helps identify the 3321 * reason for the failure. 3322 */ 3323 @FlaggedApi(FLAG_LARGE_AUDIO_FRAME) queueSecureInputBuffers( int index, @NonNull ArrayDeque<BufferInfo> bufferInfos, @NonNull ArrayDeque<CryptoInfo> cryptoInfos)3324 public final void queueSecureInputBuffers( 3325 int index, 3326 @NonNull ArrayDeque<BufferInfo> bufferInfos, 3327 @NonNull ArrayDeque<CryptoInfo> cryptoInfos) { 3328 synchronized(mBufferLock) { 3329 if (mBufferMode == BUFFER_MODE_BLOCK) { 3330 throw new IncompatibleWithBlockModelException("queueSecureInputBuffers() " 3331 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3332 + "Please use getQueueRequest() to queue buffers"); 3333 } 3334 invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); 3335 mDequeuedInputBuffers.remove(index); 3336 } 3337 try { 3338 native_queueSecureInputBuffers( 3339 index, bufferInfos.toArray(), cryptoInfos.toArray()); 3340 } catch (CryptoException | IllegalStateException | IllegalArgumentException e) { 3341 revalidateByteBuffer(mCachedInputBuffers, index, true /* input */); 3342 throw e; 3343 } 3344 } 3345 native_queueSecureInputBuffer( int index, int offset, @NonNull CryptoInfo info, long presentationTimeUs, int flags)3346 private native final void native_queueSecureInputBuffer( 3347 int index, 3348 int offset, 3349 @NonNull CryptoInfo info, 3350 long presentationTimeUs, 3351 int flags) throws CryptoException; 3352 native_queueSecureInputBuffers( int index, @NonNull Object[] bufferInfos, @NonNull Object[] cryptoInfos)3353 private native final void native_queueSecureInputBuffers( 3354 int index, 3355 @NonNull Object[] bufferInfos, 3356 @NonNull Object[] cryptoInfos) throws CryptoException, CodecException; 3357 3358 /** 3359 * Returns the index of an input buffer to be filled with valid data 3360 * or -1 if no such buffer is currently available. 3361 * This method will return immediately if timeoutUs == 0, wait indefinitely 3362 * for the availability of an input buffer if timeoutUs < 0 or wait up 3363 * to "timeoutUs" microseconds if timeoutUs > 0. 3364 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 3365 * @throws IllegalStateException if not in the Executing state, 3366 * or codec is configured in asynchronous mode. 3367 * @throws MediaCodec.CodecException upon codec error. 3368 */ dequeueInputBuffer(long timeoutUs)3369 public final int dequeueInputBuffer(long timeoutUs) { 3370 synchronized (mBufferLock) { 3371 if (mBufferMode == BUFFER_MODE_BLOCK) { 3372 throw new IncompatibleWithBlockModelException("dequeueInputBuffer() " 3373 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 3374 + "Please use MediaCodec.Callback objectes to get input buffer slots."); 3375 } 3376 } 3377 int res = native_dequeueInputBuffer(timeoutUs); 3378 if (res >= 0) { 3379 synchronized(mBufferLock) { 3380 validateInputByteBufferLocked(mCachedInputBuffers, res); 3381 } 3382 } 3383 return res; 3384 } 3385 native_dequeueInputBuffer(long timeoutUs)3386 private native final int native_dequeueInputBuffer(long timeoutUs); 3387 3388 /** 3389 * Section of memory that represents a linear block. Applications may 3390 * acquire a block via {@link LinearBlock#obtain} and queue all or part 3391 * of the block as an input buffer to a codec, or get a block allocated by 3392 * codec as an output buffer from {@link OutputFrame}. 3393 * 3394 * {@see QueueRequest#setLinearBlock} 3395 * {@see QueueRequest#setEncryptedLinearBlock} 3396 * {@see OutputFrame#getLinearBlock} 3397 */ 3398 public static final class LinearBlock { 3399 // No public constructors. LinearBlock()3400 private LinearBlock() {} 3401 3402 /** 3403 * Returns true if the buffer is mappable. 3404 * @throws IllegalStateException if invalid 3405 */ isMappable()3406 public boolean isMappable() { 3407 synchronized (mLock) { 3408 if (!mValid) { 3409 throw new IllegalStateException("The linear block is invalid"); 3410 } 3411 return mMappable; 3412 } 3413 } 3414 3415 /** 3416 * Map the memory and return the mapped region. 3417 * <p> 3418 * The returned memory region becomes inaccessible after 3419 * {@link #recycle}, or the buffer is queued to the codecs and not 3420 * returned to the client yet. 3421 * 3422 * @return mapped memory region as {@link ByteBuffer} object 3423 * @throws IllegalStateException if not mappable or invalid 3424 */ map()3425 public @NonNull ByteBuffer map() { 3426 synchronized (mLock) { 3427 if (!mValid) { 3428 throw new IllegalStateException("The linear block is invalid"); 3429 } 3430 if (!mMappable) { 3431 throw new IllegalStateException("The linear block is not mappable"); 3432 } 3433 if (mMapped == null) { 3434 mMapped = native_map(); 3435 } 3436 return mMapped; 3437 } 3438 } 3439 native_map()3440 private native ByteBuffer native_map(); 3441 3442 /** 3443 * Mark this block as ready to be recycled by the framework once it is 3444 * no longer in use. All operations to this object after 3445 * this call will cause exceptions, as well as attempt to access the 3446 * previously mapped memory region. Caller should clear all references 3447 * to this object after this call. 3448 * <p> 3449 * To avoid excessive memory consumption, it is recommended that callers 3450 * recycle buffers as soon as they no longer need the buffers 3451 * 3452 * @throws IllegalStateException if invalid 3453 */ recycle()3454 public void recycle() { 3455 synchronized (mLock) { 3456 if (!mValid) { 3457 throw new IllegalStateException("The linear block is invalid"); 3458 } 3459 if (mMapped != null) { 3460 mMapped.setAccessible(false); 3461 mMapped = null; 3462 } 3463 native_recycle(); 3464 mValid = false; 3465 mNativeContext = 0; 3466 } 3467 3468 if (!mInternal) { 3469 sPool.offer(this); 3470 } 3471 } 3472 native_recycle()3473 private native void native_recycle(); 3474 native_obtain(int capacity, String[] codecNames)3475 private native void native_obtain(int capacity, String[] codecNames); 3476 3477 @Override finalize()3478 protected void finalize() { 3479 native_recycle(); 3480 } 3481 3482 /** 3483 * Returns true if it is possible to allocate a linear block that can be 3484 * passed to all listed codecs as input buffers without copying the 3485 * content. 3486 * <p> 3487 * Note that even if this function returns true, {@link #obtain} may 3488 * still throw due to invalid arguments or allocation failure. 3489 * 3490 * @param codecNames list of codecs that the client wants to use a 3491 * linear block without copying. Null entries are 3492 * ignored. 3493 */ isCodecCopyFreeCompatible(@onNull String[] codecNames)3494 public static boolean isCodecCopyFreeCompatible(@NonNull String[] codecNames) { 3495 return native_checkCompatible(codecNames); 3496 } 3497 native_checkCompatible(@onNull String[] codecNames)3498 private static native boolean native_checkCompatible(@NonNull String[] codecNames); 3499 3500 /** 3501 * Obtain a linear block object no smaller than {@code capacity}. 3502 * If {@link #isCodecCopyFreeCompatible} with the same 3503 * {@code codecNames} returned true, the returned 3504 * {@link LinearBlock} object can be queued to the listed codecs without 3505 * copying. The returned {@link LinearBlock} object is always 3506 * read/write mappable. 3507 * 3508 * @param capacity requested capacity of the linear block in bytes 3509 * @param codecNames list of codecs that the client wants to use this 3510 * linear block without copying. Null entries are 3511 * ignored. 3512 * @return a linear block object. 3513 * @throws IllegalArgumentException if the capacity is invalid or 3514 * codecNames contains invalid name 3515 * @throws IOException if an error occurred while allocating a buffer 3516 */ obtain( int capacity, @NonNull String[] codecNames)3517 public static @Nullable LinearBlock obtain( 3518 int capacity, @NonNull String[] codecNames) { 3519 LinearBlock buffer = sPool.poll(); 3520 if (buffer == null) { 3521 buffer = new LinearBlock(); 3522 } 3523 synchronized (buffer.mLock) { 3524 buffer.native_obtain(capacity, codecNames); 3525 } 3526 return buffer; 3527 } 3528 3529 // Called from native setInternalStateLocked(long context, boolean isMappable)3530 private void setInternalStateLocked(long context, boolean isMappable) { 3531 mNativeContext = context; 3532 mMappable = isMappable; 3533 mValid = (context != 0); 3534 mInternal = true; 3535 } 3536 3537 private static final BlockingQueue<LinearBlock> sPool = 3538 new LinkedBlockingQueue<>(); 3539 3540 private final Object mLock = new Object(); 3541 private boolean mValid = false; 3542 private boolean mMappable = false; 3543 private ByteBuffer mMapped = null; 3544 private long mNativeContext = 0; 3545 private boolean mInternal = false; 3546 } 3547 3548 /** 3549 * Map a {@link HardwareBuffer} object into {@link Image}, so that the content of the buffer is 3550 * accessible. Depending on the usage and pixel format of the hardware buffer, it may not be 3551 * mappable; this method returns null in that case. 3552 * 3553 * @param hardwareBuffer {@link HardwareBuffer} to map. 3554 * @return Mapped {@link Image} object, or null if the buffer is not mappable. 3555 */ mapHardwareBuffer(@onNull HardwareBuffer hardwareBuffer)3556 public static @Nullable Image mapHardwareBuffer(@NonNull HardwareBuffer hardwareBuffer) { 3557 return native_mapHardwareBuffer(hardwareBuffer); 3558 } 3559 native_mapHardwareBuffer( @onNull HardwareBuffer hardwareBuffer)3560 private static native @Nullable Image native_mapHardwareBuffer( 3561 @NonNull HardwareBuffer hardwareBuffer); 3562 native_closeMediaImage(long context)3563 private static native void native_closeMediaImage(long context); 3564 3565 /** 3566 * Builder-like class for queue requests. Use this class to prepare a 3567 * queue request and send it. 3568 */ 3569 public final class QueueRequest { 3570 // No public constructor QueueRequest(@onNull MediaCodec codec, int index)3571 private QueueRequest(@NonNull MediaCodec codec, int index) { 3572 mCodec = codec; 3573 mIndex = index; 3574 } 3575 3576 /** 3577 * Set a linear block to this queue request. Exactly one buffer must be 3578 * set for a queue request before calling {@link #queue}. It is possible 3579 * to use the same {@link LinearBlock} object for multiple queue 3580 * requests. The behavior is undefined if the range of the buffer 3581 * overlaps for multiple requests, or the application writes into the 3582 * region being processed by the codec. 3583 * 3584 * @param block The linear block object 3585 * @param offset The byte offset into the input buffer at which the data starts. 3586 * @param size The number of bytes of valid input data. 3587 * @return this object 3588 * @throws IllegalStateException if a buffer is already set 3589 */ setLinearBlock( @onNull LinearBlock block, int offset, int size)3590 public @NonNull QueueRequest setLinearBlock( 3591 @NonNull LinearBlock block, 3592 int offset, 3593 int size) { 3594 if (!isAccessible()) { 3595 throw new IllegalStateException("The request is stale"); 3596 } 3597 if (mLinearBlock != null || mHardwareBuffer != null) { 3598 throw new IllegalStateException("Cannot set block twice"); 3599 } 3600 mLinearBlock = block; 3601 mOffset = offset; 3602 mSize = size; 3603 mCryptoInfos.clear(); 3604 return this; 3605 } 3606 3607 /** 3608 * Set a linear block that contain multiple non-encrypted access unit to this 3609 * queue request. Exactly one buffer must be set for a queue request before 3610 * calling {@link #queue}. Multiple access units if present must be laid out contiguously 3611 * and without gaps and in order. An IllegalArgumentException will be thrown 3612 * during {@link #queue} if access units are not laid out contiguously. 3613 * 3614 * @param block The linear block object 3615 * @param infos Represents {@link MediaCodec.BufferInfo} objects to mark 3616 * individual access-unit boundaries and the timestamps associated with it. 3617 * @return this object 3618 * @throws IllegalStateException if a buffer is already set 3619 */ 3620 @FlaggedApi(FLAG_LARGE_AUDIO_FRAME) setMultiFrameLinearBlock( @onNull LinearBlock block, @NonNull ArrayDeque<BufferInfo> infos)3621 public @NonNull QueueRequest setMultiFrameLinearBlock( 3622 @NonNull LinearBlock block, 3623 @NonNull ArrayDeque<BufferInfo> infos) { 3624 if (!isAccessible()) { 3625 throw new IllegalStateException("The request is stale"); 3626 } 3627 if (mLinearBlock != null || mHardwareBuffer != null) { 3628 throw new IllegalStateException("Cannot set block twice"); 3629 } 3630 mLinearBlock = block; 3631 mBufferInfos.clear(); 3632 mBufferInfos.addAll(infos); 3633 mCryptoInfos.clear(); 3634 return this; 3635 } 3636 3637 /** 3638 * Set an encrypted linear block to this queue request. Exactly one buffer must be 3639 * set for a queue request before calling {@link #queue}. It is possible 3640 * to use the same {@link LinearBlock} object for multiple queue 3641 * requests. The behavior is undefined if the range of the buffer 3642 * overlaps for multiple requests, or the application writes into the 3643 * region being processed by the codec. 3644 * 3645 * @param block The linear block object 3646 * @param offset The byte offset into the input buffer at which the data starts. 3647 * @param size The number of bytes of valid input data. 3648 * @param cryptoInfo Metadata describing the structure of the encrypted input sample. 3649 * @return this object 3650 * @throws IllegalStateException if a buffer is already set 3651 */ setEncryptedLinearBlock( @onNull LinearBlock block, int offset, int size, @NonNull MediaCodec.CryptoInfo cryptoInfo)3652 public @NonNull QueueRequest setEncryptedLinearBlock( 3653 @NonNull LinearBlock block, 3654 int offset, 3655 int size, 3656 @NonNull MediaCodec.CryptoInfo cryptoInfo) { 3657 Objects.requireNonNull(cryptoInfo); 3658 if (!isAccessible()) { 3659 throw new IllegalStateException("The request is stale"); 3660 } 3661 if (mLinearBlock != null || mHardwareBuffer != null) { 3662 throw new IllegalStateException("Cannot set block twice"); 3663 } 3664 mLinearBlock = block; 3665 mOffset = offset; 3666 mSize = size; 3667 mCryptoInfos.clear(); 3668 mCryptoInfos.add(cryptoInfo); 3669 return this; 3670 } 3671 3672 /** 3673 * Set an encrypted linear block to this queue request. Exactly one buffer must be 3674 * set for a queue request before calling {@link #queue}. The block can contain multiple 3675 * access units and if present should be laid out contiguously and without gaps. 3676 * 3677 * @param block The linear block object 3678 * @param bufferInfos ArrayDeque of {@link MediaCodec.BufferInfo} that describes the 3679 * contents in the buffer. The ArrayDeque and the BufferInfo objects 3680 * provided can be recycled by the caller for re-use. 3681 * @param cryptoInfos ArrayDeque of {@link MediaCodec.CryptoInfo} that describes the 3682 * structure of the encrypted input samples. The ArrayDeque and the 3683 * BufferInfo objects provided can be recycled by the caller for re-use. 3684 * @return this object 3685 * @throws IllegalStateException if a buffer is already set 3686 * @throws IllegalArgumentException upon if bufferInfos is empty, contains null, or if the 3687 * access units are not contiguous. 3688 */ 3689 @FlaggedApi(FLAG_LARGE_AUDIO_FRAME) setMultiFrameEncryptedLinearBlock( @onNull LinearBlock block, @NonNull ArrayDeque<MediaCodec.BufferInfo> bufferInfos, @NonNull ArrayDeque<MediaCodec.CryptoInfo> cryptoInfos)3690 public @NonNull QueueRequest setMultiFrameEncryptedLinearBlock( 3691 @NonNull LinearBlock block, 3692 @NonNull ArrayDeque<MediaCodec.BufferInfo> bufferInfos, 3693 @NonNull ArrayDeque<MediaCodec.CryptoInfo> cryptoInfos) { 3694 if (!isAccessible()) { 3695 throw new IllegalStateException("The request is stale"); 3696 } 3697 if (mLinearBlock != null || mHardwareBuffer != null) { 3698 throw new IllegalStateException("Cannot set block twice"); 3699 } 3700 mLinearBlock = block; 3701 mBufferInfos.clear(); 3702 mBufferInfos.addAll(bufferInfos); 3703 mCryptoInfos.clear(); 3704 mCryptoInfos.addAll(cryptoInfos); 3705 return this; 3706 } 3707 3708 /** 3709 * Set a hardware graphic buffer to this queue request. Exactly one buffer must 3710 * be set for a queue request before calling {@link #queue}. 3711 * <p> 3712 * Note: buffers should have format {@link HardwareBuffer#YCBCR_420_888}, 3713 * a single layer, and an appropriate usage ({@link HardwareBuffer#USAGE_CPU_READ_OFTEN} 3714 * for software codecs and {@link HardwareBuffer#USAGE_VIDEO_ENCODE} for hardware) 3715 * for codecs to recognize. Format {@link ImageFormat#PRIVATE} together with 3716 * usage {@link HardwareBuffer#USAGE_VIDEO_ENCODE} will also work for hardware codecs. 3717 * Codecs may throw exception if the buffer is not recognizable. 3718 * 3719 * @param buffer The hardware graphic buffer object 3720 * @return this object 3721 * @throws IllegalStateException if a buffer is already set 3722 */ setHardwareBuffer( @onNull HardwareBuffer buffer)3723 public @NonNull QueueRequest setHardwareBuffer( 3724 @NonNull HardwareBuffer buffer) { 3725 if (!isAccessible()) { 3726 throw new IllegalStateException("The request is stale"); 3727 } 3728 if (mLinearBlock != null || mHardwareBuffer != null) { 3729 throw new IllegalStateException("Cannot set block twice"); 3730 } 3731 mHardwareBuffer = buffer; 3732 return this; 3733 } 3734 3735 /** 3736 * Set timestamp to this queue request. 3737 * 3738 * @param presentationTimeUs The presentation timestamp in microseconds for this 3739 * buffer. This is normally the media time at which this 3740 * buffer should be presented (rendered). When using an output 3741 * surface, this will be propagated as the {@link 3742 * SurfaceTexture#getTimestamp timestamp} for the frame (after 3743 * conversion to nanoseconds). 3744 * @return this object 3745 */ setPresentationTimeUs(long presentationTimeUs)3746 public @NonNull QueueRequest setPresentationTimeUs(long presentationTimeUs) { 3747 if (!isAccessible()) { 3748 throw new IllegalStateException("The request is stale"); 3749 } 3750 mPresentationTimeUs = presentationTimeUs; 3751 return this; 3752 } 3753 3754 /** 3755 * Set flags to this queue request. 3756 * 3757 * @param flags A bitmask of flags 3758 * {@link #BUFFER_FLAG_CODEC_CONFIG} and {@link #BUFFER_FLAG_END_OF_STREAM}. 3759 * While not prohibited, most codecs do not use the 3760 * {@link #BUFFER_FLAG_KEY_FRAME} flag for input buffers. 3761 * @return this object 3762 */ setFlags(@ufferFlag int flags)3763 public @NonNull QueueRequest setFlags(@BufferFlag int flags) { 3764 if (!isAccessible()) { 3765 throw new IllegalStateException("The request is stale"); 3766 } 3767 mFlags = flags; 3768 return this; 3769 } 3770 3771 /** 3772 * Add an integer parameter. 3773 * See {@link MediaFormat} for an exhaustive list of supported keys with 3774 * values of type int, that can also be set with {@link MediaFormat#setInteger}. 3775 * 3776 * If there was {@link MediaCodec#setParameters} 3777 * call with the same key which is not processed by the codec yet, the 3778 * value set from this method will override the unprocessed value. 3779 * 3780 * @return this object 3781 */ setIntegerParameter( @onNull String key, int value)3782 public @NonNull QueueRequest setIntegerParameter( 3783 @NonNull String key, int value) { 3784 if (!isAccessible()) { 3785 throw new IllegalStateException("The request is stale"); 3786 } 3787 mTuningKeys.add(key); 3788 mTuningValues.add(Integer.valueOf(value)); 3789 return this; 3790 } 3791 3792 /** 3793 * Add a long parameter. 3794 * See {@link MediaFormat} for an exhaustive list of supported keys with 3795 * values of type long, that can also be set with {@link MediaFormat#setLong}. 3796 * 3797 * If there was {@link MediaCodec#setParameters} 3798 * call with the same key which is not processed by the codec yet, the 3799 * value set from this method will override the unprocessed value. 3800 * 3801 * @return this object 3802 */ setLongParameter( @onNull String key, long value)3803 public @NonNull QueueRequest setLongParameter( 3804 @NonNull String key, long value) { 3805 if (!isAccessible()) { 3806 throw new IllegalStateException("The request is stale"); 3807 } 3808 mTuningKeys.add(key); 3809 mTuningValues.add(Long.valueOf(value)); 3810 return this; 3811 } 3812 3813 /** 3814 * Add a float parameter. 3815 * See {@link MediaFormat} for an exhaustive list of supported keys with 3816 * values of type float, that can also be set with {@link MediaFormat#setFloat}. 3817 * 3818 * If there was {@link MediaCodec#setParameters} 3819 * call with the same key which is not processed by the codec yet, the 3820 * value set from this method will override the unprocessed value. 3821 * 3822 * @return this object 3823 */ setFloatParameter( @onNull String key, float value)3824 public @NonNull QueueRequest setFloatParameter( 3825 @NonNull String key, float value) { 3826 if (!isAccessible()) { 3827 throw new IllegalStateException("The request is stale"); 3828 } 3829 mTuningKeys.add(key); 3830 mTuningValues.add(Float.valueOf(value)); 3831 return this; 3832 } 3833 3834 /** 3835 * Add a {@link ByteBuffer} parameter. 3836 * See {@link MediaFormat} for an exhaustive list of supported keys with 3837 * values of byte buffer, that can also be set with {@link MediaFormat#setByteBuffer}. 3838 * 3839 * If there was {@link MediaCodec#setParameters} 3840 * call with the same key which is not processed by the codec yet, the 3841 * value set from this method will override the unprocessed value. 3842 * 3843 * @return this object 3844 */ setByteBufferParameter( @onNull String key, @NonNull ByteBuffer value)3845 public @NonNull QueueRequest setByteBufferParameter( 3846 @NonNull String key, @NonNull ByteBuffer value) { 3847 if (!isAccessible()) { 3848 throw new IllegalStateException("The request is stale"); 3849 } 3850 mTuningKeys.add(key); 3851 mTuningValues.add(value); 3852 return this; 3853 } 3854 3855 /** 3856 * Add a string parameter. 3857 * See {@link MediaFormat} for an exhaustive list of supported keys with 3858 * values of type string, that can also be set with {@link MediaFormat#setString}. 3859 * 3860 * If there was {@link MediaCodec#setParameters} 3861 * call with the same key which is not processed by the codec yet, the 3862 * value set from this method will override the unprocessed value. 3863 * 3864 * @return this object 3865 */ setStringParameter( @onNull String key, @NonNull String value)3866 public @NonNull QueueRequest setStringParameter( 3867 @NonNull String key, @NonNull String value) { 3868 if (!isAccessible()) { 3869 throw new IllegalStateException("The request is stale"); 3870 } 3871 mTuningKeys.add(key); 3872 mTuningValues.add(value); 3873 return this; 3874 } 3875 3876 /** 3877 * Finish building a queue request and queue the buffers with tunings. 3878 */ queue()3879 public void queue() { 3880 if (!isAccessible()) { 3881 throw new IllegalStateException("The request is stale"); 3882 } 3883 if (mLinearBlock == null && mHardwareBuffer == null) { 3884 throw new IllegalStateException("No block is set"); 3885 } 3886 setAccessible(false); 3887 if (mBufferInfos.isEmpty()) { 3888 BufferInfo info = new BufferInfo(); 3889 info.size = mSize; 3890 info.offset = mOffset; 3891 info.presentationTimeUs = mPresentationTimeUs; 3892 info.flags = mFlags; 3893 mBufferInfos.add(info); 3894 } 3895 if (mLinearBlock != null) { 3896 3897 mCodec.native_queueLinearBlock( 3898 mIndex, mLinearBlock, 3899 mCryptoInfos.isEmpty() ? null : mCryptoInfos.toArray(), 3900 mBufferInfos.toArray(), 3901 mTuningKeys, mTuningValues); 3902 } else if (mHardwareBuffer != null) { 3903 mCodec.native_queueHardwareBuffer( 3904 mIndex, mHardwareBuffer, mPresentationTimeUs, mFlags, 3905 mTuningKeys, mTuningValues); 3906 } 3907 clear(); 3908 } 3909 clear()3910 @NonNull QueueRequest clear() { 3911 mLinearBlock = null; 3912 mOffset = 0; 3913 mSize = 0; 3914 mHardwareBuffer = null; 3915 mPresentationTimeUs = 0; 3916 mFlags = 0; 3917 mBufferInfos.clear(); 3918 mCryptoInfos.clear(); 3919 mTuningKeys.clear(); 3920 mTuningValues.clear(); 3921 return this; 3922 } 3923 isAccessible()3924 boolean isAccessible() { 3925 return mAccessible; 3926 } 3927 setAccessible(boolean accessible)3928 @NonNull QueueRequest setAccessible(boolean accessible) { 3929 mAccessible = accessible; 3930 return this; 3931 } 3932 3933 private final MediaCodec mCodec; 3934 private final int mIndex; 3935 private LinearBlock mLinearBlock = null; 3936 private int mOffset = 0; 3937 private int mSize = 0; 3938 private HardwareBuffer mHardwareBuffer = null; 3939 private long mPresentationTimeUs = 0; 3940 private @BufferFlag int mFlags = 0; 3941 private final ArrayDeque<BufferInfo> mBufferInfos = new ArrayDeque<>(); 3942 private final ArrayDeque<CryptoInfo> mCryptoInfos = new ArrayDeque<>(); 3943 private final ArrayList<String> mTuningKeys = new ArrayList<>(); 3944 private final ArrayList<Object> mTuningValues = new ArrayList<>(); 3945 3946 private boolean mAccessible = false; 3947 } 3948 native_queueLinearBlock( int index, @NonNull LinearBlock block, @Nullable Object[] cryptoInfos, @NonNull Object[] bufferInfos, @NonNull ArrayList<String> keys, @NonNull ArrayList<Object> values)3949 private native void native_queueLinearBlock( 3950 int index, 3951 @NonNull LinearBlock block, 3952 @Nullable Object[] cryptoInfos, 3953 @NonNull Object[] bufferInfos, 3954 @NonNull ArrayList<String> keys, 3955 @NonNull ArrayList<Object> values); 3956 native_queueHardwareBuffer( int index, @NonNull HardwareBuffer buffer, long presentationTimeUs, int flags, @NonNull ArrayList<String> keys, @NonNull ArrayList<Object> values)3957 private native void native_queueHardwareBuffer( 3958 int index, 3959 @NonNull HardwareBuffer buffer, 3960 long presentationTimeUs, 3961 int flags, 3962 @NonNull ArrayList<String> keys, 3963 @NonNull ArrayList<Object> values); 3964 3965 private final ArrayList<QueueRequest> mQueueRequests = new ArrayList<>(); 3966 3967 /** 3968 * Return a {@link QueueRequest} object for an input slot index. 3969 * 3970 * @param index input slot index from 3971 * {@link Callback#onInputBufferAvailable} 3972 * @return queue request object 3973 * @throws IllegalStateException if not using block model 3974 * @throws IllegalArgumentException if the input slot is not available or 3975 * the index is out of range 3976 */ getQueueRequest(int index)3977 public @NonNull QueueRequest getQueueRequest(int index) { 3978 synchronized (mBufferLock) { 3979 if (mBufferMode != BUFFER_MODE_BLOCK) { 3980 throw new IllegalStateException("The codec is not configured for block model"); 3981 } 3982 if (index < 0 || index >= mQueueRequests.size()) { 3983 throw new IndexOutOfBoundsException("Expected range of index: [0," 3984 + (mQueueRequests.size() - 1) + "]; actual: " + index); 3985 } 3986 QueueRequest request = mQueueRequests.get(index); 3987 if (request == null) { 3988 throw new IllegalArgumentException("Unavailable index: " + index); 3989 } 3990 if (!request.isAccessible()) { 3991 throw new IllegalArgumentException( 3992 "The request is stale at index " + index); 3993 } 3994 return request.clear(); 3995 } 3996 } 3997 3998 /** 3999 * If a non-negative timeout had been specified in the call 4000 * to {@link #dequeueOutputBuffer}, indicates that the call timed out. 4001 */ 4002 public static final int INFO_TRY_AGAIN_LATER = -1; 4003 4004 /** 4005 * The output format has changed, subsequent data will follow the new 4006 * format. {@link #getOutputFormat()} returns the new format. Note, that 4007 * you can also use the new {@link #getOutputFormat(int)} method to 4008 * get the format for a specific output buffer. This frees you from 4009 * having to track output format changes. 4010 */ 4011 public static final int INFO_OUTPUT_FORMAT_CHANGED = -2; 4012 4013 /** 4014 * The output buffers have changed, the client must refer to the new 4015 * set of output buffers returned by {@link #getOutputBuffers} from 4016 * this point on. 4017 * 4018 * <p>Additionally, this event signals that the video scaling mode 4019 * may have been reset to the default.</p> 4020 * 4021 * @deprecated This return value can be ignored as {@link 4022 * #getOutputBuffers} has been deprecated. Client should 4023 * request a current buffer using on of the get-buffer or 4024 * get-image methods each time one has been dequeued. 4025 */ 4026 public static final int INFO_OUTPUT_BUFFERS_CHANGED = -3; 4027 4028 /** @hide */ 4029 @IntDef({ 4030 INFO_TRY_AGAIN_LATER, 4031 INFO_OUTPUT_FORMAT_CHANGED, 4032 INFO_OUTPUT_BUFFERS_CHANGED, 4033 }) 4034 @Retention(RetentionPolicy.SOURCE) 4035 public @interface OutputBufferInfo {} 4036 4037 /** 4038 * Dequeue an output buffer, block at most "timeoutUs" microseconds. 4039 * Returns the index of an output buffer that has been successfully 4040 * decoded or one of the INFO_* constants. 4041 * @param info Will be filled with buffer meta data. 4042 * @param timeoutUs The timeout in microseconds, a negative timeout indicates "infinite". 4043 * @throws IllegalStateException if not in the Executing state, 4044 * or codec is configured in asynchronous mode. 4045 * @throws MediaCodec.CodecException upon codec error. 4046 */ 4047 @OutputBufferInfo dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)4048 public final int dequeueOutputBuffer( 4049 @NonNull BufferInfo info, long timeoutUs) { 4050 synchronized (mBufferLock) { 4051 if (mBufferMode == BUFFER_MODE_BLOCK) { 4052 throw new IncompatibleWithBlockModelException("dequeueOutputBuffer() " 4053 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4054 + "Please use MediaCodec.Callback objects to get output buffer slots."); 4055 } 4056 } 4057 int res = native_dequeueOutputBuffer(info, timeoutUs); 4058 synchronized (mBufferLock) { 4059 if (res == INFO_OUTPUT_BUFFERS_CHANGED) { 4060 cacheBuffersLocked(false /* input */); 4061 } else if (res >= 0) { 4062 validateOutputByteBufferLocked(mCachedOutputBuffers, res, info); 4063 if (mHasSurface || mCachedOutputBuffers == null) { 4064 mDequeuedOutputInfos.put(res, info.dup()); 4065 } 4066 } 4067 } 4068 return res; 4069 } 4070 native_dequeueOutputBuffer( @onNull BufferInfo info, long timeoutUs)4071 private native final int native_dequeueOutputBuffer( 4072 @NonNull BufferInfo info, long timeoutUs); 4073 4074 /** 4075 * If you are done with a buffer, use this call to return the buffer to the codec 4076 * or to render it on the output surface. If you configured the codec with an 4077 * output surface, setting {@code render} to {@code true} will first send the buffer 4078 * to that output surface. The surface will release the buffer back to the codec once 4079 * it is no longer used/displayed. 4080 * 4081 * Once an output buffer is released to the codec, it MUST NOT 4082 * be used until it is later retrieved by {@link #getOutputBuffer} in response 4083 * to a {@link #dequeueOutputBuffer} return value or a 4084 * {@link Callback#onOutputBufferAvailable} callback. 4085 * 4086 * @param index The index of a client-owned output buffer previously returned 4087 * from a call to {@link #dequeueOutputBuffer}. 4088 * @param render If a valid surface was specified when configuring the codec, 4089 * passing true renders this output buffer to the surface. 4090 * @throws IllegalStateException if not in the Executing state. 4091 * @throws MediaCodec.CodecException upon codec error. 4092 */ releaseOutputBuffer(int index, boolean render)4093 public final void releaseOutputBuffer(int index, boolean render) { 4094 releaseOutputBufferInternal(index, render, false /* updatePTS */, 0 /* dummy */); 4095 } 4096 4097 /** 4098 * If you are done with a buffer, use this call to update its surface timestamp 4099 * and return it to the codec to render it on the output surface. If you 4100 * have not specified an output surface when configuring this video codec, 4101 * this call will simply return the buffer to the codec.<p> 4102 * 4103 * The timestamp may have special meaning depending on the destination surface. 4104 * 4105 * <table> 4106 * <tr><th>SurfaceView specifics</th></tr> 4107 * <tr><td> 4108 * If you render your buffer on a {@link android.view.SurfaceView}, 4109 * you can use the timestamp to render the buffer at a specific time (at the 4110 * VSYNC at or after the buffer timestamp). For this to work, the timestamp 4111 * needs to be <i>reasonably close</i> to the current {@link System#nanoTime}. 4112 * Currently, this is set as within one (1) second. A few notes: 4113 * 4114 * <ul> 4115 * <li>the buffer will not be returned to the codec until the timestamp 4116 * has passed and the buffer is no longer used by the {@link android.view.Surface}. 4117 * <li>buffers are processed sequentially, so you may block subsequent buffers to 4118 * be displayed on the {@link android.view.Surface}. This is important if you 4119 * want to react to user action, e.g. stop the video or seek. 4120 * <li>if multiple buffers are sent to the {@link android.view.Surface} to be 4121 * rendered at the same VSYNC, the last one will be shown, and the other ones 4122 * will be dropped. 4123 * <li>if the timestamp is <em>not</em> "reasonably close" to the current system 4124 * time, the {@link android.view.Surface} will ignore the timestamp, and 4125 * display the buffer at the earliest feasible time. In this mode it will not 4126 * drop frames. 4127 * <li>for best performance and quality, call this method when you are about 4128 * two VSYNCs' time before the desired render time. For 60Hz displays, this is 4129 * about 33 msec. 4130 * </ul> 4131 * </td></tr> 4132 * </table> 4133 * 4134 * Once an output buffer is released to the codec, it MUST NOT 4135 * be used until it is later retrieved by {@link #getOutputBuffer} in response 4136 * to a {@link #dequeueOutputBuffer} return value or a 4137 * {@link Callback#onOutputBufferAvailable} callback. 4138 * 4139 * @param index The index of a client-owned output buffer previously returned 4140 * from a call to {@link #dequeueOutputBuffer}. 4141 * @param renderTimestampNs The timestamp to associate with this buffer when 4142 * it is sent to the Surface. 4143 * @throws IllegalStateException if not in the Executing state. 4144 * @throws MediaCodec.CodecException upon codec error. 4145 */ releaseOutputBuffer(int index, long renderTimestampNs)4146 public final void releaseOutputBuffer(int index, long renderTimestampNs) { 4147 releaseOutputBufferInternal( 4148 index, true /* render */, true /* updatePTS */, renderTimestampNs); 4149 } 4150 releaseOutputBufferInternal( int index, boolean render, boolean updatePts, long renderTimestampNs)4151 private void releaseOutputBufferInternal( 4152 int index, boolean render, boolean updatePts, long renderTimestampNs) { 4153 BufferInfo info = null; 4154 synchronized(mBufferLock) { 4155 switch (mBufferMode) { 4156 case BUFFER_MODE_LEGACY: 4157 invalidateByteBufferLocked(mCachedOutputBuffers, index, false /* input */); 4158 mDequeuedOutputBuffers.remove(index); 4159 if (mHasSurface || mCachedOutputBuffers == null) { 4160 info = mDequeuedOutputInfos.remove(index); 4161 } 4162 break; 4163 case BUFFER_MODE_BLOCK: 4164 OutputFrame frame = mOutputFrames.get(index); 4165 frame.setAccessible(false); 4166 frame.clear(); 4167 break; 4168 default: 4169 throw new IllegalStateException( 4170 "Unrecognized buffer mode: " + mBufferMode); 4171 } 4172 } 4173 releaseOutputBuffer( 4174 index, render, updatePts, renderTimestampNs); 4175 } 4176 4177 @UnsupportedAppUsage releaseOutputBuffer( int index, boolean render, boolean updatePTS, long timeNs)4178 private native final void releaseOutputBuffer( 4179 int index, boolean render, boolean updatePTS, long timeNs); 4180 4181 /** 4182 * Signals end-of-stream on input. Equivalent to submitting an empty buffer with 4183 * {@link #BUFFER_FLAG_END_OF_STREAM} set. This may only be used with 4184 * encoders receiving input from a Surface created by {@link #createInputSurface}. 4185 * @throws IllegalStateException if not in the Executing state. 4186 * @throws MediaCodec.CodecException upon codec error. 4187 */ signalEndOfInputStream()4188 public native final void signalEndOfInputStream(); 4189 4190 /** 4191 * Call this after dequeueOutputBuffer signals a format change by returning 4192 * {@link #INFO_OUTPUT_FORMAT_CHANGED}. 4193 * You can also call this after {@link #configure} returns 4194 * successfully to get the output format initially configured 4195 * for the codec. Do this to determine what optional 4196 * configuration parameters were supported by the codec. 4197 * 4198 * @throws IllegalStateException if not in the Executing or 4199 * Configured state. 4200 * @throws MediaCodec.CodecException upon codec error. 4201 */ 4202 @NonNull getOutputFormat()4203 public final MediaFormat getOutputFormat() { 4204 return new MediaFormat(getFormatNative(false /* input */)); 4205 } 4206 4207 /** 4208 * Call this after {@link #configure} returns successfully to 4209 * get the input format accepted by the codec. Do this to 4210 * determine what optional configuration parameters were 4211 * supported by the codec. 4212 * 4213 * @throws IllegalStateException if not in the Executing or 4214 * Configured state. 4215 * @throws MediaCodec.CodecException upon codec error. 4216 */ 4217 @NonNull getInputFormat()4218 public final MediaFormat getInputFormat() { 4219 return new MediaFormat(getFormatNative(true /* input */)); 4220 } 4221 4222 /** 4223 * Returns the output format for a specific output buffer. 4224 * 4225 * @param index The index of a client-owned input buffer previously 4226 * returned from a call to {@link #dequeueInputBuffer}. 4227 * 4228 * @return the format for the output buffer, or null if the index 4229 * is not a dequeued output buffer. 4230 */ 4231 @NonNull getOutputFormat(int index)4232 public final MediaFormat getOutputFormat(int index) { 4233 return new MediaFormat(getOutputFormatNative(index)); 4234 } 4235 4236 @NonNull getFormatNative(boolean input)4237 private native final Map<String, Object> getFormatNative(boolean input); 4238 4239 @NonNull getOutputFormatNative(int index)4240 private native final Map<String, Object> getOutputFormatNative(int index); 4241 4242 // used to track dequeued buffers 4243 private static class BufferMap { 4244 // various returned representations of the codec buffer 4245 private static class CodecBuffer { 4246 private Image mImage; 4247 private ByteBuffer mByteBuffer; 4248 free()4249 public void free() { 4250 if (mByteBuffer != null) { 4251 // all of our ByteBuffers are direct 4252 java.nio.NioUtils.freeDirectBuffer(mByteBuffer); 4253 mByteBuffer = null; 4254 } 4255 if (mImage != null) { 4256 mImage.close(); 4257 mImage = null; 4258 } 4259 } 4260 setImage(@ullable Image image)4261 public void setImage(@Nullable Image image) { 4262 free(); 4263 mImage = image; 4264 } 4265 setByteBuffer(@ullable ByteBuffer buffer)4266 public void setByteBuffer(@Nullable ByteBuffer buffer) { 4267 free(); 4268 mByteBuffer = buffer; 4269 } 4270 } 4271 4272 private final Map<Integer, CodecBuffer> mMap = 4273 new HashMap<Integer, CodecBuffer>(); 4274 remove(int index)4275 public void remove(int index) { 4276 CodecBuffer buffer = mMap.get(index); 4277 if (buffer != null) { 4278 buffer.free(); 4279 mMap.remove(index); 4280 } 4281 } 4282 put(int index, @Nullable ByteBuffer newBuffer)4283 public void put(int index, @Nullable ByteBuffer newBuffer) { 4284 CodecBuffer buffer = mMap.get(index); 4285 if (buffer == null) { // likely 4286 buffer = new CodecBuffer(); 4287 mMap.put(index, buffer); 4288 } 4289 buffer.setByteBuffer(newBuffer); 4290 } 4291 put(int index, @Nullable Image newImage)4292 public void put(int index, @Nullable Image newImage) { 4293 CodecBuffer buffer = mMap.get(index); 4294 if (buffer == null) { // likely 4295 buffer = new CodecBuffer(); 4296 mMap.put(index, buffer); 4297 } 4298 buffer.setImage(newImage); 4299 } 4300 clear()4301 public void clear() { 4302 for (CodecBuffer buffer: mMap.values()) { 4303 buffer.free(); 4304 } 4305 mMap.clear(); 4306 } 4307 } 4308 4309 private ByteBuffer[] mCachedInputBuffers; 4310 private ByteBuffer[] mCachedOutputBuffers; 4311 private BitSet mValidInputIndices = new BitSet(); 4312 private BitSet mValidOutputIndices = new BitSet(); 4313 4314 private final BufferMap mDequeuedInputBuffers = new BufferMap(); 4315 private final BufferMap mDequeuedOutputBuffers = new BufferMap(); 4316 private final Map<Integer, BufferInfo> mDequeuedOutputInfos = 4317 new HashMap<Integer, BufferInfo>(); 4318 final private Object mBufferLock; 4319 invalidateByteBufferLocked( @ullable ByteBuffer[] buffers, int index, boolean input)4320 private void invalidateByteBufferLocked( 4321 @Nullable ByteBuffer[] buffers, int index, boolean input) { 4322 if (buffers == null) { 4323 if (index >= 0) { 4324 BitSet indices = input ? mValidInputIndices : mValidOutputIndices; 4325 indices.clear(index); 4326 } 4327 } else if (index >= 0 && index < buffers.length) { 4328 ByteBuffer buffer = buffers[index]; 4329 if (buffer != null) { 4330 buffer.setAccessible(false); 4331 } 4332 } 4333 } 4334 validateInputByteBufferLocked( @ullable ByteBuffer[] buffers, int index)4335 private void validateInputByteBufferLocked( 4336 @Nullable ByteBuffer[] buffers, int index) { 4337 if (buffers == null) { 4338 if (index >= 0) { 4339 mValidInputIndices.set(index); 4340 } 4341 } else if (index >= 0 && index < buffers.length) { 4342 ByteBuffer buffer = buffers[index]; 4343 if (buffer != null) { 4344 buffer.setAccessible(true); 4345 buffer.clear(); 4346 } 4347 } 4348 } 4349 revalidateByteBuffer( @ullable ByteBuffer[] buffers, int index, boolean input)4350 private void revalidateByteBuffer( 4351 @Nullable ByteBuffer[] buffers, int index, boolean input) { 4352 synchronized(mBufferLock) { 4353 if (buffers == null) { 4354 if (index >= 0) { 4355 BitSet indices = input ? mValidInputIndices : mValidOutputIndices; 4356 indices.set(index); 4357 } 4358 } else if (index >= 0 && index < buffers.length) { 4359 ByteBuffer buffer = buffers[index]; 4360 if (buffer != null) { 4361 buffer.setAccessible(true); 4362 } 4363 } 4364 } 4365 } 4366 validateOutputByteBuffersLocked( @ullable ByteBuffer[] buffers, int index, @NonNull ArrayDeque<BufferInfo> infoDeque)4367 private void validateOutputByteBuffersLocked( 4368 @Nullable ByteBuffer[] buffers, int index, @NonNull ArrayDeque<BufferInfo> infoDeque) { 4369 Optional<BufferInfo> minInfo = infoDeque.stream().min( 4370 (info1, info2) -> Integer.compare(info1.offset, info2.offset)); 4371 Optional<BufferInfo> maxInfo = infoDeque.stream().max( 4372 (info1, info2) -> Integer.compare(info1.offset, info2.offset)); 4373 if (buffers == null) { 4374 if (index >= 0) { 4375 mValidOutputIndices.set(index); 4376 } 4377 } else if (index >= 0 && index < buffers.length) { 4378 ByteBuffer buffer = buffers[index]; 4379 if (buffer != null && minInfo.isPresent() && maxInfo.isPresent()) { 4380 buffer.setAccessible(true); 4381 buffer.limit(maxInfo.get().offset + maxInfo.get().size); 4382 buffer.position(minInfo.get().offset); 4383 } 4384 } 4385 4386 } 4387 validateOutputByteBufferLocked( @ullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info)4388 private void validateOutputByteBufferLocked( 4389 @Nullable ByteBuffer[] buffers, int index, @NonNull BufferInfo info) { 4390 if (buffers == null) { 4391 if (index >= 0) { 4392 mValidOutputIndices.set(index); 4393 } 4394 } else if (index >= 0 && index < buffers.length) { 4395 ByteBuffer buffer = buffers[index]; 4396 if (buffer != null) { 4397 buffer.setAccessible(true); 4398 buffer.limit(info.offset + info.size).position(info.offset); 4399 } 4400 } 4401 } 4402 invalidateByteBuffersLocked(@ullable ByteBuffer[] buffers)4403 private void invalidateByteBuffersLocked(@Nullable ByteBuffer[] buffers) { 4404 if (buffers != null) { 4405 for (ByteBuffer buffer: buffers) { 4406 if (buffer != null) { 4407 buffer.setAccessible(false); 4408 } 4409 } 4410 } 4411 } 4412 freeByteBufferLocked(@ullable ByteBuffer buffer)4413 private void freeByteBufferLocked(@Nullable ByteBuffer buffer) { 4414 if (buffer != null /* && buffer.isDirect() */) { 4415 // all of our ByteBuffers are direct 4416 java.nio.NioUtils.freeDirectBuffer(buffer); 4417 } 4418 } 4419 freeByteBuffersLocked(@ullable ByteBuffer[] buffers)4420 private void freeByteBuffersLocked(@Nullable ByteBuffer[] buffers) { 4421 if (buffers != null) { 4422 for (ByteBuffer buffer: buffers) { 4423 freeByteBufferLocked(buffer); 4424 } 4425 } 4426 } 4427 freeAllTrackedBuffers()4428 private void freeAllTrackedBuffers() { 4429 synchronized(mBufferLock) { 4430 freeByteBuffersLocked(mCachedInputBuffers); 4431 freeByteBuffersLocked(mCachedOutputBuffers); 4432 mCachedInputBuffers = null; 4433 mCachedOutputBuffers = null; 4434 mValidInputIndices.clear(); 4435 mValidOutputIndices.clear(); 4436 mDequeuedInputBuffers.clear(); 4437 mDequeuedOutputBuffers.clear(); 4438 mQueueRequests.clear(); 4439 mOutputFrames.clear(); 4440 } 4441 } 4442 cacheBuffersLocked(boolean input)4443 private void cacheBuffersLocked(boolean input) { 4444 ByteBuffer[] buffers = null; 4445 try { 4446 buffers = getBuffers(input); 4447 invalidateByteBuffersLocked(buffers); 4448 } catch (IllegalStateException e) { 4449 // we don't get buffers in async mode 4450 } 4451 if (buffers != null) { 4452 BitSet indices = input ? mValidInputIndices : mValidOutputIndices; 4453 for (int i = 0; i < buffers.length; ++i) { 4454 ByteBuffer buffer = buffers[i]; 4455 if (buffer == null || !indices.get(i)) { 4456 continue; 4457 } 4458 buffer.setAccessible(true); 4459 if (!input) { 4460 BufferInfo info = mDequeuedOutputInfos.get(i); 4461 if (info != null) { 4462 buffer.limit(info.offset + info.size).position(info.offset); 4463 } 4464 } 4465 } 4466 indices.clear(); 4467 } 4468 if (input) { 4469 mCachedInputBuffers = buffers; 4470 } else { 4471 mCachedOutputBuffers = buffers; 4472 } 4473 } 4474 4475 /** 4476 * Retrieve the set of input buffers. Call this after start() 4477 * returns. After calling this method, any ByteBuffers 4478 * previously returned by an earlier call to this method MUST no 4479 * longer be used. 4480 * 4481 * @deprecated Use the new {@link #getInputBuffer} method instead 4482 * each time an input buffer is dequeued. 4483 * 4484 * <b>Note:</b> As of API 21, dequeued input buffers are 4485 * automatically {@link java.nio.Buffer#clear cleared}. 4486 * 4487 * <em>Do not use this method if using an input surface.</em> 4488 * 4489 * @throws IllegalStateException if not in the Executing state, 4490 * or codec is configured in asynchronous mode. 4491 * @throws MediaCodec.CodecException upon codec error. 4492 */ 4493 @NonNull getInputBuffers()4494 public ByteBuffer[] getInputBuffers() { 4495 synchronized (mBufferLock) { 4496 if (mBufferMode == BUFFER_MODE_BLOCK) { 4497 throw new IncompatibleWithBlockModelException("getInputBuffers() " 4498 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4499 + "Please obtain MediaCodec.LinearBlock or HardwareBuffer " 4500 + "objects and attach to QueueRequest objects."); 4501 } 4502 if (mCachedInputBuffers == null) { 4503 cacheBuffersLocked(true /* input */); 4504 } 4505 if (mCachedInputBuffers == null) { 4506 throw new IllegalStateException(); 4507 } 4508 // FIXME: check codec status 4509 return mCachedInputBuffers; 4510 } 4511 } 4512 4513 /** 4514 * Retrieve the set of output buffers. Call this after start() 4515 * returns and whenever dequeueOutputBuffer signals an output 4516 * buffer change by returning {@link 4517 * #INFO_OUTPUT_BUFFERS_CHANGED}. After calling this method, any 4518 * ByteBuffers previously returned by an earlier call to this 4519 * method MUST no longer be used. 4520 * 4521 * @deprecated Use the new {@link #getOutputBuffer} method instead 4522 * each time an output buffer is dequeued. This method is not 4523 * supported if codec is configured in asynchronous mode. 4524 * 4525 * <b>Note:</b> As of API 21, the position and limit of output 4526 * buffers that are dequeued will be set to the valid data 4527 * range. 4528 * 4529 * <em>Do not use this method if using an output surface.</em> 4530 * 4531 * @throws IllegalStateException if not in the Executing state, 4532 * or codec is configured in asynchronous mode. 4533 * @throws MediaCodec.CodecException upon codec error. 4534 */ 4535 @NonNull getOutputBuffers()4536 public ByteBuffer[] getOutputBuffers() { 4537 synchronized (mBufferLock) { 4538 if (mBufferMode == BUFFER_MODE_BLOCK) { 4539 throw new IncompatibleWithBlockModelException("getOutputBuffers() " 4540 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4541 + "Please use getOutputFrame to get output frames."); 4542 } 4543 if (mCachedOutputBuffers == null) { 4544 cacheBuffersLocked(false /* input */); 4545 } 4546 if (mCachedOutputBuffers == null) { 4547 throw new IllegalStateException(); 4548 } 4549 // FIXME: check codec status 4550 return mCachedOutputBuffers; 4551 } 4552 } 4553 4554 /** 4555 * Returns a {@link java.nio.Buffer#clear cleared}, writable ByteBuffer 4556 * object for a dequeued input buffer index to contain the input data. 4557 * 4558 * After calling this method any ByteBuffer or Image object 4559 * previously returned for the same input index MUST no longer 4560 * be used. 4561 * 4562 * @param index The index of a client-owned input buffer previously 4563 * returned from a call to {@link #dequeueInputBuffer}, 4564 * or received via an onInputBufferAvailable callback. 4565 * 4566 * @return the input buffer, or null if the index is not a dequeued 4567 * input buffer, or if the codec is configured for surface input. 4568 * 4569 * @throws IllegalStateException if not in the Executing state. 4570 * @throws MediaCodec.CodecException upon codec error. 4571 */ 4572 @Nullable getInputBuffer(int index)4573 public ByteBuffer getInputBuffer(int index) { 4574 synchronized (mBufferLock) { 4575 if (mBufferMode == BUFFER_MODE_BLOCK) { 4576 throw new IncompatibleWithBlockModelException("getInputBuffer() " 4577 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4578 + "Please obtain MediaCodec.LinearBlock or HardwareBuffer " 4579 + "objects and attach to QueueRequest objects."); 4580 } 4581 } 4582 ByteBuffer newBuffer = getBuffer(true /* input */, index); 4583 synchronized (mBufferLock) { 4584 invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); 4585 mDequeuedInputBuffers.put(index, newBuffer); 4586 } 4587 return newBuffer; 4588 } 4589 4590 /** 4591 * Returns a writable Image object for a dequeued input buffer 4592 * index to contain the raw input video frame. 4593 * 4594 * After calling this method any ByteBuffer or Image object 4595 * previously returned for the same input index MUST no longer 4596 * be used. 4597 * 4598 * @param index The index of a client-owned input buffer previously 4599 * returned from a call to {@link #dequeueInputBuffer}, 4600 * or received via an onInputBufferAvailable callback. 4601 * 4602 * @return the input image, or null if the index is not a 4603 * dequeued input buffer, or not a ByteBuffer that contains a 4604 * raw image. 4605 * 4606 * @throws IllegalStateException if not in the Executing state. 4607 * @throws MediaCodec.CodecException upon codec error. 4608 */ 4609 @Nullable getInputImage(int index)4610 public Image getInputImage(int index) { 4611 synchronized (mBufferLock) { 4612 if (mBufferMode == BUFFER_MODE_BLOCK) { 4613 throw new IncompatibleWithBlockModelException("getInputImage() " 4614 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4615 + "Please obtain MediaCodec.LinearBlock or HardwareBuffer " 4616 + "objects and attach to QueueRequest objects."); 4617 } 4618 } 4619 Image newImage = getImage(true /* input */, index); 4620 synchronized (mBufferLock) { 4621 invalidateByteBufferLocked(mCachedInputBuffers, index, true /* input */); 4622 mDequeuedInputBuffers.put(index, newImage); 4623 } 4624 return newImage; 4625 } 4626 4627 /** 4628 * Returns a read-only ByteBuffer for a dequeued output buffer 4629 * index. The position and limit of the returned buffer are set 4630 * to the valid output data. 4631 * 4632 * After calling this method, any ByteBuffer or Image object 4633 * previously returned for the same output index MUST no longer 4634 * be used. 4635 * 4636 * @param index The index of a client-owned output buffer previously 4637 * returned from a call to {@link #dequeueOutputBuffer}, 4638 * or received via an onOutputBufferAvailable callback. 4639 * 4640 * @return the output buffer, or null if the index is not a dequeued 4641 * output buffer, or the codec is configured with an output surface. 4642 * 4643 * @throws IllegalStateException if not in the Executing state. 4644 * @throws MediaCodec.CodecException upon codec error. 4645 */ 4646 @Nullable getOutputBuffer(int index)4647 public ByteBuffer getOutputBuffer(int index) { 4648 synchronized (mBufferLock) { 4649 if (mBufferMode == BUFFER_MODE_BLOCK) { 4650 throw new IncompatibleWithBlockModelException("getOutputBuffer() " 4651 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4652 + "Please use getOutputFrame() to get output frames."); 4653 } 4654 } 4655 ByteBuffer newBuffer = getBuffer(false /* input */, index); 4656 synchronized (mBufferLock) { 4657 invalidateByteBufferLocked(mCachedOutputBuffers, index, false /* input */); 4658 mDequeuedOutputBuffers.put(index, newBuffer); 4659 } 4660 return newBuffer; 4661 } 4662 4663 /** 4664 * Returns a read-only Image object for a dequeued output buffer 4665 * index that contains the raw video frame. 4666 * 4667 * After calling this method, any ByteBuffer or Image object previously 4668 * returned for the same output index MUST no longer be used. 4669 * 4670 * @param index The index of a client-owned output buffer previously 4671 * returned from a call to {@link #dequeueOutputBuffer}, 4672 * or received via an onOutputBufferAvailable callback. 4673 * 4674 * @return the output image, or null if the index is not a 4675 * dequeued output buffer, not a raw video frame, or if the codec 4676 * was configured with an output surface. 4677 * 4678 * @throws IllegalStateException if not in the Executing state. 4679 * @throws MediaCodec.CodecException upon codec error. 4680 */ 4681 @Nullable getOutputImage(int index)4682 public Image getOutputImage(int index) { 4683 synchronized (mBufferLock) { 4684 if (mBufferMode == BUFFER_MODE_BLOCK) { 4685 throw new IncompatibleWithBlockModelException("getOutputImage() " 4686 + "is not compatible with CONFIGURE_FLAG_USE_BLOCK_MODEL. " 4687 + "Please use getOutputFrame() to get output frames."); 4688 } 4689 } 4690 Image newImage = getImage(false /* input */, index); 4691 synchronized (mBufferLock) { 4692 invalidateByteBufferLocked(mCachedOutputBuffers, index, false /* input */); 4693 mDequeuedOutputBuffers.put(index, newImage); 4694 } 4695 return newImage; 4696 } 4697 4698 /** 4699 * A single output frame and its associated metadata. 4700 */ 4701 public static final class OutputFrame { 4702 // No public constructor OutputFrame(int index)4703 OutputFrame(int index) { 4704 mIndex = index; 4705 } 4706 4707 /** 4708 * Returns the output linear block, or null if this frame is empty. 4709 * 4710 * @throws IllegalStateException if this output frame is not linear. 4711 */ getLinearBlock()4712 public @Nullable LinearBlock getLinearBlock() { 4713 if (mHardwareBuffer != null) { 4714 throw new IllegalStateException("This output frame is not linear"); 4715 } 4716 return mLinearBlock; 4717 } 4718 4719 /** 4720 * Returns the output hardware graphic buffer, or null if this frame is empty. 4721 * 4722 * @throws IllegalStateException if this output frame is not graphic. 4723 */ getHardwareBuffer()4724 public @Nullable HardwareBuffer getHardwareBuffer() { 4725 if (mLinearBlock != null) { 4726 throw new IllegalStateException("This output frame is not graphic"); 4727 } 4728 return mHardwareBuffer; 4729 } 4730 4731 /** 4732 * Returns the presentation timestamp in microseconds. 4733 */ getPresentationTimeUs()4734 public long getPresentationTimeUs() { 4735 return mPresentationTimeUs; 4736 } 4737 4738 /** 4739 * Returns the buffer flags. 4740 */ getFlags()4741 public @BufferFlag int getFlags() { 4742 return mFlags; 4743 } 4744 4745 /* 4746 * Returns the BufferInfos associated with this OutputFrame. These BufferInfos 4747 * describes the access units present in the OutputFrame. Access units are laid 4748 * out contiguously without gaps and in order. 4749 */ 4750 @FlaggedApi(FLAG_LARGE_AUDIO_FRAME) getBufferInfos()4751 public @NonNull ArrayDeque<BufferInfo> getBufferInfos() { 4752 if (mBufferInfos.isEmpty()) { 4753 // single BufferInfo could be present. 4754 BufferInfo bufferInfo = new BufferInfo(); 4755 bufferInfo.set(0, 0, mPresentationTimeUs, mFlags); 4756 mBufferInfos.add(bufferInfo); 4757 } 4758 return mBufferInfos; 4759 } 4760 4761 /** 4762 * Returns a read-only {@link MediaFormat} for this frame. The returned 4763 * object is valid only until the client calls {@link MediaCodec#releaseOutputBuffer}. 4764 */ getFormat()4765 public @NonNull MediaFormat getFormat() { 4766 return mFormat; 4767 } 4768 4769 /** 4770 * Returns an unmodifiable set of the names of entries that has changed from 4771 * the previous frame. The entries may have been removed/changed/added. 4772 * Client can find out what the change is by querying {@link MediaFormat} 4773 * object returned from {@link #getFormat}. 4774 */ getChangedKeys()4775 public @NonNull Set<String> getChangedKeys() { 4776 if (mKeySet.isEmpty() && !mChangedKeys.isEmpty()) { 4777 mKeySet.addAll(mChangedKeys); 4778 } 4779 return Collections.unmodifiableSet(mKeySet); 4780 } 4781 clear()4782 void clear() { 4783 mLinearBlock = null; 4784 mHardwareBuffer = null; 4785 mFormat = null; 4786 mBufferInfos.clear(); 4787 mChangedKeys.clear(); 4788 mKeySet.clear(); 4789 mLoaded = false; 4790 } 4791 isAccessible()4792 boolean isAccessible() { 4793 return mAccessible; 4794 } 4795 setAccessible(boolean accessible)4796 void setAccessible(boolean accessible) { 4797 mAccessible = accessible; 4798 } 4799 setBufferInfo(MediaCodec.BufferInfo info)4800 void setBufferInfo(MediaCodec.BufferInfo info) { 4801 // since any of setBufferInfo(s) should translate to getBufferInfos, 4802 // mBufferInfos needs to be reset for every setBufferInfo(s) 4803 mBufferInfos.clear(); 4804 mPresentationTimeUs = info.presentationTimeUs; 4805 mFlags = info.flags; 4806 } 4807 setBufferInfos(ArrayDeque<BufferInfo> infos)4808 void setBufferInfos(ArrayDeque<BufferInfo> infos) { 4809 mBufferInfos.clear(); 4810 mBufferInfos.addAll(infos); 4811 } 4812 isLoaded()4813 boolean isLoaded() { 4814 return mLoaded; 4815 } 4816 setLoaded(boolean loaded)4817 void setLoaded(boolean loaded) { 4818 mLoaded = loaded; 4819 } 4820 4821 private final int mIndex; 4822 private LinearBlock mLinearBlock = null; 4823 private HardwareBuffer mHardwareBuffer = null; 4824 private long mPresentationTimeUs = 0; 4825 private @BufferFlag int mFlags = 0; 4826 private MediaFormat mFormat = null; 4827 private final ArrayDeque<BufferInfo> mBufferInfos = new ArrayDeque<>(); 4828 private final ArrayList<String> mChangedKeys = new ArrayList<>(); 4829 private final Set<String> mKeySet = new HashSet<>(); 4830 private boolean mAccessible = false; 4831 private boolean mLoaded = false; 4832 } 4833 4834 private final ArrayList<OutputFrame> mOutputFrames = new ArrayList<>(); 4835 4836 /** 4837 * Returns an {@link OutputFrame} object. 4838 * 4839 * @param index output buffer index from 4840 * {@link Callback#onOutputBufferAvailable} 4841 * @return {@link OutputFrame} object describing the output buffer 4842 * @throws IllegalStateException if not using block model 4843 * @throws IllegalArgumentException if the output buffer is not available or 4844 * the index is out of range 4845 */ getOutputFrame(int index)4846 public @NonNull OutputFrame getOutputFrame(int index) { 4847 synchronized (mBufferLock) { 4848 if (mBufferMode != BUFFER_MODE_BLOCK) { 4849 throw new IllegalStateException("The codec is not configured for block model"); 4850 } 4851 if (index < 0 || index >= mOutputFrames.size()) { 4852 throw new IndexOutOfBoundsException("Expected range of index: [0," 4853 + (mQueueRequests.size() - 1) + "]; actual: " + index); 4854 } 4855 OutputFrame frame = mOutputFrames.get(index); 4856 if (frame == null) { 4857 throw new IllegalArgumentException("Unavailable index: " + index); 4858 } 4859 if (!frame.isAccessible()) { 4860 throw new IllegalArgumentException( 4861 "The output frame is stale at index " + index); 4862 } 4863 if (!frame.isLoaded()) { 4864 native_getOutputFrame(frame, index); 4865 frame.setLoaded(true); 4866 } 4867 return frame; 4868 } 4869 } 4870 native_getOutputFrame(OutputFrame frame, int index)4871 private native void native_getOutputFrame(OutputFrame frame, int index); 4872 4873 /** 4874 * The content is scaled to the surface dimensions 4875 */ 4876 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT = 1; 4877 4878 /** 4879 * The content is scaled, maintaining its aspect ratio, the whole 4880 * surface area is used, content may be cropped. 4881 * <p class=note> 4882 * This mode is only suitable for content with 1:1 pixel aspect ratio as you cannot 4883 * configure the pixel aspect ratio for a {@link Surface}. 4884 * <p class=note> 4885 * As of {@link android.os.Build.VERSION_CODES#N} release, this mode may not work if 4886 * the video is {@linkplain MediaFormat#KEY_ROTATION rotated} by 90 or 270 degrees. 4887 */ 4888 public static final int VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING = 2; 4889 4890 /** @hide */ 4891 @IntDef({ 4892 VIDEO_SCALING_MODE_SCALE_TO_FIT, 4893 VIDEO_SCALING_MODE_SCALE_TO_FIT_WITH_CROPPING, 4894 }) 4895 @Retention(RetentionPolicy.SOURCE) 4896 public @interface VideoScalingMode {} 4897 4898 /** 4899 * If a surface has been specified in a previous call to {@link #configure} 4900 * specifies the scaling mode to use. The default is "scale to fit". 4901 * <p class=note> 4902 * The scaling mode may be reset to the <strong>default</strong> each time an 4903 * {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is received from the codec; therefore, the client 4904 * must call this method after every buffer change event (and before the first output buffer is 4905 * released for rendering) to ensure consistent scaling mode. 4906 * <p class=note> 4907 * Since the {@link #INFO_OUTPUT_BUFFERS_CHANGED} event is deprecated, this can also be done 4908 * after each {@link #INFO_OUTPUT_FORMAT_CHANGED} event. 4909 * 4910 * @throws IllegalArgumentException if mode is not recognized. 4911 * @throws IllegalStateException if in the Released state. 4912 */ setVideoScalingMode(@ideoScalingMode int mode)4913 public native final void setVideoScalingMode(@VideoScalingMode int mode); 4914 4915 /** 4916 * Sets the audio presentation. 4917 * @param presentation see {@link AudioPresentation}. In particular, id should be set. 4918 */ setAudioPresentation(@onNull AudioPresentation presentation)4919 public void setAudioPresentation(@NonNull AudioPresentation presentation) { 4920 if (presentation == null) { 4921 throw new NullPointerException("audio presentation is null"); 4922 } 4923 native_setAudioPresentation(presentation.getPresentationId(), presentation.getProgramId()); 4924 } 4925 native_setAudioPresentation(int presentationId, int programId)4926 private native void native_setAudioPresentation(int presentationId, int programId); 4927 4928 /** 4929 * Retrieve the codec name. 4930 * 4931 * If the codec was created by createDecoderByType or createEncoderByType, what component is 4932 * chosen is not known beforehand. This method returns the name of the codec that was 4933 * selected by the platform. 4934 * 4935 * <strong>Note:</strong> Implementations may provide multiple aliases (codec 4936 * names) for the same underlying codec, any of which can be used to instantiate the same 4937 * underlying codec in {@link MediaCodec#createByCodecName}. This method returns the 4938 * name used to create the codec in this case. 4939 * 4940 * @throws IllegalStateException if in the Released state. 4941 */ 4942 @NonNull getName()4943 public final String getName() { 4944 // get canonical name to handle exception 4945 String canonicalName = getCanonicalName(); 4946 return mNameAtCreation != null ? mNameAtCreation : canonicalName; 4947 } 4948 4949 /** 4950 * Retrieve the underlying codec name. 4951 * 4952 * This method is similar to {@link #getName}, except that it returns the underlying component 4953 * name even if an alias was used to create this MediaCodec object by name, 4954 * 4955 * @throws IllegalStateException if in the Released state. 4956 */ 4957 @NonNull getCanonicalName()4958 public native final String getCanonicalName(); 4959 4960 /** 4961 * Return Metrics data about the current codec instance. 4962 * 4963 * @return a {@link PersistableBundle} containing the set of attributes and values 4964 * available for the media being handled by this instance of MediaCodec 4965 * The attributes are descibed in {@link MetricsConstants}. 4966 * 4967 * Additional vendor-specific fields may also be present in 4968 * the return value. 4969 */ getMetrics()4970 public PersistableBundle getMetrics() { 4971 PersistableBundle bundle = native_getMetrics(); 4972 return bundle; 4973 } 4974 native_getMetrics()4975 private native PersistableBundle native_getMetrics(); 4976 4977 /** 4978 * Change a video encoder's target bitrate on the fly. The value is an 4979 * Integer object containing the new bitrate in bps. 4980 * 4981 * @see #setParameters(Bundle) 4982 */ 4983 public static final String PARAMETER_KEY_VIDEO_BITRATE = "video-bitrate"; 4984 4985 /** 4986 * Temporarily suspend/resume encoding of input data. While suspended 4987 * input data is effectively discarded instead of being fed into the 4988 * encoder. This parameter really only makes sense to use with an encoder 4989 * in "surface-input" mode, as the client code has no control over the 4990 * input-side of the encoder in that case. 4991 * The value is an Integer object containing the value 1 to suspend 4992 * or the value 0 to resume. 4993 * 4994 * @see #setParameters(Bundle) 4995 */ 4996 public static final String PARAMETER_KEY_SUSPEND = "drop-input-frames"; 4997 4998 /** 4999 * When {@link #PARAMETER_KEY_SUSPEND} is present, the client can also 5000 * optionally use this key to specify the timestamp (in micro-second) 5001 * at which the suspend/resume operation takes effect. 5002 * 5003 * Note that the specified timestamp must be greater than or equal to the 5004 * timestamp of any previously queued suspend/resume operations. 5005 * 5006 * The value is a long int, indicating the timestamp to suspend/resume. 5007 * 5008 * @see #setParameters(Bundle) 5009 */ 5010 public static final String PARAMETER_KEY_SUSPEND_TIME = "drop-start-time-us"; 5011 5012 /** 5013 * Specify an offset (in micro-second) to be added on top of the timestamps 5014 * onward. A typical use case is to apply an adjust to the timestamps after 5015 * a period of pause by the user. 5016 * 5017 * This parameter can only be used on an encoder in "surface-input" mode. 5018 * 5019 * The value is a long int, indicating the timestamp offset to be applied. 5020 * 5021 * @see #setParameters(Bundle) 5022 */ 5023 public static final String PARAMETER_KEY_OFFSET_TIME = "time-offset-us"; 5024 5025 /** 5026 * Request that the encoder produce a sync frame "soon". 5027 * Provide an Integer with the value 0. 5028 * 5029 * @see #setParameters(Bundle) 5030 */ 5031 public static final String PARAMETER_KEY_REQUEST_SYNC_FRAME = "request-sync"; 5032 5033 /** 5034 * Set the HDR10+ metadata on the next queued input frame. 5035 * 5036 * Provide a byte array of data that's conforming to the 5037 * user_data_registered_itu_t_t35() syntax of SEI message for ST 2094-40. 5038 *<p> 5039 * For decoders: 5040 *<p> 5041 * When a decoder is configured for one of the HDR10+ profiles that uses 5042 * out-of-band metadata (such as {@link 5043 * MediaCodecInfo.CodecProfileLevel#VP9Profile2HDR10Plus} or {@link 5044 * MediaCodecInfo.CodecProfileLevel#VP9Profile3HDR10Plus}), this 5045 * parameter sets the HDR10+ metadata on the next input buffer queued 5046 * to the decoder. A decoder supporting these profiles must propagate 5047 * the metadata to the format of the output buffer corresponding to this 5048 * particular input buffer (under key {@link MediaFormat#KEY_HDR10_PLUS_INFO}). 5049 * The metadata should be applied to that output buffer and the buffers 5050 * following it (in display order), until the next output buffer (in 5051 * display order) upon which an HDR10+ metadata is set. 5052 *<p> 5053 * This parameter shouldn't be set if the decoder is not configured for 5054 * an HDR10+ profile that uses out-of-band metadata. In particular, 5055 * it shouldn't be set for HDR10+ profiles that uses in-band metadata 5056 * where the metadata is embedded in the input buffers, for example 5057 * {@link MediaCodecInfo.CodecProfileLevel#HEVCProfileMain10HDR10Plus}. 5058 *<p> 5059 * For encoders: 5060 *<p> 5061 * When an encoder is configured for one of the HDR10+ profiles and the 5062 * operates in byte buffer input mode (instead of surface input mode), 5063 * this parameter sets the HDR10+ metadata on the next input buffer queued 5064 * to the encoder. For the HDR10+ profiles that uses out-of-band metadata 5065 * (such as {@link MediaCodecInfo.CodecProfileLevel#VP9Profile2HDR10Plus}, 5066 * or {@link MediaCodecInfo.CodecProfileLevel#VP9Profile3HDR10Plus}), 5067 * the metadata must be propagated to the format of the output buffer 5068 * corresponding to this particular input buffer (under key {@link 5069 * MediaFormat#KEY_HDR10_PLUS_INFO}). For the HDR10+ profiles that uses 5070 * in-band metadata (such as {@link 5071 * MediaCodecInfo.CodecProfileLevel#HEVCProfileMain10HDR10Plus}), the 5072 * metadata info must be embedded in the corresponding output buffer itself. 5073 *<p> 5074 * This parameter shouldn't be set if the encoder is not configured for 5075 * an HDR10+ profile, or if it's operating in surface input mode. 5076 *<p> 5077 * 5078 * @see MediaFormat#KEY_HDR10_PLUS_INFO 5079 */ 5080 public static final String PARAMETER_KEY_HDR10_PLUS_INFO = MediaFormat.KEY_HDR10_PLUS_INFO; 5081 5082 /** 5083 * Enable/disable low latency decoding mode. 5084 * When enabled, the decoder doesn't hold input and output data more than 5085 * required by the codec standards. 5086 * The value is an Integer object containing the value 1 to enable 5087 * or the value 0 to disable. 5088 * 5089 * @see #setParameters(Bundle) 5090 * @see MediaFormat#KEY_LOW_LATENCY 5091 */ 5092 public static final String PARAMETER_KEY_LOW_LATENCY = 5093 MediaFormat.KEY_LOW_LATENCY; 5094 5095 /** 5096 * Control video peek of the first frame when a codec is configured for tunnel mode with 5097 * {@link MediaFormat#KEY_AUDIO_SESSION_ID} while the {@link AudioTrack} is paused. 5098 *<p> 5099 * When disabled (1) after a {@link #flush} or {@link #start}, (2) while the corresponding 5100 * {@link AudioTrack} is paused and (3) before any buffers are queued, the first frame is not to 5101 * be rendered until either this parameter is enabled or the corresponding {@link AudioTrack} 5102 * has begun playback. Once the frame is decoded and ready to be rendered, 5103 * {@link OnFirstTunnelFrameReadyListener#onFirstTunnelFrameReady} is called but the frame is 5104 * not rendered. The surface continues to show the previously-rendered content, or black if the 5105 * surface is new. A subsequent call to {@link AudioTrack#play} renders this frame and triggers 5106 * a callback to {@link OnFrameRenderedListener#onFrameRendered}, and video playback begins. 5107 *<p> 5108 * <b>Note</b>: To clear any previously rendered content and show black, configure the 5109 * MediaCodec with {@code KEY_PUSH_BLANK_BUFFERS_ON_STOP(1)}, and call {@link #stop} before 5110 * pushing new video frames to the codec. 5111 *<p> 5112 * When enabled (1) after a {@link #flush} or {@link #start} and (2) while the corresponding 5113 * {@link AudioTrack} is paused, the first frame is rendered as soon as it is decoded, or 5114 * immediately, if it has already been decoded. If not already decoded, when the frame is 5115 * decoded and ready to be rendered, 5116 * {@link OnFirstTunnelFrameReadyListener#onFirstTunnelFrameReady} is called. The frame is then 5117 * immediately rendered and {@link OnFrameRenderedListener#onFrameRendered} is subsequently 5118 * called. 5119 *<p> 5120 * The value is an Integer object containing the value 1 to enable or the value 0 to disable. 5121 *<p> 5122 * The default for this parameter is <b>enabled</b>. Once a frame has been rendered, changing 5123 * this parameter has no effect until a subsequent {@link #flush} or 5124 * {@link #stop}/{@link #start}. 5125 * 5126 * @see #setParameters(Bundle) 5127 */ 5128 public static final String PARAMETER_KEY_TUNNEL_PEEK = "tunnel-peek"; 5129 5130 /** 5131 * Set the region of interest as QpOffset-Map on the next queued input frame. 5132 * <p> 5133 * The associated value is a byte array containing quantization parameter (QP) offsets in 5134 * raster scan order for the entire frame at 16x16 granularity. The size of the byte array 5135 * shall be ((frame_width + 15) / 16) * ((frame_height + 15) / 16), where frame_width and 5136 * frame_height correspond to width and height configured using {@link MediaFormat#KEY_WIDTH} 5137 * and {@link MediaFormat#KEY_HEIGHT} keys respectively. During encoding, if the coding unit 5138 * size is larger than 16x16, then the qpOffset information of all 16x16 blocks that 5139 * encompass the coding unit is combined and used. The QP of target block will be calculated 5140 * as 'frameQP + offsetQP'. If the result exceeds minQP or maxQP configured then the value 5141 * will be clamped. Negative offset results in blocks encoded at lower QP than frame QP and 5142 * positive offsets will result in encoding blocks at higher QP than frame QP. If the areas 5143 * of negative QP and positive QP are chosen wisely, the overall viewing experience can be 5144 * improved. 5145 * <p> 5146 * If byte array size is too small than the expected size, components may ignore the 5147 * configuration silently. If the byte array exceeds the expected size, components shall use 5148 * the initial portion and ignore the rest. 5149 * <p> 5150 * The scope of this key is throughout the encoding session until it is reconfigured during 5151 * running state. 5152 * <p> 5153 * @see #setParameters(Bundle) 5154 */ 5155 @FlaggedApi(FLAG_REGION_OF_INTEREST) 5156 public static final String PARAMETER_KEY_QP_OFFSET_MAP = "qp-offset-map"; 5157 5158 /** 5159 * Set the region of interest as QpOffset-Rects on the next queued input frame. 5160 * <p> 5161 * The associated value is a String in the format "Top1,Left1-Bottom1,Right1=Offset1;Top2, 5162 * Left2-Bottom2,Right2=Offset2;...". Co-ordinates (Top, Left), (Top, Right), (Bottom, Left) 5163 * and (Bottom, Right) form the vertices of bounding box of region of interest in pixels. 5164 * Pixel (0, 0) points to the top-left corner of the frame. Offset is the suggested 5165 * quantization parameter (QP) offset of the blocks in the bounding box. The bounding box 5166 * will get stretched outwards to align to LCU boundaries during encoding. The Qp Offset is 5167 * integral and shall be in the range [-128, 127]. The QP of target block will be calculated 5168 * as frameQP + offsetQP. If the result exceeds minQP or maxQP configured then the value will 5169 * be clamped. Negative offset results in blocks encoded at lower QP than frame QP and 5170 * positive offsets will result in blocks encoded at higher QP than frame QP. If the areas of 5171 * negative QP and positive QP are chosen wisely, the overall viewing experience can be 5172 * improved. 5173 * <p> 5174 * If Roi rect is not valid that is bounding box width is < 0 or bounding box height is < 0, 5175 * components may ignore the configuration silently. If Roi rect extends outside frame 5176 * boundaries, then rect shall be clamped to the frame boundaries. 5177 * <p> 5178 * The scope of this key is throughout the encoding session until it is reconfigured during 5179 * running state. 5180 * <p> 5181 * The maximum number of contours (rectangles) that can be specified for a given input frame 5182 * is device specific. Implementations will drop/ignore the rectangles that are beyond their 5183 * supported limit. Hence it is preferable to place the rects in descending order of 5184 * importance. Transitively, if the bounding boxes overlap, then the most preferred 5185 * rectangle's qp offset (earlier rectangle qp offset) will be used to quantize the block. 5186 * <p> 5187 * @see #setParameters(Bundle) 5188 */ 5189 @FlaggedApi(FLAG_REGION_OF_INTEREST) 5190 public static final String PARAMETER_KEY_QP_OFFSET_RECTS = "qp-offset-rects"; 5191 5192 /** 5193 * Communicate additional parameter changes to the component instance. 5194 * <b>Note:</b> Some of these parameter changes may silently fail to apply. 5195 * 5196 * @param params The bundle of parameters to set. 5197 * @throws IllegalStateException if in the Released state. 5198 */ setParameters(@ullable Bundle params)5199 public final void setParameters(@Nullable Bundle params) { 5200 if (params == null) { 5201 return; 5202 } 5203 5204 String[] keys = new String[params.size()]; 5205 Object[] values = new Object[params.size()]; 5206 5207 int i = 0; 5208 for (final String key: params.keySet()) { 5209 if (key.equals(MediaFormat.KEY_AUDIO_SESSION_ID)) { 5210 int sessionId = 0; 5211 try { 5212 sessionId = (Integer)params.get(key); 5213 } catch (Exception e) { 5214 throw new IllegalArgumentException("Wrong Session ID Parameter!"); 5215 } 5216 keys[i] = "audio-hw-sync"; 5217 values[i] = AudioSystem.getAudioHwSyncForSession(sessionId); 5218 } else { 5219 keys[i] = key; 5220 Object value = params.get(key); 5221 5222 // Bundle's byte array is a byte[], JNI layer only takes ByteBuffer 5223 if (value instanceof byte[]) { 5224 values[i] = ByteBuffer.wrap((byte[])value); 5225 } else { 5226 values[i] = value; 5227 } 5228 } 5229 ++i; 5230 } 5231 5232 setParameters(keys, values); 5233 } 5234 logAndRun(String message, Runnable r)5235 private void logAndRun(String message, Runnable r) { 5236 final String TAG = "MediaCodec"; 5237 android.util.Log.d(TAG, "enter: " + message); 5238 r.run(); 5239 android.util.Log.d(TAG, "exit : " + message); 5240 } 5241 5242 /** 5243 * Sets an asynchronous callback for actionable MediaCodec events. 5244 * 5245 * If the client intends to use the component in asynchronous mode, 5246 * a valid callback should be provided before {@link #configure} is called. 5247 * 5248 * When asynchronous callback is enabled, the client should not call 5249 * {@link #getInputBuffers}, {@link #getOutputBuffers}, 5250 * {@link #dequeueInputBuffer(long)} or {@link #dequeueOutputBuffer(BufferInfo, long)}. 5251 * <p> 5252 * Also, {@link #flush} behaves differently in asynchronous mode. After calling 5253 * {@code flush}, you must call {@link #start} to "resume" receiving input buffers, 5254 * even if an input surface was created. 5255 * 5256 * @param cb The callback that will run. Use {@code null} to clear a previously 5257 * set callback (before {@link #configure configure} is called and run 5258 * in synchronous mode). 5259 * @param handler Callbacks will happen on the handler's thread. If {@code null}, 5260 * callbacks are done on the default thread (the caller's thread or the 5261 * main thread.) 5262 */ setCallback(@ullable Callback cb, @Nullable Handler handler)5263 public void setCallback(@Nullable /* MediaCodec. */ Callback cb, @Nullable Handler handler) { 5264 if (cb != null) { 5265 synchronized (mListenerLock) { 5266 EventHandler newHandler = getEventHandlerOn(handler, mCallbackHandler); 5267 // NOTE: there are no callbacks on the handler at this time, but check anyways 5268 // even if we were to extend this to be callable dynamically, it must 5269 // be called when codec is flushed, so no messages are pending. 5270 if (newHandler != mCallbackHandler) { 5271 if (android.media.codec.Flags.setCallbackStall()) { 5272 logAndRun( 5273 "[new handler] removeMessages(SET_CALLBACK)", 5274 () -> { 5275 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 5276 }); 5277 logAndRun( 5278 "[new handler] removeMessages(CALLBACK)", 5279 () -> { 5280 mCallbackHandler.removeMessages(EVENT_CALLBACK); 5281 }); 5282 } else { 5283 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 5284 mCallbackHandler.removeMessages(EVENT_CALLBACK); 5285 } 5286 mCallbackHandler = newHandler; 5287 } 5288 } 5289 } else if (mCallbackHandler != null) { 5290 if (android.media.codec.Flags.setCallbackStall()) { 5291 logAndRun( 5292 "[null handler] removeMessages(SET_CALLBACK)", 5293 () -> { 5294 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 5295 }); 5296 logAndRun( 5297 "[null handler] removeMessages(CALLBACK)", 5298 () -> { 5299 mCallbackHandler.removeMessages(EVENT_CALLBACK); 5300 }); 5301 } else { 5302 mCallbackHandler.removeMessages(EVENT_SET_CALLBACK); 5303 mCallbackHandler.removeMessages(EVENT_CALLBACK); 5304 } 5305 } 5306 5307 if (mCallbackHandler != null) { 5308 // set java callback on main handler 5309 Message msg = mCallbackHandler.obtainMessage(EVENT_SET_CALLBACK, 0, 0, cb); 5310 mCallbackHandler.sendMessage(msg); 5311 5312 // set native handler here, don't post to handler because 5313 // it may cause the callback to be delayed and set in a wrong state. 5314 // Note that native codec may start sending events to the callback 5315 // handler after this returns. 5316 native_setCallback(cb); 5317 } 5318 } 5319 5320 /** 5321 * Sets an asynchronous callback for actionable MediaCodec events on the default 5322 * looper. 5323 * <p> 5324 * Same as {@link #setCallback(Callback, Handler)} with handler set to null. 5325 * @param cb The callback that will run. Use {@code null} to clear a previously 5326 * set callback (before {@link #configure configure} is called and run 5327 * in synchronous mode). 5328 * @see #setCallback(Callback, Handler) 5329 */ setCallback(@ullable Callback cb)5330 public void setCallback(@Nullable /* MediaCodec. */ Callback cb) { 5331 setCallback(cb, null /* handler */); 5332 } 5333 5334 /** 5335 * Listener to be called when the first output frame has been decoded 5336 * and is ready to be rendered for a codec configured for tunnel mode with 5337 * {@code KEY_AUDIO_SESSION_ID}. 5338 * 5339 * @see MediaCodec#setOnFirstTunnelFrameReadyListener 5340 */ 5341 public interface OnFirstTunnelFrameReadyListener { 5342 5343 /** 5344 * Called when the first output frame has been decoded and is ready to be 5345 * rendered. 5346 */ onFirstTunnelFrameReady(@onNull MediaCodec codec)5347 void onFirstTunnelFrameReady(@NonNull MediaCodec codec); 5348 } 5349 5350 /** 5351 * Registers a callback to be invoked when the first output frame has been decoded 5352 * and is ready to be rendered on a codec configured for tunnel mode with {@code 5353 * KEY_AUDIO_SESSION_ID}. 5354 * 5355 * @param handler the callback will be run on the handler's thread. If {@code 5356 * null}, the callback will be run on the default thread, which is the looper from 5357 * which the codec was created, or a new thread if there was none. 5358 * 5359 * @param listener the callback that will be run. If {@code null}, clears any registered 5360 * listener. 5361 */ setOnFirstTunnelFrameReadyListener( @ullable Handler handler, @Nullable OnFirstTunnelFrameReadyListener listener)5362 public void setOnFirstTunnelFrameReadyListener( 5363 @Nullable Handler handler, @Nullable OnFirstTunnelFrameReadyListener listener) { 5364 synchronized (mListenerLock) { 5365 mOnFirstTunnelFrameReadyListener = listener; 5366 if (listener != null) { 5367 EventHandler newHandler = getEventHandlerOn( 5368 handler, 5369 mOnFirstTunnelFrameReadyHandler); 5370 if (newHandler != mOnFirstTunnelFrameReadyHandler) { 5371 mOnFirstTunnelFrameReadyHandler.removeMessages(EVENT_FIRST_TUNNEL_FRAME_READY); 5372 } 5373 mOnFirstTunnelFrameReadyHandler = newHandler; 5374 } else if (mOnFirstTunnelFrameReadyHandler != null) { 5375 mOnFirstTunnelFrameReadyHandler.removeMessages(EVENT_FIRST_TUNNEL_FRAME_READY); 5376 } 5377 native_enableOnFirstTunnelFrameReadyListener(listener != null); 5378 } 5379 } 5380 native_enableOnFirstTunnelFrameReadyListener(boolean enable)5381 private native void native_enableOnFirstTunnelFrameReadyListener(boolean enable); 5382 5383 /** 5384 * Listener to be called when an output frame has rendered on the output surface 5385 * 5386 * @see MediaCodec#setOnFrameRenderedListener 5387 */ 5388 public interface OnFrameRenderedListener { 5389 5390 /** 5391 * Called when an output frame has rendered on the output surface. 5392 * <p> 5393 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 5394 * render timing samples, and can be significantly delayed and batched. Starting with 5395 * Android {@link android.os.Build.VERSION_CODES#UPSIDE_DOWN_CAKE}, a callback will always 5396 * be received for each rendered frame providing the MediaCodec is still in the executing 5397 * state when the callback is dispatched. Prior to Android 5398 * {@link android.os.Build.VERSION_CODES#UPSIDE_DOWN_CAKE}, some frames may have been 5399 * rendered even if there was no callback generated. 5400 * 5401 * @param codec the MediaCodec instance 5402 * @param presentationTimeUs the presentation time (media time) of the frame rendered. 5403 * This is usually the same as specified in {@link #queueInputBuffer}; however, 5404 * some codecs may alter the media time by applying some time-based transformation, 5405 * such as frame rate conversion. In that case, presentation time corresponds 5406 * to the actual output frame rendered. 5407 * @param nanoTime The system time when the frame was rendered. 5408 * 5409 * @see System#nanoTime 5410 */ onFrameRendered( @onNull MediaCodec codec, long presentationTimeUs, long nanoTime)5411 public void onFrameRendered( 5412 @NonNull MediaCodec codec, long presentationTimeUs, long nanoTime); 5413 } 5414 5415 /** 5416 * Registers a callback to be invoked when an output frame is rendered on the output surface. 5417 * <p> 5418 * This method can be called in any codec state, but will only have an effect in the 5419 * Executing state for codecs that render buffers to the output surface. 5420 * <p> 5421 * <strong>Note:</strong> This callback is for informational purposes only: to get precise 5422 * render timing samples, and can be significantly delayed and batched. Some frames may have 5423 * been rendered even if there was no callback generated. 5424 * 5425 * @param listener the callback that will be run 5426 * @param handler the callback will be run on the handler's thread. If {@code null}, 5427 * the callback will be run on the default thread, which is the looper 5428 * from which the codec was created, or a new thread if there was none. 5429 */ setOnFrameRenderedListener( @ullable OnFrameRenderedListener listener, @Nullable Handler handler)5430 public void setOnFrameRenderedListener( 5431 @Nullable OnFrameRenderedListener listener, @Nullable Handler handler) { 5432 synchronized (mListenerLock) { 5433 mOnFrameRenderedListener = listener; 5434 if (listener != null) { 5435 EventHandler newHandler = getEventHandlerOn(handler, mOnFrameRenderedHandler); 5436 if (newHandler != mOnFrameRenderedHandler) { 5437 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 5438 } 5439 mOnFrameRenderedHandler = newHandler; 5440 } else if (mOnFrameRenderedHandler != null) { 5441 mOnFrameRenderedHandler.removeMessages(EVENT_FRAME_RENDERED); 5442 } 5443 native_enableOnFrameRenderedListener(listener != null); 5444 } 5445 } 5446 native_enableOnFrameRenderedListener(boolean enable)5447 private native void native_enableOnFrameRenderedListener(boolean enable); 5448 5449 /** 5450 * Returns a list of vendor parameter names. 5451 * <p> 5452 * This method can be called in any codec state except for released state. 5453 * 5454 * @return a list containing supported vendor parameters; an empty 5455 * list if no vendor parameters are supported. The order of the 5456 * parameters is arbitrary. 5457 * @throws IllegalStateException if in the Released state. 5458 */ 5459 @NonNull getSupportedVendorParameters()5460 public List<String> getSupportedVendorParameters() { 5461 return native_getSupportedVendorParameters(); 5462 } 5463 5464 @NonNull native_getSupportedVendorParameters()5465 private native List<String> native_getSupportedVendorParameters(); 5466 5467 /** 5468 * Contains description of a parameter. 5469 */ 5470 public static class ParameterDescriptor { ParameterDescriptor()5471 private ParameterDescriptor() {} 5472 5473 /** 5474 * Returns the name of the parameter. 5475 */ 5476 @NonNull getName()5477 public String getName() { 5478 return mName; 5479 } 5480 5481 /** 5482 * Returns the type of the parameter. 5483 * {@link MediaFormat#TYPE_NULL} is never returned. 5484 */ 5485 @MediaFormat.Type getType()5486 public int getType() { 5487 return mType; 5488 } 5489 5490 @Override equals(Object o)5491 public boolean equals(Object o) { 5492 if (o == null) { 5493 return false; 5494 } 5495 if (!(o instanceof ParameterDescriptor)) { 5496 return false; 5497 } 5498 ParameterDescriptor other = (ParameterDescriptor) o; 5499 return this.mName.equals(other.mName) && this.mType == other.mType; 5500 } 5501 5502 @Override hashCode()5503 public int hashCode() { 5504 return Arrays.asList( 5505 (Object) mName, 5506 (Object) Integer.valueOf(mType)).hashCode(); 5507 } 5508 5509 private String mName; 5510 private @MediaFormat.Type int mType; 5511 } 5512 5513 /** 5514 * Describe a parameter with the name. 5515 * <p> 5516 * This method can be called in any codec state except for released state. 5517 * 5518 * @param name name of the parameter to describe, typically one from 5519 * {@link #getSupportedVendorParameters}. 5520 * @return {@link ParameterDescriptor} object that describes the parameter. 5521 * {@code null} if unrecognized / not able to describe. 5522 * @throws IllegalStateException if in the Released state. 5523 */ 5524 @Nullable getParameterDescriptor(@onNull String name)5525 public ParameterDescriptor getParameterDescriptor(@NonNull String name) { 5526 return native_getParameterDescriptor(name); 5527 } 5528 5529 @Nullable native_getParameterDescriptor(@onNull String name)5530 private native ParameterDescriptor native_getParameterDescriptor(@NonNull String name); 5531 5532 /** 5533 * Subscribe to vendor parameters, so that these parameters will be present in 5534 * {@link #getOutputFormat} and changes to these parameters generate 5535 * output format change event. 5536 * <p> 5537 * Unrecognized parameter names or standard (non-vendor) parameter names will be ignored. 5538 * {@link #reset} also resets the list of subscribed parameters. 5539 * If a parameter in {@code names} is already subscribed, it will remain subscribed. 5540 * <p> 5541 * This method can be called in any codec state except for released state. When called in 5542 * running state with newly subscribed parameters, it takes effect no later than the 5543 * processing of the subsequently queued buffer. For the new parameters, the codec will generate 5544 * output format change event. 5545 * <p> 5546 * Note that any vendor parameters set in a {@link #configure} or 5547 * {@link #setParameters} call are automatically subscribed. 5548 * <p> 5549 * See also {@link #INFO_OUTPUT_FORMAT_CHANGED} or {@link Callback#onOutputFormatChanged} 5550 * for output format change events. 5551 * 5552 * @param names names of the vendor parameters to subscribe. This may be an empty list, 5553 * and in that case this method will not change the list of subscribed parameters. 5554 * @throws IllegalStateException if in the Released state. 5555 */ subscribeToVendorParameters(@onNull List<String> names)5556 public void subscribeToVendorParameters(@NonNull List<String> names) { 5557 native_subscribeToVendorParameters(names); 5558 } 5559 native_subscribeToVendorParameters(@onNull List<String> names)5560 private native void native_subscribeToVendorParameters(@NonNull List<String> names); 5561 5562 /** 5563 * Unsubscribe from vendor parameters, so that these parameters will not be present in 5564 * {@link #getOutputFormat} and changes to these parameters no longer generate 5565 * output format change event. 5566 * <p> 5567 * Unrecognized parameter names, standard (non-vendor) parameter names will be ignored. 5568 * {@link #reset} also resets the list of subscribed parameters. 5569 * If a parameter in {@code names} is already unsubscribed, it will remain unsubscribed. 5570 * <p> 5571 * This method can be called in any codec state except for released state. When called in 5572 * running state with newly unsubscribed parameters, it takes effect no later than the 5573 * processing of the subsequently queued buffer. For the removed parameters, the codec will 5574 * generate output format change event. 5575 * <p> 5576 * Note that any vendor parameters set in a {@link #configure} or 5577 * {@link #setParameters} call are automatically subscribed, and with this method 5578 * they can be unsubscribed. 5579 * <p> 5580 * See also {@link #INFO_OUTPUT_FORMAT_CHANGED} or {@link Callback#onOutputFormatChanged} 5581 * for output format change events. 5582 * 5583 * @param names names of the vendor parameters to unsubscribe. This may be an empty list, 5584 * and in that case this method will not change the list of subscribed parameters. 5585 * @throws IllegalStateException if in the Released state. 5586 */ unsubscribeFromVendorParameters(@onNull List<String> names)5587 public void unsubscribeFromVendorParameters(@NonNull List<String> names) { 5588 native_unsubscribeFromVendorParameters(names); 5589 } 5590 native_unsubscribeFromVendorParameters(@onNull List<String> names)5591 private native void native_unsubscribeFromVendorParameters(@NonNull List<String> names); 5592 getEventHandlerOn( @ullable Handler handler, @NonNull EventHandler lastHandler)5593 private EventHandler getEventHandlerOn( 5594 @Nullable Handler handler, @NonNull EventHandler lastHandler) { 5595 if (handler == null) { 5596 return mEventHandler; 5597 } else { 5598 Looper looper = handler.getLooper(); 5599 if (lastHandler.getLooper() == looper) { 5600 return lastHandler; 5601 } else { 5602 return new EventHandler(this, looper); 5603 } 5604 } 5605 } 5606 5607 /** 5608 * MediaCodec callback interface. Used to notify the user asynchronously 5609 * of various MediaCodec events. 5610 */ 5611 public static abstract class Callback { 5612 /** 5613 * Called when an input buffer becomes available. 5614 * 5615 * @param codec The MediaCodec object. 5616 * @param index The index of the available input buffer. 5617 */ onInputBufferAvailable(@onNull MediaCodec codec, int index)5618 public abstract void onInputBufferAvailable(@NonNull MediaCodec codec, int index); 5619 5620 /** 5621 * Called when an output buffer becomes available. 5622 * 5623 * @param codec The MediaCodec object. 5624 * @param index The index of the available output buffer. 5625 * @param info Info regarding the available output buffer {@link MediaCodec.BufferInfo}. 5626 */ onOutputBufferAvailable( @onNull MediaCodec codec, int index, @NonNull BufferInfo info)5627 public abstract void onOutputBufferAvailable( 5628 @NonNull MediaCodec codec, int index, @NonNull BufferInfo info); 5629 5630 /** 5631 * Called when multiple access-units are available in the output. 5632 * 5633 * @param codec The MediaCodec object. 5634 * @param index The index of the available output buffer. 5635 * @param infos Infos describing the available output buffer {@link MediaCodec.BufferInfo}. 5636 * Access units present in the output buffer are laid out contiguously 5637 * without gaps and in order. 5638 */ 5639 @FlaggedApi(FLAG_LARGE_AUDIO_FRAME) onOutputBuffersAvailable( @onNull MediaCodec codec, int index, @NonNull ArrayDeque<BufferInfo> infos)5640 public void onOutputBuffersAvailable( 5641 @NonNull MediaCodec codec, int index, @NonNull ArrayDeque<BufferInfo> infos) { 5642 /* 5643 * This callback returns multiple BufferInfos when codecs are configured to operate on 5644 * large audio frame. Since at this point, we have a single large buffer, returning 5645 * each BufferInfo using 5646 * {@link Callback#onOutputBufferAvailable onOutputBufferAvailable} may cause the 5647 * index to be released to the codec using {@link MediaCodec#releaseOutputBuffer} 5648 * before all BuffersInfos can be returned to the client. 5649 * Hence this callback is required to be implemented or else an exception is thrown. 5650 */ 5651 throw new IllegalStateException( 5652 "Client must override onOutputBuffersAvailable when codec is " + 5653 "configured to operate with multiple access units"); 5654 } 5655 5656 /** 5657 * Called when the MediaCodec encountered an error 5658 * 5659 * @param codec The MediaCodec object. 5660 * @param e The {@link MediaCodec.CodecException} object describing the error. 5661 */ onError(@onNull MediaCodec codec, @NonNull CodecException e)5662 public abstract void onError(@NonNull MediaCodec codec, @NonNull CodecException e); 5663 5664 /** 5665 * Called only when MediaCodec encountered a crypto(decryption) error when using 5666 * a decoder configured with CONFIGURE_FLAG_USE_CRYPTO_ASYNC flag along with crypto 5667 * or descrambler object. 5668 * 5669 * @param codec The MediaCodec object 5670 * @param e The {@link MediaCodec.CryptoException} object with error details. 5671 */ onCryptoError(@onNull MediaCodec codec, @NonNull CryptoException e)5672 public void onCryptoError(@NonNull MediaCodec codec, @NonNull CryptoException e) { 5673 /* 5674 * A default implementation for backward compatibility. 5675 * Use of CONFIGURE_FLAG_USE_CRYPTO_ASYNC requires override of this callback 5676 * to receive CrytoInfo. Without an orverride an exception is thrown. 5677 */ 5678 throw new IllegalStateException( 5679 "Client must override onCryptoError when the codec is " + 5680 "configured with CONFIGURE_FLAG_USE_CRYPTO_ASYNC.", e); 5681 } 5682 5683 /** 5684 * Called when the output format has changed 5685 * 5686 * @param codec The MediaCodec object. 5687 * @param format The new output format. 5688 */ onOutputFormatChanged( @onNull MediaCodec codec, @NonNull MediaFormat format)5689 public abstract void onOutputFormatChanged( 5690 @NonNull MediaCodec codec, @NonNull MediaFormat format); 5691 } 5692 postEventFromNative( int what, int arg1, int arg2, @Nullable Object obj)5693 private void postEventFromNative( 5694 int what, int arg1, int arg2, @Nullable Object obj) { 5695 synchronized (mListenerLock) { 5696 EventHandler handler = mEventHandler; 5697 if (what == EVENT_CALLBACK) { 5698 handler = mCallbackHandler; 5699 } else if (what == EVENT_FIRST_TUNNEL_FRAME_READY) { 5700 handler = mOnFirstTunnelFrameReadyHandler; 5701 } else if (what == EVENT_FRAME_RENDERED) { 5702 handler = mOnFrameRenderedHandler; 5703 } 5704 if (handler != null) { 5705 Message msg = handler.obtainMessage(what, arg1, arg2, obj); 5706 handler.sendMessage(msg); 5707 } 5708 } 5709 } 5710 5711 @UnsupportedAppUsage setParameters(@onNull String[] keys, @NonNull Object[] values)5712 private native final void setParameters(@NonNull String[] keys, @NonNull Object[] values); 5713 5714 /** 5715 * Get the codec info. If the codec was created by createDecoderByType 5716 * or createEncoderByType, what component is chosen is not known beforehand, 5717 * and thus the caller does not have the MediaCodecInfo. 5718 * @throws IllegalStateException if in the Released state. 5719 */ 5720 @NonNull getCodecInfo()5721 public MediaCodecInfo getCodecInfo() { 5722 // Get the codec name first. If the codec is already released, 5723 // IllegalStateException will be thrown here. 5724 String name = getName(); 5725 synchronized (mCodecInfoLock) { 5726 if (mCodecInfo == null) { 5727 // Get the codec info for this codec itself first. Only initialize 5728 // the full codec list if this somehow fails because it can be slow. 5729 mCodecInfo = getOwnCodecInfo(); 5730 if (mCodecInfo == null) { 5731 mCodecInfo = MediaCodecList.getInfoFor(name); 5732 } 5733 } 5734 return mCodecInfo; 5735 } 5736 } 5737 5738 @NonNull getOwnCodecInfo()5739 private native final MediaCodecInfo getOwnCodecInfo(); 5740 5741 @NonNull 5742 @UnsupportedAppUsage getBuffers(boolean input)5743 private native final ByteBuffer[] getBuffers(boolean input); 5744 5745 @Nullable getBuffer(boolean input, int index)5746 private native final ByteBuffer getBuffer(boolean input, int index); 5747 5748 @Nullable getImage(boolean input, int index)5749 private native final Image getImage(boolean input, int index); 5750 native_init()5751 private static native final void native_init(); 5752 native_setup( @onNull String name, boolean nameIsType, boolean encoder, int pid, int uid)5753 private native final void native_setup( 5754 @NonNull String name, boolean nameIsType, boolean encoder, int pid, int uid); 5755 native_finalize()5756 private native final void native_finalize(); 5757 5758 static { 5759 System.loadLibrary("media_jni"); native_init()5760 native_init(); 5761 } 5762 5763 @UnsupportedAppUsage(maxTargetSdk = Build.VERSION_CODES.P, trackingBug = 115609023) 5764 private long mNativeContext = 0; 5765 private final Lock mNativeContextLock = new ReentrantLock(); 5766 lockAndGetContext()5767 private final long lockAndGetContext() { 5768 mNativeContextLock.lock(); 5769 return mNativeContext; 5770 } 5771 setAndUnlockContext(long context)5772 private final void setAndUnlockContext(long context) { 5773 mNativeContext = context; 5774 mNativeContextLock.unlock(); 5775 } 5776 5777 /** @hide */ 5778 public static class MediaImage extends Image { 5779 private final boolean mIsReadOnly; 5780 private final int mWidth; 5781 private final int mHeight; 5782 private final int mFormat; 5783 private long mTimestamp; 5784 private final Plane[] mPlanes; 5785 private final ByteBuffer mBuffer; 5786 private final ByteBuffer mInfo; 5787 private final int mXOffset; 5788 private final int mYOffset; 5789 private final long mBufferContext; 5790 5791 private final static int TYPE_YUV = 1; 5792 5793 private final int mTransform = 0; //Default no transform 5794 private final int mScalingMode = 0; //Default frozen scaling mode 5795 5796 @Override getFormat()5797 public int getFormat() { 5798 throwISEIfImageIsInvalid(); 5799 return mFormat; 5800 } 5801 5802 @Override getHeight()5803 public int getHeight() { 5804 throwISEIfImageIsInvalid(); 5805 return mHeight; 5806 } 5807 5808 @Override getWidth()5809 public int getWidth() { 5810 throwISEIfImageIsInvalid(); 5811 return mWidth; 5812 } 5813 5814 @Override getTransform()5815 public int getTransform() { 5816 throwISEIfImageIsInvalid(); 5817 return mTransform; 5818 } 5819 5820 @Override getScalingMode()5821 public int getScalingMode() { 5822 throwISEIfImageIsInvalid(); 5823 return mScalingMode; 5824 } 5825 5826 @Override getTimestamp()5827 public long getTimestamp() { 5828 throwISEIfImageIsInvalid(); 5829 return mTimestamp; 5830 } 5831 5832 @Override 5833 @NonNull getPlanes()5834 public Plane[] getPlanes() { 5835 throwISEIfImageIsInvalid(); 5836 return Arrays.copyOf(mPlanes, mPlanes.length); 5837 } 5838 5839 @Override close()5840 public void close() { 5841 if (mIsImageValid) { 5842 if (mBuffer != null) { 5843 java.nio.NioUtils.freeDirectBuffer(mBuffer); 5844 } 5845 if (mBufferContext != 0) { 5846 native_closeMediaImage(mBufferContext); 5847 } 5848 mIsImageValid = false; 5849 } 5850 } 5851 5852 /** 5853 * Set the crop rectangle associated with this frame. 5854 * <p> 5855 * The crop rectangle specifies the region of valid pixels in the image, 5856 * using coordinates in the largest-resolution plane. 5857 */ 5858 @Override setCropRect(@ullable Rect cropRect)5859 public void setCropRect(@Nullable Rect cropRect) { 5860 if (mIsReadOnly) { 5861 throw new ReadOnlyBufferException(); 5862 } 5863 super.setCropRect(cropRect); 5864 } 5865 MediaImage( @onNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect)5866 public MediaImage( 5867 @NonNull ByteBuffer buffer, @NonNull ByteBuffer info, boolean readOnly, 5868 long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect) { 5869 mTimestamp = timestamp; 5870 mIsImageValid = true; 5871 mIsReadOnly = buffer.isReadOnly(); 5872 mBuffer = buffer.duplicate(); 5873 5874 // save offsets and info 5875 mXOffset = xOffset; 5876 mYOffset = yOffset; 5877 mInfo = info; 5878 5879 mBufferContext = 0; 5880 5881 int cbPlaneOffset = -1; 5882 int crPlaneOffset = -1; 5883 int planeOffsetInc = -1; 5884 int pixelStride = -1; 5885 5886 // read media-info. See MediaImage2 5887 if (info.remaining() == 104) { 5888 int type = info.getInt(); 5889 if (type != TYPE_YUV) { 5890 throw new UnsupportedOperationException("unsupported type: " + type); 5891 } 5892 int numPlanes = info.getInt(); 5893 if (numPlanes != 3) { 5894 throw new RuntimeException("unexpected number of planes: " + numPlanes); 5895 } 5896 mWidth = info.getInt(); 5897 mHeight = info.getInt(); 5898 if (mWidth < 1 || mHeight < 1) { 5899 throw new UnsupportedOperationException( 5900 "unsupported size: " + mWidth + "x" + mHeight); 5901 } 5902 int bitDepth = info.getInt(); 5903 if (bitDepth != 8 && bitDepth != 10) { 5904 throw new UnsupportedOperationException("unsupported bit depth: " + bitDepth); 5905 } 5906 int bitDepthAllocated = info.getInt(); 5907 if (bitDepthAllocated != 8 && bitDepthAllocated != 16) { 5908 throw new UnsupportedOperationException( 5909 "unsupported allocated bit depth: " + bitDepthAllocated); 5910 } 5911 if (bitDepth == 8 && bitDepthAllocated == 8) { 5912 mFormat = ImageFormat.YUV_420_888; 5913 planeOffsetInc = 1; 5914 pixelStride = 2; 5915 } else if (bitDepth == 10 && bitDepthAllocated == 16) { 5916 mFormat = ImageFormat.YCBCR_P010; 5917 planeOffsetInc = 2; 5918 pixelStride = 4; 5919 } else { 5920 throw new UnsupportedOperationException("couldn't infer ImageFormat" 5921 + " bitDepth: " + bitDepth + " bitDepthAllocated: " + bitDepthAllocated); 5922 } 5923 5924 mPlanes = new MediaPlane[numPlanes]; 5925 for (int ix = 0; ix < numPlanes; ix++) { 5926 int planeOffset = info.getInt(); 5927 int colInc = info.getInt(); 5928 int rowInc = info.getInt(); 5929 int horiz = info.getInt(); 5930 int vert = info.getInt(); 5931 if (horiz != vert || horiz != (ix == 0 ? 1 : 2)) { 5932 throw new UnsupportedOperationException("unexpected subsampling: " 5933 + horiz + "x" + vert + " on plane " + ix); 5934 } 5935 if (colInc < 1 || rowInc < 1) { 5936 throw new UnsupportedOperationException("unexpected strides: " 5937 + colInc + " pixel, " + rowInc + " row on plane " + ix); 5938 } 5939 buffer.clear(); 5940 buffer.position(mBuffer.position() + planeOffset 5941 + (xOffset / horiz) * colInc + (yOffset / vert) * rowInc); 5942 buffer.limit(buffer.position() + Utils.divUp(bitDepth, 8) 5943 + (mHeight / vert - 1) * rowInc + (mWidth / horiz - 1) * colInc); 5944 mPlanes[ix] = new MediaPlane(buffer.slice(), rowInc, colInc); 5945 if ((mFormat == ImageFormat.YUV_420_888 || mFormat == ImageFormat.YCBCR_P010) 5946 && ix == 1) { 5947 cbPlaneOffset = planeOffset; 5948 } else if ((mFormat == ImageFormat.YUV_420_888 5949 || mFormat == ImageFormat.YCBCR_P010) && ix == 2) { 5950 crPlaneOffset = planeOffset; 5951 } 5952 } 5953 } else { 5954 throw new UnsupportedOperationException( 5955 "unsupported info length: " + info.remaining()); 5956 } 5957 5958 // Validate chroma semiplanerness. 5959 if (mFormat == ImageFormat.YCBCR_P010) { 5960 if (crPlaneOffset != cbPlaneOffset + planeOffsetInc) { 5961 throw new UnsupportedOperationException("Invalid plane offsets" 5962 + " cbPlaneOffset: " + cbPlaneOffset + " crPlaneOffset: " + crPlaneOffset); 5963 } 5964 if (mPlanes[1].getPixelStride() != pixelStride 5965 || mPlanes[2].getPixelStride() != pixelStride) { 5966 throw new UnsupportedOperationException("Invalid pixelStride"); 5967 } 5968 } 5969 5970 if (cropRect == null) { 5971 cropRect = new Rect(0, 0, mWidth, mHeight); 5972 } 5973 cropRect.offset(-xOffset, -yOffset); 5974 super.setCropRect(cropRect); 5975 } 5976 MediaImage( @onNull ByteBuffer[] buffers, int[] rowStrides, int[] pixelStrides, int width, int height, int format, boolean readOnly, long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect, long context)5977 public MediaImage( 5978 @NonNull ByteBuffer[] buffers, int[] rowStrides, int[] pixelStrides, 5979 int width, int height, int format, boolean readOnly, 5980 long timestamp, int xOffset, int yOffset, @Nullable Rect cropRect, long context) { 5981 if (buffers.length != rowStrides.length || buffers.length != pixelStrides.length) { 5982 throw new IllegalArgumentException( 5983 "buffers, rowStrides and pixelStrides should have the same length"); 5984 } 5985 mWidth = width; 5986 mHeight = height; 5987 mFormat = format; 5988 mTimestamp = timestamp; 5989 mIsImageValid = true; 5990 mIsReadOnly = readOnly; 5991 mBuffer = null; 5992 mInfo = null; 5993 mPlanes = new MediaPlane[buffers.length]; 5994 for (int i = 0; i < buffers.length; ++i) { 5995 mPlanes[i] = new MediaPlane(buffers[i], rowStrides[i], pixelStrides[i]); 5996 } 5997 5998 // save offsets and info 5999 mXOffset = xOffset; 6000 mYOffset = yOffset; 6001 6002 if (cropRect == null) { 6003 cropRect = new Rect(0, 0, mWidth, mHeight); 6004 } 6005 cropRect.offset(-xOffset, -yOffset); 6006 super.setCropRect(cropRect); 6007 6008 mBufferContext = context; 6009 } 6010 6011 private class MediaPlane extends Plane { MediaPlane(@onNull ByteBuffer buffer, int rowInc, int colInc)6012 public MediaPlane(@NonNull ByteBuffer buffer, int rowInc, int colInc) { 6013 mData = buffer; 6014 mRowInc = rowInc; 6015 mColInc = colInc; 6016 } 6017 6018 @Override getRowStride()6019 public int getRowStride() { 6020 throwISEIfImageIsInvalid(); 6021 return mRowInc; 6022 } 6023 6024 @Override getPixelStride()6025 public int getPixelStride() { 6026 throwISEIfImageIsInvalid(); 6027 return mColInc; 6028 } 6029 6030 @Override 6031 @NonNull getBuffer()6032 public ByteBuffer getBuffer() { 6033 throwISEIfImageIsInvalid(); 6034 return mData; 6035 } 6036 6037 private final int mRowInc; 6038 private final int mColInc; 6039 private final ByteBuffer mData; 6040 } 6041 } 6042 6043 public final static class MetricsConstants 6044 { MetricsConstants()6045 private MetricsConstants() {} 6046 6047 /** 6048 * Key to extract the codec being used 6049 * from the {@link MediaCodec#getMetrics} return value. 6050 * The value is a String. 6051 */ 6052 public static final String CODEC = "android.media.mediacodec.codec"; 6053 6054 /** 6055 * Key to extract the MIME type 6056 * from the {@link MediaCodec#getMetrics} return value. 6057 * The value is a String. 6058 */ 6059 public static final String MIME_TYPE = "android.media.mediacodec.mime"; 6060 6061 /** 6062 * Key to extract what the codec mode 6063 * from the {@link MediaCodec#getMetrics} return value. 6064 * The value is a String. Values will be one of the constants 6065 * {@link #MODE_AUDIO} or {@link #MODE_VIDEO}. 6066 */ 6067 public static final String MODE = "android.media.mediacodec.mode"; 6068 6069 /** 6070 * The value returned for the key {@link #MODE} when the 6071 * codec is a audio codec. 6072 */ 6073 public static final String MODE_AUDIO = "audio"; 6074 6075 /** 6076 * The value returned for the key {@link #MODE} when the 6077 * codec is a video codec. 6078 */ 6079 public static final String MODE_VIDEO = "video"; 6080 6081 /** 6082 * Key to extract the flag indicating whether the codec is running 6083 * as an encoder or decoder from the {@link MediaCodec#getMetrics} return value. 6084 * The value is an integer. 6085 * A 0 indicates decoder; 1 indicates encoder. 6086 */ 6087 public static final String ENCODER = "android.media.mediacodec.encoder"; 6088 6089 /** 6090 * Key to extract the flag indicating whether the codec is running 6091 * in secure (DRM) mode from the {@link MediaCodec#getMetrics} return value. 6092 * The value is an integer. 6093 */ 6094 public static final String SECURE = "android.media.mediacodec.secure"; 6095 6096 /** 6097 * Key to extract the width (in pixels) of the video track 6098 * from the {@link MediaCodec#getMetrics} return value. 6099 * The value is an integer. 6100 */ 6101 public static final String WIDTH = "android.media.mediacodec.width"; 6102 6103 /** 6104 * Key to extract the height (in pixels) of the video track 6105 * from the {@link MediaCodec#getMetrics} return value. 6106 * The value is an integer. 6107 */ 6108 public static final String HEIGHT = "android.media.mediacodec.height"; 6109 6110 /** 6111 * Key to extract the rotation (in degrees) to properly orient the video 6112 * from the {@link MediaCodec#getMetrics} return. 6113 * The value is a integer. 6114 */ 6115 public static final String ROTATION = "android.media.mediacodec.rotation"; 6116 6117 } 6118 } 6119
