1# Copyright 2022 The Android Open Source Project
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#      http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Utility functions for processing video recordings.
15"""
16# Each item in this list corresponds to quality levels defined per
17# CamcorderProfile. For Video ITS, we will currently test below qualities
18# only if supported by the camera device.
19
20
21import logging
22import os.path
23import re
24import subprocess
25import error_util
26import image_processing_utils
27
28
29COLORSPACE_HDR = 'bt2020'
30HR_TO_SEC = 3600
31INDEX_FIRST_SUBGROUP = 1
32MIN_TO_SEC = 60
33
34ITS_SUPPORTED_QUALITIES = (
35    'HIGH',
36    '2160P',
37    '1080P',
38    '720P',
39    '480P',
40    'CIF',
41    'QCIF',
42    'QVGA',
43    'LOW',
44    'VGA'
45)
46
47LOW_RESOLUTION_SIZES = (
48    '176x144',
49    '192x144',
50    '352x288',
51    '384x288',
52    '320x240',
53)
54
55LOWEST_RES_TESTED_AREA = 640*360
56
57VIDEO_QUALITY_SIZE = {
58    # '480P', '1080P', HIGH' & 'LOW' are not included as they are DUT-dependent
59    '2160P': '3840x2160',
60    '720P': '1280x720',
61    'VGA': '640x480',
62    'CIF': '352x288',
63    'QVGA': '320x240',
64    'QCIF': '176x144',
65}
66
67
68def get_largest_common_preview_video_size(cam, camera_id):
69  """Returns the largest, common size between preview and video.
70
71  Args:
72    cam: camera object.
73    camera_id: str; camera ID.
74
75  Returns:
76    largest_common_size: str; largest common size between preview & video.
77  """
78  supported_preview_sizes = cam.get_all_supported_preview_sizes(camera_id)
79  supported_video_qualities = cam.get_supported_video_qualities(camera_id)
80  logging.debug('Supported video profiles & IDs: %s', supported_video_qualities)
81
82  # Make a list of supported_video_sizes from video qualities
83  supported_video_sizes = []
84  for quality in supported_video_qualities:
85    video_quality = quality.split(':')[0]  # form is ['CIF:3', '480P:4', ...]
86    if video_quality in VIDEO_QUALITY_SIZE:
87      supported_video_sizes.append(VIDEO_QUALITY_SIZE[video_quality])
88  logging.debug(
89      'Supported video sizes: %s', supported_video_sizes)
90
91  # Use areas of video sizes to find the largest common size
92  size_to_area = lambda s: int(s.split('x')[0])*int(s.split('x')[1])
93  largest_common_size = ''
94  largest_area = 0
95  common_sizes = list(set(supported_preview_sizes) & set(supported_video_sizes))
96  for size in common_sizes:
97    area = size_to_area(size)
98    if area > largest_area:
99      largest_area = area
100      largest_common_size = size
101  if not largest_common_size:
102    raise AssertionError('No common size between Preview and Video!')
103  logging.debug('Largest common size: %s', largest_common_size)
104  return largest_common_size
105
106
107def get_lowest_common_preview_video_size(
108    supported_preview_sizes, supported_video_qualities, min_area):
109  """Returns the common, smallest size above minimum in preview and video.
110
111  Args:
112    supported_preview_sizes: str; preview size (ex. '1920x1080')
113    supported_video_qualities: str; video recording quality and id pair
114    (ex. '480P:4', '720P:5'')
115    min_area: int; filter to eliminate smaller sizes (ex. 640*480)
116  Returns:
117    smallest_common_size: str; smallest, common size between preview and video
118    smallest_common_video_quality: str; video recording quality such as 480P
119  """
120
121  # Make dictionary on video quality and size according to compatibility
122  supported_video_size_to_quality = {}
123  for quality in supported_video_qualities:
124    video_quality = quality.split(':')[0]
125    if video_quality in VIDEO_QUALITY_SIZE:
126      video_size = VIDEO_QUALITY_SIZE[video_quality]
127      supported_video_size_to_quality[video_size] = video_quality
128  logging.debug(
129      'Supported video size to quality: %s', supported_video_size_to_quality)
130
131  # Use areas of video sizes to find the smallest, common size
132  size_to_area = lambda s: int(s.split('x')[0])*int(s.split('x')[1])
133  smallest_common_size = ''
134  smallest_area = float('inf')
135  for size in supported_preview_sizes:
136    if size in supported_video_size_to_quality:
137      area = size_to_area(size)
138      if smallest_area > area >= min_area:
139        smallest_area = area
140        smallest_common_size = size
141  logging.debug('Lowest common size: %s', smallest_common_size)
142
143  # Find video quality of resolution with resolution as key
144  smallest_common_video_quality = (
145      supported_video_size_to_quality[smallest_common_size])
146  logging.debug(
147      'Lowest common size video quality: %s', smallest_common_video_quality)
148
149  return smallest_common_size, smallest_common_video_quality
150
151
152def log_ffmpeg_version():
153  """Logs the ffmpeg version being used."""
154
155  ffmpeg_version_cmd = ('ffmpeg -version')
156  p = subprocess.Popen(ffmpeg_version_cmd, shell=True, stdout=subprocess.PIPE)
157  output, _ = p.communicate()
158  if p.poll() != 0:
159    raise error_util.CameraItsError('Error running ffmpeg version cmd.')
160  decoded_output = output.decode('utf-8')
161  logging.debug('ffmpeg version: %s', decoded_output.split(' ')[2])
162
163
164def extract_key_frames_from_video(log_path, video_file_name):
165  """Returns a list of extracted key frames.
166
167  Ffmpeg tool is used to extract key frames from the video at path
168  os.path.join(log_path, video_file_name).
169  The extracted key frames will have the name video_file_name with "_key_frame"
170  suffix to identify the frames for video of each quality. Since there can be
171  multiple key frames, each key frame image will be differentiated with it's
172  frame index. All the extracted key frames will be available in jpeg format
173  at the same path as the video file.
174
175  The run time flag '-loglevel quiet' hides the information from terminal.
176  In order to see the detailed output of ffmpeg command change the loglevel
177  option to 'info'.
178
179  Args:
180    log_path: path for video file directory.
181    video_file_name: name of the video file.
182  Returns:
183    key_frame_files: a sorted list of files which contains a name per key
184      frame. Ex: VID_20220325_050918_0_preview_1920x1440_key_frame_0001.png
185  """
186  ffmpeg_image_name = f'{os.path.splitext(video_file_name)[0]}_key_frame'
187  ffmpeg_image_file_path = os.path.join(
188      log_path, ffmpeg_image_name + '_%04d.png')
189  cmd = ['ffmpeg',
190         '-skip_frame',
191         'nokey',
192         '-i',
193         os.path.join(log_path, video_file_name),
194         '-vsync',
195         'vfr',
196         '-frame_pts',
197         'true',
198         ffmpeg_image_file_path,
199         '-loglevel',
200         'quiet',
201        ]
202  logging.debug('Extracting key frames from: %s', video_file_name)
203  _ = subprocess.call(cmd,
204                      stdin=subprocess.DEVNULL,
205                      stdout=subprocess.DEVNULL,
206                      stderr=subprocess.DEVNULL)
207  arr = os.listdir(os.path.join(log_path))
208  key_frame_files = []
209  for file in arr:
210    if '.png' in file and not os.path.isdir(file) and ffmpeg_image_name in file:
211      key_frame_files.append(file)
212  key_frame_files.sort()
213  logging.debug('Extracted key frames: %s', key_frame_files)
214  logging.debug('Length of key_frame_files: %d', len(key_frame_files))
215  if not key_frame_files:
216    raise AssertionError('No key frames extracted. Check source video.')
217
218  return key_frame_files
219
220
221def get_key_frame_to_process(key_frame_files):
222  """Returns the key frame file from the list of key_frame_files.
223
224  If the size of the list is 1 then the file in the list will be returned else
225  the file with highest frame_index will be returned for further processing.
226
227  Args:
228    key_frame_files: A list of key frame files.
229  Returns:
230    key_frame_file to be used for further processing.
231  """
232  if not key_frame_files:
233    raise AssertionError('key_frame_files list is empty.')
234  key_frame_files.sort()
235  return key_frame_files[-1]
236
237
238def extract_all_frames_from_video(log_path, video_file_name, img_format):
239  """Extracts and returns a list of all extracted frames.
240
241  Ffmpeg tool is used to extract all frames from the video at path
242  <log_path>/<video_file_name>. The extracted key frames will have the name
243  video_file_name with "_frame" suffix to identify the frames for video of each
244  size. Each frame image will be differentiated with its frame index. All
245  extracted key frames will be available in the provided img_format format at
246  the same path as the video file.
247
248  The run time flag '-loglevel quiet' hides the information from terminal.
249  In order to see the detailed output of ffmpeg command change the loglevel
250  option to 'info'.
251
252  Args:
253    log_path: str; path for video file directory
254    video_file_name: str; name of the video file.
255    img_format: str; type of image to export frames into. ex. 'png'
256  Returns:
257    key_frame_files: An ordered list of paths for each frame extracted from the
258                     video
259  """
260  logging.debug('Extracting all frames')
261  ffmpeg_image_name = f"{video_file_name.split('.')[0]}_frame"
262  logging.debug('ffmpeg_image_name: %s', ffmpeg_image_name)
263  ffmpeg_image_file_names = (
264      f'{os.path.join(log_path, ffmpeg_image_name)}_%04d.{img_format}')
265  cmd = [
266      'ffmpeg', '-i', os.path.join(log_path, video_file_name),
267      '-vsync', 'passthrough',  # prevents frame drops during decoding
268      ffmpeg_image_file_names, '-loglevel', 'quiet'
269  ]
270  _ = subprocess.call(cmd,
271                      stdin=subprocess.DEVNULL,
272                      stdout=subprocess.DEVNULL,
273                      stderr=subprocess.DEVNULL)
274
275  file_list = sorted(
276      [_ for _ in os.listdir(log_path) if (_.endswith(img_format)
277                                           and ffmpeg_image_name in _)])
278  if not file_list:
279    raise AssertionError('No frames extracted. Check source video.')
280
281  return file_list
282
283
284def extract_last_key_frame_from_recording(log_path, file_name):
285  """Extract last key frame from recordings.
286
287  Args:
288    log_path: str; file location
289    file_name: str file name for saved video
290
291  Returns:
292    numpy image of last key frame
293  """
294  key_frame_files = extract_key_frames_from_video(log_path, file_name)
295  logging.debug('key_frame_files: %s', key_frame_files)
296
297  # Get the last_key_frame file to process.
298  last_key_frame_file = get_key_frame_to_process(key_frame_files)
299  logging.debug('last_key_frame: %s', last_key_frame_file)
300
301  # Convert last_key_frame to numpy array
302  np_image = image_processing_utils.convert_image_to_numpy_array(
303      os.path.join(log_path, last_key_frame_file))
304  logging.debug('last key frame image shape: %s', np_image.shape)
305
306  return np_image
307
308
309def get_average_frame_rate(video_file_name_with_path):
310  """Get average frame rate assuming variable frame rate video.
311
312  Args:
313    video_file_name_with_path: path to the video to be analyzed
314  Returns:
315    Float. average frames per second.
316  """
317
318  cmd = ['ffprobe',
319         '-v',
320         'quiet',
321         '-show_streams',
322         '-select_streams',
323         'v:0',  # first video stream
324         video_file_name_with_path
325        ]
326  logging.debug('Getting frame rate')
327  raw_output = ''
328  try:
329    raw_output = subprocess.check_output(cmd,
330                                         stdin=subprocess.DEVNULL,
331                                         stderr=subprocess.STDOUT)
332  except subprocess.CalledProcessError as e:
333    raise AssertionError(str(e.output)) from e
334  if raw_output:
335    output = str(raw_output.decode('utf-8')).strip()
336    logging.debug('ffprobe command %s output: %s', ' '.join(cmd), output)
337    average_frame_rate_data = (
338        re.search(r'avg_frame_rate=*([0-9]+/[0-9]+)', output)
339        .group(INDEX_FIRST_SUBGROUP)
340    )
341    average_frame_rate = (int(average_frame_rate_data.split('/')[0]) /
342                          int(average_frame_rate_data.split('/')[1]))
343    logging.debug('Average FPS: %.4f', average_frame_rate)
344    return average_frame_rate
345  else:
346    raise AssertionError('ffprobe failed to provide frame rate data')
347
348
349def get_frame_deltas(video_file_name_with_path, timestamp_type='pts'):
350  """Get list of time diffs between frames.
351
352  Args:
353    video_file_name_with_path: path to the video to be analyzed
354    timestamp_type: 'pts' or 'dts'
355  Returns:
356    List of floats. Time diffs between frames in seconds.
357  """
358
359  cmd = ['ffprobe',
360         '-show_entries',
361         f'frame=pkt_{timestamp_type}_time',
362         '-select_streams',
363         'v',
364         video_file_name_with_path
365         ]
366  logging.debug('Getting frame deltas')
367  raw_output = ''
368  try:
369    raw_output = subprocess.check_output(cmd,
370                                         stdin=subprocess.DEVNULL,
371                                         stderr=subprocess.STDOUT)
372  except subprocess.CalledProcessError as e:
373    raise AssertionError(str(e.output)) from e
374  if raw_output:
375    output = str(raw_output.decode('utf-8')).strip().split('\n')
376    deltas = []
377    prev_time = None
378    for line in output:
379      if timestamp_type not in line:
380        continue
381      curr_time = float(re.search(r'time= *([0-9][0-9\.]*)', line)
382                        .group(INDEX_FIRST_SUBGROUP))
383      if prev_time is not None:
384        deltas.append(curr_time - prev_time)
385      prev_time = curr_time
386    logging.debug('Frame deltas: %s', deltas)
387    return deltas
388  else:
389    raise AssertionError('ffprobe failed to provide frame delta data')
390
391
392def get_video_colorspace(log_path, video_file_name):
393  """Get the video colorspace.
394
395  Args:
396    log_path: path for video file directory
397    video_file_name: name of the video file
398  Returns:
399    video colorspace, e.g. BT.2020 or BT.709
400  """
401
402  cmd = ['ffprobe',
403         '-show_streams',
404         '-select_streams',
405         'v:0',
406         '-of',
407         'json',
408         '-i',
409         os.path.join(log_path, video_file_name)
410         ]
411  logging.debug('Get the video colorspace')
412  raw_output = ''
413  try:
414    raw_output = subprocess.check_output(cmd,
415                                         stdin=subprocess.DEVNULL,
416                                         stderr=subprocess.STDOUT)
417  except subprocess.CalledProcessError as e:
418    raise AssertionError(str(e.output)) from e
419
420  logging.debug('raw_output: %s', raw_output)
421  if raw_output:
422    colorspace = ''
423    output = str(raw_output.decode('utf-8')).strip().split('\n')
424    logging.debug('output: %s', output)
425    for line in output:
426      logging.debug('line: %s', line)
427      metadata = re.search(r'"color_space": ("[a-z0-9]*")', line)
428      if metadata:
429        colorspace = metadata.group(INDEX_FIRST_SUBGROUP)
430    logging.debug('Colorspace: %s', colorspace)
431    return colorspace
432  else:
433    raise AssertionError('ffprobe failed to provide color space')
434