1/* Copyright 2020 The TensorFlow Authors. All Rights Reserved. 2 3Licensed under the Apache License, Version 2.0 (the "License"); 4you may not use this file except in compliance with the License. 5You may obtain a copy of the License at 6 7 http://www.apache.org/licenses/LICENSE-2.0 8 9Unless required by applicable law or agreed to in writing, software 10distributed under the License is distributed on an "AS IS" BASIS, 11WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12See the License for the specific language governing permissions and 13limitations under the License. 14==============================================================================*/ 15 16syntax = "proto2"; 17 18package tflite.task.vision; 19 20// Results of performing image segmentation. 21// Note that at the time, a single `Segmentation` element is expected to be 22// returned; the field is made repeated for later extension to e.g. instance 23// segmentation models, which may return one segmentation per object. 24message SegmentationResult { 25 repeated Segmentation segmentation = 1; 26} 27 28// Next Id: 6 29message Segmentation { 30 // Confidence mask. This is a flattened 2D-array in row major order. For each 31 // pixel, the value indicates the prediction confidence usually in the [0, 1] 32 // range where higher values represent a stronger confidence. Ultimately this 33 // is model specific, and other range of values might be used. 34 message ConfidenceMask { 35 repeated float value = 1 [packed = true]; 36 } 37 38 // List of confidence masks with respect to the model output depth (this depth 39 // represents how many classes are supported). Note: some models have a single 40 // class (e.g. a sky segmentation model) which turns into a single confidence 41 // mask in this list. 42 message ConfidenceMasks { 43 repeated ConfidenceMask confidence_mask = 1; 44 } 45 46 // IMPORTANT: segmentation masks are not direcly suited for display, in 47 // particular: 48 // * they are relative to the unrotated input frame, i.e. *not* taking into 49 // account the `Orientation` flag of the input FrameBuffer, 50 // * their dimensions are intrinsic to the model, i.e. *not* dependent on the 51 // input FrameBuffer dimensions. 52 // 53 // Example of such post-processing, assuming: 54 // * an input FrameBuffer with width=640, height=480, orientation=kLeftBottom 55 // (i.e. the image will be rotated 90° clockwise during preprocessing to 56 // make it "upright"), 57 // * a model outputting masks of size 224x224. 58 // In order to be directly displayable on top of the input image assumed to 59 // be displayed *with* the `Orientation` flag taken into account (according to 60 // the EXIF specification [1]), the masks need to be: 61 // * re-scaled to 640 x 480, 62 // * then rotated 90° clockwise. 63 // 64 // [1]: http://jpegclub.org/exif_orientation.html 65 oneof mask_oneof { 66 // Category mask. This is a flattened 2D-array of size `width` x `height`, 67 // in row major order. The value of each pixel in this mask represents the 68 // class to which the pixel belongs. 69 // See `colored_labels` for instructions on how to get pixel labels and 70 // display color. 71 bytes category_mask = 1; 72 73 // One confidence masks of size `width` x `height` for each of the supported 74 // classes. The value of each pixel in these masks represents the confidence 75 // score for this particular class. 76 // See `colored_labels` for instructions on how to get pixel labels and 77 // display color. 78 ConfidenceMasks confidence_masks = 4; 79 } 80 // The width of the mask. This is an intrinsic parameter of the model being 81 // used, and does not depend on the input image dimensions. 82 optional int32 width = 2; 83 // The height of the mask. This is an intrinsic parameter of the model being 84 // used, and does not depend on the input image dimensions. 85 optional int32 height = 3; 86 87 // Defines a label associated with an RGB color, for display purposes. 88 message ColoredLabel { 89 // The RGB color components for the label, in the [0, 255] range. 90 optional uint32 r = 1; 91 optional uint32 g = 2; 92 optional uint32 b = 3; 93 // The class name, as provided in the label map packed in the TFLite Model 94 // Metadata. 95 optional string class_name = 4; 96 // The display name, as provided in the label map (if available) packed in 97 // the TFLite Model Metadata. See `display_names_locale` field in 98 // ImageSegmenterOptions. 99 optional string display_name = 5; 100 } 101 102 // The list of colored labels for all the supported categories. Depending on 103 // which is present, this list is in 1:1 correspondence with: 104 // * `category_mask` pixel values, i.e. a pixel with value `i` is 105 // associated with `colored_labels[i]`, 106 // * `confidence_masks` indices, i.e. `confidence_masks[i]` is associated with 107 // `colored_labels[i]`. 108 repeated ColoredLabel colored_labels = 5; 109} 110