1 /* Copyright 2019 The TensorFlow Authors. All Rights Reserved.
2 
3 Licensed under the Apache License, Version 2.0 (the "License");
4 you may not use this file except in compliance with the License.
5 You may obtain a copy of the License at
6 
7     http://www.apache.org/licenses/LICENSE-2.0
8 
9 Unless required by applicable law or agreed to in writing, software
10 distributed under the License is distributed on an "AS IS" BASIS,
11 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12 See the License for the specific language governing permissions and
13 limitations under the License.
14 ==============================================================================*/
15 
16 package org.tensorflow.lite.support.image;
17 
18 import android.graphics.PointF;
19 import android.graphics.RectF;
20 import java.util.ArrayList;
21 import java.util.List;
22 import java.util.ListIterator;
23 import org.tensorflow.lite.support.common.Operator;
24 import org.tensorflow.lite.support.common.SequentialProcessor;
25 import org.tensorflow.lite.support.common.SupportPreconditions;
26 import org.tensorflow.lite.support.common.TensorOperator;
27 import org.tensorflow.lite.support.image.ops.Rot90Op;
28 import org.tensorflow.lite.support.image.ops.TensorOperatorWrapper;
29 
30 /**
31  * ImageProcessor is a helper class for preprocessing and postprocessing {@link TensorImage}. It
32  * could transform a {@link TensorImage} to another by executing a chain of {@link ImageOperator}.
33  *
34  * <p>Example Usage:
35  *
36  * <pre>
37  *   ImageProcessor processor = new ImageProcessor.Builder()
38  *       .add(new ResizeOp(224, 224, ResizeMethod.NEAREST_NEIGHBOR)
39  *       .add(new Rot90Op())
40  *       .add(new NormalizeOp(127.5f, 127.5f))
41  *       .build();
42  *   TensorImage anotherTensorImage = processor.process(tensorImage);
43  * </pre>
44  *
45  * <p><b>WARNING:</b> Instances of an {@code ImageProcessor} are <b>not</b> thread-safe with {@link
46  * #updateNumberOfRotations}. Updating the number of rotations and then processing images (using
47  * {@link #process}) must be protected from concurrent access. It is recommended to create separate
48  * {@code ImageProcessor} instances for each thread. If multiple threads access a {@code
49  * ImageProcessor} concurrently, it must be synchronized externally.
50  *
51  * @see ImageProcessor.Builder to build a {@link ImageProcessor} instance
52  * @see ImageProcessor#process(TensorImage) to apply the processor on a {@link TensorImage}
53  */
54 public class ImageProcessor extends SequentialProcessor<TensorImage> {
ImageProcessor(Builder builder)55   private ImageProcessor(Builder builder) {
56     super(builder);
57   }
58 
59   /**
60    * Transforms a point from coordinates system of the result image back to the one of the input
61    * image.
62    *
63    * @param point the point from the result coordinates system.
64    * @param inputImageHeight the height of input image.
65    * @param inputImageWidth the width of input image.
66    * @return the point with the coordinates from the coordinates system of the input image.
67    */
inverseTransform(PointF point, int inputImageHeight, int inputImageWidth)68   public PointF inverseTransform(PointF point, int inputImageHeight, int inputImageWidth) {
69     List<Integer> widths = new ArrayList<>();
70     List<Integer> heights = new ArrayList<>();
71     int currentWidth = inputImageWidth;
72     int currentHeight = inputImageHeight;
73     for (Operator<TensorImage> op : operatorList) {
74       widths.add(currentWidth);
75       heights.add(currentHeight);
76       ImageOperator imageOperator = (ImageOperator) op;
77       int newHeight = imageOperator.getOutputImageHeight(currentHeight, currentWidth);
78       int newWidth = imageOperator.getOutputImageWidth(currentHeight, currentWidth);
79       currentHeight = newHeight;
80       currentWidth = newWidth;
81     }
82     ListIterator<Operator<TensorImage>> opIterator = operatorList.listIterator(operatorList.size());
83     ListIterator<Integer> widthIterator = widths.listIterator(widths.size());
84     ListIterator<Integer> heightIterator = heights.listIterator(heights.size());
85     while (opIterator.hasPrevious()) {
86       ImageOperator imageOperator = (ImageOperator) opIterator.previous();
87       int height = heightIterator.previous();
88       int width = widthIterator.previous();
89       point = imageOperator.inverseTransform(point, height, width);
90     }
91     return point;
92   }
93 
94   /**
95    * Transforms a rectangle from coordinates system of the result image back to the one of the input
96    * image.
97    *
98    * @param rect the rectangle from the result coordinates system.
99    * @param inputImageHeight the height of input image.
100    * @param inputImageWidth the width of input image.
101    * @return the rectangle with the coordinates from the coordinates system of the input image.
102    */
inverseTransform(RectF rect, int inputImageHeight, int inputImageWidth)103   public RectF inverseTransform(RectF rect, int inputImageHeight, int inputImageWidth) {
104     // when rotation is involved, corner order may change - top left changes to bottom right, .etc
105     PointF p1 =
106         inverseTransform(new PointF(rect.left, rect.top), inputImageHeight, inputImageWidth);
107     PointF p2 =
108         inverseTransform(new PointF(rect.right, rect.bottom), inputImageHeight, inputImageWidth);
109     return new RectF(
110         Math.min(p1.x, p2.x), Math.min(p1.y, p2.y), Math.max(p1.x, p2.x), Math.max(p1.y, p2.y));
111   }
112 
113   /**
114    * The Builder to create an ImageProcessor, which could be executed later.
115    *
116    * @see #add(TensorOperator) to add a general TensorOperator
117    * @see #add(ImageOperator) to add an ImageOperator
118    * @see #build() complete the building process and get a built Processor
119    */
120   public static class Builder extends SequentialProcessor.Builder<TensorImage> {
Builder()121     public Builder() {
122       super();
123     }
124 
125     /**
126      * Adds an {@link ImageOperator} into the Operator chain.
127      *
128      * @param op the Operator instance to be executed then
129      */
add(ImageOperator op)130     public Builder add(ImageOperator op) {
131       super.add(op);
132       return this;
133     }
134 
135     /**
136      * Adds a {@link TensorOperator} into the Operator chain. In execution, the processor calls
137      * {@link TensorImage#getTensorBuffer()} to transform the {@link TensorImage} by transforming
138      * the underlying {@link org.tensorflow.lite.support.tensorbuffer.TensorBuffer}.
139      *
140      * @param op the Operator instance to be executed then
141      */
add(TensorOperator op)142     public Builder add(TensorOperator op) {
143       return add(new TensorOperatorWrapper(op));
144     }
145 
146     /** Completes the building process and gets the {@link ImageProcessor} instance. */
147     @Override
build()148     public ImageProcessor build() {
149       return new ImageProcessor(this);
150     }
151   }
152 
153   /**
154    * Updates the number of rotations for the first {@link Rot90Op} in this {@link ImageProcessor}.
155    *
156    * <p><b>WARNING:</b>this method is <b>not</b> thread-safe. Updating the number of rotations and
157    * then processing images (using {@link #process}) must be protected from concurrent access with
158    * additional synchronization.
159    *
160    * @param k the number of rotations
161    * @throws IllegalStateException if {@link Rot90Op} has not been added to this {@link
162    *     ImageProcessor}
163    */
updateNumberOfRotations(int k)164   public void updateNumberOfRotations(int k) {
165     updateNumberOfRotations(k, /*occurrence=*/ 0);
166   }
167 
168   /**
169    * Updates the number of rotations for the {@link Rot90Op} specified by {@code occurrence} in this
170    * {@link ImageProcessor}.
171    *
172    * <p><b>WARNING:</b>this method is <b>not</b> thread-safe. Updating the number of rotations and
173    * then processing images (using {@link #process}) must be protected from concurrent access with
174    * additional synchronization.
175    *
176    * @param k the number of rotations
177    * @param occurrence the index of perticular {@link Rot90Op} in this {@link ImageProcessor}. For
178    *     example, if the second {@link Rot90Op} needs to be updated, {@code occurrence} should be
179    *     set to 1.
180    * @throws IndexOutOfBoundsException if {@code occurrence} is negative or is not less than the
181    *     number of {@link Rot90Op} in this {@link ImageProcessor}
182    * @throws IllegalStateException if {@link Rot90Op} has not been added to this {@link
183    *     ImageProcessor}
184    */
updateNumberOfRotations(int k, int occurrence)185   public synchronized void updateNumberOfRotations(int k, int occurrence) {
186     SupportPreconditions.checkState(
187         operatorIndex.containsKey(Rot90Op.class.getName()),
188         "The Rot90Op has not been added to the ImageProcessor.");
189 
190     List<Integer> indexes = operatorIndex.get(Rot90Op.class.getName());
191     SupportPreconditions.checkElementIndex(occurrence, indexes.size(), "occurrence");
192 
193     // The index of the Rot90Op to be replaced in operatorList.
194     int index = indexes.get(occurrence);
195     Rot90Op newRot = new Rot90Op(k);
196     operatorList.set(index, newRot);
197   }
198 }
199