1 /*
2  * Copyright (C) 2016 The Android Open Source Project
3  *
4  * Licensed under the Apache License, Version 2.0 (the "License");
5  * you may not use this file except in compliance with the License.
6  * You may obtain a copy of the License at
7  *
8  *      http://www.apache.org/licenses/LICENSE-2.0
9  *
10  * Unless required by applicable law or agreed to in writing, software
11  * distributed under the License is distributed on an "AS IS" BASIS,
12  * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13  * See the License for the specific language governing permissions and
14  * limitations under the License.
15  */
16 package com.google.android.exoplayer2.text.ttml;
17 
18 import android.graphics.Bitmap;
19 import android.graphics.BitmapFactory;
20 import android.text.SpannableStringBuilder;
21 import android.util.Base64;
22 import android.util.Pair;
23 import androidx.annotation.Nullable;
24 import com.google.android.exoplayer2.C;
25 import com.google.android.exoplayer2.text.Cue;
26 import com.google.android.exoplayer2.util.Assertions;
27 import java.util.ArrayList;
28 import java.util.HashMap;
29 import java.util.List;
30 import java.util.Map;
31 import java.util.TreeMap;
32 import java.util.TreeSet;
33 import org.checkerframework.checker.nullness.qual.MonotonicNonNull;
34 
35 /**
36  * A package internal representation of TTML node.
37  */
38 /* package */ final class TtmlNode {
39 
40   public static final String TAG_TT = "tt";
41   public static final String TAG_HEAD = "head";
42   public static final String TAG_BODY = "body";
43   public static final String TAG_DIV = "div";
44   public static final String TAG_P = "p";
45   public static final String TAG_SPAN = "span";
46   public static final String TAG_BR = "br";
47   public static final String TAG_STYLE = "style";
48   public static final String TAG_STYLING = "styling";
49   public static final String TAG_LAYOUT = "layout";
50   public static final String TAG_REGION = "region";
51   public static final String TAG_METADATA = "metadata";
52   public static final String TAG_IMAGE = "image";
53   public static final String TAG_DATA = "data";
54   public static final String TAG_INFORMATION = "information";
55 
56   public static final String ANONYMOUS_REGION_ID = "";
57   public static final String ATTR_ID = "id";
58   public static final String ATTR_TTS_ORIGIN = "origin";
59   public static final String ATTR_TTS_EXTENT = "extent";
60   public static final String ATTR_TTS_DISPLAY_ALIGN = "displayAlign";
61   public static final String ATTR_TTS_BACKGROUND_COLOR = "backgroundColor";
62   public static final String ATTR_TTS_FONT_STYLE = "fontStyle";
63   public static final String ATTR_TTS_FONT_SIZE = "fontSize";
64   public static final String ATTR_TTS_FONT_FAMILY = "fontFamily";
65   public static final String ATTR_TTS_FONT_WEIGHT = "fontWeight";
66   public static final String ATTR_TTS_COLOR = "color";
67   public static final String ATTR_TTS_RUBY = "ruby";
68   public static final String ATTR_TTS_RUBY_POSITION = "rubyPosition";
69   public static final String ATTR_TTS_TEXT_DECORATION = "textDecoration";
70   public static final String ATTR_TTS_TEXT_ALIGN = "textAlign";
71   public static final String ATTR_TTS_TEXT_COMBINE = "textCombine";
72   public static final String ATTR_TTS_WRITING_MODE = "writingMode";
73 
74   // Values for ruby
75   public static final String RUBY_CONTAINER = "container";
76   public static final String RUBY_BASE = "base";
77   public static final String RUBY_BASE_CONTAINER = "baseContainer";
78   public static final String RUBY_TEXT = "text";
79   public static final String RUBY_TEXT_CONTAINER = "textContainer";
80   public static final String RUBY_DELIMITER = "delimiter";
81 
82   // Values for rubyPosition
83   public static final String RUBY_BEFORE = "before";
84   public static final String RUBY_AFTER = "after";
85   // Values for textDecoration
86   public static final String LINETHROUGH = "linethrough";
87   public static final String NO_LINETHROUGH = "nolinethrough";
88   public static final String UNDERLINE = "underline";
89   public static final String NO_UNDERLINE = "nounderline";
90   public static final String ITALIC = "italic";
91   public static final String BOLD = "bold";
92 
93   // Values for textAlign
94   public static final String LEFT = "left";
95   public static final String CENTER = "center";
96   public static final String RIGHT = "right";
97   public static final String START = "start";
98   public static final String END = "end";
99 
100   // Values for textCombine
101   public static final String COMBINE_NONE = "none";
102   public static final String COMBINE_ALL = "all";
103 
104   // Values for writingMode
105   public static final String VERTICAL = "tb";
106   public static final String VERTICAL_LR = "tblr";
107   public static final String VERTICAL_RL = "tbrl";
108 
109   @Nullable public final String tag;
110   @Nullable public final String text;
111   public final boolean isTextNode;
112   public final long startTimeUs;
113   public final long endTimeUs;
114   @Nullable public final TtmlStyle style;
115   @Nullable private final String[] styleIds;
116   public final String regionId;
117   @Nullable public final String imageId;
118   @Nullable public final TtmlNode parent;
119 
120   private final HashMap<String, Integer> nodeStartsByRegion;
121   private final HashMap<String, Integer> nodeEndsByRegion;
122 
123   private @MonotonicNonNull List<TtmlNode> children;
124 
buildTextNode(String text)125   public static TtmlNode buildTextNode(String text) {
126     return new TtmlNode(
127         /* tag= */ null,
128         TtmlRenderUtil.applyTextElementSpacePolicy(text),
129         /* startTimeUs= */ C.TIME_UNSET,
130         /* endTimeUs= */ C.TIME_UNSET,
131         /* style= */ null,
132         /* styleIds= */ null,
133         ANONYMOUS_REGION_ID,
134         /* imageId= */ null,
135         /* parent= */ null);
136   }
137 
buildNode( @ullable String tag, long startTimeUs, long endTimeUs, @Nullable TtmlStyle style, @Nullable String[] styleIds, String regionId, @Nullable String imageId, @Nullable TtmlNode parent)138   public static TtmlNode buildNode(
139       @Nullable String tag,
140       long startTimeUs,
141       long endTimeUs,
142       @Nullable TtmlStyle style,
143       @Nullable String[] styleIds,
144       String regionId,
145       @Nullable String imageId,
146       @Nullable TtmlNode parent) {
147     return new TtmlNode(
148         tag, /* text= */ null, startTimeUs, endTimeUs, style, styleIds, regionId, imageId, parent);
149   }
150 
TtmlNode( @ullable String tag, @Nullable String text, long startTimeUs, long endTimeUs, @Nullable TtmlStyle style, @Nullable String[] styleIds, String regionId, @Nullable String imageId, @Nullable TtmlNode parent)151   private TtmlNode(
152       @Nullable String tag,
153       @Nullable String text,
154       long startTimeUs,
155       long endTimeUs,
156       @Nullable TtmlStyle style,
157       @Nullable String[] styleIds,
158       String regionId,
159       @Nullable String imageId,
160       @Nullable TtmlNode parent) {
161     this.tag = tag;
162     this.text = text;
163     this.imageId = imageId;
164     this.style = style;
165     this.styleIds = styleIds;
166     this.isTextNode = text != null;
167     this.startTimeUs = startTimeUs;
168     this.endTimeUs = endTimeUs;
169     this.regionId = Assertions.checkNotNull(regionId);
170     this.parent = parent;
171     nodeStartsByRegion = new HashMap<>();
172     nodeEndsByRegion = new HashMap<>();
173   }
174 
isActive(long timeUs)175   public boolean isActive(long timeUs) {
176     return (startTimeUs == C.TIME_UNSET && endTimeUs == C.TIME_UNSET)
177         || (startTimeUs <= timeUs && endTimeUs == C.TIME_UNSET)
178         || (startTimeUs == C.TIME_UNSET && timeUs < endTimeUs)
179         || (startTimeUs <= timeUs && timeUs < endTimeUs);
180   }
181 
addChild(TtmlNode child)182   public void addChild(TtmlNode child) {
183     if (children == null) {
184       children = new ArrayList<>();
185     }
186     children.add(child);
187   }
188 
getChild(int index)189   public TtmlNode getChild(int index) {
190     if (children == null) {
191       throw new IndexOutOfBoundsException();
192     }
193     return children.get(index);
194   }
195 
getChildCount()196   public int getChildCount() {
197     return children == null ? 0 : children.size();
198   }
199 
getEventTimesUs()200   public long[] getEventTimesUs() {
201     TreeSet<Long> eventTimeSet = new TreeSet<>();
202     getEventTimes(eventTimeSet, false);
203     long[] eventTimes = new long[eventTimeSet.size()];
204     int i = 0;
205     for (long eventTimeUs : eventTimeSet) {
206       eventTimes[i++] = eventTimeUs;
207     }
208     return eventTimes;
209   }
210 
getEventTimes(TreeSet<Long> out, boolean descendsPNode)211   private void getEventTimes(TreeSet<Long> out, boolean descendsPNode) {
212     boolean isPNode = TAG_P.equals(tag);
213     boolean isDivNode = TAG_DIV.equals(tag);
214     if (descendsPNode || isPNode || (isDivNode && imageId != null)) {
215       if (startTimeUs != C.TIME_UNSET) {
216         out.add(startTimeUs);
217       }
218       if (endTimeUs != C.TIME_UNSET) {
219         out.add(endTimeUs);
220       }
221     }
222     if (children == null) {
223       return;
224     }
225     for (int i = 0; i < children.size(); i++) {
226       children.get(i).getEventTimes(out, descendsPNode || isPNode);
227     }
228   }
229 
230   @Nullable
getStyleIds()231   public String[] getStyleIds() {
232     return styleIds;
233   }
234 
getCues( long timeUs, Map<String, TtmlStyle> globalStyles, Map<String, TtmlRegion> regionMap, Map<String, String> imageMap)235   public List<Cue> getCues(
236       long timeUs,
237       Map<String, TtmlStyle> globalStyles,
238       Map<String, TtmlRegion> regionMap,
239       Map<String, String> imageMap) {
240 
241     List<Pair<String, String>> regionImageOutputs = new ArrayList<>();
242     traverseForImage(timeUs, regionId, regionImageOutputs);
243 
244     TreeMap<String, Cue.Builder> regionTextOutputs = new TreeMap<>();
245     traverseForText(timeUs, false, regionId, regionTextOutputs);
246     traverseForStyle(timeUs, globalStyles, regionTextOutputs);
247 
248     List<Cue> cues = new ArrayList<>();
249 
250     // Create image based cues.
251     for (Pair<String, String> regionImagePair : regionImageOutputs) {
252       @Nullable String encodedBitmapData = imageMap.get(regionImagePair.second);
253       if (encodedBitmapData == null) {
254         // Image reference points to an invalid image. Do nothing.
255         continue;
256       }
257 
258       byte[] bitmapData = Base64.decode(encodedBitmapData, Base64.DEFAULT);
259       Bitmap bitmap = BitmapFactory.decodeByteArray(bitmapData, /* offset= */ 0, bitmapData.length);
260       TtmlRegion region = Assertions.checkNotNull(regionMap.get(regionImagePair.first));
261 
262       cues.add(
263           new Cue.Builder()
264               .setBitmap(bitmap)
265               .setPosition(region.position)
266               .setPositionAnchor(Cue.ANCHOR_TYPE_START)
267               .setLine(region.line, Cue.LINE_TYPE_FRACTION)
268               .setLineAnchor(region.lineAnchor)
269               .setSize(region.width)
270               .setBitmapHeight(region.height)
271               .build());
272     }
273 
274     // Create text based cues.
275     for (Map.Entry<String, Cue.Builder> entry : regionTextOutputs.entrySet()) {
276       TtmlRegion region = Assertions.checkNotNull(regionMap.get(entry.getKey()));
277       Cue.Builder regionOutput = entry.getValue();
278       cleanUpText((SpannableStringBuilder) Assertions.checkNotNull(regionOutput.getText()));
279       regionOutput.setLine(region.line, region.lineType);
280       regionOutput.setLineAnchor(region.lineAnchor);
281       regionOutput.setPosition(region.position);
282       regionOutput.setSize(region.width);
283       regionOutput.setTextSize(region.textSize, region.textSizeType);
284       cues.add(regionOutput.build());
285     }
286 
287     return cues;
288   }
289 
traverseForImage( long timeUs, String inheritedRegion, List<Pair<String, String>> regionImageList)290   private void traverseForImage(
291       long timeUs, String inheritedRegion, List<Pair<String, String>> regionImageList) {
292     String resolvedRegionId = ANONYMOUS_REGION_ID.equals(regionId) ? inheritedRegion : regionId;
293     if (isActive(timeUs) && TAG_DIV.equals(tag) && imageId != null) {
294       regionImageList.add(new Pair<>(resolvedRegionId, imageId));
295       return;
296     }
297     for (int i = 0; i < getChildCount(); ++i) {
298       getChild(i).traverseForImage(timeUs, resolvedRegionId, regionImageList);
299     }
300   }
301 
traverseForText( long timeUs, boolean descendsPNode, String inheritedRegion, Map<String, Cue.Builder> regionOutputs)302   private void traverseForText(
303       long timeUs,
304       boolean descendsPNode,
305       String inheritedRegion,
306       Map<String, Cue.Builder> regionOutputs) {
307     nodeStartsByRegion.clear();
308     nodeEndsByRegion.clear();
309     if (TAG_METADATA.equals(tag)) {
310       // Ignore metadata tag.
311       return;
312     }
313 
314     String resolvedRegionId = ANONYMOUS_REGION_ID.equals(regionId) ? inheritedRegion : regionId;
315 
316     if (isTextNode && descendsPNode) {
317       getRegionOutputText(resolvedRegionId, regionOutputs).append(Assertions.checkNotNull(text));
318     } else if (TAG_BR.equals(tag) && descendsPNode) {
319       getRegionOutputText(resolvedRegionId, regionOutputs).append('\n');
320     } else if (isActive(timeUs)) {
321       // This is a container node, which can contain zero or more children.
322       for (Map.Entry<String, Cue.Builder> entry : regionOutputs.entrySet()) {
323         nodeStartsByRegion.put(
324             entry.getKey(), Assertions.checkNotNull(entry.getValue().getText()).length());
325       }
326 
327       boolean isPNode = TAG_P.equals(tag);
328       for (int i = 0; i < getChildCount(); i++) {
329         getChild(i).traverseForText(timeUs, descendsPNode || isPNode, resolvedRegionId,
330             regionOutputs);
331       }
332       if (isPNode) {
333         TtmlRenderUtil.endParagraph(getRegionOutputText(resolvedRegionId, regionOutputs));
334       }
335 
336       for (Map.Entry<String, Cue.Builder> entry : regionOutputs.entrySet()) {
337         nodeEndsByRegion.put(
338             entry.getKey(), Assertions.checkNotNull(entry.getValue().getText()).length());
339       }
340     }
341   }
342 
getRegionOutputText( String resolvedRegionId, Map<String, Cue.Builder> regionOutputs)343   private static SpannableStringBuilder getRegionOutputText(
344       String resolvedRegionId, Map<String, Cue.Builder> regionOutputs) {
345     if (!regionOutputs.containsKey(resolvedRegionId)) {
346       Cue.Builder regionOutput = new Cue.Builder();
347       regionOutput.setText(new SpannableStringBuilder());
348       regionOutputs.put(resolvedRegionId, regionOutput);
349     }
350     return (SpannableStringBuilder)
351         Assertions.checkNotNull(regionOutputs.get(resolvedRegionId).getText());
352   }
353 
traverseForStyle( long timeUs, Map<String, TtmlStyle> globalStyles, Map<String, Cue.Builder> regionOutputs)354   private void traverseForStyle(
355       long timeUs, Map<String, TtmlStyle> globalStyles, Map<String, Cue.Builder> regionOutputs) {
356     if (!isActive(timeUs)) {
357       return;
358     }
359     for (Map.Entry<String, Integer> entry : nodeEndsByRegion.entrySet()) {
360       String regionId = entry.getKey();
361       int start = nodeStartsByRegion.containsKey(regionId) ? nodeStartsByRegion.get(regionId) : 0;
362       int end = entry.getValue();
363       if (start != end) {
364         Cue.Builder regionOutput = Assertions.checkNotNull(regionOutputs.get(regionId));
365         applyStyleToOutput(globalStyles, regionOutput, start, end);
366       }
367     }
368     for (int i = 0; i < getChildCount(); ++i) {
369       getChild(i).traverseForStyle(timeUs, globalStyles, regionOutputs);
370     }
371   }
372 
applyStyleToOutput( Map<String, TtmlStyle> globalStyles, Cue.Builder regionOutput, int start, int end)373   private void applyStyleToOutput(
374       Map<String, TtmlStyle> globalStyles, Cue.Builder regionOutput, int start, int end) {
375     @Nullable TtmlStyle resolvedStyle = TtmlRenderUtil.resolveStyle(style, styleIds, globalStyles);
376     @Nullable SpannableStringBuilder text = (SpannableStringBuilder) regionOutput.getText();
377     if (text == null) {
378       text = new SpannableStringBuilder();
379       regionOutput.setText(text);
380     }
381     if (resolvedStyle != null) {
382       TtmlRenderUtil.applyStylesToSpan(text, start, end, resolvedStyle, parent);
383       regionOutput.setVerticalType(resolvedStyle.getVerticalType());
384     }
385   }
386 
cleanUpText(SpannableStringBuilder builder)387   private static void cleanUpText(SpannableStringBuilder builder) {
388     // Having joined the text elements, we need to do some final cleanup on the result.
389     // Remove any text covered by a DeleteTextSpan (e.g. ruby text).
390     DeleteTextSpan[] deleteTextSpans = builder.getSpans(0, builder.length(), DeleteTextSpan.class);
391     for (DeleteTextSpan deleteTextSpan : deleteTextSpans) {
392       builder.replace(builder.getSpanStart(deleteTextSpan), builder.getSpanEnd(deleteTextSpan), "");
393     }
394     // Collapse multiple consecutive spaces into a single space.
395     for (int i = 0; i < builder.length(); i++) {
396       if (builder.charAt(i) == ' ') {
397         int j = i + 1;
398         while (j < builder.length() && builder.charAt(j) == ' ') {
399           j++;
400         }
401         int spacesToDelete = j - (i + 1);
402         if (spacesToDelete > 0) {
403           builder.delete(i, i + spacesToDelete);
404         }
405       }
406     }
407     // Remove any spaces from the start of each line.
408     if (builder.length() > 0 && builder.charAt(0) == ' ') {
409       builder.delete(0, 1);
410     }
411     for (int i = 0; i < builder.length() - 1; i++) {
412       if (builder.charAt(i) == '\n' && builder.charAt(i + 1) == ' ') {
413         builder.delete(i + 1, i + 2);
414       }
415     }
416     // Remove any spaces from the end of each line.
417     if (builder.length() > 0 && builder.charAt(builder.length() - 1) == ' ') {
418       builder.delete(builder.length() - 1, builder.length());
419     }
420     for (int i = 0; i < builder.length() - 1; i++) {
421       if (builder.charAt(i) == ' ' && builder.charAt(i + 1) == '\n') {
422         builder.delete(i, i + 1);
423       }
424     }
425     // Trim a trailing newline, if there is one.
426     if (builder.length() > 0 && builder.charAt(builder.length() - 1) == '\n') {
427       builder.delete(builder.length() - 1, builder.length());
428     }
429   }
430 
431 }
432