#1583 - Parsing base64 encoded image data from metadata

demos/main/src/main/assets/media.exolist.json

@@ -2,6 +2,11 @@
   {
     "name": "YouTube DASH",
     "samples": [
+      {
+        "name": "DVB Image sub",
+        "uri": "https://livesim.dashif.org/dash/vod/testpic_2s/img_subs.mpd",
+        "extension": "mpd"
+      },
       {
         "name": "Google Glass (MP4,H264)",
         "uri": "https://www.youtube.com/api/manifest/dash/id/bf5bb2419360daf1/source/youtube?as=fmp4_audio_clear,fmp4_sd_hd_clear&sparams=ip,ipbits,expire,source,id,as&ip=0.0.0.0&ipbits=0&expire=19000000000&signature=51AF5F39AB0CEC3E5497CD9C900EBFEAECCCB5C7.8506521BFC350652163895D4C26DEE124209AA9E&key=ik0",

library/core/src/main/java/com/google/android/exoplayer2/text/ttml/TtmlDecoder.java

@@ -68,6 +68,8 @@ public final class TtmlDecoder extends SimpleSubtitleDecoder {
   private static final String ATTR_END = "end";
   private static final String ATTR_STYLE = "style";
   private static final String ATTR_REGION = "region";
+  private static final String ATTR_IMAGE = "backgroundImage";
+
 
   private static final Pattern CLOCK_TIME =
       Pattern.compile("^([0-9][0-9]+):([0-9][0-9]):([0-9][0-9])"
@@ -105,6 +107,7 @@ public final class TtmlDecoder extends SimpleSubtitleDecoder {
       XmlPullParser xmlParser = xmlParserFactory.newPullParser();
       Map<String, TtmlStyle> globalStyles = new HashMap<>();
       Map<String, TtmlRegion> regionMap = new HashMap<>();
+      Map<String, String> imageMap = new HashMap<>();
       regionMap.put(TtmlNode.ANONYMOUS_REGION_ID, new TtmlRegion(null));
       ByteArrayInputStream inputStream = new ByteArrayInputStream(bytes, 0, length);
       xmlParser.setInput(inputStream, null);
@@ -127,7 +130,7 @@ public final class TtmlDecoder extends SimpleSubtitleDecoder {
               Log.i(TAG, "Ignoring unsupported tag: " + xmlParser.getName());
               unsupportedNodeDepth++;
             } else if (TtmlNode.TAG_HEAD.equals(name)) {
-              parseHeader(xmlParser, globalStyles, regionMap, cellResolution);
+              parseHeader(xmlParser, globalStyles, regionMap, cellResolution, imageMap);
             } else {
               try {
                 TtmlNode node = parseNode(xmlParser, parent, regionMap, frameAndTickRate);
@@ -145,7 +148,7 @@ public final class TtmlDecoder extends SimpleSubtitleDecoder {
             parent.addChild(TtmlNode.buildTextNode(xmlParser.getText()));
           } else if (eventType == XmlPullParser.END_TAG) {
             if (xmlParser.getName().equals(TtmlNode.TAG_TT)) {
-              ttmlSubtitle = new TtmlSubtitle(nodeStack.peek(), globalStyles, regionMap);
+              ttmlSubtitle = new TtmlSubtitle(nodeStack.peek(), globalStyles, regionMap, imageMap);
             }
             nodeStack.pop();
           }
@@ -230,7 +233,8 @@ public final class TtmlDecoder extends SimpleSubtitleDecoder {
       XmlPullParser xmlParser,
       Map<String, TtmlStyle> globalStyles,
       Map<String, TtmlRegion> globalRegions,
-      CellResolution cellResolution)
+      CellResolution cellResolution,
+      Map<String, String> imageMap)
       throws IOException, XmlPullParserException {
     do {
       xmlParser.next();
@@ -250,11 +254,29 @@ public final class TtmlDecoder extends SimpleSubtitleDecoder {
         if (ttmlRegion != null) {
           globalRegions.put(ttmlRegion.id, ttmlRegion);
         }
+      } else if(XmlPullParserUtil.isStartTag(xmlParser, TtmlNode.TAG_METADATA)){
+        parseMetaData(xmlParser, imageMap);
       }
+
     } while (!XmlPullParserUtil.isEndTag(xmlParser, TtmlNode.TAG_HEAD));
     return globalStyles;
   }
 
+  public void parseMetaData(XmlPullParser xmlParser, Map<String, String> imageMap) throws IOException, XmlPullParserException {
+    do {
+      xmlParser.next();
+      if (XmlPullParserUtil.isStartTag(xmlParser, TtmlNode.TAG_SMPTE_IMAGE)) {
+        for (int i = 0; i < xmlParser.getAttributeCount(); i++) {
+          String id = XmlPullParserUtil.getAttributeValue(xmlParser, "id");
+          if(id != null){
+            String base64 = xmlParser.nextText();
+            imageMap.put(id, base64);
+          }
+        }
+      }
+    } while (!XmlPullParserUtil.isEndTag(xmlParser, TtmlNode.TAG_METADATA));
+  }
+
   /**
    * Parses a region declaration.
    *
@@ -457,6 +479,7 @@ public final class TtmlDecoder extends SimpleSubtitleDecoder {
     long startTime = C.TIME_UNSET;
     long endTime = C.TIME_UNSET;
     String regionId = TtmlNode.ANONYMOUS_REGION_ID;
+    String imageId = "";
     String[] styleIds = null;
     int attributeCount = parser.getAttributeCount();
     TtmlStyle style = parseStyleAttributes(parser, null);
@@ -487,6 +510,9 @@ public final class TtmlDecoder extends SimpleSubtitleDecoder {
             regionId = value;
           }
           break;
+        case ATTR_IMAGE:
+          imageId = value.substring(1);
+          break;
         default:
           // Do nothing.
           break;
@@ -509,7 +535,7 @@ public final class TtmlDecoder extends SimpleSubtitleDecoder {
         endTime = parent.endTimeUs;
       }
     }
-    return TtmlNode.buildNode(parser.getName(), startTime, endTime, style, styleIds, regionId);
+    return TtmlNode.buildNode(parser.getName(), startTime, endTime, style, styleIds, regionId, imageId);
   }
 
   private static boolean isSupportedTag(String tag) {

library/core/src/main/java/com/google/android/exoplayer2/text/ttml/TtmlNode.java

@@ -15,10 +15,16 @@
  */
 package com.google.android.exoplayer2.text.ttml;
 
+import android.graphics.Bitmap;
+import android.graphics.BitmapFactory;
 import android.text.SpannableStringBuilder;
+import android.util.Base64;
+import android.util.Pair;
+
 import com.google.android.exoplayer2.C;
 import com.google.android.exoplayer2.text.Cue;
 import com.google.android.exoplayer2.util.Assertions;
+
 import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
@@ -44,9 +50,9 @@ import java.util.TreeSet;
   public static final String TAG_LAYOUT = "layout";
   public static final String TAG_REGION = "region";
   public static final String TAG_METADATA = "metadata";
-  public static final String TAG_SMPTE_IMAGE = "smpte:image";
+  public static final String TAG_SMPTE_IMAGE = "image";
-  public static final String TAG_SMPTE_DATA = "smpte:data";
+  public static final String TAG_SMPTE_DATA = "data";
-  public static final String TAG_SMPTE_INFORMATION = "smpte:information";
+  public static final String TAG_SMPTE_INFORMATION = "information";
 
   public static final String ANONYMOUS_REGION_ID = "";
   public static final String ATTR_ID = "id";
@@ -82,6 +88,7 @@ import java.util.TreeSet;
   public final long endTimeUs;
   public final TtmlStyle style;
   public final String regionId;
+  public final String imageId;
 
   private final String[] styleIds;
   private final HashMap<String, Integer> nodeStartsByRegion;
@@ -91,18 +98,19 @@ import java.util.TreeSet;
 
   public static TtmlNode buildTextNode(String text) {
     return new TtmlNode(null, TtmlRenderUtil.applyTextElementSpacePolicy(text), C.TIME_UNSET,
-        C.TIME_UNSET, null, null, ANONYMOUS_REGION_ID);
+        C.TIME_UNSET, null, null, ANONYMOUS_REGION_ID, null);
   }
 
   public static TtmlNode buildNode(String tag, long startTimeUs, long endTimeUs,
-      TtmlStyle style, String[] styleIds, String regionId) {
+                                   TtmlStyle style, String[] styleIds, String regionId, String imageId) {
-    return new TtmlNode(tag, null, startTimeUs, endTimeUs, style, styleIds, regionId);
+    return new TtmlNode(tag, null, startTimeUs, endTimeUs, style, styleIds, regionId, imageId);
   }
 
   private TtmlNode(String tag, String text, long startTimeUs, long endTimeUs,
-      TtmlStyle style, String[] styleIds, String regionId) {
+                   TtmlStyle style, String[] styleIds, String regionId, String imageId) {
     this.tag = tag;
     this.text = text;
+    this.imageId = imageId;
     this.style = style;
     this.styleIds = styleIds;
     this.isTextNode = text != null;
@@ -172,11 +180,37 @@ import java.util.TreeSet;
   }
 
   public List<Cue> getCues(long timeUs, Map<String, TtmlStyle> globalStyles,
-      Map<String, TtmlRegion> regionMap) {
+                           Map<String, TtmlRegion> regionMap, Map<String, String> imageMap) {
+
     TreeMap<String, SpannableStringBuilder> regionOutputs = new TreeMap<>();
+    List<Pair<String, String>> regionImageList = new ArrayList<>();
+
     traverseForText(timeUs, false, regionId, regionOutputs);
     traverseForStyle(timeUs, globalStyles, regionOutputs);
+    traverseForImage(timeUs, regionId, regionImageList);
+
     List<Cue> cues = new ArrayList<>();
+
+    // Create text based cues
+    for (Pair<String, String> regionImagePair : regionImageList) {
+      String base64 = imageMap.get(regionImagePair.second);
+      byte[] decodedString = Base64.decode(base64, Base64.DEFAULT);
+      Bitmap decodedByte = BitmapFactory.decodeByteArray(decodedString, 0, decodedString.length);
+      TtmlRegion region = regionMap.get(regionImagePair.first);
+
+      cues.add(
+          new Cue(decodedByte,
+              region.position,
+              Cue.TYPE_UNSET,
+              region.line,
+              region.lineAnchor,
+              region.width,
+              Cue.DIMEN_UNSET
+          )
+      );
+    }
+
+    // Create image based cues
     for (Entry<String, SpannableStringBuilder> entry : regionOutputs.entrySet()) {
       TtmlRegion region = regionMap.get(entry.getKey());
       cues.add(
@@ -195,6 +229,19 @@ import java.util.TreeSet;
     return cues;
   }
 
+  private void traverseForImage(long timeUs, String inheritedRegion, List<Pair<String, String>> regionImageList) {
+      // TODO isActive needed?
+
+    String resolvedRegionId = ANONYMOUS_REGION_ID.equals(regionId) ? inheritedRegion : regionId;
+    if (TAG_DIV.equals(tag) && imageId != null) {
+      regionImageList.add(new Pair<>(resolvedRegionId, imageId));
+    }
+
+    for (int i = 0; i < getChildCount(); ++i) {
+      getChild(i).traverseForImage(timeUs, resolvedRegionId, regionImageList);
+    }
+  }
+
   private void traverseForText(
       long timeUs,
       boolean descendsPNode,

library/core/src/main/java/com/google/android/exoplayer2/text/ttml/TtmlSubtitle.java

@@ -32,11 +32,14 @@ import java.util.Map;
   private final long[] eventTimesUs;
   private final Map<String, TtmlStyle> globalStyles;
   private final Map<String, TtmlRegion> regionMap;
+  private final Map<String, String> imageMap;
+
 
   public TtmlSubtitle(TtmlNode root, Map<String, TtmlStyle> globalStyles,
-      Map<String, TtmlRegion> regionMap) {
+      Map<String, TtmlRegion> regionMap, Map<String, String> imageMap) {
     this.root = root;
     this.regionMap = regionMap;
+    this.imageMap = imageMap;
     this.globalStyles =
         globalStyles != null ? Collections.unmodifiableMap(globalStyles) : Collections.emptyMap();
     this.eventTimesUs = root.getEventTimesUs();
@@ -65,7 +68,7 @@ import java.util.Map;
 
   @Override
   public List<Cue> getCues(long timeUs) {
-    return root.getCues(timeUs, globalStyles, regionMap);
+    return root.getCues(timeUs, globalStyles, regionMap, imageMap);
   }
 
   /* @VisibleForTesting */