Skip to content

Commit

Permalink
google#1583 - Parsing base64 encoded image data from metadata
Browse files Browse the repository at this point in the history
  • Loading branch information
@szaboa
szaboa committed Nov 6, 2018
1 parent 3479765 commit 254a586
Show file tree
Hide file tree
Showing 4 changed files with 95 additions and 14 deletions.
5 changes: 5 additions & 0 deletions demos/main/src/main/assets/media.exolist.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,11 @@
{
"name": "YouTube DASH",
"samples": [
{
"name": "DVB Image sub",
"uri": "https://livesim.dashif.org/dash/vod/testpic_2s/img_subs.mpd",
"extension": "mpd"
},
{
"name": "Google Glass (MP4,H264)",
"uri": "https://www.youtube.com/api/manifest/dash/id/bf5bb2419360daf1/source/youtube?as=fmp4_audio_clear,fmp4_sd_hd_clear&sparams=ip,ipbits,expire,source,id,as&ip=0.0.0.0&ipbits=0&expire=19000000000&signature=51AF5F39AB0CEC3E5497CD9C900EBFEAECCCB5C7.8506521BFC350652163895D4C26DEE124209AA9E&key=ik0",
Expand Down
34 changes: 30 additions & 4 deletions library/core/src/main/java/com/google/android/exoplayer2/text/ttml/TtmlDecoder.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,8 @@ public final class TtmlDecoder extends SimpleSubtitleDecoder {
private static final String ATTR_END = "end";
private static final String ATTR_STYLE = "style";
private static final String ATTR_REGION = "region";
private static final String ATTR_IMAGE = "backgroundImage";


private static final Pattern CLOCK_TIME =
Pattern.compile("^([0-9][0-9]+):([0-9][0-9]):([0-9][0-9])"
Expand Down Expand Up @@ -105,6 +107,7 @@ protected TtmlSubtitle decode(byte[] bytes, int length, boolean reset)
XmlPullParser xmlParser = xmlParserFactory.newPullParser();
Map<String, TtmlStyle> globalStyles = new HashMap<>();
Map<String, TtmlRegion> regionMap = new HashMap<>();
Map<String, String> imageMap = new HashMap<>();
regionMap.put(TtmlNode.ANONYMOUS_REGION_ID, new TtmlRegion(null));
ByteArrayInputStream inputStream = new ByteArrayInputStream(bytes, 0, length);
xmlParser.setInput(inputStream, null);
Expand All @@ -127,7 +130,7 @@ protected TtmlSubtitle decode(byte[] bytes, int length, boolean reset)
Log.i(TAG, "Ignoring unsupported tag: " + xmlParser.getName());
unsupportedNodeDepth++;
} else if (TtmlNode.TAG_HEAD.equals(name)) {
parseHeader(xmlParser, globalStyles, regionMap, cellResolution);
parseHeader(xmlParser, globalStyles, regionMap, cellResolution, imageMap);
} else {
try {
TtmlNode node = parseNode(xmlParser, parent, regionMap, frameAndTickRate);
Expand All @@ -145,7 +148,7 @@ protected TtmlSubtitle decode(byte[] bytes, int length, boolean reset)
parent.addChild(TtmlNode.buildTextNode(xmlParser.getText()));
} else if (eventType == XmlPullParser.END_TAG) {
if (xmlParser.getName().equals(TtmlNode.TAG_TT)) {
ttmlSubtitle = new TtmlSubtitle(nodeStack.peek(), globalStyles, regionMap);
ttmlSubtitle = new TtmlSubtitle(nodeStack.peek(), globalStyles, regionMap, imageMap);
}
nodeStack.pop();
}
Expand Down Expand Up @@ -230,7 +233,8 @@ private Map<String, TtmlStyle> parseHeader(
XmlPullParser xmlParser,
Map<String, TtmlStyle> globalStyles,
Map<String, TtmlRegion> globalRegions,
CellResolution cellResolution)
CellResolution cellResolution,
Map<String, String> imageMap)
throws IOException, XmlPullParserException {
do {
xmlParser.next();
Expand All @@ -250,11 +254,29 @@ private Map<String, TtmlStyle> parseHeader(
if (ttmlRegion != null) {
globalRegions.put(ttmlRegion.id, ttmlRegion);
}
} else if(XmlPullParserUtil.isStartTag(xmlParser, TtmlNode.TAG_METADATA)){
parseMetaData(xmlParser, imageMap);
}

} while (!XmlPullParserUtil.isEndTag(xmlParser, TtmlNode.TAG_HEAD));
return globalStyles;
}

public void parseMetaData(XmlPullParser xmlParser, Map<String, String> imageMap) throws IOException, XmlPullParserException {
do {
xmlParser.next();
if (XmlPullParserUtil.isStartTag(xmlParser, TtmlNode.TAG_SMPTE_IMAGE)) {
for (int i = 0; i < xmlParser.getAttributeCount(); i++) {
String id = XmlPullParserUtil.getAttributeValue(xmlParser, "id");
if(id != null){
String base64 = xmlParser.nextText();
imageMap.put(id, base64);
}
}
}
} while (!XmlPullParserUtil.isEndTag(xmlParser, TtmlNode.TAG_METADATA));
}

/**
* Parses a region declaration.
*
Expand Down Expand Up @@ -457,6 +479,7 @@ private TtmlNode parseNode(XmlPullParser parser, TtmlNode parent,
long startTime = C.TIME_UNSET;
long endTime = C.TIME_UNSET;
String regionId = TtmlNode.ANONYMOUS_REGION_ID;
String imageId = "";
String[] styleIds = null;
int attributeCount = parser.getAttributeCount();
TtmlStyle style = parseStyleAttributes(parser, null);
Expand Down Expand Up @@ -487,6 +510,9 @@ private TtmlNode parseNode(XmlPullParser parser, TtmlNode parent,
regionId = value;
}
break;
case ATTR_IMAGE:
imageId = value.substring(1);
break;
default:
// Do nothing.
break;
Expand All @@ -509,7 +535,7 @@ private TtmlNode parseNode(XmlPullParser parser, TtmlNode parent,
endTime = parent.endTimeUs;
}
}
return TtmlNode.buildNode(parser.getName(), startTime, endTime, style, styleIds, regionId);
return TtmlNode.buildNode(parser.getName(), startTime, endTime, style, styleIds, regionId, imageId);
}

private static boolean isSupportedTag(String tag) {
Expand Down
63 changes: 55 additions & 8 deletions library/core/src/main/java/com/google/android/exoplayer2/text/ttml/TtmlNode.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,16 @@
*/
package com.google.android.exoplayer2.text.ttml;

import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.text.SpannableStringBuilder;
import android.util.Base64;
import android.util.Pair;

import com.google.android.exoplayer2.C;
import com.google.android.exoplayer2.text.Cue;
import com.google.android.exoplayer2.util.Assertions;

import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
Expand All @@ -44,9 +50,9 @@
public static final String TAG_LAYOUT = "layout";
public static final String TAG_REGION = "region";
public static final String TAG_METADATA = "metadata";
public static final String TAG_SMPTE_IMAGE = "smpte:image";
public static final String TAG_SMPTE_DATA = "smpte:data";
public static final String TAG_SMPTE_INFORMATION = "smpte:information";
public static final String TAG_SMPTE_IMAGE = "image";
public static final String TAG_SMPTE_DATA = "data";
public static final String TAG_SMPTE_INFORMATION = "information";

public static final String ANONYMOUS_REGION_ID = "";
public static final String ATTR_ID = "id";
Expand Down Expand Up @@ -82,6 +88,7 @@
public final long endTimeUs;
public final TtmlStyle style;
public final String regionId;
public final String imageId;

private final String[] styleIds;
private final HashMap<String, Integer> nodeStartsByRegion;
Expand All @@ -91,18 +98,19 @@

public static TtmlNode buildTextNode(String text) {
return new TtmlNode(null, TtmlRenderUtil.applyTextElementSpacePolicy(text), C.TIME_UNSET,
C.TIME_UNSET, null, null, ANONYMOUS_REGION_ID);
C.TIME_UNSET, null, null, ANONYMOUS_REGION_ID, null);
}

public static TtmlNode buildNode(String tag, long startTimeUs, long endTimeUs,
TtmlStyle style, String[] styleIds, String regionId) {
return new TtmlNode(tag, null, startTimeUs, endTimeUs, style, styleIds, regionId);
TtmlStyle style, String[] styleIds, String regionId, String imageId) {
return new TtmlNode(tag, null, startTimeUs, endTimeUs, style, styleIds, regionId, imageId);
}

private TtmlNode(String tag, String text, long startTimeUs, long endTimeUs,
TtmlStyle style, String[] styleIds, String regionId) {
TtmlStyle style, String[] styleIds, String regionId, String imageId) {
this.tag = tag;
this.text = text;
this.imageId = imageId;
this.style = style;
this.styleIds = styleIds;
this.isTextNode = text != null;
Expand Down Expand Up @@ -172,11 +180,37 @@ public String[] getStyleIds() {
}

public List<Cue> getCues(long timeUs, Map<String, TtmlStyle> globalStyles,
Map<String, TtmlRegion> regionMap) {
Map<String, TtmlRegion> regionMap, Map<String, String> imageMap) {

TreeMap<String, SpannableStringBuilder> regionOutputs = new TreeMap<>();
List<Pair<String, String>> regionImageList = new ArrayList<>();

traverseForText(timeUs, false, regionId, regionOutputs);
traverseForStyle(timeUs, globalStyles, regionOutputs);
traverseForImage(timeUs, regionId, regionImageList);

List<Cue> cues = new ArrayList<>();

// Create text based cues
for (Pair<String, String> regionImagePair : regionImageList) {
String base64 = imageMap.get(regionImagePair.second);
byte[] decodedString = Base64.decode(base64, Base64.DEFAULT);
Bitmap decodedByte = BitmapFactory.decodeByteArray(decodedString, 0, decodedString.length);
TtmlRegion region = regionMap.get(regionImagePair.first);

cues.add(
new Cue(decodedByte,
region.position,
Cue.TYPE_UNSET,
region.line,
region.lineAnchor,
region.width,
Cue.DIMEN_UNSET
)
);
}

// Create image based cues
for (Entry<String, SpannableStringBuilder> entry : regionOutputs.entrySet()) {
TtmlRegion region = regionMap.get(entry.getKey());
cues.add(
Expand All @@ -195,6 +229,19 @@ public List<Cue> getCues(long timeUs, Map<String, TtmlStyle> globalStyles,
return cues;
}

private void traverseForImage(long timeUs, String inheritedRegion, List<Pair<String, String>> regionImageList) {
// TODO isActive needed?

String resolvedRegionId = ANONYMOUS_REGION_ID.equals(regionId) ? inheritedRegion : regionId;
if (TAG_DIV.equals(tag) && imageId != null) {
regionImageList.add(new Pair<>(resolvedRegionId, imageId));
}

for (int i = 0; i < getChildCount(); ++i) {
getChild(i).traverseForImage(timeUs, resolvedRegionId, regionImageList);
}
}

private void traverseForText(
long timeUs,
boolean descendsPNode,
Expand Down
7 changes: 5 additions & 2 deletions library/core/src/main/java/com/google/android/exoplayer2/text/ttml/TtmlSubtitle.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,14 @@
private final long[] eventTimesUs;
private final Map<String, TtmlStyle> globalStyles;
private final Map<String, TtmlRegion> regionMap;
private final Map<String, String> imageMap;


public TtmlSubtitle(TtmlNode root, Map<String, TtmlStyle> globalStyles,
Map<String, TtmlRegion> regionMap) {
Map<String, TtmlRegion> regionMap, Map<String, String> imageMap) {
this.root = root;
this.regionMap = regionMap;
this.imageMap = imageMap;
this.globalStyles =
globalStyles != null ? Collections.unmodifiableMap(globalStyles) : Collections.emptyMap();
this.eventTimesUs = root.getEventTimesUs();
Expand Down Expand Up @@ -65,7 +68,7 @@ public long getEventTime(int index) {

@Override
public List<Cue> getCues(long timeUs) {
return root.getCues(timeUs, globalStyles, regionMap);
return root.getCues(timeUs, globalStyles, regionMap, imageMap);
}

/* @VisibleForTesting */
Expand Down

0 comments on commit 254a586

Please sign in to comment.