YARN-1462. Made RM write application tags to timeline server and expo…

…sed them to users via generic history web UI and REST API. Contributed by Xuan Gong.

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/main/java/org/apache/hadoop/mapred/NotRunningJob.java

@@ -90,7 +90,7 @@ private ApplicationReport getUnknownApplicationReport() {
  return ApplicationReport.newInstance(unknownAppId, unknownAttemptId,
  "N/A", "N/A", "N/A", "N/A", 0, null, YarnApplicationState.NEW, "N/A",
  "N/A", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f,
- YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
+ YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null);
  }
 
  NotRunningJob(ApplicationReport applicationReport, JobState jobState) {

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestClientServiceDelegate.java

@@ -516,7 +516,7 @@ private ApplicationReport getFinishedApplicationReport() {
  return ApplicationReport.newInstance(appId, attemptId, "user", "queue",
  "appname", "host", 124, null, YarnApplicationState.FINISHED,
  "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null,
- "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
+ "N/A", 0.0f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null);
  }
 
  private ApplicationReport getRunningApplicationReport(String host, int port) {
@@ -526,7 +526,7 @@ private ApplicationReport getRunningApplicationReport(String host, int port) {
  return ApplicationReport.newInstance(appId, attemptId, "user", "queue",
  "appname", host, port, null, YarnApplicationState.RUNNING, "diagnostics",
  "url", 0, 0, FinalApplicationStatus.UNDEFINED, null, "N/A", 0.0f,
- YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
+ YarnConfiguration.DEFAULT_APPLICATION_TYPE, null, null);
  }
 
  private ResourceMgrDelegate getRMDelegate() throws IOException {

hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-jobclient/src/test/java/org/apache/hadoop/mapred/TestYARNRunner.java

@@ -196,7 +196,7 @@ public ClientServiceDelegate answer(InvocationOnMock invocation)
  ApplicationReport.newInstance(appId, null, "tmp", "tmp", "tmp",
  "tmp", 0, null, YarnApplicationState.FINISHED, "tmp", "tmp",
  0l, 0l, FinalApplicationStatus.SUCCEEDED, null, null, 0f,
- "tmp", null));
+ "tmp", null, null));
  yarnRunner.killJob(jobId);
  verify(clientDelegate).killJob(jobId);
  }

hadoop-yarn-project/CHANGES.txt

@@ -20,6 +20,9 @@ Release 2.7.1 - UNRELEASED
  YARN-3489. RMServerUtils.validateResourceRequests should only obtain queue 
  info once. (Varun Saxena via wangda)
 
+ YARN-1462. Made RM write application tags to timeline server and exposed them
+ to users via generic history web UI and REST API. (Xuan Gong via zjshen)
+
  OPTIMIZATIONS
 
  YARN-3006. Improve the error message when attempting manual failover with 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ApplicationReport.java

@@ -59,7 +59,8 @@ public static ApplicationReport newInstance(ApplicationId applicationId,
  YarnApplicationState state, String diagnostics, String url,
  long startTime, long finishTime, FinalApplicationStatus finalStatus,
  ApplicationResourceUsageReport appResources, String origTrackingUrl,
- float progress, String applicationType, Token amRmToken) {
+ float progress, String applicationType, Token amRmToken,
+ Set<String> tags) {
  ApplicationReport report = Records.newRecord(ApplicationReport.class);
  report.setApplicationId(applicationId);
  report.setCurrentApplicationAttemptId(applicationAttemptId);
@@ -80,6 +81,7 @@ public static ApplicationReport newInstance(ApplicationId applicationId,
  report.setProgress(progress);
  report.setApplicationType(applicationType);
  report.setAMRMToken(amRmToken);
+ report.setApplicationTags(tags);
  return report;
  }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/ProtocolHATestBase.java

@@ -647,7 +647,7 @@ public ApplicationReport createFakeAppReport() {
  "fakeQueue", "fakeApplicationName", "localhost", 0, null,
  YarnApplicationState.FINISHED, "fake an application report", "",
  1000l, 1200l, FinalApplicationStatus.FAILED, null, "", 50f,
- "fakeApplicationType", null);
+ "fakeApplicationType", null, null);
  return report;
  }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAHSClient.java

@@ -334,7 +334,7 @@ private void createAppReports() {
  "queue", "appname", "host", 124, null,
  YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
  FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN",
- null);
+ null, null);
  List<ApplicationReport> applicationReports =
  new ArrayList<ApplicationReport>();
  applicationReports.add(newApplicationReport);
@@ -391,7 +391,7 @@ private void createAppReports() {
  "queue2", "appname2", "host2", 125, null,
  YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
  FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f,
- "NON-YARN", null);
+ "NON-YARN", null, null);
  applicationReports.add(newApplicationReport2);
 
  ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
@@ -401,7 +401,7 @@ private void createAppReports() {
  "queue3", "appname3", "host3", 126, null,
  YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
  FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f,
- "MAPREDUCE", null);
+ "MAPREDUCE", null, null);
  applicationReports.add(newApplicationReport3);
 
  ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
@@ -411,7 +411,7 @@ private void createAppReports() {
  "queue4", "appname4", "host4", 127, null,
  YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
  FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f,
- "NON-MAPREDUCE", null);
+ "NON-MAPREDUCE", null, null);
  applicationReports.add(newApplicationReport4);
  reports = applicationReports;
  }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java

@@ -580,7 +580,8 @@ private List<ApplicationReport> createAppReports() {
  applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
  "user", "queue", "appname", "host", 124, null,
  YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+ null);
  List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
  applicationReports.add(newApplicationReport);
  List<ApplicationAttemptReport> appAttempts = new ArrayList<ApplicationAttemptReport>();
@@ -655,7 +656,7 @@ private List<ApplicationReport> createAppReports() {
  "user2", "queue2", "appname2", "host2", 125, null,
  YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
  FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", 
- null);
+  null, null);
  applicationReports.add(newApplicationReport2);
 
  ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
@@ -664,7 +665,7 @@ private List<ApplicationReport> createAppReports() {
  "user3", "queue3", "appname3", "host3", 126, null,
  YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
  FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE",
- null);
+  null, null);
  applicationReports.add(newApplicationReport3);
 
  ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
@@ -675,7 +676,7 @@ private List<ApplicationReport> createAppReports() {
  "user4", "queue4", "appname4", "host4", 127, null,
  YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
  FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f,
- "NON-MAPREDUCE", null);
+ "NON-MAPREDUCE", null, null);
  applicationReports.add(newApplicationReport4);
  return applicationReports;
  }

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestYarnCLI.java

@@ -103,7 +103,7 @@ public void testGetApplicationReport() throws Exception {
  "user", "queue", "appname", "host", 124, null,
  YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
  FinalApplicationStatus.SUCCEEDED, usageReport, "N/A", 0.53789f, "YARN",
- null);
+ null, null);
  when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
  newApplicationReport);
  int result = cli.run(new String[] { "application", "-status", applicationId.toString() });
@@ -347,7 +347,8 @@ public void testGetApplications() throws Exception {
  applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
  "user", "queue", "appname", "host", 124, null,
  YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+ null);
  List<ApplicationReport> applicationReports = new ArrayList<ApplicationReport>();
  applicationReports.add(newApplicationReport);
 
@@ -357,7 +358,7 @@ public void testGetApplications() throws Exception {
  "user2", "queue2", "appname2", "host2", 125, null,
  YarnApplicationState.FINISHED, "diagnostics2", "url2", 2, 2,
  FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.63789f, "NON-YARN", 
- null);
+  null, null);
  applicationReports.add(newApplicationReport2);
 
  ApplicationId applicationId3 = ApplicationId.newInstance(1234, 7);
@@ -366,7 +367,7 @@ public void testGetApplications() throws Exception {
  "user3", "queue3", "appname3", "host3", 126, null,
  YarnApplicationState.RUNNING, "diagnostics3", "url3", 3, 3,
  FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.73789f, "MAPREDUCE", 
- null);
+ null, null);
  applicationReports.add(newApplicationReport3);
 
  ApplicationId applicationId4 = ApplicationId.newInstance(1234, 8);
@@ -375,7 +376,7 @@ public void testGetApplications() throws Exception {
  "user4", "queue4", "appname4", "host4", 127, null,
  YarnApplicationState.FAILED, "diagnostics4", "url4", 4, 4,
  FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.83789f, "NON-MAPREDUCE",
- null);
+ null, null);
  applicationReports.add(newApplicationReport4);
 
  ApplicationId applicationId5 = ApplicationId.newInstance(1234, 9);
@@ -384,7 +385,7 @@ public void testGetApplications() throws Exception {
  "user5", "queue5", "appname5", "host5", 128, null,
  YarnApplicationState.ACCEPTED, "diagnostics5", "url5", 5, 5,
  FinalApplicationStatus.KILLED, null, "N/A", 0.93789f, "HIVE",
- null);
+ null, null);
  applicationReports.add(newApplicationReport5);
 
  ApplicationId applicationId6 = ApplicationId.newInstance(1234, 10);
@@ -393,7 +394,7 @@ public void testGetApplications() throws Exception {
  "user6", "queue6", "appname6", "host6", 129, null,
  YarnApplicationState.SUBMITTED, "diagnostics6", "url6", 6, 6,
  FinalApplicationStatus.KILLED, null, "N/A", 0.99789f, "PIG",
- null);
+ null, null);
  applicationReports.add(newApplicationReport6);
 
  // Test command yarn application -list
@@ -817,7 +818,8 @@ public void testKillApplication() throws Exception {
  applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
  "user", "queue", "appname", "host", 124, null,
  YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+ null);
  when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
  newApplicationReport2);
  int result = cli.run(new String[] { "application","-kill", applicationId.toString() });
@@ -830,7 +832,8 @@ public void testKillApplication() throws Exception {
  applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
  "user", "queue", "appname", "host", 124, null,
  YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+ null);
  when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
  newApplicationReport);
  result = cli.run(new String[] { "application","-kill", applicationId.toString() });
@@ -865,7 +868,8 @@ public void testMoveApplicationAcrossQueues() throws Exception {
  applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
  "user", "queue", "appname", "host", 124, null,
  YarnApplicationState.FINISHED, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+ null);
  when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
  newApplicationReport2);
  int result = cli.run(new String[] { "application", "-movetoqueue",
@@ -880,7 +884,8 @@ public void testMoveApplicationAcrossQueues() throws Exception {
  applicationId, ApplicationAttemptId.newInstance(applicationId, 1),
  "user", "queue", "appname", "host", 124, null,
  YarnApplicationState.RUNNING, "diagnostics", "url", 0, 0,
- FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null);
+ FinalApplicationStatus.SUCCEEDED, null, "N/A", 0.53789f, "YARN", null,
+ null);
  when(client.getApplicationReport(any(ApplicationId.class))).thenReturn(
  newApplicationReport);
  result = cli.run(new String[] { "application", "-movetoqueue",

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/TestApplicatonReport.java

@@ -58,7 +58,8 @@ protected static ApplicationReport createApplicationReport(
  ApplicationReport.newInstance(appId, appAttemptId, "user", "queue",
  "appname", "host", 124, null, YarnApplicationState.FINISHED,
  "diagnostics", "url", 0, 0, FinalApplicationStatus.SUCCEEDED, null,
- "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null);
+ "N/A", 0.53789f, YarnConfiguration.DEFAULT_APPLICATION_TYPE, null,
+ null);
  return appReport;
  }
 

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerImpl.java

@@ -139,7 +139,7 @@ private ApplicationReport convertToApplicationReport(
  appHistory.getYarnApplicationState(), appHistory.getDiagnosticsInfo(),
  trackingUrl, appHistory.getStartTime(), appHistory.getFinishTime(),
  appHistory.getFinalApplicationStatus(), null, "", 100,
- appHistory.getApplicationType(), null);
+ appHistory.getApplicationType(), null, null);
  }
 
  private ApplicationAttemptHistoryData getLastAttempt(ApplicationId appId)

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/ApplicationHistoryManagerOnTimelineStore.java

@@ -19,11 +19,14 @@
 package org.apache.hadoop.yarn.server.applicationhistoryservice;
 
 import java.io.IOException;
+import java.util.Collection;
 import java.util.EnumSet;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.LinkedHashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
@@ -243,6 +246,7 @@ private static ApplicationReportExt convertToApplicationReport(
  FinalApplicationStatus finalStatus = FinalApplicationStatus.UNDEFINED;
  YarnApplicationState state = null;
  ApplicationResourceUsageReport appResources = null;
+ Set<String> appTags = null;
  Map<ApplicationAccessType, String> appViewACLs =
  new HashMap<ApplicationAccessType, String>();
  Map<String, Object> entityInfo = entity.getOtherInfo();
@@ -264,7 +268,7 @@ private static ApplicationReportExt convertToApplicationReport(
  ConverterUtils.toApplicationId(entity.getEntityId()),
  latestApplicationAttemptId, user, queue, name, null, -1, null, state,
  diagnosticsInfo, null, createdTime, finishedTime, finalStatus, null,
- null, 1.0F, type, null), appViewACLs);
+ null, 1.0F, type, null, appTags), appViewACLs);
  }
  if (entityInfo.containsKey(ApplicationMetricsConstants.QUEUE_ENTITY_INFO)) {
  queue =
@@ -289,6 +293,17 @@ private static ApplicationReportExt convertToApplicationReport(
  appResources=ApplicationResourceUsageReport
  .newInstance(0, 0, null, null, null, memorySeconds, vcoreSeconds);
  }
+ if (entityInfo.containsKey(ApplicationMetricsConstants.APP_TAGS_INFO)) {
+ appTags = new HashSet<String>();
+ Object obj = entityInfo.get(ApplicationMetricsConstants.APP_TAGS_INFO);
+ if (obj != null && obj instanceof Collection<?>) {
+ for(Object o : (Collection<?>)obj) {
+ if (o != null) {
+ appTags.add(o.toString());
+ }
+ }
+ }
+ }
  }
  List<TimelineEvent> events = entity.getEvents();
  if (events != null) {
@@ -340,7 +355,7 @@ private static ApplicationReportExt convertToApplicationReport(
  ConverterUtils.toApplicationId(entity.getEntityId()),
  latestApplicationAttemptId, user, queue, name, null, -1, null, state,
  diagnosticsInfo, null, createdTime, finishedTime, finalStatus, appResources,
- null, 1.0F, type, null), appViewACLs);
+ null, 1.0F, type, null, appTags), appViewACLs);
  }
 
  private static ApplicationAttemptReport convertToApplicationAttemptReport(

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestApplicationHistoryManagerOnTimelineStore.java

@@ -22,7 +22,9 @@
 import java.util.Arrays;
 import java.util.Collection;
 import java.util.HashMap;
+import java.util.HashSet;
 import java.util.Map;
+import java.util.Set;
 
 import org.apache.hadoop.conf.Configuration;
 import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
@@ -183,6 +185,9 @@ public ApplicationReport run() throws Exception {
  Assert.assertEquals(Integer.MAX_VALUE + 2L, app.getStartTime());
  Assert.assertEquals(Integer.MAX_VALUE + 3L, app.getFinishTime());
  Assert.assertTrue(Math.abs(app.getProgress() - 1.0F) < 0.0001);
+ Assert.assertEquals(2, app.getApplicationTags().size());
+ Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_1"));
+ Assert.assertTrue(app.getApplicationTags().contains("Test_APP_TAGS_2"));
  // App 2 doesn't have the ACLs, such that the default ACLs " " will be used.
  // Nobody except admin and owner has access to the details of the app.
  if ((i == 1 && callerUGI != null &&
@@ -469,6 +474,10 @@ private static TimelineEntity createApplicationTimelineEntity(
  entityInfo.put(ApplicationMetricsConstants.APP_VIEW_ACLS_ENTITY_INFO,
  "user2");
  }
+ Set<String> appTags = new HashSet<String>();
+ appTags.add("Test_APP_TAGS_1");
+ appTags.add("Test_APP_TAGS_2");
+ entityInfo.put(ApplicationMetricsConstants.APP_TAGS_INFO, appTags);
  entity.setOtherInfo(entityInfo);
  TimelineEvent tEvent = new TimelineEvent();
  tEvent.setEventType(ApplicationMetricsConstants.CREATED_EVENT_TYPE);

hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/metrics/ApplicationMetricsConstants.java

@@ -73,4 +73,5 @@ public class ApplicationMetricsConstants {
  public static final String LATEST_APP_ATTEMPT_EVENT_INFO =
  "YARN_APPLICATION_LATEST_APP_ATTEMPT";
 
+ public static final String APP_TAGS_INFO = "YARN_APPLICATION_TAGS";
 }