[Webportal] Support Job Priority in job-list

src/database-controller/src/common/framework.js

@@ -186,7 +186,7 @@ class Snapshot {
  );
  const jobPriority = _.get(
  loadedConfig,
- 'extras.hivedscheduler.jobPriorityClass',
+ 'extras.hivedScheduler.jobPriorityClass',
  null,
  );
  // Job status change notification

src/rest-server/docs/swagger.yaml

@@ -17,10 +17,11 @@ info:
  version 2.2.4: support sorting by completionTime in get the list of jobs
  version 2.2.5: add alert related api
  version 2.2.6: update type of taskUid
+ version 2.2.7: add jobPriority list jobs parameter
  license:
  name: MIT License
  url: "https:/microsoft/pai/blob/master/LICENSE"
- version: 2.2.6
+ version: 2.2.7
 externalDocs:
  description: Find out more about OpenPAI
  url: "https:/microsoft/pai"
@@ -1150,6 +1151,11 @@ paths:
  description: filter jobs with tags. When multiple tags are specified, every job selected should have none of these tags
  schema:
  type: string
+ - name: jobPriority
+ in: query
+ description: filter jobs with jobPriority, fields include oppo, test, prod, and default (default means jobPriorityClass in job config is null)
+ schema:
+ type: string
  - name: offset
  in: query
  description: list job offset
@@ -1164,7 +1170,7 @@ paths:
  in: query
  description: 'order of job list.
  It follows the format <field>,<ASC|DESC>, default value is "submissionTime,DESC".
- Available fields include: jobName, submissionTime, username, vc, retries, totalTaskNumber, totalGpuNumber, state, completionTime.
+ Available fields include: jobName, submissionTime, username, vc, retries, totalTaskNumber, totalGpuNumber, state, completionTime, jobPriority.
  CompletionTime maybe null for some jobs, these jobs will be returned at the end of the list when sorting by ASC order & at the beginning when sorting by DESC order.'
  schema:
  type: string
@@ -1197,6 +1203,7 @@ paths:
  completedTime: 0
  appExitCode: 0
  virtualCluster: unknown
+ jobPriority: prod
  "500":
  $ref: "#/components/responses/UnknownError"
  "/api/v2/jobs/{user}~{job}":
@@ -2223,6 +2230,10 @@ components:
  debugId:
  type: string
  description: md5 hash name for the job in framework controller, used for debug purpose
+ jobPriority:
+ type: string
+ nullable: true
+ description: job priority
  required:
  - name
  - username

src/rest-server/src/controllers/v2/job.js

@@ -81,6 +81,22 @@ const list = asyncHandler(async (req, res) => {
  { virtualCluster: { [Op.substring]: req.query.keyword } },
  ];
  }
+ if ('jobPriority' in req.query) {
+ const jobPriorityFilter = req.query.jobPriority.split(',');
+ const index = jobPriorityFilter.indexOf('default');
+ if (index !== -1) {
+ jobPriorityFilter.splice(index, 1);
+ if (filters[Op.or] === undefined) {
+ filters[Op.or] = [];
+ }
+ filters[Op.or].push({ jobPriority: { [Op.is]: null } });
+ if (jobPriorityFilter.length > 0) {
+ filters[Op.or].push({ jobPriority: jobPriorityFilter });
+ }
+ } else {
+ filters.jobPriority = jobPriorityFilter;
+ }
+ }
  if ('order' in req.query) {
  const [field, ordering] = req.query.order.split(',');
  if (
@@ -94,6 +110,7 @@ const list = asyncHandler(async (req, res) => {
  'totalGpuNumber',
  'state',
  'completionTime',
+ 'jobPriority',
  ].includes(field)
  ) {
  if (ordering === 'ASC' || ordering === 'DESC') {
@@ -106,6 +123,10 @@ const list = asyncHandler(async (req, res) => {
  const orderingWithNulls =
  ordering === 'ASC' ? 'ASC NULLS LAST' : 'DESC NULLS FIRST';
  order.push(['completionTime', orderingWithNulls]);
+ } else if (field === 'jobPriority') {
+ const orderingWithNulls =
+ ordering === 'ASC' ? 'ASC NULLS LAST' : 'DESC NULLS FIRST';
+ order.push(['jobPriority', orderingWithNulls]);
  } else {
  order.push([field, ordering]);
  }
@@ -129,6 +150,7 @@ const list = asyncHandler(async (req, res) => {
  'totalGpuNumber',
  'totalTaskNumber',
  'totalTaskRoleNumber',
+ 'jobPriority',
  'retries',
  'retryDelayTime',
  'platformRetries',

src/rest-server/src/models/v2/job/k8s.js

@@ -75,6 +75,7 @@ const convertFrameworkSummary = (framework) => {
  totalGpuNumber: framework.totalGpuNumber,
  totalTaskNumber: framework.totalTaskNumber,
  totalTaskRoleNumber: framework.totalTaskRoleNumber,
+ jobPriority: framework.jobPriority,
  };
 };
 
@@ -187,6 +188,7 @@ const convertFrameworkDetail = async (
  debugId: frameworkWithLatestAttempt.metadata.name,
  name: jobName,
  tags: tags.reduce((arr, curr) => [...arr, curr.name], []),
+ jobPriority: frameworkWithLatestAttempt.jobPriority,
  jobStatus: {
  username: userName,
  state: convertState(

src/rest-server/src/models/v2/utils/frameworkConverter.js

@@ -337,6 +337,7 @@ const convertToJobAttempt = async (framework) => {
  0,
  );
  const totalTaskRoleNumber = framework.spec.taskRoles.length;
+ const jobPriority = framework.jobPriority;
  const diagnostics = completionStatus ? completionStatus.diagnostics : null;
  const exitDiagnostics = generateExitDiagnostics(diagnostics);
  const appExitTriggerMessage =
@@ -417,6 +418,7 @@ const convertToJobAttempt = async (framework) => {
  totalGpuNumber,
  totalTaskNumber,
  totalTaskRoleNumber,
+ jobPriority,
  taskRoles,
  };
 };

src/webportal/src/app/job/job-view/fabric/JobList/Filter.js

@@ -11,11 +11,13 @@ class Filter {
  */
  constructor(
  keyword = '',
+ priorities = new Set(),
  users = new Set(),
  virtualClusters = new Set(),
  statuses = new Set(),
  ) {
  this.keyword = keyword;
+ this.priorities = priorities;
  this.users = users;
  this.virtualClusters = virtualClusters;
  this.statuses = statuses;
@@ -26,6 +28,7 @@ class Filter {
  save() {
  const content = JSON.stringify({
  users: Array.from(this.users),
+ priorities: Array.from(this.priorities),
  virtualClusters: Array.from(this.virtualClusters),
  statuses: Array.from(this.statuses),
  keyword: this.keyword,
@@ -36,10 +39,19 @@ class Filter {
  load() {
  try {
  const content = window.localStorage.getItem(LOCAL_STORAGE_KEY);
- const { users, virtualClusters, statuses, keyword } = JSON.parse(content);
+ const {
+ priorities,
+ users,
+ virtualClusters,
+ statuses,
+ keyword,
+ } = JSON.parse(content);
  if (Array.isArray(users)) {
  this.users = new Set(users);
  }
+ if (Array.isArray(priorities)) {
+ this.priorities = new Set(priorities);
+ }
  if (Array.isArray(virtualClusters)) {
  this.virtualClusters = new Set(virtualClusters);
  }
@@ -53,7 +65,7 @@ class Filter {
  }
 
  apply() {
- const { keyword, users, virtualClusters, statuses } = this;
+ const { keyword, priorities, users, virtualClusters, statuses } = this;
 
  const query = {};
  if (keyword && keyword !== '') {
@@ -62,6 +74,22 @@ class Filter {
  if (users && users.size > 0) {
  query.username = Array.from(users).join(',');
  }
+ if (priorities && priorities.size > 0) {
+ query.jobPriority = Array.from(priorities)
+ .map(priority => {
+ switch (priority) {
+ case 'Opportunistic':
+ return 'oppo';
+ case 'Product':
+ return 'prod';
+ case 'Test':
+ return 'test';
+ default:
+ return 'default';
+ }
+ })
+ .join(',');
+ }
  if (virtualClusters && virtualClusters.size > 0) {
  query.vc = Array.from(virtualClusters).join(',');
  }

src/webportal/src/app/job/job-view/fabric/JobList/Ordering.js

@@ -40,6 +40,7 @@ export default class Ordering {
  'status',
  'taskCount',
  'gpuCount',
+ 'jobPriority',
  ].includes(field)
  ) {
  this.field = field;
@@ -78,6 +79,8 @@ export default class Ordering {
  query = 'totalTaskNumber';
  } else if (field === 'gpuCount') {
  query = 'totalGpuNumber';
+ } else if (field === 'jobPriority') {
+ query = 'jobPriority';
  }
 
  return { order: `${query},${descending ? 'DESC' : 'ASC'}` };

src/webportal/src/app/job/job-view/fabric/JobList/Table.jsx

@@ -187,6 +187,26 @@ export default function Table() {
  headerClassName: FontClassNames.medium,
  isResizable: true,
  });
+ const priorityColumn = applySortProps({
+ key: 'jobPriority',
+ minWidth: 95,
+ name: 'Priority',
+ className: FontClassNames.mediumPlus,
+ headerClassName: FontClassNames.medium,
+ isResizable: true,
+ onRender(job) {
+ switch (job.jobPriority) {
+ case 'oppo':
+ return 'Opportunistic';
+ case 'test':
+ return 'Test';
+ case 'prod':
+ return 'Product';
+ default:
+ return 'Default';
+ }
+ },
+ });
  const statusColumn = applySortProps({
  key: 'status',
  minWidth: 100,
@@ -272,6 +292,7 @@ export default function Table() {
  retriesColumn,
  taskCountColumn,
  gpuCountColumn,
+ priorityColumn,
  statusColumn,
  actionsColumn,
  ];