[Webportal] Support Dshuttle storage (#4935)

* update * update * support other dshuttle types * support other ufs type in storage detail page * update * update
microsoft · Oct 14, 2020 · fe8d1ee · fe8d1ee
1 parent 3b80b51
commit fe8d1ee
Show file tree

Hide file tree

Showing 6 changed files with 202 additions and 48 deletions.
diff --git a/src/webportal/src/app/job-submission/components/data/team-detail.jsx b/src/webportal/src/app/job-submission/components/data/team-detail.jsx
@@ -34,45 +34,84 @@ export default function TeamDetail({ isOpen = false, config, hide }) {
  hide();
  };
 
- const columes = [
- {
- key: 'containerPath',
- name: 'Path',
- headerClassName: FontClassNames.semibold,
- minWidth: 120,
- onRender: item => {
- return (
- <div className={FontClassNames.small}>{`/mnt/${item.name}`}</div>
- );
- },
- },
- {
- key: 'serverType',
- name: 'Server Type',
- headerClassName: FontClassNames.semibold,
- minWidth: 80,
- onRender: item => {
- if (item === undefined) {
- return <div className={FontClassNames.small}>{'Invalid Server'}</div>;
- } else {
- return <div className={FontClassNames.small}>{item.type}</div>;
- }
+ const columes = config => {
+ const result = [
+ {
+ key: 'containerPath',
+ name: 'Path',
+ headerClassName: FontClassNames.semibold,
+ minWidth: 120,
+ onRender: item => {
+ return (
+ <div className={FontClassNames.small}>{`/mnt/${item.name}`}</div>
+ );
+ },
  },
- },
- {
- key: 'serverPath',
- name: 'Server Path(Server Root Path as bold)',
- headerClassName: FontClassNames.semibold,
- minWidth: 400,
- onRender: item => {
- if (item === undefined) {
- return <div className={FontClassNames.small}>{'Invalid Server'}</div>;
- } else {
- return SERVER_PATH[item.type](item);
- }
+ {
+ key: 'serverType',
+ name: 'Server Type',
+ headerClassName: FontClassNames.semibold,
+ minWidth: 80,
+ onRender: item => {
+ if (item === undefined) {
+ return (
+ <div className={FontClassNames.small}>{'Invalid Server'}</div>
+ );
+ } else {
+ return <div className={FontClassNames.small}>{item.type}</div>;
+ }
+ },
  },
- },
- {
+ ];
+ if (config.type === 'dshuttle') {
+ result.push({
+ key: 'ufsType',
+ name: 'UFS Type',
+ headerClassName: FontClassNames.semibold,
+ minWidth: 80,
+ onRender: item => {
+ if (item === undefined) {
+ return <div className={FontClassNames.small}>{'Invalid Type'}</div>;
+ } else if (item.data.ufsType === 'wasb') {
+ return 'azureBlob';
+ } else {
+ return item.data.ufsType;
+ }
+ },
+ });
+ result.push({
+ key: 'ufsServerPath',
+ name: 'UFS Server Path(Server Root Path as bold)',
+ headerClassName: FontClassNames.semibold,
+ minWidth: 350,
+ onRender: item => {
+ if (item === undefined) {
+ return (
+ <div className={FontClassNames.small}>{'Invalid Server'}</div>
+ );
+ } else {
+ return SERVER_PATH[item.type](item);
+ }
+ },
+ });
+ } else {
+ result.push({
+ key: 'serverPath',
+ name: 'Server Path(Server Root Path as bold)',
+ headerClassName: FontClassNames.semibold,
+ minWidth: 400,
+ onRender: item => {
+ if (item === undefined) {
+ return (
+ <div className={FontClassNames.small}>{'Invalid Server'}</div>
+ );
+ } else {
+ return SERVER_PATH[item.type](item);
+ }
+ },
+ });
+ }
+ result.push({
  key: 'permission',
  name: 'Permission',
  headerClassName: FontClassNames.semibold,
@@ -84,8 +123,10 @@ export default function TeamDetail({ isOpen = false, config, hide }) {
  </div>
  );
  },
- },
- ];
+ });
+
+ return result;
+ };
 
  return (
  <Dialog
@@ -177,14 +218,22 @@ export default function TeamDetail({ isOpen = false, config, hide }) {
  </Stack>
  )}
  <Text variant='large'>How to use data ?</Text>
- <Text styles={{ root: { marginLeft: `${spacing.m}` } }}>
- By selecting team storage, the storage server will be automatically
- mounted to Path when job runs. You could copy/read/write like local
- folder.
- </Text>
+ {config.type === 'dshuttle' && (
+ <Text styles={{ root: { marginLeft: `${spacing.m}` } }}>
+ By selecting team storage, the server path will be automatically
+ mounted to path when job running. Please treat is as local folder.
+ </Text>
+ )}
+ {config.type !== 'dshuttle' && (
+ <Text styles={{ root: { marginLeft: `${spacing.m}` } }}>
+ By selecting team storage, the storage server will be automatically
+ mounted to Path when job runs. You could copy/read/write like local
+ folder.
+ </Text>
+ )}
  <Text variant='large'>Details</Text>
  <DetailsList
- columns={columes}
+ columns={columes(config)}
  disableSelectionZone
  selectionMode={SelectionMode.none}
  items={[config]}
@@ -320,6 +369,25 @@ export const NAS_TIPS = {
  <span> to upload data directly.</span>
  </div>
  ),
+ dshuttle: (
+ <div>
+ <div style={{ fontWeight: FontWeights.semibold }}>Dshuttle</div>
+ <span>
+ Storage Dshuttle is configured for group. It used as a fast data cache
+ and try to speed up I/O intensive workload. It&apos;s a readonly
+ storage. For more detail, please refer to
+ </span>
+ <span> </span>
+ <Link
+ href='https:/microsoft/dshuttle'
+ target='_blank'
+ style={{ fontWeight: FontWeights.semibold }}
+ >
+ Dshuttle doc
+ </Link>
+ <span> or contact cluster amdin.</span>
+ </div>
+ ),
 };
 
 export const SERVER_PATH = {
@@ -353,4 +421,14 @@ export const SERVER_PATH = {
  {storage.data.path || '/'}
  </div>
  ),
+ dshuttle: storage => (
+ <div className={FontClassNames.semibold}>
+ <b>
+ {storage.data.ufsType === 'wasb'
+ ? `${storage.data.accountName}.blob.core.windows.net/${storage.data.containerName}`
+ : storage.data.ufsUri}
+ </b>
+ {storage.data.path || '/'}
+ </div>
+ ),
 };
diff --git a/src/webportal/src/app/job-submission/models/data/mount-directories.js b/src/webportal/src/app/job-submission/models/data/mount-directories.js
@@ -329,8 +329,13 @@ export class MountDirectories {
  'azurefile://' + data.accountName + '/' + data.shareName;
  break;
  case 'azureBlob':
+ returnValue = `accountName: ${data.accountName} containerName: ${data.containerName}`;
+ break;
+ case 'dshuttle':
  returnValue =
- 'azureblob://' + data.accountName + '/' + data.containerName;
+ data.ufsType === 'wasb'
+ ? `Azure Blob (accountName: ${data.accountName} containerName: ${data.containerName})`
+ : data.ufsUri;
  break;
  case 'hdfs':
  returnValue = 'hdfs://' + data.namenode + ':' + data.port;
@@ -351,7 +356,7 @@ export class MountDirectories {
  serverRootPath +
  this.normalizePath(serverRootPath.endsWith('/') ? '' : '/' + path);
  newTeamDataList.push(
- new InputData(`/mnt/${storage.name}`, serverPath, storage.name),
+ new InputData(`/mnt/${storage.name}`, serverPath, storage.type),
  );
  }
  });

diff --git a/src/webportal/src/app/job-submission/utils/conn.js b/src/webportal/src/app/job-submission/utils/conn.js
@@ -7,6 +7,7 @@ import config from '../../config/webportal.config';
 import yaml from 'js-yaml';
 import { get } from 'lodash';
 import urljoin from 'url-join';
+import { getDeshuttleStorageDetails } from './utils';
 
 const token = cookies.get('token');
 
@@ -96,6 +97,27 @@ export async function fetchStorageDetails(configNames) {
  if (defaultStorageNames.includes(detail.name)) {
  detail.default = true;
  }
+ if (detail.type === 'dshuttle') {
+ const res = await fetch('dshuttle/api/v1/master/info', {
+ headers: {
+ Authorization: `Bearer ${token}`,
+ },
+ });
+ if (res.ok) {
+ const json = await res.json();
+ if (
+ detail.data.dshuttlePath &&
+ json.mountPoints[detail.data.dshuttlePath]
+ ) {
+ detail.data = {
+ ...detail.data,
+ ...getDeshuttleStorageDetails(
+ json.mountPoints[detail.data.dshuttlePath],
+ ),
+ };
+ }
+ }
+ }
  details.push(detail);
  }
  }

diff --git a/src/webportal/src/app/job-submission/utils/utils.js b/src/webportal/src/app/job-submission/utils/utils.js
@@ -218,3 +218,16 @@ export function getStoragePlugin(extras) {
  }
  return plugins.find(plugin => plugin.plugin === 'teamwise_storage');
 }
+
+export function getDeshuttleStorageDetails(dshuttleMountPoint) {
+ const ufsType = dshuttleMountPoint.ufsType;
+ const ufsUri = dshuttleMountPoint.ufsUri;
+ if (ufsType === 'wasb') {
+ const strings = ufsUri.split('//')[1].split('@');
+ const accountName = strings[1].split('.')[0];
+ const containerName = strings[0];
+ return { ufsType, accountName, containerName };
+ } else {
+ return { ufsType, ufsUri };
+ }
+}
diff --git a/src/webportal/src/app/user/fabric/conn.js b/src/webportal/src/app/user/fabric/conn.js
@@ -6,6 +6,7 @@ import cookies from 'js-cookie';
 import config from '../../config/webportal.config';
 import { checkToken } from '../user-auth/user-auth.component';
 import { clearToken } from '../user-logout/user-logout.component';
+import { getDeshuttleStorageDetails } from '../../job-submission/utils/utils';
 
 const client = new PAIV2.OpenPAIClient({
  rest_server_uri: new URL(config.restServerUri, window.location.href),
@@ -177,8 +178,32 @@ export const listStorageDetailRequest = async () => {
  return wrapper(async () => {
  const storageSummary = await client.storage.getStorages();
  const details = [];
+ const token = checkToken();
  for (const storage of storageSummary.storages) {
- details.push(await client.storage.getStorage(storage.name));
+ const detail = await client.storage.getStorage(storage.name);
+ if (detail.type === 'dshuttle') {
+ const res = await fetch('dshuttle/api/v1/master/info', {
+ headers: {
+ Authorization: `Bearer ${token}`,
+ },
+ });
+ if (res.ok) {
+ const json = await res.json();
+ if (
+ detail.data.dshuttlePath &&
+ json.mountPoints[detail.data.dshuttlePath]
+ ) {
+ detail.data = {
+ ...detail.data,
+ ...getDeshuttleStorageDetails(
+ json.mountPoints[detail.data.dshuttlePath],
+ ),
+ };
+ }
+ }
+ }
+
+ details.push(detail);
  }
  return details;
  });

diff --git a/src/webportal/src/app/user/fabric/user-profile/storage-list.jsx b/src/webportal/src/app/user/fabric/user-profile/storage-list.jsx
@@ -43,6 +43,17 @@ function getStorageServerUri(storageDetail) {
  {data.containerName}; <b>{'Path: '}</b>
  </>
  );
+ case 'dshuttle':
+ return data.ufsType === 'wasb' ? (
+ <>
+ {'AzureBlob '}
+ <b>{'StorageAccount: '}</b>
+ {data.accountName}; <b>{'Container: '}</b>
+ {data.containerName}; <b>{'Path: '}</b>
+ </>
+ ) : (
+ data.ufsUri
+ );
  case 'hdfs':
  return `hdfs://${data.namenode}:${data.port}`;
  case 'unknown':