diff --git a/src/webportal/src/app/job-submission/components/data/team-detail.jsx b/src/webportal/src/app/job-submission/components/data/team-detail.jsx index d0e7a13019..726e550d50 100644 --- a/src/webportal/src/app/job-submission/components/data/team-detail.jsx +++ b/src/webportal/src/app/job-submission/components/data/team-detail.jsx @@ -34,45 +34,84 @@ export default function TeamDetail({ isOpen = false, config, hide }) { hide(); }; - const columes = [ - { - key: 'containerPath', - name: 'Path', - headerClassName: FontClassNames.semibold, - minWidth: 120, - onRender: item => { - return ( -
{`/mnt/${item.name}`}
- ); - }, - }, - { - key: 'serverType', - name: 'Server Type', - headerClassName: FontClassNames.semibold, - minWidth: 80, - onRender: item => { - if (item === undefined) { - return
{'Invalid Server'}
; - } else { - return
{item.type}
; - } + const columes = config => { + const result = [ + { + key: 'containerPath', + name: 'Path', + headerClassName: FontClassNames.semibold, + minWidth: 120, + onRender: item => { + return ( +
{`/mnt/${item.name}`}
+ ); + }, }, - }, - { - key: 'serverPath', - name: 'Server Path(Server Root Path as bold)', - headerClassName: FontClassNames.semibold, - minWidth: 400, - onRender: item => { - if (item === undefined) { - return
{'Invalid Server'}
; - } else { - return SERVER_PATH[item.type](item); - } + { + key: 'serverType', + name: 'Server Type', + headerClassName: FontClassNames.semibold, + minWidth: 80, + onRender: item => { + if (item === undefined) { + return ( +
{'Invalid Server'}
+ ); + } else { + return
{item.type}
; + } + }, }, - }, - { + ]; + if (config.type === 'dshuttle') { + result.push({ + key: 'ufsType', + name: 'UFS Type', + headerClassName: FontClassNames.semibold, + minWidth: 80, + onRender: item => { + if (item === undefined) { + return
{'Invalid Type'}
; + } else if (item.data.ufsType === 'wasb') { + return 'azureBlob'; + } else { + return item.data.ufsType; + } + }, + }); + result.push({ + key: 'ufsServerPath', + name: 'UFS Server Path(Server Root Path as bold)', + headerClassName: FontClassNames.semibold, + minWidth: 350, + onRender: item => { + if (item === undefined) { + return ( +
{'Invalid Server'}
+ ); + } else { + return SERVER_PATH[item.type](item); + } + }, + }); + } else { + result.push({ + key: 'serverPath', + name: 'Server Path(Server Root Path as bold)', + headerClassName: FontClassNames.semibold, + minWidth: 400, + onRender: item => { + if (item === undefined) { + return ( +
{'Invalid Server'}
+ ); + } else { + return SERVER_PATH[item.type](item); + } + }, + }); + } + result.push({ key: 'permission', name: 'Permission', headerClassName: FontClassNames.semibold, @@ -84,8 +123,10 @@ export default function TeamDetail({ isOpen = false, config, hide }) { ); }, - }, - ]; + }); + + return result; + }; return ( )} How to use data ? - - By selecting team storage, the storage server will be automatically - mounted to Path when job runs. You could copy/read/write like local - folder. - + {config.type === 'dshuttle' && ( + + By selecting team storage, the server path will be automatically + mounted to path when job running. Please treat is as local folder. + + )} + {config.type !== 'dshuttle' && ( + + By selecting team storage, the storage server will be automatically + mounted to Path when job runs. You could copy/read/write like local + folder. + + )} Details to upload data directly. ), + dshuttle: ( +
+
Dshuttle
+ + Storage Dshuttle is configured for group. It used as a fast data cache + and try to speed up I/O intensive workload. It's a readonly + storage. For more detail, please refer to + + + + Dshuttle doc + + or contact cluster amdin. +
+ ), }; export const SERVER_PATH = { @@ -353,4 +421,14 @@ export const SERVER_PATH = { {storage.data.path || '/'} ), + dshuttle: storage => ( +
+ + {storage.data.ufsType === 'wasb' + ? `${storage.data.accountName}.blob.core.windows.net/${storage.data.containerName}` + : storage.data.ufsUri} + + {storage.data.path || '/'} +
+ ), }; diff --git a/src/webportal/src/app/job-submission/models/data/mount-directories.js b/src/webportal/src/app/job-submission/models/data/mount-directories.js index 467582c0e5..0a515a4ed3 100644 --- a/src/webportal/src/app/job-submission/models/data/mount-directories.js +++ b/src/webportal/src/app/job-submission/models/data/mount-directories.js @@ -329,8 +329,13 @@ export class MountDirectories { 'azurefile://' + data.accountName + '/' + data.shareName; break; case 'azureBlob': + returnValue = `accountName: ${data.accountName} containerName: ${data.containerName}`; + break; + case 'dshuttle': returnValue = - 'azureblob://' + data.accountName + '/' + data.containerName; + data.ufsType === 'wasb' + ? `Azure Blob (accountName: ${data.accountName} containerName: ${data.containerName})` + : data.ufsUri; break; case 'hdfs': returnValue = 'hdfs://' + data.namenode + ':' + data.port; @@ -351,7 +356,7 @@ export class MountDirectories { serverRootPath + this.normalizePath(serverRootPath.endsWith('/') ? '' : '/' + path); newTeamDataList.push( - new InputData(`/mnt/${storage.name}`, serverPath, storage.name), + new InputData(`/mnt/${storage.name}`, serverPath, storage.type), ); } }); diff --git a/src/webportal/src/app/job-submission/utils/conn.js b/src/webportal/src/app/job-submission/utils/conn.js index fd3013f951..a78c0d7b08 100644 --- a/src/webportal/src/app/job-submission/utils/conn.js +++ b/src/webportal/src/app/job-submission/utils/conn.js @@ -7,6 +7,7 @@ import config from '../../config/webportal.config'; import yaml from 'js-yaml'; import { get } from 'lodash'; import urljoin from 'url-join'; +import { getDeshuttleStorageDetails } from './utils'; const token = cookies.get('token'); @@ -96,6 +97,27 @@ export async function fetchStorageDetails(configNames) { if (defaultStorageNames.includes(detail.name)) { detail.default = true; } + if (detail.type === 'dshuttle') { + const res = await fetch('dshuttle/api/v1/master/info', { + headers: { + Authorization: `Bearer ${token}`, + }, + }); + if (res.ok) { + const json = await res.json(); + if ( + detail.data.dshuttlePath && + json.mountPoints[detail.data.dshuttlePath] + ) { + detail.data = { + ...detail.data, + ...getDeshuttleStorageDetails( + json.mountPoints[detail.data.dshuttlePath], + ), + }; + } + } + } details.push(detail); } } diff --git a/src/webportal/src/app/job-submission/utils/utils.js b/src/webportal/src/app/job-submission/utils/utils.js index dada62c611..5785501b26 100644 --- a/src/webportal/src/app/job-submission/utils/utils.js +++ b/src/webportal/src/app/job-submission/utils/utils.js @@ -218,3 +218,16 @@ export function getStoragePlugin(extras) { } return plugins.find(plugin => plugin.plugin === 'teamwise_storage'); } + +export function getDeshuttleStorageDetails(dshuttleMountPoint) { + const ufsType = dshuttleMountPoint.ufsType; + const ufsUri = dshuttleMountPoint.ufsUri; + if (ufsType === 'wasb') { + const strings = ufsUri.split('//')[1].split('@'); + const accountName = strings[1].split('.')[0]; + const containerName = strings[0]; + return { ufsType, accountName, containerName }; + } else { + return { ufsType, ufsUri }; + } +} diff --git a/src/webportal/src/app/user/fabric/conn.js b/src/webportal/src/app/user/fabric/conn.js index 90c9fceee3..4131d5c7d6 100644 --- a/src/webportal/src/app/user/fabric/conn.js +++ b/src/webportal/src/app/user/fabric/conn.js @@ -6,6 +6,7 @@ import cookies from 'js-cookie'; import config from '../../config/webportal.config'; import { checkToken } from '../user-auth/user-auth.component'; import { clearToken } from '../user-logout/user-logout.component'; +import { getDeshuttleStorageDetails } from '../../job-submission/utils/utils'; const client = new PAIV2.OpenPAIClient({ rest_server_uri: new URL(config.restServerUri, window.location.href), @@ -177,8 +178,32 @@ export const listStorageDetailRequest = async () => { return wrapper(async () => { const storageSummary = await client.storage.getStorages(); const details = []; + const token = checkToken(); for (const storage of storageSummary.storages) { - details.push(await client.storage.getStorage(storage.name)); + const detail = await client.storage.getStorage(storage.name); + if (detail.type === 'dshuttle') { + const res = await fetch('dshuttle/api/v1/master/info', { + headers: { + Authorization: `Bearer ${token}`, + }, + }); + if (res.ok) { + const json = await res.json(); + if ( + detail.data.dshuttlePath && + json.mountPoints[detail.data.dshuttlePath] + ) { + detail.data = { + ...detail.data, + ...getDeshuttleStorageDetails( + json.mountPoints[detail.data.dshuttlePath], + ), + }; + } + } + } + + details.push(detail); } return details; }); diff --git a/src/webportal/src/app/user/fabric/user-profile/storage-list.jsx b/src/webportal/src/app/user/fabric/user-profile/storage-list.jsx index 79a912e9ba..fd0f445b72 100644 --- a/src/webportal/src/app/user/fabric/user-profile/storage-list.jsx +++ b/src/webportal/src/app/user/fabric/user-profile/storage-list.jsx @@ -43,6 +43,17 @@ function getStorageServerUri(storageDetail) { {data.containerName}; {'Path: '} ); + case 'dshuttle': + return data.ufsType === 'wasb' ? ( + <> + {'AzureBlob '} + {'StorageAccount: '} + {data.accountName}; {'Container: '} + {data.containerName}; {'Path: '} + + ) : ( + data.ufsUri + ); case 'hdfs': return `hdfs://${data.namenode}:${data.port}`; case 'unknown':