pull code (#8)

* Add module userControl; Modify gitignore; * modify gateway module. * publicservice安装的时候加上gateway的ip和端口替换脚本 * 控制台增加jdbc的参数 * Fix JDBCEngineExecutorManagerImpl And JDBCEntranceParser using spring annotations does not work problem close #252 * Added JDBC script type support close #254 * Update start-usercontrol.sh * Update stop-usercontrol.sh * remove legacy code from metadata module * StorageResultSetWriter toString() needs to add schema * resolve checkConsumerHealthy NullPointerException * resolve checkConsumerHealthy NullPointerException * modify gateway module. * refine usercontrol module. * Add module userControl; Modify gitignore; * modify gateway module. * Update start-usercontrol.sh * Update stop-usercontrol.sh * modify gateway module. * refine usercontrol module. * Get JDBC parameters from the console close #258 * Get JDBC parameters from the console close #258 * Get JDBC parameters from the console close #258 * Automated installation enhancements close #260 * update version from 0.9.2 to 0.9.3 close #261 * Automated installation enhancements close #260 * add userControl module to pom * update version from 0.9.2 to 0.9.3 Co-authored-by: tangjianfeng <[email protected]> Co-authored-by: chaogefeng <[email protected]> Co-authored-by: shanhuang <[email protected]> Co-authored-by: wForget <[email protected]> Co-authored-by: cooperyang <[email protected]>
apache · Jan 17, 2020 · 2bf7aec · 2bf7aec
1 parent 0f6cf3f
commit 2bf7aec
Show file tree

Hide file tree

Showing 93 changed files with 1,411 additions and 175 deletions.
diff --git a/.gitignore b/.gitignore
@@ -49,4 +49,14 @@ ujes/engine/target/
 ujes/enginemanager/target/
 ujes/entrance/target/
 
-ujes/definedEngines/mlsql/entrance/target
+ujes/definedEngines/mlsql/entrance/target/
+assembly/public-module/target/
+bml/bml-engine-hook/target/
+bml/bmlclient/target/
+bml/bmlcommon/target/
+bml/bmlserver/target/
+core/hadoop-common/target/
+publicService/workspace/client/workspace-httpclient/target/
+userControl/target/
+
+
diff --git a/assembly/pom.xml b/assembly/pom.xml
@@ -21,7 +21,7 @@
  <parent>
  <groupId>com.webank.wedatasphere.linkis</groupId>
  <artifactId>linkis</artifactId>
- <version>0.9.2</version>
+ <version>0.9.3</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
 

diff --git a/assembly/public-module/pom.xml b/assembly/public-module/pom.xml
@@ -5,7 +5,7 @@
  <parent>
  <artifactId>linkis</artifactId>
  <groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.2</version>
+ <version>0.9.3</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
 

diff --git a/bin/checkServices.sh b/bin/checkServices.sh
@@ -20,8 +20,11 @@ shellDir=`dirname $0`
 workDir=`cd ${shellDir}/..;pwd`
 
 ##load config
-source ${workDir}/conf/config.sh
-
+#source ${workDir}/conf/config.sh
+export LINKIS_DSS_CONF_FILE=${LINKIS_DSS_CONF_FILE:-"${workDir}/conf/config.sh"}
+export DISTRIBUTION=${DISTRIBUTION:-"${workDir}/conf/config.sh"}
+source ${LINKIS_DSS_CONF_FILE}
+source ${DISTRIBUTION}
 MICRO_SERVICE_NAME=$1
 MICRO_SERVICE_IP=$2
 MICRO_SERVICE_PORT=$3
@@ -56,7 +59,11 @@ function executeCMD(){
 }
 
 echo "Start to Check if your microservice:$MICRO_SERVICE_NAME is normal via telnet"
-
+echo "--------------------------------------------------------------------------------------------------------------------------"
+echo $MICRO_SERVICE_NAME
+echo $MICRO_SERVICE_IP
+echo $MICRO_SERVICE_PORT
+echo "--------------------------------------------------------------------------------------------------------------------------"
 result=`echo -e "\n" | telnet $MICRO_SERVICE_IP $MICRO_SERVICE_PORT 2>/dev/null | grep Connected | wc -l`
 if [ $result -eq 1 ]; then
  echo "$MICRO_SERVICE_NAME is ok."

diff --git a/bin/install.sh b/bin/install.sh
@@ -15,14 +15,12 @@
 # limitations under the License.
 #
 
-
 #Actively load user env
 source ~/.bash_profile
 
 shellDir=`dirname $0`
 workDir=`cd ${shellDir}/..;pwd`
 
-
 #To be compatible with MacOS and Linux
 txt=""
 if [[ "$OSTYPE" == "darwin"* ]]; then
@@ -57,14 +55,14 @@ fi
 }
 
 function checkPythonAndJava(){
- python --version
- isSuccess "execute python --version"
- java -version
- isSuccess "execute java --version"
+ python --version
+ isSuccess "execute python --version"
+ java -version
+ isSuccess "execute java --version"
 }
 
 function checkHadoopAndHive(){
- hadoopVersion="`hdfs version`"
+ hadoopVersion="`hdfs version`"
  defaultHadoopVersion="2.7"
  checkversion "$hadoopVersion" $defaultHadoopVersion hadoop
  checkversion "$(whereis hive)" "1.2" hive
@@ -118,15 +116,18 @@ need_cmd() {
 
 
 sh ${workDir}/bin/checkEnv.sh
-isSuccess "failed to check env"
+isSuccess "check env"
 
 ##load config
 echo "step1:load config "
-source ${workDir}/conf/config.sh
-source ${workDir}/conf/db.sh
+export LINKIS_CONFIG_PATH=${LINKIS_CONFIG_PATH:-"${workDir}/conf/config.sh"}
+export LINKIS_DB_CONFIG_PATH=${LINKIS_DB_CONFIG_PATH:-"${workDir}/conf/db.sh"}
+export DISTRIBUTION=${DISTRIBUTION:-"${workDir}/conf/config.sh"}
+source ${LINKIS_CONFIG_PATH}
+source ${LINKIS_DB_CONFIG_PATH}
+source ${DISTRIBUTION}
 isSuccess "load config"
 
-
 local_host="`hostname --fqdn`"
 
 ipaddr=$(ip addr | awk '/^[0-9]+: / {}; /inet.*global/ {print gensub(/(.*)\/(.*)/, "\\1", "g", $2)}')
@@ -171,27 +172,26 @@ function copyFile(){
 
 }
 
-##env check
-echo "Please enter the mode selection such as: 1"
-echo " 1: Lite"
-echo " 2: Simple"
-echo " 3: Standard"
-echo ""
+##install mode choice
 
-INSTALL_MODE=1
+if [ "$INSTALL_MODE" == "" ];then
+ echo "Please enter the mode selection such as: 1"
+ echo " 1: Lite"
+ echo " 2: Simple"
+ echo " 3: Standard"
+ echo ""
+ read -p "Please input the choice:" idx
+ INSTALL_MODE=$idx
+fi
 
-read -p "Please input the choice:" idx
-if [[ '1' = "$idx" ]];then
- INSTALL_MODE=1
- echo "You chose Lite installation mode"
+if [[ '1' = "$INSTALL_MODE" ]];then
+ echo "You chose Lite installation mode" 
  checkPythonAndJava
-elif [[ '2' = "$idx" ]];then
- INSTALL_MODE=2
+elif [[ '2' = "$INSTALL_MODE" ]];then
  echo "You chose Simple installation mode"
  checkPythonAndJava
  checkHadoopAndHive
-elif [[ '3' = "$idx" ]];then
- INSTALL_MODE=3
+elif [[ '3' = "$INSTALL_MODE" ]];then
  echo "You chose Standard installation mode"
  checkPythonAndJava
  checkHadoopAndHive
@@ -201,7 +201,6 @@ else
  exit 1
 fi
 
-
 ##env check
 echo "Do you want to clear Linkis table information in the database?"
 echo " 1: Do not execute table-building statements"
@@ -227,22 +226,68 @@ echo "create hdfs directory and local directory"
 if [ "$WORKSPACE_USER_ROOT_PATH" != "" ]
 then
  localRootDir=$WORKSPACE_USER_ROOT_PATH
- if [[ $WORKSPACE_USER_ROOT_PATH == file://* ]]
- then
+ if [[ $WORKSPACE_USER_ROOT_PATH == file://* ]];then
  localRootDir=${WORKSPACE_USER_ROOT_PATH#file://}
+ mkdir -p $localRootDir/$deployUser
+ sudo chmod -R 775 $localRootDir/$deployUser
+ elif [[ $WORKSPACE_USER_ROOT_PATH == hdfs://* ]];then
+ localRootDir=${WORKSPACE_USER_ROOT_PATH#hdfs://}
+ hdfs dfs -mkdir -p $localRootDir/$deployUser
+ hdfs dfs -chmod -R 775 $localRootDir/$deployUser
+ else
+ echo "does not support $WORKSPACE_USER_ROOT_PATH filesystem types"
  fi
- mkdir -p $localRootDir/$deployUser
 fi
-isSuccess "create local directory"
+isSuccess "create $WORKSPACE_USER_ROOT_PATH directory"
+
+
 if [ "$HDFS_USER_ROOT_PATH" != "" ]
 then
- hdfs dfs -mkdir -p $HDFS_USER_ROOT_PATH/$deployUser
+ localRootDir=$HDFS_USER_ROOT_PATH
+ if [[ $HDFS_USER_ROOT_PATH == file://* ]];then
+ localRootDir=${HDFS_USER_ROOT_PATH#file://}
+ mkdir -p $localRootDir/$deployUser
+ sudo chmod -R 775 $localRootDir/$deployUser
+ elif [[ $HDFS_USER_ROOT_PATH == hdfs://* ]];then
+ localRootDir=${HDFS_USER_ROOT_PATH#hdfs://}
+ hdfs dfs -mkdir -p $localRootDir/$deployUser
+ hdfs dfs -chmod -R 775 $localRootDir/$deployUser
+ else
+ echo "does not support $HDFS_USER_ROOT_PATH filesystem types"
+ fi
 fi
+isSuccess "create $HDFS_USER_ROOT_PATH directory"
 
 
-##stop server
-#echo "step2,stop server"
-#sh ${workDir}/bin/stop-all.sh
+if [ "$RESULT_SET_ROOT_PATH" != "" ]
+then
+ localRootDir=$RESULT_SET_ROOT_PATH
+ if [[ $RESULT_SET_ROOT_PATH == file://* ]];then
+ localRootDir=${RESULT_SET_ROOT_PATH#file://}
+ mkdir -p $localRootDir/$deployUser
+ sudo chmod -R 775 $localRootDir/$deployUser
+ elif [[ $RESULT_SET_ROOT_PATH == hdfs://* ]];then
+ localRootDir=${RESULT_SET_ROOT_PATH#hdfs://}
+ hdfs dfs -mkdir -p $localRootDir/$deployUser
+ hdfs dfs -chmod -R 775 $localRootDir/$deployUser
+ else
+ echo "does not support $RESULT_SET_ROOT_PATH filesystem types" 
+ fi
+fi
+isSuccess "create $RESULT_SET_ROOT_PATH directory"
+
+##init db
+if [[ '2' = "$MYSQL_INSTALL_MODE" ]];then
+ mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/linkis_ddl.sql"
+ isSuccess "source linkis_ddl.sql"
+ mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/linkis_dml.sql"
+ isSuccess "source linkis_dml.sql"
+ echo "Rebuild the table"
+fi
+
+##Deal special symbol '#'
+HIVE_META_PASSWORD=$(echo ${HIVE_META_PASSWORD//'#'/'\#'})
+MYSQL_PASSWORD=$(echo ${MYSQL_PASSWORD//'#'/'\#'})
 
 ##Eurkea install
 SERVER_NAME=eureka
@@ -255,6 +300,7 @@ then
  SERVER_IP=$local_host
 fi
 EUREKA_URL=http://$SERVER_IP:$EUREKA_PORT/eureka/
+
 if ! executeCMD $SERVER_IP "test -e $SERVER_HOME"; then
  executeCMD $SERVER_IP "sudo mkdir -p $SERVER_HOME;sudo chown -R $deployUser:$deployUser $SERVER_HOME"
  isSuccess "create the dir of $SERVER_HOME"
@@ -372,16 +418,6 @@ echo "subsitution linkis.properties of $SERVER_NAME"
 echo "<----------------$SERVER_NAME:end------------------->"
 ##ResourceManager install end
 
-##init db
-if [[ '2' = "$MYSQL_INSTALL_MODE" ]];then
- mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/linkis_ddl.sql"
- isSuccess "source linkis_ddl.sql"
- mysql -h$MYSQL_HOST -P$MYSQL_PORT -u$MYSQL_USER -p$MYSQL_PASSWORD -D$MYSQL_DB --default-character-set=utf8 -e "source ${workDir}/db/linkis_dml.sql"
- isSuccess "source linkis_dml.sql"
- echo "Rebuild the table"
-fi
-
-
 
 ##PythonEM install
 PACKAGE_DIR=linkis/ujes/python
@@ -466,6 +502,7 @@ then
 fi
 if [ "$HIVE_META_PASSWORD" != "" ]
 then
+ HIVE_META_PASSWORD=$(echo ${HIVE_META_PASSWORD//'#'/'\#'})
  executeCMD $SERVER_IP "sed -i ${txt} \"s#hive.meta.password.*#hive.meta.password=$HIVE_META_PASSWORD#g\" $SERVER_CONF_PATH"
 fi
 isSuccess "subsitution linkis.properties of $SERVER_NAME"

diff --git a/bin/start-all.sh b/bin/start-all.sh
@@ -25,13 +25,14 @@ echo ${info}
 source /etc/profile
 source ~/.bash_profile
 
-workDir=`dirname "${BASH_SOURCE-$0}"`
-workDir=`cd "$workDir"; pwd`
-
-
-CONF_DIR="${workDir}"/../conf
-CONF_FILE=${CONF_DIR}/config.sh
-
+shellDir=`dirname $0`
+workDir=`cd ${shellDir}/..;pwd`
+
+CONF_DIR="${workDir}"/conf
+export LINKIS_DSS_CONF_FILE=${LINKIS_DSS_CONF_FILE:-"${CONF_DIR}/config.sh"}
+export DISTRIBUTION=${DISTRIBUTION:-"${CONF_DIR}/config.sh"}
+#source $LINKIS_DSS_CONF_FILE
+source ${DISTRIBUTION}
 function isSuccess(){
 if [ $? -ne 0 ]; then
  echo "Failed to " + $1
@@ -74,11 +75,11 @@ function executeCMD(){
 #if there is no LINKIS_INSTALL_HOME，we need to source config again
 if [ -z ${LINKIS_INSTALL_HOME} ];then
  echo "Info: LINKIS_INSTALL_HOME does not exist, we will source config"
- if [ ! -f "${CONF_FILE}" ];then
+ if [ ! -f "${LINKIS_DSS_CONF_FILE}" ];then
  echo "Error: can not find config file, start applications failed"
  exit 1
  else
- source ${CONF_FILE}
+ source ${LINKIS_DSS_CONF_FILE}
  fi
 fi
 APP_PREFIX="linkis-"
@@ -209,7 +210,7 @@ if ! executeCMD $SERVER_IP "test -e $SERVER_BIN"; then
  return
 fi
 
-sh $workDir/checkServices.sh $SERVER_NAME $SERVER_IP $SERVER_PORT
+sh $workDir/bin/checkServices.sh $SERVER_NAME $SERVER_IP $SERVER_PORT
 isSuccess "start $SERVER_NAME "
 echo "<-------------------------------->"
 sleep 3
@@ -282,4 +283,4 @@ SERVER_IP=$JDBC_INSTALL_IP
 SERVER_PORT=$JDBC_ENTRANCE_PORT
 checkServer
 
-echo "Linkis started successfully"
+echo "Linkis started successfully"
diff --git a/bin/stop-all.sh b/bin/stop-all.sh
@@ -30,8 +30,9 @@ workDir=`cd "$workDir"; pwd`
 
 
 CONF_DIR="${workDir}"/../conf
-CONF_FILE=${CONF_DIR}/config.sh
-
+export LINKIS_DSS_CONF_FILE=${LINKIS_DSS_CONF_FILE:-"${CONF_DIR}/config.sh"}
+export DISTRIBUTION=${DISTRIBUTION:-"${CONF_DIR}/config.sh"}
+source ${DISTRIBUTION}
 function isSuccess(){
 if [ $? -ne 0 ]; then
  echo "ERROR: " + $1
@@ -74,11 +75,11 @@ function executeCMD(){
 #if there is no LINKIS_INSTALL_HOME，we need to source config again
 if [ -z ${LINKIS_INSTALL_HOME} ];then
  echo "Warning: LINKIS_INSTALL_HOME does not exist, we will source config"
- if [ ! -f "${CONF_FILE}" ];then
+ if [ ! -f "${LINKIS_DSS_CONF_FILE}" ];then
  echo "Error: can not find config file, stop applications failed"
  exit 1
  else
- source ${CONF_FILE}
+ source ${LINKIS_DSS_CONF_FILE}
  fi
 fi
 APP_PREFIX="linkis-"

diff --git a/bml/bml-engine-hook/pom.xml b/bml/bml-engine-hook/pom.xml
@@ -5,7 +5,7 @@
  <parent>
  <artifactId>linkis</artifactId>
  <groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.2</version>
+ <version>0.9.3</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
 

diff --git a/bml/bmlclient/pom.xml b/bml/bmlclient/pom.xml
@@ -5,7 +5,7 @@
  <parent>
  <artifactId>linkis</artifactId>
  <groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.2</version>
+ <version>0.9.3</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
 

diff --git a/bml/bmlcommon/pom.xml b/bml/bmlcommon/pom.xml
@@ -5,7 +5,7 @@
  <parent>
  <artifactId>linkis</artifactId>
  <groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.2</version>
+ <version>0.9.3</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>
 

diff --git a/bml/bmlserver/pom.xml b/bml/bmlserver/pom.xml
@@ -5,7 +5,7 @@
  <parent>
  <artifactId>linkis</artifactId>
  <groupId>com.webank.wedatasphere.linkis</groupId>
- <version>0.9.2</version>
+ <version>0.9.3</version>
  </parent>
  <modelVersion>4.0.0</modelVersion>