apache · MountHuang · Nov 20, 2019 · Nov 19, 2019
diff --git a/...loudProtocol/src/main/scala/com/webank/wedatasphere/linkis/protocol/mdq/MDQProtocol.scala b/...loudProtocol/src/main/scala/com/webank/wedatasphere/linkis/protocol/mdq/MDQProtocol.scala
@@ -0,0 +1,45 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.protocol.mdq
+
+import java.util
+trait MDQProtocol {
+
+}
+
+
+case class DDLRequest(params:util.Map[String, Object]) extends MDQProtocol
+
+/**
+ * 返回DDL语句
+ */
+case class DDLResponse(code:String) extends MDQProtocol
+
+abstract class DDLExecute(code:String) extends MDQProtocol
+
+/**
+ * sparkEngine返回执行是否成功
+ * @param status true is 成功， false is 失败
+ * @param code 返回的代码
+ */
+case class DDLExecuteResponse(status:Boolean, code:String, user:String) extends DDLExecute(code:String)
+
+
+/**
+ * 返回是否在MDQ收尾成功，包括数据库插入等操作
+ * @param status
+ */
+case class DDLCompleteResponse(status:Boolean) extends MDQProtocol
diff --git a/db/linkis_ddl.sql b/db/linkis_ddl.sql
@@ -368,3 +368,103 @@ CREATE TABLE `linkis_var_key` (
  PRIMARY KEY (`id`),
  KEY `application_id` (`application_id`)
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;
+
+-- ----------------------------
+-- Table structure for linkis_mdq_access
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_mdq_access`;
+CREATE TABLE `linkis_mdq_access` (
+ `id` bigint(20) NOT NULL AUTO_INCREMENT,
+ `table_id` bigint(20) NOT NULL,
+ `visitor` varchar(16) COLLATE utf8_bin NOT NULL,
+ `fields` varchar(255) COLLATE utf8_bin DEFAULT NULL,
+ `application_id` int(4) NOT NULL,
+ `access_time` datetime NOT NULL,
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+-- ----------------------------
+-- Table structure for linkis_mdq_field
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_mdq_field`;
+CREATE TABLE `linkis_mdq_field` (
+ `id` bigint(20) NOT NULL AUTO_INCREMENT,
+ `table_id` bigint(20) NOT NULL,
+ `name` varchar(64) COLLATE utf8_bin NOT NULL,
+ `alias` varchar(64) COLLATE utf8_bin DEFAULT NULL,
+ `type` varchar(64) COLLATE utf8_bin NOT NULL,
+ `comment` varchar(255) COLLATE utf8_bin DEFAULT NULL,
+ `express` varchar(255) COLLATE utf8_bin DEFAULT NULL,
+ `rule` varchar(128) COLLATE utf8_bin DEFAULT NULL,
+ `is_partition_field` tinyint(1) NOT NULL,
+ `is_primary` tinyint(1) NOT NULL,
+ `length` int(11) DEFAULT NULL,
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+-- ----------------------------
+-- Table structure for linkis_mdq_import
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_mdq_import`;
+CREATE TABLE `linkis_mdq_import` (
+ `id` bigint(20) NOT NULL AUTO_INCREMENT,
+ `table_id` bigint(20) NOT NULL,
+ `import_type` int(4) NOT NULL,
+ `args` varchar(255) COLLATE utf8_bin NOT NULL,
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+-- ----------------------------
+-- Table structure for linkis_mdq_lineage
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_mdq_lineage`;
+CREATE TABLE `linkis_mdq_lineage` (
+ `id` bigint(20) NOT NULL AUTO_INCREMENT,
+ `table_id` bigint(20) DEFAULT NULL,
+ `source_table` varchar(64) COLLATE utf8_bin DEFAULT NULL,
+ `update_time` datetime DEFAULT NULL,
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+-- ----------------------------
+-- Table structure for linkis_mdq_table
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_mdq_table`;
+CREATE TABLE `linkis_mdq_table` (
+ `id` bigint(255) NOT NULL AUTO_INCREMENT,
+ `database` varchar(64) COLLATE utf8_bin NOT NULL,
+ `name` varchar(64) COLLATE utf8_bin NOT NULL,
+ `alias` varchar(64) COLLATE utf8_bin DEFAULT NULL,
+ `creator` varchar(16) COLLATE utf8_bin NOT NULL,
+ `comment` varchar(255) COLLATE utf8_bin DEFAULT NULL,
+ `create_time` datetime NOT NULL,
+ `product_name` varchar(64) COLLATE utf8_bin DEFAULT NULL,
+ `project_name` varchar(255) COLLATE utf8_bin DEFAULT NULL,
+ `usage` varchar(128) COLLATE utf8_bin DEFAULT NULL,
+ `lifecycle` int(4) NOT NULL,
+ `use_way` int(4) NOT NULL,
+ `is_import` tinyint(1) NOT NULL,
+ `model_level` int(4) NOT NULL,
+ `is_external_use` tinyint(1) NOT NULL,
+ `is_partition_table` tinyint(1) NOT NULL,
+ `is_available` tinyint(1) NOT NULL,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `database` (`database`,`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+-- ----------------------------
+-- Table structure for linkis_mdq_table_info
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_mdq_table_info`;
+CREATE TABLE `linkis_mdq_table_info` (
+ `id` bigint(20) NOT NULL AUTO_INCREMENT,
+ `table_id` bigint(20) NOT NULL,
+ `table_last_update_time` datetime NOT NULL,
+ `row_num` bigint(20) NOT NULL,
+ `file_num` int(11) NOT NULL,
+ `table_size` varchar(32) COLLATE utf8_bin NOT NULL,
+ `partitions_num` int(11) NOT NULL,
+ `update_time` datetime NOT NULL,
+ `field_num` int(11) NOT NULL,
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
diff --git a/db/module/linkis_metadata.sql b/db/module/linkis_metadata.sql
@@ -0,0 +1,101 @@
+SET FOREIGN_KEY_CHECKS=0;
+
+-- ----------------------------
+-- Table structure for linkis_mdq_access
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_mdq_access`;
+CREATE TABLE `linkis_mdq_access` (
+ `id` bigint(20) NOT NULL AUTO_INCREMENT,
+ `table_id` bigint(20) NOT NULL,
+ `visitor` varchar(16) COLLATE utf8_bin NOT NULL,
+ `fields` varchar(255) COLLATE utf8_bin DEFAULT NULL,
+ `application_id` int(4) NOT NULL,
+ `access_time` datetime NOT NULL,
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+-- ----------------------------
+-- Table structure for linkis_mdq_field
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_mdq_field`;
+CREATE TABLE `linkis_mdq_field` (
+ `id` bigint(20) NOT NULL AUTO_INCREMENT,
+ `table_id` bigint(20) NOT NULL,
+ `name` varchar(64) COLLATE utf8_bin NOT NULL,
+ `alias` varchar(64) COLLATE utf8_bin DEFAULT NULL,
+ `type` varchar(64) COLLATE utf8_bin NOT NULL,
+ `comment` varchar(255) COLLATE utf8_bin DEFAULT NULL,
+ `express` varchar(255) COLLATE utf8_bin DEFAULT NULL,
+ `rule` varchar(128) COLLATE utf8_bin DEFAULT NULL,
+ `is_partition_field` tinyint(1) NOT NULL,
+ `is_primary` tinyint(1) NOT NULL,
+ `length` int(11) DEFAULT NULL,
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+-- ----------------------------
+-- Table structure for linkis_mdq_import
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_mdq_import`;
+CREATE TABLE `linkis_mdq_import` (
+ `id` bigint(20) NOT NULL AUTO_INCREMENT,
+ `table_id` bigint(20) NOT NULL,
+ `import_type` int(4) NOT NULL,
+ `args` varchar(255) COLLATE utf8_bin NOT NULL,
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+-- ----------------------------
+-- Table structure for linkis_mdq_lineage
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_mdq_lineage`;
+CREATE TABLE `linkis_mdq_lineage` (
+ `id` bigint(20) NOT NULL AUTO_INCREMENT,
+ `table_id` bigint(20) DEFAULT NULL,
+ `source_table` varchar(64) COLLATE utf8_bin DEFAULT NULL,
+ `update_time` datetime DEFAULT NULL,
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+-- ----------------------------
+-- Table structure for linkis_mdq_table
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_mdq_table`;
+CREATE TABLE `linkis_mdq_table` (
+ `id` bigint(255) NOT NULL AUTO_INCREMENT,
+ `database` varchar(64) COLLATE utf8_bin NOT NULL,
+ `name` varchar(64) COLLATE utf8_bin NOT NULL,
+ `alias` varchar(64) COLLATE utf8_bin DEFAULT NULL,
+ `creator` varchar(16) COLLATE utf8_bin NOT NULL,
+ `comment` varchar(255) COLLATE utf8_bin DEFAULT NULL,
+ `create_time` datetime NOT NULL,
+ `product_name` varchar(64) COLLATE utf8_bin DEFAULT NULL,
+ `project_name` varchar(255) COLLATE utf8_bin DEFAULT NULL,
+ `usage` varchar(128) COLLATE utf8_bin DEFAULT NULL,
+ `lifecycle` int(4) NOT NULL,
+ `use_way` int(4) NOT NULL,
+ `is_import` tinyint(1) NOT NULL,
+ `model_level` int(4) NOT NULL,
+ `is_external_use` tinyint(1) NOT NULL,
+ `is_partition_table` tinyint(1) NOT NULL,
+ `is_available` tinyint(1) NOT NULL,
+ PRIMARY KEY (`id`),
+ UNIQUE KEY `database` (`database`,`name`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
+
+-- ----------------------------
+-- Table structure for linkis_mdq_table_info
+-- ----------------------------
+DROP TABLE IF EXISTS `linkis_mdq_table_info`;
+CREATE TABLE `linkis_mdq_table_info` (
+ `id` bigint(20) NOT NULL AUTO_INCREMENT,
+ `table_id` bigint(20) NOT NULL,
+ `table_last_update_time` datetime NOT NULL,
+ `row_num` bigint(20) NOT NULL,
+ `file_num` int(11) NOT NULL,
+ `table_size` varchar(32) COLLATE utf8_bin NOT NULL,
+ `partitions_num` int(11) NOT NULL,
+ `update_time` datetime NOT NULL,
+ `field_num` int(11) NOT NULL,
+ PRIMARY KEY (`id`)
+) ENGINE=InnoDB DEFAULT CHARSET=utf8 COLLATE=utf8_bin;
diff --git a/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/dao/MdqDao.java b/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/dao/MdqDao.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2019 WeBank
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package com.webank.wedatasphere.linkis.metadata.dao;
+
+
+import com.webank.wedatasphere.linkis.metadata.domain.mdq.po.MdqField;
+import com.webank.wedatasphere.linkis.metadata.domain.mdq.po.MdqImport;
+import com.webank.wedatasphere.linkis.metadata.domain.mdq.po.MdqLineage;
+import com.webank.wedatasphere.linkis.metadata.domain.mdq.po.MdqTable;
+import org.apache.ibatis.annotations.Param;
+
+import java.util.List;
+
+
+public interface MdqDao {
+
+ void activateTable(Long tableId);
+
+ MdqTable selectTableByName(@Param("database") String database, @Param("tableName") String tableName, @Param("user") String user);
+
+ List<MdqField> listMdqFieldByTableId(Long tableId);
+
+ void insertTable(MdqTable table);
+
+ void insertFields(@Param("mdqFieldList") List<MdqField> mdqFieldList);
+
+ void insertImport(MdqImport mdqImport);
+
+ void insertLineage(MdqLineage mdqLineage);
+
+ MdqTable selectTableForUpdate(@Param("database") String database, @Param("tableName") String tableName);
+
+ void deleteTableBaseInfo(Long id);
+}
diff --git a/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/dao/impl/MdqDao.xml b/metadata/src/main/java/com/webank/wedatasphere/linkis/metadata/dao/impl/MdqDao.xml
@@ -0,0 +1,90 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<!--
+~ Copyright 2019 WeBank
+~
+~ Licensed under the Apache License, Version 2.0 (the "License");
+~ you may not use this file except in compliance with the License.
+~ You may obtain a copy of the License at
+~
+~ http://www.apache.org/licenses/LICENSE-2.0
+~
+~ Unless required by applicable law or agreed to in writing, software
+~ distributed under the License is distributed on an "AS IS" BASIS,
+~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+~ See the License for the specific language governing permissions and
+~ limitations under the License.
+-->
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
+<mapper namespace="com.webank.wedatasphere.linkis.metadata.dao.MdqDao">
+
+ <sql id="table_List">
+ id, `database`, `name`, `alias`, `creator`, `comment`,`create_time`,
+ `product_name`,`project_name`,`usage`,`lifecycle`,`use_way`,`is_import`,`model_level`,`is_external_use`,
+ `is_partition_table`,`is_available`
+ </sql>
+
+ <sql id="field_List">
+ id, `table_id`, `name`, `alias`, `type`, `comment`,`express`,
+ `rule`,`is_partition_field`,`is_primary`,`length`
+ </sql>
+
+ <sql id="import_List">
+ id, `table_id`, `import_type`, `args`
+ </sql>
+
+ <sql id="lineage_List">
+ id, `table_id`, `source_table`, `update_time`
+ </sql>
+
+ <update id="activateTable">
+ update linkis_mdq_table set is_available = true where id = #{tableId}
+ </update>
+
+ <select id="selectTableByName" resultType="com.webank.wedatasphere.linkis.metadata.domain.mdq.po.MdqTable">
+ select * from linkis_mdq_table
+ WHERE `database` = #{database} AND `name` = #{tableName} AND creator = #{user} AND is_available = true
+ </select>
+
+ <select id="listMdqFieldByTableId" resultType="com.webank.wedatasphere.linkis.metadata.domain.mdq.po.MdqField">
+ select * from linkis_mdq_field
+ WHERE table_id = #{tableId}
+ </select>
+
+ <insert id="insertTable" useGeneratedKeys="true" keyProperty="id" parameterType="com.webank.wedatasphere.linkis.metadata.domain.mdq.po.MdqTable">
+ INSERT INTO linkis_mdq_table(<include refid="table_List"/>) VALUES (
+ #{id},#{database},#{name},#{alias},#{creator},#{comment},#{createTime},#{productName},#{projectName},#{usage},
+ #{lifecycle},#{useWay},#{isImport},#{modelLevel},#{isExternalUse},#{isPartitionTable},#{isAvailable}
+ )
+ </insert>
+
+ <insert id="insertFields" parameterType="com.webank.wedatasphere.linkis.metadata.domain.mdq.po.MdqField">
+ INSERT INTO linkis_mdq_field
+ (<include refid="field_List"/>)
+ VALUES
+ <foreach collection="mdqFieldList" item="fields" separator="," >
+ (#{fields.id},#{fields.tableId},#{fields.name},#{fields.alias},#{fields.type},#{fields.comment},
+ #{fields.express},#{fields.rule},#{fields.isPartitionField},#{fields.isPrimary},#{fields.length})
+ </foreach>
+ </insert>
+
+ <insert id="insertImport" parameterType="com.webank.wedatasphere.linkis.metadata.domain.mdq.po.MdqImport">
+ INSERT INTO linkis_mdq_import(<include refid="import_List"/>) VALUES (
+ #{id},#{tableId},#{importType},#{args}
+ )
+ </insert>
+
+ <insert id="insertLineage" parameterType="com.webank.wedatasphere.linkis.metadata.domain.mdq.po.MdqLineage">
+ INSERT INTO linkis_mdq_lineage(<include refid="lineage_List"/>) VALUES (
+ #{id},#{tableId},#{sourceTable},#{updateTime}
+ )
+ </insert>
+
+ <select id="selectTableForUpdate" resultType="com.webank.wedatasphere.linkis.metadata.domain.mdq.po.MdqTable">
+ select * from linkis_mdq_table WHERE `database` = #{database} AND `name` = #{tableName} for UPDATE;
+ </select>
+
+ <delete id="deleteTableBaseInfo">
+ DELETE FROM linkis_mdq_table WHERE id = #{id}
+ </delete>
+
+</mapper>