diff --git a/.gitignore b/.gitignore
index 7f93f82..087cd1d 100644
--- a/.gitignore
+++ b/.gitignore
@@ -28,3 +28,4 @@ node_modules
*.pyc
.gradle
+*pid.lock
\ No newline at end of file
diff --git a/Makefile b/Makefile
index 499442f..33cd60c 100644
--- a/Makefile
+++ b/Makefile
@@ -45,8 +45,16 @@ unpack-flink:
mkdir -p tmp && rm -f tmp/flink-1.12.3-bin-scala_2.11.tgz && wget -P tmp/ https://mirrors.huaweicloud.com/apache/flink/flink-1.12.3/flink-1.12.3-bin-scala_2.11.tgz
rm -rf tmp/jax/flink-1.12.3 && tar -zxf tmp/flink-1.12.3-bin-scala_2.11.tgz -C tmp/jax && rm -f tmp/flink-1.12.3-bin-scala_2.11.tgz
+
+.PHONY: unpack-builtin-stages
+unpack-builtin-stages:
+ mkdir -p /tmp/jax/builtin-stages && rm -rf /tmp/jax/builtin-stages
+ cp jax-flink-job-1_9/target/jax-flink-job-1_9-${JAX_VERSION}.jar tmp/jax/jax/jar_dir/ && mv tmp/jax/jax/jar_dir/jax-flink-job-1_9-${JAX_VERSION}.jar tmp/jax/jax/jar_dir/jax-flink-job-1_9_jars.jar
+ cp jax-flink-job/target/jax-flink-job-${JAX_VERSION}.jar tmp/jax/jax/jar_dir/ && mv tmp/jax/jax/jar_dir/jax-flink-job-${JAX_VERSION}.jar tmp/jax/jax/jar_dir/jax-flink-job_jars.jar
+
+
.PHONY: package-all
-package-all: unpack-jax unpack-flink
+package-all: unpack-jax unpack-flink unpack-builtin-stages
cd tmp && tar -zcf jax-all-${JAX_VERSION}-${DATE}-${COMMIT_ID}.tar.gz jax/
.PHONY: image
diff --git a/README.md b/README.md
index a27925d..2b3b7ec 100644
--- a/README.md
+++ b/README.md
@@ -1,29 +1,206 @@
-# Package
+
+# Introduction 简介
+
+JAX项目由**擎创数据团队**研发并开源的数据中台项目, 帮助企业构建数据中台,提供数据集成、数据处理、数据开发、作业管理等数据服务;
+
+JAX提供可拖拽的流批作业开发,可以轻松实现实时和离线的数据清洗、指标计算、数据探索等功能;
+
+JAX是一款轻量级的数据平台,JAX本身并不强制与某种数据存储、计算框架绑定,也不与数据仓库、数据湖绑定,JAX更多的是构建一层数据抽象层,将企业中纷繁复杂的大数据基础设施统一起来,用户可以以平滑的方式将既有的数据存储和计算资源与JAX配合工作。JAX也有能力对接多种多样的数据库,例如关系型数据库、数据仓库、时序数据库、文档数据库等。也支持用户可选择不同类型的计算资源, 既可用选Hadoop, 也可以使用Kubernetes提供计算资源。
+
+
+# Features 功能
+
+### 功能概况
+作业管理
+* 流作业开发和管理
+* 批作业开发和管理
+
+算子管理
+* 拓展包管理
+* 算子列表:
+
+系统管理
+* 集群管理
+* 框架管理
+
+
+### 功能详细介绍和使用
+
+请参照 [Jax功能模块](docs/JaxFeatures.md) 详细了解Jax功能模块。
+
+
+
+# QuickStart 快速构建和体验
+
+依赖环境: docker, docker-compose
+
+预编译好的JAX已发布到 aliyun docker镜像, 这里提供与MySQL, Flink, Kafka等服务快速集成部署的docker-compose.yml
+
+
+### 创建docker-compose.yml
+
+使用如下命令新建 docker-compose.yml 或手动创建下文内容的 docker-compose.yml:
```sh
-$ make package-all
-```
+tee docker-compose.yml <<'EOF'
-# Docker
+version: "3"
+services:
+ web:
+ image: registry.cn-hangzhou.aliyuncs.com/eoitek/jax:1.0.0
+ depends_on:
+ - db
+ - taskmanager
+ environment:
+ MYSQL_HOST: db
+ MYSQL_USER: root
+ MYSQL_PASSWORD: my-secret-pw
+ ports:
+ - "49999:9999"
+ volumes:
+ - web-data:/app/jax/jar_dir
+ networks:
+ - jax
+ db:
+ image: mysql:5.7.25
+ environment:
+ MYSQL_ROOT_PASSWORD: my-secret-pw
+ MYSQL_DATABASE: jax_db
+ networks:
+ - jax
+ volumes:
+ - db-data:/var/lib/mysql
+ taskmanager:
+ image: ${FLINK_IMAGE:-flink:1.9.1-scala_2.11}
+ command: taskmanager
+ depends_on:
+ - jobmanager
+ environment:
+ - |
+ FLINK_PROPERTIES=
+ jobmanager.rpc.address: jobmanager
+ taskmanager.numberOfTaskSlots: 4
+ networks:
+ - jax
+ jobmanager:
+ image: ${FLINK_IMAGE:-flink:1.9.1-scala_2.11}
+ command: jobmanager
+ environment:
+ - |
+ FLINK_PROPERTIES=
+ jobmanager.rpc.address: jobmanager
+ ports:
+ - "48081:8081"
+ networks:
+ - jax
+ zookeeper:
+ image: wurstmeister/zookeeper
+ networks:
+ - jax
+ kafka:
+ image: wurstmeister/kafka
+ depends_on:
+ - zookeeper
+ environment:
+ KAFKA_ADVERTISED_HOST_NAME: ${KAFKA_AD_IP:-kafka}
+ KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
+ KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+ KAFKA_BROKER_ID: 1
+ KAFKA_CREATE_TOPICS: sourceTopic:2:1,sinkTopic:2:1
+ networks:
+ - jax
-requirements
+networks:
+ jax:
+
+volumes:
+ db-data:
+ web-data:
+
+EOF
```
-node/npm/yarn
-wget
-docker/docker compose
+
+其中的 registry.cn-hangzhou.aliyuncs.com/eoitek/jax:1.0.0 即为已准备好的JAX docker镜像
+
+
+### 一键启动各服务
+
+```sh
+
+docker-compose up
+
```
-With flink 1.9 standalone cluster:
+服务验证和访问:
+- 访问JAX服务: http://{ip}:49999
+- 访问Flink Standalone 测试集群: http://{ip}:48081
+
+
+
+
+# Compile and Deploy 编译部署
+
+
+### Requriments 编译环境要求
+
+- 基础命令: tar, make, java;
+- 前端命令: nodejs, npm, yarn/yarnpkg, vue-cli;
+- Docker命令: docker, docker-compose ;
+
+
+
+### Compile 编译部署命令
+
+使用make命令(基于Makefile) 进行项目编译和打包
```sh
-$ make image
-$ docker-compose up
+# 编译并打包
+make package-all
+
+# 或者直接: 打包+ 制作Docker镜像
+make image
+
```
-With flink 1.12 standalone cluster:
+编译打包好的tar.gz文件即位于 tmp 目录下的 jax-all-xxx.tar.gz 压缩文件;
+解压后, 需要对 jax/application.yml配置文件 配置好正确的如下变量,才能start.sh启动
+- jax.home(或$JAX_HOME环境变量) 为解压的JAX安装目录;
+- spring.datasource中配置正确的MySQL url账号密码
+
+
+配置成功后, 启动JAX服务
```sh
-$ make image
-$ FLINK_IMAGE=flink:1.12.3-scala_2.11 docker-compose up
+cd $JAX_HOME
+
+./start.sh
+
+
+# 项目启动成功, 由如下打印
+ _ _ |_ _ _|_. ___ _ | _
+| | |\/|_)(_| | |_\ |_)||_|_\
+ / |
+ 3.1.2
+Jax Application is Ready
+
```
+
+
+
+### 详细编译部署手册
+
+项目编译和部署详细文档, 可参见 [编译部署](docs/CompileAndDeploy.md)
+
+
+
+# Documents
+
+- 编译部署和环境准备相关问题, 可参见 [** 常见问题列表 **](https://datasalon.yuque.com/staff-dg3tgh/pg6cpg/uem0ig)
+- 生产运维中常见问题, 可参考 [** 生产运维常见问题 **](https://datasalon.yuque.com/staff-dg3tgh/pg6cpg/uem0ig)
+
+
+# Contributing
+
+# Communication
+
diff --git a/docker-compose.yml b/docker-compose.yml
index afbdf04..b2fc686 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -10,11 +10,22 @@ services:
MYSQL_USER: root
MYSQL_PASSWORD: my-secret-pw
ports:
- - "9999:9999"
+ - "49999:9999"
volumes:
- web-data:/app/jax/jar_dir
networks:
- jax
+ db:
+ image: mysql:5.7.25
+ environment:
+ MYSQL_ROOT_PASSWORD: my-secret-pw
+ MYSQL_DATABASE: jax_db
+ ports:
+ - "43306:3306"
+ networks:
+ - jax
+ volumes:
+ - db-data:/var/lib/mysql
taskmanager:
image: ${FLINK_IMAGE:-flink:1.9.1-scala_2.11}
command: taskmanager
@@ -35,18 +46,25 @@ services:
FLINK_PROPERTIES=
jobmanager.rpc.address: jobmanager
ports:
- - "8081:8081"
+ - "48081:8081"
networks:
- jax
- db:
- image: mysql:5.7.25
+ zookeeper:
+ image: wurstmeister/zookeeper
+ networks:
+ - jax
+ kafka:
+ image: wurstmeister/kafka
+ depends_on:
+ - zookeeper
environment:
- MYSQL_ROOT_PASSWORD: my-secret-pw
- MYSQL_DATABASE: jax_db
+ KAFKA_ADVERTISED_HOST_NAME: ${KAFKA_AD_IP:-kafka}
+ KAFKA_LISTENERS: PLAINTEXT://0.0.0.0:9092
+ KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+ KAFKA_BROKER_ID: 1
+ KAFKA_CREATE_TOPICS: sourceTopic:2:1,sinkTopic:2:1
networks:
- jax
- volumes:
- - db-data:/var/lib/mysql
networks:
jax:
diff --git a/docs/CompileAndDeploy.md b/docs/CompileAndDeploy.md
new file mode 100644
index 0000000..859fce9
--- /dev/null
+++ b/docs/CompileAndDeploy.md
@@ -0,0 +1,122 @@
+
+
+### Environment Requriment
+
+nodejs, npm, yarn, vue-cli
+tar dos2unix
+
+### Jax运行环境
+
+### Jax项目编译环境
+
+Jax项目手动编译, 可在Window/Linux/Unix 准备好如下命令
+- java
+- maven
+- nodejs 10.0+
+- yarn/yarnpkg
+- vue-cli
+- dos2unix
+- make
+- docker-compose
+
+
+
+### Download 预编译版下载
+
+
+
+### Compile & Package 编译打包
+
+
+cd 到jax的根目录, 使用make 命令编译和打包出整个jax-all-1.0.0-xxx.tar.gz安装包(位于其tmp目录下);
+
+make package-all主要包括:
+- mvn clean package -DskipTests 后端编译打包
+- yarn install && yarn build 前端编译打包
+- wget flink-xx.tgz && tar -zxf flink-xx.tgz 下载并解压安装flink/spark
+
+
+```sh
+cd jax
+
+make package-all
+
+tar -xvf tmp/jax-all-*.tar.gz -C /opt/jax
+
+```
+
+编译打包注意点:
+- 如果make package-all失败, 可用尝试分别对后端项目(mvn clean package )和 前端项目(cd jax-ui && yarnpkg install & yarnpkg build) 进行编译测试, 确认环境没问题;
+- make packag-all中使用 ./mvnw 第一次执行时可能很慢(在下载maven/wrapper包), 需耐心等待,不宜打断;
+- 编译中若遇到报错,可参考 [常见问题列表](https://datasalon.yuque.com/staff-dg3tgh/pg6cpg/uem0ig)
+
+
+### Deploy 项目部署
+
+项目启动前, 需要确保Hadoop已安装好;
+
+解压jax-all-xxx.tar.gz 项目后, 直接执行其 start.sh脚本即可启动JAX项目, 如见到打印: Jax Application is Ready, 则一般项目启动成功;
+```sh
+$ ./start.sh
+
+# 项目启动成功, 由如下打印
+ _ _ |_ _ _|_. ___ _ | _
+| | |\/|_)(_| | |_\ |_)||_|_\
+ / |
+ 3.1.2
+Jax Application is Ready
+
+```
+
+如果项目启动成功, 默认jax-web访问端口是9999, 访问 http://{hostname}:9999 即可;
+
+如果没有此打印,或者 9999端口未开,则可能项目启动失败, 具体原因详解 logs/jax-web.error.log 排查;
+- 一般启动失败是 jax/application.yml中配置缺失或配错导致;
+- 注意其中 jax.home, MySQL的DB和账号密码;
+
+
+##### 开启Debug
+
+```sh
+export JAX_WEB_DEBUG_PORT=45000
+```
+在start.sh 脚本中,若JAX_WEB_DEBUG_PORT存在则会增加jvm debug参数, 方便进行远程Debug和调试;
+
+
+### Docker镜像和运行
+
+提前确认相关环境命令:
+- node/npm/yarn
+- mvn/make
+- docker/docker-compose
+
+
+首先需要生成 jax docker镜像
+```sh
+make image
+```
+
+
+默认安装的flink计算引擎是1.9.1版本,并会启1个Flink Standalone模式容器,
+本测试Docker Demo中会启1个Zookeeper/Kafka服务, 需要指定一个用于外网可访问的Kafka广播IP环境变量: $KAFKA_AD_IP
+可先通过如下命令生成 KAFKA_AD_IP=宿主机IP变量, 再 docker-compose up 构建和启动 mysql,redis,kafka,jax等服务;
+
+```sh
+export KAFKA_AD_IP=$(ip addr|grep "global ens"| tail -n1|awk '{print $2}'|awk -F"/" '{print $1}')
+# 或者在命令行直接赋值环境变量: KAFKA_AD_IP
+# export KAFKA_AD_IP=192.168.51.124
+echo $KAFKA_AD_IP
+```
+
+With flink 1.9 standalone cluster:
+```sh
+docker-compose up
+```
+
+With flink 1.12 standalone cluster:
+
+```sh
+$ make image
+$ FLINK_IMAGE=flink:1.12.3-scala_2.11 docker-compose up
+```
+
diff --git a/docs/JaxFeatures.md b/docs/JaxFeatures.md
new file mode 100644
index 0000000..52f7ba3
--- /dev/null
+++ b/docs/JaxFeatures.md
@@ -0,0 +1,66 @@
+# Jax 功能模块介绍
+
+## 1.作业管理
+
+#### 1.1 流作业开发和管理
+
+流作业模块: 提供基于Flink算子的流任务编排和启停管理
+
+
+用户可在拖拽出3种不同算子(输入/输出/计算),构建自己的flink DAG计算逻辑;
+
+
+JAX的可视化编排能够提供用户直观的操作界面,用户得以轻松编排出作业。同时,可视化还提供强大的作业调试功能,用户可以在发布作业前进行调试,以确保逻辑的正确性。
+
+JAX的可视化编排与市面上的其他类似产品是有本质的区别。JAX编排形成的是一个单独的实时处理或离线处理作业。而许多其他产品中的DAG是表示一组作业的相互依赖关系,并通过DAG的描述关系,先后执行其中的作业,其中的作业可以是实时处理也可以是离线作业。
+所以JAX的可视化编排更像是给Apache Beam的披上了一层外衣(但JAX底层并不基于Apache Beam),而其他产品更像是Azkaban。可视化作业编排能够同时支持实时(Flink)处理和离线(Spark)处理
+
+
+#### 1.2 批作业开发和管理
+
+与流作业管理类似, 提供基于Spark的批作业管理
+
+
+
+
+
+
+## 2. 算子管理
+
+#### 2.1 拓展包管理
+
+
+扩展包用于扩展可视化作业编排、SQL作业、机器学习工作台中的算子。可使用编译自带的
+
+用户可以通过编写Java或Python代码,实现自定义的算子。JAX能通过扩展包管理功能识别这些算子,从而在上述功能中直接使用。
+
+有了扩展包的支持,可以说JAX平台的能力得以无限扩展。用户只需要编写算子的核心的处理代码,无需开发前端的表单,JAX能够自动识别参数,并转化为表单。
+
+
+#### 2.2 算子列表:
+
+
+算子管理模块主要展示[拓展包管理]中加载的各算子详情;
+
+
+
+## 3. 系统管理
+
+#### 3.1 框架管理
+
+
+关键管理用于定义Flink或Spark计算框架的基础配置和环境变量;
+框架管理提供类似模块的功能, 将自带的flink/spark安装包结合默认参数提供给用户开箱即用;
+用户可根据自身需要参考框架模板,配置自定义的 flink/spark计算引擎和相关参数;
+
+
+#### 3.2 集群管理
+
+集群是JAX运行作业的基础设施。集群管理是JAX对接企业中已有的数据平台的基本方法。借此,JAX得以将作业运行在企业已有的数据平台上。
+
+集群管理支持对接常见的大数据平台(Hadoop、CDH、TDH、Fusioninsight、Kubernetes等)。作为轻量级平台,JAX不对集群本身做复杂的管理,仅提供基本的信息展示
+
+
+
+
+
diff --git "a/docs/images/\346\211\271\344\275\234\344\270\232\347\256\241\347\220\206.png" "b/docs/images/\346\211\271\344\275\234\344\270\232\347\256\241\347\220\206.png"
new file mode 100644
index 0000000..fc6f0f2
Binary files /dev/null and "b/docs/images/\346\211\271\344\275\234\344\270\232\347\256\241\347\220\206.png" differ
diff --git "a/docs/images/\346\213\223\345\261\225\345\214\205\347\256\241\347\220\206.png" "b/docs/images/\346\213\223\345\261\225\345\214\205\347\256\241\347\220\206.png"
new file mode 100644
index 0000000..f28af3e
Binary files /dev/null and "b/docs/images/\346\213\223\345\261\225\345\214\205\347\256\241\347\220\206.png" differ
diff --git "a/docs/images/\346\241\206\346\236\266\347\256\241\347\220\206.png" "b/docs/images/\346\241\206\346\236\266\347\256\241\347\220\206.png"
new file mode 100644
index 0000000..2258b10
Binary files /dev/null and "b/docs/images/\346\241\206\346\236\266\347\256\241\347\220\206.png" differ
diff --git "a/docs/images/\346\265\201\344\275\234\344\270\232pipeline.png" "b/docs/images/\346\265\201\344\275\234\344\270\232pipeline.png"
new file mode 100644
index 0000000..15a22cc
Binary files /dev/null and "b/docs/images/\346\265\201\344\275\234\344\270\232pipeline.png" differ
diff --git "a/docs/images/\346\265\201\344\275\234\344\270\232\346\213\226\346\213\275\347\274\226\350\276\221.png" "b/docs/images/\346\265\201\344\275\234\344\270\232\346\213\226\346\213\275\347\274\226\350\276\221.png"
new file mode 100644
index 0000000..25db071
Binary files /dev/null and "b/docs/images/\346\265\201\344\275\234\344\270\232\346\213\226\346\213\275\347\274\226\350\276\221.png" differ
diff --git "a/docs/images/\347\256\227\345\255\220\347\256\241\347\220\206.png" "b/docs/images/\347\256\227\345\255\220\347\256\241\347\220\206.png"
new file mode 100644
index 0000000..91627a0
Binary files /dev/null and "b/docs/images/\347\256\227\345\255\220\347\256\241\347\220\206.png" differ
diff --git "a/docs/images/\351\233\206\347\276\244\347\256\241\347\220\206.png" "b/docs/images/\351\233\206\347\276\244\347\256\241\347\220\206.png"
new file mode 100644
index 0000000..1fa8395
Binary files /dev/null and "b/docs/images/\351\233\206\347\276\244\347\256\241\347\220\206.png" differ
diff --git a/jax-web/pom.xml b/jax-web/pom.xml
index daace4a..37603c1 100644
--- a/jax-web/pom.xml
+++ b/jax-web/pom.xml
@@ -118,6 +118,24 @@
+
+ org.projectlombok
+ lombok
+ 1.16.22
+ test
+
+
+ io.springfox
+ springfox-swagger2
+
+
+ io.springfox
+ springfox-swagger-ui
+
+
+ junit
+ junit
+
@@ -213,51 +231,4 @@
-
-
-
- dev_skip
-
-
- dev_skip
-
- false
-
-
-
-
-
- com.mycila
- license-maven-plugin
-
- true
-
-
-
- org.apache.maven.plugins
- maven-checkstyle-plugin
-
- true
-
-
-
- org.apache.maven.plugins
- maven-enforcer-plugin
-
- true
-
-
-
- org.apache.maven.plugins
- maven-javadoc-plugin
-
- true
-
-
-
-
-
-
-
-
\ No newline at end of file
diff --git a/jax-web/release/debug/debug.sh b/jax-web/release/debug/debug.sh
index 15b8ded..e370c7f 100755
--- a/jax-web/release/debug/debug.sh
+++ b/jax-web/release/debug/debug.sh
@@ -32,6 +32,7 @@ LoggingConfig4Log4j="file:${DIR}/flink-debug-docker.properties"
DEBUG_JAVA_OPTS="-Xmx1024m -Xms1024m ${DEBUG_JAVA_OPTS}"
-exec "$JAVA" ${DEBUG_JAVA_OPTS} -Dlogback.configurationFile="${LoggingConfig}" -Dlog4j.configuration="${LoggingConfig4Log4j}" "$@"
+# exec "$JAVA" ${DEBUG_JAVA_OPTS} -Dlogback.configurationFile="${LoggingConfig}" -Dlog4j.configuration="${LoggingConfig4Log4j}" "$@" > debug_out.log 2>&1
+$JAVA ${DEBUG_JAVA_OPTS} -Dlogback.configurationFile="${LoggingConfig}" -Dlog4j.configuration="${LoggingConfig4Log4j}" "$@" 2>&1 |tee $DIR/out_debug.log
diff --git a/jax-web/src/main/java/com/eoi/jax/web/common/config/AppConfig.java b/jax-web/src/main/java/com/eoi/jax/web/common/config/AppConfig.java
index 3dce51e..4cefeb4 100644
--- a/jax-web/src/main/java/com/eoi/jax/web/common/config/AppConfig.java
+++ b/jax-web/src/main/java/com/eoi/jax/web/common/config/AppConfig.java
@@ -28,7 +28,9 @@ public class AppConfig {
public static final String JAX_JOB_JAR_LIB = "jaxjobjarlib";
public static final String JAX_JOB_WORK_DIR = "jaxjobworkdir";
public static final String JAX_JOB_PYTHON_DIR = "jaxjobpythondir";
- public static final String HADOOP_ETC_RELATIVE = "etc/hadoop";
+ public static final String HADOOP_ETC_RELATIVE = "etc";
+ public static final String HADOOP_ETC_HADOOP_RELATIVE = "etc/hadoop";
+ public static final String HADOOP_CONF = "conf";
public static final String HADOOP_YARN_BIN_RELATIVE = "bin/yarn";
public static final String FLINK_BIN_RELATIVE = "bin/flink";
public static final String SPARK_BIN_RELATIVE = "bin/spark-submit";
diff --git a/jax-web/src/main/java/com/eoi/jax/web/common/config/RouterConfig.java b/jax-web/src/main/java/com/eoi/jax/web/common/config/RouterConfig.java
index 3d6b2b6..db48b4e 100644
--- a/jax-web/src/main/java/com/eoi/jax/web/common/config/RouterConfig.java
+++ b/jax-web/src/main/java/com/eoi/jax/web/common/config/RouterConfig.java
@@ -90,5 +90,10 @@ protected Resource getResource(String resourcePath,
new FileSystemResource(ConfigLoader.load().jax.getWebsite().getIndex());
}
});
+
+
+ registry.addResourceHandler("swagger-ui.html")
+ .addResourceLocations("classpath:/META-INF/resources/");
+
}
}
diff --git a/jax-web/src/main/java/com/eoi/jax/web/common/config/SwaggerConfig.java b/jax-web/src/main/java/com/eoi/jax/web/common/config/SwaggerConfig.java
new file mode 100644
index 0000000..5a87826
--- /dev/null
+++ b/jax-web/src/main/java/com/eoi/jax/web/common/config/SwaggerConfig.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package com.eoi.jax.web.common.config;
+
+import io.swagger.annotations.ApiOperation;
+import org.springframework.boot.autoconfigure.condition.ConditionalOnProperty;
+import org.springframework.context.annotation.Bean;
+import org.springframework.context.annotation.Configuration;
+import springfox.documentation.builders.ApiInfoBuilder;
+import springfox.documentation.builders.PathSelectors;
+import springfox.documentation.builders.RequestHandlerSelectors;
+import springfox.documentation.service.ApiInfo;
+import springfox.documentation.spi.DocumentationType;
+import springfox.documentation.spring.web.plugins.Docket;
+import springfox.documentation.swagger2.annotations.EnableSwagger2;
+
+@Configuration
+@EnableSwagger2
+@ConditionalOnProperty(prefix = "jax.swagger", name = "swagger-ui-open", havingValue = "true")
+public class SwaggerConfig {
+
+ @Bean
+ public Docket createRestApi() {
+ ApiInfo apiInfo = new ApiInfoBuilder()
+ .title("JAX REST API")
+ .version("v1")
+ .description("")
+ .termsOfServiceUrl("")
+ .build();
+ return new Docket(DocumentationType.SWAGGER_2)
+ .apiInfo(apiInfo)
+ .enable(true) // 默认开启
+ .select()
+ .apis(RequestHandlerSelectors.withMethodAnnotation(ApiOperation.class))
+ .paths(PathSelectors.any())
+ .build();
+
+ }
+
+
+}
diff --git a/jax-web/src/main/java/com/eoi/jax/web/common/util/Common.java b/jax-web/src/main/java/com/eoi/jax/web/common/util/Common.java
index 8043bed..6f7e85c 100644
--- a/jax-web/src/main/java/com/eoi/jax/web/common/util/Common.java
+++ b/jax-web/src/main/java/com/eoi/jax/web/common/util/Common.java
@@ -17,6 +17,7 @@
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.io.resource.ClassPathResource;
import cn.hutool.core.util.StrUtil;
+import com.eoi.jax.web.common.config.ConfigLoader;
import java.io.File;
import java.nio.file.Path;
@@ -29,6 +30,8 @@
import java.util.function.Predicate;
public class Common {
+ public static final String VAR_JAX_HOME = "${JAX_HOME}";
+
private Common() {
// forbid init instance
}
@@ -114,4 +117,13 @@ public static byte[] readClassPathResource(String path, ClassLoader loader) {
ClassPathResource resource = new ClassPathResource(path, loader);
return resource.readBytes();
}
+
+ public static String replaceJaxHomeAsPath(String oldPath) {
+ if (oldPath.contains(VAR_JAX_HOME)) {
+ String replacedPath = StrUtil.replace(oldPath, VAR_JAX_HOME, ConfigLoader.load().jax.getHome());
+ return Paths.get(replacedPath).toString();
+ }
+ return oldPath;
+ }
+
}
diff --git a/jax-web/src/main/java/com/eoi/jax/web/common/util/HadoopUtil.java b/jax-web/src/main/java/com/eoi/jax/web/common/util/HadoopUtil.java
index d4477ac..5a8249e 100644
--- a/jax-web/src/main/java/com/eoi/jax/web/common/util/HadoopUtil.java
+++ b/jax-web/src/main/java/com/eoi/jax/web/common/util/HadoopUtil.java
@@ -17,6 +17,7 @@
import cn.hutool.core.io.FileUtil;
import cn.hutool.core.util.StrUtil;
import com.eoi.jax.web.common.ResponseCode;
+import com.eoi.jax.web.common.config.AppConfig;
import com.eoi.jax.web.common.exception.BizException;
import com.eoi.jax.web.dao.entity.TbCluster;
import com.eoi.jax.web.provider.cluster.ClusterVariable;
@@ -36,6 +37,8 @@
import java.io.FileInputStream;
import java.io.IOException;
import java.io.OutputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Collection;
@@ -78,6 +81,10 @@ public static String readHdfsServer(String configHome) {
public static String readYarnWebUrl(String configHome) {
Configuration config = readConfig(configHome);
+ return getYarnWebUrl(config);
+ }
+
+ public static String getYarnWebUrl(Configuration config) {
String schema = "http://";
String webUrl = null;
// HA
@@ -224,4 +231,42 @@ private static Path getUploadPath(FileSystem fs, String fileName) throws IOExcep
}
return finalPath;
}
+
+ public static Configuration getConfFromHadoopHome(String hadoopHome) {
+ String confDir = getConfDirFromHadoopHome(hadoopHome);
+ if (null != confDir) {
+ return readConfig(confDir);
+ }
+ return null;
+ }
+
+ public static String getConfDirFromHadoopHome(String hadoopHome) {
+ String confDir = null;
+ if (null == hadoopHome || hadoopHome.isEmpty()) {
+ return confDir;
+ }
+ String[] possiblePaths = new String[4];
+ try {
+ possiblePaths[0] = Paths.get(hadoopHome, AppConfig.HADOOP_ETC_RELATIVE).toString(); // eoitek hadoop
+ possiblePaths[1] = Paths.get(hadoopHome,AppConfig.HADOOP_ETC_HADOOP_RELATIVE).toString(); // apache hadoop
+ possiblePaths[2] = Paths.get(hadoopHome,AppConfig.HADOOP_CONF).toString(); // old hadoop version
+ if (null != System.getenv(AppConfig.HADOOP_CONF_DIR)) {
+ possiblePaths[3] = Paths.get(System.getenv(AppConfig.HADOOP_CONF_DIR)).toString(); // from env
+ }
+
+ for (String possibleHadoopConfPath :possiblePaths) {
+ if (Files.exists(Paths.get(possibleHadoopConfPath))
+ && Files.exists(Paths.get(possibleHadoopConfPath,"core-site.xml"))) {
+ confDir = possibleHadoopConfPath;
+ if (null != confDir) {
+ return confDir;
+ }
+ }
+ }
+ } catch (RuntimeException e) {
+ logger.warn("Catch Failure: parse HADOOP_CONF_DIR by hadoopHome=" + hadoopHome + ",cause: " + e.getMessage(),e);
+ }
+ return confDir;
+ }
+
}
diff --git a/jax-web/src/main/java/com/eoi/jax/web/controller/ClusterController.java b/jax-web/src/main/java/com/eoi/jax/web/controller/ClusterController.java
index 91756d9..f33a6e1 100644
--- a/jax-web/src/main/java/com/eoi/jax/web/controller/ClusterController.java
+++ b/jax-web/src/main/java/com/eoi/jax/web/controller/ClusterController.java
@@ -17,8 +17,14 @@
import com.eoi.jax.web.common.ResponseResult;
import com.eoi.jax.web.model.cluster.ClusterReq;
import com.eoi.jax.web.model.cluster.ClusterResp;
+import com.eoi.jax.web.model.cluster.bean.FlinkStandaloneClusterBean;
+import com.eoi.jax.web.model.cluster.bean.SparkStandaloneClusterBean;
+import com.eoi.jax.web.model.cluster.bean.YarnClusterBean;
+import com.eoi.jax.web.model.cluster.config.BeanConfigParser;
+import com.eoi.jax.web.model.cluster.config.ConfigDescription;
import com.eoi.jax.web.provider.resource.ClusterResourcePool;
import com.eoi.jax.web.service.ClusterService;
+import io.swagger.annotations.ApiOperation;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
@@ -29,6 +35,7 @@
import org.springframework.web.bind.annotation.RestController;
import java.util.List;
+import java.util.Map;
@RestController
public class ClusterController extends V1Controller {
@@ -36,16 +43,19 @@ public class ClusterController extends V1Controller {
@Autowired
private ClusterService clusterService;
+ @ApiOperation("获取cluster列表")
@GetMapping("cluster")
public ResponseResult> list() {
return new ResponseResult>().setEntity(clusterService.listCluster());
}
+ @ApiOperation("获取cluster详情")
@GetMapping("cluster/{clusterName}")
public ResponseResult get(@PathVariable("clusterName") String clusterName) {
return new ResponseResult().setEntity(clusterService.getCluster(clusterName));
}
+ @ApiOperation("新建cluster")
@PostMapping("cluster/{clusterName}")
public ResponseResult create(@PathVariable("clusterName") String clusterName,
@RequestBody ClusterReq req) {
@@ -53,6 +63,7 @@ public ResponseResult create(@PathVariable("clusterName") String cl
return new ResponseResult().setEntity(clusterService.createCluster(req));
}
+ @ApiOperation("更新cluster")
@PutMapping("cluster/{clusterName}")
public ResponseResult update(@PathVariable("clusterName") String clusterName,
@RequestBody ClusterReq req) {
@@ -60,19 +71,36 @@ public ResponseResult update(@PathVariable("clusterName") String cl
return new ResponseResult().setEntity(clusterService.updateCluster(req));
}
+ @ApiOperation("删除cluster详情")
@DeleteMapping("cluster/{clusterName}")
public ResponseResult delete(@PathVariable("clusterName") String clusterName) {
return new ResponseResult().setEntity(clusterService.deleteCluster(clusterName));
}
+ @ApiOperation("获取cluster详情")
@GetMapping("cluster/{clusterName}/resource")
public ResponseResult getResource(@PathVariable("clusterName") String clusterName) {
return new ResponseResult().setEntity(clusterService.getResource(clusterName));
}
+ @ApiOperation("获取cluster列表")
@GetMapping("cluster-resource")
public ResponseResult> listResource() {
return new ResponseResult>().setEntity(clusterService.listClusterResource());
}
+ @ApiOperation("获取cluster-动态表")
+ @GetMapping("cluster-options")
+ public ResponseResult