natas_hw
/
dolphinscheduler0701
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Remove mapper usage in tools (#15073)

This commit is contained in:
Wenjun Ruan 2023-10-25 22:58:47 +08:00 committed by GitHub
parent ae847ba331
commit a5284e4024
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
45 changed files with 718 additions and 586 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
dolphinscheduler-alert/dolphinscheduler-alert-server/src/main/resources/application.yaml
View File

@ -40,6 +40,20 @@ spring:
leak-detection-threshold: 0
initialization-fail-timeout: 1
# Mybatis-plus configuration, you don't need to change it
mybatis-plus:
mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
type-aliases-package: org.apache.dolphinscheduler.dao.entity
configuration:
cache-enabled: false
call-setters-on-nulls: true
map-underscore-to-camel-case: true
jdbc-type-for-null: NULL
global-config:
db-config:
id-type: auto
banner: false
server:
port: 50053

14
dolphinscheduler-api/src/main/resources/application.yaml
View File

@ -88,6 +88,20 @@ springdoc:
path: /swagger-ui.html
packages-to-scan: org.apache.dolphinscheduler.api
# Mybatis-plus configuration, you don't need to change it
mybatis-plus:
mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
type-aliases-package: org.apache.dolphinscheduler.dao.entity
configuration:
cache-enabled: false
call-setters-on-nulls: true
map-underscore-to-camel-case: true
jdbc-type-for-null: NULL
global-config:
db-config:
id-type: auto
banner: false
management:
endpoints:
web:

13
dolphinscheduler-api/src/test/resources/application.yaml
View File

@ -29,6 +29,19 @@ spring:
username: sa
password: ""
mybatis-plus:
mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
type-aliases-package: org.apache.dolphinscheduler.dao.entity
configuration:
cache-enabled: false
call-setters-on-nulls: true
map-underscore-to-camel-case: true
jdbc-type-for-null: NULL
global-config:
db-config:
id-type: auto
banner: false
registry:
type: zookeeper

3
dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/DaoPluginConfiguration.java
View File

@ -20,6 +20,7 @@
package org.apache.dolphinscheduler.dao.plugin.api;
import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect;
import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor;
import com.baomidou.mybatisplus.annotation.DbType;
@ -33,4 +34,6 @@ public interface DaoPluginConfiguration {
DatabaseMonitor databaseMonitor();
DatabaseDialect databaseDialect();
}

26
dolphinscheduler-dao-plugin/dolphinscheduler-dao-api/src/main/java/org/apache/dolphinscheduler/dao/plugin/api/dialect/DatabaseDialect.java Normal file
View File

@ -0,0 +1,26 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.plugin.api.dialect;
public interface DatabaseDialect {
boolean tableExists(String tableName);
boolean columnExists(String tableName, String columnName);
}

7
dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/H2DaoPluginConfiguration.java
View File

@ -21,7 +21,9 @@
package org.apache.dolphinscheduler.dao.plugin.h2;
import org.apache.dolphinscheduler.dao.plugin.api.DaoPluginConfiguration;
import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect;
import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor;
import org.apache.dolphinscheduler.dao.plugin.h2.dialect.H2Dialect;
import org.apache.dolphinscheduler.dao.plugin.h2.monitor.H2Monitor;
import javax.sql.DataSource;
@ -49,4 +51,9 @@ public class H2DaoPluginConfiguration implements DaoPluginConfiguration {
return new H2Monitor(dataSource);
}
@Override
public DatabaseDialect databaseDialect() {
return new H2Dialect();
}
}

33
dolphinscheduler-dao-plugin/dolphinscheduler-dao-h2/src/main/java/org/apache/dolphinscheduler/dao/plugin/h2/dialect/H2Dialect.java Normal file
View File

@ -0,0 +1,33 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.plugin.h2.dialect;
import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect;
public class H2Dialect implements DatabaseDialect {
@Override
public boolean tableExists(String tableName) {
throw new UnsupportedOperationException();
}
@Override
public boolean columnExists(String tableName, String columnName) {
throw new UnsupportedOperationException();
}
}

7
dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/MysqlDaoPluginConfiguration.java
View File

@ -20,7 +20,9 @@
package org.apache.dolphinscheduler.dao.plugin.mysql;
import org.apache.dolphinscheduler.dao.plugin.api.DaoPluginConfiguration;
import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect;
import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor;
import org.apache.dolphinscheduler.dao.plugin.mysql.dialect.MysqlDialect;
import org.apache.dolphinscheduler.dao.plugin.mysql.monitor.MysqlMonitor;
import javax.sql.DataSource;
@ -47,4 +49,9 @@ public class MysqlDaoPluginConfiguration implements DaoPluginConfiguration {
public DatabaseMonitor databaseMonitor() {
return new MysqlMonitor(dataSource);
}
@Override
public DatabaseDialect databaseDialect() {
return new MysqlDialect(dataSource);
}
}

61
dolphinscheduler-dao-plugin/dolphinscheduler-dao-mysql/src/main/java/org/apache/dolphinscheduler/dao/plugin/mysql/dialect/MysqlDialect.java Normal file
View File

@ -0,0 +1,61 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
*/
package org.apache.dolphinscheduler.dao.plugin.mysql.dialect;
import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect;
import java.sql.Connection;
import java.sql.ResultSet;
import javax.sql.DataSource;
import lombok.SneakyThrows;
public class MysqlDialect implements DatabaseDialect {
private final DataSource dataSource;
public MysqlDialect(DataSource dataSource) {
this.dataSource = dataSource;
}
@SneakyThrows
@Override
public boolean tableExists(String tableName) {
try (
Connection conn = dataSource.getConnection();
ResultSet rs = conn.getMetaData().getTables(conn.getCatalog(), conn.getSchema(), tableName, null)) {
return rs.next();
}
}
@SneakyThrows
@Override
public boolean columnExists(String tableName, String columnName) {
try (
Connection conn = dataSource.getConnection();
ResultSet rs =
conn.getMetaData().getColumns(conn.getCatalog(), conn.getSchema(), tableName, columnName)) {
return rs.next();
}
}
}

7
dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/PostgresqlDaoPluginConfiguration.java
View File

@ -21,7 +21,9 @@
package org.apache.dolphinscheduler.dao.plugin.postgresql;
import org.apache.dolphinscheduler.dao.plugin.api.DaoPluginConfiguration;
import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect;
import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor;
import org.apache.dolphinscheduler.dao.plugin.postgresql.dialect.PostgresqlDialect;
import org.apache.dolphinscheduler.dao.plugin.postgresql.monitor.PostgresqlMonitor;
import javax.sql.DataSource;
@ -48,4 +50,9 @@ public class PostgresqlDaoPluginConfiguration implements DaoPluginConfiguration
public DatabaseMonitor databaseMonitor() {
return new PostgresqlMonitor(dataSource);
}
@Override
public DatabaseDialect databaseDialect() {
return new PostgresqlDialect(dataSource);
}
}

79
dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/PostgreSQLUpgradeDao.java → dolphinscheduler-dao-plugin/dolphinscheduler-dao-postgresql/src/main/java/org/apache/dolphinscheduler/dao/plugin/postgresql/dialect/PostgresqlDialect.java
View File

@ -15,9 +15,9 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.tools.datasource.dao;
package org.apache.dolphinscheduler.dao.plugin.postgresql.dialect;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect;
import java.sql.Connection;
import java.sql.PreparedStatement;
@ -26,31 +26,37 @@ import java.sql.SQLException;
import javax.sql.DataSource;
import lombok.extern.slf4j.Slf4j;
import lombok.SneakyThrows;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
public class PostgresqlDialect implements DatabaseDialect {
@Service
@Slf4j
@Profile("postgresql")
public class PostgreSQLUpgradeDao extends UpgradeDao {
private final DataSource dataSource;
private PostgreSQLUpgradeDao(DataSource dataSource) {
super(dataSource);
public PostgresqlDialect(DataSource dataSource) {
this.dataSource = dataSource;
}
@SneakyThrows
@Override
protected String initSqlPath() {
return "create/release-1.2.0_schema/postgresql";
public boolean tableExists(String tableName) {
try (
Connection conn = dataSource.getConnection();
ResultSet rs = conn.getMetaData().getTables(conn.getCatalog(), getSchema(), tableName, null)) {
return rs.next();
}
}
@SneakyThrows
@Override
public DbType getDbType() {
return DbType.POSTGRESQL;
public boolean columnExists(String tableName, String columnName) {
try (
Connection conn = dataSource.getConnection();
ResultSet rs = conn.getMetaData().getTables(conn.getCatalog(), getSchema(), tableName, null)) {
return rs.next();
}
}
public String getSchema() {
private String getSchema() throws SQLException {
try (
Connection conn = dataSource.getConnection();
PreparedStatement pstmt = conn.prepareStatement("select current_schema()");
@ -60,48 +66,7 @@ public class PostgreSQLUpgradeDao extends UpgradeDao {
return resultSet.getString(1);
}
}
} catch (SQLException e) {
log.error(e.getMessage(), e);
}
return "";
}
/**
* determines whether a table exists
*
* @param tableName tableName
* @return if table exist return true，else return false
*/
@Override
public boolean isExistsTable(String tableName) {
try (
Connection conn = dataSource.getConnection();
ResultSet rs = conn.getMetaData().getTables(conn.getCatalog(), getSchema(), tableName, null)) {
return rs.next();
} catch (SQLException e) {
log.error(e.getMessage(), e);
throw new RuntimeException(e.getMessage(), e);
}
}
/**
* determines whether a field exists in the specified table
*
* @param tableName tableName
* @param columnName columnName
* @return if column name exist return true，else return false
*/
@Override
public boolean isExistsColumn(String tableName, String columnName) {
try (
Connection conn = dataSource.getConnection();
ResultSet rs = conn.getMetaData().getColumns(conn.getCatalog(), getSchema(), tableName, columnName)) {
return rs.next();
} catch (SQLException e) {
log.error(e.getMessage(), e);
throw new RuntimeException(e.getMessage(), e);
}
}
}

48
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/DaoConfiguration.java
View File

@ -20,12 +20,60 @@
package org.apache.dolphinscheduler.dao;
import org.apache.dolphinscheduler.dao.plugin.api.DaoPluginConfiguration;
import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect;
import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.EnableAutoConfiguration;
import org.springframework.boot.jdbc.init.DataSourceScriptDatabaseInitializer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.context.annotation.Configuration;
import com.baomidou.mybatisplus.annotation.DbType;
import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor;
import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor;
@Configuration
@EnableAutoConfiguration
@ComponentScan({"org.apache.dolphinscheduler.dao.plugin"})
@MapperScan(basePackages = "org.apache.dolphinscheduler.dao.mapper", sqlSessionFactoryRef = "sqlSessionFactory")
public class DaoConfiguration {
/**
* Inject this field to make sure the database is initialized, this can solve the table not found issue #8432.
*/
@Autowired(required = false)
public DataSourceScriptDatabaseInitializer dataSourceScriptDatabaseInitializer;
/**
* Inject this field to make sure the DaoPluginConfiguration is initialized before SpringConnectionFactory.
*/
@Autowired
public DaoPluginConfiguration daoPluginConfiguration;
@Bean
public MybatisPlusInterceptor paginationInterceptor(DbType dbType) {
MybatisPlusInterceptor interceptor = new MybatisPlusInterceptor();
interceptor.addInnerInterceptor(new PaginationInnerInterceptor(dbType));
return interceptor;
}
@Bean
public DbType dbType() {
return daoPluginConfiguration.dbType();
}
@Bean
public DatabaseMonitor databaseMonitor() {
return daoPluginConfiguration.databaseMonitor();
}
@Bean
public DatabaseDialect databaseDialect() {
return daoPluginConfiguration.databaseDialect();
}
}

111
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/datasource/SpringConnectionFactory.java
View File

@ -1,111 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.datasource;
import org.apache.dolphinscheduler.dao.plugin.api.DaoPluginConfiguration;
import org.apache.dolphinscheduler.dao.plugin.api.monitor.DatabaseMonitor;
import org.apache.ibatis.session.SqlSessionFactory;
import org.apache.ibatis.type.JdbcType;
import javax.sql.DataSource;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.jdbc.init.DataSourceScriptDatabaseInitializer;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.io.support.PathMatchingResourcePatternResolver;
import org.springframework.core.io.support.ResourcePatternResolver;
import org.springframework.jdbc.datasource.DataSourceTransactionManager;
import com.baomidou.mybatisplus.annotation.DbType;
import com.baomidou.mybatisplus.annotation.IdType;
import com.baomidou.mybatisplus.core.MybatisConfiguration;
import com.baomidou.mybatisplus.core.config.GlobalConfig;
import com.baomidou.mybatisplus.extension.plugins.MybatisPlusInterceptor;
import com.baomidou.mybatisplus.extension.plugins.inner.PaginationInnerInterceptor;
import com.baomidou.mybatisplus.extension.spring.MybatisSqlSessionFactoryBean;
@Configuration
public class SpringConnectionFactory {
/**
* Inject this field to make sure the database is initialized, this can solve the table not found issue #8432.
*/
@Autowired(required = false)
public DataSourceScriptDatabaseInitializer dataSourceScriptDatabaseInitializer;
/**
* Inject this field to make sure the DaoPluginConfiguration is initialized before SpringConnectionFactory.
*/
@Autowired
public DaoPluginConfiguration daoPluginConfiguration;
@Bean
public MybatisPlusInterceptor paginationInterceptor(DbType dbType) {
MybatisPlusInterceptor interceptor = new MybatisPlusInterceptor();
interceptor.addInnerInterceptor(new PaginationInnerInterceptor(dbType));
return interceptor;
}
@Bean
public DataSourceTransactionManager transactionManager(DataSource dataSource) {
return new DataSourceTransactionManager(dataSource);
}
@Bean
public SqlSessionFactory sqlSessionFactory(DataSource dataSource,
GlobalConfig globalConfig,
DbType dbType) throws Exception {
MybatisConfiguration configuration = new MybatisConfiguration();
configuration.setMapUnderscoreToCamelCase(true);
configuration.setCacheEnabled(false);
configuration.setCallSettersOnNulls(true);
configuration.setJdbcTypeForNull(JdbcType.NULL);
configuration.addInterceptor(paginationInterceptor(dbType));
MybatisSqlSessionFactoryBean sqlSessionFactoryBean = new MybatisSqlSessionFactoryBean();
sqlSessionFactoryBean.setConfiguration(configuration);
sqlSessionFactoryBean.setDataSource(dataSource);
sqlSessionFactoryBean.setGlobalConfig(globalConfig);
sqlSessionFactoryBean.setTypeAliasesPackage("org.apache.dolphinscheduler.dao.entity");
ResourcePatternResolver resolver = new PathMatchingResourcePatternResolver();
// todo: if the different database has different sql, we need to add the different mapper.
sqlSessionFactoryBean
.setMapperLocations(resolver.getResources("org/apache/dolphinscheduler/dao/mapper/*Mapper.xml"));
return sqlSessionFactoryBean.getObject();
}
@Bean
public GlobalConfig globalConfig() {
return new GlobalConfig().setDbConfig(new GlobalConfig.DbConfig()
.setIdType(IdType.AUTO)).setBanner(false);
}
@Bean
public DbType dbType() {
return daoPluginConfiguration.dbType();
}
@Bean
public DatabaseMonitor databaseMonitor() {
return daoPluginConfiguration.databaseMonitor();
}
}

60
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDaoTest.java
View File

@ -1,60 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.upgrade;
import java.util.HashMap;
import java.util.Map;
import javax.sql.DataSource;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.context.ActiveProfiles;
@ActiveProfiles("h2")
public class ProcessDefinitionDaoTest {
@Autowired
private DataSource dataSource;
final ProcessDefinitionDao processDefinitionDao = new ProcessDefinitionDao();
@Test
public void testQueryAllProcessDefinition() {
// Map<Integer, String> processDefinitionJsonMap =
// processDefinitionDao.queryAllProcessDefinition(dataSource.getConnection());
// assertThat(processDefinitionJsonMap.size(),greaterThanOrEqualTo(0));
}
@Test
public void testUpdateProcessDefinitionJson() {
Map<Integer, String> processDefinitionJsonMap = new HashMap<>();
processDefinitionJsonMap.put(1, "test");
// processDefinitionDao.updateProcessDefinitionJson(dataSource.getConnection(),processDefinitionJsonMap);
}
@Test
public void testQueryAllProcessDefinitionException() {
// processDefinitionDao.queryAllProcessDefinition(null);
}
@Test
public void testUpdateProcessDefinitionJsonException() {
Assertions.assertThrows(Exception.class, () -> processDefinitionDao.updateProcessDefinitionJson(null, null));
}
}

50
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDaoTest.java
View File

@ -1,50 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.upgrade;
import org.apache.dolphinscheduler.dao.DaoConfiguration;
import javax.sql.DataSource;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.junit.jupiter.MockitoExtension;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.ActiveProfiles;
@ActiveProfiles("h2")
@SpringBootTest(classes = DaoConfiguration.class)
@ExtendWith(MockitoExtension.class)
@SpringBootApplication(scanBasePackageClasses = DaoConfiguration.class)
public class WorkerGroupDaoTest {
@Autowired
protected DataSource dataSource;
@Test
public void testQueryQueryAllOldWorkerGroupException() throws Exception {
Assertions.assertThrows(Exception.class, () -> {
WorkerGroupDao workerGroupDao = new WorkerGroupDao();
workerGroupDao.queryAllOldWorkerGroup(null);
});
}
}

13
dolphinscheduler-dao/src/test/resources/application.yaml
View File

@ -26,3 +26,16 @@ spring:
url: jdbc:h2:mem:dolphinscheduler;MODE=MySQL;DB_CLOSE_DELAY=-1;DATABASE_TO_LOWER=true;
username: sa
password: ""
mybatis-plus:
mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
type-aliases-package: org.apache.dolphinscheduler.dao.entity
configuration:
cache-enabled: false
call-setters-on-nulls: true
map-underscore-to-camel-case: true
jdbc-type-for-null: NULL
global-config:
db-config:
id-type: auto
banner: false

15
dolphinscheduler-master/src/main/resources/application.yaml
View File

@ -71,6 +71,21 @@ spring:
org.quartz.jobStore.driverDelegateClass: org.quartz.impl.jdbcjobstore.PostgreSQLDelegate
org.quartz.jobStore.clusterCheckinInterval: 5000
# Mybatis-plus configuration, you don't need to change it
mybatis-plus:
mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
type-aliases-package: org.apache.dolphinscheduler.dao.entity
configuration:
cache-enabled: false
call-setters-on-nulls: true
map-underscore-to-camel-case: true
jdbc-type-for-null: NULL
global-config:
db-config:
id-type: auto
banner: false
registry:
type: zookeeper
zookeeper:

13
dolphinscheduler-standalone-server/src/main/resources/application.yaml
View File

@ -76,6 +76,19 @@ spring:
pathmatch:
matching-strategy: ANT_PATH_MATCHER
mybatis-plus:
mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
type-aliases-package: org.apache.dolphinscheduler.dao.entity
configuration:
cache-enabled: false
call-setters-on-nulls: true
map-underscore-to-camel-case: true
jdbc-type-for-null: NULL
global-config:
db-config:
id-type: auto
banner: false
registry:
type: zookeeper
zookeeper:

3
dolphinscheduler-tools/pom.xml
View File

@ -130,19 +130,16 @@
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>mysql</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>mysql</groupId>
<artifactId>mysql-connector-java</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.testcontainers</groupId>
<artifactId>postgresql</artifactId>
<scope>test</scope>
</dependency>
</dependencies>

46
dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/DolphinSchedulerManager.java
View File

@ -17,57 +17,45 @@
package org.apache.dolphinscheduler.tools.datasource;
import org.apache.dolphinscheduler.dao.upgrade.SchemaUtils;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.tools.datasource.dao.UpgradeDao;
import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect;
import org.apache.dolphinscheduler.tools.datasource.upgrader.DolphinSchedulerUpgrader;
import org.apache.dolphinscheduler.tools.datasource.upgrader.DolphinSchedulerVersion;
import org.apache.dolphinscheduler.tools.datasource.upgrader.UpgradeDao;
import org.apache.dolphinscheduler.tools.datasource.utils.SchemaUtils;
import org.apache.commons.collections4.CollectionUtils;
import java.io.IOException;
import java.sql.Connection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Function;
import java.util.stream.Collectors;
import javax.sql.DataSource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
@Service
@Slf4j
public class DolphinSchedulerManager {
private final UpgradeDao upgradeDao;
@Autowired
private UpgradeDao upgradeDao;
@Autowired
private DatabaseDialect databaseDialect;
private Map<DolphinSchedulerVersion, DolphinSchedulerUpgrader> upgraderMap = new HashMap<>();
public DolphinSchedulerManager(DataSource dataSource, List<UpgradeDao> daos,
List<DolphinSchedulerUpgrader> dolphinSchedulerUpgraders) throws Exception {
final DbType type = getCurrentDbType(dataSource);
upgradeDao = daos.stream()
.filter(it -> it.getDbType() == type)
.findFirst()
.orElseThrow(() -> new RuntimeException(
"Cannot find UpgradeDao implementation for db type: " + type));
public DolphinSchedulerManager(List<DolphinSchedulerUpgrader> dolphinSchedulerUpgraders) throws Exception {
if (CollectionUtils.isNotEmpty(dolphinSchedulerUpgraders)) {
upgraderMap = dolphinSchedulerUpgraders.stream()
.collect(Collectors.toMap(DolphinSchedulerUpgrader::getCurrentVersion, Function.identity()));
}
}
private DbType getCurrentDbType(DataSource dataSource) throws Exception {
try (Connection conn = dataSource.getConnection()) {
String name = conn.getMetaData().getDatabaseProductName().toUpperCase();
return DbType.valueOf(name);
}
}
public void initDolphinScheduler() {
this.initDolphinSchedulerSchema();
}
@ -78,9 +66,9 @@ public class DolphinSchedulerManager {
*/
public boolean schemaIsInitialized() {
// Determines whether the dolphinscheduler table structure has been init
if (upgradeDao.isExistsTable("t_escheduler_version")
|| upgradeDao.isExistsTable("t_ds_version")
|| upgradeDao.isExistsTable("t_escheduler_queue")) {
if (databaseDialect.tableExists("t_escheduler_version")
|| databaseDialect.tableExists("t_ds_version")
|| databaseDialect.tableExists("t_escheduler_queue")) {
log.info("The database has been initialized. Skip the initialization step");
return true;
}
@ -100,13 +88,13 @@ public class DolphinSchedulerManager {
} else {
String version;
// Gets the version of the current system
if (upgradeDao.isExistsTable("t_escheduler_version")) {
if (databaseDialect.tableExists("t_escheduler_version")) {
version = upgradeDao.getCurrentVersion("t_escheduler_version");
} else if (upgradeDao.isExistsTable("t_ds_version")) {
} else if (databaseDialect.tableExists("t_ds_version")) {
version = upgradeDao.getCurrentVersion("t_ds_version");
} else if (upgradeDao.isExistsColumn("t_escheduler_queue", "create_time")) {
} else if (databaseDialect.columnExists("t_escheduler_queue", "create_time")) {
version = "1.0.1";
} else if (upgradeDao.isExistsTable("t_escheduler_queue")) {
} else if (databaseDialect.tableExists("t_escheduler_queue")) {
version = "1.0.0";
} else {
log.error("Unable to determine current software version, so cannot upgrade");

56
dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/InitDolphinScheduler.java
View File

@ -1,56 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.tools.datasource;
import org.apache.dolphinscheduler.dao.DaoConfiguration;
import lombok.extern.slf4j.Slf4j;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.ImportAutoConfiguration;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Component;
@ImportAutoConfiguration(DaoConfiguration.class)
@SpringBootApplication
public class InitDolphinScheduler {
public static void main(String[] args) {
SpringApplication.run(InitDolphinScheduler.class, args);
}
@Component
@Profile("init")
@Slf4j
static class InitRunner implements CommandLineRunner {
private final DolphinSchedulerManager dolphinSchedulerManager;
InitRunner(DolphinSchedulerManager dolphinSchedulerManager) {
this.dolphinSchedulerManager = dolphinSchedulerManager;
}
@Override
public void run(String... args) {
dolphinSchedulerManager.initDolphinScheduler();
log.info("init DolphinScheduler finished");
}
}
}

3
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/JsonSplitDao.java → dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/JsonSplitDao.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.upgrade;
package org.apache.dolphinscheduler.tools.datasource.dao;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;
@ -28,6 +28,7 @@ import java.util.List;
import lombok.extern.slf4j.Slf4j;
@Deprecated
@Slf4j
public class JsonSplitDao {

90
dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/MySQLUpgradeDao.java
View File

@ -1,90 +0,0 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.tools.datasource.dao;
import org.apache.dolphinscheduler.spi.enums.DbType;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
import javax.sql.DataSource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.context.annotation.Profile;
import org.springframework.stereotype.Service;
@Service
@Slf4j
@Profile("mysql")
public class MySQLUpgradeDao extends UpgradeDao {
private MySQLUpgradeDao(DataSource dataSource) {
super(dataSource);
}
@Override
protected String initSqlPath() {
return "create/release-1.0.0_schema/mysql";
}
@Override
public DbType getDbType() {
return DbType.MYSQL;
}
/**
* determines whether a table exists
* @param tableName tableName
* @return if table exist return true，else return false
*/
@Override
public boolean isExistsTable(String tableName) {
try (
Connection conn = dataSource.getConnection();
ResultSet rs = conn.getMetaData().getTables(conn.getCatalog(), conn.getSchema(), tableName, null)) {
return rs.next();
} catch (SQLException e) {
log.error(e.getMessage(), e);
throw new RuntimeException(e.getMessage(), e);
}
}
/**
* determines whether a field exists in the specified table
* @param tableName tableName
* @param columnName columnName
* @return if column name exist return true，else return false
*/
@Override
public boolean isExistsColumn(String tableName, String columnName) {
try (
Connection conn = dataSource.getConnection();
ResultSet rs =
conn.getMetaData().getColumns(conn.getCatalog(), conn.getSchema(), tableName, columnName)) {
return rs.next();
} catch (SQLException e) {
log.error(e.getMessage(), e);
throw new RuntimeException(e.getMessage(), e);
}
}
}

3
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProcessDefinitionDao.java → dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProcessDefinitionDao.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.upgrade;
package org.apache.dolphinscheduler.tools.datasource.dao;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.common.enums.Flag;
@ -33,6 +33,7 @@ import java.util.Map;
import lombok.extern.slf4j.Slf4j;
@Deprecated
@Slf4j
public class ProcessDefinitionDao {

3
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ProjectDao.java → dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ProjectDao.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.upgrade;
package org.apache.dolphinscheduler.tools.datasource.dao;
import org.apache.dolphinscheduler.common.utils.CodeGenerateUtils;
@ -27,6 +27,7 @@ import java.util.Map;
import lombok.extern.slf4j.Slf4j;
@Deprecated
@Slf4j
public class ProjectDao {

1
dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ResourceDao.java
View File

@ -34,6 +34,7 @@ import com.google.common.base.Strings;
/**
* resource dao
*/
@Deprecated
@Slf4j
public class ResourceDao {

3
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/ScheduleDao.java → dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/ScheduleDao.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.upgrade;
package org.apache.dolphinscheduler.tools.datasource.dao;
import java.sql.Connection;
import java.sql.PreparedStatement;
@ -26,6 +26,7 @@ import java.util.Map;
import lombok.extern.slf4j.Slf4j;
@Deprecated
@Slf4j
public class ScheduleDao {

3
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/WorkerGroupDao.java → dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/WorkerGroupDao.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.upgrade;
package org.apache.dolphinscheduler.tools.datasource.dao;
import java.sql.Connection;
import java.sql.PreparedStatement;
@ -25,6 +25,7 @@ import java.util.Map;
import lombok.extern.slf4j.Slf4j;
@Deprecated
@Slf4j
public class WorkerGroupDao {

52
dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/dao/UpgradeDao.java → dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/UpgradeDao.java
View File

@ -15,11 +15,12 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.tools.datasource.dao;
package org.apache.dolphinscheduler.tools.datasource.upgrader;
import org.apache.dolphinscheduler.common.sql.SqlScriptRunner;
import org.apache.dolphinscheduler.dao.upgrade.SchemaUtils;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.apache.dolphinscheduler.dao.plugin.api.dialect.DatabaseDialect;
import org.apache.dolphinscheduler.tools.datasource.dao.ResourceDao;
import org.apache.dolphinscheduler.tools.datasource.utils.SchemaUtils;
import java.io.FileNotFoundException;
import java.sql.Connection;
@ -31,34 +32,33 @@ import javax.sql.DataSource;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.baomidou.mybatisplus.annotation.DbType;
@Slf4j
public abstract class UpgradeDao {
@Service
public class UpgradeDao {
private static final String T_VERSION_NAME = "t_escheduler_version";
private static final String T_NEW_VERSION_NAME = "t_ds_version";
protected final DataSource dataSource;
@Autowired
private DataSource dataSource;
protected UpgradeDao(DataSource dataSource) {
this.dataSource = dataSource;
}
@Autowired
private DbType dbType;
protected abstract String initSqlPath();
public abstract DbType getDbType();
public void initSchema() {
// Execute the dolphinscheduler full sql
runInitSql(getDbType());
}
@Autowired
private DatabaseDialect databaseDialect;
/**
* run init sql to init db schema
*
* @param dbType db type
*/
private void runInitSql(DbType dbType) {
String sqlFilePath = String.format("sql/dolphinscheduler_%s.sql", dbType.getDescp());
public void initSchema() {
// Execute the dolphinscheduler full sql
String sqlFilePath = String.format("sql/dolphinscheduler_%s.sql", dbType.getDb());
SqlScriptRunner sqlScriptRunner = new SqlScriptRunner(dataSource, sqlFilePath);
try {
sqlScriptRunner.execute();
@ -68,10 +68,6 @@ public abstract class UpgradeDao {
}
}
public abstract boolean isExistsTable(String tableName);
public abstract boolean isExistsColumn(String tableName, String columnName);
public String getCurrentVersion(String versionName) {
String sql = String.format("select version from %s", versionName);
String version = null;
@ -112,17 +108,17 @@ public abstract class UpgradeDao {
private void upgradeDolphinSchedulerDML(String schemaDir, String scriptFile) {
String schemaVersion = schemaDir.split("_")[0];
String sqlFilePath =
String.format("sql/upgrade/%s/%s/%s", schemaDir, getDbType().name().toLowerCase(), scriptFile);
String.format("sql/upgrade/%s/%s/%s", schemaDir, dbType.getDb(), scriptFile);
try {
// Execute the upgraded dolphinscheduler dml
SqlScriptRunner sqlScriptRunner = new SqlScriptRunner(dataSource, sqlFilePath);
sqlScriptRunner.execute();
try (Connection connection = dataSource.getConnection()) {
String upgradeSQL;
if (isExistsTable(T_VERSION_NAME)) {
if (databaseDialect.tableExists(T_VERSION_NAME)) {
// Change version in the version table to the new version
upgradeSQL = String.format("update %s set version = ?", T_VERSION_NAME);
} else if (isExistsTable(T_NEW_VERSION_NAME)) {
} else if (databaseDialect.tableExists(T_NEW_VERSION_NAME)) {
// Change version in the version table to the new version
upgradeSQL = String.format("update %s set version = ?", T_NEW_VERSION_NAME);
} else {
@ -151,7 +147,7 @@ public abstract class UpgradeDao {
*/
public void upgradeDolphinSchedulerDDL(String schemaDir, String scriptFile) {
String sqlFilePath =
String.format("sql/upgrade/%s/%s/%s", schemaDir, getDbType().name().toLowerCase(), scriptFile);
String.format("sql/upgrade/%s/%s/%s", schemaDir, dbType.getDb(), scriptFile);
SqlScriptRunner sqlScriptRunner = new SqlScriptRunner(dataSource, sqlFilePath);
try {
// Execute the dolphinscheduler ddl.sql for the upgrade

4
dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v130/V130DolphinSchedulerUpgrader.java
View File

@ -18,8 +18,8 @@
package org.apache.dolphinscheduler.tools.datasource.upgrader.v130;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.upgrade.ProcessDefinitionDao;
import org.apache.dolphinscheduler.dao.upgrade.WorkerGroupDao;
import org.apache.dolphinscheduler.tools.datasource.dao.ProcessDefinitionDao;
import org.apache.dolphinscheduler.tools.datasource.dao.WorkerGroupDao;
import org.apache.dolphinscheduler.tools.datasource.upgrader.DolphinSchedulerUpgrader;
import org.apache.dolphinscheduler.tools.datasource.upgrader.DolphinSchedulerVersion;

2
dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v132/V132DolphinSchedulerUpgrader.java
View File

@ -18,8 +18,8 @@
package org.apache.dolphinscheduler.tools.datasource.upgrader.v132;
import org.apache.dolphinscheduler.common.utils.JSONUtils;
import org.apache.dolphinscheduler.dao.upgrade.ProcessDefinitionDao;
import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo;
import org.apache.dolphinscheduler.tools.datasource.dao.ProcessDefinitionDao;
import org.apache.dolphinscheduler.tools.datasource.upgrader.DolphinSchedulerUpgrader;
import org.apache.dolphinscheduler.tools.datasource.upgrader.DolphinSchedulerVersion;

10
dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v200/V200DolphinSchedulerUpgrader.java
View File

@ -32,15 +32,15 @@ import org.apache.dolphinscheduler.dao.entity.ProcessDefinition;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
import org.apache.dolphinscheduler.dao.entity.ProcessTaskRelationLog;
import org.apache.dolphinscheduler.dao.entity.TaskDefinitionLog;
import org.apache.dolphinscheduler.dao.upgrade.JsonSplitDao;
import org.apache.dolphinscheduler.dao.upgrade.ProcessDefinitionDao;
import org.apache.dolphinscheduler.dao.upgrade.ProjectDao;
import org.apache.dolphinscheduler.dao.upgrade.ScheduleDao;
import org.apache.dolphinscheduler.plugin.task.api.model.ResourceInfo;
import org.apache.dolphinscheduler.plugin.task.api.parameters.TaskTimeoutParameter;
import org.apache.dolphinscheduler.tools.datasource.dao.UpgradeDao;
import org.apache.dolphinscheduler.tools.datasource.dao.JsonSplitDao;
import org.apache.dolphinscheduler.tools.datasource.dao.ProcessDefinitionDao;
import org.apache.dolphinscheduler.tools.datasource.dao.ProjectDao;
import org.apache.dolphinscheduler.tools.datasource.dao.ScheduleDao;
import org.apache.dolphinscheduler.tools.datasource.upgrader.DolphinSchedulerUpgrader;
import org.apache.dolphinscheduler.tools.datasource.upgrader.DolphinSchedulerVersion;
import org.apache.dolphinscheduler.tools.datasource.upgrader.UpgradeDao;
import org.apache.commons.collections4.CollectionUtils;

275
dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/upgrader/v320/V320DolphinSchedulerUpgrader.java
View File

@ -18,59 +18,41 @@
package org.apache.dolphinscheduler.tools.datasource.upgrader.v320;
import org.apache.dolphinscheduler.common.constants.Constants;
import org.apache.dolphinscheduler.dao.entity.ProcessDefinitionLog;
import org.apache.dolphinscheduler.dao.entity.ProcessInstance;
import org.apache.dolphinscheduler.dao.entity.Schedule;
import org.apache.dolphinscheduler.dao.entity.TaskInstance;
import org.apache.dolphinscheduler.dao.entity.User;
import org.apache.dolphinscheduler.dao.mapper.ProcessDefinitionLogMapper;
import org.apache.dolphinscheduler.dao.mapper.ProcessInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.ScheduleMapper;
import org.apache.dolphinscheduler.dao.mapper.TaskInstanceMapper;
import org.apache.dolphinscheduler.dao.mapper.UserMapper;
import org.apache.dolphinscheduler.tools.datasource.dao.UpgradeDao;
import org.apache.dolphinscheduler.tools.datasource.upgrader.DolphinSchedulerUpgrader;
import org.apache.dolphinscheduler.tools.datasource.upgrader.DolphinSchedulerVersion;
import org.apache.dolphinscheduler.tools.datasource.upgrader.UpgradeDao;
import org.apache.commons.collections4.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.commons.collections4.MapUtils;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
import javax.sql.DataSource;
import lombok.SneakyThrows;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.context.annotation.Lazy;
import org.springframework.stereotype.Component;
import com.baomidou.mybatisplus.core.conditions.query.LambdaQueryWrapper;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
@Slf4j
@Component
public class V320DolphinSchedulerUpgrader implements DolphinSchedulerUpgrader {
@Autowired
private ProcessInstanceMapper processInstanceMapper;
private DataSource dataSource;
@Autowired
private ProcessDefinitionLogMapper processDefinitionLogMapper;
@Autowired
private ScheduleMapper scheduleMapper;
@Autowired
private UserMapper userMapper;
@Autowired
private TaskInstanceMapper taskInstanceMapper;
@Lazy()
@Autowired
private UpgradeDao upgradeDao;
@SneakyThrows
@Override
public void doUpgrade() {
upgradeWorkflowInstance();
@ -80,68 +62,219 @@ public class V320DolphinSchedulerUpgrader implements DolphinSchedulerUpgrader {
}
private void upgradeWorkflowInstance() {
Map<Integer, String> userMap = userMapper.selectList(new QueryWrapper<>())
.stream()
.collect(Collectors.toMap(User::getId, User::getUserName));
Map<Integer, String> userMap = getUserMap();
while (true) {
LambdaQueryWrapper<ProcessInstance> wrapper = new QueryWrapper<ProcessInstance>()
.lambda()
.eq(ProcessInstance::getProjectCode, null)
.last("limit 1000");
List<ProcessInstance> needUpdateWorkflowInstance = processInstanceMapper.selectList(wrapper);
if (CollectionUtils.isEmpty(needUpdateWorkflowInstance)) {
List<Map<String, Object>> needUpdateWorkflowInstances = getProcessInstanceWhichProjectCodeIsNull();
if (CollectionUtils.isEmpty(needUpdateWorkflowInstances)) {
return;
}
needUpdateWorkflowInstance.parallelStream()
needUpdateWorkflowInstances.parallelStream()
.forEach(processInstance -> {
ProcessDefinitionLog processDefinitionLog =
processDefinitionLogMapper.queryByDefinitionCodeAndVersion(
processInstance.getProcessDefinitionCode(),
processInstance.getProcessDefinitionVersion());
Schedule schedule =
scheduleMapper.queryByProcessDefinitionCode(processInstance.getProcessDefinitionCode());
if (processDefinitionLog != null) {
processInstance.setProjectCode(processDefinitionLog.getProjectCode());
processInstance.setTenantCode(
StringUtils.defaultIfEmpty(schedule.getTenantCode(), Constants.DEFAULT));
processInstance.setExecutorName(userMap.get(processInstance.getExecutorId()));
} else {
processInstance.setProjectCode(-1L);
Integer id = (Integer) processInstance.get("id");
Long processDefinitionCode = (Long) processInstance.get("process_definition_code");
Integer processDefinitionVersion = (Integer) processInstance.get("process_definition_version");
Map<String, Object> processDefinitionLog =
getProcessDefinitionLogByCode(processDefinitionCode, processDefinitionVersion);
Long projectCode = -1L;
String tenantCode = null;
String executorName = null;
if (MapUtils.isNotEmpty(processDefinitionLog)) {
Map<String, Object> scheduler = getSchedulerByProcessDefinitionCode(processDefinitionCode);
projectCode = processDefinitionLog.get("project_code") == null ? -1L
: (Long) processDefinitionLog.get("project_code");
tenantCode = scheduler.get("tenant_code") == null ? Constants.DEFAULT
: (String) scheduler.get("tenant_code");
executorName = userMap.get((Integer) processInstance.get("executor_id"));
}
processInstanceMapper.updateById(processInstance);
updateProjectCodeInProcessInstance(id, projectCode, tenantCode, executorName);
});
log.info("Success upgrade workflow instance, current batch size: {}", needUpdateWorkflowInstance.size());
log.info("Success upgrade workflow instance, current batch size: {}", needUpdateWorkflowInstances.size());
}
}
private void upgradeTaskInstance() {
while (true) {
LambdaQueryWrapper<TaskInstance> wrapper = new QueryWrapper<TaskInstance>()
.lambda()
.eq(TaskInstance::getProjectCode, null)
.last("limit 1000");
List<TaskInstance> taskInstances = taskInstanceMapper.selectList(wrapper);
List<Map<String, Object>> taskInstances = getTaskInstanceWhichProjectCodeIsNull();
if (CollectionUtils.isEmpty(taskInstances)) {
return;
}
taskInstances.parallelStream()
.forEach(taskInstance -> {
ProcessInstance processInstance =
processInstanceMapper.selectById(taskInstance.getProcessInstanceId());
if (processInstance == null) {
taskInstance.setProjectCode(-1L);
} else {
taskInstance.setProjectCode(processInstance.getProjectCode());
taskInstance.setProcessInstanceName(processInstance.getName());
taskInstance.setExecutorName(processInstance.getExecutorName());
Integer id = (Integer) taskInstance.get("id");
Integer processInstanceId = (Integer) taskInstance.get("process_instance_id");
Map<String, Object> processInstance = getProcessInstanceById(processInstanceId);
Long projectCode = -1L;
String processInstanceName = null;
String executorName = null;
if (MapUtils.isNotEmpty(processInstance)) {
projectCode = processInstance.get("project_code") == null ? -1L
: (Long) processInstance.get("project_code");
processInstanceName = (String) processInstance.get("name");
executorName = (String) processInstance.get("executor_name");
}
taskInstanceMapper.updateById(taskInstance);
updateProjectCodeInTaskInstance(id, projectCode, processInstanceName, executorName);
});
log.info("Success upgrade task instance, current batch size: {}", taskInstances.size());
}
}
private List<Map<String, Object>> getTaskInstanceWhichProjectCodeIsNull() {
List<Map<String, Object>> processInstanceList = new ArrayList<>();
try (
Connection connection = dataSource.getConnection();
PreparedStatement preparedStatement = connection.prepareStatement(
"select id, process_instance_id from t_ds_task_instance where project_code is null limit 1000");
ResultSet resultSet = preparedStatement.executeQuery()) {
while (resultSet.next()) {
Map<String, Object> row = new HashMap<>();
row.put("id", resultSet.getInt("id"));
row.put("process_instance_id", resultSet.getInt("process_instance_id"));
processInstanceList.add(row);
}
return processInstanceList;
} catch (Exception ex) {
throw new RuntimeException("Query t_ds_process_instance error", ex);
}
}
private Map<Integer, String> getUserMap() {
Map<Integer, String> userMap = new HashMap<>();
try (
Connection connection = dataSource.getConnection();
PreparedStatement preparedStatement =
connection.prepareStatement("select id, user_name from t_ds_user");
ResultSet resultSet = preparedStatement.executeQuery()) {
while (resultSet.next()) {
userMap.put(resultSet.getInt("id"), resultSet.getString("user_name"));
}
} catch (Exception ex) {
throw new RuntimeException("Query t_ds_user error", ex);
}
return userMap;
}
private List<Map<String, Object>> getProcessInstanceWhichProjectCodeIsNull() {
List<Map<String, Object>> processInstanceList = new ArrayList<>();
try (
Connection connection = dataSource.getConnection();
PreparedStatement preparedStatement = connection.prepareStatement(
"select id, process_definition_code, process_definition_version, executor_id from t_ds_process_instance where project_code is null limit 1000");
ResultSet resultSet = preparedStatement.executeQuery()) {
while (resultSet.next()) {
Map<String, Object> row = new HashMap<>();
row.put("id", resultSet.getInt("id"));
row.put("process_definition_code", resultSet.getLong("process_definition_code"));
row.put("process_definition_version", resultSet.getInt("process_definition_version"));
row.put("executor_id", resultSet.getInt("executor_id"));
processInstanceList.add(row);
}
return processInstanceList;
} catch (Exception ex) {
throw new RuntimeException("Query t_ds_process_instance error", ex);
}
}
private Map<String, Object> getProcessInstanceById(Integer processInstanceId) {
try (
Connection connection = dataSource.getConnection();
PreparedStatement preparedStatement = connection.prepareStatement(
"select project_code, name, executor_name from t_ds_process_instance where id = ?");) {
preparedStatement.setInt(1, processInstanceId);
try (ResultSet resultSet = preparedStatement.executeQuery()) {
while (resultSet.next()) {
Map<String, Object> row = new HashMap<>();
row.put("project_code", resultSet.getLong("project_code"));
row.put("name", resultSet.getString("name"));
row.put("executor_name", resultSet.getString("executor_name"));
return row;
}
}
return Collections.emptyMap();
} catch (Exception ex) {
throw new RuntimeException("Query t_ds_process_instance error", ex);
}
}
private Map<String, Object> getProcessDefinitionLogByCode(Long processDefinitionCode,
Integer processDefinitionVersion) {
try (
Connection connection = dataSource.getConnection();
PreparedStatement preparedStatement = connection.prepareStatement(
"select project_code from t_ds_process_definition_log where code = ? and version = ?")) {
preparedStatement.setLong(1, processDefinitionCode);
preparedStatement.setInt(2, processDefinitionVersion);
try (ResultSet resultSet = preparedStatement.executeQuery()) {
while (resultSet.next()) {
Map<String, Object> row = new HashMap<>();
row.put("project_code", resultSet.getLong("project_code"));
return row;
}
}
return Collections.emptyMap();
} catch (Exception ex) {
throw new RuntimeException("Query t_ds_process_definition_log error", ex);
}
}
private Map<String, Object> getSchedulerByProcessDefinitionCode(Long processDefinitionCode) {
try (
Connection connection = dataSource.getConnection();
PreparedStatement preparedStatement = connection
.prepareStatement("select * from t_ds_schedules where process_definition_code = ?")) {
preparedStatement.setLong(1, processDefinitionCode);
try (ResultSet resultSet = preparedStatement.executeQuery()) {
while (resultSet.next()) {
Map<String, Object> row = new HashMap<>();
row.put("tenant_code", resultSet.getString("tenant_code"));
return row;
}
}
return Collections.emptyMap();
} catch (Exception ex) {
throw new RuntimeException("Query t_ds_schedules error", ex);
}
}
private void updateProjectCodeInProcessInstance(Integer processInstanceId, Long projectCode, String tenantCode,
String executorName) {
try (
Connection connection = dataSource.getConnection();
PreparedStatement preparedStatement = connection.prepareStatement(
"update t_ds_process_instance set project_code = ?, tenant_code = ?, executor_name = ? where id = ?")) {
preparedStatement.setLong(1, projectCode);
preparedStatement.setString(2, tenantCode);
preparedStatement.setString(3, executorName);
preparedStatement.setInt(4, processInstanceId);
preparedStatement.executeUpdate();
} catch (Exception ex) {
throw new RuntimeException("Update t_ds_process_instance error", ex);
}
}
private void updateProjectCodeInTaskInstance(Integer id, Long projectCode, String processInstanceName,
String executorName) {
try (
Connection connection = dataSource.getConnection();
PreparedStatement preparedStatement = connection.prepareStatement(
"update t_ds_task_instance set project_code = ?, process_instance_name = ?, executor_name = ? where id = ?")) {
preparedStatement.setLong(1, projectCode);
preparedStatement.setString(2, processInstanceName);
preparedStatement.setString(3, executorName);
preparedStatement.setInt(4, id);
preparedStatement.executeUpdate();
} catch (Exception ex) {
throw new RuntimeException("Update t_ds_process_instance error", ex);
}
}
@Override
public DolphinSchedulerVersion getCurrentVersion() {
return DolphinSchedulerVersion.V3_2_0;

2
dolphinscheduler-dao/src/main/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtils.java → dolphinscheduler-tools/src/main/java/org/apache/dolphinscheduler/tools/datasource/utils/SchemaUtils.java
View File

@ -15,7 +15,7 @@
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.upgrade;
package org.apache.dolphinscheduler.tools.datasource.utils;
import org.apache.dolphinscheduler.common.utils.FileUtils;

14
dolphinscheduler-tools/src/main/resources/application.yaml
View File

@ -34,6 +34,20 @@ spring:
leak-detection-threshold: 0
initialization-fail-timeout: 1
# Mybatis-plus configuration, you don't need to change it
mybatis-plus:
mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
type-aliases-package: org.apache.dolphinscheduler.dao.entity
configuration:
cache-enabled: false
call-setters-on-nulls: true
map-underscore-to-camel-case: true
jdbc-type-for-null: NULL
global-config:
db-config:
id-type: auto
banner: false
demo:
tenant-code: default
domain-name: localhost

36
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/upgrade/SchemaUtilsTest.java → dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/utils/SchemaUtilsTest.java
View File

@ -1,21 +1,24 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
* distributed with this work for additional information
* regarding copyright ownership. The ASF licenses this file
* to you under the Apache License, Version 2.0 (the
* "License"); you may not use this file except in compliance
* with the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing,
* software distributed under the License is distributed on an
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
* KIND, either express or implied. See the License for the
* specific language governing permissions and limitations
* under the License.
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.dao.upgrade;
package org.apache.dolphinscheduler.tools.datasource.utils;
import org.apache.commons.collections4.CollectionUtils;
@ -25,10 +28,10 @@ import java.util.List;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.Test;
public class SchemaUtilsTest {
class SchemaUtilsTest {
@Test
public void testIsAGreatVersion() {
void testIsAGreatVersion() {
// param is null
try {
SchemaUtils.isAGreatVersion(null, null);
@ -64,7 +67,7 @@ public class SchemaUtilsTest {
}
@Test
public void testGetAllSchemaList() {
void testGetAllSchemaList() {
List<String> list = null;
try {
list = SchemaUtils.getAllSchemaList();
@ -73,4 +76,5 @@ public class SchemaUtilsTest {
}
Assertions.assertFalse(CollectionUtils.isEmpty(list), "Can not find any schema files");
}
}

13
dolphinscheduler-tools/src/test/resources/application-mysql.yaml
View File

@ -34,6 +34,19 @@ spring:
leak-detection-threshold: 0
initialization-fail-timeout: 1
mybatis-plus:
mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
type-aliases-package: org.apache.dolphinscheduler.dao.entity
configuration:
cache-enabled: false
call-setters-on-nulls: true
map-underscore-to-camel-case: true
jdbc-type-for-null: NULL
global-config:
db-config:
id-type: auto
banner: false
demo:
tenant-code: default
domain-name: localhost

13
dolphinscheduler-tools/src/test/resources/application-postgresql.yaml
View File

@ -34,6 +34,19 @@ spring:
leak-detection-threshold: 0
initialization-fail-timeout: 1
mybatis-plus:
mapper-locations: classpath:org/apache/dolphinscheduler/dao/mapper/*Mapper.xml
type-aliases-package: org.apache.dolphinscheduler.dao.entity
configuration:
cache-enabled: false
call-setters-on-nulls: true
map-underscore-to-camel-case: true
jdbc-type-for-null: NULL
global-config:
db-config:
id-type: auto
banner: false
demo:
tenant-code: default
domain-name: localhost

2
dolphinscheduler-tools/src/test/resources/logback.xml
View File

@ -22,7 +22,7 @@
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<encoder>
<pattern>
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - [WorkflowInstance-%X{workflowInstanceId:-0}][TaskInstance-%X{taskInstanceId:-0}] - %msg%n
[%level] %date{yyyy-MM-dd HH:mm:ss.SSS Z} %logger{96}:[%line] - %msg%n
</pattern>
<charset>UTF-8</charset>
</encoder>

16
dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl.sql Normal file
View File

@ -0,0 +1,16 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

16
dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_ddl_post.sql Normal file
View File

@ -0,0 +1,16 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

16
dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/mysql/dolphinscheduler_dml.sql Normal file
View File

@ -0,0 +1,16 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

16
dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl.sql Normal file
View File

@ -0,0 +1,16 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

16
dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_ddl_post.sql Normal file
View File

@ -0,0 +1,16 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

16
dolphinscheduler-tools/src/test/resources/sql/upgrade/3.3.0_schema/postgresql/dolphinscheduler_dml.sql Normal file
View File

@ -0,0 +1,16 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/