natas_hw
/
dolphinscheduler0701
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[Chore] Use testcontainer origin exposed port rather than generate random port (#16195)

This commit is contained in:
Wenjun Ruan 2024-06-21 22:19:54 +08:00 committed by GitHub
parent 4a76a07c9c
commit f0e0395b36
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
11 changed files with 53 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
dolphinscheduler-dao/src/test/java/org/apache/dolphinscheduler/dao/repository/impl/CommandDaoImplTest.java
View File

@ -29,7 +29,6 @@ import org.apache.dolphinscheduler.common.enums.WarningType;
import org.apache.dolphinscheduler.common.utils.DateUtils;
import org.apache.dolphinscheduler.dao.BaseDaoTest;
import org.apache.dolphinscheduler.dao.entity.Command;
import org.apache.dolphinscheduler.dao.mapper.CommandMapper;
import org.apache.dolphinscheduler.dao.repository.CommandDao;
import org.apache.commons.lang3.RandomUtils;
@ -39,34 +38,28 @@ import java.util.stream.Collectors;
import org.junit.jupiter.api.RepeatedTest;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.test.annotation.DirtiesContext;
import com.baomidou.mybatisplus.core.conditions.query.QueryWrapper;
@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
class CommandDaoImplTest extends BaseDaoTest {
@Autowired
private CommandDao commandDao;
@Autowired
private CommandMapper commandMapper;
@RepeatedTest(value = 100)
@RepeatedTest(value = 10)
void fetchCommandByIdSlot() {
// clear all commands
commandMapper.delete(new QueryWrapper<Command>().ge("id", -1));
int totalSlot = RandomUtils.nextInt(1, 10);
int currentSlotIndex = RandomUtils.nextInt(0, totalSlot);
int fetchSize = RandomUtils.nextInt(10, 100);
int idStep = RandomUtils.nextInt(1, 5);
int commandSize = RandomUtils.nextInt(currentSlotIndex, 1000);
// Generate commandSize commands
int id = 0;
int id = 1;
for (int j = 0; j < commandSize; j++) {
id += idStep;
Command command = generateCommand(CommandType.START_PROCESS, 0);
command.setId(id);
commandDao.insert(command);
id += idStep;
}
List<Command> commands = commandDao.queryCommandByIdSlot(currentSlotIndex, totalSlot, idStep, fetchSize);

1
dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-etcd/src/test/java/org/apache/dolphinscheduler/plugin/registry/etcd/EtcdRegistryTestCase.java
View File

@ -51,6 +51,7 @@ public class EtcdRegistryTestCase extends RegistryTestCase<EtcdRegistry> {
.build()
.cluster();
etcdCluster.start();
System.clearProperty("registry.endpoints");
System.setProperty("registry.endpoints",
etcdCluster.clientEndpoints().stream().map(URI::toString).collect(Collectors.joining(",")));
}

10
dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/MysqlJdbcRegistryTestCase.java
View File

@ -17,8 +17,6 @@
package org.apache.dolphinscheduler.plugin.registry.jdbc;
import org.apache.commons.lang3.RandomUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Statement;
@ -37,8 +35,6 @@ import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.lifecycle.Startables;
import org.testcontainers.utility.DockerImageName;
import com.google.common.collect.Lists;
@ActiveProfiles("mysql")
class MysqlJdbcRegistryTestCase extends JdbcRegistryTestCase {
@ -55,11 +51,11 @@ class MysqlJdbcRegistryTestCase extends JdbcRegistryTestCase {
.withExposedPorts(3306)
.waitingFor(Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(300)));
int exposedPort = RandomUtils.nextInt(10000, 65535);
mysqlContainer.setPortBindings(Lists.newArrayList(exposedPort + ":3306"));
Startables.deepStart(Stream.of(mysqlContainer)).join();
String jdbcUrl = "jdbc:mysql://localhost:" + exposedPort + "/dolphinscheduler?useSSL=false&serverTimezone=UTC";
String jdbcUrl = "jdbc:mysql://localhost:" + mysqlContainer.getMappedPort(3306)
+ "/dolphinscheduler?useSSL=false&serverTimezone=UTC";
System.clearProperty("spring.datasource.url");
System.setProperty("spring.datasource.url", jdbcUrl);
try (

9
dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-jdbc/src/test/java/org/apache/dolphinscheduler/plugin/registry/jdbc/PostgresqlJdbcRegistryTestCase.java
View File

@ -17,8 +17,6 @@
package org.apache.dolphinscheduler.plugin.registry.jdbc;
import org.apache.commons.lang3.RandomUtils;
import java.sql.Connection;
import java.sql.DriverManager;
import java.sql.Statement;
@ -37,8 +35,6 @@ import org.testcontainers.containers.PostgreSQLContainer;
import org.testcontainers.lifecycle.Startables;
import org.testcontainers.utility.DockerImageName;
import com.google.common.collect.Lists;
@ActiveProfiles("postgresql")
@SpringBootTest(classes = {JdbcRegistryProperties.class})
@SpringBootApplication(scanBasePackageClasses = JdbcRegistryProperties.class)
@ -55,12 +51,11 @@ public class PostgresqlJdbcRegistryTestCase extends JdbcRegistryTestCase {
.withDatabaseName("dolphinscheduler")
.withNetwork(Network.newNetwork())
.withExposedPorts(5432);
int exposedPort = RandomUtils.nextInt(10000, 65535);
postgresqlContainer.setPortBindings(Lists.newArrayList(exposedPort + ":5432"));
Startables.deepStart(Stream.of(postgresqlContainer)).join();
String jdbcUrl = "jdbc:postgresql://localhost:" + exposedPort + "/dolphinscheduler";
String jdbcUrl = "jdbc:postgresql://localhost:" + postgresqlContainer.getMappedPort(5432) + "/dolphinscheduler";
System.clearProperty("spring.datasource.url");
System.setProperty("spring.datasource.url", jdbcUrl);
try (
Connection connection = DriverManager.getConnection(jdbcUrl, "root", "root");

12
dolphinscheduler-registry/dolphinscheduler-registry-plugins/dolphinscheduler-registry-zookeeper/src/test/java/org/apache/dolphinscheduler/plugin/registry/zookeeper/ZookeeperRegistryTestCase.java
View File

@ -19,8 +19,6 @@ package org.apache.dolphinscheduler.plugin.registry.zookeeper;
import org.apache.dolphinscheduler.plugin.registry.RegistryTestCase;
import org.apache.commons.lang3.RandomUtils;
import java.util.stream.Stream;
import lombok.SneakyThrows;
@ -35,8 +33,6 @@ import org.testcontainers.containers.Network;
import org.testcontainers.lifecycle.Startables;
import org.testcontainers.utility.DockerImageName;
import com.google.common.collect.Lists;
@SpringBootTest(classes = ZookeeperRegistryProperties.class)
@SpringBootApplication(scanBasePackageClasses = ZookeeperRegistryProperties.class)
class ZookeeperRegistryTestCase extends RegistryTestCase<ZookeeperRegistry> {
@ -52,11 +48,11 @@ class ZookeeperRegistryTestCase extends RegistryTestCase<ZookeeperRegistry> {
@BeforeAll
public static void setUpTestingServer() {
zookeeperContainer = new GenericContainer<>(DockerImageName.parse("zookeeper:3.8"))
.withNetwork(NETWORK);
int randomPort = RandomUtils.nextInt(10000, 65535);
zookeeperContainer.setPortBindings(Lists.newArrayList(randomPort + ":2181"));
.withNetwork(NETWORK)
.withExposedPorts(2181);
Startables.deepStart(Stream.of(zookeeperContainer)).join();
System.setProperty("registry.zookeeper.connect-string", "localhost:" + randomPort);
System.clearProperty("registry.zookeeper.connect-string");
System.setProperty("registry.zookeeper.connect-string", "localhost:" + zookeeperContainer.getMappedPort(2181));
}
@SneakyThrows

1
dolphinscheduler-storage-plugin/dolphinscheduler-storage-hdfs/src/test/java/org/apache/dolphinscheduler/plugin/storage/hdfs/LocalStorageOperatorTest.java
View File

@ -52,6 +52,7 @@ class LocalStorageOperatorTest {
@BeforeEach
public void setup() {
Files.createDirectories(Paths.get(resourceBaseDir));
System.clearProperty(Constants.RESOURCE_UPLOAD_PATH);
System.setProperty(Constants.RESOURCE_UPLOAD_PATH, resourceBaseDir);
LocalStorageOperatorFactory localStorageOperatorFactory = new LocalStorageOperatorFactory();

8
dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/jupiter/DolphinSchedulerDatabaseContainerExtension.java
View File

@ -20,7 +20,6 @@ package org.apache.dolphinscheduler.tools.datasource.jupiter;
import java.util.HashMap;
import java.util.Map;
import java.util.ServiceLoader;
import java.util.stream.Stream;
import lombok.extern.slf4j.Slf4j;
@ -28,7 +27,6 @@ import org.junit.jupiter.api.extension.AfterAllCallback;
import org.junit.jupiter.api.extension.BeforeAllCallback;
import org.junit.jupiter.api.extension.ExtensionContext;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.lifecycle.Startables;
import org.testcontainers.utility.DockerImageName;
@Slf4j
@ -39,12 +37,6 @@ public class DolphinSchedulerDatabaseContainerExtension implements BeforeAllCall
@Override
public void beforeAll(ExtensionContext context) {
databaseContainer = getDataSourceContainer(context);
log.info("Create {} successfully.", databaseContainer.getDockerImageName());
databaseContainer.start();
log.info("Starting {}...", databaseContainer.getDockerImageName());
Startables.deepStart(Stream.of(databaseContainer)).join();
log.info("{} started", databaseContainer.getDockerImageName());
}

2
dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/mysql/DolphinSchedulerMysqlProfile.java
View File

@ -27,6 +27,7 @@ import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ActiveProfiles;
@Inherited
@ -34,6 +35,7 @@ import org.springframework.test.context.ActiveProfiles;
@Retention(RetentionPolicy.RUNTIME)
@ActiveProfiles("mysql")
@SpringBootTest(classes = {UpgradeDolphinScheduler.class, DaoConfiguration.class})
@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
public @interface DolphinSchedulerMysqlProfile {
}

21
dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/mysql/MysqlDatabaseContainerProvider.java
View File

@ -20,15 +20,20 @@ package org.apache.dolphinscheduler.tools.datasource.mysql;
import org.apache.dolphinscheduler.tools.datasource.jupiter.DatabaseContainerProvider;
import org.apache.dolphinscheduler.tools.datasource.jupiter.DolphinSchedulerDatabaseContainer;
import java.util.stream.Stream;
import lombok.extern.slf4j.Slf4j;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.MySQLContainer;
import org.testcontainers.containers.Network;
import org.testcontainers.containers.wait.strategy.Wait;
import org.testcontainers.lifecycle.Startables;
import org.testcontainers.utility.DockerImageName;
import com.google.auto.service.AutoService;
import com.google.common.collect.Lists;
@Slf4j
@AutoService(DatabaseContainerProvider.class)
public class MysqlDatabaseContainerProvider implements DatabaseContainerProvider {
@ -43,7 +48,19 @@ public class MysqlDatabaseContainerProvider implements DatabaseContainerProvider
.withNetwork(NETWORK)
.withExposedPorts(3306)
.waitingFor(Wait.forHealthcheck());
mysqlContainer.setPortBindings(Lists.newArrayList("3306:3306"));
log.info("Create {} successfully.", mysqlContainer.getDockerImageName());
mysqlContainer.start();
log.info("Starting {}...", mysqlContainer.getDockerImageName());
Startables.deepStart(Stream.of(mysqlContainer)).join();
log.info("{} started", mysqlContainer.getDockerImageName());
String jdbcUrl = "jdbc:mysql://localhost:" + mysqlContainer.getMappedPort(3306)
+ "/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8";
System.clearProperty("spring.datasource.url");
System.setProperty("spring.datasource.url", jdbcUrl);
return mysqlContainer;
}

2
dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/postgresql/DolphinSchedulerPostgresqlProfile.java
View File

@ -27,12 +27,14 @@ import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.annotation.DirtiesContext;
import org.springframework.test.context.ActiveProfiles;
@Inherited
@Target(ElementType.TYPE)
@Retention(RetentionPolicy.RUNTIME)
@ActiveProfiles("postgresql")
@SpringBootTest(classes = {UpgradeDolphinScheduler.class, DaoConfiguration.class})
@DirtiesContext(classMode = DirtiesContext.ClassMode.BEFORE_EACH_TEST_METHOD)
public @interface DolphinSchedulerPostgresqlProfile {
}

16
dolphinscheduler-tools/src/test/java/org/apache/dolphinscheduler/tools/datasource/postgresql/PostgresqlDatabaseContainerProvider.java
View File

@ -20,15 +20,17 @@ package org.apache.dolphinscheduler.tools.datasource.postgresql;
import org.apache.dolphinscheduler.tools.datasource.jupiter.DatabaseContainerProvider;
import org.apache.dolphinscheduler.tools.datasource.jupiter.DolphinSchedulerDatabaseContainer;
import java.util.stream.Stream;
import lombok.extern.slf4j.Slf4j;
import org.testcontainers.containers.GenericContainer;
import org.testcontainers.containers.Network;
import org.testcontainers.containers.PostgreSQLContainer;
import org.testcontainers.lifecycle.Startables;
import org.testcontainers.utility.DockerImageName;
import com.google.auto.service.AutoService;
import com.google.common.collect.Lists;
@Slf4j
@AutoService(DatabaseContainerProvider.class)
@ -47,8 +49,18 @@ public class PostgresqlDatabaseContainerProvider implements DatabaseContainerPro
.withDatabaseName("dolphinscheduler")
.withNetwork(NETWORK)
.withExposedPorts(5432);
postgresqlContainer.setPortBindings(Lists.newArrayList("5432:5432"));
log.info("Create {} successfully.", postgresqlContainer.getDockerImageName());
postgresqlContainer.start();
log.info("Starting {}...", postgresqlContainer.getDockerImageName());
Startables.deepStart(Stream.of(postgresqlContainer)).join();
log.info("{} started", postgresqlContainer.getDockerImageName());
String jdbcUrl = "jdbc:mysql://localhost:" + postgresqlContainer.getMappedPort(5432)
+ "/dolphinscheduler?useUnicode=true&characterEncoding=UTF-8";
System.clearProperty("spring.datasource.url");
System.setProperty("spring.datasource.url", jdbcUrl);
return postgresqlContainer;
}