natas_hw
/
dolphinscheduler
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

[CI][E2E] Try to fix e2e timeout error (#14690) 

* try to fix e2e timeout error

* remove use file

(cherry picked from commit dec5228d62)
This commit is contained in:
xiangzihao 2023-08-03 17:08:37 +08:00 committed by Jay Chung
parent 3499c25f84
commit ce5c8f530b
8 changed files with 63 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

8
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml
View File

@ -15,7 +15,7 @@
# limitations under the License.
#
version: "2.1"
version: "3.8"
services:
dolphinscheduler:
@ -23,14 +23,14 @@ services:
environment:
MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true'
expose:
- 12345
ports:
- "12345:12345"
networks:
- e2e
healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s
timeout: 60s
timeout: 5s
retries: 120
networks:

14
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml
View File

@ -15,7 +15,7 @@
# limitations under the License.
#
version: "2.1"
version: "3.8"
services:
dolphinscheduler:
@ -23,14 +23,14 @@ services:
environment:
MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true'
expose:
- 12345
ports:
- "12345:12345"
networks:
- e2e
healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s
timeout: 300s
timeout: 5s
retries: 120
depends_on:
clickhouse:
@ -45,8 +45,8 @@ services:
nofile:
soft: 262144
hard: 262144
expose:
- 8123
ports:
- "8123:8123"
mem_limit: 3G
mem_reservation: 1G
networks:
@ -54,7 +54,7 @@ services:
healthcheck:
test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"]
interval: 5s
timeout: 300s
timeout: 5s
retries: 120
networks:

35
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-hive/docker-compose.yaml
View File

@ -23,15 +23,16 @@ services:
environment:
MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true'
expose:
- 12345
ports:
- "12345:12345"
networks:
- e2e
healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s
timeout: 300s
timeout: 5s
retries: 120
start_period: 60s
depends_on:
hive-server:
condition: service_healthy
@ -43,12 +44,12 @@ services:
- ./hadoop-hive.env
networks:
- e2e
expose:
- "50070"
ports:
- "50070:50070"
healthcheck:
test: [ "CMD", "curl", "http://localhost:50070/" ]
interval: 5s
timeout: 300s
timeout: 5s
retries: 120
datanode:
image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8
@ -58,12 +59,12 @@ services:
SERVICE_PRECONDITION: "namenode:50070"
networks:
- e2e
expose:
- "50075"
ports:
- "50075:50075"
healthcheck:
test: [ "CMD", "curl", "http://localhost:50075" ]
interval: 5s
timeout: 300s
timeout: 5s
retries: 120
hive-server:
image: bde2020/hive:2.3.2-postgresql-metastore
@ -74,8 +75,8 @@ services:
environment:
HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore"
SERVICE_PRECONDITION: "hive-metastore:9083"
expose:
- "10000"
ports:
- "10000:10000"
depends_on:
datanode:
condition: service_healthy
@ -84,7 +85,7 @@ services:
healthcheck:
test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check -e "show databases;"
interval: 5s
timeout: 300s
timeout: 5s
retries: 120
hive-metastore:
image: bde2020/hive:2.3.2-postgresql-metastore
@ -95,8 +96,8 @@ services:
- e2e
environment:
SERVICE_PRECONDITION: "namenode:50070 datanode:50075 hive-metastore-postgresql:5432"
expose:
- "9083"
ports:
- "9083:9083"
depends_on:
hive-metastore-postgresql:
condition: service_healthy
@ -104,12 +105,12 @@ services:
image: bde2020/hive-metastore-postgresql:2.3.0
networks:
- e2e
expose:
- "5432"
ports:
- "5432:5432"
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 5s
timeout: 300s
timeout: 5s
retries: 120

14
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml
View File

@ -15,7 +15,7 @@
# limitations under the License.
#
version: "2.1"
version: "3.8"
services:
dolphinscheduler:
@ -23,8 +23,8 @@ services:
environment:
MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true'
expose:
- 12345
ports:
- "12345:12345"
networks:
- e2e
volumes:
@ -33,7 +33,7 @@ services:
healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s
timeout: 300s
timeout: 5s
retries: 120
depends_on:
mysql:
@ -46,12 +46,12 @@ services:
MYSQL_ROOT_PASSWORD: 123456
networks:
- e2e
expose:
- 3306
ports:
- "3306:3306"
healthcheck:
test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD
interval: 5s
timeout: 300s
timeout: 5s
retries: 120
networks:

10
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml
View File

@ -15,7 +15,7 @@
# limitations under the License.
#
version: "2.1"
version: "3.8"
services:
dolphinscheduler:
@ -23,8 +23,8 @@ services:
environment:
MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true'
expose:
- 12345
ports:
- "12345:12345"
networks:
- e2e
healthcheck:
@ -42,8 +42,8 @@ services:
POSTGRES_PASSWORD: postgres
networks:
- e2e
expose:
- 5432
ports:
- "5432:5432"
healthcheck:
test: ["CMD-SHELL", "pg_isready -U postgres"]
interval: 5s

14
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml
View File

@ -15,7 +15,7 @@
# limitations under the License.
#
version: "2.1"
version: "3.8"
services:
dolphinscheduler:
@ -23,28 +23,28 @@ services:
environment:
MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true'
expose:
- 12345
ports:
- "12345:12345"
networks:
- e2e
healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s
timeout: 300s
timeout: 5s
retries: 120
depends_on:
sqlserver:
condition: service_healthy
sqlserver:
image: alexk002/sqlserver2019_demo:1
expose:
- 1433
ports:
- "1433:1433"
networks:
- e2e
healthcheck:
test: /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P OcP2020123 -Q "SELECT 1"
interval: 5s
timeout: 300s
timeout: 5s
retries: 120
networks:

10
dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml
View File

@ -23,14 +23,14 @@ services:
environment:
MASTER_MAX_CPU_LOAD_AVG: 100
WORKER_TENANT_AUTO_CREATE: 'true'
expose:
- 12345
ports:
- "12345:12345"
networks:
- e2e
healthcheck:
test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ]
interval: 5s
timeout: 300s
timeout: 5s
retries: 120
volumes:
- ./common.properties:/opt/dolphinscheduler/conf/common.properties
@ -46,7 +46,7 @@ services:
stdin_open: true
command: server /data --console-address ":9001"
ports:
- 9000:9000
- "9000:9000"
networks:
- e2e
environment:
@ -55,7 +55,7 @@ services:
healthcheck:
test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
interval: 5s
timeout: 300s
timeout: 5s
retries: 120
mc:
image: minio/mc:RELEASE.2022-01-07T06-01-38Z

15
dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/DolphinSchedulerExtension.java
View File

@ -64,6 +64,8 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
private final int LOCAL_PORT = 5173;
private final int DOCKER_PORT = 12345;
private RemoteWebDriver driver;
private DockerComposeContainer<?> compose;
private BrowserWebDriverContainer<?> browser;
@ -72,6 +74,8 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
private Path record;
private final String serviceName = "dolphinscheduler_1";
@Override
@SuppressWarnings("UnstableApiUsage")
public void beforeAll(ExtensionContext context) throws IOException {
@ -89,7 +93,7 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
setBrowserContainerByOsName();
if (compose != null) {
Testcontainers.exposeHostPorts(compose.getServicePort("dolphinscheduler_1", 12345));
Testcontainers.exposeHostPorts(compose.getServicePort(serviceName, DOCKER_PORT));
browser.withAccessToHost(true);
}
browser.start();
@ -123,7 +127,7 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
compose = createDockerCompose(context);
compose.start();
address = HostAndPort.fromParts("host.testcontainers.internal", compose.getServicePort("dolphinscheduler_1", 12345));
address = HostAndPort.fromParts("host.testcontainers.internal", compose.getServicePort(serviceName, DOCKER_PORT));
rootPath = "/dolphinscheduler/ui/";
}
@ -202,9 +206,10 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall
compose = new DockerComposeContainer<>(files)
.withPull(true)
.withTailChildContainers(true)
.withExposedService("dolphinscheduler_1", 12345)
.withLogConsumer("dolphinscheduler_1", outputFrame -> LOGGER.info(outputFrame.getUtf8String()))
.waitingFor("dolphinscheduler_1", Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(300)));
.withLocalCompose(true)
.withExposedService(serviceName, DOCKER_PORT, Wait.forListeningPort().withStartupTimeout(Duration.ofSeconds(300)))
.withLogConsumer(serviceName, outputFrame -> LOGGER.info(outputFrame.getUtf8String()))
.waitingFor(serviceName, Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(300)));
return compose;
}