From ce5c8f530b42daf56fe19af0c846e6003bd2b960 Mon Sep 17 00:00:00 2001 From: xiangzihao <460888207@qq.com> Date: Thu, 3 Aug 2023 17:08:37 +0800 Subject: [PATCH] [CI][E2E] Try to fix e2e timeout error (#14690) * try to fix e2e timeout error * remove use file (cherry picked from commit dec5228d62400dda1fd3aef2d6a12473f86fd57b) --- .../docker/basic/docker-compose.yaml | 8 ++--- .../datasource-clickhouse/docker-compose.yaml | 14 ++++---- .../datasource-hive/docker-compose.yaml | 35 ++++++++++--------- .../datasource-mysql/docker-compose.yaml | 14 ++++---- .../datasource-postgresql/docker-compose.yaml | 10 +++--- .../datasource-sqlserver/docker-compose.yaml | 14 ++++---- .../docker/file-manage/docker-compose.yaml | 10 +++--- .../e2e/core/DolphinSchedulerExtension.java | 15 +++++--- 8 files changed, 63 insertions(+), 57 deletions(-) diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml index f12ddd9342..89d645c2c1 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/basic/docker-compose.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -version: "2.1" +version: "3.8" services: dolphinscheduler: @@ -23,14 +23,14 @@ services: environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' - expose: - - 12345 + ports: + - "12345:12345" networks: - e2e healthcheck: test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] interval: 5s - timeout: 60s + timeout: 5s retries: 120 networks: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml index 12279aaf44..3b9ccc0f93 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-clickhouse/docker-compose.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -version: "2.1" +version: "3.8" services: dolphinscheduler: @@ -23,14 +23,14 @@ services: environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' - expose: - - 12345 + ports: + - "12345:12345" networks: - e2e healthcheck: test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] interval: 5s - timeout: 300s + timeout: 5s retries: 120 depends_on: clickhouse: @@ -45,8 +45,8 @@ services: nofile: soft: 262144 hard: 262144 - expose: - - 8123 + ports: + - "8123:8123" mem_limit: 3G mem_reservation: 1G networks: @@ -54,7 +54,7 @@ services: healthcheck: test: ["CMD", "wget", "--spider", "-q", "localhost:8123/ping"] interval: 5s - timeout: 300s + timeout: 5s retries: 120 networks: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-hive/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-hive/docker-compose.yaml index a976244753..b7609ff874 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-hive/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-hive/docker-compose.yaml @@ -23,15 +23,16 @@ services: environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' - expose: - - 12345 + ports: + - "12345:12345" networks: - e2e healthcheck: test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] interval: 5s - timeout: 300s + timeout: 5s retries: 120 + start_period: 60s depends_on: hive-server: condition: service_healthy @@ -43,12 +44,12 @@ services: - ./hadoop-hive.env networks: - e2e - expose: - - "50070" + ports: + - "50070:50070" healthcheck: test: [ "CMD", "curl", "http://localhost:50070/" ] interval: 5s - timeout: 300s + timeout: 5s retries: 120 datanode: image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 @@ -58,12 +59,12 @@ services: SERVICE_PRECONDITION: "namenode:50070" networks: - e2e - expose: - - "50075" + ports: + - "50075:50075" healthcheck: test: [ "CMD", "curl", "http://localhost:50075" ] interval: 5s - timeout: 300s + timeout: 5s retries: 120 hive-server: image: bde2020/hive:2.3.2-postgresql-metastore @@ -74,8 +75,8 @@ services: environment: HIVE_CORE_CONF_javax_jdo_option_ConnectionURL: "jdbc:postgresql://hive-metastore/metastore" SERVICE_PRECONDITION: "hive-metastore:9083" - expose: - - "10000" + ports: + - "10000:10000" depends_on: datanode: condition: service_healthy @@ -84,7 +85,7 @@ services: healthcheck: test: beeline -u "jdbc:hive2://127.0.0.1:10000/default" -n health_check -e "show databases;" interval: 5s - timeout: 300s + timeout: 5s retries: 120 hive-metastore: image: bde2020/hive:2.3.2-postgresql-metastore @@ -95,8 +96,8 @@ services: - e2e environment: SERVICE_PRECONDITION: "namenode:50070 datanode:50075 hive-metastore-postgresql:5432" - expose: - - "9083" + ports: + - "9083:9083" depends_on: hive-metastore-postgresql: condition: service_healthy @@ -104,12 +105,12 @@ services: image: bde2020/hive-metastore-postgresql:2.3.0 networks: - e2e - expose: - - "5432" + ports: + - "5432:5432" healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 5s - timeout: 300s + timeout: 5s retries: 120 diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml index 98d7bac656..099736805e 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-mysql/docker-compose.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -version: "2.1" +version: "3.8" services: dolphinscheduler: @@ -23,8 +23,8 @@ services: environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' - expose: - - 12345 + ports: + - "12345:12345" networks: - e2e volumes: @@ -33,7 +33,7 @@ services: healthcheck: test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] interval: 5s - timeout: 300s + timeout: 5s retries: 120 depends_on: mysql: @@ -46,12 +46,12 @@ services: MYSQL_ROOT_PASSWORD: 123456 networks: - e2e - expose: - - 3306 + ports: + - "3306:3306" healthcheck: test: mysqladmin ping -h 127.0.0.1 -u root --password=$$MYSQL_ROOT_PASSWORD interval: 5s - timeout: 300s + timeout: 5s retries: 120 networks: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml index c78f3e73e6..7d1e38832c 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-postgresql/docker-compose.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -version: "2.1" +version: "3.8" services: dolphinscheduler: @@ -23,8 +23,8 @@ services: environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' - expose: - - 12345 + ports: + - "12345:12345" networks: - e2e healthcheck: @@ -42,8 +42,8 @@ services: POSTGRES_PASSWORD: postgres networks: - e2e - expose: - - 5432 + ports: + - "5432:5432" healthcheck: test: ["CMD-SHELL", "pg_isready -U postgres"] interval: 5s diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml index 1a1fbc82e2..cf94105313 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/datasource-sqlserver/docker-compose.yaml @@ -15,7 +15,7 @@ # limitations under the License. # -version: "2.1" +version: "3.8" services: dolphinscheduler: @@ -23,28 +23,28 @@ services: environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' - expose: - - 12345 + ports: + - "12345:12345" networks: - e2e healthcheck: test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] interval: 5s - timeout: 300s + timeout: 5s retries: 120 depends_on: sqlserver: condition: service_healthy sqlserver: image: alexk002/sqlserver2019_demo:1 - expose: - - 1433 + ports: + - "1433:1433" networks: - e2e healthcheck: test: /opt/mssql-tools/bin/sqlcmd -S localhost -U sa -P OcP2020123 -Q "SELECT 1" interval: 5s - timeout: 300s + timeout: 5s retries: 120 networks: diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml index 41c8b7945c..9a46ed02ad 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-case/src/test/resources/docker/file-manage/docker-compose.yaml @@ -23,14 +23,14 @@ services: environment: MASTER_MAX_CPU_LOAD_AVG: 100 WORKER_TENANT_AUTO_CREATE: 'true' - expose: - - 12345 + ports: + - "12345:12345" networks: - e2e healthcheck: test: [ "CMD", "curl", "http://localhost:12345/actuator/health" ] interval: 5s - timeout: 300s + timeout: 5s retries: 120 volumes: - ./common.properties:/opt/dolphinscheduler/conf/common.properties @@ -46,7 +46,7 @@ services: stdin_open: true command: server /data --console-address ":9001" ports: - - 9000:9000 + - "9000:9000" networks: - e2e environment: @@ -55,7 +55,7 @@ services: healthcheck: test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"] interval: 5s - timeout: 300s + timeout: 5s retries: 120 mc: image: minio/mc:RELEASE.2022-01-07T06-01-38Z diff --git a/dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/DolphinSchedulerExtension.java b/dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/DolphinSchedulerExtension.java index edb10caf77..ce96044777 100644 --- a/dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/DolphinSchedulerExtension.java +++ b/dolphinscheduler-e2e/dolphinscheduler-e2e-core/src/main/java/org/apache/dolphinscheduler/e2e/core/DolphinSchedulerExtension.java @@ -64,6 +64,8 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall private final int LOCAL_PORT = 5173; + private final int DOCKER_PORT = 12345; + private RemoteWebDriver driver; private DockerComposeContainer compose; private BrowserWebDriverContainer browser; @@ -72,6 +74,8 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall private Path record; + private final String serviceName = "dolphinscheduler_1"; + @Override @SuppressWarnings("UnstableApiUsage") public void beforeAll(ExtensionContext context) throws IOException { @@ -89,7 +93,7 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall setBrowserContainerByOsName(); if (compose != null) { - Testcontainers.exposeHostPorts(compose.getServicePort("dolphinscheduler_1", 12345)); + Testcontainers.exposeHostPorts(compose.getServicePort(serviceName, DOCKER_PORT)); browser.withAccessToHost(true); } browser.start(); @@ -123,7 +127,7 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall compose = createDockerCompose(context); compose.start(); - address = HostAndPort.fromParts("host.testcontainers.internal", compose.getServicePort("dolphinscheduler_1", 12345)); + address = HostAndPort.fromParts("host.testcontainers.internal", compose.getServicePort(serviceName, DOCKER_PORT)); rootPath = "/dolphinscheduler/ui/"; } @@ -202,9 +206,10 @@ final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCall compose = new DockerComposeContainer<>(files) .withPull(true) .withTailChildContainers(true) - .withExposedService("dolphinscheduler_1", 12345) - .withLogConsumer("dolphinscheduler_1", outputFrame -> LOGGER.info(outputFrame.getUtf8String())) - .waitingFor("dolphinscheduler_1", Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(300))); + .withLocalCompose(true) + .withExposedService(serviceName, DOCKER_PORT, Wait.forListeningPort().withStartupTimeout(Duration.ofSeconds(300))) + .withLogConsumer(serviceName, outputFrame -> LOGGER.info(outputFrame.getUtf8String())) + .waitingFor(serviceName, Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(300))); return compose; }