From 66618f009c080a77235a5b5e4748f269fb9dbd10 Mon Sep 17 00:00:00 2001 From: EricGao888 Date: Thu, 30 May 2024 17:45:42 +0800 Subject: [PATCH] add unit tests --- ...erverlessSparkDataSourceProcessorTest.java | 213 +++++++++--------- .../AliyunServerlessSparkParameters.java | 20 +- .../AliyunServerlessSparkTask.java | 3 - .../AliyunServerlessSparkTaskChannel.java | 16 ++ ...iyunServerlessSparkTaskChannelFactory.java | 16 ++ .../task/aliyunserverlessspark/RunState.java | 16 ++ .../AliyunServerlessSparkTaskTest.java | 87 +++---- .../fields/use-aliyun-serverless-spark.ts | 54 ----- 8 files changed, 207 insertions(+), 218 deletions(-) diff --git a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-aliyunserverlessspark/src/test/java/org/apache/dolphinscheduler/plugin/datasource/aliyunserverlessspark/AliyunServerlessSparkDataSourceProcessorTest.java b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-aliyunserverlessspark/src/test/java/org/apache/dolphinscheduler/plugin/datasource/aliyunserverlessspark/AliyunServerlessSparkDataSourceProcessorTest.java index 79c26e8b2f..50aae38487 100644 --- a/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-aliyunserverlessspark/src/test/java/org/apache/dolphinscheduler/plugin/datasource/aliyunserverlessspark/AliyunServerlessSparkDataSourceProcessorTest.java +++ b/dolphinscheduler-datasource-plugin/dolphinscheduler-datasource-aliyunserverlessspark/src/test/java/org/apache/dolphinscheduler/plugin/datasource/aliyunserverlessspark/AliyunServerlessSparkDataSourceProcessorTest.java @@ -1,107 +1,106 @@ -///* -// * Licensed to the Apache Software Foundation (ASF) under one or more -// * contributor license agreements. See the NOTICE file distributed with -// * this work for additional information regarding copyright ownership. -// * The ASF licenses this file to You under the Apache License, Version 2.0 -// * (the "License"); you may not use this file except in compliance with -// * the License. You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, software -// * distributed under the License is distributed on an "AS IS" BASIS, -// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// * See the License for the specific language governing permissions and -// * limitations under the License. -// */ -// -//package org.apache.dolphinscheduler.plugin.datasource.aliyunserverlessspark; -// -//import org.apache.dolphinscheduler.plugin.datasource.zeppelin.param.AliyunServerlessSparkConnectionParam; -//import org.apache.dolphinscheduler.plugin.datasource.zeppelin.param.AliyunServerlessSparkDataSourceParamDTO; -//import org.apache.dolphinscheduler.plugin.datasource.zeppelin.param.AliyunServerlessSparkDataSourceProcessor; -//import org.apache.dolphinscheduler.spi.enums.DbType; -// -//import org.junit.jupiter.api.Assertions; -//import org.junit.jupiter.api.BeforeEach; -//import org.junit.jupiter.api.Test; -//import org.junit.jupiter.api.extension.ExtendWith; -//import org.mockito.MockedConstruction; -//import org.mockito.Mockito; -//import org.mockito.junit.jupiter.MockitoExtension; -// -//@ExtendWith(MockitoExtension.class) -//public class AliyunServerlessSparkDataSourceProcessorTest { -// -// private AliyunServerlessSparkDataSourceProcessor zeppelinDataSourceProcessor; -// -// private String connectJson = -// "{\"username\":\"lucky\",\"password\":\"123456\",\"restEndpoint\":\"https://dolphinscheduler.com:8080\"}"; -// -// @BeforeEach -// public void init() { -// zeppelinDataSourceProcessor = new AliyunServerlessSparkDataSourceProcessor(); -// } -// -// @Test -// void testCheckDatasourceParam() { -// AliyunServerlessSparkDataSourceParamDTO zeppelinDataSourceParamDTO = new AliyunServerlessSparkDataSourceParamDTO(); -// Assertions.assertThrows(IllegalArgumentException.class, -// () -> zeppelinDataSourceProcessor.checkDatasourceParam(zeppelinDataSourceParamDTO)); -// zeppelinDataSourceParamDTO.setRestEndpoint("http://dolphinscheduler.com:8080"); -// Assertions.assertThrows(IllegalArgumentException.class, -// () -> zeppelinDataSourceProcessor.checkDatasourceParam(zeppelinDataSourceParamDTO)); -// zeppelinDataSourceParamDTO.setUserName("root"); -// Assertions -// .assertDoesNotThrow(() -> zeppelinDataSourceProcessor.checkDatasourceParam(zeppelinDataSourceParamDTO)); -// } -// -// @Test -// void testGetDatasourceUniqueId() { -// AliyunServerlessSparkConnectionParam zeppelinConnectionParam = new AliyunServerlessSparkConnectionParam(); -// zeppelinConnectionParam.setRestEndpoint("https://dolphinscheduler.com:8080"); -// zeppelinConnectionParam.setUsername("root"); -// zeppelinConnectionParam.setPassword("123456"); -// Assertions.assertEquals("zeppelin@https://dolphinscheduler.com:8080@root@123456", -// zeppelinDataSourceProcessor.getDatasourceUniqueId(zeppelinConnectionParam, DbType.ZEPPELIN)); -// -// } -// -// @Test -// void testCreateDatasourceParamDTO() { -// AliyunServerlessSparkDataSourceParamDTO zeppelinDataSourceParamDTO = -// (AliyunServerlessSparkDataSourceParamDTO) zeppelinDataSourceProcessor.createDatasourceParamDTO(connectJson); -// Assertions.assertEquals("lucky", zeppelinDataSourceParamDTO.getUserName()); -// Assertions.assertEquals("123456", zeppelinDataSourceParamDTO.getPassword()); -// Assertions.assertEquals("https://dolphinscheduler.com:8080", zeppelinDataSourceParamDTO.getRestEndpoint()); -// } -// -// @Test -// void testCreateConnectionParams() { -// AliyunServerlessSparkDataSourceParamDTO zeppelinDataSourceParamDTO = -// (AliyunServerlessSparkDataSourceParamDTO) zeppelinDataSourceProcessor.createDatasourceParamDTO(connectJson); -// AliyunServerlessSparkConnectionParam zeppelinConnectionParam = -// zeppelinDataSourceProcessor.createConnectionParams(zeppelinDataSourceParamDTO); -// Assertions.assertEquals("lucky", zeppelinConnectionParam.getUsername()); -// Assertions.assertEquals("123456", zeppelinConnectionParam.getPassword()); -// Assertions.assertEquals("https://dolphinscheduler.com:8080", zeppelinConnectionParam.getRestEndpoint()); -// } -// -// @Test -// void testTestConnection() { -// AliyunServerlessSparkDataSourceParamDTO zeppelinDataSourceParamDTO = -// (AliyunServerlessSparkDataSourceParamDTO) zeppelinDataSourceProcessor.createDatasourceParamDTO(connectJson); -// AliyunServerlessSparkConnectionParam connectionParam = -// zeppelinDataSourceProcessor.createConnectionParams(zeppelinDataSourceParamDTO); -// Assertions.assertFalse(zeppelinDataSourceProcessor.checkDataSourceConnectivity(connectionParam)); -// try ( -// MockedConstruction sshClientWrapperMockedConstruction = -// Mockito.mockConstruction(ZeppelinClientWrapper.class, (mock, context) -> { -// Mockito.when( -// mock.checkConnect(connectionParam.getUsername(), connectionParam.getPassword())) -// .thenReturn(true); -// })) { -// Assertions.assertTrue(zeppelinDataSourceProcessor.checkDataSourceConnectivity(connectionParam)); -// } -// } -//} +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.plugin.datasource.aliyunserverlessspark; + +import org.apache.dolphinscheduler.plugin.datasource.aliyunserverlessspark.param.AliyunServerlessSparkConnectionParam; +import org.apache.dolphinscheduler.plugin.datasource.aliyunserverlessspark.param.AliyunServerlessSparkDataSourceParamDTO; +import org.apache.dolphinscheduler.plugin.datasource.aliyunserverlessspark.param.AliyunServerlessSparkDataSourceProcessor; +import org.apache.dolphinscheduler.spi.enums.DbType; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.ExtendWith; +import org.mockito.MockedConstruction; +import org.mockito.Mockito; +import org.mockito.junit.jupiter.MockitoExtension; + +@ExtendWith(MockitoExtension.class) +public class AliyunServerlessSparkDataSourceProcessorTest { + + private AliyunServerlessSparkDataSourceProcessor aliyunServerlessSparkDataSourceProcessor; + + private String connectJson = + "{\"accessKeyId\":\"mockAccessKeyId\",\"accessKeySecret\":\"mockAccessKeySecret\",\"regionId\":\"cn-hangzhou\"}"; + + @BeforeEach + public void init() { + aliyunServerlessSparkDataSourceProcessor = new AliyunServerlessSparkDataSourceProcessor(); + } + + @Test + void testCheckDatasourceParam() { + AliyunServerlessSparkDataSourceParamDTO aliyunServerlessSparkDataSourceParamDTO = new AliyunServerlessSparkDataSourceParamDTO(); + aliyunServerlessSparkDataSourceParamDTO.setRegionId("cn-hangzhou"); + Assertions.assertThrows(IllegalArgumentException.class, + () -> aliyunServerlessSparkDataSourceProcessor.checkDatasourceParam(aliyunServerlessSparkDataSourceParamDTO)); + aliyunServerlessSparkDataSourceParamDTO.setAccessKeyId("mockAccessKeyId"); + aliyunServerlessSparkDataSourceParamDTO.setAccessKeySecret("mockAccessKeySecret"); + Assertions + .assertDoesNotThrow(() -> aliyunServerlessSparkDataSourceProcessor.checkDatasourceParam(aliyunServerlessSparkDataSourceParamDTO)); + } + + @Test + void testGetDatasourceUniqueId() { + AliyunServerlessSparkConnectionParam aliyunServerlessSparkConnectionParam = new AliyunServerlessSparkConnectionParam(); + aliyunServerlessSparkConnectionParam.setRegionId("cn-hangzhou"); + aliyunServerlessSparkConnectionParam.setAccessKeyId("mockAccessKeyId"); + aliyunServerlessSparkConnectionParam.setAccessKeySecret("mockAccessKeySecret"); + Assertions.assertEquals("aliyun_serverless_spark@cn-hangzhou@mockAccessKeyId@mockAccessKeySecret", + aliyunServerlessSparkDataSourceProcessor.getDatasourceUniqueId(aliyunServerlessSparkConnectionParam, DbType.ALIYUN_SERVERLESS_SPARK)); + } + + @Test + void testCreateDatasourceParamDTO() { + AliyunServerlessSparkDataSourceParamDTO aliyunServerlessSparkDataSourceParamDTO = + (AliyunServerlessSparkDataSourceParamDTO) aliyunServerlessSparkDataSourceProcessor.createDatasourceParamDTO(connectJson); + Assertions.assertEquals("cn-hangzhou", aliyunServerlessSparkDataSourceParamDTO.getRegionId()); + Assertions.assertEquals("mockAccessKeyId", aliyunServerlessSparkDataSourceParamDTO.getAccessKeyId()); + Assertions.assertEquals("mockAccessKeySecret", aliyunServerlessSparkDataSourceParamDTO.getAccessKeySecret()); + } + + @Test + void testCreateConnectionParams() { + AliyunServerlessSparkDataSourceParamDTO aliyunServerlessSparkDataSourceParamDTO = + (AliyunServerlessSparkDataSourceParamDTO) aliyunServerlessSparkDataSourceProcessor.createDatasourceParamDTO(connectJson); + AliyunServerlessSparkConnectionParam aliyunServerlessSparkConnectionParam = + aliyunServerlessSparkDataSourceProcessor.createConnectionParams(aliyunServerlessSparkDataSourceParamDTO); + Assertions.assertEquals("cn-hangzhou", aliyunServerlessSparkConnectionParam.getRegionId()); + Assertions.assertEquals("mockAccessKeyId", aliyunServerlessSparkConnectionParam.getAccessKeyId()); + Assertions.assertEquals("mockAccessKeySecret", aliyunServerlessSparkConnectionParam.getAccessKeySecret()); + } + + @Test + void testTestConnection() { + AliyunServerlessSparkDataSourceParamDTO aliyunServerlessSparkDataSourceParamDTO = + (AliyunServerlessSparkDataSourceParamDTO) aliyunServerlessSparkDataSourceProcessor.createDatasourceParamDTO(connectJson); + AliyunServerlessSparkConnectionParam connectionParam = + aliyunServerlessSparkDataSourceProcessor.createConnectionParams(aliyunServerlessSparkDataSourceParamDTO); + Assertions.assertTrue(aliyunServerlessSparkDataSourceProcessor.checkDataSourceConnectivity(connectionParam)); + try ( + MockedConstruction AliyunServerlessSparkClientWrapper = + Mockito.mockConstruction(AliyunServerlessSparkClientWrapper.class, (mock, context) -> { + Mockito.when( + mock.checkConnect(connectionParam.getAccessKeyId(), connectionParam.getAccessKeySecret(), connectionParam.getRegionId())) + .thenReturn(true); + })) { + Assertions.assertTrue(aliyunServerlessSparkDataSourceProcessor.checkDataSourceConnectivity(connectionParam)); + } + } +} diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/main/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/AliyunServerlessSparkParameters.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/main/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/AliyunServerlessSparkParameters.java index 7b9415b38f..8e69dc735f 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/main/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/AliyunServerlessSparkParameters.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/main/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/AliyunServerlessSparkParameters.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.dolphinscheduler.plugin.task.aliyunserverlessspark; import lombok.Data; @@ -12,10 +28,6 @@ import java.util.List; @Data @Slf4j public class AliyunServerlessSparkParameters extends AbstractParameters { - // connection configurations -// private String regionId; -// private String accessKeyId; -// private String accessKeySecret; // spark job configurations private String workspaceId; private String resourceQueueId; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/main/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/AliyunServerlessSparkTask.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/main/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/AliyunServerlessSparkTask.java index 017ce7b7cb..d6d60db030 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/main/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/AliyunServerlessSparkTask.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/main/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/AliyunServerlessSparkTask.java @@ -66,8 +66,6 @@ public class AliyunServerlessSparkTask extends AbstractRemoteTask { private String jobRunId; - private RunState previousState; - private RunState currentState; private String accessKeyId; @@ -107,7 +105,6 @@ public class AliyunServerlessSparkTask extends AbstractRemoteTask { throw new AliyunServerlessSparkTaskException("Failed to build Aliyun-Serverless-Spark client!"); } - previousState = RunState.Submitted; currentState = RunState.Submitted; } diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/main/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/AliyunServerlessSparkTaskChannel.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/main/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/AliyunServerlessSparkTaskChannel.java index 5a49beecb3..0f67c94457 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/main/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/AliyunServerlessSparkTaskChannel.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/main/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/AliyunServerlessSparkTaskChannel.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.dolphinscheduler.plugin.task.aliyunserverlessspark; import org.apache.dolphinscheduler.common.utils.JSONUtils; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/main/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/AliyunServerlessSparkTaskChannelFactory.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/main/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/AliyunServerlessSparkTaskChannelFactory.java index 02b02212fa..908c562e2e 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/main/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/AliyunServerlessSparkTaskChannelFactory.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/main/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/AliyunServerlessSparkTaskChannelFactory.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.dolphinscheduler.plugin.task.aliyunserverlessspark; import org.apache.dolphinscheduler.plugin.task.api.TaskChannel; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/main/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/RunState.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/main/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/RunState.java index 656513a6a4..0d617170d3 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/main/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/RunState.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/main/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/RunState.java @@ -1,3 +1,19 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.dolphinscheduler.plugin.task.aliyunserverlessspark; import java.util.Objects; diff --git a/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/test/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/AliyunServerlessSparkTaskTest.java b/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/test/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/AliyunServerlessSparkTaskTest.java index ec2f4c3826..1a360ea8a8 100644 --- a/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/test/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/AliyunServerlessSparkTaskTest.java +++ b/dolphinscheduler-task-plugin/dolphinscheduler-task-aliyunserverlessspark/src/test/java/org/apache/dolphinscheduler/plugin/task/aliyunserverlessspark/AliyunServerlessSparkTaskTest.java @@ -1,64 +1,51 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.dolphinscheduler.plugin.task.aliyunserverlessspark; +import org.apache.dolphinscheduler.plugin.task.api.TaskException; +import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext; + import java.util.Collections; +import org.junit.jupiter.api.BeforeEach; +import org.mockito.InjectMocks; +import org.mockito.Mock; + import com.aliyun.emr_serverless_spark20230808.Client; import com.aliyun.emr_serverless_spark20230808.models.StartJobRunResponse; import com.aliyun.emr_serverless_spark20230808.models.Tag; import com.aliyun.tea.TeaException; public class AliyunServerlessSparkTaskTest { - public static void main(String[] args) throws Exception { - com.aliyun.emr_serverless_spark20230808.Client client = AliyunServerlessSparkTaskTest.createClient(); - com.aliyun.emr_serverless_spark20230808.models.StartJobRunRequest startJobRunRequest = new com.aliyun.emr_serverless_spark20230808.models.StartJobRunRequest(); - startJobRunRequest.setRegionId("cn-hangzhou"); - startJobRunRequest.setResourceQueueId("root_queue"); - startJobRunRequest.setCodeType("JAR"); - startJobRunRequest.setName("ds-test"); - startJobRunRequest.setReleaseVersion("esr-2.1-native (Spark 3.3.1, Scala 2.12, Native Runtime)"); - Tag tag = new Tag(); - tag.setKey("environment"); - tag.setValue("production"); - startJobRunRequest.setTags(Collections.singletonList(tag)); - com.aliyun.emr_serverless_spark20230808.models.JobDriver.JobDriverSparkSubmit jobDriverSparkSubmit = new com.aliyun.emr_serverless_spark20230808.models.JobDriver.JobDriverSparkSubmit() - .setEntryPoint("oss://datadev-oss-hdfs-test/spark-resource/examples/jars/spark-examples_2.12-3.3.1.jar") - .setEntryPointArguments(java.util.Arrays.asList( - "1" - )) - .setSparkSubmitParameters("--class org.apache.spark.examples.SparkPi --conf spark.executor.cores=4 --conf spark.executor.memory=20g --conf spark.driver.cores=4 --conf spark.driver.memory=8g --conf spark.executor.instances=1"); - com.aliyun.emr_serverless_spark20230808.models.JobDriver jobDriver = new com.aliyun.emr_serverless_spark20230808.models.JobDriver() - .setSparkSubmit(jobDriverSparkSubmit); - startJobRunRequest.setJobDriver(jobDriver); - com.aliyun.teautil.models.RuntimeOptions runtime = new com.aliyun.teautil.models.RuntimeOptions(); - java.util.Map headers = new java.util.HashMap<>(); - try { - // 复制代码运行请自行打印 API 的返回值 - StartJobRunResponse startJobRunResponse = client.startJobRunWithOptions("w-f7b841e8c73211be", startJobRunRequest, headers, runtime); - System.out.println(startJobRunResponse.getBody().getRequestId()); - System.out.println(startJobRunResponse.getBody().getJobRunId()); - } catch (TeaException error) { - // 此处仅做打印展示，请谨慎对待异常处理，在工程项目中切勿直接忽略异常。 - // 错误 message - System.out.println(error.getMessage()); - // 诊断地址 - System.out.println(error.getData().get("Recommend")); - com.aliyun.teautil.Common.assertAsString(error.message); - } catch (Exception _error) { - TeaException error = new TeaException(_error.getMessage(), _error); - // 此处仅做打印展示，请谨慎对待异常处理，在工程项目中切勿直接忽略异常。 - // 错误 message - System.out.println(error.getMessage()); - // 诊断地址 - System.out.println(error.getData().get("Recommend")); - com.aliyun.teautil.Common.assertAsString(error.message); - } + @Mock + private TaskExecutionContext mockTaskExecutionContext; + + @Mock + private Client mockAliyunServerlessSparkClient; + + @InjectMocks + private AliyunServerlessSparkTask aliyunServerlessSparkTask; + + @BeforeEach + public void before() { + } - public static Client createClient() throws Exception { - com.aliyun.teaopenapi.models.Config config = new com.aliyun.teaopenapi.models.Config() - .setAccessKeyId(System.getenv("ALIBABA_CLOUD_ACCESS_KEY_ID")) - .setAccessKeySecret(System.getenv("ALIBABA_CLOUD_ACCESS_KEY_SECRET")); - config.endpoint = "emr-serverless-spark.cn-hangzhou.aliyuncs.com"; - return new com.aliyun.emr_serverless_spark20230808.Client(config); + public void testInit() { + } } diff --git a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-aliyun-serverless-spark.ts b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-aliyun-serverless-spark.ts index b7ccb5f1f4..7c1777a740 100644 --- a/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-aliyun-serverless-spark.ts +++ b/dolphinscheduler-ui/src/views/projects/task/components/node/fields/use-aliyun-serverless-spark.ts @@ -23,60 +23,6 @@ export function useAliyunServerlessSpark(model: { [field: string]: any }): IJson return [ // mandatory field - // { - // type: 'input', - // field: 'regionId', - // name: t('project.node.region_id'), - // props: { - // placeholder: t('project.node.region_id_tips') - // }, - // validate: { - // trigger: ['input', 'blur'], - // required: true, - // validator(validate: any, value: string) { - // if (!value) { - // return new Error(t('project.node.region_id_tips')) - // } - // } - // } - // }, - - // { - // type: 'input', - // field: 'accessKeyId', - // name: t('project.node.access_key_id'), - // props: { - // placeholder: t('project.node.access_key_id_tips') - // }, - // validate: { - // trigger: ['input', 'blur'], - // required: true, - // validator(validate: any, value: string) { - // if (!value) { - // return new Error(t('project.node.access_key_id_tips')) - // } - // } - // } - // }, - - // { - // type: 'input', - // field: 'accessKeySecret', - // name: t('project.node.access_key_secret'), - // props: { - // placeholder: t('project.node.access_key_secret_tips') - // }, - // validate: { - // trigger: ['input', 'blur'], - // required: true, - // validator(validate: any, value: string) { - // if (!value) { - // return new Error(t('project.node.access_key_secret_tips')) - // } - // } - // } - // }, - { type: 'input', field: 'workspaceId',