add unit tests

This commit is contained in:
EricGao888 2024-05-30 17:45:42 +08:00
parent f8a49cc647
commit 66618f009c
8 changed files with 207 additions and 218 deletions

View File

@ -1,107 +1,106 @@
///*
// * Licensed to the Apache Software Foundation (ASF) under one or more
// * contributor license agreements. See the NOTICE file distributed with
// * this work for additional information regarding copyright ownership.
// * The ASF licenses this file to You under the Apache License, Version 2.0
// * (the "License"); you may not use this file except in compliance with
// * the License. You may obtain a copy of the License at
// *
// * http://www.apache.org/licenses/LICENSE-2.0
// *
// * Unless required by applicable law or agreed to in writing, software
// * distributed under the License is distributed on an "AS IS" BASIS,
// * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// * See the License for the specific language governing permissions and
// * limitations under the License.
// */
//
//package org.apache.dolphinscheduler.plugin.datasource.aliyunserverlessspark;
//
//import org.apache.dolphinscheduler.plugin.datasource.zeppelin.param.AliyunServerlessSparkConnectionParam;
//import org.apache.dolphinscheduler.plugin.datasource.zeppelin.param.AliyunServerlessSparkDataSourceParamDTO;
//import org.apache.dolphinscheduler.plugin.datasource.zeppelin.param.AliyunServerlessSparkDataSourceProcessor;
//import org.apache.dolphinscheduler.spi.enums.DbType;
//
//import org.junit.jupiter.api.Assertions;
//import org.junit.jupiter.api.BeforeEach;
//import org.junit.jupiter.api.Test;
//import org.junit.jupiter.api.extension.ExtendWith;
//import org.mockito.MockedConstruction;
//import org.mockito.Mockito;
//import org.mockito.junit.jupiter.MockitoExtension;
//
//@ExtendWith(MockitoExtension.class)
//public class AliyunServerlessSparkDataSourceProcessorTest {
//
// private AliyunServerlessSparkDataSourceProcessor zeppelinDataSourceProcessor;
//
// private String connectJson =
// "{\"username\":\"lucky\",\"password\":\"123456\",\"restEndpoint\":\"https://dolphinscheduler.com:8080\"}";
//
// @BeforeEach
// public void init() {
// zeppelinDataSourceProcessor = new AliyunServerlessSparkDataSourceProcessor();
// }
//
// @Test
// void testCheckDatasourceParam() {
// AliyunServerlessSparkDataSourceParamDTO zeppelinDataSourceParamDTO = new AliyunServerlessSparkDataSourceParamDTO();
// Assertions.assertThrows(IllegalArgumentException.class,
// () -> zeppelinDataSourceProcessor.checkDatasourceParam(zeppelinDataSourceParamDTO));
// zeppelinDataSourceParamDTO.setRestEndpoint("http://dolphinscheduler.com:8080");
// Assertions.assertThrows(IllegalArgumentException.class,
// () -> zeppelinDataSourceProcessor.checkDatasourceParam(zeppelinDataSourceParamDTO));
// zeppelinDataSourceParamDTO.setUserName("root");
// Assertions
// .assertDoesNotThrow(() -> zeppelinDataSourceProcessor.checkDatasourceParam(zeppelinDataSourceParamDTO));
// }
//
// @Test
// void testGetDatasourceUniqueId() {
// AliyunServerlessSparkConnectionParam zeppelinConnectionParam = new AliyunServerlessSparkConnectionParam();
// zeppelinConnectionParam.setRestEndpoint("https://dolphinscheduler.com:8080");
// zeppelinConnectionParam.setUsername("root");
// zeppelinConnectionParam.setPassword("123456");
// Assertions.assertEquals("zeppelin@https://dolphinscheduler.com:8080@root@123456",
// zeppelinDataSourceProcessor.getDatasourceUniqueId(zeppelinConnectionParam, DbType.ZEPPELIN));
//
// }
//
// @Test
// void testCreateDatasourceParamDTO() {
// AliyunServerlessSparkDataSourceParamDTO zeppelinDataSourceParamDTO =
// (AliyunServerlessSparkDataSourceParamDTO) zeppelinDataSourceProcessor.createDatasourceParamDTO(connectJson);
// Assertions.assertEquals("lucky", zeppelinDataSourceParamDTO.getUserName());
// Assertions.assertEquals("123456", zeppelinDataSourceParamDTO.getPassword());
// Assertions.assertEquals("https://dolphinscheduler.com:8080", zeppelinDataSourceParamDTO.getRestEndpoint());
// }
//
// @Test
// void testCreateConnectionParams() {
// AliyunServerlessSparkDataSourceParamDTO zeppelinDataSourceParamDTO =
// (AliyunServerlessSparkDataSourceParamDTO) zeppelinDataSourceProcessor.createDatasourceParamDTO(connectJson);
// AliyunServerlessSparkConnectionParam zeppelinConnectionParam =
// zeppelinDataSourceProcessor.createConnectionParams(zeppelinDataSourceParamDTO);
// Assertions.assertEquals("lucky", zeppelinConnectionParam.getUsername());
// Assertions.assertEquals("123456", zeppelinConnectionParam.getPassword());
// Assertions.assertEquals("https://dolphinscheduler.com:8080", zeppelinConnectionParam.getRestEndpoint());
// }
//
// @Test
// void testTestConnection() {
// AliyunServerlessSparkDataSourceParamDTO zeppelinDataSourceParamDTO =
// (AliyunServerlessSparkDataSourceParamDTO) zeppelinDataSourceProcessor.createDatasourceParamDTO(connectJson);
// AliyunServerlessSparkConnectionParam connectionParam =
// zeppelinDataSourceProcessor.createConnectionParams(zeppelinDataSourceParamDTO);
// Assertions.assertFalse(zeppelinDataSourceProcessor.checkDataSourceConnectivity(connectionParam));
// try (
// MockedConstruction<ZeppelinClientWrapper> sshClientWrapperMockedConstruction =
// Mockito.mockConstruction(ZeppelinClientWrapper.class, (mock, context) -> {
// Mockito.when(
// mock.checkConnect(connectionParam.getUsername(), connectionParam.getPassword()))
// .thenReturn(true);
// })) {
// Assertions.assertTrue(zeppelinDataSourceProcessor.checkDataSourceConnectivity(connectionParam));
// }
// }
//}
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.datasource.aliyunserverlessspark;
import org.apache.dolphinscheduler.plugin.datasource.aliyunserverlessspark.param.AliyunServerlessSparkConnectionParam;
import org.apache.dolphinscheduler.plugin.datasource.aliyunserverlessspark.param.AliyunServerlessSparkDataSourceParamDTO;
import org.apache.dolphinscheduler.plugin.datasource.aliyunserverlessspark.param.AliyunServerlessSparkDataSourceProcessor;
import org.apache.dolphinscheduler.spi.enums.DbType;
import org.junit.jupiter.api.Assertions;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Disabled;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.MockedConstruction;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
@ExtendWith(MockitoExtension.class)
public class AliyunServerlessSparkDataSourceProcessorTest {
private AliyunServerlessSparkDataSourceProcessor aliyunServerlessSparkDataSourceProcessor;
private String connectJson =
"{\"accessKeyId\":\"mockAccessKeyId\",\"accessKeySecret\":\"mockAccessKeySecret\",\"regionId\":\"cn-hangzhou\"}";
@BeforeEach
public void init() {
aliyunServerlessSparkDataSourceProcessor = new AliyunServerlessSparkDataSourceProcessor();
}
@Test
void testCheckDatasourceParam() {
AliyunServerlessSparkDataSourceParamDTO aliyunServerlessSparkDataSourceParamDTO = new AliyunServerlessSparkDataSourceParamDTO();
aliyunServerlessSparkDataSourceParamDTO.setRegionId("cn-hangzhou");
Assertions.assertThrows(IllegalArgumentException.class,
() -> aliyunServerlessSparkDataSourceProcessor.checkDatasourceParam(aliyunServerlessSparkDataSourceParamDTO));
aliyunServerlessSparkDataSourceParamDTO.setAccessKeyId("mockAccessKeyId");
aliyunServerlessSparkDataSourceParamDTO.setAccessKeySecret("mockAccessKeySecret");
Assertions
.assertDoesNotThrow(() -> aliyunServerlessSparkDataSourceProcessor.checkDatasourceParam(aliyunServerlessSparkDataSourceParamDTO));
}
@Test
void testGetDatasourceUniqueId() {
AliyunServerlessSparkConnectionParam aliyunServerlessSparkConnectionParam = new AliyunServerlessSparkConnectionParam();
aliyunServerlessSparkConnectionParam.setRegionId("cn-hangzhou");
aliyunServerlessSparkConnectionParam.setAccessKeyId("mockAccessKeyId");
aliyunServerlessSparkConnectionParam.setAccessKeySecret("mockAccessKeySecret");
Assertions.assertEquals("aliyun_serverless_spark@cn-hangzhou@mockAccessKeyId@mockAccessKeySecret",
aliyunServerlessSparkDataSourceProcessor.getDatasourceUniqueId(aliyunServerlessSparkConnectionParam, DbType.ALIYUN_SERVERLESS_SPARK));
}
@Test
void testCreateDatasourceParamDTO() {
AliyunServerlessSparkDataSourceParamDTO aliyunServerlessSparkDataSourceParamDTO =
(AliyunServerlessSparkDataSourceParamDTO) aliyunServerlessSparkDataSourceProcessor.createDatasourceParamDTO(connectJson);
Assertions.assertEquals("cn-hangzhou", aliyunServerlessSparkDataSourceParamDTO.getRegionId());
Assertions.assertEquals("mockAccessKeyId", aliyunServerlessSparkDataSourceParamDTO.getAccessKeyId());
Assertions.assertEquals("mockAccessKeySecret", aliyunServerlessSparkDataSourceParamDTO.getAccessKeySecret());
}
@Test
void testCreateConnectionParams() {
AliyunServerlessSparkDataSourceParamDTO aliyunServerlessSparkDataSourceParamDTO =
(AliyunServerlessSparkDataSourceParamDTO) aliyunServerlessSparkDataSourceProcessor.createDatasourceParamDTO(connectJson);
AliyunServerlessSparkConnectionParam aliyunServerlessSparkConnectionParam =
aliyunServerlessSparkDataSourceProcessor.createConnectionParams(aliyunServerlessSparkDataSourceParamDTO);
Assertions.assertEquals("cn-hangzhou", aliyunServerlessSparkConnectionParam.getRegionId());
Assertions.assertEquals("mockAccessKeyId", aliyunServerlessSparkConnectionParam.getAccessKeyId());
Assertions.assertEquals("mockAccessKeySecret", aliyunServerlessSparkConnectionParam.getAccessKeySecret());
}
@Test
void testTestConnection() {
AliyunServerlessSparkDataSourceParamDTO aliyunServerlessSparkDataSourceParamDTO =
(AliyunServerlessSparkDataSourceParamDTO) aliyunServerlessSparkDataSourceProcessor.createDatasourceParamDTO(connectJson);
AliyunServerlessSparkConnectionParam connectionParam =
aliyunServerlessSparkDataSourceProcessor.createConnectionParams(aliyunServerlessSparkDataSourceParamDTO);
Assertions.assertTrue(aliyunServerlessSparkDataSourceProcessor.checkDataSourceConnectivity(connectionParam));
try (
MockedConstruction<AliyunServerlessSparkClientWrapper> AliyunServerlessSparkClientWrapper =
Mockito.mockConstruction(AliyunServerlessSparkClientWrapper.class, (mock, context) -> {
Mockito.when(
mock.checkConnect(connectionParam.getAccessKeyId(), connectionParam.getAccessKeySecret(), connectionParam.getRegionId()))
.thenReturn(true);
})) {
Assertions.assertTrue(aliyunServerlessSparkDataSourceProcessor.checkDataSourceConnectivity(connectionParam));
}
}
}

View File

@ -1,3 +1,19 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.aliyunserverlessspark;
import lombok.Data;
@ -12,10 +28,6 @@ import java.util.List;
@Data
@Slf4j
public class AliyunServerlessSparkParameters extends AbstractParameters {
// connection configurations
// private String regionId;
// private String accessKeyId;
// private String accessKeySecret;
// spark job configurations
private String workspaceId;
private String resourceQueueId;

View File

@ -66,8 +66,6 @@ public class AliyunServerlessSparkTask extends AbstractRemoteTask {
private String jobRunId;
private RunState previousState;
private RunState currentState;
private String accessKeyId;
@ -107,7 +105,6 @@ public class AliyunServerlessSparkTask extends AbstractRemoteTask {
throw new AliyunServerlessSparkTaskException("Failed to build Aliyun-Serverless-Spark client!");
}
previousState = RunState.Submitted;
currentState = RunState.Submitted;
}

View File

@ -1,3 +1,19 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.aliyunserverlessspark;
import org.apache.dolphinscheduler.common.utils.JSONUtils;

View File

@ -1,3 +1,19 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.aliyunserverlessspark;
import org.apache.dolphinscheduler.plugin.task.api.TaskChannel;

View File

@ -1,3 +1,19 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.aliyunserverlessspark;
import java.util.Objects;

View File

@ -1,64 +1,51 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.apache.dolphinscheduler.plugin.task.aliyunserverlessspark;
import org.apache.dolphinscheduler.plugin.task.api.TaskException;
import org.apache.dolphinscheduler.plugin.task.api.TaskExecutionContext;
import java.util.Collections;
import org.junit.jupiter.api.BeforeEach;
import org.mockito.InjectMocks;
import org.mockito.Mock;
import com.aliyun.emr_serverless_spark20230808.Client;
import com.aliyun.emr_serverless_spark20230808.models.StartJobRunResponse;
import com.aliyun.emr_serverless_spark20230808.models.Tag;
import com.aliyun.tea.TeaException;
public class AliyunServerlessSparkTaskTest {
public static void main(String[] args) throws Exception {
com.aliyun.emr_serverless_spark20230808.Client client = AliyunServerlessSparkTaskTest.createClient();
com.aliyun.emr_serverless_spark20230808.models.StartJobRunRequest startJobRunRequest = new com.aliyun.emr_serverless_spark20230808.models.StartJobRunRequest();
startJobRunRequest.setRegionId("cn-hangzhou");
startJobRunRequest.setResourceQueueId("root_queue");
startJobRunRequest.setCodeType("JAR");
startJobRunRequest.setName("ds-test");
startJobRunRequest.setReleaseVersion("esr-2.1-native (Spark 3.3.1, Scala 2.12, Native Runtime)");
Tag tag = new Tag();
tag.setKey("environment");
tag.setValue("production");
startJobRunRequest.setTags(Collections.singletonList(tag));
com.aliyun.emr_serverless_spark20230808.models.JobDriver.JobDriverSparkSubmit jobDriverSparkSubmit = new com.aliyun.emr_serverless_spark20230808.models.JobDriver.JobDriverSparkSubmit()
.setEntryPoint("oss://datadev-oss-hdfs-test/spark-resource/examples/jars/spark-examples_2.12-3.3.1.jar")
.setEntryPointArguments(java.util.Arrays.asList(
"1"
))
.setSparkSubmitParameters("--class org.apache.spark.examples.SparkPi --conf spark.executor.cores=4 --conf spark.executor.memory=20g --conf spark.driver.cores=4 --conf spark.driver.memory=8g --conf spark.executor.instances=1");
com.aliyun.emr_serverless_spark20230808.models.JobDriver jobDriver = new com.aliyun.emr_serverless_spark20230808.models.JobDriver()
.setSparkSubmit(jobDriverSparkSubmit);
startJobRunRequest.setJobDriver(jobDriver);
com.aliyun.teautil.models.RuntimeOptions runtime = new com.aliyun.teautil.models.RuntimeOptions();
java.util.Map<String, String> headers = new java.util.HashMap<>();
try {
// 复制代码运行请自行打印 API 的返回值
StartJobRunResponse startJobRunResponse = client.startJobRunWithOptions("w-f7b841e8c73211be", startJobRunRequest, headers, runtime);
System.out.println(startJobRunResponse.getBody().getRequestId());
System.out.println(startJobRunResponse.getBody().getJobRunId());
} catch (TeaException error) {
// 此处仅做打印展示请谨慎对待异常处理在工程项目中切勿直接忽略异常
// 错误 message
System.out.println(error.getMessage());
// 诊断地址
System.out.println(error.getData().get("Recommend"));
com.aliyun.teautil.Common.assertAsString(error.message);
} catch (Exception _error) {
TeaException error = new TeaException(_error.getMessage(), _error);
// 此处仅做打印展示请谨慎对待异常处理在工程项目中切勿直接忽略异常
// 错误 message
System.out.println(error.getMessage());
// 诊断地址
System.out.println(error.getData().get("Recommend"));
com.aliyun.teautil.Common.assertAsString(error.message);
}
@Mock
private TaskExecutionContext mockTaskExecutionContext;
@Mock
private Client mockAliyunServerlessSparkClient;
@InjectMocks
private AliyunServerlessSparkTask aliyunServerlessSparkTask;
@BeforeEach
public void before() {
}
public static Client createClient() throws Exception {
com.aliyun.teaopenapi.models.Config config = new com.aliyun.teaopenapi.models.Config()
.setAccessKeyId(System.getenv("ALIBABA_CLOUD_ACCESS_KEY_ID"))
.setAccessKeySecret(System.getenv("ALIBABA_CLOUD_ACCESS_KEY_SECRET"));
config.endpoint = "emr-serverless-spark.cn-hangzhou.aliyuncs.com";
return new com.aliyun.emr_serverless_spark20230808.Client(config);
public void testInit() {
}
}

View File

@ -23,60 +23,6 @@ export function useAliyunServerlessSpark(model: { [field: string]: any }): IJson
return [
// mandatory field
// {
// type: 'input',
// field: 'regionId',
// name: t('project.node.region_id'),
// props: {
// placeholder: t('project.node.region_id_tips')
// },
// validate: {
// trigger: ['input', 'blur'],
// required: true,
// validator(validate: any, value: string) {
// if (!value) {
// return new Error(t('project.node.region_id_tips'))
// }
// }
// }
// },
// {
// type: 'input',
// field: 'accessKeyId',
// name: t('project.node.access_key_id'),
// props: {
// placeholder: t('project.node.access_key_id_tips')
// },
// validate: {
// trigger: ['input', 'blur'],
// required: true,
// validator(validate: any, value: string) {
// if (!value) {
// return new Error(t('project.node.access_key_id_tips'))
// }
// }
// }
// },
// {
// type: 'input',
// field: 'accessKeySecret',
// name: t('project.node.access_key_secret'),
// props: {
// placeholder: t('project.node.access_key_secret_tips')
// },
// validate: {
// trigger: ['input', 'blur'],
// required: true,
// validator(validate: any, value: string) {
// if (!value) {
// return new Error(t('project.node.access_key_secret_tips'))
// }
// }
// }
// },
{
type: 'input',
field: 'workspaceId',