[feature-#14449] add secret in k8s Task (#14930)
* [feature-#14449] add secret * [feature-#14449] upgrade UT and rename parameter ```secret``` * [feature-#14449] fix UT * [feature-#14449] fix UT * [feature-#14449] improvement * [feature-#14449] format --------- Co-authored-by: fuchanghai <‘2875334588@qq.com’>
This commit is contained in:
fuchanghai
GitHub
parent
1e2fddf958
commit
f90eadd40f
|
|
@ -32,6 +32,7 @@ public class K8sTaskMainParameters {
|
|||
private String image;
|
||||
private String command;
|
||||
private String args;
|
||||
private String pullSecret;
|
||||
private String namespaceName;
|
||||
private String clusterName;
|
||||
private String imagePullPolicy;
|
||||
|
|
|
|||
|
|
@ -17,6 +17,7 @@
|
|||
|
||||
package org.apache.dolphinscheduler.plugin.task.api.k8s.impl;
|
||||
|
||||
import static java.util.Collections.singletonList;
|
||||
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.API_VERSION;
|
||||
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.CPU;
|
||||
import static org.apache.dolphinscheduler.plugin.task.api.TaskConstants.EXIT_CODE_FAILURE;
|
||||
|
|
@ -67,6 +68,7 @@ import org.slf4j.Logger;
|
|||
import io.fabric8.kubernetes.api.model.Affinity;
|
||||
import io.fabric8.kubernetes.api.model.AffinityBuilder;
|
||||
import io.fabric8.kubernetes.api.model.EnvVar;
|
||||
import io.fabric8.kubernetes.api.model.LocalObjectReference;
|
||||
import io.fabric8.kubernetes.api.model.NodeSelectorTerm;
|
||||
import io.fabric8.kubernetes.api.model.Quantity;
|
||||
import io.fabric8.kubernetes.api.model.ResourceRequirements;
|
||||
|
|
@ -91,10 +93,11 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor {
|
|||
super(logger, taskRequest);
|
||||
}
|
||||
|
||||
public Job buildK8sJob(K8sTaskMainParameters k8STaskMainParameters) {
|
||||
public void buildK8sJob(K8sTaskMainParameters k8STaskMainParameters) {
|
||||
String taskInstanceId = String.valueOf(taskRequest.getTaskInstanceId());
|
||||
String taskName = taskRequest.getTaskName().toLowerCase(Locale.ROOT);
|
||||
String image = k8STaskMainParameters.getImage();
|
||||
String pullSecret = k8STaskMainParameters.getPullSecret();
|
||||
String namespaceName = k8STaskMainParameters.getNamespaceName();
|
||||
String imagePullPolicy = k8STaskMainParameters.getImagePullPolicy();
|
||||
Map<String, String> otherParams = k8STaskMainParameters.getParamsMap();
|
||||
|
|
@ -155,7 +158,7 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor {
|
|||
.endRequiredDuringSchedulingIgnoredDuringExecution()
|
||||
.endNodeAffinity().build();
|
||||
|
||||
JobBuilder jobBuilder = new JobBuilder()
|
||||
job = new JobBuilder()
|
||||
.withApiVersion(API_VERSION)
|
||||
.withNewMetadata()
|
||||
.withName(k8sJobName)
|
||||
|
|
@ -178,14 +181,16 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor {
|
|||
.withResources(new ResourceRequirements(limitRes, reqRes))
|
||||
.withEnv(envVars)
|
||||
.endContainer()
|
||||
.withImagePullSecrets(
|
||||
StringUtils.isEmpty(pullSecret) ? null : singletonList(new LocalObjectReference(pullSecret)))
|
||||
.withRestartPolicy(RESTART_POLICY)
|
||||
.withAffinity(affinity)
|
||||
.endSpec()
|
||||
.endTemplate()
|
||||
.withBackoffLimit(retryNum)
|
||||
.endSpec();
|
||||
.endSpec()
|
||||
.build();
|
||||
|
||||
return jobBuilder.build();
|
||||
}
|
||||
|
||||
public void registerBatchJobWatcher(Job job, String taskInstanceId, TaskResponse taskResponse) {
|
||||
|
|
@ -322,7 +327,7 @@ public class K8sTaskExecutor extends AbstractK8sTaskExecutor {
|
|||
JSONUtils.parseObject(k8sParameterStr, K8sTaskMainParameters.class);
|
||||
try {
|
||||
log.info("[K8sJobExecutor-{}-{}] start to submit job", taskName, taskInstanceId);
|
||||
job = buildK8sJob(k8STaskMainParameters);
|
||||
buildK8sJob(k8STaskMainParameters);
|
||||
stopJobOnK8s(k8sParameterStr);
|
||||
String namespaceName = k8STaskMainParameters.getNamespaceName();
|
||||
k8sUtils.createJob(namespaceName, job);
|
||||
|
|
|
|||
|
|
@ -38,6 +38,7 @@ public class K8sTaskParameters extends AbstractParameters {
|
|||
private String namespace;
|
||||
private String command;
|
||||
private String args;
|
||||
private String pullSecret;
|
||||
private String imagePullPolicy;
|
||||
private double minCpuCores;
|
||||
private double minMemorySpace;
|
||||
|
|
|
|||
|
|
@ -81,7 +81,8 @@ public class K8sTaskExecutorTest {
|
|||
k8sTaskMainParameters.setCommand("[\"perl\" ,\"-Mbignum=bpi\", \"-wle\", \"print bpi(2000)\"]");
|
||||
k8sTaskMainParameters.setLabelMap(labelMap);
|
||||
k8sTaskMainParameters.setNodeSelectorRequirements(Arrays.asList(requirement));
|
||||
job = k8sTaskExecutor.buildK8sJob(k8sTaskMainParameters);
|
||||
k8sTaskExecutor.buildK8sJob(k8sTaskMainParameters);
|
||||
job = k8sTaskExecutor.getJob();
|
||||
}
|
||||
@Test
|
||||
public void testGetK8sJobStatusNormal() {
|
||||
|
|
|
|||
|
|
@ -87,6 +87,7 @@ public class K8sTask extends AbstractK8sTask {
|
|||
String namespaceName = namespace.get(NAMESPACE_NAME);
|
||||
String clusterName = namespace.get(CLUSTER);
|
||||
k8sTaskMainParameters.setImage(k8sTaskParameters.getImage());
|
||||
k8sTaskMainParameters.setPullSecret(k8sTaskParameters.getPullSecret());
|
||||
k8sTaskMainParameters.setNamespaceName(namespaceName);
|
||||
k8sTaskMainParameters.setClusterName(clusterName);
|
||||
k8sTaskMainParameters.setMinCpuCores(k8sTaskParameters.getMinCpuCores());
|
||||
|
|
|
|||
|
|
@ -46,6 +46,8 @@ public class K8sTaskTest {
|
|||
private final String image = "ds-dev";
|
||||
private final String imagePullPolicy = "IfNotPresent";
|
||||
|
||||
private final String pullSecret = "ds-secret";
|
||||
|
||||
private final String namespace = "{\"name\":\"default\",\"cluster\":\"lab\"}";
|
||||
|
||||
private final double minCpuCores = 2;
|
||||
|
|
@ -74,6 +76,7 @@ public class K8sTaskTest {
|
|||
k8sTaskParameters.setArgs(args);
|
||||
k8sTaskParameters.setCustomizedLabels(labels);
|
||||
k8sTaskParameters.setNodeSelectors(nodeSelectorExpressions);
|
||||
k8sTaskParameters.setPullSecret(pullSecret);
|
||||
TaskExecutionContext taskRequest = new TaskExecutionContext();
|
||||
taskRequest.setTaskInstanceId(taskInstanceId);
|
||||
taskRequest.setTaskName(taskName);
|
||||
|
|
@ -99,7 +102,7 @@ public class K8sTaskTest {
|
|||
@Test
|
||||
public void testBuildCommandNormal() {
|
||||
String expectedStr =
|
||||
"{\"image\":\"ds-dev\",\"command\":\"[\\\"/bin/bash\\\", \\\"-c\\\"]\",\"args\":\"[\\\"echo hello world\\\"]\",\"namespaceName\":\"default\",\"clusterName\":\"lab\",\"imagePullPolicy\":\"IfNotPresent\",\"minCpuCores\":2.0,\"minMemorySpace\":10.0,\"paramsMap\":{\"day\":\"20220507\"},\"labelMap\":{\"test\":\"1234\"},\"nodeSelectorRequirements\":[{\"key\":\"node-label\",\"operator\":\"In\",\"values\":[\"1234\",\"12345\"]}]}";
|
||||
"{\"image\":\"ds-dev\",\"command\":\"[\\\"/bin/bash\\\", \\\"-c\\\"]\",\"args\":\"[\\\"echo hello world\\\"]\",\"pullSecret\":\"ds-secret\",\"namespaceName\":\"default\",\"clusterName\":\"lab\",\"imagePullPolicy\":\"IfNotPresent\",\"minCpuCores\":2.0,\"minMemorySpace\":10.0,\"paramsMap\":{\"day\":\"20220507\"},\"labelMap\":{\"test\":\"1234\"},\"nodeSelectorRequirements\":[{\"key\":\"node-label\",\"operator\":\"In\",\"values\":[\"1234\",\"12345\"]}]}";
|
||||
String commandStr = k8sTask.buildCommand();
|
||||
Assertions.assertEquals(expectedStr, commandStr);
|
||||
}
|
||||
|
|
@ -107,7 +110,7 @@ public class K8sTaskTest {
|
|||
@Test
|
||||
public void testGetParametersNormal() {
|
||||
String expectedStr =
|
||||
"K8sTaskParameters(image=ds-dev, namespace={\"name\":\"default\",\"cluster\":\"lab\"}, command=[\"/bin/bash\", \"-c\"], args=[\"echo hello world\"], imagePullPolicy=IfNotPresent, minCpuCores=2.0, minMemorySpace=10.0, customizedLabels=[Label(label=test, value=1234)], nodeSelectors=[NodeSelectorExpression(key=node-label, operator=In, values=1234,12345)])";
|
||||
"K8sTaskParameters(image=ds-dev, namespace={\"name\":\"default\",\"cluster\":\"lab\"}, command=[\"/bin/bash\", \"-c\"], args=[\"echo hello world\"], pullSecret=ds-secret, imagePullPolicy=IfNotPresent, minCpuCores=2.0, minMemorySpace=10.0, customizedLabels=[Label(label=test, value=1234)], nodeSelectors=[NodeSelectorExpression(key=node-label, operator=In, values=1234,12345)])";
|
||||
String result = k8sTask.getParameters().toString();
|
||||
Assertions.assertEquals(expectedStr, result);
|
||||
}
|
||||
|
|
|
|||
|
|
@ -397,6 +397,8 @@ export default {
|
|||
image_tips: 'Please enter image',
|
||||
image_pull_policy: 'Image pull policy',
|
||||
image_pull_policy_tips: 'Please select a image pull policy (required)',
|
||||
pull_secret: 'Pull secret',
|
||||
pull_secret_tips: 'Please enter pull secret',
|
||||
command: 'Command',
|
||||
command_tips:
|
||||
'Please enter the container execution command, for example: ["printenv"]',
|
||||
|
|
|
|||
|
|
@ -394,6 +394,8 @@ export default {
|
|||
image_tips: '请输入镜像',
|
||||
image_pull_policy: '镜像拉取策略',
|
||||
image_pull_policy_tips: '请选择镜像拉取策略（必选）',
|
||||
pull_secret: '仓库密钥',
|
||||
pull_secret_tips: '请输入仓库密钥',
|
||||
command: '容器执行命令',
|
||||
command_tips: '请输入容器执行命令，例如：["printenv"]',
|
||||
args: '执行命令参数',
|
||||
|
|
|
|||
|
|
@ -79,6 +79,14 @@ export function useK8s(model: { [field: string]: any }): IJsonItem[] {
|
|||
},
|
||||
value: 'IfNotPresent'
|
||||
},
|
||||
{
|
||||
type: 'input',
|
||||
field: 'pullSecret',
|
||||
name: t('project.node.pull_secret'),
|
||||
props: {
|
||||
placeholder: t('project.node.pull_secret_tips')
|
||||
}
|
||||
},
|
||||
{
|
||||
type: 'input',
|
||||
field: 'command',
|
||||
|
|
|
|||
|
|
@ -359,6 +359,7 @@ export function formatParams(data: INodeData): {
|
|||
taskParams.args = data.args
|
||||
taskParams.customizedLabels = data.customizedLabels
|
||||
taskParams.nodeSelectors = data.nodeSelectors
|
||||
taskParams.pullSecret = data.pullSecret
|
||||
}
|
||||
|
||||
if (data.taskType === 'JUPYTER') {
|
||||
|
|
|
|||
|
|
@ -378,6 +378,7 @@ interface ITaskParams {
|
|||
minMemorySpace?: string
|
||||
image?: string
|
||||
imagePullPolicy?: string
|
||||
pullSecret?: string
|
||||
command?: string
|
||||
args?: string
|
||||
customizedLabels?: ILabel[]
|
||||
|
|
|
|||
Loading…
Reference in New Issue