1123 lines
34 KiB
Go
1123 lines
34 KiB
Go
/*
|
|
* SPDX-FileCopyrightText: Copyright (c) 2022 Atalaya Tech. Inc
|
|
* SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
|
|
* SPDX-License-Identifier: Apache-2.0
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
* Modifications Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES
|
|
*/
|
|
|
|
package controller
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"testing"
|
|
|
|
dynamoCommon "github.com/ai-dynamo/dynamo/deploy/cloud/operator/api/dynamo/common"
|
|
"github.com/ai-dynamo/dynamo/deploy/cloud/operator/api/v1alpha1"
|
|
commonconsts "github.com/ai-dynamo/dynamo/deploy/cloud/operator/internal/consts"
|
|
"github.com/ai-dynamo/dynamo/deploy/cloud/operator/internal/controller_common"
|
|
"github.com/ai-dynamo/dynamo/deploy/cloud/operator/internal/dynamo"
|
|
"github.com/google/go-cmp/cmp"
|
|
"github.com/onsi/gomega"
|
|
"github.com/onsi/gomega/format"
|
|
istioNetworking "istio.io/api/networking/v1beta1"
|
|
networkingv1beta1 "istio.io/client-go/pkg/apis/networking/v1beta1"
|
|
appsv1 "k8s.io/api/apps/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
networkingv1 "k8s.io/api/networking/v1"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/util/intstr"
|
|
"k8s.io/client-go/kubernetes/scheme"
|
|
"k8s.io/client-go/tools/record"
|
|
"k8s.io/utils/ptr"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1"
|
|
volcanov1beta1 "volcano.sh/apis/pkg/apis/scheduling/v1beta1"
|
|
)
|
|
|
|
func TestIsDeploymentReady(t *testing.T) {
|
|
type args struct {
|
|
deployment *appsv1.Deployment
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
args args
|
|
want bool
|
|
}{
|
|
{
|
|
name: "deployment is nil",
|
|
args: args{
|
|
deployment: nil,
|
|
},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "not ready",
|
|
args: args{
|
|
deployment: &appsv1.Deployment{
|
|
Spec: appsv1.DeploymentSpec{},
|
|
Status: appsv1.DeploymentStatus{
|
|
Conditions: []appsv1.DeploymentCondition{
|
|
{
|
|
Type: appsv1.DeploymentAvailable,
|
|
Status: corev1.ConditionFalse,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "not ready (paused)",
|
|
args: args{
|
|
deployment: &appsv1.Deployment{
|
|
Spec: appsv1.DeploymentSpec{
|
|
Paused: true,
|
|
},
|
|
},
|
|
},
|
|
want: false,
|
|
},
|
|
{
|
|
name: "ready",
|
|
args: args{
|
|
deployment: &appsv1.Deployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Generation: 1,
|
|
},
|
|
Spec: appsv1.DeploymentSpec{
|
|
Replicas: &[]int32{1}[0],
|
|
},
|
|
Status: appsv1.DeploymentStatus{
|
|
ObservedGeneration: 1,
|
|
UpdatedReplicas: 1,
|
|
AvailableReplicas: 1,
|
|
Conditions: []appsv1.DeploymentCondition{
|
|
{
|
|
Type: appsv1.DeploymentAvailable,
|
|
Status: corev1.ConditionTrue,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
want: true,
|
|
},
|
|
{
|
|
name: "ready (no desired replicas)",
|
|
args: args{
|
|
deployment: &appsv1.Deployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Generation: 1,
|
|
},
|
|
Spec: appsv1.DeploymentSpec{
|
|
Replicas: &[]int32{0}[0],
|
|
},
|
|
},
|
|
},
|
|
want: true,
|
|
},
|
|
{
|
|
name: "not ready (condition false)",
|
|
args: args{
|
|
deployment: &appsv1.Deployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Generation: 1,
|
|
},
|
|
Spec: appsv1.DeploymentSpec{
|
|
Replicas: &[]int32{1}[0],
|
|
},
|
|
Status: appsv1.DeploymentStatus{
|
|
ObservedGeneration: 1,
|
|
UpdatedReplicas: 1,
|
|
AvailableReplicas: 1,
|
|
Conditions: []appsv1.DeploymentCondition{
|
|
{
|
|
Type: appsv1.DeploymentAvailable,
|
|
Status: corev1.ConditionFalse,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
want: false,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
if got := IsDeploymentReady(tt.args.deployment); got != tt.want {
|
|
t.Errorf("IsDeploymentReady() = %v, want %v", got, tt.want)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
type mockEtcdStorage struct {
|
|
deleteKeysFunc func(ctx context.Context, prefix string) error
|
|
}
|
|
|
|
func (m *mockEtcdStorage) DeleteKeys(ctx context.Context, prefix string) error {
|
|
return m.deleteKeysFunc(ctx, prefix)
|
|
}
|
|
|
|
func TestDynamoComponentDeploymentReconciler_FinalizeResource(t *testing.T) {
|
|
type fields struct {
|
|
EtcdStorage etcdStorage
|
|
}
|
|
type args struct {
|
|
ctx context.Context
|
|
dynamoComponentDeployment *v1alpha1.DynamoComponentDeployment
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
fields fields
|
|
args args
|
|
wantErr bool
|
|
}{
|
|
{
|
|
name: "delete etcd keys",
|
|
fields: fields{
|
|
EtcdStorage: &mockEtcdStorage{
|
|
deleteKeysFunc: func(ctx context.Context, prefix string) error {
|
|
if prefix == "/default/components/service1" {
|
|
return nil
|
|
}
|
|
return fmt.Errorf("invalid prefix: %s", prefix)
|
|
},
|
|
},
|
|
},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
dynamoComponentDeployment: &v1alpha1.DynamoComponentDeployment{
|
|
Spec: v1alpha1.DynamoComponentDeploymentSpec{
|
|
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
|
|
ServiceName: "service1",
|
|
DynamoNamespace: &[]string{"default"}[0],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "delete etcd keys (error)",
|
|
fields: fields{
|
|
EtcdStorage: &mockEtcdStorage{
|
|
deleteKeysFunc: func(ctx context.Context, prefix string) error {
|
|
return fmt.Errorf("invalid prefix: %s", prefix)
|
|
},
|
|
},
|
|
},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
dynamoComponentDeployment: &v1alpha1.DynamoComponentDeployment{
|
|
Spec: v1alpha1.DynamoComponentDeploymentSpec{
|
|
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
|
|
ServiceName: "service1",
|
|
DynamoNamespace: &[]string{"default"}[0],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
wantErr: true,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
r := &DynamoComponentDeploymentReconciler{
|
|
EtcdStorage: tt.fields.EtcdStorage,
|
|
}
|
|
if err := r.FinalizeResource(tt.args.ctx, tt.args.dynamoComponentDeployment); (err != nil) != tt.wantErr {
|
|
t.Errorf("DynamoComponentDeploymentReconciler.FinalizeResource() error = %v, wantErr %v", err, tt.wantErr)
|
|
}
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDynamoComponentDeploymentReconciler_generateIngress(t *testing.T) {
|
|
type fields struct {
|
|
}
|
|
type args struct {
|
|
ctx context.Context
|
|
opt generateResourceOption
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
fields fields
|
|
args args
|
|
want *networkingv1.Ingress
|
|
want1 bool
|
|
wantErr bool
|
|
}{
|
|
{
|
|
name: "generate ingress",
|
|
fields: fields{},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
opt: generateResourceOption{
|
|
dynamoComponentDeployment: &v1alpha1.DynamoComponentDeployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "service1",
|
|
Namespace: "default",
|
|
},
|
|
Spec: v1alpha1.DynamoComponentDeploymentSpec{
|
|
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
|
|
ServiceName: "service1",
|
|
DynamoNamespace: &[]string{"default"}[0],
|
|
Ingress: &v1alpha1.IngressSpec{
|
|
Enabled: true,
|
|
Host: "someservice",
|
|
IngressControllerClassName: &[]string{"nginx"}[0],
|
|
UseVirtualService: false,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
want: &networkingv1.Ingress{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "service1",
|
|
Namespace: "default",
|
|
},
|
|
Spec: networkingv1.IngressSpec{
|
|
IngressClassName: &[]string{"nginx"}[0],
|
|
Rules: []networkingv1.IngressRule{
|
|
{
|
|
Host: "someservice.local",
|
|
IngressRuleValue: networkingv1.IngressRuleValue{
|
|
HTTP: &networkingv1.HTTPIngressRuleValue{
|
|
Paths: []networkingv1.HTTPIngressPath{
|
|
{
|
|
Path: "/",
|
|
PathType: &[]networkingv1.PathType{networkingv1.PathTypePrefix}[0],
|
|
Backend: networkingv1.IngressBackend{
|
|
Service: &networkingv1.IngressServiceBackend{
|
|
Name: "service1",
|
|
Port: networkingv1.ServiceBackendPort{Number: commonconsts.DynamoServicePort},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
want1: false,
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "generate ingress, disabled",
|
|
fields: fields{},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
opt: generateResourceOption{
|
|
dynamoComponentDeployment: &v1alpha1.DynamoComponentDeployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "service1",
|
|
Namespace: "default",
|
|
},
|
|
Spec: v1alpha1.DynamoComponentDeploymentSpec{
|
|
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
|
|
ServiceName: "service1",
|
|
DynamoNamespace: &[]string{"default"}[0],
|
|
Ingress: &v1alpha1.IngressSpec{
|
|
Enabled: false,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
want: &networkingv1.Ingress{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "service1",
|
|
Namespace: "default",
|
|
},
|
|
},
|
|
want1: true,
|
|
wantErr: false,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
g := gomega.NewGomegaWithT(t)
|
|
r := &DynamoComponentDeploymentReconciler{}
|
|
got, got1, err := r.generateIngress(tt.args.ctx, tt.args.opt)
|
|
if (err != nil) != tt.wantErr {
|
|
t.Errorf("DynamoComponentDeploymentReconciler.generateIngress() error = %v, wantErr %v", err, tt.wantErr)
|
|
return
|
|
}
|
|
g.Expect(got).To(gomega.Equal(tt.want))
|
|
g.Expect(got1).To(gomega.Equal(tt.want1))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDynamoComponentDeploymentReconciler_generateVirtualService(t *testing.T) {
|
|
type fields struct {
|
|
}
|
|
type args struct {
|
|
ctx context.Context
|
|
opt generateResourceOption
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
fields fields
|
|
args args
|
|
want *networkingv1beta1.VirtualService
|
|
want1 bool
|
|
wantErr bool
|
|
}{
|
|
{
|
|
name: "generate virtual service, disabled in operator config",
|
|
fields: fields{},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
opt: generateResourceOption{
|
|
dynamoComponentDeployment: &v1alpha1.DynamoComponentDeployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "service1",
|
|
Namespace: "default",
|
|
},
|
|
Spec: v1alpha1.DynamoComponentDeploymentSpec{
|
|
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
|
|
ServiceName: "service1",
|
|
DynamoNamespace: &[]string{"default"}[0],
|
|
Ingress: &v1alpha1.IngressSpec{
|
|
Enabled: true,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
want: &networkingv1beta1.VirtualService{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "service1",
|
|
Namespace: "default",
|
|
},
|
|
},
|
|
want1: true,
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "generate virtual service, enabled in operator config",
|
|
fields: fields{},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
opt: generateResourceOption{
|
|
dynamoComponentDeployment: &v1alpha1.DynamoComponentDeployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "service1",
|
|
Namespace: "default",
|
|
},
|
|
Spec: v1alpha1.DynamoComponentDeploymentSpec{
|
|
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
|
|
ServiceName: "service1",
|
|
DynamoNamespace: &[]string{"default"}[0],
|
|
Ingress: &v1alpha1.IngressSpec{
|
|
Enabled: true,
|
|
Host: "someservice",
|
|
UseVirtualService: true,
|
|
VirtualServiceGateway: &[]string{"istio-system/ingress-alb"}[0],
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
want: &networkingv1beta1.VirtualService{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "service1",
|
|
Namespace: "default",
|
|
},
|
|
Spec: istioNetworking.VirtualService{
|
|
Hosts: []string{"someservice.local"},
|
|
Gateways: []string{"istio-system/ingress-alb"},
|
|
Http: []*istioNetworking.HTTPRoute{
|
|
{
|
|
Match: []*istioNetworking.HTTPMatchRequest{
|
|
{
|
|
Uri: &istioNetworking.StringMatch{
|
|
MatchType: &istioNetworking.StringMatch_Prefix{Prefix: "/"},
|
|
},
|
|
},
|
|
},
|
|
Route: []*istioNetworking.HTTPRouteDestination{
|
|
{
|
|
Destination: &istioNetworking.Destination{
|
|
Host: "service1",
|
|
Port: &istioNetworking.PortSelector{
|
|
Number: commonconsts.DynamoServicePort,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
want1: false,
|
|
wantErr: false,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
g := gomega.NewGomegaWithT(t)
|
|
r := &DynamoComponentDeploymentReconciler{}
|
|
got, got1, err := r.generateVirtualService(tt.args.ctx, tt.args.opt)
|
|
if (err != nil) != tt.wantErr {
|
|
t.Errorf("DynamoComponentDeploymentReconciler.generateVirtualService() error = %v, wantErr %v", err, tt.wantErr)
|
|
return
|
|
}
|
|
g.Expect(got).To(gomega.Equal(tt.want))
|
|
g.Expect(got1).To(gomega.Equal(tt.want1))
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestDynamoComponentDeploymentReconciler_generateVolcanoPodGroup(t *testing.T) {
|
|
type fields struct {
|
|
Client client.Client
|
|
Recorder record.EventRecorder
|
|
Config controller_common.Config
|
|
EtcdStorage etcdStorage
|
|
}
|
|
type args struct {
|
|
ctx context.Context
|
|
opt generateResourceOption
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
fields fields
|
|
args args
|
|
want *volcanov1beta1.PodGroup
|
|
want1 bool
|
|
wantErr bool
|
|
}{
|
|
{
|
|
name: "generate volcano pod group",
|
|
args: args{
|
|
ctx: context.Background(),
|
|
opt: generateResourceOption{
|
|
dynamoComponentDeployment: &v1alpha1.DynamoComponentDeployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "service1",
|
|
Namespace: "default",
|
|
},
|
|
Spec: v1alpha1.DynamoComponentDeploymentSpec{
|
|
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
|
|
Multinode: &v1alpha1.MultinodeSpec{
|
|
NodeCount: 2,
|
|
},
|
|
ServiceName: "service1",
|
|
DynamoNamespace: &[]string{"default"}[0],
|
|
},
|
|
},
|
|
},
|
|
instanceID: ptr.To(5),
|
|
},
|
|
},
|
|
want: &volcanov1beta1.PodGroup{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "service1-5",
|
|
Namespace: "default",
|
|
Labels: map[string]string{
|
|
"instance-id": "5",
|
|
},
|
|
},
|
|
Spec: volcanov1beta1.PodGroupSpec{
|
|
MinMember: 2,
|
|
},
|
|
},
|
|
want1: false,
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "nil instanceID",
|
|
args: args{
|
|
ctx: context.Background(),
|
|
opt: generateResourceOption{
|
|
dynamoComponentDeployment: &v1alpha1.DynamoComponentDeployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "service-nil-instanceid",
|
|
Namespace: "default",
|
|
},
|
|
Spec: v1alpha1.DynamoComponentDeploymentSpec{
|
|
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
|
|
ServiceName: "service-nil-instanceid",
|
|
DynamoNamespace: &[]string{"default"}[0],
|
|
Multinode: &v1alpha1.MultinodeSpec{
|
|
NodeCount: 2,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
instanceID: nil,
|
|
},
|
|
},
|
|
want: nil,
|
|
want1: false,
|
|
wantErr: true,
|
|
},
|
|
{
|
|
name: "negative instanceID",
|
|
args: args{
|
|
ctx: context.Background(),
|
|
opt: generateResourceOption{
|
|
dynamoComponentDeployment: &v1alpha1.DynamoComponentDeployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "service-negative-instanceid",
|
|
Namespace: "default",
|
|
},
|
|
Spec: v1alpha1.DynamoComponentDeploymentSpec{
|
|
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
|
|
ServiceName: "service-negative-instanceid",
|
|
DynamoNamespace: &[]string{"default"}[0],
|
|
Multinode: &v1alpha1.MultinodeSpec{
|
|
NodeCount: 2,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
instanceID: ptr.To(-1),
|
|
},
|
|
},
|
|
want: nil,
|
|
want1: false,
|
|
wantErr: true,
|
|
},
|
|
}
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
g := gomega.NewGomegaWithT(t)
|
|
r := &DynamoComponentDeploymentReconciler{
|
|
Client: tt.fields.Client,
|
|
Recorder: tt.fields.Recorder,
|
|
Config: tt.fields.Config,
|
|
EtcdStorage: tt.fields.EtcdStorage,
|
|
}
|
|
got, got1, err := r.generateVolcanoPodGroup(tt.args.ctx, tt.args.opt)
|
|
if (err != nil) != tt.wantErr {
|
|
t.Errorf("DynamoComponentDeploymentReconciler.generateVolcanoPodGroup() error = %v, wantErr %v", err, tt.wantErr)
|
|
return
|
|
}
|
|
if diff := cmp.Diff(tt.want, got); diff != "" {
|
|
t.Errorf("Mismatch (-expected +actual):\n%s", diff)
|
|
}
|
|
g.Expect(got).To(gomega.Equal(tt.want))
|
|
g.Expect(got1).To(gomega.Equal(tt.want1))
|
|
})
|
|
}
|
|
}
|
|
|
|
type mockDockerSecretRetriever struct {
|
|
GetSecretsFunc func(namespace, imageName string) ([]string, error)
|
|
}
|
|
|
|
func (m *mockDockerSecretRetriever) GetSecrets(namespace, imageName string) ([]string, error) {
|
|
return m.GetSecretsFunc(namespace, imageName)
|
|
}
|
|
|
|
func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.T) {
|
|
var limit = ptr.To(resource.MustParse("250Mi"))
|
|
limit.SetMilli(ptr.To(resource.MustParse("1Gi")).MilliValue() / 2)
|
|
type fields struct {
|
|
Client client.Client
|
|
Recorder record.EventRecorder
|
|
Config controller_common.Config
|
|
EtcdStorage etcdStorage
|
|
DockerSecretRetriever *mockDockerSecretRetriever
|
|
}
|
|
type args struct {
|
|
ctx context.Context
|
|
opt generateResourceOption
|
|
// Add expected ServiceAccountName if you want to verify it's picked up
|
|
// For now, we'll ensure a default one exists for the happy path
|
|
mockServiceAccounts []client.Object
|
|
}
|
|
tests := []struct {
|
|
name string
|
|
fields fields
|
|
args args
|
|
want *leaderworkersetv1.LeaderWorkerSet
|
|
want1 bool // toDelete
|
|
wantErr bool
|
|
}{
|
|
{
|
|
name: "generateLeaderWorkerSet - nominal case",
|
|
fields: fields{
|
|
Recorder: record.NewFakeRecorder(100),
|
|
Config: controller_common.Config{}, // Provide default or test-specific config
|
|
DockerSecretRetriever: &mockDockerSecretRetriever{
|
|
GetSecretsFunc: func(namespace, imageName string) ([]string, error) {
|
|
return []string{}, nil
|
|
},
|
|
},
|
|
},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
opt: generateResourceOption{
|
|
dynamoComponentDeployment: &v1alpha1.DynamoComponentDeployment{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-lws-deploy",
|
|
Namespace: "default",
|
|
OwnerReferences: []metav1.OwnerReference{
|
|
{
|
|
Kind: "DynamoGraphDeployment",
|
|
Name: "test-lws-deploy",
|
|
},
|
|
},
|
|
},
|
|
Spec: v1alpha1.DynamoComponentDeploymentSpec{
|
|
BackendFramework: string(dynamo.BackendFrameworkVLLM),
|
|
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
|
|
Envs: []corev1.EnvVar{
|
|
{
|
|
Name: "TEST_ENV_FROM_DYNAMO_COMPONENT_DEPLOYMENT_SPEC",
|
|
Value: "test_value_from_dynamo_component_deployment_spec",
|
|
},
|
|
},
|
|
ComponentType: string(commonconsts.ComponentTypeWorker),
|
|
SubComponentType: "test-sub-component",
|
|
ServiceName: "test-lws-deploy-service",
|
|
DynamoNamespace: &[]string{"default"}[0],
|
|
Multinode: &v1alpha1.MultinodeSpec{
|
|
NodeCount: 2,
|
|
},
|
|
Resources: &dynamoCommon.Resources{
|
|
Requests: &dynamoCommon.ResourceItem{
|
|
CPU: "300m",
|
|
Memory: "500Mi",
|
|
},
|
|
Limits: &dynamoCommon.ResourceItem{
|
|
GPU: "1",
|
|
Memory: "20Gi",
|
|
CPU: "10",
|
|
},
|
|
},
|
|
ExtraPodMetadata: &dynamoCommon.ExtraPodMetadata{
|
|
Annotations: map[string]string{
|
|
"nvidia.com/annotation1": "annotation1",
|
|
},
|
|
Labels: map[string]string{
|
|
"nvidia.com/label1": "label1",
|
|
},
|
|
},
|
|
ExtraPodSpec: &dynamoCommon.ExtraPodSpec{
|
|
PodSpec: &corev1.PodSpec{
|
|
TerminationGracePeriodSeconds: ptr.To(int64(10)),
|
|
Containers: []corev1.Container{
|
|
{
|
|
Image: "another-image:latest",
|
|
},
|
|
},
|
|
},
|
|
MainContainer: &corev1.Container{
|
|
Image: "test-image:latest",
|
|
Command: []string{
|
|
"sh",
|
|
"-c",
|
|
},
|
|
Args: []string{
|
|
"some dynamo command",
|
|
},
|
|
Env: []corev1.EnvVar{
|
|
{
|
|
Name: "TEST_ENV_FROM_EXTRA_POD_SPEC",
|
|
Value: "test_value_from_extra_pod_spec",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
instanceID: ptr.To(0),
|
|
},
|
|
// Define a mock ServiceAccount that should be found by r.List
|
|
mockServiceAccounts: []client.Object{
|
|
&corev1.ServiceAccount{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "default-test-sa", // Name it will be resolved to
|
|
Namespace: "default", // Must match dynamoComponentDeployment.Namespace
|
|
Labels: map[string]string{
|
|
commonconsts.KubeLabelDynamoComponentPod: commonconsts.KubeLabelValueTrue,
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
want: &leaderworkersetv1.LeaderWorkerSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "test-lws-deploy-0",
|
|
Namespace: "default",
|
|
Labels: map[string]string{
|
|
"instance-id": "0",
|
|
},
|
|
},
|
|
Spec: leaderworkersetv1.LeaderWorkerSetSpec{
|
|
Replicas: ptr.To(int32(1)),
|
|
StartupPolicy: leaderworkersetv1.LeaderCreatedStartupPolicy,
|
|
LeaderWorkerTemplate: leaderworkersetv1.LeaderWorkerTemplate{
|
|
Size: ptr.To(int32(2)),
|
|
LeaderTemplate: &corev1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{
|
|
"instance-id": "0",
|
|
commonconsts.KubeLabelMetricsEnabled: commonconsts.KubeLabelValueTrue,
|
|
"role": "leader",
|
|
"nvidia.com/label1": "label1",
|
|
commonconsts.KubeLabelDynamoComponentType: commonconsts.ComponentTypeWorker,
|
|
commonconsts.KubeLabelDynamoSubComponentType: "test-sub-component",
|
|
commonconsts.KubeLabelDynamoGraphDeploymentName: "",
|
|
},
|
|
Annotations: map[string]string{
|
|
"scheduling.k8s.io/group-name": "test-lws-deploy-0",
|
|
"nvidia.com/annotation1": "annotation1",
|
|
},
|
|
},
|
|
Spec: corev1.PodSpec{
|
|
SchedulerName: "volcano",
|
|
TerminationGracePeriodSeconds: ptr.To(int64(10)),
|
|
Volumes: []corev1.Volume{
|
|
{
|
|
Name: "shared-memory",
|
|
VolumeSource: corev1.VolumeSource{
|
|
EmptyDir: &corev1.EmptyDirVolumeSource{
|
|
Medium: corev1.StorageMediumMemory,
|
|
SizeLimit: func() *resource.Quantity { q := resource.MustParse(commonconsts.DefaultSharedMemorySize); return &q }(),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
RestartPolicy: corev1.RestartPolicyAlways,
|
|
Containers: []corev1.Container{
|
|
{
|
|
Image: "another-image:latest",
|
|
},
|
|
{
|
|
Name: commonconsts.MainContainerName,
|
|
Image: "test-image:latest",
|
|
Command: []string{"sh", "-c"},
|
|
Args: []string{"ray start --head --port=6379 && some dynamo command"},
|
|
Env: []corev1.EnvVar{
|
|
{Name: "DYN_NAMESPACE", Value: "default"},
|
|
{Name: "DYN_PARENT_DGD_K8S_NAME", Value: "test-lws-deploy"},
|
|
{Name: "DYN_PARENT_DGD_K8S_NAMESPACE", Value: "default"},
|
|
{Name: "DYN_SYSTEM_ENABLED", Value: "true"},
|
|
{Name: "DYN_SYSTEM_PORT", Value: "9090"},
|
|
{Name: "DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS", Value: "[\"generate\"]"},
|
|
{Name: "TEST_ENV_FROM_DYNAMO_COMPONENT_DEPLOYMENT_SPEC", Value: "test_value_from_dynamo_component_deployment_spec"},
|
|
{Name: "TEST_ENV_FROM_EXTRA_POD_SPEC", Value: "test_value_from_extra_pod_spec"},
|
|
},
|
|
Ports: []corev1.ContainerPort{
|
|
{
|
|
Protocol: corev1.ProtocolTCP, Name: commonconsts.DynamoSystemPortName, ContainerPort: commonconsts.DynamoSystemPort,
|
|
},
|
|
},
|
|
VolumeMounts: []corev1.VolumeMount{
|
|
{
|
|
Name: "shared-memory",
|
|
MountPath: commonconsts.DefaultSharedMemoryMountPath,
|
|
},
|
|
},
|
|
Resources: corev1.ResourceRequirements{
|
|
Requests: corev1.ResourceList{
|
|
corev1.ResourceCPU: resource.MustParse("300m"),
|
|
corev1.ResourceMemory: resource.MustParse("500Mi"),
|
|
},
|
|
Limits: corev1.ResourceList{
|
|
corev1.ResourceMemory: resource.MustParse("20Gi"),
|
|
corev1.ResourceCPU: resource.MustParse("10"),
|
|
"nvidia.com/gpu": resource.MustParse("1"),
|
|
},
|
|
},
|
|
LivenessProbe: &corev1.Probe{
|
|
ProbeHandler: corev1.ProbeHandler{
|
|
HTTPGet: &corev1.HTTPGetAction{
|
|
Path: "/live",
|
|
Port: intstr.FromString(commonconsts.DynamoSystemPortName),
|
|
},
|
|
},
|
|
TimeoutSeconds: 30,
|
|
PeriodSeconds: 5,
|
|
SuccessThreshold: 0,
|
|
FailureThreshold: 1,
|
|
},
|
|
ReadinessProbe: &corev1.Probe{
|
|
ProbeHandler: corev1.ProbeHandler{
|
|
HTTPGet: &corev1.HTTPGetAction{
|
|
Path: "/health",
|
|
Port: intstr.FromString(commonconsts.DynamoSystemPortName),
|
|
},
|
|
},
|
|
TimeoutSeconds: 30,
|
|
PeriodSeconds: 10,
|
|
SuccessThreshold: 0,
|
|
FailureThreshold: 60,
|
|
},
|
|
StartupProbe: &corev1.Probe{
|
|
ProbeHandler: corev1.ProbeHandler{
|
|
HTTPGet: &corev1.HTTPGetAction{
|
|
Path: "/live",
|
|
Port: intstr.FromString(commonconsts.DynamoSystemPortName),
|
|
},
|
|
},
|
|
TimeoutSeconds: 5,
|
|
PeriodSeconds: 10,
|
|
SuccessThreshold: 0,
|
|
FailureThreshold: 720,
|
|
},
|
|
},
|
|
},
|
|
ImagePullSecrets: nil, // Assuming default config gives empty secret name
|
|
ServiceAccountName: "default-test-sa", // Updated to reflect mocked SA
|
|
},
|
|
},
|
|
WorkerTemplate: corev1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Labels: map[string]string{
|
|
"instance-id": "0",
|
|
commonconsts.KubeLabelMetricsEnabled: commonconsts.KubeLabelValueTrue,
|
|
"role": "worker",
|
|
"nvidia.com/label1": "label1",
|
|
commonconsts.KubeLabelDynamoComponentType: commonconsts.ComponentTypeWorker,
|
|
commonconsts.KubeLabelDynamoSubComponentType: "test-sub-component",
|
|
commonconsts.KubeLabelDynamoGraphDeploymentName: "",
|
|
},
|
|
Annotations: map[string]string{
|
|
"scheduling.k8s.io/group-name": "test-lws-deploy-0",
|
|
"nvidia.com/annotation1": "annotation1",
|
|
},
|
|
},
|
|
Spec: corev1.PodSpec{
|
|
TerminationGracePeriodSeconds: ptr.To(int64(10)),
|
|
SchedulerName: "volcano",
|
|
Volumes: []corev1.Volume{
|
|
{
|
|
Name: "shared-memory",
|
|
VolumeSource: corev1.VolumeSource{
|
|
EmptyDir: &corev1.EmptyDirVolumeSource{
|
|
Medium: corev1.StorageMediumMemory,
|
|
SizeLimit: func() *resource.Quantity { q := resource.MustParse(commonconsts.DefaultSharedMemorySize); return &q }(),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
RestartPolicy: corev1.RestartPolicyAlways,
|
|
Containers: []corev1.Container{
|
|
{
|
|
Image: "another-image:latest",
|
|
},
|
|
{
|
|
Name: commonconsts.MainContainerName,
|
|
Image: "test-image:latest",
|
|
Command: []string{"sh", "-c"},
|
|
Args: []string{"ray start --address=$(LWS_LEADER_ADDRESS):6379 --block"},
|
|
Env: []corev1.EnvVar{
|
|
{Name: "DYN_NAMESPACE", Value: "default"},
|
|
{Name: "DYN_PARENT_DGD_K8S_NAME", Value: "test-lws-deploy"},
|
|
{Name: "DYN_PARENT_DGD_K8S_NAMESPACE", Value: "default"},
|
|
{Name: "DYN_SYSTEM_ENABLED", Value: "true"},
|
|
{Name: "DYN_SYSTEM_PORT", Value: "9090"},
|
|
{Name: "DYN_SYSTEM_USE_ENDPOINT_HEALTH_STATUS", Value: "[\"generate\"]"},
|
|
{Name: "TEST_ENV_FROM_DYNAMO_COMPONENT_DEPLOYMENT_SPEC", Value: "test_value_from_dynamo_component_deployment_spec"},
|
|
{Name: "TEST_ENV_FROM_EXTRA_POD_SPEC", Value: "test_value_from_extra_pod_spec"},
|
|
},
|
|
Ports: []corev1.ContainerPort{
|
|
{
|
|
Protocol: corev1.ProtocolTCP, Name: commonconsts.DynamoSystemPortName, ContainerPort: commonconsts.DynamoSystemPort,
|
|
},
|
|
},
|
|
VolumeMounts: []corev1.VolumeMount{
|
|
{
|
|
Name: "shared-memory",
|
|
MountPath: commonconsts.DefaultSharedMemoryMountPath,
|
|
},
|
|
},
|
|
Resources: corev1.ResourceRequirements{
|
|
Limits: corev1.ResourceList{
|
|
corev1.ResourceMemory: resource.MustParse("20Gi"),
|
|
corev1.ResourceCPU: resource.MustParse("10"),
|
|
"nvidia.com/gpu": resource.MustParse("1"),
|
|
},
|
|
Requests: corev1.ResourceList{
|
|
corev1.ResourceCPU: resource.MustParse("300m"),
|
|
corev1.ResourceMemory: resource.MustParse("500Mi"),
|
|
},
|
|
},
|
|
},
|
|
},
|
|
ImagePullSecrets: nil,
|
|
ServiceAccountName: "default-test-sa", // Updated to reflect mocked SA
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
want1: false,
|
|
wantErr: false,
|
|
},
|
|
{
|
|
name: "nil instanceID", // This test should fail before r.List is called in generatePodTemplateSpec
|
|
fields: fields{
|
|
Recorder: record.NewFakeRecorder(100),
|
|
},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
opt: generateResourceOption{
|
|
dynamoComponentDeployment: &v1alpha1.DynamoComponentDeployment{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "test-lws-nil-id", Namespace: "default"},
|
|
Spec: v1alpha1.DynamoComponentDeploymentSpec{
|
|
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
|
|
Multinode: &v1alpha1.MultinodeSpec{
|
|
NodeCount: 2,
|
|
},
|
|
Resources: &dynamoCommon.Resources{
|
|
Limits: &dynamoCommon.ResourceItem{
|
|
GPU: "1",
|
|
},
|
|
},
|
|
ExtraPodSpec: &dynamoCommon.ExtraPodSpec{
|
|
MainContainer: &corev1.Container{
|
|
Image: "test-image:latest",
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
instanceID: nil,
|
|
},
|
|
mockServiceAccounts: []client.Object{ // Provide a default SA for consistency, though not strictly needed here
|
|
&corev1.ServiceAccount{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "default-test-sa", Namespace: "default", // Match namespace
|
|
Labels: map[string]string{commonconsts.KubeLabelDynamoComponentPod: commonconsts.KubeLabelValueTrue},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
want: nil,
|
|
want1: false,
|
|
wantErr: true,
|
|
},
|
|
{
|
|
name: "error from generateLeaderPodTemplateSpec", // This case involves an error from generatePodTemplateSpec
|
|
fields: fields{
|
|
Recorder: record.NewFakeRecorder(100),
|
|
},
|
|
args: args{
|
|
ctx: context.Background(),
|
|
opt: generateResourceOption{
|
|
dynamoComponentDeployment: &v1alpha1.DynamoComponentDeployment{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "test-lws-leader-err", Namespace: "default"},
|
|
Spec: v1alpha1.DynamoComponentDeploymentSpec{
|
|
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
|
|
Multinode: &v1alpha1.MultinodeSpec{
|
|
NodeCount: 2,
|
|
},
|
|
Resources: &dynamoCommon.Resources{
|
|
Limits: &dynamoCommon.ResourceItem{
|
|
GPU: "1",
|
|
},
|
|
},
|
|
ExtraPodSpec: &dynamoCommon.ExtraPodSpec{
|
|
MainContainer: &corev1.Container{
|
|
Image: "", // Image is missing, will cause error in generatePodTemplateSpec
|
|
},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
instanceID: ptr.To(0),
|
|
},
|
|
// No specific SA needed if error is before SA listing, but good to be consistent
|
|
mockServiceAccounts: []client.Object{
|
|
&corev1.ServiceAccount{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "default-test-sa", Namespace: "default", // Match namespace
|
|
Labels: map[string]string{commonconsts.KubeLabelDynamoComponentPod: commonconsts.KubeLabelValueTrue},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
want: nil,
|
|
want1: false,
|
|
wantErr: true,
|
|
},
|
|
}
|
|
|
|
// Initialize scheme & add API types
|
|
s := scheme.Scheme
|
|
if err := v1alpha1.AddToScheme(s); err != nil {
|
|
t.Fatalf("Failed to add v1alpha1 to scheme: %v", err)
|
|
}
|
|
if err := corev1.AddToScheme(s); err != nil {
|
|
t.Fatalf("Failed to add corev1 to scheme: %v", err)
|
|
}
|
|
// Add LeaderWorkerSet to scheme if not already present globally for tests
|
|
if err := leaderworkersetv1.AddToScheme(s); err != nil {
|
|
t.Fatalf("Failed to add leaderworkersetv1 to scheme: %v", err)
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
format.MaxLength = 0
|
|
g := gomega.NewGomegaWithT(t)
|
|
|
|
// Build initial objects for fake client for this test case
|
|
var initialClientObjects []client.Object
|
|
if tt.args.opt.dynamoComponentDeployment != nil {
|
|
initialClientObjects = append(initialClientObjects, tt.args.opt.dynamoComponentDeployment)
|
|
}
|
|
if len(tt.args.mockServiceAccounts) > 0 {
|
|
initialClientObjects = append(initialClientObjects, tt.args.mockServiceAccounts...)
|
|
}
|
|
|
|
fakeKubeClient := fake.NewClientBuilder().
|
|
WithScheme(s).
|
|
WithObjects(initialClientObjects...).
|
|
Build()
|
|
|
|
r := &DynamoComponentDeploymentReconciler{
|
|
Client: fakeKubeClient, // Use the fake client
|
|
Recorder: tt.fields.Recorder,
|
|
Config: tt.fields.Config,
|
|
EtcdStorage: tt.fields.EtcdStorage,
|
|
DockerSecretRetriever: tt.fields.DockerSecretRetriever,
|
|
// Scheme: s, // Pass scheme if reconciler uses it directly, often client uses it
|
|
}
|
|
got, got1, err := r.generateLeaderWorkerSet(tt.args.ctx, tt.args.opt)
|
|
if (err != nil) != tt.wantErr {
|
|
t.Errorf("DynamoComponentDeploymentReconciler.generateLeaderWorkerSet() error = %v, wantErr %v", err, tt.wantErr)
|
|
return
|
|
}
|
|
if diff := cmp.Diff(tt.want, got); diff != "" {
|
|
t.Errorf("Mismatch (-expected +actual):\n%s", diff)
|
|
}
|
|
// Use gomega.Equal for deep comparison of complex structs
|
|
g.Expect(got).To(gomega.BeEquivalentTo(tt.want))
|
|
g.Expect(got1).To(gomega.BeEquivalentTo(tt.want1))
|
|
})
|
|
}
|
|
}
|