dynamo/deploy/operator/internal/controller_common/resource_test.go

773 lines
23 KiB
Go

/*
* SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
* SPDX-License-Identifier: Apache-2.0
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package controller_common
import (
"testing"
appsv1 "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1"
"github.com/ai-dynamo/dynamo/deploy/operator/internal/consts"
"github.com/bsm/gomega"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"sigs.k8s.io/controller-runtime/pkg/client"
)
func TestGetSpecChangeResult(t *testing.T) {
tests := []struct {
name string
current client.Object
desired client.Object
expectedHash bool
expectedError bool
}{
{
name: "no change in hash with deployment spec and env variables",
current: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": map[string]interface{}{
"name": "nim-deployment",
"namespace": "default",
},
"spec": map[string]interface{}{
"replicas": int64(2),
"selector": map[string]interface{}{
"matchLabels": map[string]interface{}{
"app": "nim",
},
},
"template": map[string]interface{}{
"metadata": map[string]interface{}{
"labels": map[string]interface{}{
"app": "nim",
},
},
"spec": map[string]interface{}{
"containers": []interface{}{
map[string]interface{}{
"name": "nim",
"image": "nim:v0.1.0",
"ports": []interface{}{
map[string]interface{}{
"containerPort": int64(80),
},
},
"env": []interface{}{
map[string]interface{}{"name": "ENV_VAR1", "value": "value1"},
map[string]interface{}{"name": "ENV_VAR2", "value": "value2"},
},
},
},
},
},
},
},
},
desired: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": map[string]interface{}{
"name": "nim-deployment",
"namespace": "default",
},
"spec": map[string]interface{}{
"replicas": int64(2),
"selector": map[string]interface{}{
"matchLabels": map[string]interface{}{
"app": "nim",
},
},
"template": map[string]interface{}{
"metadata": map[string]interface{}{
"labels": map[string]interface{}{
"app": "nim",
},
},
"spec": map[string]interface{}{
"containers": []interface{}{
map[string]interface{}{
"name": "nim",
"image": "nim:v0.1.0",
"ports": []interface{}{
map[string]interface{}{
"containerPort": int64(80),
},
},
"env": []interface{}{
map[string]interface{}{"name": "ENV_VAR1", "value": "value1"},
map[string]interface{}{"name": "ENV_VAR2", "value": "value2"},
},
},
},
},
},
},
},
},
expectedHash: false,
expectedError: false,
},
{
name: "no change in hash with deployment spec and env variables, change in order",
current: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": map[string]interface{}{
"name": "nim-deployment",
"namespace": "default",
},
"spec": map[string]interface{}{
"replicas": int64(2),
"selector": map[string]interface{}{
"matchLabels": map[string]interface{}{
"app": "nim",
},
},
"template": map[string]interface{}{
"metadata": map[string]interface{}{
"labels": map[string]interface{}{
"app": "nim",
},
},
"spec": map[string]interface{}{
"containers": []interface{}{
map[string]interface{}{
"name": "nim",
"image": "nim:v0.1.0",
"ports": []interface{}{
map[string]interface{}{
"containerPort": int64(80),
},
},
"env": []interface{}{
map[string]interface{}{"name": "ENV_VAR1", "value": "value1"},
map[string]interface{}{"name": "ENV_VAR2", "value": "value2"},
},
},
},
},
},
},
},
},
desired: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": map[string]interface{}{
"name": "nim-deployment",
"namespace": "default",
},
"spec": map[string]interface{}{
"replicas": int64(2),
"selector": map[string]interface{}{
"matchLabels": map[string]interface{}{
"app": "nim",
},
},
"template": map[string]interface{}{
"metadata": map[string]interface{}{
"labels": map[string]interface{}{
"app": "nim",
},
},
"spec": map[string]interface{}{
"containers": []interface{}{
map[string]interface{}{
"name": "nim",
"image": "nim:v0.1.0",
"ports": []interface{}{
map[string]interface{}{
"containerPort": int64(80),
},
},
"env": []interface{}{
map[string]interface{}{"name": "ENV_VAR2", "value": "value2"},
map[string]interface{}{"name": "ENV_VAR1", "value": "value1"},
},
},
},
},
},
},
},
},
expectedHash: false,
expectedError: false,
},
{
name: "no change in hash with change in metadata and status",
current: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": map[string]interface{}{
"name": "nim-deployment",
"namespace": "default",
},
"spec": map[string]interface{}{
"replicas": int64(2),
"selector": map[string]interface{}{
"matchLabels": map[string]interface{}{
"app": "nim",
},
},
"template": map[string]interface{}{
"metadata": map[string]interface{}{
"labels": map[string]interface{}{
"app": "nim",
},
},
"spec": map[string]interface{}{
"containers": []interface{}{
map[string]interface{}{
"name": "nim",
"image": "nim:v0.1.0",
"ports": []interface{}{
map[string]interface{}{
"containerPort": int64(80),
},
}, // switch order of env
"env": []interface{}{
map[string]interface{}{"name": "ENV_VAR1", "value": "value1"},
map[string]interface{}{"name": "ENV_VAR2", "value": "value2"},
},
},
},
},
},
},
},
},
desired: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": map[string]interface{}{
"name": "nim-deployment",
"namespace": "default",
"blah": "blah",
},
"spec": map[string]interface{}{
"replicas": int64(2),
"selector": map[string]interface{}{
"matchLabels": map[string]interface{}{
"app": "nim",
},
},
"template": map[string]interface{}{
"metadata": map[string]interface{}{
"labels": map[string]interface{}{
"app": "nim",
},
},
"spec": map[string]interface{}{
"containers": []interface{}{
map[string]interface{}{
"name": "nim",
"image": "nim:v0.1.0",
"ports": []interface{}{
map[string]interface{}{
"containerPort": int64(80),
},
},
"env": []interface{}{
map[string]interface{}{"name": "ENV_VAR1", "value": "value1"},
map[string]interface{}{"name": "ENV_VAR2", "value": "value2"},
},
},
},
},
},
},
"status": map[string]interface{}{
"ready": true,
},
},
},
expectedHash: false,
expectedError: false,
},
{
name: "change in hash with change in value of elements",
current: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": map[string]interface{}{
"name": "nim-deployment",
"namespace": "default",
},
"spec": map[string]interface{}{
"replicas": int64(2),
"selector": map[string]interface{}{
"matchLabels": map[string]interface{}{
"app": "nim",
},
},
"template": map[string]interface{}{
"metadata": map[string]interface{}{
"labels": map[string]interface{}{
"app": "nim",
},
},
"spec": map[string]interface{}{
"containers": []interface{}{
map[string]interface{}{
"name": "nim",
"image": "nim:v0.1.0",
"ports": []interface{}{
map[string]interface{}{
"containerPort": int64(80),
},
},
"env": []interface{}{
map[string]interface{}{"name": "ENV_VAR1", "value": "value2"},
map[string]interface{}{"name": "ENV_VAR2", "value": "value1"},
},
},
},
},
},
},
},
},
desired: &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": map[string]interface{}{
"name": "nim-deployment",
"namespace": "default",
},
"spec": map[string]interface{}{
"replicas": int64(3),
"selector": map[string]interface{}{
"matchLabels": map[string]interface{}{
"app": "nim",
},
},
"template": map[string]interface{}{
"metadata": map[string]interface{}{
"labels": map[string]interface{}{
"app": "nim",
},
},
"spec": map[string]interface{}{
"containers": []interface{}{
map[string]interface{}{
"name": "nim",
"image": "nim:v0.1.0",
"ports": []interface{}{
map[string]interface{}{
"containerPort": int64(80),
},
},
"env": []interface{}{
map[string]interface{}{"name": "ENV_VAR1", "value": "asdf"},
map[string]interface{}{"name": "ENV_VAR2", "value": "jljl"},
},
},
},
},
},
},
},
},
expectedHash: true,
expectedError: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
hash, err := GetSpecHash(tt.current)
if err != nil {
t.Errorf("failed to get spec hash in test for resource %s: %s", tt.current.GetName(), err)
}
// Set both hash and generation annotations (generation=1 simulates initial state)
updateAnnotations(tt.current, hash, 1)
result, err := GetSpecChangeResult(tt.current, tt.desired)
if err != nil {
t.Errorf("failed to check if spec has changed in test for resource %s: %s", tt.current.GetName(), err)
}
if tt.expectedHash && !result.NeedsUpdate {
t.Errorf("GetSpecChangeResult() NeedsUpdate = %v, want %v", result.NeedsUpdate, tt.expectedHash)
}
if !tt.expectedHash && result.NeedsUpdate {
t.Errorf("GetSpecChangeResult() NeedsUpdate = %v, want %v", result.NeedsUpdate, tt.expectedHash)
}
})
}
}
func TestGetSpecChangeResult_GenerationTracking(t *testing.T) {
tests := []struct {
name string
currentGeneration int64
lastAppliedGeneration string // empty string means annotation not set
lastAppliedHash string // empty string means annotation not set, "match" means compute from desired
desiredReplicas int64 // different from current (2) means hash will differ
expectNeedsUpdate bool
expectManualChangeDetected bool
expectNewGeneration int64 // 0 means don't check
}{
{
name: "no change - generations and hash match",
currentGeneration: 5,
lastAppliedGeneration: "5",
lastAppliedHash: "match",
desiredReplicas: 2, // same as current
expectNeedsUpdate: false,
},
{
name: "manual change detected - generation increased",
currentGeneration: 7,
lastAppliedGeneration: "5",
lastAppliedHash: "match",
desiredReplicas: 2,
expectNeedsUpdate: true,
expectManualChangeDetected: true,
expectNewGeneration: 8, // current(7) + 1
},
{
// Upgrade scenario: hash matches but no generation annotation yet.
// We do a full update to ensure spec is correct (could have been manual edits
// before we added generation tracking).
name: "missing generation annotation - full update for safety",
currentGeneration: 5,
lastAppliedGeneration: "", // missing
lastAppliedHash: "match",
desiredReplicas: 2,
expectNeedsUpdate: true,
expectNewGeneration: 6, // current + 1
},
{
name: "missing hash annotation - needs full update",
currentGeneration: 5,
lastAppliedGeneration: "5",
lastAppliedHash: "", // missing
desiredReplicas: 2,
expectNeedsUpdate: true,
expectNewGeneration: 6, // current(5) + 1
},
{
name: "hash changed - needs full update",
currentGeneration: 5,
lastAppliedGeneration: "5",
lastAppliedHash: "match",
desiredReplicas: 3, // different from current (2)
expectNeedsUpdate: true,
expectNewGeneration: 6, // current(5) + 1
},
{
name: "corrupted generation annotation - needs full update",
currentGeneration: 5,
lastAppliedGeneration: "invalid",
lastAppliedHash: "match",
desiredReplicas: 2,
expectNeedsUpdate: true,
expectNewGeneration: 6, // current(5) + 1
},
{
name: "both annotations missing - needs full update",
currentGeneration: 5,
lastAppliedGeneration: "",
lastAppliedHash: "",
desiredReplicas: 2,
expectNeedsUpdate: true,
expectNewGeneration: 6, // current(5) + 1
},
{
name: "manual change with hash also changed",
currentGeneration: 7,
lastAppliedGeneration: "5",
lastAppliedHash: "match",
desiredReplicas: 3, // different
expectNeedsUpdate: true,
expectManualChangeDetected: false, // hash change takes precedence
expectNewGeneration: 8,
},
{
// Generation=0 can occur with CRDs that don't have generation tracking enabled,
// or as a safety net for edge cases. When gen=0, we skip generation-based
// manual change detection and rely solely on hash comparison.
name: "generation zero - skip generation check",
currentGeneration: 0,
lastAppliedGeneration: "0",
lastAppliedHash: "match",
desiredReplicas: 2,
expectNeedsUpdate: false, // gen check skipped when gen=0, hash matches
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := gomega.NewGomegaWithT(t)
// Create current resource
current := &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": map[string]interface{}{
"name": "test-deployment",
"namespace": "default",
"generation": tt.currentGeneration,
"annotations": map[string]interface{}{},
},
"spec": map[string]interface{}{
"replicas": int64(2),
},
},
}
// Create desired resource
desired := &unstructured.Unstructured{
Object: map[string]interface{}{
"apiVersion": "apps/v1",
"kind": "Deployment",
"metadata": map[string]interface{}{
"name": "test-deployment",
"namespace": "default",
},
"spec": map[string]interface{}{
"replicas": tt.desiredReplicas,
},
},
}
// Set annotations based on test case
// "match" means the lastAppliedHash should match the CURRENT spec's hash
// (simulating that operator last applied what's currently in the cluster)
annotations := make(map[string]string)
if tt.lastAppliedHash == "match" {
hash, err := GetSpecHash(current)
g.Expect(err).To(gomega.BeNil())
annotations[NvidiaAnnotationHashKey] = hash
} else if tt.lastAppliedHash != "" {
annotations[NvidiaAnnotationHashKey] = tt.lastAppliedHash
}
if tt.lastAppliedGeneration != "" {
annotations[NvidiaAnnotationGenerationKey] = tt.lastAppliedGeneration
}
if len(annotations) > 0 {
current.SetAnnotations(annotations)
}
result, err := GetSpecChangeResult(current, desired)
g.Expect(err).To(gomega.BeNil())
g.Expect(result.NeedsUpdate).To(gomega.Equal(tt.expectNeedsUpdate), "NeedsUpdate mismatch")
g.Expect(result.ManualChangeDetected).To(gomega.Equal(tt.expectManualChangeDetected), "ManualChangeDetected mismatch")
if tt.expectNewGeneration != 0 {
g.Expect(result.NewGeneration).To(gomega.Equal(tt.expectNewGeneration), "NewGeneration mismatch")
}
})
}
}
func TestCopySpec(t *testing.T) {
src := appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "nim-deployment",
Namespace: "default",
},
Spec: appsv1.DeploymentSpec{
Replicas: &[]int32{2}[0],
},
}
dst := appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "nim-deployment",
Namespace: "default",
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: "apps/v1",
Kind: "Deployment",
Name: "nim-deployment",
UID: "1234567890",
},
},
},
Spec: appsv1.DeploymentSpec{
Replicas: &[]int32{1}[0],
},
}
err := CopySpec(&src, &dst)
if err != nil {
t.Errorf("failed to copy spec in test for resource %s: %s", src.GetName(), err)
}
expected := appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "nim-deployment",
Namespace: "default",
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: "apps/v1",
Kind: "Deployment",
Name: "nim-deployment",
UID: "1234567890",
},
},
},
Spec: appsv1.DeploymentSpec{
Replicas: &[]int32{2}[0],
},
}
g := gomega.NewGomegaWithT(t)
g.Expect(dst).To(gomega.Equal(expected))
}
func TestGetResourcesConfig(t *testing.T) {
tests := []struct {
name string
resources *v1alpha1.Resources
expectedGPULimit corev1.ResourceName
expectedGPUValue string
expectError bool
}{
{
name: "limits.gpu defined with no gpuType",
resources: &v1alpha1.Resources{
Limits: &v1alpha1.ResourceItem{
GPU: "4",
},
},
expectedGPULimit: corev1.ResourceName(consts.KubeResourceGPUNvidia),
expectedGPUValue: "4",
expectError: false,
},
{
name: "limits.gpu defined with custom gpuType",
resources: &v1alpha1.Resources{
Limits: &v1alpha1.ResourceItem{
GPU: "8",
GPUType: "gpu.custom-type.com/test",
},
},
expectedGPULimit: corev1.ResourceName("gpu.custom-type.com/test"),
expectedGPUValue: "8",
expectError: false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := gomega.NewGomegaWithT(t)
result, err := GetResourcesConfig(tt.resources)
if tt.expectError {
g.Expect(err).To(gomega.HaveOccurred())
return
}
g.Expect(err).To(gomega.BeNil())
g.Expect(result).ToNot(gomega.BeNil())
g.Expect(result.Limits).ToNot(gomega.BeNil())
gpuQuantity, exists := result.Limits[tt.expectedGPULimit]
g.Expect(exists).To(gomega.BeTrue(), "GPU resource %s should exist in limits", tt.expectedGPULimit)
g.Expect(gpuQuantity.String()).To(gomega.Equal(tt.expectedGPUValue))
})
}
}
func TestAppendUniqueImagePullSecrets(t *testing.T) {
tests := []struct {
name string
existing []corev1.LocalObjectReference
additional []corev1.LocalObjectReference
expected []corev1.LocalObjectReference
}{
{
name: "empty existing, empty additional",
existing: []corev1.LocalObjectReference{},
additional: []corev1.LocalObjectReference{},
expected: []corev1.LocalObjectReference{},
},
{
name: "empty existing, some additional",
existing: []corev1.LocalObjectReference{},
additional: []corev1.LocalObjectReference{{Name: "secret-a"}, {Name: "secret-b"}},
expected: []corev1.LocalObjectReference{{Name: "secret-a"}, {Name: "secret-b"}},
},
{
name: "some existing, empty additional",
existing: []corev1.LocalObjectReference{{Name: "secret-a"}},
additional: []corev1.LocalObjectReference{},
expected: []corev1.LocalObjectReference{{Name: "secret-a"}},
},
{
name: "no duplicates",
existing: []corev1.LocalObjectReference{{Name: "secret-a"}},
additional: []corev1.LocalObjectReference{{Name: "secret-b"}, {Name: "secret-c"}},
expected: []corev1.LocalObjectReference{{Name: "secret-a"}, {Name: "secret-b"}, {Name: "secret-c"}},
},
{
name: "all duplicates",
existing: []corev1.LocalObjectReference{{Name: "secret-a"}, {Name: "secret-b"}},
additional: []corev1.LocalObjectReference{{Name: "secret-a"}, {Name: "secret-b"}},
expected: []corev1.LocalObjectReference{{Name: "secret-a"}, {Name: "secret-b"}},
},
{
name: "some duplicates",
existing: []corev1.LocalObjectReference{{Name: "secret-a"}, {Name: "secret-b"}},
additional: []corev1.LocalObjectReference{{Name: "secret-b"}, {Name: "secret-c"}},
expected: []corev1.LocalObjectReference{{Name: "secret-a"}, {Name: "secret-b"}, {Name: "secret-c"}},
},
{
name: "duplicates within additional",
existing: []corev1.LocalObjectReference{{Name: "secret-a"}},
additional: []corev1.LocalObjectReference{{Name: "secret-b"}, {Name: "secret-b"}, {Name: "secret-c"}},
expected: []corev1.LocalObjectReference{{Name: "secret-a"}, {Name: "secret-b"}, {Name: "secret-c"}},
},
{
name: "nil existing",
existing: nil,
additional: []corev1.LocalObjectReference{{Name: "secret-a"}},
expected: []corev1.LocalObjectReference{{Name: "secret-a"}},
},
{
name: "nil additional",
existing: []corev1.LocalObjectReference{{Name: "secret-a"}},
additional: nil,
expected: []corev1.LocalObjectReference{{Name: "secret-a"}},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := gomega.NewGomegaWithT(t)
result := AppendUniqueImagePullSecrets(tt.existing, tt.additional)
g.Expect(result).To(gomega.Equal(tt.expected))
})
}
}