Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Allow specifying affinity overrides in synthesizer spec #271

Merged
merged 3 commits into from
Jan 22, 2025
Merged

Allow specifying affinity overrides in synthesizer spec #271

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
0d6ace5
Add affinity overrides, merge in and add tests
NickKeller Jan 22, 2025
bf2543f
Regenerate docs with proper version
NickKeller Jan 22, 2025
39e4a21
Add missing branch for merging nodeaffinity
NickKeller Jan 22, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
927 changes: 927 additions & 0 deletions api/v1/config/crd/eno.azure.io_synthesizers.yaml
View file
Open in desktop

Large diffs are not rendered by default.

1 change: 1 addition & 0 deletions api/v1/synthesizer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,7 @@ type PodOverrides struct {
Labels map[string]string `json:"labels,omitempty"`
Annotations map[string]string `json:"annotations,omitempty"`
Resources corev1.ResourceRequirements `json:"resources,omitempty"`
Affinity *corev1.Affinity `json:"affinity,omitempty"`
}

type SynthesizerStatus struct {
Expand Down
6 changes: 6 additions & 0 deletions api/v1/zz_generated.deepcopy.go
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions docs/api.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -156,6 +156,7 @@ _Appears in:_
| `labels` _object (keys:string, values:string)_ | | | |
| `annotations` _object (keys:string, values:string)_ | | | |
| `resources` _[ResourceRequirements](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#resourcerequirements-v1-core)_ | | | |
| `affinity` _[Affinity](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.22/#affinity-v1-core)_ | | | |


#### Ref
Expand Down
1 change: 1 addition & 0 deletions go.work.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ cloud.google.com/go v0.112.1/go.mod h1:+Vbu+Y1UU+I1rjmzeMOb/8RfkKJK2Gyxi1X6jJCZL
cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls=
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
cloud.google.com/go/compute/metadata v0.3.0/go.mod h1:zFmK7XCadkQkj6TtorcaGlCW1hT1fIilQDwofLpJ20k=
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
github.com/AdamKorcz/go-118-fuzz-build v0.0.0-20230306123547-8075edf89bb0/go.mod h1:OahwfttHWG6eJ0clwcfBAHoDI6X/LV/15hx/wlMZSrU=
github.com/Masterminds/vcs v1.13.3/go.mod h1:TiE7xuEjl1N4j016moRd6vezp6e6Lz23gypeXfzXeW8=
Expand Down
36 changes: 36 additions & 0 deletions internal/controllers/synthesis/pod.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"slices"
"strconv"

"github.com/imdario/mergo"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/ptr"
Expand Down Expand Up @@ -146,6 +147,41 @@ func newPod(cfg *Config, comp *apiv1.Composition, syn *apiv1.Synthesizer) *corev
}
}

// now that taints/toleration defaults have been set, time to merge in any overrides specified on the synthesizer
if syn.Spec.PodOverrides.Affinity != nil {
// do the merge
// easy one first
if syn.Spec.PodOverrides.Affinity.PodAffinity != nil {
pod.Spec.Affinity.PodAffinity = syn.Spec.PodOverrides.Affinity.PodAffinity
}

// merge in antiaffinity
if syn.Spec.PodOverrides.Affinity.PodAntiAffinity != nil {
// preferred is specified above so we want to append to that if specified
_ = mergo.Merge(&pod.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution,
syn.Spec.PodOverrides.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution,
mergo.WithAppendSlice,
mergo.WithoutDereference,
mergo.WithSliceDeepCopy)
// we can just overwrite the required one if specified
pod.Spec.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution = syn.Spec.PodOverrides.Affinity.PodAntiAffinity.RequiredDuringSchedulingIgnoredDuringExecution
}

if syn.Spec.PodOverrides.Affinity.NodeAffinity != nil {
// only need to merge the nodeaffinity terms if cfg.NodeAffinity was specified
// easy way to check is if it's not empty
if pod.Spec.Affinity.NodeAffinity != nil {
_ = mergo.Merge(&pod.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms,
syn.Spec.PodOverrides.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms,
mergo.WithAppendSlice,
mergo.WithoutDereference,
mergo.WithSliceDeepCopy)
}
} else {
// cfg.NodeAffinity was not specified, so we can just overwrite the nodeaffinity
pod.Spec.Affinity.NodeAffinity = syn.Spec.PodOverrides.Affinity.NodeAffinity
}
}
return pod
}

Expand Down
89 changes: 89 additions & 0 deletions internal/controllers/synthesis/pod_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,10 +4,12 @@ import (
"testing"

apiv1 "github.com/Azure/eno/api/v1"
"github.com/Azure/eno/internal/manager"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

var newPodTests = []struct {
Expand Down Expand Up @@ -122,6 +124,93 @@ var newPodTests = []struct {
assert.Contains(t, p.Spec.Containers[0].Env, corev1.EnvVar{Name: "COMPOSITION_NAME", Value: "test-composition"})
},
},
{
Name: "With affinity overrides",
Assert: func(t *testing.T, p *corev1.Pod) {
// assert that node affinity is set, and node affinity has both what is hardcoded and what is in the synth
// basically this is just testing that the merge appends arrays as necessary
require.NotNil(t, p.Spec.Affinity.NodeAffinity)
assert.Equal(t, 2, len(p.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms))
assert.Equal(t, corev1.NodeSelectorOpIn, p.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Operator)
assert.Equal(t, "foo", p.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Key)
assert.Equal(t, "bar", p.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[0].MatchExpressions[0].Values[0])

assert.Equal(t, corev1.NodeSelectorOpIn, p.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[1].MatchExpressions[0].Operator)
assert.Equal(t, []string{"testvalue"}, p.Spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution.NodeSelectorTerms[1].MatchExpressions[0].Values)

require.NotNil(t, p.Spec.Affinity.PodAffinity)
assert.Equal(t, "kubernetes.io/hostname", p.Spec.Affinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].TopologyKey)
require.NotNil(t, p.Spec.Affinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector)
assert.Equal(t, "testpodaffinitylabelvalue", p.Spec.Affinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].LabelSelector.MatchLabels["testpodaffinitylabelkey"])

require.NotNil(t, p.Spec.Affinity.PodAntiAffinity)
assert.Equal(t, 2, len(p.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution))

assert.Equal(t, int32(100), p.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].Weight)
assert.Equal(t, "kubernetes.io/hostname", p.Spec.Affinity.PodAffinity.RequiredDuringSchedulingIgnoredDuringExecution[0].TopologyKey)
require.NotNil(t, p.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].PodAffinityTerm.LabelSelector)
assert.Equal(t, manager.ManagerLabelValue, p.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[0].PodAffinityTerm.LabelSelector.MatchLabels[manager.ManagerLabelKey])

assert.Equal(t, int32(100), p.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[1].Weight)
assert.Equal(t, "kubernetes.io/hostname", p.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[1].PodAffinityTerm.TopologyKey)
require.NotNil(t, p.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[1].PodAffinityTerm.LabelSelector)
assert.Equal(t, "testpodantiaffinitylabelvalue", p.Spec.Affinity.PodAntiAffinity.PreferredDuringSchedulingIgnoredDuringExecution[1].PodAffinityTerm.LabelSelector.MatchLabels["testpodantiaffinitylabelkey"])
},
Cfg: &Config{
NodeAffinityKey: "foo",
NodeAffinityValue: "bar",
},
Synth: &apiv1.Synthesizer{
Spec: apiv1.SynthesizerSpec{
PodOverrides: apiv1.PodOverrides{
Affinity: &corev1.Affinity{
NodeAffinity: &corev1.NodeAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
NodeSelectorTerms: []corev1.NodeSelectorTerm{
{
MatchExpressions: []corev1.NodeSelectorRequirement{
{
Key: "testkey",
Operator: corev1.NodeSelectorOpIn,
Values: []string{"testvalue"},
},
},
},
},
},
},
PodAffinity: &corev1.PodAffinity{
RequiredDuringSchedulingIgnoredDuringExecution: []corev1.PodAffinityTerm{
{
TopologyKey: "kubernetes.io/hostname",
LabelSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"testpodaffinitylabelkey": "testpodaffinitylabelvalue",
},
},
},
},
},
PodAntiAffinity: &corev1.PodAntiAffinity{
PreferredDuringSchedulingIgnoredDuringExecution: []corev1.WeightedPodAffinityTerm{
{
Weight: 100,
PodAffinityTerm: corev1.PodAffinityTerm{
TopologyKey: "kubernetes.io/hostname",
LabelSelector: &metav1.LabelSelector{
MatchLabels: map[string]string{
"testpodantiaffinitylabelkey": "testpodantiaffinitylabelvalue",
},
},
},
},
},
},
},
},
},
},
},
}

func TestNewPod(t *testing.T) {
Expand Down
Loading