1 package staticpodscheduler_test
2
3 import (
4 "context"
5 "embed"
6 "io/fs"
7 "os"
8 "path/filepath"
9 "strings"
10 "testing"
11
12 "github.com/go-logr/logr/testr"
13 "github.com/spf13/afero"
14 "github.com/stretchr/testify/require"
15 corev1 "k8s.io/api/core/v1"
16 metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
17 kruntime "k8s.io/apimachinery/pkg/runtime"
18 utilruntime "k8s.io/apimachinery/pkg/util/runtime"
19 clientgoscheme "k8s.io/client-go/kubernetes/scheme"
20 ctrl "sigs.k8s.io/controller-runtime"
21 "sigs.k8s.io/controller-runtime/pkg/client"
22 "sigs.k8s.io/controller-runtime/pkg/client/fake"
23
24 calico "edge-infra.dev/pkg/k8s/net/calico"
25
26 v1ienode "edge-infra.dev/pkg/sds/ien/k8s/apis/v1"
27 "edge-infra.dev/pkg/sds/ien/k8s/controllers/nodeagent/config"
28 "edge-infra.dev/pkg/sds/ien/k8s/controllers/nodeagent/plugins/staticpodscheduler"
29 "edge-infra.dev/pkg/sds/ien/k8s/controllers/nodeagent/plugins/staticpodscheduler/pkg/render"
30 nodemeta "edge-infra.dev/pkg/sds/ien/node"
31 "edge-infra.dev/pkg/sds/ien/topology"
32 )
33
34
35 var embedFS embed.FS
36
37 var (
38 manifestDir = "/etc/kubernetes/manifests"
39 )
40
41 var (
42 lanOutageDetectorManifestFilename = "lan-outage-detector.yaml"
43 lanOutageDetectorManifestFilepath = filepath.Join(manifestDir, lanOutageDetectorManifestFilename)
44
45 expectedLANOutageDetectorManifest []byte
46 )
47
48 var (
49 etcdManagerManifestFilename = "etcd-manager.yaml"
50 etcdManagerManifestFilepath = filepath.Join(manifestDir, etcdManagerManifestFilename)
51
52 expectedEtcdManagerManifest []byte
53 )
54
55 var (
56 deviceAgentManifestFilename = "device-agent.yaml"
57 deviceAgentManifestFilepath = filepath.Join(manifestDir, deviceAgentManifestFilename)
58
59 expectedDeviceAgentsManifest []byte
60 )
61
62 var (
63 cpGuardianManifestFilename = "control-plane-guardian.yaml"
64 cpGuardianManifestFilepath = filepath.Join(manifestDir, cpGuardianManifestFilename)
65
66 expectedCPGuardianManifest []byte
67 )
68
69 var (
70 hostname = "desired-host"
71 lanOutageDetectorImage = "us-east1-docker.pkg.dev/ret-edge-pltf-infra/workloads/lanoutagedetector"
72 lanOutageSchedulerImage = "us-east1-docker.pkg.dev/ret-edge-pltf-infra/workloads/lanoutagescheduler"
73 admissionImage = "us-east1-docker.pkg.dev/ret-edge-pltf-infra/workloads/k8s-admission"
74 etcdManagerImage = "us-east1-docker.pkg.dev/ret-edge-pltf-infra/workloads/etcdmanager"
75 deviceAgentImage = "us-east1-docker.pkg.dev/ret-edge-pltf-infra/workloads/deviceagent"
76 cpGuardianImage = "us-east1-docker.pkg.dev/ret-edge-pltf-infra/workloads/controlplaneguardian"
77 cpPromoterImage = "us-east1-docker.pkg.dev/ret-edge-pltf-infra/workloads/controlplanepromoter"
78 controlPlaneIP = "1.2.3.4/32"
79 macAddress = "AB:CD:EF:GH:IJ:KL"
80 thickPOS = true
81 webhookName = "admission"
82 )
83
84 func TestMain(m *testing.M) {
85 os.Exit(m.Run())
86 }
87
88 func setupTestCtx(t *testing.T) context.Context {
89 logOptions := testr.Options{
90 LogTimestamp: true,
91 Verbosity: -1,
92 }
93
94 return ctrl.LoggerInto(context.Background(), testr.NewWithOptions(t, logOptions))
95 }
96
97 func TestStaticPodScheduler(t *testing.T) {
98 ctx := setupTestCtx(t)
99 scheduler := staticpodscheduler.Plugin{}
100
101 os.Setenv(render.NodeNameEnvVar, hostname)
102 os.Setenv(render.LANOutageDetectorImageKey, lanOutageDetectorImage)
103 os.Setenv(render.LANOutageSchedulerImageKey, lanOutageSchedulerImage)
104 os.Setenv(render.AdmissionImageKey, admissionImage)
105 os.Setenv(render.EtcdManagerImageKey, etcdManagerImage)
106 os.Setenv(render.DeviceAgentImageKey, deviceAgentImage)
107 os.Setenv(render.AdmissionWebhookEnvVar, webhookName)
108 os.Setenv(render.CPGuardianImageKey, cpGuardianImage)
109 os.Setenv(render.CPPromoterImageKey, cpPromoterImage)
110
111 tests := map[string]struct {
112 nodeRole v1ienode.Role
113 filepath string
114 expectWritten bool
115 expectedContent []byte
116 }{
117 "EtcdManagerManifest_Worker": {
118 nodeRole: v1ienode.Worker,
119 filepath: etcdManagerManifestFilepath,
120 expectWritten: false,
121 expectedContent: nil,
122 },
123 "LANOutageDetectorManifest_Worker": {
124 nodeRole: v1ienode.Worker,
125 filepath: lanOutageDetectorManifestFilepath,
126 expectWritten: true,
127 expectedContent: expectedLANOutageDetectorManifest,
128 },
129 "DeviceAgentManifest_Worker": {
130 nodeRole: v1ienode.Worker,
131 filepath: deviceAgentManifestFilepath,
132 expectWritten: true,
133 expectedContent: expectedDeviceAgentsManifest,
134 },
135 "CPGuardianManifest_Worker": {
136 nodeRole: v1ienode.Worker,
137 filepath: cpGuardianManifestFilepath,
138 expectWritten: true,
139 expectedContent: expectedCPGuardianManifest,
140 },
141 "EtcdManagerManifest_Controlplane": {
142 nodeRole: v1ienode.ControlPlane,
143 filepath: etcdManagerManifestFilepath,
144 expectWritten: true,
145 expectedContent: expectedEtcdManagerManifest,
146 },
147 "LANOutageDetectorManifest_Controlplane": {
148 nodeRole: v1ienode.ControlPlane,
149 filepath: lanOutageDetectorManifestFilepath,
150 expectWritten: true,
151 expectedContent: expectedLANOutageDetectorManifest,
152 },
153 "DeviceAgentManifest_Controlplane": {
154 nodeRole: v1ienode.ControlPlane,
155 filepath: deviceAgentManifestFilepath,
156 expectWritten: true,
157 expectedContent: expectedDeviceAgentsManifest,
158 },
159 "CPGuardianManifest_Controlplane": {
160 nodeRole: v1ienode.ControlPlane,
161 filepath: cpGuardianManifestFilepath,
162 expectWritten: true,
163 expectedContent: expectedCPGuardianManifest,
164 },
165 }
166 for name, tc := range tests {
167 t.Run(name, func(t *testing.T) {
168 ienode := getIENode(hostname, tc.nodeRole)
169 var controlPlaneNode, workerNode *corev1.Node
170 switch tc.nodeRole {
171 case v1ienode.ControlPlane:
172 controlPlaneNode = getControlPlane(hostname)
173 workerNode = getWorkerNode("worker")
174 case v1ienode.Worker:
175 workerNode = getWorkerNode(hostname)
176 controlPlaneNode = getControlPlane("controlplane")
177 }
178 c := getMockKubeClient(ienode, controlPlaneNode, getTopologyConfigMap(), workerNode)
179 cfg := config.NewConfig(c, getMockKubeReader(), nil, config.Flags{})
180 memFs := afero.NewMemMapFs()
181 require.NoError(t, embedFiles(memFs))
182 cfg = cfg.WithFs(memFs)
183
184 _, err := scheduler.Reconcile(ctx, ienode, cfg)
185 require.NoError(t, err)
186
187 content, err := afero.ReadFile(memFs, tc.filepath)
188 switch tc.expectWritten {
189 case true:
190 require.NoError(t, err)
191 require.Equal(t, string(tc.expectedContent), string(content))
192 case false:
193 require.ErrorIs(t, err, os.ErrNotExist)
194 }
195 })
196 }
197 }
198
199 func getMockKubeClient(initObjs ...client.Object) client.Client {
200 return fake.NewClientBuilder().WithScheme(createScheme()).WithObjects(initObjs...).Build()
201 }
202
203 func getMockKubeReader(initObjs ...client.Object) client.Reader {
204 return fake.NewClientBuilder().WithScheme(createScheme()).WithObjects(initObjs...).Build()
205 }
206
207 func createScheme() *kruntime.Scheme {
208 scheme := kruntime.NewScheme()
209 utilruntime.Must(clientgoscheme.AddToScheme(scheme))
210 utilruntime.Must(v1ienode.AddToScheme(scheme))
211 return scheme
212 }
213
214 func getControlPlane(name string) *corev1.Node {
215 return &corev1.Node{
216 ObjectMeta: metav1.ObjectMeta{
217 Name: name,
218 Labels: map[string]string{
219 nodemeta.RoleLabel: string(v1ienode.ControlPlane),
220 },
221 Annotations: map[string]string{
222 calico.IPAnnotation: controlPlaneIP,
223 },
224 },
225 }
226 }
227
228 func getWorkerNode(name string) *corev1.Node {
229 return &corev1.Node{
230 ObjectMeta: metav1.ObjectMeta{
231 Name: name,
232 Labels: map[string]string{
233 nodemeta.RoleLabel: string(v1ienode.Worker),
234 },
235 Annotations: map[string]string{
236 calico.IPAnnotation: controlPlaneIP,
237 },
238 },
239 }
240 }
241
242 func getIENode(name string, role v1ienode.Role) *v1ienode.IENode {
243 return &v1ienode.IENode{
244 ObjectMeta: metav1.ObjectMeta{
245 Name: name,
246 },
247 Spec: v1ienode.IENodeSpec{
248 Network: []v1ienode.Network{
249 {
250 MacAddress: macAddress,
251 },
252 },
253 Role: role,
254 },
255 }
256 }
257
258 func getTopologyConfigMap() *corev1.ConfigMap {
259 tpInfo := topology.New()
260 tpInfo.ThickPOS = thickPOS
261 return tpInfo.ToConfigMap()
262 }
263
264 func embedFiles(memFS afero.Fs) error {
265 return fs.WalkDir(embedFS, "testdata/embedfs", func(path string, dir os.DirEntry, err error) error {
266 if err != nil {
267 return err
268 }
269 if dir.IsDir() && (dir.Name() == "testdata" || strings.Contains(path, "embedfs")) {
270 return nil
271 }
272 if dir.IsDir() {
273 return memFS.Mkdir(path, 0644)
274 }
275 file, err := embedFS.ReadFile(path)
276 if err != nil {
277 return err
278 }
279
280 path = strings.TrimPrefix(path, "testdata/embedfs")
281 return afero.WriteFile(memFS, path, file, 0644)
282 })
283 }
284
View as plain text