1 package clusterctl
2
3 import (
4 "context"
5 "database/sql"
6 "encoding/json"
7 "net/http"
8 "net/http/httptest"
9 "os"
10 "strings"
11 "testing"
12 "time"
13
14 "cloud.google.com/go/secretmanager/apiv1/secretmanagerpb"
15 containerAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/container/v1beta1"
16 "github.com/golang/mock/gomock"
17 "github.com/google/uuid"
18 "github.com/stretchr/testify/assert"
19 "github.com/stretchr/testify/suite"
20 "google.golang.org/grpc/codes"
21 "google.golang.org/grpc/status"
22 "k8s.io/apimachinery/pkg/runtime"
23 "sigs.k8s.io/controller-runtime/pkg/client"
24 "sigs.k8s.io/controller-runtime/pkg/client/fake"
25
26 bsltypes "edge-infra.dev/pkg/edge/api/bsl/types"
27 "edge-infra.dev/pkg/edge/api/graph/model"
28 "edge-infra.dev/pkg/edge/api/mocks"
29 "edge-infra.dev/pkg/edge/api/services"
30 "edge-infra.dev/pkg/edge/api/testutils/seededpostgres"
31 "edge-infra.dev/pkg/edge/api/types"
32 "edge-infra.dev/pkg/edge/controllers/clusterctl/pkg/plugins"
33 "edge-infra.dev/pkg/edge/controllers/clusterctl/pkg/plugins/clustersecrets"
34 loglevels "edge-infra.dev/pkg/edge/controllers/clusterctl/pkg/plugins/log-levels"
35 "edge-infra.dev/pkg/edge/controllers/clusterctl/pkg/plugins/multikustomization"
36 "edge-infra.dev/pkg/edge/k8objectsutils"
37 "edge-infra.dev/pkg/edge/registration"
38 ipranger "edge-infra.dev/pkg/f8n/ipranger/server"
39 "edge-infra.dev/pkg/k8s/runtime/controller"
40 ff "edge-infra.dev/pkg/lib/featureflag"
41 fftest "edge-infra.dev/pkg/lib/featureflag/testutil"
42 "edge-infra.dev/test"
43 "edge-infra.dev/test/framework"
44 "edge-infra.dev/test/framework/gcp"
45 "edge-infra.dev/test/framework/integration"
46 "edge-infra.dev/test/framework/k8s"
47 "edge-infra.dev/test/framework/k8s/envtest"
48 )
49
50 var trackSecretManagerSecrets = make(map[string]struct{})
51
52 func TestMain(m *testing.M) {
53 framework.HandleFlags()
54 os.Exit(m.Run())
55 }
56
57 type Suite struct {
58 *framework.Framework
59 *k8s.K8s
60 Scheme *runtime.Scheme
61 ctx context.Context
62 timeout time.Duration
63 tick time.Duration
64 ClusterClient ContainerClusterClientFunc
65 ProjectID string
66 Banner *model.Banner
67 Organization string
68 ClusterName string
69 Location string
70 NodeVersion string
71 MachineType string
72 NumNodes int
73 TopLevelProjectID string
74 TopLevelCNRMSA string
75 DB *sql.DB
76 }
77
78 func TestClusterController(t *testing.T) {
79 testEnv := envtest.Setup()
80 defer testEnv.Stop()
81
82 sp, err := seededpostgres.New()
83 if err != nil {
84 t.Fatal(err)
85 }
86 defer sp.Close()
87
88 db, err := sp.DB()
89 if err != nil {
90 t.Fatal(err)
91 }
92 defer db.Close()
93
94 registerTestPlugins(t, db)
95 scheme := createScheme()
96
97
98 fftest.MustInitTestFeatureFlags(map[string]bool{
99 ff.UseMasterAuthorizedNetworks: true,
100 })
101
102
103 var createClient ContainerClusterClientFunc
104 var waitForSetTimeout time.Duration
105 if integration.IsIntegrationTest() {
106 createClient = k8objectsutils.CreateClient
107
108 waitForSetTimeout = 5 * time.Second
109 } else {
110 k8s.Timeouts.Tick = 50 * time.Millisecond
111 fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build()
112 createClient = func(_ containerAPI.ContainerCluster, _ client.Options) (client.Client, error) {
113 return fakeClient, nil
114 }
115 }
116 sert := assert.New(t)
117 totpSecret := "totp-secret"
118 srv := httptest.NewServer(http.HandlerFunc(registration.GraphQLHandler(sert, registration.WithTotpSecret(totpSecret))))
119 iprsrv := httptest.NewServer(http.HandlerFunc(mockIPRanger()))
120 iprhost := strings.TrimPrefix(iprsrv.URL, "http://")
121 bslConfig := bsltypes.BSPConfig{
122 OrganizationPrefix: "edge-test1",
123 Endpoint: "https://api.ncr.com",
124 Root: "/customers",
125 }
126 cfg := Config{
127 CreateClient: createClient,
128 EdgeAPI: srv.URL,
129 IPRangerClient: ipranger.NewClient(iprhost),
130 DefaultRequeue: 10 * time.Millisecond,
131 TopLevelProjectID: "ret-edge-test",
132 TopLevelCNRMSA: "top-level@ret-edge-test.iam.gserviceaccount.com",
133 TotpSecret: totpSecret,
134 Domain: "edge-domain.ncr.com",
135 BSLConfig: bslConfig,
136 DatasyncDNSName: "edge-dev.ncr.com",
137 DatasyncDNSZone: "datasync-dns-zone",
138 DatabaseName: "postgres",
139 DB: db,
140 WaitForSetTimeout: waitForSetTimeout,
141 GCPRegion: "us-east1",
142 GCPZone: "c",
143 ClusterReconcilerConcurrency: 4,
144 GKEClusterReconcilerConcurrency: 4,
145 PluginConcurrency: 4,
146 HelmCacheLimit: 10,
147 EdgeSecMaxLeasePeriod: "48h",
148 EdgeSecMaxValidityPeriod: "60d",
149 }
150 mgr, _, err := Create(cfg, controller.WithCfg(testEnv.Config), controller.WithMetricsAddress("0"))
151 test.NoError(err)
152
153 k := k8s.New(testEnv.Config, k8s.WithCtrlManager(mgr), k8s.WithKonfigKonnector())
154
155 f := framework.New("clusterctl").
156 Component("clusterctl").
157 Register(k)
158
159
160 bannerEdgeID, bannerName, err := getBannerFromDatabase(db)
161 if err != nil {
162 t.Fatal(err)
163 }
164
165 s := &Suite{
166 Framework: f,
167 K8s: k,
168 ctx: context.Background(),
169 timeout: k8s.Timeouts.DefaultTimeout,
170 tick: k8s.Timeouts.Tick,
171 Scheme: scheme,
172 ClusterClient: createClient,
173 ProjectID: cfg.TopLevelProjectID,
174 Banner: &model.Banner{
175 Name: bannerName,
176 BannerEdgeID: bannerEdgeID,
177 },
178 Organization: "edge-dev0-test",
179 ClusterName: uuid.New().String(),
180 Location: "us-east1-c",
181 NodeVersion: "1.21.9-gke.300",
182 MachineType: "e2-highmem-2",
183 NumNodes: 3,
184 TopLevelCNRMSA: cfg.TopLevelCNRMSA,
185 TopLevelProjectID: cfg.TopLevelProjectID,
186 DB: db,
187 }
188
189 if integration.IsIntegrationTest() {
190 s.ProjectID = gcp.GCloud.ProjectID
191
192
193
194
195 }
196
197 suite.Run(t, s)
198 }
199
200 func getBannerFromDatabase(db *sql.DB) (id, name string, err error) {
201 err = db.QueryRow("SELECT banner_edge_id, banner_name FROM banners LIMIT 1").Scan(&id, &name)
202 return
203 }
204
205 func registerTestPlugins(t *testing.T, db *sql.DB) {
206 mockCtrl := gomock.NewController(t)
207 mockSecretManager := mocks.NewMockSecretManagerService(mockCtrl)
208
209 mockSecretManager.EXPECT().GetLatestSecretValueInfo(gomock.Any(), gomock.Any()).AnyTimes().Return(&secretmanagerpb.SecretVersion{}, status.Error(codes.NotFound, "secret not found"))
210 mockSecretManager.EXPECT().GetLatestSecretValue(gomock.Any(), gomock.Any()).AnyTimes().Return([]byte{}, nil)
211 mockSecretManager.EXPECT().GetSecretVersionValue(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return([]byte{}, status.Error(codes.NotFound, "secret not found"))
212 mockSecretManager.EXPECT().
213 AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
214 DoAndReturn(func(_ context.Context, secretID string, _ []byte, _ map[string]string, _ bool, _ *time.Time, _ string) error {
215 trackSecretManagerSecrets[secretID] = struct{}{}
216 return nil
217 }).AnyTimes()
218 mockSecretManager.EXPECT().DeleteSecret(gomock.Any(), gomock.Any()).
219 DoAndReturn(func(_ context.Context, secretID string) error {
220 delete(trackSecretManagerSecrets, secretID)
221 return nil
222 }).AnyTimes()
223 plugins.Register(clustersecrets.Plugin{
224 SecretManagerProvider: func(_ context.Context, _ string) (types.SecretManagerService, error) {
225 return mockSecretManager, nil
226 },
227 TopLevelProjectID: "t",
228 })
229 plugins.Register(multikustomization.NewPlugin(services.NewStoreClusterService(nil, nil, db, nil, nil, nil)))
230 plugins.Register(loglevels.LogLevelsPlugin{
231 DB: db,
232 })
233 plugins.Register(plugins.RemoteAccessIPPlugin{
234 DB: db,
235 })
236 }
237
238 func mockIPRanger() func(w http.ResponseWriter, r *http.Request) {
239 return func(w http.ResponseWriter, _ *http.Request) {
240 netcfg := ipranger.NetcfgResp{
241 Network: "networks/test",
242 Subnetwork: "subnetworks/test",
243 Netmask: "/21",
244 }
245 _ = json.NewEncoder(w).Encode(netcfg)
246 }
247 }
248
View as plain text