package clusterctl
import (
"context"
"database/sql"
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"time"
"cloud.google.com/go/secretmanager/apiv1/secretmanagerpb"
containerAPI "github.com/GoogleCloudPlatform/k8s-config-connector/pkg/clients/generated/apis/container/v1beta1"
"github.com/golang/mock/gomock"
"github.com/google/uuid"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/suite"
"google.golang.org/grpc/codes"
"google.golang.org/grpc/status"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
bsltypes "edge-infra.dev/pkg/edge/api/bsl/types"
"edge-infra.dev/pkg/edge/api/graph/model"
"edge-infra.dev/pkg/edge/api/mocks"
"edge-infra.dev/pkg/edge/api/services"
"edge-infra.dev/pkg/edge/api/testutils/seededpostgres"
"edge-infra.dev/pkg/edge/api/types"
"edge-infra.dev/pkg/edge/controllers/clusterctl/pkg/plugins"
"edge-infra.dev/pkg/edge/controllers/clusterctl/pkg/plugins/clustersecrets"
loglevels "edge-infra.dev/pkg/edge/controllers/clusterctl/pkg/plugins/log-levels"
"edge-infra.dev/pkg/edge/controllers/clusterctl/pkg/plugins/multikustomization"
"edge-infra.dev/pkg/edge/k8objectsutils"
"edge-infra.dev/pkg/edge/registration"
ipranger "edge-infra.dev/pkg/f8n/ipranger/server"
"edge-infra.dev/pkg/k8s/runtime/controller"
ff "edge-infra.dev/pkg/lib/featureflag"
fftest "edge-infra.dev/pkg/lib/featureflag/testutil"
"edge-infra.dev/test"
"edge-infra.dev/test/framework"
"edge-infra.dev/test/framework/gcp"
"edge-infra.dev/test/framework/integration"
"edge-infra.dev/test/framework/k8s"
"edge-infra.dev/test/framework/k8s/envtest"
)
var trackSecretManagerSecrets = make(map[string]struct{})
func TestMain(m *testing.M) {
framework.HandleFlags()
os.Exit(m.Run())
}
type Suite struct {
*framework.Framework
*k8s.K8s
Scheme *runtime.Scheme
ctx context.Context
timeout time.Duration
tick time.Duration
ClusterClient ContainerClusterClientFunc
ProjectID string
Banner *model.Banner
Organization string
ClusterName string
Location string
NodeVersion string
MachineType string
NumNodes int
TopLevelProjectID string
TopLevelCNRMSA string
DB *sql.DB
}
func TestClusterController(t *testing.T) {
testEnv := envtest.Setup()
defer testEnv.Stop() //nolint: errcheck
sp, err := seededpostgres.New()
if err != nil {
t.Fatal(err)
}
defer sp.Close()
db, err := sp.DB()
if err != nil {
t.Fatal(err)
}
defer db.Close()
registerTestPlugins(t, db)
scheme := createScheme()
// mock feature flags
fftest.MustInitTestFeatureFlags(map[string]bool{
ff.UseMasterAuthorizedNetworks: true,
})
// func for creating k8 client from container cluster
var createClient ContainerClusterClientFunc
var waitForSetTimeout time.Duration
if integration.IsIntegrationTest() {
createClient = k8objectsutils.CreateClient
// Enable WaitForSet during integration testing.
waitForSetTimeout = 5 * time.Second
} else {
k8s.Timeouts.Tick = 50 * time.Millisecond
fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build()
createClient = func(_ containerAPI.ContainerCluster, _ client.Options) (client.Client, error) {
return fakeClient, nil
}
}
sert := assert.New(t)
totpSecret := "totp-secret"
srv := httptest.NewServer(http.HandlerFunc(registration.GraphQLHandler(sert, registration.WithTotpSecret(totpSecret))))
iprsrv := httptest.NewServer(http.HandlerFunc(mockIPRanger()))
iprhost := strings.TrimPrefix(iprsrv.URL, "http://")
bslConfig := bsltypes.BSPConfig{
OrganizationPrefix: "edge-test1",
Endpoint: "https://api.ncr.com",
Root: "/customers",
}
cfg := Config{
CreateClient: createClient,
EdgeAPI: srv.URL,
IPRangerClient: ipranger.NewClient(iprhost),
DefaultRequeue: 10 * time.Millisecond,
TopLevelProjectID: "ret-edge-test",
TopLevelCNRMSA: "top-level@ret-edge-test.iam.gserviceaccount.com",
TotpSecret: totpSecret,
Domain: "edge-domain.ncr.com",
BSLConfig: bslConfig,
DatasyncDNSName: "edge-dev.ncr.com",
DatasyncDNSZone: "datasync-dns-zone",
DatabaseName: "postgres",
DB: db,
WaitForSetTimeout: waitForSetTimeout,
GCPRegion: "us-east1",
GCPZone: "c",
ClusterReconcilerConcurrency: 4,
GKEClusterReconcilerConcurrency: 4,
PluginConcurrency: 4,
HelmCacheLimit: 10,
EdgeSecMaxLeasePeriod: "48h",
EdgeSecMaxValidityPeriod: "60d",
}
mgr, _, err := Create(cfg, controller.WithCfg(testEnv.Config), controller.WithMetricsAddress("0"))
test.NoError(err)
k := k8s.New(testEnv.Config, k8s.WithCtrlManager(mgr), k8s.WithKonfigKonnector())
f := framework.New("clusterctl").
Component("clusterctl").
Register(k)
// Due to foreign key constraints, an existing banner_edge_id is needed when inserting clusters into the database
bannerEdgeID, bannerName, err := getBannerFromDatabase(db)
if err != nil {
t.Fatal(err)
}
s := &Suite{
Framework: f,
K8s: k,
ctx: context.Background(),
timeout: k8s.Timeouts.DefaultTimeout,
tick: k8s.Timeouts.Tick,
Scheme: scheme,
ClusterClient: createClient,
ProjectID: cfg.TopLevelProjectID,
Banner: &model.Banner{
Name: bannerName,
BannerEdgeID: bannerEdgeID,
},
Organization: "edge-dev0-test",
ClusterName: uuid.New().String(),
Location: "us-east1-c",
NodeVersion: "1.21.9-gke.300",
MachineType: "e2-highmem-2",
NumNodes: 3,
TopLevelCNRMSA: cfg.TopLevelCNRMSA,
TopLevelProjectID: cfg.TopLevelProjectID,
DB: db,
}
if integration.IsIntegrationTest() {
s.ProjectID = gcp.GCloud.ProjectID
// TODO set up Cluster, Banner and Organization for integration testing
//s.Banner =
//s.Organization =
//s.ClusterName =
}
suite.Run(t, s)
}
func getBannerFromDatabase(db *sql.DB) (id, name string, err error) {
err = db.QueryRow("SELECT banner_edge_id, banner_name FROM banners LIMIT 1").Scan(&id, &name)
return
}
func registerTestPlugins(t *testing.T, db *sql.DB) {
mockCtrl := gomock.NewController(t)
mockSecretManager := mocks.NewMockSecretManagerService(mockCtrl)
mockSecretManager.EXPECT().GetLatestSecretValueInfo(gomock.Any(), gomock.Any()).AnyTimes().Return(&secretmanagerpb.SecretVersion{}, status.Error(codes.NotFound, "secret not found"))
mockSecretManager.EXPECT().GetLatestSecretValue(gomock.Any(), gomock.Any()).AnyTimes().Return([]byte{}, nil)
mockSecretManager.EXPECT().GetSecretVersionValue(gomock.Any(), gomock.Any(), gomock.Any()).AnyTimes().Return([]byte{}, status.Error(codes.NotFound, "secret not found"))
mockSecretManager.EXPECT().
AddSecret(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()).
DoAndReturn(func(_ context.Context, secretID string, _ []byte, _ map[string]string, _ bool, _ *time.Time, _ string) error {
trackSecretManagerSecrets[secretID] = struct{}{}
return nil
}).AnyTimes()
mockSecretManager.EXPECT().DeleteSecret(gomock.Any(), gomock.Any()).
DoAndReturn(func(_ context.Context, secretID string) error {
delete(trackSecretManagerSecrets, secretID)
return nil
}).AnyTimes()
plugins.Register(clustersecrets.Plugin{
SecretManagerProvider: func(_ context.Context, _ string) (types.SecretManagerService, error) {
return mockSecretManager, nil
},
TopLevelProjectID: "t",
})
plugins.Register(multikustomization.NewPlugin(services.NewStoreClusterService(nil, nil, db, nil, nil, nil)))
plugins.Register(loglevels.LogLevelsPlugin{
DB: db,
})
plugins.Register(plugins.RemoteAccessIPPlugin{
DB: db,
})
}
func mockIPRanger() func(w http.ResponseWriter, r *http.Request) {
return func(w http.ResponseWriter, _ *http.Request) {
netcfg := ipranger.NetcfgResp{
Network: "networks/test",
Subnetwork: "subnetworks/test",
Netmask: "/21",
}
_ = json.NewEncoder(w).Encode(netcfg)
}
}