package services import ( "context" "database/sql" "encoding/hex" "encoding/json" "errors" "fmt" "slices" "strings" "time" "github.com/google/uuid" "gopkg.in/yaml.v2" sqlerr "edge-infra.dev/pkg/edge/api/apierror/sql" "edge-infra.dev/pkg/edge/api/clients" "edge-infra.dev/pkg/edge/api/graph/mapper" "edge-infra.dev/pkg/edge/api/graph/model" edgenode "edge-infra.dev/pkg/edge/api/services/edgenode/common" sqlquery "edge-infra.dev/pkg/edge/api/sql" "edge-infra.dev/pkg/edge/api/status" "edge-infra.dev/pkg/edge/api/utils" "edge-infra.dev/pkg/lib/crypto" "edge-infra.dev/pkg/lib/runtime/version" cc "edge-infra.dev/pkg/sds/clustersecrets/common" dsv1 "edge-infra.dev/pkg/sds/devices/k8s/apis/v1" v1ien "edge-infra.dev/pkg/sds/ien/k8s/apis/v1" ) //go:generate mockgen -destination=../mocks/mock_terminal_service.go -package=mocks edge-infra.dev/pkg/edge/api/services TerminalService type TerminalService interface { CreateTerminalEntry(ctx context.Context, newTerminal *model.TerminalCreateInput, activationCode crypto.Credential) (*model.Terminal, error) UpdateTerminalEntry(ctx context.Context, terminal *model.TerminalIDInput) (*model.Terminal, error) DeleteTerminalEntry(ctx context.Context, terminalID string) error CreateTerminalInterfaceEntry(ctx context.Context, terminalID string, newTerminalInterface *model.TerminalInterfaceCreateInput) (*model.TerminalInterface, error) UpdateTerminalInterfaceEntry(ctx context.Context, updatedInterface *model.TerminalInterfaceIDInput) (*model.TerminalInterface, error) DeleteTerminalInterfaceEntry(ctx context.Context, terminalInterfaceID string) (*model.Terminal, error) CreateTerminalAddressEntry(ctx context.Context, terminalInterfaceID string, newTerminalAddress *model.TerminalAddressCreateInput) (*model.TerminalAddress, error) UpdateTerminalAddressEntry(ctx context.Context, updatedAddress *model.TerminalAddressIDInput) (*model.TerminalAddress, error) DeleteTerminalAddressEntry(ctx context.Context, terminalAddressID string) (*model.Terminal, error) GetTerminal(ctx context.Context, terminalID string, getLabels *bool) (*model.Terminal, error) GetTerminals(ctx context.Context, clusterEdgeID *string, terminalHostname *string) ([]*model.Terminal, error) GetBannerTerminals(ctx context.Context, bannerEdgeID, projectID string) ([]*model.Terminal, error) CreateDSDSIENodeCR(terminal *model.Terminal, clusterNetworkServices []*model.ClusterNetworkServiceInfo, customLabels map[string]string, edgeVersion string) (string, error) CreateCICIENodeCR(terminal *model.Terminal, clusterNetworkServices []*model.ClusterNetworkServiceInfo, customLabels map[string]string, edgeVersion string) (string, error) GetTerminalFromInterface(ctx context.Context, terminalInterfaceID string) (*model.Terminal, error) GetTerminalFromAddress(ctx context.Context, terminalAddressID string) (*model.Terminal, error) GetTerminalBootstrapConfig(ctx context.Context, terminal *model.Terminal, clusterNetworkServices []*model.ClusterNetworkServiceInfo, breakGlassSecret, grubSecret cc.Secret, bootstrapAck, isFirstNode bool, organization string, bootstrapTokenValues []*model.KeyValues, clusterCaHash, endpoint string) (string, error) RemoveClusterEdgeBootstrapToken(ctx context.Context, clusterEdgeID string) error CreateTerminalDiskEntry(ctx context.Context, terminalID string, newTerminalDisk *model.TerminalDiskCreateInput) (*model.TerminalDisk, error) DeleteTerminalDiskEntry(ctx context.Context, terminalDiskID string) (*model.Terminal, error) UpdateTerminalDiskEntry(ctx context.Context, terminalDiskID string, updatedDisk model.TerminalDiskUpdateInput) (*model.TerminalDisk, error) GetTerminalFromDisk(ctx context.Context, terminalDiskID string) (*model.Terminal, error) GetTerminalNodeStatus(ctx context.Context, clusterEdgeID, terminalID, hostname string) (*model.TerminalStatus, error) GetNodeReplicationStatus(ctx context.Context, clusterEdgeID, hostname string) (*model.ReplicationStatus, error) GetTerminalsByClusterID(ctx context.Context, clusterEdgeID string) ([]*model.Terminal, error) TerminalDevices(ctx context.Context, terminalID string) (*model.TerminalDevices, error) } type terminalService struct { SQLDB *sql.DB BQClient clients.BQClient LabelService LabelService StoreClusterService StoreClusterService } var ( getLabel = true ErrDuplicateTerminalDiskDevicePaths = errors.New("terminals cannot have disks with duplicate device paths") ) func (t *terminalService) CreateTerminalEntry(ctx context.Context, newTerminal *model.TerminalCreateInput, activationCode crypto.Credential) (*model.Terminal, error) { if err := utils.ValidateTerminalCreateInput(newTerminal); err != nil { return nil, err } transaction, err := t.SQLDB.BeginTx(ctx, nil) if err != nil { return nil, err } defer transaction.Rollback() //nolint row := transaction.QueryRowContext(ctx, sqlquery.GetClusterNameByClusterEdgeIDQuery, newTerminal.ClusterEdgeID) var clusterName string err = row.Scan(&clusterName) if err != nil { return nil, err } // create terminal if len(newTerminal.Interfaces) == 0 { return nil, errors.New("unable to create terminal with no interfaces") } var hostname string if newTerminal.Hostname == nil { hostname = utils.CreateIENodeHostname(newTerminal.Interfaces[0].MacAddress) } else { hostname = strings.ToLower(*newTerminal.Hostname) } newTerminal.Hostname = &hostname // check existing hostnames if err := t.checkHostnames(ctx, hostname, newTerminal); err != nil { return nil, err } // check lane value is unique if newTerminal.Lane != nil && *newTerminal.Lane != "" { if err := t.checkLaneDuplicates(ctx, *newTerminal.Lane, newTerminal.ClusterEdgeID); err != nil { return nil, err } } terminal := utils.CreateTerminalModel(uuid.NewString(), newTerminal.ClusterEdgeID, newTerminal.Lane, newTerminal.Role, newTerminal.Class, newTerminal.DiscoverDisks, newTerminal.BootDisk, clusterName, *newTerminal.Hostname, newTerminal.ExistingEfiPart, *newTerminal.SwapEnabled) // validate terminal if err = utils.ValidateTerminal(&terminal); err != nil { return nil, err } hash := activationCode.Hashed() hashEncoded := hex.EncodeToString(hash) args := []interface{}{ terminal.TerminalID, terminal.Lane, terminal.Role, terminal.Class, terminal.DiscoverDisks, terminal.BootDisk, terminal.ClusterEdgeID, terminal.Hostname, hashEncoded, terminal.ExistingEfiPart, terminal.SwapEnabled, } if _, err = transaction.ExecContext(ctx, sqlquery.TerminalCreateQuery, args...); err != nil { return nil, err } // check for duplicate device paths if hasDuplicateDevicePathsCreate(newTerminal.Disks) { return nil, ErrDuplicateTerminalDiskDevicePaths } terminalDisk, err := t.createTerminalDiskEntries(ctx, transaction, newTerminal.Disks, terminal.TerminalID) if err != nil { return nil, err } terminal.Disks = terminalDisk // create interfaces terminal.Interfaces, err = t.createTerminalInterfaceEntries(ctx, transaction, newTerminal.Interfaces, terminal.TerminalID) if err != nil { if rollbackErr := transaction.Rollback(); rollbackErr != nil { return nil, rollbackErr } return nil, err } if err = transaction.Commit(); err != nil { return nil, err } return &terminal, nil } // hasDuplicateDevicePathsCreate returns true if the provided diskList contains any disks to be created with the same device path func hasDuplicateDevicePathsCreate(diskList []*model.TerminalDiskCreateInput) bool { diskPaths := []string{} for _, disk := range diskList { if slices.Contains(diskPaths, disk.DevicePath) { return true } diskPaths = append(diskPaths, disk.DevicePath) } return false } func (t *terminalService) createTerminalInterfaceEntries(ctx context.Context, transaction *sql.Tx, newInterfaces []*model.TerminalInterfaceCreateInput, terminalID string) ([]*model.TerminalInterface, error) { terminalInterfaces := []*model.TerminalInterface{} for _, newInterface := range newInterfaces { terminalInterface := utils.CreateTerminalIfaceModel(uuid.NewString(), strings.ToLower(newInterface.MacAddress), newInterface.Dhcp4, newInterface.Dhcp6, newInterface.Gateway4, newInterface.Gateway6, terminalID) args := []interface{}{ terminalInterface.TerminalInterfaceID, terminalInterface.MacAddress, terminalInterface.Dhcp4, terminalInterface.Dhcp6, terminalInterface.Gateway4, terminalInterface.Gateway6, terminalID, } if _, err := transaction.ExecContext(ctx, sqlquery.TerminalInterfaceCreateQuery, args...); err != nil { return nil, err } // create addresses var err error terminalInterface.Addresses, err = t.createTerminalAddressEntries(ctx, transaction, newInterface.Addresses, &terminalInterface) if err != nil { return nil, err } if err := utils.ValidateAllTerminalAddresses(terminalInterface.Dhcp4, terminalInterface.Addresses); err != nil { return nil, err } terminalInterfaces = append(terminalInterfaces, &terminalInterface) } return terminalInterfaces, nil } func (t *terminalService) createTerminalAddressEntries(ctx context.Context, transaction *sql.Tx, newAddresses []*model.TerminalAddressCreateInput, iface *model.TerminalInterface) ([]*model.TerminalAddress, error) { terminalAddresses := []*model.TerminalAddress{} for _, newAddress := range newAddresses { terminalAddress := utils.CreateTerminalAddressModel(uuid.NewString(), newAddress.IP, newAddress.PrefixLen, newAddress.Family, iface.TerminalInterfaceID) // validate terminal address if err := utils.ValidateTerminalAddress(&terminalAddress); err != nil { return nil, err } args := []interface{}{ terminalAddress.TerminalAddressID, terminalAddress.IP, terminalAddress.PrefixLen, terminalAddress.Family, iface.TerminalInterfaceID, } if _, err := transaction.ExecContext(ctx, sqlquery.TerminalAddressCreateQuery, args...); err != nil { return nil, err } terminalAddresses = append(terminalAddresses, &terminalAddress) } return terminalAddresses, nil } func (t *terminalService) UpdateTerminalEntry(ctx context.Context, updateTerminal *model.TerminalIDInput) (*model.Terminal, error) { if err := utils.ValidateTerminalUpdateInput(updateTerminal.TerminalValues); err != nil { return nil, err } transaction, err := t.SQLDB.BeginTx(ctx, nil) if err != nil { return nil, err } defer transaction.Rollback() //nolint // get the current values for the terminal in the db currentTerminal, err := t.GetTerminal(ctx, updateTerminal.TerminalID, &getLabel) if err != nil { return nil, err } // check lane value is unique if updateTerminal.TerminalValues.Lane != nil && *updateTerminal.TerminalValues.Lane != "" { if err := t.checkLaneDuplicates(ctx, *updateTerminal.TerminalValues.Lane, currentTerminal.ClusterEdgeID); err != nil { return nil, err } } // check activation code is available before setting primary interface if updateTerminal.TerminalValues.PrimaryInterface != nil { ac, err := edgenode.FetchActivationCode(ctx, t.SQLDB, updateTerminal.TerminalID) if err != nil { return nil, err } if ac == nil || len(*ac) == 0 { return nil, errors.New("cannot change primary interface, terminal already bootstrapped") } } // set current terminal values currentTerminal = utils.UpdateTerminal(currentTerminal, updateTerminal) // validate the current terminal if err = utils.ValidateTerminal(currentTerminal); err != nil { return nil, err } args := []interface{}{ currentTerminal.Lane, currentTerminal.Role, currentTerminal.Class, currentTerminal.DiscoverDisks, currentTerminal.BootDisk, currentTerminal.PrimaryInterface, currentTerminal.ExistingEfiPart, currentTerminal.SwapEnabled, currentTerminal.TerminalID, } // update terminal in db if _, err = transaction.ExecContext(ctx, sqlquery.TerminalUpdateQuery, args...); err != nil { return nil, err } // check for duplicate device paths in provided update disks if hasDuplicateDevicePathsUpdate(updateTerminal.TerminalValues.Disks) { return nil, ErrDuplicateTerminalDiskDevicePaths } terminalDisks, err := t.updateTerminalDiskEntries(ctx, transaction, updateTerminal.TerminalValues.Disks) if err != nil { if strings.Contains(err.Error(), "duplicate key value violates unique constraint \"unique_terminal_id_device_path\"") { return nil, ErrDuplicateTerminalDiskDevicePaths // return neat error } return nil, err } currentTerminal.Disks = terminalDisks // update interfaces if err := t.updateTerminalInterfaceEntries(ctx, transaction, updateTerminal.TerminalValues.Interfaces, currentTerminal.Interfaces); err != nil { if rollbackErr := transaction.Rollback(); rollbackErr != nil { return nil, rollbackErr } return nil, err } if err = transaction.Commit(); err != nil { return nil, err } currentTerminal, err = t.GetTerminal(ctx, updateTerminal.TerminalID, &getLabel) if err != nil { return nil, err } return currentTerminal, nil } // hasDuplicateDevicePathsUpdate returns true if the provided diskList contains any disks to be updated with the same device path // only checks provided update disks, not existing disks - that is handled by DB func hasDuplicateDevicePathsUpdate(diskList []*model.TerminalDiskIDInput) bool { diskPaths := []string{} for _, disk := range diskList { if disk == nil || disk.TerminalDiskValues == nil || disk.TerminalDiskValues.DevicePath == nil { continue } diskPath := *disk.TerminalDiskValues.DevicePath if slices.Contains(diskPaths, diskPath) { return true } diskPaths = append(diskPaths, diskPath) } return false } func (t *terminalService) UpdateTerminalInterfaceEntry(ctx context.Context, updatedInterface *model.TerminalInterfaceIDInput) (*model.TerminalInterface, error) { //nolint if err := utils.ValidateTerminalInterfaceUpdateInput(updatedInterface.TerminalInterfaceValues); err != nil { return nil, err } updateIfaces := []*model.TerminalInterfaceIDInput{updatedInterface} currentIface, err := t.getTerminalInterface(ctx, updatedInterface.TerminalInterfaceID) if err != nil { return nil, err } currentIfaces := []*model.TerminalInterface{currentIface} transaction, err := t.SQLDB.BeginTx(ctx, nil) if err != nil { return nil, err } if err := t.updateTerminalInterfaceEntries(ctx, transaction, updateIfaces, currentIfaces); err != nil { if rollbackErr := transaction.Rollback(); rollbackErr != nil { return nil, rollbackErr } return nil, err } if err := transaction.Commit(); err != nil { return nil, err } iface, err := t.getTerminalInterface(ctx, updatedInterface.TerminalInterfaceID) if err != nil { return nil, err } return iface, nil } func (t *terminalService) updateTerminalInterfaceEntries(ctx context.Context, transaction *sql.Tx, updateInterfaces []*model.TerminalInterfaceIDInput, currentInterfaces []*model.TerminalInterface) error { for _, updateInterface := range updateInterfaces { interfaceValues := updateInterface.TerminalInterfaceValues // update the current interface's values currentInterface, err := utils.UpdateTerminalInterface(currentInterfaces, updateInterface) if err != nil { return err } args := []interface{}{ currentInterface.MacAddress, currentInterface.Dhcp4, currentInterface.Dhcp6, currentInterface.Gateway4, currentInterface.Gateway6, currentInterface.TerminalInterfaceID, } // update interface in db if _, err = transaction.ExecContext(ctx, sqlquery.TerminalInterfaceUpdateQuery, args...); err != nil { return err } // update addresses if necessary if len(interfaceValues.Addresses) > 0 { if err := t.updateTerminalAddressEntries(ctx, transaction, interfaceValues.Addresses, currentInterface.Addresses); err != nil { return err } } // validate all interface addresses after update if err := utils.ValidateAllTerminalAddresses(currentInterface.Dhcp4, currentInterface.Addresses); err != nil { return err } } return nil } func (t *terminalService) updateTerminalAddressEntries(ctx context.Context, transaction *sql.Tx, updateAddresses []*model.TerminalAddressIDInput, currentAddresses []*model.TerminalAddress) error { for _, updateAddress := range updateAddresses { // update current address with new values currentAddress, err := utils.UpdateTerminalAddress(currentAddresses, updateAddress) if err != nil { return err } // validate address if err = utils.ValidateTerminalAddress(currentAddress); err != nil { return err } args := []interface{}{ currentAddress.IP, currentAddress.PrefixLen, currentAddress.Family, currentAddress.TerminalAddressID, } // update address in db if _, err = transaction.ExecContext(ctx, sqlquery.TerminalAddressUpdateQuery, args...); err != nil { return err } } return nil } func (t *terminalService) DeleteTerminalEntry(ctx context.Context, terminalID string) error { _, err := t.SQLDB.ExecContext(ctx, sqlquery.TerminalDeleteQuery, terminalID) return err } func (t *terminalService) CreateTerminalInterfaceEntry(ctx context.Context, terminalID string, newTerminalInterface *model.TerminalInterfaceCreateInput) (*model.TerminalInterface, error) { if err := utils.ValidateTerminalInterfaceCreateInput(newTerminalInterface); err != nil { return nil, err } transaction, err := t.SQLDB.BeginTx(ctx, nil) if err != nil { return nil, err } newIfaceList := []*model.TerminalInterfaceCreateInput{newTerminalInterface} ifaceList, err := t.createTerminalInterfaceEntries(ctx, transaction, newIfaceList, terminalID) if err != nil { if rollbackErr := transaction.Rollback(); rollbackErr != nil { return nil, rollbackErr } return nil, err } if err = transaction.Commit(); err != nil { return nil, err } if len(ifaceList) == 0 { return nil, errors.New("no interface to return - potential error creating interface entry") } return ifaceList[0], nil } func (t *terminalService) DeleteTerminalInterfaceEntry(ctx context.Context, terminalInterfaceID string) (*model.Terminal, error) { terminal, err := t.GetTerminalFromInterface(ctx, terminalInterfaceID) if err != nil { return nil, err } if len(terminal.Interfaces) == 1 { return nil, errors.New("unable to delete the last interface of the terminal") } for i, iface := range terminal.Interfaces { if iface.TerminalInterfaceID == terminalInterfaceID { terminal.Interfaces[i] = terminal.Interfaces[len(terminal.Interfaces)-1] terminal.Interfaces = terminal.Interfaces[:len(terminal.Interfaces)-1] break } } _, err = t.SQLDB.ExecContext(ctx, sqlquery.TerminalInterfaceDeleteQuery, terminalInterfaceID) return terminal, err } func (t *terminalService) CreateTerminalAddressEntry(ctx context.Context, terminalInterfaceID string, newTerminalAddress *model.TerminalAddressCreateInput) (*model.TerminalAddress, error) { address := utils.CreateTerminalAddressModel(uuid.NewString(), newTerminalAddress.IP, newTerminalAddress.PrefixLen, newTerminalAddress.Family, terminalInterfaceID) args := []interface{}{ address.TerminalAddressID, address.IP, address.PrefixLen, address.Family, address.TerminalInterfaceID, } _, err := t.SQLDB.ExecContext(ctx, sqlquery.TerminalAddressCreateQuery, args...) if err != nil { return nil, err } return &address, nil } func (t *terminalService) UpdateTerminalAddressEntry(ctx context.Context, updatedAddress *model.TerminalAddressIDInput) (*model.TerminalAddress, error) { //nolint updateAddresses := []*model.TerminalAddressIDInput{updatedAddress} currentAddr, err := t.getTerminalAddress(ctx, updatedAddress.TerminalAddressID) if err != nil { return nil, err } currentAddrs := []*model.TerminalAddress{currentAddr} transaction, err := t.SQLDB.BeginTx(ctx, nil) if err != nil { return nil, err } if err := t.updateTerminalAddressEntries(ctx, transaction, updateAddresses, currentAddrs); err != nil { if rollbackErr := transaction.Rollback(); rollbackErr != nil { return nil, rollbackErr } return nil, err } if err := transaction.Commit(); err != nil { return nil, err } addr, err := t.getTerminalAddress(ctx, updatedAddress.TerminalAddressID) if err != nil { return nil, err } return addr, nil } func (t *terminalService) DeleteTerminalAddressEntry(ctx context.Context, terminalAddressID string) (*model.Terminal, error) { terminal, err := t.GetTerminalFromAddress(ctx, terminalAddressID) if err != nil { return nil, err } out: for _, iface := range terminal.Interfaces { for i, address := range iface.Addresses { if address.TerminalAddressID == terminalAddressID { iface.Addresses[i] = iface.Addresses[len(iface.Addresses)-1] iface.Addresses = iface.Addresses[:len(iface.Addresses)-1] // validate that the address can be deleted if err := utils.ValidateAllTerminalAddresses(iface.Dhcp4, iface.Addresses); err != nil { return nil, err } break out } } } _, err = t.SQLDB.ExecContext(ctx, sqlquery.TerminalAddressDeleteQuery, terminalAddressID) return terminal, err } func (t *terminalService) GetTerminal(ctx context.Context, terminalID string, getLabel *bool) (*model.Terminal, error) { row := t.SQLDB.QueryRowContext(ctx, sqlquery.GetTerminalByIDQuery, terminalID) terminal, err := t.scanTerminalRow(row) if err != nil { return nil, err } terminal.Interfaces, err = t.getTerminalInterfaceByTerminalID(ctx, &terminalID) if err != nil { return nil, err } terminal.Disks, err = t.getTerminalDisks(ctx, &terminalID) if err != nil { return nil, err } if getLabel != nil && *getLabel { terminalLabelSvc := NewTerminalLabelService(t.SQLDB, nil, nil, nil, t.LabelService) terminalLabels, err := terminalLabelSvc.GetTerminalLabels(ctx, model.SearchTerminalLabelInput{ TerminalID: &terminalID, }) if err != nil { return nil, err } terminal.Labels = terminalLabels } term, err := t.updateTerminalVersions(ctx, []*model.Terminal{terminal}) if err != nil { return nil, err } terminal = term[0] return terminal, nil } func (t *terminalService) GetTerminals(ctx context.Context, clusterEdgeID *string, terminalHostname *string) ([]*model.Terminal, error) { switch { case clusterEdgeID == nil && terminalHostname == nil: return t.getAllTerminals(ctx) case clusterEdgeID != nil && terminalHostname == nil: return t.getClusterEdgeIDTerminals(ctx, *clusterEdgeID) case clusterEdgeID == nil && terminalHostname != nil: return t.getHostnameTerminals(ctx, *terminalHostname) default: return t.getClusterEdgeIDAndHostnameTerminals(ctx, *clusterEdgeID, *terminalHostname) } } func (t *terminalService) GetNodeReplicationStatus(ctx context.Context, clusterEdgeID, hostname string) (*model.ReplicationStatus, error) { var name string row := t.SQLDB.QueryRowContext(ctx, sqlquery.GetNodeReplicaName, clusterEdgeID, fmt.Sprintf("%q", hostname)) if err := row.Scan(&name); err != nil { if errors.Is(err, sql.ErrNoRows) { return &model.ReplicationStatus{ Status: "NotFound", }, nil } return nil, sqlerr.Wrap(err) } row = t.SQLDB.QueryRowContext(ctx, sqlquery.GetReplicationStatusByName, clusterEdgeID, name) var replicationStatus model.ReplicationStatus if err := row.Scan(&replicationStatus.Name, &replicationStatus.Status); err != nil { if errors.Is(err, sql.ErrNoRows) { return &model.ReplicationStatus{ Status: "NotFound", }, nil } return nil, sqlerr.Wrap(err) } replicationStatus.Status = strings.Trim(replicationStatus.Status, "\"") return &replicationStatus, nil } func (t *terminalService) GetTerminalNodeStatus(ctx context.Context, clusterEdgeID, terminalID, hostname string) (*model.TerminalStatus, error) { var ( nodeReady = "" ieNodeReady = "" terminalStatus = &model.TerminalStatus{} notReported = make(map[string]bool) ) row := t.SQLDB.QueryRowContext(ctx, sqlquery.GetTerminalNodeStatus, "Node", clusterEdgeID, hostname, false) err := row.Scan(&nodeReady) if err != nil && !errors.Is(err, sql.ErrNoRows) { return nil, err } // Handling cases where the Kubernetes Node Status is reported as UNKNOWN // We are treating this as being not reported if nodeReady == status.Unknown || errors.Is(err, sql.ErrNoRows) { notReported["Node"] = true } row = t.SQLDB.QueryRowContext(ctx, sqlquery.GetTerminalNodeStatus, v1ien.IENodeGVK.Kind, clusterEdgeID, hostname, false) if err := row.Scan(&ieNodeReady); err != nil { if !errors.Is(err, sql.ErrNoRows) { return nil, err } notReported[v1ien.IENodeGVK.Kind] = true } // Case 1: Node Status = True, Node Agent (IENode) Status = True, and (Activation Code Consumed = True || Activation Code Consumed = False) // @returns status = Ready // @returns message = Node Ready if nodeReady == status.IsReady && ieNodeReady == status.IsReady { terminalStatus.Status = status.Ready terminalStatus.Message = status.NodeReadyMessage return terminalStatus, nil } // Case 2: Node Status = Error, Node Agent (IENode) Status = Error // @returns status = Error // @returns message = Node status + node agent status condition message if nodeReady == status.NotReady || ieNodeReady == status.NotReady { //nolint:nestif aggregatedErr := make([]string, 0) if nodeReady == status.NotReady { nodeErrMessage := "" if err := t.getNodeErrorMessage(ctx, clusterEdgeID, "Node", hostname, &nodeErrMessage, &aggregatedErr); err != nil { return nil, err } } if ieNodeReady == status.NotReady { ieNodeErrMessage := "" if err := t.getNodeErrorMessage(ctx, clusterEdgeID, v1ien.IENodeGVK.Kind, hostname, &ieNodeErrMessage, &aggregatedErr); err != nil { return nil, err } } terminalStatus.Status = status.Error terminalStatus.Message = status.MergeErrorMessages(aggregatedErr) return terminalStatus, nil } activationCode := "" row = t.SQLDB.QueryRowContext(ctx, edgenode.GetActivationCode, terminalID) if err := row.Scan(&activationCode); err != nil { return nil, err } if status.IsNotReported("Node", notReported) || status.IsNotReported(v1ien.IENodeGVK.Kind, notReported) { switch { case status.IsNotReported("Node", notReported) && status.IsNotReported(v1ien.IENodeGVK.Kind, notReported) && activationCode == "": // Case 3: Node Status = N/A, Node Agent Status (IENode) = N/A, and Activation Code Consumed = True // @returns status = N/A // @returns message = Status for node missing terminalStatus.Status = status.NotAvailable terminalStatus.Message = status.NotAvailableMessage case activationCode != "": // Case 4: Node Status = N/A, Node Agent (IENode) Status = N/A, and Activation Code Consumed = False // @returns status = Awaiting Installation // @returns message = Activation code not consumed terminalStatus.Status = status.AwaitingInstallation terminalStatus.Message = status.UnusedActivationCodeMessage // Case 5: Node Status = Unknown, Node Agent Status (IENode) = "", and Activation Code Consumed = True // @returns status = Disconnected // @returns message = Node is disconnected or turned off case activationCode == "": terminalStatus.Status = status.Disconnected terminalStatus.Message = status.DisconnectedMessage } } return terminalStatus, nil } func (t *terminalService) getNodeErrorMessage(ctx context.Context, clusterEdgeID, kind, hostName string, errMessage *string, aggregatedErr *[]string) error { row := t.SQLDB.QueryRowContext(ctx, sqlquery.GetTerminalNodeStatusMessage, kind, clusterEdgeID, hostName, false) if err := row.Scan(errMessage); err != nil { if !errors.Is(err, sql.ErrNoRows) { return err } } else { *aggregatedErr = append(*aggregatedErr, *errMessage) } return nil } func (t *terminalService) GetBannerTerminals(ctx context.Context, bannerEdgeID string, projectID string) ([]*model.Terminal, error) { // get all cluster edge ids for the banner rows, err := t.SQLDB.QueryContext(ctx, sqlquery.GetClustersByBannerEdgeIDQuery, bannerEdgeID) if err != nil { return nil, err } // version information will eventually be returned by the above query once ctlfish moves over to the SQLDB versionMap, err := t.getTerminalVersionMapFromProjectID(ctx, projectID) if err != nil { return nil, err } terminalMap := make(map[string]*model.Terminal) ifaceMap := make(map[string]*model.TerminalInterface) addressMap := make(map[string]*model.TerminalAddress) for rows.Next() { terminal, ifaceBanner, addressBanner := model.Terminal{}, utils.NullIface{}, utils.NullAddress{} err := rows.Scan(&terminal.TerminalID, &terminal.Lane, &terminal.Role, &terminal.Class, &terminal.DiscoverDisks, &terminal.BootDisk, &terminal.PrimaryInterface, &terminal.ExistingEfiPart, &terminal.SwapEnabled, &terminal.ClusterEdgeID, &terminal.ClusterName, &terminal.Hostname, &ifaceBanner.TerminalInterfaceID, &ifaceBanner.MacAddress, &ifaceBanner.Dhcp4, &ifaceBanner.Dhcp6, &ifaceBanner.Gateway4, &ifaceBanner.Gateway6, &ifaceBanner.TerminalID, &addressBanner.TerminalAddressID, &addressBanner.IP, &addressBanner.PrefixLen, &addressBanner.Family, &addressBanner.TerminalInterfaceID) if err != nil { return nil, err } if _, exists := terminalMap[terminal.TerminalID]; !exists { terminalMap[terminal.TerminalID] = &terminal } if ifaceBanner.TerminalInterfaceID.Valid { iface, err := utils.ConvertNullIface(ifaceBanner) if err != nil { return nil, err } if _, exists := ifaceMap[iface.TerminalInterfaceID]; !exists { ifaceMap[iface.TerminalInterfaceID] = iface } } if addressBanner.TerminalAddressID.Valid { address, err := utils.ConvertNullAddr(addressBanner) if err != nil { return nil, err } if _, exists := addressMap[address.TerminalAddressID]; !exists { addressMap[address.TerminalAddressID] = address } } } if err := rows.Err(); err != nil { return nil, sqlerr.Wrap(err) } terminals := mapper.UnpackBannerTerminals(terminalMap, ifaceMap, addressMap, versionMap) for _, terminal := range terminals { terminalDisks, err := t.getTerminalDisks(ctx, &terminal.TerminalID) if err != nil { return nil, err } terminal.Disks = terminalDisks } return terminals, nil } func (t *terminalService) getAllTerminals(ctx context.Context) ([]*model.Terminal, error) { rows, err := t.SQLDB.QueryContext(ctx, sqlquery.GetAllTerminalsQuery) if err != nil { return nil, err } terminals, err := t.scanTerminalRows(rows) if err != nil { return nil, err } for _, terminal := range terminals { terminalDisks, err := t.getTerminalDisks(ctx, &terminal.TerminalID) if err != nil { return nil, err } terminal.Disks = terminalDisks terminal.Interfaces, err = t.getTerminalInterfaceByTerminalID(ctx, &terminal.TerminalID) if err != nil { return nil, err } terminalLabelSvc := NewTerminalLabelService(t.SQLDB, nil, nil, nil, t.LabelService) terminalLabels, err := terminalLabelSvc.GetTerminalLabels(ctx, model.SearchTerminalLabelInput{ TerminalID: &terminal.TerminalID, }) if err != nil { return nil, err } terminal.Labels = terminalLabels } return t.updateTerminalVersions(ctx, terminals) } func (t *terminalService) getClusterEdgeIDTerminals(ctx context.Context, clusterEdgeID string) ([]*model.Terminal, error) { //nolint terminals, err := t.GetTerminalsByClusterID(ctx, clusterEdgeID) if err != nil { return nil, err } for _, terminal := range terminals { terminalDisks, err := t.getTerminalDisks(ctx, &terminal.TerminalID) if err != nil { return nil, err } terminal.Disks = terminalDisks terminal.Interfaces, err = t.getTerminalInterfaceByTerminalID(ctx, &terminal.TerminalID) if err != nil { return nil, err } terminalLabelSvc := NewTerminalLabelService(t.SQLDB, nil, nil, nil, t.LabelService) terminalLabels, err := terminalLabelSvc.GetTerminalLabels(ctx, model.SearchTerminalLabelInput{ TerminalID: &terminal.TerminalID, }) if err != nil { return nil, err } terminal.Labels = terminalLabels } return t.updateTerminalVersions(ctx, terminals) } func (t *terminalService) GetTerminalsByClusterID(ctx context.Context, clusterEdgeID string) ([]*model.Terminal, error) { rows, err := t.SQLDB.QueryContext(ctx, sqlquery.GetTerminalByClusterEdgeIDQuery, clusterEdgeID) if err != nil { return nil, err } terminals, err := t.scanTerminalRows(rows) if err != nil { return nil, err } return terminals, nil } func (t *terminalService) getHostnameTerminals(ctx context.Context, terminalHostname string) ([]*model.Terminal, error) { //nolint rows, err := t.SQLDB.QueryContext(ctx, sqlquery.GetTerminalByHostnameQuery, terminalHostname) if err != nil { return nil, err } terminals, err := t.scanTerminalRows(rows) if err != nil { return nil, err } for _, terminal := range terminals { terminalDisks, err := t.getTerminalDisks(ctx, &terminal.TerminalID) if err != nil { return nil, err } terminal.Disks = terminalDisks terminal.Interfaces, err = t.getTerminalInterfaceByTerminalID(ctx, &terminal.TerminalID) if err != nil { return nil, err } terminalLabelSvc := NewTerminalLabelService(t.SQLDB, nil, nil, nil, t.LabelService) terminalLabels, err := terminalLabelSvc.GetTerminalLabels(ctx, model.SearchTerminalLabelInput{ TerminalID: &terminal.TerminalID, }) if err != nil { return nil, err } terminal.Labels = terminalLabels } return t.updateTerminalVersions(ctx, terminals) } func (t *terminalService) getClusterEdgeIDAndHostnameTerminals(ctx context.Context, clusterEdgeID string, terminalHostname string) ([]*model.Terminal, error) { rows, err := t.SQLDB.QueryContext(ctx, sqlquery.GetTerminalByClusterEdgeIDAndHostnameQuery, clusterEdgeID, terminalHostname) if err != nil { return nil, err } terminals, err := t.scanTerminalRows(rows) if err != nil { return nil, err } for _, terminal := range terminals { terminalDisks, err := t.getTerminalDisks(ctx, &terminal.TerminalID) if err != nil { return nil, err } terminal.Disks = terminalDisks terminal.Interfaces, err = t.getTerminalInterfaceByTerminalID(ctx, &terminal.TerminalID) if err != nil { return nil, err } terminalLabelSvc := NewTerminalLabelService(t.SQLDB, nil, nil, nil, t.LabelService) terminalLabels, err := terminalLabelSvc.GetTerminalLabels(ctx, model.SearchTerminalLabelInput{ TerminalID: &terminal.TerminalID, }) if err != nil { return nil, err } terminal.Labels = terminalLabels } return t.updateTerminalVersions(ctx, terminals) } func (t *terminalService) getTerminalInterface(ctx context.Context, interfaceID string) (*model.TerminalInterface, error) { row := t.SQLDB.QueryRowContext(ctx, sqlquery.GetTerminalInterfaceQuery, interfaceID) iface, err := t.scanTerminalIfaceRow(row) if err != nil { return nil, err } iface.Addresses, err = t.getTerminalAddressByInterfaceID(ctx, &iface.TerminalInterfaceID) if err != nil { return nil, err } return iface, nil } func (t *terminalService) getTerminalInterfaceByTerminalID(ctx context.Context, terminalID *string) ([]*model.TerminalInterface, error) { rows, err := t.SQLDB.QueryContext(ctx, sqlquery.GetTerminalInterfaceByTerminalIDQuery, terminalID) if err != nil { return nil, err } ifaces, err := t.scanTerminalIfaceRows(rows) if err != nil { return nil, err } for _, iface := range ifaces { iface.Addresses, err = t.getTerminalAddressByInterfaceID(ctx, &iface.TerminalInterfaceID) if err != nil { return nil, err } } return ifaces, nil } func (t *terminalService) getTerminalAddress(ctx context.Context, addressID string) (*model.TerminalAddress, error) { row := t.SQLDB.QueryRowContext(ctx, sqlquery.GetTerminalAddressQuery, addressID) addr, err := t.scanTerminalAddressRow(row) if err != nil { return nil, err } return addr, nil } func (t *terminalService) getTerminalAddressByInterfaceID(ctx context.Context, terminalInterfaceID *string) ([]*model.TerminalAddress, error) { rows, err := t.SQLDB.QueryContext(ctx, sqlquery.GetTerminalAddressByInterfaceIDQuery, terminalInterfaceID) if err != nil { return nil, err } addresses, err := t.scanTerminalAddressRows(rows) if err != nil { return nil, err } return addresses, nil } func (t *terminalService) scanTerminalRow(row *sql.Row) (*model.Terminal, error) { terminal := &model.Terminal{} err := row.Scan(&terminal.TerminalID, &terminal.Lane, &terminal.Role, &terminal.ClusterEdgeID, &terminal.ClusterName, &terminal.Class, &terminal.DiscoverDisks, &terminal.BootDisk, &terminal.PrimaryInterface, &terminal.ExistingEfiPart, &terminal.SwapEnabled, &terminal.Hostname) if err != nil { return nil, err } return terminal, nil } func (t *terminalService) scanTerminalRows(rows *sql.Rows) ([]*model.Terminal, error) { terminals := []*model.Terminal{} for rows.Next() { terminal := &model.Terminal{} err := rows.Scan(&terminal.TerminalID, &terminal.Lane, &terminal.Role, &terminal.ClusterEdgeID, &terminal.ClusterName, &terminal.Class, &terminal.DiscoverDisks, &terminal.BootDisk, &terminal.PrimaryInterface, &terminal.ExistingEfiPart, &terminal.SwapEnabled, &terminal.Hostname) if err != nil { return nil, err } terminals = append(terminals, terminal) } if err := rows.Err(); err != nil { return nil, sqlerr.Wrap(err) } return terminals, nil } func (t *terminalService) scanTerminalIfaceRow(row *sql.Row) (*model.TerminalInterface, error) { iface := &model.TerminalInterface{} if err := row.Scan(&iface.TerminalInterfaceID, &iface.MacAddress, &iface.Dhcp4, &iface.Dhcp6, &iface.Gateway4, &iface.Gateway6, &iface.TerminalID); err != nil { return nil, err } return iface, nil } func (t *terminalService) scanTerminalIfaceRows(rows *sql.Rows) ([]*model.TerminalInterface, error) { terminalInterfaces := []*model.TerminalInterface{} for rows.Next() { terminalInterface := model.TerminalInterface{} err := rows.Scan(&terminalInterface.TerminalInterfaceID, &terminalInterface.MacAddress, &terminalInterface.Dhcp4, &terminalInterface.Dhcp6, &terminalInterface.Gateway4, &terminalInterface.Gateway6, &terminalInterface.TerminalID) if err != nil { return nil, err } terminalInterfaces = append(terminalInterfaces, &terminalInterface) } if err := rows.Err(); err != nil { return nil, sqlerr.Wrap(err) } return terminalInterfaces, nil } func (t *terminalService) scanTerminalAddressRow(row *sql.Row) (*model.TerminalAddress, error) { addr := &model.TerminalAddress{} if err := row.Scan(&addr.TerminalAddressID, &addr.IP, &addr.PrefixLen, &addr.Family, &addr.TerminalInterfaceID); err != nil { return nil, err } return addr, nil } func (t *terminalService) scanTerminalAddressRows(rows *sql.Rows) ([]*model.TerminalAddress, error) { terminalAddresses := []*model.TerminalAddress{} for rows.Next() { terminalAddress := model.TerminalAddress{} if err := rows.Scan(&terminalAddress.TerminalAddressID, &terminalAddress.IP, &terminalAddress.PrefixLen, &terminalAddress.Family, &terminalAddress.TerminalInterfaceID); err != nil { return nil, err } terminalAddresses = append(terminalAddresses, &terminalAddress) } if err := rows.Err(); err != nil { return nil, sqlerr.Wrap(err) } return terminalAddresses, nil } func (t *terminalService) CreateDSDSIENodeCR(terminal *model.Terminal, clusterNetworkServices []*model.ClusterNetworkServiceInfo, customLabels map[string]string, edgeVersion string) (string, error) { dsdsIENode := mapper.TerminalToIENode(terminal, clusterNetworkServices, customLabels, edgeVersion) dsdsIENodeBase64, err := utils.ConvertStructToBase64(dsdsIENode) if err != nil { return "", err } return dsdsIENodeBase64, nil } func (t *terminalService) CreateCICIENodeCR(terminal *model.Terminal, clusterNetworkServices []*model.ClusterNetworkServiceInfo, customLabels map[string]string, edgeVersion string) (string, error) { terminalCR := mapper.TerminalToCICIENode(terminal, clusterNetworkServices, customLabels, edgeVersion) terminalCRBase64, err := utils.ConvertStructToBase64(terminalCR) if err != nil { return "", err } return terminalCRBase64, nil } func (t *terminalService) GetTerminalFromInterface(ctx context.Context, terminalInterfaceID string) (*model.Terminal, error) { row := t.SQLDB.QueryRowContext(ctx, sqlquery.GetTerminalIDFromInterfaceQuery, terminalInterfaceID) var terminalID string if err := row.Scan(&terminalID); err != nil { return nil, err } terminal, err := t.GetTerminal(ctx, terminalID, &getLabel) if err != nil { return nil, err } return terminal, nil } func (t *terminalService) GetTerminalFromAddress(ctx context.Context, terminalAddressID string) (*model.Terminal, error) { row := t.SQLDB.QueryRowContext(ctx, sqlquery.GetInterfaceIDFromAddressQuery, terminalAddressID) var ifaceID string if err := row.Scan(&ifaceID); err != nil { return nil, err } row = t.SQLDB.QueryRowContext(ctx, sqlquery.GetTerminalIDFromInterfaceQuery, ifaceID) var terminalID string if err := row.Scan(&terminalID); err != nil { return nil, err } terminal, err := t.GetTerminal(ctx, terminalID, &getLabel) if err != nil { return nil, err } return terminal, nil } func (t *terminalService) TerminalDevices(ctx context.Context, terminalID string) (*model.TerminalDevices, error) { fetchLabels := false terminal, err := t.GetTerminal(ctx, terminalID, &fetchLabels) if err != nil { return &model.TerminalDevices{}, err } var value string row := t.SQLDB.QueryRowContext(ctx, sqlquery.GetTerminalDevices, terminal.Hostname, terminal.ClusterEdgeID) if err := row.Scan(&value); err != nil { return &model.TerminalDevices{}, err } deviceMap := dsv1.DeviceStatusesSpec{Devices: map[string][]dsv1.DeviceState{}} if err = json.Unmarshal([]byte(value), &deviceMap.Devices); err != nil { return &model.TerminalDevices{}, fmt.Errorf("failed to convert device map: %v", err) } terminalDevices := &model.TerminalDevices{Classes: make([]*model.ClassDeviceMap, len(deviceMap.Devices))} classIdx := 0 for class, classDevices := range deviceMap.Devices { devices := make([]*model.Device, len(classDevices)) devIdx := 0 for _, dev := range classDevices { devices[devIdx] = &model.Device{Name: dev.Name} devIdx++ } terminalDevices.Classes[classIdx] = &model.ClassDeviceMap{ Name: class, Devices: devices, } classIdx++ } return terminalDevices, nil } func (t *terminalService) checkHostnames(ctx context.Context, nodeName string, newTerminal *model.TerminalCreateInput) error { // get all of the hostnames in the cluster rows, err := t.SQLDB.QueryContext(ctx, sqlquery.GetAllHostnamesForAClusterQuery, newTerminal.ClusterEdgeID) if err != nil { return err } var hostname string for rows.Next() { err := rows.Scan(&hostname) if err != nil { return err } if hostname == nodeName { return fmt.Errorf("cannot create terminal - hostname %s already exists", hostname) } } if err := rows.Err(); err != nil { return sqlerr.Wrap(err) } return nil } func (t *terminalService) RemoveClusterEdgeBootstrapToken(ctx context.Context, clusterEdgeID string) error { if clusterEdgeID == "" { return nil } return t.uploadClusterEdgeBootstrapToken(ctx, "", clusterEdgeID) } func (t *terminalService) uploadClusterEdgeBootstrapToken(ctx context.Context, edgeBootstrapTokenHash, clusterEdgeID string) error { expirationTime := time.Now().UTC().Add(time.Hour * 2).Format(mapper.TimeFormat) _, err := t.SQLDB.ExecContext(ctx, sqlquery.UploadEdgeBootstrapToken, edgeBootstrapTokenHash, expirationTime, clusterEdgeID) return err } func (t *terminalService) checkLaneDuplicates(ctx context.Context, newLane, clusterEdgeID string) error { // check that a lane is unique for a cluster rows, err := t.SQLDB.QueryContext(ctx, sqlquery.GetAllLanesForAClusterQuery, clusterEdgeID) if err != nil { return err } for rows.Next() { var hostname string var lane sql.NullString if err := rows.Scan(&lane, &hostname); err != nil { return err } if lane.Valid && lane.String == newLane { return fmt.Errorf("cannot create terminal - lane %s already in use by terminal %s", newLane, hostname) } } if err := rows.Err(); err != nil { return sqlerr.Wrap(err) } return nil } func (t *terminalService) GetTerminalBootstrapConfig(ctx context.Context, terminal *model.Terminal, clusterNetworkServices []*model.ClusterNetworkServiceInfo, breakGlassSecret, grubSecret cc.Secret, bootstrapAck, isFirstNode bool, organization string, bootstrapTokenValues []*model.KeyValues, clusterCaHash, endpoint string) (string, error) { edgeInfraVersion := version.New().SemVer grubEntry := strings.Split(grubSecret.String(), "\n") if len(grubEntry) != 3 { return "", errors.New("invalid grub credential returned to bootstrap config") } grubLine := strings.Split(grubEntry[1], " ") if len(grubEntry) != 3 { return "", errors.New("invalid grub hash returned to bootstrap config") } payload := mapper.TerminalBootstrapPayload{ Terminal: terminal, EdgeInfraVersion: edgeInfraVersion, ClusterNetworkServices: clusterNetworkServices, ShadowFileLine: breakGlassSecret.String(), GrubFileEntry: grubLine[2], FirstNode: isFirstNode, BootstrapTokenValues: bootstrapTokenValues, ClusterCaHash: clusterCaHash, BootstrapAck: bootstrapAck, Organization: organization, Endpoint: endpoint, } if isFirstNode { edgeBootstrapToken, err := crypto.GenerateRandomEdgeBootstrapToken() if err != nil { return "", err } if err := t.uploadClusterEdgeBootstrapToken(ctx, hex.EncodeToString(edgeBootstrapToken.Hashed()), terminal.ClusterEdgeID); err != nil { return "", err } payload.EdgeBootstrapToken = edgeBootstrapToken.Plain() } config, err := payload.Yaml() if err != nil { return "", err } yaml, err := yaml.Marshal(&config) if err != nil { return "", err } return string(yaml), nil } func (t *terminalService) getProjectIDbyClusterEdgeID(ctx context.Context, clusterEdgeID string) (string, error) { var projectID string row := t.SQLDB.QueryRowContext(ctx, sqlquery.GetProjectIDByClusterEdgeID, clusterEdgeID) if err := row.Scan(&projectID); err != nil { return "", err } return projectID, nil } func (t *terminalService) getNodeResources(ctx context.Context, projectID string) ([]string, error) { // returns the terminal version from BQ and maps to the hostname logRequest := mapper.GetNodes() logRequest.GetClusterEdgeID = true return t.BQClient.GetKubeResource(ctx, projectID, nil, logRequest) } func (t *terminalService) getTerminalVersionMapFromProjectID(ctx context.Context, projectID string) (map[string]map[string]string, error) { // get node resources from BQ then create a map of hostnames to node versions nodes, err := t.getNodeResources(ctx, projectID) if err != nil { return nil, err } return mapper.GenerateTerminalVersionMap(nodes) } func (t *terminalService) updateTerminalVersions(ctx context.Context, terminals []*model.Terminal) ([]*model.Terminal, error) { if len(terminals) == 0 { return terminals, nil } projectID, err := t.getProjectIDbyClusterEdgeID(ctx, terminals[0].ClusterEdgeID) if err != nil { return nil, err } versionMap, err := t.getTerminalVersionMapFromProjectID(ctx, projectID) if err != nil { return nil, err } for _, terminal := range terminals { terminal.Version = versionMap[terminal.ClusterEdgeID][terminal.Hostname] } return terminals, nil } func NewTerminalService(sqlDB *sql.DB, labelSvc LabelService) TerminalService { return &terminalService{ SQLDB: sqlDB, LabelService: labelSvc, } } func NewTerminalServiceBQ(sqlDB *sql.DB, BQClient clients.BQClient, labelSvc LabelService) TerminalService { return &terminalService{ SQLDB: sqlDB, LabelService: labelSvc, BQClient: BQClient, } }