package ipranger import ( "context" "errors" "fmt" "net/netip" "strconv" "strings" "sync" "go4.org/netipx" "google.golang.org/api/compute/v1" ) const Netmask = 21 const rangePerSubnet = 30 const rangePerRequest = 2 const allIPs = "10.0.0.0/8" const gcpIPs = "10.128.0.0/9" type IPRanger struct { mu sync.Mutex subnetSvc computeSubnetSvc // stores maps gcp projects to subnet range information stores map[string]*rangeStore } type Subnet struct { Network string Name string Ref string region string aliasRangeQuota int ipRange string } type rangeStore struct { // subnets map is e.g.: // region0: // subnet0: {} // subnet1: {} // region1: // subnet2: {} subnets map[string]map[string]Subnet ipset *netipx.IPSet } func New(ctx context.Context) (*IPRanger, error) { subnetSvc, err := newGCPSubnetService(ctx) if err != nil { return nil, err } return &IPRanger{ subnetSvc: subnetSvc, }, nil } // FindOrCreateSubnet returns an available subnet if one exists, otherwise // it creates and returns information about a new one. This is the primary // function that should be used by callers, as it controls concurrent access // to the underlying state and performs the entire sequence of operations // needed to get a suitable subnet func (r *IPRanger) FindOrCreateSubnet(project, region string) (Subnet, error) { r.mu.Lock() defer r.mu.Unlock() // get current state from gcp err := r.InitState(project) if err != nil { return Subnet{}, fmt.Errorf("failed to initialize state. err: %v", err) } // find subnet with range quota ok, s := r.FindAvailableSubnet(project, region) if ok { return s, nil } // if no subnets have quota, create new one s, err = r.CreateNewSubnet(project, region) if err != nil { return Subnet{}, fmt.Errorf("failed to create subnet. region: %v err: %v", region, err) } return s, nil } // CreateNewSubnet needs no introduction func (r *IPRanger) CreateNewSubnet(project, region string) (Subnet, error) { // find unused subnet ip range prefix, _, ok := r.stores[project].ipset.RemoveFreePrefix(Netmask) if !ok { return Subnet{}, errors.New("no ip ranges remaining") } name := r.nextSubnetName(project) newSubnet, err := r.subnetSvc.Create(project, prefix.String(), region, name) if err != nil { return Subnet{}, err } // if successful, update store to use ipset with new range removed // and include new subnet // r.store.ipset = newSet // r.store.subnets[region][subnet.Name] = subnet return fromComputeSubnetwork(newSubnet), nil } // FindAvailableSubnet locates a subnet that has enough spare secondary ranges // to be suitable for use in the creation of a gke cluster func (r *IPRanger) FindAvailableSubnet(project, region string) (ok bool, s Subnet) { // find existing subnet with available alias range quota subnets := r.stores[project].subnets[region] if subnets == nil { return false, Subnet{} } for _, s := range subnets { if s.aliasRangeQuota >= rangePerRequest { return true, s } } return false, Subnet{} } // nextSubnetName finds the next unused name that can be used to create // a new subnet func (r *IPRanger) nextSubnetName(project string) string { var next int64 for _, v := range r.stores[project].subnets { for name := range v { snum := strings.TrimPrefix(name, "subnet") if snum == name { // name wasnt prefixed with "subnet", maybe "default", thats fine // since it wont overlap with this naming scheme continue } // if we cant parse a number, the name was malformed and thus wont // collide with the new name we provide. thats also fine num, _ := strconv.ParseInt(snum, 10, 64) if num >= next { // if max is N, next num is N+1 next = num + 1 } } } return fmt.Sprintf("subnet%d", next) } func fromComputeSubnetwork(csn *compute.Subnetwork) Subnet { return Subnet{ Network: csn.Network, Name: csn.Name, Ref: csn.SelfLink, region: fromRegionURI(csn.Region), aliasRangeQuota: rangePerSubnet - len(csn.SecondaryIpRanges), ipRange: csn.IpCidrRange, } } // buildRangeStore converts gcp types to a local representation with additional // information, like a processable ipset func buildRangeStore(subnets computeSubnetAggregatedList) (*rangeStore, error) { store := &rangeStore{ subnets: map[string]map[string]Subnet{}, } var existingIPs netipx.IPSetBuilder for regionRegion, v := range subnets { // aggregated list keys are e.g. regions/us-east1 rr := strings.Split(regionRegion, "/") if len(rr) != 2 { return nil, fmt.Errorf("invalid region key %v. expected 'regions/$region'", regionRegion) } region := rr[1] store.subnets[region] = map[string]Subnet{} // each region key in region->subnets for _, sn := range v.Subnetworks { subnet := fromComputeSubnetwork(sn) // parse ip cidr ranges into set subnetPrefix, err := netip.ParsePrefix(subnet.ipRange) if err != nil { return nil, fmt.Errorf("failed to parse ip and prefix from subnet range '%s', err: %v", subnet.ipRange, err) } existingIPs.AddPrefix(subnetPrefix) for _, aliasRange := range sn.SecondaryIpRanges { aliasPrefix, err := netip.ParsePrefix(aliasRange.IpCidrRange) if err != nil { return nil, fmt.Errorf("failed to parse ip and prefix from secondary range '%s', err: %v", aliasRange.IpCidrRange, err) } existingIPs.AddPrefix(aliasPrefix) } store.subnets[region][subnet.Name] = subnet } } existingSet, err := existingIPs.IPSet() if err != nil { return nil, fmt.Errorf("failed to build IPSet, err: %v", err) } var availableIPs netipx.IPSetBuilder // start with all private ips availableIPs.AddPrefix(netip.MustParsePrefix(allIPs)) // remove in use ranges availableIPs.RemoveSet(existingSet) // remove ips reserved by gcp availableIPs.RemovePrefix(netip.MustParsePrefix(gcpIPs)) // build available set from what remains availableSet, err := availableIPs.IPSet() if err != nil { return nil, err } store.ipset = availableSet return store, nil } // initState fetches the current state of networks from gcp and builds // a local representation for further processing func (r *IPRanger) InitState(project string) error { if r.stores == nil { r.stores = make(map[string]*rangeStore) } subnets, err := r.subnetSvc.List(project) if err != nil { return fmt.Errorf("failed to list subnets. err: %v", err) } store, err := buildRangeStore(subnets) if err != nil { return err } r.stores[project] = store return nil } func fromRegionURI(uri string) string { split := strings.Split(uri, "/") return split[len(split)-1] } func toRegionURI(project, region string) string { return fmt.Sprintf("https://www.googleapis.com/compute/v1/projects/%s/regions/%s", project, region) }