main.go

Documentation: edge-infra.dev/hack/tools/gcs-file-cleaner

     1  package main
     2  
     3  import (
     4  	"context"
     5  	"fmt"
     6  	"io"
     7  	"os"
     8  	"strings"
     9  
    10  	"cloud.google.com/go/storage"
    11  
    12  	"google.golang.org/api/iterator"
    13  )
    14  
    15  var buckets = []string{
    16  	// select project_id from banners
    17  	"ret-edge-nw6y96roiq14yk81p58g2",
    18  	"ret-edge-2rvkbyt0pspbaunh79jv2",
    19  	"ret-edge-v4hsjv7qedilr89s9wx5q",
    20  	"ret-edge-m8yzq5ckmrg0xbywbx5pn",
    21  	"ret-edge-tf31hknvuc8chx7ii96rw",
    22  	"ret-edge-0p3gmoszouh9lrghf6nj5",
    23  	"ret-edge-f94mtrmimvlypgpcxp0nw",
    24  	"ret-edge-782vn2w9xibt9ycx4643f",
    25  	"ret-edge-mb17p8vt99u3bthmfbkug",
    26  	"ret-edge-3u27t1ttkk4ngtkmk842b",
    27  	"ret-edge-qlc2lkjpbdh01ndmowrj2",
    28  	"ret-edge-b79we3ikmc7j9mihuwst2",
    29  	"ret-edge-kijwfr92bmik7lyifh9kw",
    30  	"ret-edge-q40buawbn75r8em5rs8yt",
    31  }
    32  
    33  type gcsFileModifier = func(ctx context.Context, obj *storage.ObjectHandle, file, filename string)
    34  
    35  func main() {
    36  	//os.Setenv("GOOGLE_APPLICATION_CREDENTIALS", "<chariot-sa>") may not be needed
    37  
    38  	// TODO use these gcsFileModifier function in order: 1) replaceExternalSecretAPIVersion 2) deleteOldExternalSecret
    39  	var apply gcsFileModifier = deleteInvalidKustomization
    40  	filter := "fluxcfg"
    41  
    42  	ctx := context.Background()
    43  	// Determine storage options
    44  	cl, err := storage.NewClient(ctx)
    45  	if err != nil {
    46  		fmt.Println("failed to create gcs storage client", err)
    47  		os.Exit(1)
    48  	}
    49  
    50  	for _, b := range buckets {
    51  		fmt.Println("processing bucket:", b)
    52  		bucket := cl.Bucket(b)
    53  		it := bucket.Objects(ctx, nil)
    54  		for {
    55  			attrs, err := it.Next()
    56  			if err == iterator.Done {
    57  				break
    58  			}
    59  			if err != nil {
    60  				fmt.Println("failed to list file in gcs bucket", b, err)
    61  				panic(err)
    62  			}
    63  			if strings.Contains(attrs.Name, filter) { //nolint: nestif
    64  				obj := bucket.Object(attrs.Name)
    65  				rdr, err := obj.NewReader(ctx)
    66  				if err != nil {
    67  					fmt.Println("failed to create reader for file in chariot folder", attrs.Name, err)
    68  					os.Exit(1)
    69  				}
    70  				data, err := io.ReadAll(rdr)
    71  				if err != nil || rdr.Close() != nil {
    72  					fmt.Println("failed to read file in chariot folder", attrs.Name, err)
    73  					os.Exit(1)
    74  				}
    75  				apply(ctx, obj, attrs.Name, string(data))
    76  			}
    77  		}
    78  	}
    79  }
    80  
    81  // deleteOldExternalSecret delete old external secrets i.e. external-secrets.io/v1alpha1
    82  func deleteOldExternalSecret(ctx context.Context, obj *storage.ObjectHandle, filename, fileData string) { //nolint
    83  	if strings.Contains(fileData, "{\"kind\":\"ExternalSecret\"") &&
    84  		strings.Contains(fileData, "\"apiVersion\":\"external-secrets.io/v1alpha1\"") {
    85  		printFile(filename, fileData)
    86  		//if err := obj.Delete(ctx); err != nil {
    87  		//	fmt.Println("deleteOldExternalSecret: failed to delete", filename, err)
    88  		//	os.Exit(1)
    89  		//}
    90  	}
    91  }
    92  
    93  // deleteInvalidKustomization delete invalid Kustomization
    94  func deleteInvalidKustomization(ctx context.Context, obj *storage.ObjectHandle, filename, fileData string) { //nolint
    95  	if strings.Contains(fileData, "{\"kind\":\"Kustomization\"") &&
    96  		strings.Contains(fileData, "\"apiVersion\":\"v1beta2\"") {
    97  		printFile(filename, fileData)
    98  		if err := obj.Delete(ctx); err != nil {
    99  			fmt.Println("deleteInvalidKustomization: failed to delete", filename, err)
   100  			os.Exit(1)
   101  		}
   102  	}
   103  }
   104  
   105  // removeClusterStatus delete Cluster status field
   106  func removeClusterStatus(ctx context.Context, obj *storage.ObjectHandle, filename, fileData string) { //nolint
   107  	if strings.Contains(fileData, "{\"kind\":\"Cluster\"") {
   108  		printFile(filename, fileData)
   109  		newYaml := strings.ReplaceAll(fileData, ",\"status\":{\"ready\":false,\"message\":\"\"}", "")
   110  		wc := obj.NewWriter(ctx)
   111  		_, err := wc.Write([]byte(newYaml))
   112  		if err != nil {
   113  			fmt.Println("removeClusterStatus: failed to write to", filename, err)
   114  			os.Exit(1)
   115  		}
   116  		if err := wc.Close(); err != nil {
   117  			fmt.Println("removeClusterStatus: failed to close", filename, err)
   118  			os.Exit(1)
   119  		}
   120  	}
   121  }
   122  
   123  // printFile keep track of modified files just in case, customize this to your liking
   124  func printFile(filename, fileData string) {
   125  	fmt.Println(filename, "------------------------------------------")
   126  	fmt.Println(fileData)
   127  	fmt.Println("------------------------------------------")
   128  }
   129
View as plain text