From 81c87ad56334d7b639c5d3076b27cbb58e347a21 Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Mon, 4 Feb 2019 15:31:36 +0100 Subject: [PATCH] Initial version --- .gitignore | 1 + auth.go | 38 ++++++++++++++ gcr.go | 151 +++++++++++++++++++++++++++++++++++++++++++++++++++++ main.go | 68 ++++++++++++++++++++++++ 4 files changed, 258 insertions(+) create mode 100644 .gitignore create mode 100644 auth.go create mode 100644 gcr.go create mode 100644 main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8dbcad6 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +gcr-clean diff --git a/auth.go b/auth.go new file mode 100644 index 0000000..d6f9051 --- /dev/null +++ b/auth.go @@ -0,0 +1,38 @@ +package main + +import ( + "io/ioutil" + "os" + + "github.com/genuinetools/reg/repoutils" + log "github.com/sirupsen/logrus" +) + +func getAuth() string { + if auth != "" { + return auth + } + + // If specified use Application Default Credentials + if _, err := os.Stat(cfg.GoogleApplicationCredentials); err == nil { + jsonData, err := ioutil.ReadFile(cfg.GoogleApplicationCredentials) + if err != nil { + log.WithError(err).Fatal("Unable to read GoogleApplicationCredentials file") + } + + auth = string(jsonData) + } + + // No luck yet? Try Docker auth + if auth == "" { + if ac, err := repoutils.GetAuthConfig("", "", cfg.Registry); err == nil && ac.Password != "" { + auth = ac.Password + } + } + + if auth == "" { + log.Fatal("No valid credentials found for registry") + } + + return auth +} diff --git a/gcr.go b/gcr.go new file mode 100644 index 0000000..69928bd --- /dev/null +++ b/gcr.go @@ -0,0 +1,151 @@ +package main + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + "strings" + "sync" + + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" +) + +func deleteTags(delChan <-chan deleteRequest, wg *sync.WaitGroup) { + limiter := make(chan struct{}, cfg.Parallel) + + for req := range delChan { + limiter <- struct{}{} + wg.Add(1) + + go func(req deleteRequest, limiter <-chan struct{}, wg *sync.WaitGroup) { + defer func() { <-limiter }() + defer wg.Done() + + logger := log.WithFields(log.Fields{ + "manifest": req.Digest, + "repo": req.Repo, + }) + + if !cfg.NoOp { + if _, err := request(http.MethodDelete, fmt.Sprintf("%s/manifests/%s", req.Repo, req.Digest)); err != nil { + logger.WithError(err).Error("Failed to delete manifest") + return + } + } + logger.WithField("noop", cfg.NoOp).Info("Manifest deleted") + }(req, limiter, wg) + } +} + +func fetchRepositories(projectIDs []string, delChan chan deleteRequest, wg *sync.WaitGroup) error { + defer wg.Done() + log.Info("Fetching repositories...") + + response := struct { + Repositories []string `json:"repositories"` + }{} + + body, err := request(http.MethodGet, "_catalog") + if err != nil { + return errors.Wrap(err, "Could not fetch catalog") + } + + if err := json.NewDecoder(body).Decode(&response); err != nil { + return errors.Wrap(err, "Unable to unmarshal JSON response") + } + + for _, repo := range response.Repositories { + process := false + for _, projectID := range projectIDs { + if strings.HasPrefix(repo, projectID) { + process = true + continue + } + } + + if !process { + log.WithField("repo", repo).Debug("Not in project scope, ignoring") + continue + } + + wg.Add(1) + go func(repo string) { + defer wg.Done() + if err := fetchUntaggedManifests(repo, delChan, wg); err != nil { + log.WithField("repo", repo).WithError(err).Error("Unable to fetch manifests") + } + }(repo) + } + + return nil +} + +func fetchUntaggedManifests(repo string, delChan chan deleteRequest, wg *sync.WaitGroup) error { + body, err := request(http.MethodGet, fmt.Sprintf("%s/tags/list", repo)) + if err != nil { + return errors.Wrap(err, "Unable to list tags") + } + + response := struct { + Manifests map[string]struct { + Tags []string `json:"tag"` + } `json:"manifest"` + }{} + + if err := json.NewDecoder(body).Decode(&response); err != nil { + return errors.Wrap(err, "Unable to unmarshal JSON response") + } + + for digest, info := range response.Manifests { + if len(info.Tags) == 0 { + delChan <- deleteRequest{repo, digest} + continue + } + + log.WithFields(log.Fields{ + "repo": repo, + "digest": digest, + "tags": len(info.Tags), + }).Debug("Manifest has tags, ignoring") + } + + return nil +} + +func request(method string, path string) (io.Reader, error) { + logger := log.WithFields(log.Fields{ + "method": method, + "path": path, + }) + + uri := fmt.Sprintf("https://%s/v2/%s", cfg.Registry, path) + + req, err := http.NewRequest(method, uri, nil) + if err != nil { + return nil, errors.Wrap(err, "Unable to create HTTP request") + } + + req.SetBasicAuth("_json_key", getAuth()) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + logger.WithError(err).Debug("HTTP request failed") + return nil, errors.Wrap(err, "HTTP request failed") + } + defer resp.Body.Close() + + if resp.StatusCode > 299 { + logger.WithField("status", resp.StatusCode).Debug("Status code indicated error") + return nil, errors.Errorf("HTTP request failed with status HTTP %d", resp.StatusCode) + } + + logger.Debug("Request success") + + buf := new(bytes.Buffer) + _, err = io.Copy(buf, resp.Body) + + return buf, errors.Wrap(err, "Unable to read response body") +} diff --git a/main.go b/main.go new file mode 100644 index 0000000..1ad50f7 --- /dev/null +++ b/main.go @@ -0,0 +1,68 @@ +package main + +import ( + "fmt" + "os" + "sync" + + log "github.com/sirupsen/logrus" + + "github.com/Luzifer/rconfig" +) + +type deleteRequest struct{ Repo, Digest string } + +var ( + cfg = struct { + GoogleApplicationCredentials string `flag:"account" default:"" description:"Path to account.json file with GCR access"` + Listen string `flag:"listen" default:":3000" description:"Port/IP to listen on"` + LogLevel string `flag:"log-level" default:"info" description:"Log level (debug, info, warn, error, fatal)"` + NoOp bool `flag:"noop,n" default:"true" description:"Do not execute destructive DELETE operation"` + Parallel int `flag:"parallel,p" default:"10" description:"How many deletions to execute in parallel"` + Registry string `flag:"registry" default:"gcr.io" description:"The registry used (gcr.io, eu.gcr.io, us.gcr.io, ...)"` + VersionAndExit bool `flag:"version" default:"false" description:"Prints current version and exits"` + }{} + + auth string + + version = "dev" +) + +func init() { + rconfig.AutoEnv(true) + if err := rconfig.ParseAndValidate(&cfg); err != nil { + log.Fatalf("Unable to parse commandline options: %s", err) + } + + if cfg.VersionAndExit { + fmt.Printf("gcr-clean %s\n", version) + os.Exit(0) + } + + if l, err := log.ParseLevel(cfg.LogLevel); err != nil { + log.WithError(err).Fatal("Unable to parse log level") + } else { + log.SetLevel(l) + } +} + +func main() { + args := rconfig.Args() + if len(args) < 2 { + log.Fatal("Expecting one or more positional arguments: gcr-clean [project id...]") + } + + var ( + wg = new(sync.WaitGroup) + delChan = make(chan deleteRequest, 1000) + ) + + go deleteTags(delChan, wg) + + wg.Add(1) + if err := fetchRepositories(args[1:], delChan, wg); err != nil { + log.WithError(err).Error("An error occurred while fetching repos") + } + + wg.Wait() +}