Initial version

.gitignore

@@ -0,0 +1 @@
+gcr-clean

auth.go

@@ -0,0 +1,38 @@
+package main
+
+import (
+	"io/ioutil"
+	"os"
+
+	"github.com/genuinetools/reg/repoutils"
+	log "github.com/sirupsen/logrus"
+)
+
+func getAuth() string {
+	if auth != "" {
+		return auth
+	}
+
+	// If specified use Application Default Credentials
+	if _, err := os.Stat(cfg.GoogleApplicationCredentials); err == nil {
+		jsonData, err := ioutil.ReadFile(cfg.GoogleApplicationCredentials)
+		if err != nil {
+			log.WithError(err).Fatal("Unable to read GoogleApplicationCredentials file")
+		}
+
+		auth = string(jsonData)
+	}
+
+	// No luck yet? Try Docker auth
+	if auth == "" {
+		if ac, err := repoutils.GetAuthConfig("", "", cfg.Registry); err == nil && ac.Password != "" {
+			auth = ac.Password
+		}
+	}
+
+	if auth == "" {
+		log.Fatal("No valid credentials found for registry")
+	}
+
+	return auth
+}

gcr.go

@@ -0,0 +1,151 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"strings"
+	"sync"
+
+	"github.com/pkg/errors"
+	log "github.com/sirupsen/logrus"
+)
+
+func deleteTags(delChan <-chan deleteRequest, wg *sync.WaitGroup) {
+	limiter := make(chan struct{}, cfg.Parallel)
+
+	for req := range delChan {
+		limiter <- struct{}{}
+		wg.Add(1)
+
+		go func(req deleteRequest, limiter <-chan struct{}, wg *sync.WaitGroup) {
+			defer func() { <-limiter }()
+			defer wg.Done()
+
+			logger := log.WithFields(log.Fields{
+				"manifest": req.Digest,
+				"repo":     req.Repo,
+			})
+
+			if !cfg.NoOp {
+				if _, err := request(http.MethodDelete, fmt.Sprintf("%s/manifests/%s", req.Repo, req.Digest)); err != nil {
+					logger.WithError(err).Error("Failed to delete manifest")
+					return
+				}
+			}
+			logger.WithField("noop", cfg.NoOp).Info("Manifest deleted")
+		}(req, limiter, wg)
+	}
+}
+
+func fetchRepositories(projectIDs []string, delChan chan deleteRequest, wg *sync.WaitGroup) error {
+	defer wg.Done()
+	log.Info("Fetching repositories...")
+
+	response := struct {
+		Repositories []string `json:"repositories"`
+	}{}
+
+	body, err := request(http.MethodGet, "_catalog")
+	if err != nil {
+		return errors.Wrap(err, "Could not fetch catalog")
+	}
+
+	if err := json.NewDecoder(body).Decode(&response); err != nil {
+		return errors.Wrap(err, "Unable to unmarshal JSON response")
+	}
+
+	for _, repo := range response.Repositories {
+		process := false
+		for _, projectID := range projectIDs {
+			if strings.HasPrefix(repo, projectID) {
+				process = true
+				continue
+			}
+		}
+
+		if !process {
+			log.WithField("repo", repo).Debug("Not in project scope, ignoring")
+			continue
+		}
+
+		wg.Add(1)
+		go func(repo string) {
+			defer wg.Done()
+			if err := fetchUntaggedManifests(repo, delChan, wg); err != nil {
+				log.WithField("repo", repo).WithError(err).Error("Unable to fetch manifests")
+			}
+		}(repo)
+	}
+
+	return nil
+}
+
+func fetchUntaggedManifests(repo string, delChan chan deleteRequest, wg *sync.WaitGroup) error {
+	body, err := request(http.MethodGet, fmt.Sprintf("%s/tags/list", repo))
+	if err != nil {
+		return errors.Wrap(err, "Unable to list tags")
+	}
+
+	response := struct {
+		Manifests map[string]struct {
+			Tags []string `json:"tag"`
+		} `json:"manifest"`
+	}{}
+
+	if err := json.NewDecoder(body).Decode(&response); err != nil {
+		return errors.Wrap(err, "Unable to unmarshal JSON response")
+	}
+
+	for digest, info := range response.Manifests {
+		if len(info.Tags) == 0 {
+			delChan <- deleteRequest{repo, digest}
+			continue
+		}
+
+		log.WithFields(log.Fields{
+			"repo":   repo,
+			"digest": digest,
+			"tags":   len(info.Tags),
+		}).Debug("Manifest has tags, ignoring")
+	}
+
+	return nil
+}
+
+func request(method string, path string) (io.Reader, error) {
+	logger := log.WithFields(log.Fields{
+		"method": method,
+		"path":   path,
+	})
+
+	uri := fmt.Sprintf("https://%s/v2/%s", cfg.Registry, path)
+
+	req, err := http.NewRequest(method, uri, nil)
+	if err != nil {
+		return nil, errors.Wrap(err, "Unable to create HTTP request")
+	}
+
+	req.SetBasicAuth("_json_key", getAuth())
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		logger.WithError(err).Debug("HTTP request failed")
+		return nil, errors.Wrap(err, "HTTP request failed")
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode > 299 {
+		logger.WithField("status", resp.StatusCode).Debug("Status code indicated error")
+		return nil, errors.Errorf("HTTP request failed with status HTTP %d", resp.StatusCode)
+	}
+
+	logger.Debug("Request success")
+
+	buf := new(bytes.Buffer)
+	_, err = io.Copy(buf, resp.Body)
+
+	return buf, errors.Wrap(err, "Unable to read response body")
+}

main.go

@@ -0,0 +1,68 @@
+package main
+
+import (
+	"fmt"
+	"os"
+	"sync"
+
+	log "github.com/sirupsen/logrus"
+
+	"github.com/Luzifer/rconfig"
+)
+
+type deleteRequest struct{ Repo, Digest string }
+
+var (
+	cfg = struct {
+		GoogleApplicationCredentials string `flag:"account" default:"" description:"Path to account.json file with GCR access"`
+		Listen                       string `flag:"listen" default:":3000" description:"Port/IP to listen on"`
+		LogLevel                     string `flag:"log-level" default:"info" description:"Log level (debug, info, warn, error, fatal)"`
+		NoOp                         bool   `flag:"noop,n" default:"true" description:"Do not execute destructive DELETE operation"`
+		Parallel                     int    `flag:"parallel,p" default:"10" description:"How many deletions to execute in parallel"`
+		Registry                     string `flag:"registry" default:"gcr.io" description:"The registry used (gcr.io, eu.gcr.io, us.gcr.io, ...)"`
+		VersionAndExit               bool   `flag:"version" default:"false" description:"Prints current version and exits"`
+	}{}
+
+	auth string
+
+	version = "dev"
+)
+
+func init() {
+	rconfig.AutoEnv(true)
+	if err := rconfig.ParseAndValidate(&cfg); err != nil {
+		log.Fatalf("Unable to parse commandline options: %s", err)
+	}
+
+	if cfg.VersionAndExit {
+		fmt.Printf("gcr-clean %s\n", version)
+		os.Exit(0)
+	}
+
+	if l, err := log.ParseLevel(cfg.LogLevel); err != nil {
+		log.WithError(err).Fatal("Unable to parse log level")
+	} else {
+		log.SetLevel(l)
+	}
+}
+
+func main() {
+	args := rconfig.Args()
+	if len(args) < 2 {
+		log.Fatal("Expecting one or more positional arguments: gcr-clean <project id> [project id...]")
+	}
+
+	var (
+		wg      = new(sync.WaitGroup)
+		delChan = make(chan deleteRequest, 1000)
+	)
+
+	go deleteTags(delChan, wg)
+
+	wg.Add(1)
+	if err := fetchRepositories(args[1:], delChan, wg); err != nil {
+		log.WithError(err).Error("An error occurred while fetching repos")
+	}
+
+	wg.Wait()
+}