From b38cd766abcfef677e7e4f95b05fd3bd36f576af Mon Sep 17 00:00:00 2001 From: Knut Ahlers Date: Sat, 27 Jul 2019 02:16:03 +0200 Subject: [PATCH] First running version --- .gitignore | 2 + gitea.go | 58 ++++++++++++++++++ go.mod | 15 +++++ go.sum | 62 +++++++++++++++++++ main.go | 177 +++++++++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 314 insertions(+) create mode 100644 .gitignore create mode 100644 gitea.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..8443666 --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +create-gitea-migration +.env diff --git a/gitea.go b/gitea.go new file mode 100644 index 0000000..5862006 --- /dev/null +++ b/gitea.go @@ -0,0 +1,58 @@ +package main + +import ( + "net/url" + + "github.com/google/go-github/github" +) + +type createMigrationRequest struct { + AuthPassword string `json:"auth_password"` + AuthUsername string `json:"auth_username"` + CloneAddr string `json:"clone_addr"` + Description string `json:"description"` + Issues bool `json:"issues"` + //Labels bool `json:"labels"` + //Milestones bool `json:"milestones"` + Mirror bool `json:"mirror"` + Private bool `json:"private"` + PullRequests bool `json:"pull_requests"` + //Releases bool `json:"releases"` + RepoName string `json:"repo_name"` + Uid int64 `json:"uid"` + Wiki bool `json:"wiki"` +} + +func createMigrationRequestFromGithubRepo(gr *github.Repository) createMigrationRequest { + cmr := createMigrationRequest{ + CloneAddr: strFromPtr(gr.CloneURL), + Description: strFromPtr(gr.Description), + Issues: boolFromPtr(gr.HasIssues), + Mirror: true, + Private: boolFromPtr(gr.Private), + PullRequests: boolFromPtr(gr.HasIssues), + RepoName: strFromPtr(gr.Name), + Uid: cfg.TargetUser, + Wiki: boolFromPtr(gr.HasWiki), + } + + if boolFromPtr(gr.Private) { + uri, _ := url.Parse(strFromPtr(gr.CloneURL)) + uri.User = url.UserPassword("api", cfg.GithubToken) + cmr.CloneAddr = uri.String() + } + + return cmr +} + +func boolFromPtr(in *bool) bool { + return in != nil && *in +} + +func strFromPtr(in *string) string { + if in == nil { + return "" + } + + return *in +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..acb2c39 --- /dev/null +++ b/go.mod @@ -0,0 +1,15 @@ +module github.com/Luzifer/create-gitea-migration + +go 1.12 + +require ( + github.com/Luzifer/rconfig v2.2.0+incompatible // indirect + github.com/Luzifer/rconfig/v2 v2.2.1 + github.com/google/go-github v17.0.0+incompatible + github.com/google/go-querystring v1.0.0 // indirect + github.com/onsi/ginkgo v1.8.0 // indirect + github.com/onsi/gomega v1.5.0 // indirect + github.com/pkg/errors v0.8.1 + github.com/sirupsen/logrus v1.4.2 + golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..950e6b2 --- /dev/null +++ b/go.sum @@ -0,0 +1,62 @@ +cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= +github.com/Luzifer/rconfig v2.2.0+incompatible h1:Kle3+rshPM7LxciOheaR4EfHUzibkDDGws04sefQ5m8= +github.com/Luzifer/rconfig v2.2.0+incompatible/go.mod h1:9pet6z2+mm/UAB0jF/rf0s62USfHNolzgR6Q4KpsJI0= +github.com/Luzifer/rconfig/v2 v2.2.1 h1:zcDdLQlnlzwcBJ8E0WFzOkQE1pCMn3EbX0dFYkeTczg= +github.com/Luzifer/rconfig/v2 v2.2.1/go.mod h1:OKIX0/JRZrPJ/ZXXWklQEFXA6tBfWaljZbW37w+sqBw= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= +github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= +github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= +github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.8.0 h1:VkHVNpR4iVnU8XQR6DBm8BqYjN7CRzw+xKUbVVbbW9w= +github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.5.0 h1:izbySO9zDPmjJ8rDjLvkA2zJHIo+HkYXHnf7eN7SSyo= +github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd h1:nTDtHvHSdCn1m6ITfMRqtOd/9+7a3s8RBNOZ3eYZzJA= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e h1:bRhVy7zSSasaqNksaRZiA5EEI+Ei4I1nO5Jh72wfHlg= +golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f h1:wMNYb4v58l5UBM7MYRLPG6ZhfOqbKu7X5eyFl8ZhKvA= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/validator.v2 v2.0.0-20180514200540-135c24b11c19 h1:WB265cn5OpO+hK3pikC9hpP1zI/KTwmyMFKloW9eOVc= +gopkg.in/validator.v2 v2.0.0-20180514200540-135c24b11c19/go.mod h1:o4V0GXN9/CAmCsvJ0oXYZvrZOe7syiDZSN1GWGZTGzc= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go new file mode 100644 index 0000000..ddb50f1 --- /dev/null +++ b/main.go @@ -0,0 +1,177 @@ +package main + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "os" + "regexp" + "strings" + + "github.com/Luzifer/rconfig/v2" + "github.com/google/go-github/github" + "github.com/pkg/errors" + log "github.com/sirupsen/logrus" + "golang.org/x/oauth2" +) + +var ( + cfg = struct { + DryRun bool `flag:"dry-run,n" default:"false" description:"Only report actions to be done, don't execute them"` + GiteaToken string `flag:"gitea-token" default:"" description:"Token to interact with Gitea instance" validate:"nonzero"` + GiteaURL string `flag:"gitea-url" default:"" description:"URL of the Gitea instance" validate:"nonzero"` + GithubToken string `flag:"github-token" default:"" description:"Github access token" validate:"nonzero"` + LogLevel string `flag:"log-level" default:"info" description:"Log level (debug, info, warn, error, fatal)"` + MigratePrivate bool `flag:"migrate-private" default:"true" description:"Migrate private repos (the given Github Token will be entered as sync credential!)"` + SourceExpression string `flag:"source-expression" default:"" description:"Regular expression to match the full name of the source repo (i.e. '^Luzifer/.*$')" validate:"nonzero"` + TargetUser int64 `flag:"target-user" default:"0" description:"ID of the User / Organization in Gitea to assign the repo to" validate:"nonzero"` + TargetUserName string `flag:"target-user-name" default:"" description:"Username of the given ID (to check whether repo already exists)" validate:"nonzero"` + VersionAndExit bool `flag:"version" default:"false" description:"Prints current version and exits"` + }{} + + version = "dev" +) + +func init() { + rconfig.AutoEnv(true) + if err := rconfig.ParseAndValidate(&cfg); err != nil { + log.Fatalf("Unable to parse commandline options: %s", err) + } + + if cfg.VersionAndExit { + fmt.Printf("create-gitea-migration %s\n", version) + os.Exit(0) + } + + if l, err := log.ParseLevel(cfg.LogLevel); err != nil { + log.WithError(err).Fatal("Unable to parse log level") + } else { + log.SetLevel(l) + } +} + +func main() { + log.WithFields(log.Fields{ + "dry-run": cfg.DryRun, + "source": cfg.SourceExpression, + "target-user": cfg.TargetUserName, + "version": version, + }).Info("create-gitea-migration started") + + log.Info("Collecting source repos...") + repos, err := fetchGithubRepos() + if err != nil { + log.WithError(err).Fatal("Failed to fetch repos") + } + + log.Info("Creating target repos...") + for _, r := range repos { + if err := giteaCreateMigration(r); err != nil { + log.WithError(err).Error("Unable to create mirror") + } + } +} + +func fetchGithubRepos() ([]*github.Repository, error) { + ctx := context.Background() + + ts := oauth2.StaticTokenSource( + &oauth2.Token{AccessToken: cfg.GithubToken}, + ) + tc := oauth2.NewClient(ctx, ts) + + client := github.NewClient(tc) + + opt := &github.RepositoryListOptions{ + ListOptions: github.ListOptions{PerPage: 100}, + } + + sourceExpr := regexp.MustCompile(cfg.SourceExpression) + + // get all pages of results + var allRepos []*github.Repository + for { + repos, resp, err := client.Repositories.List(ctx, "", opt) + if err != nil { + return nil, errors.Wrap(err, "Unable to list repos") + } + + for _, r := range repos { + if !sourceExpr.MatchString(*r.FullName) { + continue + } + + allRepos = append(allRepos, r) + } + + if resp.NextPage == 0 { + break + } + + opt.Page = resp.NextPage + } + + return allRepos, nil +} + +func giteaCreateMigration(gr *github.Repository) error { + logger := log.WithFields(log.Fields{ + "repo": strFromPtr(gr.Name), + "private": boolFromPtr(gr.Private), + }) + + req, _ := http.NewRequest(http.MethodGet, giteaURL(strings.Join([]string{"api/v1/repos", cfg.TargetUserName, strFromPtr(gr.Name)}, "/")), nil) + req.Header.Set("Authorization", "token "+cfg.GiteaToken) + + resp, err := http.DefaultClient.Do(req) + if err != nil { + return errors.Wrap(err, "Unable to create repo in Gitea") + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + logger.Info("Repo already exists, no action required") + return nil + } + + cmr := createMigrationRequestFromGithubRepo(gr) + + body := new(bytes.Buffer) + if err := json.NewEncoder(body).Encode(cmr); err != nil { + return errors.Wrap(err, "Unable to marshal creation request") + } + + if cfg.DryRun { + logger.Warn("Repo not found, will be created in real run (dry-run enabled)") + return nil + } + + logger.Info("Repo not found, creating") + + req, _ = http.NewRequest(http.MethodPost, giteaURL("api/v1/repos/migrate"), body) + req.Header.Set("Content-Type", "application/json") + req.Header.Set("Authorization", "token "+cfg.GiteaToken) + + resp, err = http.DefaultClient.Do(req) + if err != nil { + return errors.Wrap(err, "Unable to create repo in Gitea") + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusCreated { + body, _ := ioutil.ReadAll(resp.Body) + return errors.Errorf("Unable to create repo in Gitea: Status %d: %s", resp.StatusCode, body) + } + + return nil +} + +func giteaURL(path string) string { + return strings.Join([]string{ + strings.TrimRight(cfg.GiteaURL, "/"), + strings.TrimLeft(path, "/"), + }, "/") +}