diff --git a/cmd/dump_repo.go b/cmd/dump_repo.go
new file mode 100644
index 0000000000..cea640b534
--- /dev/null
+++ b/cmd/dump_repo.go
@@ -0,0 +1,162 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cmd
+
+import (
+ "context"
+ "errors"
+ "strings"
+
+ "code.gitea.io/gitea/modules/convert"
+ "code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/migrations"
+ "code.gitea.io/gitea/modules/migrations/base"
+ "code.gitea.io/gitea/modules/setting"
+ "code.gitea.io/gitea/modules/structs"
+
+ "github.com/urfave/cli"
+)
+
+// CmdDumpRepository represents the available dump repository sub-command.
+var CmdDumpRepository = cli.Command{
+ Name: "dump-repo",
+ Usage: "Dump the repository from git/github/gitea/gitlab",
+ Description: "This is a command for dumping the repository data.",
+ Action: runDumpRepository,
+ Flags: []cli.Flag{
+ cli.StringFlag{
+ Name: "git_service",
+ Value: "",
+ Usage: "Git service, git, github, gitea, gitlab. If clone_addr could be recognized, this could be ignored.",
+ },
+ cli.StringFlag{
+ Name: "repo_dir, r",
+ Value: "./data",
+ Usage: "Repository dir path to store the data",
+ },
+ cli.StringFlag{
+ Name: "clone_addr",
+ Value: "",
+ Usage: "The URL will be clone, currently could be a git/github/gitea/gitlab http/https URL",
+ },
+ cli.StringFlag{
+ Name: "auth_username",
+ Value: "",
+ Usage: "The username to visit the clone_addr",
+ },
+ cli.StringFlag{
+ Name: "auth_password",
+ Value: "",
+ Usage: "The password to visit the clone_addr",
+ },
+ cli.StringFlag{
+ Name: "auth_token",
+ Value: "",
+ Usage: "The personal token to visit the clone_addr",
+ },
+ cli.StringFlag{
+ Name: "owner_name",
+ Value: "",
+ Usage: "The data will be stored on a directory with owner name if not empty",
+ },
+ cli.StringFlag{
+ Name: "repo_name",
+ Value: "",
+ Usage: "The data will be stored on a directory with repository name if not empty",
+ },
+ cli.StringFlag{
+ Name: "units",
+ Value: "",
+ Usage: `Which items will be migrated, one or more units should be separated as comma.
+wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.`,
+ },
+ },
+}
+
+func runDumpRepository(ctx *cli.Context) error {
+ if err := initDB(); err != nil {
+ return err
+ }
+
+ log.Trace("AppPath: %s", setting.AppPath)
+ log.Trace("AppWorkPath: %s", setting.AppWorkPath)
+ log.Trace("Custom path: %s", setting.CustomPath)
+ log.Trace("Log path: %s", setting.LogRootPath)
+ setting.InitDBConfig()
+
+ var (
+ serviceType structs.GitServiceType
+ cloneAddr = ctx.String("clone_addr")
+ serviceStr = ctx.String("git_service")
+ )
+
+ if strings.HasPrefix(strings.ToLower(cloneAddr), "https://github.com/") {
+ serviceStr = "github"
+ } else if strings.HasPrefix(strings.ToLower(cloneAddr), "https://gitlab.com/") {
+ serviceStr = "gitlab"
+ } else if strings.HasPrefix(strings.ToLower(cloneAddr), "https://gitea.com/") {
+ serviceStr = "gitea"
+ }
+ if serviceStr == "" {
+ return errors.New("git_service missed or clone_addr cannot be recognized")
+ }
+ serviceType = convert.ToGitServiceType(serviceStr)
+
+ var opts = base.MigrateOptions{
+ GitServiceType: serviceType,
+ CloneAddr: cloneAddr,
+ AuthUsername: ctx.String("auth_username"),
+ AuthPassword: ctx.String("auth_password"),
+ AuthToken: ctx.String("auth_token"),
+ RepoName: ctx.String("repo_name"),
+ }
+
+ if len(ctx.String("units")) == 0 {
+ opts.Wiki = true
+ opts.Issues = true
+ opts.Milestones = true
+ opts.Labels = true
+ opts.Releases = true
+ opts.Comments = true
+ opts.PullRequests = true
+ opts.ReleaseAssets = true
+ } else {
+ units := strings.Split(ctx.String("units"), ",")
+ for _, unit := range units {
+ switch strings.ToLower(unit) {
+ case "wiki":
+ opts.Wiki = true
+ case "issues":
+ opts.Issues = true
+ case "milestones":
+ opts.Milestones = true
+ case "labels":
+ opts.Labels = true
+ case "releases":
+ opts.Releases = true
+ case "release_assets":
+ opts.ReleaseAssets = true
+ case "comments":
+ opts.Comments = true
+ case "pull_requests":
+ opts.PullRequests = true
+ }
+ }
+ }
+
+ if err := migrations.DumpRepository(
+ context.Background(),
+ ctx.String("repo_dir"),
+ ctx.String("owner_name"),
+ opts,
+ ); err != nil {
+ log.Fatal("Failed to dump repository: %v", err)
+ return err
+ }
+
+ log.Trace("Dump finished!!!")
+
+ return nil
+}
diff --git a/cmd/restore_repo.go b/cmd/restore_repo.go
new file mode 100644
index 0000000000..541995879b
--- /dev/null
+++ b/cmd/restore_repo.go
@@ -0,0 +1,119 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cmd
+
+import (
+ "context"
+ "strings"
+
+ "code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/migrations"
+ "code.gitea.io/gitea/modules/migrations/base"
+ "code.gitea.io/gitea/modules/setting"
+ "code.gitea.io/gitea/modules/storage"
+ pull_service "code.gitea.io/gitea/services/pull"
+
+ "github.com/urfave/cli"
+)
+
+// CmdRestoreRepository represents the available restore a repository sub-command.
+var CmdRestoreRepository = cli.Command{
+ Name: "restore-repo",
+ Usage: "Restore the repository from disk",
+ Description: "This is a command for restoring the repository data.",
+ Action: runRestoreRepository,
+ Flags: []cli.Flag{
+ cli.StringFlag{
+ Name: "repo_dir, r",
+ Value: "./data",
+ Usage: "Repository dir path to restore from",
+ },
+ cli.StringFlag{
+ Name: "owner_name",
+ Value: "",
+ Usage: "Restore destination owner name",
+ },
+ cli.StringFlag{
+ Name: "repo_name",
+ Value: "",
+ Usage: "Restore destination repository name",
+ },
+ cli.StringFlag{
+ Name: "units",
+ Value: "",
+ Usage: `Which items will be restored, one or more units should be separated as comma.
+wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.`,
+ },
+ },
+}
+
+func runRestoreRepository(ctx *cli.Context) error {
+ if err := initDB(); err != nil {
+ return err
+ }
+
+ log.Trace("AppPath: %s", setting.AppPath)
+ log.Trace("AppWorkPath: %s", setting.AppWorkPath)
+ log.Trace("Custom path: %s", setting.CustomPath)
+ log.Trace("Log path: %s", setting.LogRootPath)
+ setting.InitDBConfig()
+
+ if err := storage.Init(); err != nil {
+ return err
+ }
+
+ if err := pull_service.Init(); err != nil {
+ return err
+ }
+
+ var opts = base.MigrateOptions{
+ RepoName: ctx.String("repo_name"),
+ }
+
+ if len(ctx.String("units")) == 0 {
+ opts.Wiki = true
+ opts.Issues = true
+ opts.Milestones = true
+ opts.Labels = true
+ opts.Releases = true
+ opts.Comments = true
+ opts.PullRequests = true
+ opts.ReleaseAssets = true
+ } else {
+ units := strings.Split(ctx.String("units"), ",")
+ for _, unit := range units {
+ switch strings.ToLower(unit) {
+ case "wiki":
+ opts.Wiki = true
+ case "issues":
+ opts.Issues = true
+ case "milestones":
+ opts.Milestones = true
+ case "labels":
+ opts.Labels = true
+ case "releases":
+ opts.Releases = true
+ case "release_assets":
+ opts.ReleaseAssets = true
+ case "comments":
+ opts.Comments = true
+ case "pull_requests":
+ opts.PullRequests = true
+ }
+ }
+ }
+
+ if err := migrations.RestoreRepository(
+ context.Background(),
+ ctx.String("repo_dir"),
+ ctx.String("owner_name"),
+ ctx.String("repo_name"),
+ ); err != nil {
+ log.Fatal("Failed to restore repository: %v", err)
+ return err
+ }
+
+ return nil
+}
diff --git a/docs/content/doc/usage/command-line.en-us.md b/docs/content/doc/usage/command-line.en-us.md
index a09d5dde73..98d047fb48 100644
--- a/docs/content/doc/usage/command-line.en-us.md
+++ b/docs/content/doc/usage/command-line.en-us.md
@@ -441,3 +441,28 @@ Manage running server operations:
- `--host value`, `-H value`: Mail server host (defaults to: 127.0.0.1:25)
- `--send-to value`, `-s value`: Email address(es) to send to
- `--subject value`, `-S value`: Subject header of sent emails
+
+### dump-repo
+
+Dump-repo dumps repository data from git/github/gitea/gitlab:
+
+- Options:
+ - `--git_service service` : Git service, it could be `git`, `github`, `gitea`, `gitlab`, If clone_addr could be recognized, this could be ignored.
+ - `--repo_dir dir`, `-r dir`: Repository dir path to store the data
+ - `--clone_addr addr`: The URL will be clone, currently could be a git/github/gitea/gitlab http/https URL. i.e. https://github.com/lunny/tango.git
+ - `--auth_username lunny`: The username to visit the clone_addr
+ - `--auth_password <password>`: The password to visit the clone_addr
+ - `--auth_token <token>`: The personal token to visit the clone_addr
+ - `--owner_name lunny`: The data will be stored on a directory with owner name if not empty
+ - `--repo_name tango`: The data will be stored on a directory with repository name if not empty
+ - `--units <units>`: Which items will be migrated, one or more units should be separated as comma. wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.
+
+### restore-repo
+
+Restore-repo restore repository data from disk dir:
+
+- Options:
+ - `--repo_dir dir`, `-r dir`: Repository dir path to restore from
+ - `--owner_name lunny`: Restore destination owner name
+ - `--repo_name tango`: Restore destination repository name
+ - `--units <units>`: Which items will be restored, one or more units should be separated as comma. wiki, issues, labels, releases, release_assets, milestones, pull_requests, comments are allowed. Empty means all units.
\ No newline at end of file
diff --git a/main.go b/main.go
index 8ee6ffa92c..6cbdc24401 100644
--- a/main.go
+++ b/main.go
@@ -72,6 +72,8 @@ arguments - which can alternatively be run by running the subcommand web.`
cmd.Cmdembedded,
cmd.CmdMigrateStorage,
cmd.CmdDocs,
+ cmd.CmdDumpRepository,
+ cmd.CmdRestoreRepository,
}
// Now adjust these commands to add our global configuration options
diff --git a/models/admin.go b/models/admin.go
index 420adbcda9..4635676d0c 100644
--- a/models/admin.go
+++ b/models/admin.go
@@ -132,3 +132,16 @@ func DeleteNoticesByIDs(ids []int64) error {
Delete(new(Notice))
return err
}
+
+// GetAdminUser returns the first administrator
+func GetAdminUser() (*User, error) {
+ var admin User
+ has, err := x.Where("is_admin=?", true).Get(&admin)
+ if err != nil {
+ return nil, err
+ } else if !has {
+ return nil, ErrUserNotExist{}
+ }
+
+ return &admin, nil
+}
diff --git a/models/task.go b/models/task.go
index b86314b449..b729bb8632 100644
--- a/models/task.go
+++ b/models/task.go
@@ -211,10 +211,6 @@ func FinishMigrateTask(task *Task) error {
if _, err := sess.ID(task.ID).Cols("status", "end_time").Update(task); err != nil {
return err
}
- task.Repo.Status = RepositoryReady
- if _, err := sess.ID(task.RepoID).Cols("status").Update(task.Repo); err != nil {
- return err
- }
return sess.Commit()
}
diff --git a/modules/migrations/base/comment.go b/modules/migrations/base/comment.go
index 4a653e474b..3c32e63b82 100644
--- a/modules/migrations/base/comment.go
+++ b/modules/migrations/base/comment.go
@@ -9,10 +9,10 @@ import "time"
// Comment is a standard comment information
type Comment struct {
- IssueIndex int64
- PosterID int64
- PosterName string
- PosterEmail string
+ IssueIndex int64 `yaml:"issue_index"`
+ PosterID int64 `yaml:"poster_id"`
+ PosterName string `yaml:"poster_name"`
+ PosterEmail string `yaml:"poster_email"`
Created time.Time
Updated time.Time
Content string
diff --git a/modules/migrations/base/downloader.go b/modules/migrations/base/downloader.go
index 5c47ed5305..afa99105c9 100644
--- a/modules/migrations/base/downloader.go
+++ b/modules/migrations/base/downloader.go
@@ -7,20 +7,13 @@ package base
import (
"context"
- "io"
"time"
"code.gitea.io/gitea/modules/structs"
)
-// AssetDownloader downloads an asset (attachment) for a release
-type AssetDownloader interface {
- GetAsset(relTag string, relID, id int64) (io.ReadCloser, error)
-}
-
// Downloader downloads the site repo informations
type Downloader interface {
- AssetDownloader
SetContext(context.Context)
GetRepoInfo() (*Repository, error)
GetTopics() ([]string, error)
diff --git a/modules/migrations/base/issue.go b/modules/migrations/base/issue.go
index f9dc8b93fe..8b1b461244 100644
--- a/modules/migrations/base/issue.go
+++ b/modules/migrations/base/issue.go
@@ -10,15 +10,15 @@ import "time"
// Issue is a standard issue information
type Issue struct {
Number int64
- PosterID int64
- PosterName string
- PosterEmail string
+ PosterID int64 `yaml:"poster_id"`
+ PosterName string `yaml:"poster_name"`
+ PosterEmail string `yaml:"poster_email"`
Title string
Content string
Ref string
Milestone string
State string // closed, open
- IsLocked bool
+ IsLocked bool `yaml:"is_locked"`
Created time.Time
Updated time.Time
Closed *time.Time
diff --git a/modules/migrations/base/options.go b/modules/migrations/base/options.go
index dbc40b138a..3c9b2c22fc 100644
--- a/modules/migrations/base/options.go
+++ b/modules/migrations/base/options.go
@@ -31,5 +31,6 @@ type MigrateOptions struct {
Releases bool
Comments bool
PullRequests bool
+ ReleaseAssets bool
MigrateToRepoID int64
}
diff --git a/modules/migrations/base/pullrequest.go b/modules/migrations/base/pullrequest.go
index ee612fbb8e..6411137d0a 100644
--- a/modules/migrations/base/pullrequest.go
+++ b/modules/migrations/base/pullrequest.go
@@ -13,11 +13,11 @@ import (
// PullRequest defines a standard pull request information
type PullRequest struct {
Number int64
- OriginalNumber int64
+ OriginalNumber int64 `yaml:"original_number"`
Title string
- PosterName string
- PosterID int64
- PosterEmail string
+ PosterName string `yaml:"poster_name"`
+ PosterID int64 `yaml:"poster_id"`
+ PosterEmail string `yaml:"poster_email"`
Content string
Milestone string
State string
@@ -25,14 +25,14 @@ type PullRequest struct {
Updated time.Time
Closed *time.Time
Labels []*Label
- PatchURL string
+ PatchURL string `yaml:"patch_url"`
Merged bool
- MergedTime *time.Time
- MergeCommitSHA string
+ MergedTime *time.Time `yaml:"merged_time"`
+ MergeCommitSHA string `yaml:"merge_commit_sha"`
Head PullRequestBranch
Base PullRequestBranch
Assignees []string
- IsLocked bool
+ IsLocked bool `yaml:"is_locked"`
Reactions []*Reaction
}
@@ -43,11 +43,11 @@ func (p *PullRequest) IsForkPullRequest() bool {
// PullRequestBranch represents a pull request branch
type PullRequestBranch struct {
- CloneURL string
+ CloneURL string `yaml:"clone_url"`
Ref string
SHA string
- RepoName string
- OwnerName string
+ RepoName string `yaml:"repo_name"`
+ OwnerName string `yaml:"owner_name"`
}
// RepoPath returns pull request repo path
diff --git a/modules/migrations/base/reaction.go b/modules/migrations/base/reaction.go
index b79223d4cd..1519499134 100644
--- a/modules/migrations/base/reaction.go
+++ b/modules/migrations/base/reaction.go
@@ -6,7 +6,7 @@ package base
// Reaction represents a reaction to an issue/pr/comment.
type Reaction struct {
- UserID int64
- UserName string
+ UserID int64 `yaml:"user_id"`
+ UserName string `yaml:"user_name"`
Content string
}
diff --git a/modules/migrations/base/release.go b/modules/migrations/base/release.go
index c9b26ab1da..8b4339928b 100644
--- a/modules/migrations/base/release.go
+++ b/modules/migrations/base/release.go
@@ -4,32 +4,37 @@
package base
-import "time"
+import (
+ "io"
+ "time"
+)
// ReleaseAsset represents a release asset
type ReleaseAsset struct {
ID int64
Name string
- ContentType *string
+ ContentType *string `yaml:"content_type"`
Size *int
- DownloadCount *int
+ DownloadCount *int `yaml:"download_count"`
Created time.Time
Updated time.Time
- DownloadURL *string
+ DownloadURL *string `yaml:"download_url"`
+ // if DownloadURL is nil, the function should be invoked
+ DownloadFunc func() (io.ReadCloser, error) `yaml:"-"`
}
// Release represents a release
type Release struct {
- TagName string
- TargetCommitish string
+ TagName string `yaml:"tag_name"`
+ TargetCommitish string `yaml:"target_commitish"`
Name string
Body string
Draft bool
Prerelease bool
- PublisherID int64
- PublisherName string
- PublisherEmail string
- Assets []ReleaseAsset
+ PublisherID int64 `yaml:"publisher_id"`
+ PublisherName string `yaml:"publisher_name"`
+ PublisherEmail string `yaml:"publisher_email"`
+ Assets []*ReleaseAsset
Created time.Time
Published time.Time
}
diff --git a/modules/migrations/base/repo.go b/modules/migrations/base/repo.go
index d26a911854..693a96314d 100644
--- a/modules/migrations/base/repo.go
+++ b/modules/migrations/base/repo.go
@@ -9,10 +9,10 @@ package base
type Repository struct {
Name string
Owner string
- IsPrivate bool
- IsMirror bool
+ IsPrivate bool `yaml:"is_private"`
+ IsMirror bool `yaml:"is_mirror"`
Description string
- CloneURL string
- OriginalURL string
+ CloneURL string `yaml:"clone_url"`
+ OriginalURL string `yaml:"original_url"`
DefaultBranch string
}
diff --git a/modules/migrations/base/review.go b/modules/migrations/base/review.go
index 0a9d03dae9..6344f0384d 100644
--- a/modules/migrations/base/review.go
+++ b/modules/migrations/base/review.go
@@ -17,29 +17,29 @@ const (
// Review is a standard review information
type Review struct {
ID int64
- IssueIndex int64
- ReviewerID int64
- ReviewerName string
+ IssueIndex int64 `yaml:"issue_index"`
+ ReviewerID int64 `yaml:"reviewer_id"`
+ ReviewerName string `yaml:"reviewer_name"`
Official bool
- CommitID string
+ CommitID string `yaml:"commit_id"`
Content string
- CreatedAt time.Time
- State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT
+ CreatedAt time.Time `yaml:"created_at"`
+ State string // PENDING, APPROVED, REQUEST_CHANGES, or COMMENT
Comments []*ReviewComment
}
// ReviewComment represents a review comment
type ReviewComment struct {
ID int64
- InReplyTo int64
+ InReplyTo int64 `yaml:"in_reply_to"`
Content string
- TreePath string
- DiffHunk string
+ TreePath string `yaml:"tree_path"`
+ DiffHunk string `yaml:"diff_hunk"`
Position int
Line int
- CommitID string
- PosterID int64
+ CommitID string `yaml:"commit_id"`
+ PosterID int64 `yaml:"poster_id"`
Reactions []*Reaction
- CreatedAt time.Time
- UpdatedAt time.Time
+ CreatedAt time.Time `yaml:"created_at"`
+ UpdatedAt time.Time `yaml:"updated_at"`
}
diff --git a/modules/migrations/base/uploader.go b/modules/migrations/base/uploader.go
index 07c2bb0d42..dfcf81d052 100644
--- a/modules/migrations/base/uploader.go
+++ b/modules/migrations/base/uploader.go
@@ -11,7 +11,7 @@ type Uploader interface {
CreateRepo(repo *Repository, opts MigrateOptions) error
CreateTopics(topic ...string) error
CreateMilestones(milestones ...*Milestone) error
- CreateReleases(downloader Downloader, releases ...*Release) error
+ CreateReleases(releases ...*Release) error
SyncTags() error
CreateLabels(labels ...*Label) error
CreateIssues(issues ...*Issue) error
@@ -19,5 +19,6 @@ type Uploader interface {
CreatePullRequests(prs ...*PullRequest) error
CreateReviews(reviews ...*Review) error
Rollback() error
+ Finish() error
Close()
}
diff --git a/modules/migrations/dump.go b/modules/migrations/dump.go
new file mode 100644
index 0000000000..3c3b9a1753
--- /dev/null
+++ b/modules/migrations/dump.go
@@ -0,0 +1,591 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "os"
+ "path/filepath"
+ "time"
+
+ "code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/git"
+ "code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/migrations/base"
+ "code.gitea.io/gitea/modules/repository"
+
+ "gopkg.in/yaml.v2"
+)
+
+var (
+ _ base.Uploader = &RepositoryDumper{}
+)
+
+// RepositoryDumper implements an Uploader to the local directory
+type RepositoryDumper struct {
+ ctx context.Context
+ baseDir string
+ repoOwner string
+ repoName string
+ opts base.MigrateOptions
+ milestoneFile *os.File
+ labelFile *os.File
+ releaseFile *os.File
+ issueFile *os.File
+ commentFiles map[int64]*os.File
+ pullrequestFile *os.File
+ reviewFiles map[int64]*os.File
+
+ gitRepo *git.Repository
+ prHeadCache map[string]struct{}
+}
+
+// NewRepositoryDumper creates an gitea Uploader
+func NewRepositoryDumper(ctx context.Context, baseDir, repoOwner, repoName string, opts base.MigrateOptions) (*RepositoryDumper, error) {
+ baseDir = filepath.Join(baseDir, repoOwner, repoName)
+ if err := os.MkdirAll(baseDir, os.ModePerm); err != nil {
+ return nil, err
+ }
+ return &RepositoryDumper{
+ ctx: ctx,
+ opts: opts,
+ baseDir: baseDir,
+ repoOwner: repoOwner,
+ repoName: repoName,
+ prHeadCache: make(map[string]struct{}),
+ commentFiles: make(map[int64]*os.File),
+ reviewFiles: make(map[int64]*os.File),
+ }, nil
+}
+
+// MaxBatchInsertSize returns the table's max batch insert size
+func (g *RepositoryDumper) MaxBatchInsertSize(tp string) int {
+ return 1000
+}
+
+func (g *RepositoryDumper) gitPath() string {
+ return filepath.Join(g.baseDir, "git")
+}
+
+func (g *RepositoryDumper) wikiPath() string {
+ return filepath.Join(g.baseDir, "wiki")
+}
+
+func (g *RepositoryDumper) commentDir() string {
+ return filepath.Join(g.baseDir, "comments")
+}
+
+func (g *RepositoryDumper) reviewDir() string {
+ return filepath.Join(g.baseDir, "reviews")
+}
+
+func (g *RepositoryDumper) setURLToken(remoteAddr string) (string, error) {
+ if len(g.opts.AuthToken) > 0 || len(g.opts.AuthUsername) > 0 {
+ u, err := url.Parse(remoteAddr)
+ if err != nil {
+ return "", err
+ }
+ u.User = url.UserPassword(g.opts.AuthUsername, g.opts.AuthPassword)
+ if len(g.opts.AuthToken) > 0 {
+ u.User = url.UserPassword("oauth2", g.opts.AuthToken)
+ }
+ remoteAddr = u.String()
+ }
+
+ return remoteAddr, nil
+}
+
+// CreateRepo creates a repository
+func (g *RepositoryDumper) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error {
+ f, err := os.Create(filepath.Join(g.baseDir, "repo.yml"))
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ bs, err := yaml.Marshal(map[string]interface{}{
+ "name": repo.Name,
+ "owner": repo.Owner,
+ "description": repo.Description,
+ "clone_addr": opts.CloneAddr,
+ "original_url": repo.OriginalURL,
+ "is_private": opts.Private,
+ "service_type": opts.GitServiceType,
+ "wiki": opts.Wiki,
+ "issues": opts.Issues,
+ "milestones": opts.Milestones,
+ "labels": opts.Labels,
+ "releases": opts.Releases,
+ "comments": opts.Comments,
+ "pulls": opts.PullRequests,
+ "assets": opts.ReleaseAssets,
+ })
+ if err != nil {
+ return err
+ }
+
+ if _, err := f.Write(bs); err != nil {
+ return err
+ }
+
+ repoPath := g.gitPath()
+ if err := os.MkdirAll(repoPath, os.ModePerm); err != nil {
+ return err
+ }
+
+ migrateTimeout := 2 * time.Hour
+
+ remoteAddr, err := g.setURLToken(repo.CloneURL)
+ if err != nil {
+ return err
+ }
+
+ err = git.Clone(remoteAddr, repoPath, git.CloneRepoOptions{
+ Mirror: true,
+ Quiet: true,
+ Timeout: migrateTimeout,
+ })
+ if err != nil {
+ return fmt.Errorf("Clone: %v", err)
+ }
+
+ if opts.Wiki {
+ wikiPath := g.wikiPath()
+ wikiRemotePath := repository.WikiRemoteURL(remoteAddr)
+ if len(wikiRemotePath) > 0 {
+ if err := os.MkdirAll(wikiPath, os.ModePerm); err != nil {
+ return fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
+ }
+
+ if err := git.Clone(wikiRemotePath, wikiPath, git.CloneRepoOptions{
+ Mirror: true,
+ Quiet: true,
+ Timeout: migrateTimeout,
+ Branch: "master",
+ }); err != nil {
+ log.Warn("Clone wiki: %v", err)
+ if err := os.RemoveAll(wikiPath); err != nil {
+ return fmt.Errorf("Failed to remove %s: %v", wikiPath, err)
+ }
+ }
+ }
+ }
+
+ g.gitRepo, err = git.OpenRepository(g.gitPath())
+ return err
+}
+
+// Close closes this uploader
+func (g *RepositoryDumper) Close() {
+ if g.gitRepo != nil {
+ g.gitRepo.Close()
+ }
+ if g.milestoneFile != nil {
+ g.milestoneFile.Close()
+ }
+ if g.labelFile != nil {
+ g.labelFile.Close()
+ }
+ if g.releaseFile != nil {
+ g.releaseFile.Close()
+ }
+ if g.issueFile != nil {
+ g.issueFile.Close()
+ }
+ for _, f := range g.commentFiles {
+ f.Close()
+ }
+ if g.pullrequestFile != nil {
+ g.pullrequestFile.Close()
+ }
+ for _, f := range g.reviewFiles {
+ f.Close()
+ }
+}
+
+// CreateTopics creates topics
+func (g *RepositoryDumper) CreateTopics(topics ...string) error {
+ f, err := os.Create(filepath.Join(g.baseDir, "topic.yml"))
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ bs, err := yaml.Marshal(map[string]interface{}{
+ "topics": topics,
+ })
+ if err != nil {
+ return err
+ }
+
+ if _, err := f.Write(bs); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// CreateMilestones creates milestones
+func (g *RepositoryDumper) CreateMilestones(milestones ...*base.Milestone) error {
+ var err error
+ if g.milestoneFile == nil {
+ g.milestoneFile, err = os.Create(filepath.Join(g.baseDir, "milestone.yml"))
+ if err != nil {
+ return err
+ }
+ }
+
+ bs, err := yaml.Marshal(milestones)
+ if err != nil {
+ return err
+ }
+
+ if _, err := g.milestoneFile.Write(bs); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// CreateLabels creates labels
+func (g *RepositoryDumper) CreateLabels(labels ...*base.Label) error {
+ var err error
+ if g.labelFile == nil {
+ g.labelFile, err = os.Create(filepath.Join(g.baseDir, "label.yml"))
+ if err != nil {
+ return err
+ }
+ }
+
+ bs, err := yaml.Marshal(labels)
+ if err != nil {
+ return err
+ }
+
+ if _, err := g.labelFile.Write(bs); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// CreateReleases creates releases
+func (g *RepositoryDumper) CreateReleases(releases ...*base.Release) error {
+ if g.opts.ReleaseAssets {
+ for _, release := range releases {
+ attachDir := filepath.Join("release_assets", release.TagName)
+ if err := os.MkdirAll(filepath.Join(g.baseDir, attachDir), os.ModePerm); err != nil {
+ return err
+ }
+ for _, asset := range release.Assets {
+ attachLocalPath := filepath.Join(attachDir, asset.Name)
+ // download attachment
+
+ err := func(attachPath string) error {
+ var rc io.ReadCloser
+ var err error
+ if asset.DownloadURL == nil {
+ rc, err = asset.DownloadFunc()
+ if err != nil {
+ return err
+ }
+ } else {
+ resp, err := http.Get(*asset.DownloadURL)
+ if err != nil {
+ return err
+ }
+ rc = resp.Body
+ }
+ defer rc.Close()
+
+ fw, err := os.Create(attachPath)
+ if err != nil {
+ return fmt.Errorf("Create: %v", err)
+ }
+ defer fw.Close()
+
+ _, err = io.Copy(fw, rc)
+ return err
+ }(filepath.Join(g.baseDir, attachLocalPath))
+ if err != nil {
+ return err
+ }
+ asset.DownloadURL = &attachLocalPath // to save the filepath on the yml file, change the source
+ }
+ }
+ }
+
+ var err error
+ if g.releaseFile == nil {
+ g.releaseFile, err = os.Create(filepath.Join(g.baseDir, "release.yml"))
+ if err != nil {
+ return err
+ }
+ }
+
+ bs, err := yaml.Marshal(releases)
+ if err != nil {
+ return err
+ }
+
+ if _, err := g.releaseFile.Write(bs); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// SyncTags syncs releases with tags in the database
+func (g *RepositoryDumper) SyncTags() error {
+ return nil
+}
+
+// CreateIssues creates issues
+func (g *RepositoryDumper) CreateIssues(issues ...*base.Issue) error {
+ var err error
+ if g.issueFile == nil {
+ g.issueFile, err = os.Create(filepath.Join(g.baseDir, "issue.yml"))
+ if err != nil {
+ return err
+ }
+ }
+
+ bs, err := yaml.Marshal(issues)
+ if err != nil {
+ return err
+ }
+
+ if _, err := g.issueFile.Write(bs); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+func (g *RepositoryDumper) createItems(dir string, itemFiles map[int64]*os.File, itemsMap map[int64][]interface{}) error {
+ if err := os.MkdirAll(dir, os.ModePerm); err != nil {
+ return err
+ }
+
+ for number, items := range itemsMap {
+ var err error
+ itemFile := itemFiles[number]
+ if itemFile == nil {
+ itemFile, err = os.Create(filepath.Join(dir, fmt.Sprintf("%d.yml", number)))
+ if err != nil {
+ return err
+ }
+ itemFiles[number] = itemFile
+ }
+
+ bs, err := yaml.Marshal(items)
+ if err != nil {
+ return err
+ }
+
+ if _, err := itemFile.Write(bs); err != nil {
+ return err
+ }
+ }
+
+ return nil
+}
+
+// CreateComments creates comments of issues
+func (g *RepositoryDumper) CreateComments(comments ...*base.Comment) error {
+ var commentsMap = make(map[int64][]interface{}, len(comments))
+ for _, comment := range comments {
+ commentsMap[comment.IssueIndex] = append(commentsMap[comment.IssueIndex], comment)
+ }
+
+ return g.createItems(g.commentDir(), g.commentFiles, commentsMap)
+}
+
+// CreatePullRequests creates pull requests
+func (g *RepositoryDumper) CreatePullRequests(prs ...*base.PullRequest) error {
+ for _, pr := range prs {
+ // download patch file
+ err := func() error {
+ u, err := g.setURLToken(pr.PatchURL)
+ if err != nil {
+ return err
+ }
+ resp, err := http.Get(u)
+ if err != nil {
+ return err
+ }
+ defer resp.Body.Close()
+ pullDir := filepath.Join(g.gitPath(), "pulls")
+ if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
+ return err
+ }
+ fPath := filepath.Join(pullDir, fmt.Sprintf("%d.patch", pr.Number))
+ f, err := os.Create(fPath)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+ if _, err = io.Copy(f, resp.Body); err != nil {
+ return err
+ }
+ pr.PatchURL = "git/pulls/" + fmt.Sprintf("%d.patch", pr.Number)
+
+ return nil
+ }()
+ if err != nil {
+ return err
+ }
+
+ // set head information
+ pullHead := filepath.Join(g.gitPath(), "refs", "pull", fmt.Sprintf("%d", pr.Number))
+ if err := os.MkdirAll(pullHead, os.ModePerm); err != nil {
+ return err
+ }
+ p, err := os.Create(filepath.Join(pullHead, "head"))
+ if err != nil {
+ return err
+ }
+ _, err = p.WriteString(pr.Head.SHA)
+ p.Close()
+ if err != nil {
+ return err
+ }
+
+ if pr.IsForkPullRequest() && pr.State != "closed" {
+ if pr.Head.OwnerName != "" {
+ remote := pr.Head.OwnerName
+ _, ok := g.prHeadCache[remote]
+ if !ok {
+ // git remote add
+ // TODO: how to handle private CloneURL?
+ err := g.gitRepo.AddRemote(remote, pr.Head.CloneURL, true)
+ if err != nil {
+ log.Error("AddRemote failed: %s", err)
+ } else {
+ g.prHeadCache[remote] = struct{}{}
+ ok = true
+ }
+ }
+
+ if ok {
+ _, err = git.NewCommand("fetch", remote, pr.Head.Ref).RunInDir(g.gitPath())
+ if err != nil {
+ log.Error("Fetch branch from %s failed: %v", pr.Head.CloneURL, err)
+ } else {
+ headBranch := filepath.Join(g.gitPath(), "refs", "heads", pr.Head.OwnerName, pr.Head.Ref)
+ if err := os.MkdirAll(filepath.Dir(headBranch), os.ModePerm); err != nil {
+ return err
+ }
+ b, err := os.Create(headBranch)
+ if err != nil {
+ return err
+ }
+ _, err = b.WriteString(pr.Head.SHA)
+ b.Close()
+ if err != nil {
+ return err
+ }
+ }
+ }
+ }
+ }
+ }
+
+ var err error
+ if g.pullrequestFile == nil {
+ if err := os.MkdirAll(g.baseDir, os.ModePerm); err != nil {
+ return err
+ }
+ g.pullrequestFile, err = os.Create(filepath.Join(g.baseDir, "pull_request.yml"))
+ if err != nil {
+ return err
+ }
+ }
+
+ bs, err := yaml.Marshal(prs)
+ if err != nil {
+ return err
+ }
+
+ if _, err := g.pullrequestFile.Write(bs); err != nil {
+ return err
+ }
+
+ return nil
+}
+
+// CreateReviews create pull request reviews
+func (g *RepositoryDumper) CreateReviews(reviews ...*base.Review) error {
+ var reviewsMap = make(map[int64][]interface{}, len(reviews))
+ for _, review := range reviews {
+ reviewsMap[review.IssueIndex] = append(reviewsMap[review.IssueIndex], review)
+ }
+
+ return g.createItems(g.reviewDir(), g.reviewFiles, reviewsMap)
+}
+
+// Rollback when migrating failed, this will rollback all the changes.
+func (g *RepositoryDumper) Rollback() error {
+ g.Close()
+ return os.RemoveAll(g.baseDir)
+}
+
+// Finish when migrating succeed, this will update something.
+func (g *RepositoryDumper) Finish() error {
+ return nil
+}
+
+// DumpRepository dump repository according MigrateOptions to a local directory
+func DumpRepository(ctx context.Context, baseDir, ownerName string, opts base.MigrateOptions) error {
+ downloader, err := newDownloader(ctx, ownerName, opts)
+ if err != nil {
+ return err
+ }
+ uploader, err := NewRepositoryDumper(ctx, baseDir, ownerName, opts.RepoName, opts)
+ if err != nil {
+ return err
+ }
+
+ if err := migrateRepository(downloader, uploader, opts); err != nil {
+ if err1 := uploader.Rollback(); err1 != nil {
+ log.Error("rollback failed: %v", err1)
+ }
+ return err
+ }
+ return nil
+}
+
+// RestoreRepository restore a repository from the disk directory
+func RestoreRepository(ctx context.Context, baseDir string, ownerName, repoName string) error {
+ doer, err := models.GetAdminUser()
+ if err != nil {
+ return err
+ }
+ var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, repoName)
+ downloader, err := NewRepositoryRestorer(ctx, baseDir, ownerName, repoName)
+ if err != nil {
+ return err
+ }
+ if err = migrateRepository(downloader, uploader, base.MigrateOptions{
+ Wiki: true,
+ Issues: true,
+ Milestones: true,
+ Labels: true,
+ Releases: true,
+ Comments: true,
+ PullRequests: true,
+ ReleaseAssets: true,
+ }); err != nil {
+ if err1 := uploader.Rollback(); err1 != nil {
+ log.Error("rollback failed: %v", err1)
+ }
+ return err
+ }
+ return nil
+}
diff --git a/modules/migrations/error.go b/modules/migrations/error.go
index b2e2315fc8..462ba29026 100644
--- a/modules/migrations/error.go
+++ b/modules/migrations/error.go
@@ -14,6 +14,9 @@ import (
var (
// ErrNotSupported returns the error not supported
ErrNotSupported = errors.New("not supported")
+
+ // ErrRepoNotCreated returns the error that repository not created
+ ErrRepoNotCreated = errors.New("repository is not created yet")
)
// IsRateLimitError returns true if the err is github.RateLimitError
diff --git a/modules/migrations/git.go b/modules/migrations/git.go
index 0aad8dbef5..88222086e4 100644
--- a/modules/migrations/git.go
+++ b/modules/migrations/git.go
@@ -6,7 +6,6 @@ package migrations
import (
"context"
- "io"
"code.gitea.io/gitea/modules/migrations/base"
)
@@ -65,11 +64,6 @@ func (g *PlainGitDownloader) GetReleases() ([]*base.Release, error) {
return nil, ErrNotSupported
}
-// GetAsset returns an asset
-func (g *PlainGitDownloader) GetAsset(_ string, _, _ int64) (io.ReadCloser, error) {
- return nil, ErrNotSupported
-}
-
// GetIssues returns issues according page and perPage
func (g *PlainGitDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
return nil, false, ErrNotSupported
diff --git a/modules/migrations/gitea_downloader.go b/modules/migrations/gitea_downloader.go
index 0509c708bf..0c690464fa 100644
--- a/modules/migrations/gitea_downloader.go
+++ b/modules/migrations/gitea_downloader.go
@@ -268,13 +268,27 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
for _, asset := range rel.Attachments {
size := int(asset.Size)
dlCount := int(asset.DownloadCount)
- r.Assets = append(r.Assets, base.ReleaseAsset{
+ r.Assets = append(r.Assets, &base.ReleaseAsset{
ID: asset.ID,
Name: asset.Name,
Size: &size,
DownloadCount: &dlCount,
Created: asset.Created,
DownloadURL: &asset.DownloadURL,
+ DownloadFunc: func() (io.ReadCloser, error) {
+ asset, _, err := g.client.GetReleaseAttachment(g.repoOwner, g.repoName, rel.ID, asset.ID)
+ if err != nil {
+ return nil, err
+ }
+ // FIXME: for a private download?
+ resp, err := http.Get(asset.DownloadURL)
+ if err != nil {
+ return nil, err
+ }
+
+ // resp.Body is closed by the uploader
+ return resp.Body, nil
+ },
})
}
return r
@@ -310,21 +324,6 @@ func (g *GiteaDownloader) GetReleases() ([]*base.Release, error) {
return releases, nil
}
-// GetAsset returns an asset
-func (g *GiteaDownloader) GetAsset(_ string, relID, id int64) (io.ReadCloser, error) {
- asset, _, err := g.client.GetReleaseAttachment(g.repoOwner, g.repoName, relID, id)
- if err != nil {
- return nil, err
- }
- resp, err := http.Get(asset.DownloadURL)
- if err != nil {
- return nil, err
- }
-
- // resp.Body is closed by the uploader
- return resp.Body, nil
-}
-
func (g *GiteaDownloader) getIssueReactions(index int64) ([]*base.Reaction, error) {
var reactions []*base.Reaction
if err := g.client.CheckServerVersionConstraint(">=1.11"); err != nil {
diff --git a/modules/migrations/gitea_uploader.go b/modules/migrations/gitea_uploader.go
index 91ddda9c39..6118b3b5c1 100644
--- a/modules/migrations/gitea_uploader.go
+++ b/modules/migrations/gitea_uploader.go
@@ -10,7 +10,6 @@ import (
"context"
"fmt"
"io"
- "net/http"
"net/url"
"os"
"path/filepath"
@@ -28,6 +27,7 @@ import (
"code.gitea.io/gitea/modules/storage"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/timeutil"
+ "code.gitea.io/gitea/modules/uri"
"code.gitea.io/gitea/services/pull"
gouuid "github.com/google/uuid"
@@ -86,6 +86,22 @@ func (g *GiteaLocalUploader) MaxBatchInsertSize(tp string) int {
return 10
}
+func fullURL(opts base.MigrateOptions, remoteAddr string) (string, error) {
+ var fullRemoteAddr = remoteAddr
+ if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
+ u, err := url.Parse(remoteAddr)
+ if err != nil {
+ return "", err
+ }
+ u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
+ if len(opts.AuthToken) > 0 {
+ u.User = url.UserPassword("oauth2", opts.AuthToken)
+ }
+ fullRemoteAddr = u.String()
+ }
+ return fullRemoteAddr, nil
+}
+
// CreateRepo creates a repository
func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.MigrateOptions) error {
owner, err := models.GetUserByName(g.repoOwner)
@@ -93,19 +109,10 @@ func (g *GiteaLocalUploader) CreateRepo(repo *base.Repository, opts base.Migrate
return err
}
- var remoteAddr = repo.CloneURL
- if len(opts.AuthToken) > 0 || len(opts.AuthUsername) > 0 {
- u, err := url.Parse(repo.CloneURL)
- if err != nil {
- return err
- }
- u.User = url.UserPassword(opts.AuthUsername, opts.AuthPassword)
- if len(opts.AuthToken) > 0 {
- u.User = url.UserPassword("oauth2", opts.AuthToken)
- }
- remoteAddr = u.String()
+ remoteAddr, err := fullURL(opts, repo.CloneURL)
+ if err != nil {
+ return err
}
-
var r *models.Repository
if opts.MigrateToRepoID <= 0 {
r, err = repo_module.CreateRepository(g.doer, owner, models.CreateRepoOptions{
@@ -224,7 +231,7 @@ func (g *GiteaLocalUploader) CreateLabels(labels ...*base.Label) error {
}
// CreateReleases creates releases
-func (g *GiteaLocalUploader) CreateReleases(downloader base.Downloader, releases ...*base.Release) error {
+func (g *GiteaLocalUploader) CreateReleases(releases ...*base.Release) error {
var rels = make([]*models.Release, 0, len(releases))
for _, release := range releases {
var rel = models.Release{
@@ -283,25 +290,27 @@ func (g *GiteaLocalUploader) CreateReleases(downloader base.Downloader, releases
// download attachment
err = func() error {
+ // asset.DownloadURL maybe a local file
var rc io.ReadCloser
if asset.DownloadURL == nil {
- rc, err = downloader.GetAsset(rel.TagName, rel.ID, asset.ID)
+ rc, err = asset.DownloadFunc()
if err != nil {
return err
}
} else {
- resp, err := http.Get(*asset.DownloadURL)
+ rc, err = uri.Open(*asset.DownloadURL)
if err != nil {
return err
}
- rc = resp.Body
}
+ defer rc.Close()
_, err = storage.Attachments.Save(attach.RelativePath(), rc)
return err
}()
if err != nil {
return err
}
+
rel.Attachments = append(rel.Attachments, &attach)
}
@@ -559,11 +568,12 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
// download patch file
err := func() error {
- resp, err := http.Get(pr.PatchURL)
+ // pr.PatchURL maybe a local file
+ ret, err := uri.Open(pr.PatchURL)
if err != nil {
return err
}
- defer resp.Body.Close()
+ defer ret.Close()
pullDir := filepath.Join(g.repo.RepoPath(), "pulls")
if err = os.MkdirAll(pullDir, os.ModePerm); err != nil {
return err
@@ -573,7 +583,7 @@ func (g *GiteaLocalUploader) newPullRequest(pr *base.PullRequest) (*models.PullR
return err
}
defer f.Close()
- _, err = io.Copy(f, resp.Body)
+ _, err = io.Copy(f, ret)
return err
}()
if err != nil {
@@ -859,3 +869,13 @@ func (g *GiteaLocalUploader) Rollback() error {
}
return nil
}
+
+// Finish when migrating success, this will do some status update things.
+func (g *GiteaLocalUploader) Finish() error {
+ if g.repo == nil || g.repo.ID <= 0 {
+ return ErrRepoNotCreated
+ }
+
+ g.repo.Status = models.RepositoryReady
+ return models.UpdateRepositoryCols(g.repo, "status")
+}
diff --git a/modules/migrations/gitea_uploader_test.go b/modules/migrations/gitea_uploader_test.go
index 8432a1eecd..3c7def4675 100644
--- a/modules/migrations/gitea_uploader_test.go
+++ b/modules/migrations/gitea_uploader_test.go
@@ -52,6 +52,7 @@ func TestGiteaUploadRepo(t *testing.T) {
repo := models.AssertExistsAndLoadBean(t, &models.Repository{OwnerID: user.ID, Name: repoName}).(*models.Repository)
assert.True(t, repo.HasWiki())
+ assert.EqualValues(t, models.RepositoryReady, repo.Status)
milestones, err := models.GetMilestones(models.GetMilestonesOption{
RepoID: repo.ID,
diff --git a/modules/migrations/github.go b/modules/migrations/github.go
index 7aa1e57274..178517ba42 100644
--- a/modules/migrations/github.go
+++ b/modules/migrations/github.go
@@ -291,7 +291,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
}
for _, asset := range rel.Assets {
- r.Assets = append(r.Assets, base.ReleaseAsset{
+ r.Assets = append(r.Assets, &base.ReleaseAsset{
ID: *asset.ID,
Name: *asset.Name,
ContentType: asset.ContentType,
@@ -299,6 +299,16 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
DownloadCount: asset.DownloadCount,
Created: asset.CreatedAt.Time,
Updated: asset.UpdatedAt.Time,
+ DownloadFunc: func() (io.ReadCloser, error) {
+ asset, redir, err := g.client.Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, *asset.ID, http.DefaultClient)
+ if err != nil {
+ return nil, err
+ }
+ if asset == nil {
+ return ioutil.NopCloser(bytes.NewBufferString(redir)), nil
+ }
+ return asset, nil
+ },
})
}
return r
@@ -330,18 +340,6 @@ func (g *GithubDownloaderV3) GetReleases() ([]*base.Release, error) {
return releases, nil
}
-// GetAsset returns an asset
-func (g *GithubDownloaderV3) GetAsset(_ string, _, id int64) (io.ReadCloser, error) {
- asset, redir, err := g.client.Repositories.DownloadReleaseAsset(g.ctx, g.repoOwner, g.repoName, id, http.DefaultClient)
- if err != nil {
- return nil, err
- }
- if asset == nil {
- return ioutil.NopCloser(bytes.NewBufferString(redir)), nil
- }
- return asset, nil
-}
-
// GetIssues returns issues according start and limit
func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
if perPage > g.maxPerPage {
@@ -363,6 +361,7 @@ func (g *GithubDownloaderV3) GetIssues(page, perPage int) ([]*base.Issue, bool,
if err != nil {
return nil, false, fmt.Errorf("error while listing repos: %v", err)
}
+ log.Trace("Request get issues %d/%d, but in fact get %d", perPage, page, len(issues))
g.rate = &resp.Rate
for _, issue := range issues {
if issue.IsPullRequest() {
diff --git a/modules/migrations/gitlab.go b/modules/migrations/gitlab.go
index b1027c4f64..e3fa956758 100644
--- a/modules/migrations/gitlab.go
+++ b/modules/migrations/gitlab.go
@@ -295,12 +295,32 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
}
for k, asset := range rel.Assets.Links {
- r.Assets = append(r.Assets, base.ReleaseAsset{
+ r.Assets = append(r.Assets, &base.ReleaseAsset{
ID: int64(asset.ID),
Name: asset.Name,
ContentType: &rel.Assets.Sources[k].Format,
Size: &zero,
DownloadCount: &zero,
+ DownloadFunc: func() (io.ReadCloser, error) {
+ link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, rel.TagName, asset.ID, gitlab.WithContext(g.ctx))
+ if err != nil {
+ return nil, err
+ }
+
+ req, err := http.NewRequest("GET", link.URL, nil)
+ if err != nil {
+ return nil, err
+ }
+ req = req.WithContext(g.ctx)
+
+ resp, err := http.DefaultClient.Do(req)
+ if err != nil {
+ return nil, err
+ }
+
+ // resp.Body is closed by the uploader
+ return resp.Body, nil
+ },
})
}
return r
@@ -329,28 +349,6 @@ func (g *GitlabDownloader) GetReleases() ([]*base.Release, error) {
return releases, nil
}
-// GetAsset returns an asset
-func (g *GitlabDownloader) GetAsset(tag string, _, id int64) (io.ReadCloser, error) {
- link, _, err := g.client.ReleaseLinks.GetReleaseLink(g.repoID, tag, int(id), gitlab.WithContext(g.ctx))
- if err != nil {
- return nil, err
- }
-
- req, err := http.NewRequest("GET", link.URL, nil)
- if err != nil {
- return nil, err
- }
- req = req.WithContext(g.ctx)
-
- resp, err := http.DefaultClient.Do(req)
- if err != nil {
- return nil, err
- }
-
- // resp.Body is closed by the uploader
- return resp.Body, nil
-}
-
// GetIssues returns issues according start and limit
// Note: issue label description and colors are not supported by the go-gitlab library at this time
func (g *GitlabDownloader) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
diff --git a/modules/migrations/migrate.go b/modules/migrations/migrate.go
index b3ecb8114a..4c15626e57 100644
--- a/modules/migrations/migrate.go
+++ b/modules/migrations/migrate.go
@@ -73,10 +73,30 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string,
if err != nil {
return nil, err
}
+ downloader, err := newDownloader(ctx, ownerName, opts)
+ if err != nil {
+ return nil, err
+ }
+ var uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
+ uploader.gitServiceType = opts.GitServiceType
+
+ if err := migrateRepository(downloader, uploader, opts); err != nil {
+ if err1 := uploader.Rollback(); err1 != nil {
+ log.Error("rollback failed: %v", err1)
+ }
+ if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil {
+ log.Error("create respotiry notice failed: ", err2)
+ }
+ return nil, err
+ }
+ return uploader.repo, nil
+}
+
+func newDownloader(ctx context.Context, ownerName string, opts base.MigrateOptions) (base.Downloader, error) {
var (
downloader base.Downloader
- uploader = NewGiteaLocalUploader(ctx, doer, ownerName, opts.RepoName)
+ err error
)
for _, factory := range factories {
@@ -101,24 +121,10 @@ func MigrateRepository(ctx context.Context, doer *models.User, ownerName string,
log.Trace("Will migrate from git: %s", opts.OriginalURL)
}
- uploader.gitServiceType = opts.GitServiceType
-
if setting.Migrations.MaxAttempts > 1 {
downloader = base.NewRetryDownloader(ctx, downloader, setting.Migrations.MaxAttempts, setting.Migrations.RetryBackoff)
}
-
- if err := migrateRepository(downloader, uploader, opts); err != nil {
- if err1 := uploader.Rollback(); err1 != nil {
- log.Error("rollback failed: %v", err1)
- }
-
- if err2 := models.CreateRepositoryNotice(fmt.Sprintf("Migrate repository from %s failed: %v", opts.OriginalURL, err)); err2 != nil {
- log.Error("create repository notice failed: ", err2)
- }
- return nil, err
- }
-
- return uploader.repo, nil
+ return downloader, nil
}
// migrateRepository will download information and then upload it to Uploader, this is a simple
@@ -204,7 +210,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
relBatchSize = len(releases)
}
- if err := uploader.CreateReleases(downloader, releases[:relBatchSize]...); err != nil {
+ if err := uploader.CreateReleases(releases[:relBatchSize]...); err != nil {
return err
}
releases = releases[relBatchSize:]
@@ -235,31 +241,30 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
return err
}
- if !opts.Comments {
- continue
- }
-
- var allComments = make([]*base.Comment, 0, commentBatchSize)
- for _, issue := range issues {
- comments, err := downloader.GetComments(issue.Number)
- if err != nil {
- return err
- }
-
- allComments = append(allComments, comments...)
-
- if len(allComments) >= commentBatchSize {
- if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
+ if opts.Comments {
+ var allComments = make([]*base.Comment, 0, commentBatchSize)
+ for _, issue := range issues {
+ log.Trace("migrating issue %d's comments", issue.Number)
+ comments, err := downloader.GetComments(issue.Number)
+ if err != nil {
return err
}
- allComments = allComments[commentBatchSize:]
- }
- }
+ allComments = append(allComments, comments...)
- if len(allComments) > 0 {
- if err := uploader.CreateComments(allComments...); err != nil {
- return err
+ if len(allComments) >= commentBatchSize {
+ if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
+ return err
+ }
+
+ allComments = allComments[commentBatchSize:]
+ }
+ }
+
+ if len(allComments) > 0 {
+ if err := uploader.CreateComments(allComments...); err != nil {
+ return err
+ }
}
}
@@ -282,65 +287,64 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
return err
}
- if !opts.Comments {
- continue
- }
-
- // plain comments
- var allComments = make([]*base.Comment, 0, commentBatchSize)
- for _, pr := range prs {
- comments, err := downloader.GetComments(pr.Number)
- if err != nil {
- return err
- }
-
- allComments = append(allComments, comments...)
-
- if len(allComments) >= commentBatchSize {
- if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
+ if opts.Comments {
+ // plain comments
+ var allComments = make([]*base.Comment, 0, commentBatchSize)
+ for _, pr := range prs {
+ log.Trace("migrating pull request %d's comments", pr.Number)
+ comments, err := downloader.GetComments(pr.Number)
+ if err != nil {
return err
}
- allComments = allComments[commentBatchSize:]
- }
- }
- if len(allComments) > 0 {
- if err := uploader.CreateComments(allComments...); err != nil {
- return err
- }
- }
- // migrate reviews
- var allReviews = make([]*base.Review, 0, reviewBatchSize)
- for _, pr := range prs {
- number := pr.Number
+ allComments = append(allComments, comments...)
- // on gitlab migrations pull number change
- if pr.OriginalNumber > 0 {
- number = pr.OriginalNumber
- }
-
- reviews, err := downloader.GetReviews(number)
- if pr.OriginalNumber > 0 {
- for i := range reviews {
- reviews[i].IssueIndex = pr.Number
+ if len(allComments) >= commentBatchSize {
+ if err := uploader.CreateComments(allComments[:commentBatchSize]...); err != nil {
+ return err
+ }
+ allComments = allComments[commentBatchSize:]
}
}
- if err != nil {
- return err
- }
-
- allReviews = append(allReviews, reviews...)
-
- if len(allReviews) >= reviewBatchSize {
- if err := uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil {
+ if len(allComments) > 0 {
+ if err := uploader.CreateComments(allComments...); err != nil {
return err
}
- allReviews = allReviews[reviewBatchSize:]
}
- }
- if len(allReviews) > 0 {
- if err := uploader.CreateReviews(allReviews...); err != nil {
- return err
+
+ // migrate reviews
+ var allReviews = make([]*base.Review, 0, reviewBatchSize)
+ for _, pr := range prs {
+ number := pr.Number
+
+ // on gitlab migrations pull number change
+ if pr.OriginalNumber > 0 {
+ number = pr.OriginalNumber
+ }
+
+ reviews, err := downloader.GetReviews(number)
+ if pr.OriginalNumber > 0 {
+ for i := range reviews {
+ reviews[i].IssueIndex = pr.Number
+ }
+ }
+ if err != nil {
+ return err
+ }
+
+ allReviews = append(allReviews, reviews...)
+
+ if len(allReviews) >= reviewBatchSize {
+ if err := uploader.CreateReviews(allReviews[:reviewBatchSize]...); err != nil {
+ return err
+ }
+ allReviews = allReviews[reviewBatchSize:]
+ }
+ }
+ if len(allReviews) > 0 {
+ if err := uploader.CreateReviews(allReviews...); err != nil {
+ return err
+ }
}
}
@@ -350,7 +354,7 @@ func migrateRepository(downloader base.Downloader, uploader base.Uploader, opts
}
}
- return nil
+ return uploader.Finish()
}
// Init migrations service
diff --git a/modules/migrations/restore.go b/modules/migrations/restore.go
new file mode 100644
index 0000000000..5550aaeb03
--- /dev/null
+++ b/modules/migrations/restore.go
@@ -0,0 +1,276 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package migrations
+
+import (
+ "context"
+ "fmt"
+ "io/ioutil"
+ "os"
+ "path/filepath"
+ "strconv"
+
+ "code.gitea.io/gitea/modules/migrations/base"
+
+ "gopkg.in/yaml.v2"
+)
+
+// RepositoryRestorer implements an Downloader from the local directory
+type RepositoryRestorer struct {
+ ctx context.Context
+ baseDir string
+ repoOwner string
+ repoName string
+}
+
+// NewRepositoryRestorer creates a repository restorer which could restore repository from a dumped folder
+func NewRepositoryRestorer(ctx context.Context, baseDir string, owner, repoName string) (*RepositoryRestorer, error) {
+ baseDir, err := filepath.Abs(baseDir)
+ if err != nil {
+ return nil, err
+ }
+ return &RepositoryRestorer{
+ ctx: ctx,
+ baseDir: baseDir,
+ repoOwner: owner,
+ repoName: repoName,
+ }, nil
+}
+
+func (r *RepositoryRestorer) commentDir() string {
+ return filepath.Join(r.baseDir, "comments")
+}
+
+func (r *RepositoryRestorer) reviewDir() string {
+ return filepath.Join(r.baseDir, "reviews")
+}
+
+// SetContext set context
+func (r *RepositoryRestorer) SetContext(ctx context.Context) {
+ r.ctx = ctx
+}
+
+// GetRepoInfo returns a repository information
+func (r *RepositoryRestorer) GetRepoInfo() (*base.Repository, error) {
+ p := filepath.Join(r.baseDir, "repo.yml")
+ bs, err := ioutil.ReadFile(p)
+ if err != nil {
+ return nil, err
+ }
+
+ var opts = make(map[string]string)
+ err = yaml.Unmarshal(bs, &opts)
+ if err != nil {
+ return nil, err
+ }
+
+ isPrivate, _ := strconv.ParseBool(opts["is_private"])
+
+ return &base.Repository{
+ Owner: r.repoOwner,
+ Name: r.repoName,
+ IsPrivate: isPrivate,
+ Description: opts["description"],
+ OriginalURL: opts["original_url"],
+ CloneURL: opts["clone_addr"],
+ DefaultBranch: opts["default_branch"],
+ }, nil
+}
+
+// GetTopics return github topics
+func (r *RepositoryRestorer) GetTopics() ([]string, error) {
+ p := filepath.Join(r.baseDir, "topic.yml")
+
+ var topics = struct {
+ Topics []string `yaml:"topics"`
+ }{}
+
+ bs, err := ioutil.ReadFile(p)
+ if err != nil {
+ return nil, err
+ }
+
+ err = yaml.Unmarshal(bs, &topics)
+ if err != nil {
+ return nil, err
+ }
+ return topics.Topics, nil
+}
+
+// GetMilestones returns milestones
+func (r *RepositoryRestorer) GetMilestones() ([]*base.Milestone, error) {
+ var milestones = make([]*base.Milestone, 0, 10)
+ p := filepath.Join(r.baseDir, "milestone.yml")
+ _, err := os.Stat(p)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, nil
+ }
+ return nil, err
+ }
+
+ bs, err := ioutil.ReadFile(p)
+ if err != nil {
+ return nil, err
+ }
+
+ err = yaml.Unmarshal(bs, &milestones)
+ if err != nil {
+ return nil, err
+ }
+ return milestones, nil
+}
+
+// GetReleases returns releases
+func (r *RepositoryRestorer) GetReleases() ([]*base.Release, error) {
+ var releases = make([]*base.Release, 0, 10)
+ p := filepath.Join(r.baseDir, "release.yml")
+ _, err := os.Stat(p)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, nil
+ }
+ return nil, err
+ }
+
+ bs, err := ioutil.ReadFile(p)
+ if err != nil {
+ return nil, err
+ }
+
+ err = yaml.Unmarshal(bs, &releases)
+ if err != nil {
+ return nil, err
+ }
+ for _, rel := range releases {
+ for _, asset := range rel.Assets {
+ *asset.DownloadURL = "file://" + filepath.Join(r.baseDir, *asset.DownloadURL)
+ }
+ }
+ return releases, nil
+}
+
+// GetLabels returns labels
+func (r *RepositoryRestorer) GetLabels() ([]*base.Label, error) {
+ var labels = make([]*base.Label, 0, 10)
+ p := filepath.Join(r.baseDir, "label.yml")
+ _, err := os.Stat(p)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, nil
+ }
+ return nil, err
+ }
+
+ bs, err := ioutil.ReadFile(p)
+ if err != nil {
+ return nil, err
+ }
+
+ err = yaml.Unmarshal(bs, &labels)
+ if err != nil {
+ return nil, err
+ }
+ return labels, nil
+}
+
+// GetIssues returns issues according start and limit
+func (r *RepositoryRestorer) GetIssues(page, perPage int) ([]*base.Issue, bool, error) {
+ var issues = make([]*base.Issue, 0, 10)
+ p := filepath.Join(r.baseDir, "issue.yml")
+ _, err := os.Stat(p)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, true, nil
+ }
+ return nil, false, err
+ }
+
+ bs, err := ioutil.ReadFile(p)
+ if err != nil {
+ return nil, false, err
+ }
+
+ err = yaml.Unmarshal(bs, &issues)
+ if err != nil {
+ return nil, false, err
+ }
+ return issues, true, nil
+}
+
+// GetComments returns comments according issueNumber
+func (r *RepositoryRestorer) GetComments(issueNumber int64) ([]*base.Comment, error) {
+ var comments = make([]*base.Comment, 0, 10)
+ p := filepath.Join(r.commentDir(), fmt.Sprintf("%d.yml", issueNumber))
+ _, err := os.Stat(p)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, nil
+ }
+ return nil, err
+ }
+
+ bs, err := ioutil.ReadFile(p)
+ if err != nil {
+ return nil, err
+ }
+
+ err = yaml.Unmarshal(bs, &comments)
+ if err != nil {
+ return nil, err
+ }
+ return comments, nil
+}
+
+// GetPullRequests returns pull requests according page and perPage
+func (r *RepositoryRestorer) GetPullRequests(page, perPage int) ([]*base.PullRequest, bool, error) {
+ var pulls = make([]*base.PullRequest, 0, 10)
+ p := filepath.Join(r.baseDir, "pull_request.yml")
+ _, err := os.Stat(p)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, true, nil
+ }
+ return nil, false, err
+ }
+
+ bs, err := ioutil.ReadFile(p)
+ if err != nil {
+ return nil, false, err
+ }
+
+ err = yaml.Unmarshal(bs, &pulls)
+ if err != nil {
+ return nil, false, err
+ }
+ for _, pr := range pulls {
+ pr.PatchURL = "file://" + filepath.Join(r.baseDir, pr.PatchURL)
+ }
+ return pulls, true, nil
+}
+
+// GetReviews returns pull requests review
+func (r *RepositoryRestorer) GetReviews(pullRequestNumber int64) ([]*base.Review, error) {
+ var reviews = make([]*base.Review, 0, 10)
+ p := filepath.Join(r.reviewDir(), fmt.Sprintf("%d.yml", pullRequestNumber))
+ _, err := os.Stat(p)
+ if err != nil {
+ if os.IsNotExist(err) {
+ return nil, nil
+ }
+ return nil, err
+ }
+
+ bs, err := ioutil.ReadFile(p)
+ if err != nil {
+ return nil, err
+ }
+
+ err = yaml.Unmarshal(bs, &reviews)
+ if err != nil {
+ return nil, err
+ }
+ return reviews, nil
+}
diff --git a/modules/uri/uri.go b/modules/uri/uri.go
new file mode 100644
index 0000000000..0967a0802f
--- /dev/null
+++ b/modules/uri/uri.go
@@ -0,0 +1,40 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package uri
+
+import (
+ "fmt"
+ "io"
+ "net/http"
+ "net/url"
+ "os"
+ "strings"
+)
+
+// ErrURISchemeNotSupported represents a scheme error
+type ErrURISchemeNotSupported struct {
+ Scheme string
+}
+
+func (e ErrURISchemeNotSupported) Error() string {
+ return fmt.Sprintf("Unsupported scheme: %v", e.Scheme)
+}
+
+// Open open a local file or a remote file
+func Open(uriStr string) (io.ReadCloser, error) {
+ u, err := url.Parse(uriStr)
+ if err != nil {
+ return nil, err
+ }
+ switch strings.ToLower(u.Scheme) {
+ case "http", "https":
+ f, err := http.Get(uriStr)
+ return f.Body, err
+ case "file":
+ return os.Open(u.Path)
+ default:
+ return nil, ErrURISchemeNotSupported{Scheme: u.Scheme}
+ }
+}
diff --git a/modules/uri/uri_test.go b/modules/uri/uri_test.go
new file mode 100644
index 0000000000..8cadd6b918
--- /dev/null
+++ b/modules/uri/uri_test.go
@@ -0,0 +1,20 @@
+// Copyright 2020 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package uri
+
+import (
+ "path/filepath"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+)
+
+func TestReadURI(t *testing.T) {
+ p, err := filepath.Abs("./uri.go")
+ assert.NoError(t, err)
+ f, err := Open("file://" + p)
+ assert.NoError(t, err)
+ defer f.Close()
+}
diff --git a/routers/api/v1/repo/migrate.go b/routers/api/v1/repo/migrate.go
index ab480c29aa..3fd9300904 100644
--- a/routers/api/v1/repo/migrate.go
+++ b/routers/api/v1/repo/migrate.go
@@ -176,11 +176,8 @@ func Migrate(ctx *context.APIContext, form api.MigrateRepoOptions) {
}
if err == nil {
- repo.Status = models.RepositoryReady
- if err := models.UpdateRepositoryCols(repo, "status"); err == nil {
- notification.NotifyMigrateRepository(ctx.User, repoOwner, repo)
- return
- }
+ notification.NotifyMigrateRepository(ctx.User, repoOwner, repo)
+ return
}
if repo != nil {