From 8cd987af0c4669623ec350a50096433583e74828 Mon Sep 17 00:00:00 2001
From: Sandro Santilli <strk@kbt.io>
Date: Sun, 31 Dec 2017 15:45:46 +0100
Subject: [PATCH] Add repo-sync-releases admin command (#3254)
* Add repo-sync-releases admin command
Will help recovering corrupted database, see #3247
* Load repos in chunks of 10, exit with error if unable to get a list, scan private repos, fix typo
* Fix debug output about num releases
* Introduce RepositoryListDefaultPageSize constant, set to 64
Use it from the new admin command
* Use RepositoryListDefaultPageSize in more places
* Document RepositoryListDefaultPageSize
---
cmd/admin.go | 75 ++++++++++++++++++++++++++++++++++++++++
models/issue_indexer.go | 2 +-
models/migrations/v39.go | 2 +-
models/repo_indexer.go | 2 +-
models/repo_list.go | 7 ++++
5 files changed, 85 insertions(+), 3 deletions(-)
diff --git a/cmd/admin.go b/cmd/admin.go
index 928bc1ebba..4df28358a2 100644
--- a/cmd/admin.go
+++ b/cmd/admin.go
@@ -8,7 +8,9 @@ package cmd
import (
"fmt"
+ "code.gitea.io/git"
"code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/setting"
"github.com/urfave/cli"
@@ -24,6 +26,7 @@ to make automatic initialization process more smoothly`,
Subcommands: []cli.Command{
subcmdCreateUser,
subcmdChangePassword,
+ subcmdRepoSyncReleases,
},
}
@@ -76,6 +79,12 @@ to make automatic initialization process more smoothly`,
},
},
}
+
+ subcmdRepoSyncReleases = cli.Command{
+ Name: "repo-sync-releases",
+ Usage: "Synchronize repository releases with tags",
+ Action: runRepoSyncReleases,
+ }
)
func runChangePassword(c *cli.Context) error {
@@ -145,3 +154,69 @@ func runCreateUser(c *cli.Context) error {
fmt.Printf("New user '%s' has been successfully created!\n", c.String("name"))
return nil
}
+
+func runRepoSyncReleases(c *cli.Context) error {
+
+ setting.NewContext()
+ models.LoadConfigs()
+
+ setting.NewXORMLogService(false)
+ if err := models.SetEngine(); err != nil {
+ return fmt.Errorf("models.SetEngine: %v", err)
+ }
+
+ log.Trace("Synchronizing repository releases (this may take a while)")
+ for page := 1; ; page++ {
+ repos, count, err := models.SearchRepositoryByName(&models.SearchRepoOptions{
+ Page: page,
+ PageSize: models.RepositoryListDefaultPageSize,
+ Private: true,
+ })
+ if err != nil {
+ log.Fatal(4, "SearchRepositoryByName: %v", err)
+ return err
+ }
+ if len(repos) == 0 {
+ break
+ }
+ log.Trace("Processing next %d repos of %d", len(repos), count)
+ for _, repo := range repos {
+ log.Trace("Synchronizing repo %s with path %s", repo.FullName(), repo.RepoPath())
+ gitRepo, err := git.OpenRepository(repo.RepoPath())
+ if err != nil {
+ log.Warn("OpenRepository: %v", err)
+ continue
+ }
+
+ oldnum, err := models.GetReleaseCountByRepoID(repo.ID,
+ models.FindReleasesOptions{
+ IncludeDrafts: false,
+ IncludeTags: true,
+ })
+ if err != nil {
+ log.Warn(" GetReleaseCountByRepoID: %v", err)
+ }
+ log.Trace(" currentNumReleases is %d, running SyncReleasesWithTags", oldnum)
+
+ if err = models.SyncReleasesWithTags(repo, gitRepo); err != nil {
+ log.Warn(" SyncReleasesWithTags: %v", err)
+ continue
+ }
+
+ count, err = models.GetReleaseCountByRepoID(repo.ID,
+ models.FindReleasesOptions{
+ IncludeDrafts: false,
+ IncludeTags: true,
+ })
+ if err != nil {
+ log.Warn(" GetReleaseCountByRepoID: %v", err)
+ continue
+ }
+
+ log.Trace(" repo %s releases synchronized to tags: from %d to %d",
+ repo.FullName(), oldnum, count)
+ }
+ }
+
+ return nil
+}
diff --git a/models/issue_indexer.go b/models/issue_indexer.go
index 26d053a5d7..922b66f95e 100644
--- a/models/issue_indexer.go
+++ b/models/issue_indexer.go
@@ -29,7 +29,7 @@ func populateIssueIndexer() error {
for page := 1; ; page++ {
repos, _, err := SearchRepositoryByName(&SearchRepoOptions{
Page: page,
- PageSize: 10,
+ PageSize: RepositoryListDefaultPageSize,
OrderBy: SearchOrderByID,
Private: true,
Collaborate: util.OptionalBoolFalse,
diff --git a/models/migrations/v39.go b/models/migrations/v39.go
index 3547ef1f9e..42197e80f2 100644
--- a/models/migrations/v39.go
+++ b/models/migrations/v39.go
@@ -31,7 +31,7 @@ func releaseAddColumnIsTagAndSyncTags(x *xorm.Engine) error {
// For the sake of SQLite3, we can't use x.Iterate here.
offset := 0
- pageSize := 20
+ pageSize := models.RepositoryListDefaultPageSize
for {
repos := make([]*models.Repository, 0, pageSize)
if err := x.Table("repository").Asc("id").Limit(pageSize, offset).Find(&repos); err != nil {
diff --git a/models/repo_indexer.go b/models/repo_indexer.go
index 4877d339f8..8cc08904ed 100644
--- a/models/repo_indexer.go
+++ b/models/repo_indexer.go
@@ -81,7 +81,7 @@ func populateRepoIndexer() error {
for page := 1; ; page++ {
repos, _, err := SearchRepositoryByName(&SearchRepoOptions{
Page: page,
- PageSize: 10,
+ PageSize: RepositoryListDefaultPageSize,
OrderBy: SearchOrderByID,
Private: true,
})
diff --git a/models/repo_list.go b/models/repo_list.go
index d13266f208..bc9b831d30 100644
--- a/models/repo_list.go
+++ b/models/repo_list.go
@@ -13,6 +13,13 @@ import (
"github.com/go-xorm/builder"
)
+// RepositoryListDefaultPageSize is the default number of repositories
+// to load in memory when running administrative tasks on all (or almost
+// all) of them.
+// The number should be low enough to avoid filling up all RAM with
+// repository data...
+const RepositoryListDefaultPageSize = 64
+
// RepositoryList contains a list of repositories
type RepositoryList []*Repository