serv/forgejo
1
0
Fork 0
mirror of https://codeberg.org/forgejo/forgejo.git synced 2024-12-27 22:23:50 +03:00
Code Issues Projects Releases Packages Wiki Activity Actions 8

Improve mirror iterator (#18928)

Browse source 
* Improve mirror iterator

* fix test
This commit is contained in:
Lunny Xiao 2022-03-01 03:41:06 +08:00 committed by GitHub
parent 59959ab222
commit b75ad7b87f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 19 additions and 19 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
models/repo/mirror.go
View file

@ -120,11 +120,12 @@ func DeleteMirrorByRepoID(repoID int64) error {
}
// MirrorsIterate iterates all mirror repositories.
func MirrorsIterate(f func(idx int, bean interface{}) error) error {
func MirrorsIterate(limit int, f func(idx int, bean interface{}) error) error {
return db.GetEngine(db.DefaultContext).
Where("next_update_unix<=?", time.Now().Unix()).
And("next_update_unix!=0").
OrderBy("updated_unix ASC").
Limit(limit).
Iterate(new(Mirror), f)
}

3
models/repo/pushmirror.go
View file

@ -101,10 +101,11 @@ func GetPushMirrorsByRepoID(repoID int64) ([]*PushMirror, error) {
}
// PushMirrorsIterate iterates all push-mirror repositories.
func PushMirrorsIterate(f func(idx int, bean interface{}) error) error {
func PushMirrorsIterate(limit int, f func(idx int, bean interface{}) error) error {
return db.GetEngine(db.DefaultContext).
Where("last_update + (`interval` / ?) <= ?", time.Second, time.Now().Unix()).
And("`interval` != 0").
OrderBy("last_update ASC").
Limit(limit).
Iterate(new(PushMirror), f)
}

2
models/repo/pushmirror_test.go
View file

@ -40,7 +40,7 @@ func TestPushMirrorsIterate(t *testing.T) {
time.Sleep(1 * time.Millisecond)
PushMirrorsIterate(func(idx int, bean interface{}) error {
PushMirrorsIterate(1, func(idx int, bean interface{}) error {
m, ok := bean.(*PushMirror)
assert.True(t, ok)
assert.Equal(t, "test-1", m.RemoteName)

30
services/mirror/mirror.go
View file

@ -55,9 +55,7 @@ func Update(ctx context.Context, pullLimit, pushLimit int) error {
}
log.Trace("Doing: Update")
requested := 0
handler := func(idx int, bean interface{}, limit int) error {
handler := func(idx int, bean interface{}) error {
var item SyncRequest
var repo *repo_model.Repository
if m, ok := bean.(*repo_model.Mirror); ok {
@ -104,35 +102,35 @@ func Update(ctx context.Context, pullLimit, pushLimit int) error {
}
return err
}
requested++
if limit > 0 && requested > limit {
return errLimit
}
return nil
}
pullMirrorsRequested := 0
if pullLimit != 0 {
requested = 0
if err := repo_model.MirrorsIterate(func(idx int, bean interface{}) error {
return handler(idx, bean, pullLimit)
if err := repo_model.MirrorsIterate(pullLimit, func(idx int, bean interface{}) error {
if err := handler(idx, bean); err != nil {
return err
}
pullMirrorsRequested++
return nil
}); err != nil && err != errLimit {
log.Error("MirrorsIterate: %v", err)
return err
}
pullMirrorsRequested, requested = requested, 0
}
pushMirrorsRequested := 0
if pushLimit != 0 {
requested = 0
if err := repo_model.PushMirrorsIterate(func(idx int, bean interface{}) error {
return handler(idx, bean, pushLimit)
if err := repo_model.PushMirrorsIterate(pushLimit, func(idx int, bean interface{}) error {
if err := handler(idx, bean); err != nil {
return err
}
pushMirrorsRequested++
return nil
}); err != nil && err != errLimit {
log.Error("PushMirrorsIterate: %v", err)
return err
}
pushMirrorsRequested, requested = requested, 0
}
log.Trace("Finished: Update: %d pull mirrors and %d push mirrors queued", pullMirrorsRequested, pushMirrorsRequested)
return nil