From eca05b09aa269dda1309ee77ac750e29e71c3fd3 Mon Sep 17 00:00:00 2001
From: Lauris BH <lauris@nix.lv>
Date: Thu, 26 Oct 2017 04:37:33 +0300
Subject: [PATCH] Add commit count caching (#2774)
* Add commit count caching
* Small refactoring
* Add different key prefix for refs and commits
* Add configuratuion option to allow to change caching time or disable it
---
conf/app.ini | 3 ++
models/repo.go | 11 ++++++
models/update.go | 13 +++++--
modules/cache/cache.go | 72 ++++++++++++++++++++++++++++++++++++++
modules/context/repo.go | 21 +++++++++--
modules/setting/setting.go | 38 +++++++++++++-------
routers/admin/admin.go | 6 ++--
routers/init.go | 3 ++
routers/repo/commit.go | 4 +--
routers/routes/routes.go | 10 +++---
10 files changed, 153 insertions(+), 28 deletions(-)
create mode 100644 modules/cache/cache.go
diff --git a/conf/app.ini b/conf/app.ini
index 0ce8aae52e..3a850106c1 100644
--- a/conf/app.ini
+++ b/conf/app.ini
@@ -339,6 +339,9 @@ INTERVAL = 60
; redis: network=tcp,addr=:6379,password=macaron,db=0,pool_size=100,idle_timeout=180
; memcache: `127.0.0.1:11211`
HOST =
+; Time to keep items in cache if not used, default is 16 hours.
+; Setting it to 0 disables caching
+ITEM_TTL = 16h
[session]
; Either "memory", "file", or "redis", default is "memory"
diff --git a/models/repo.go b/models/repo.go
index 1b1be62f87..eca71568ee 100644
--- a/models/repo.go
+++ b/models/repo.go
@@ -258,6 +258,17 @@ func (repo *Repository) APIFormat(mode AccessMode) *api.Repository {
return repo.innerAPIFormat(mode, false)
}
+// GetCommitsCountCacheKey returns cache key used for commits count caching.
+func (repo *Repository) GetCommitsCountCacheKey(contextName string, isRef bool) string {
+ var prefix string
+ if isRef {
+ prefix = "ref"
+ } else {
+ prefix = "commit"
+ }
+ return fmt.Sprintf("commits-count-%d-%s-%s", repo.ID, prefix, contextName)
+}
+
func (repo *Repository) innerAPIFormat(mode AccessMode, isParent bool) *api.Repository {
var parent *api.Repository
diff --git a/models/update.go b/models/update.go
index 62d13ce209..82369bf636 100644
--- a/models/update.go
+++ b/models/update.go
@@ -11,7 +11,7 @@ import (
"strings"
"code.gitea.io/git"
-
+ "code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/log"
)
@@ -205,19 +205,26 @@ func pushUpdate(opts PushUpdateOptions) (repo *Repository, err error) {
var commits = &PushCommits{}
if strings.HasPrefix(opts.RefFullName, git.TagPrefix) {
// If is tag reference
+ tagName := opts.RefFullName[len(git.TagPrefix):]
if isDelRef {
- err = pushUpdateDeleteTag(repo, gitRepo, opts.RefFullName[len(git.TagPrefix):])
+ err = pushUpdateDeleteTag(repo, gitRepo, tagName)
if err != nil {
return nil, fmt.Errorf("pushUpdateDeleteTag: %v", err)
}
} else {
- err = pushUpdateAddTag(repo, gitRepo, opts.RefFullName[len(git.TagPrefix):])
+ // Clear cache for tag commit count
+ cache.Remove(repo.GetCommitsCountCacheKey(tagName, true))
+ err = pushUpdateAddTag(repo, gitRepo, tagName)
if err != nil {
return nil, fmt.Errorf("pushUpdateAddTag: %v", err)
}
}
} else if !isDelRef {
// If is branch reference
+
+ // Clear cache for branch commit count
+ cache.Remove(repo.GetCommitsCountCacheKey(opts.RefFullName[len(git.BranchPrefix):], true))
+
newCommit, err := gitRepo.GetCommit(opts.NewCommitID)
if err != nil {
return nil, fmt.Errorf("gitRepo.GetCommit: %v", err)
diff --git a/modules/cache/cache.go b/modules/cache/cache.go
new file mode 100644
index 0000000000..0a73ae8ae9
--- /dev/null
+++ b/modules/cache/cache.go
@@ -0,0 +1,72 @@
+// Copyright 2017 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package cache
+
+import (
+ "code.gitea.io/gitea/modules/setting"
+
+ mc "github.com/go-macaron/cache"
+)
+
+var conn mc.Cache
+
+// NewContext start cache service
+func NewContext() error {
+ if setting.CacheService == nil || conn != nil {
+ return nil
+ }
+
+ var err error
+ conn, err = mc.NewCacher(setting.CacheService.Adapter, mc.Options{
+ Adapter: setting.CacheService.Adapter,
+ AdapterConfig: setting.CacheService.Conn,
+ Interval: setting.CacheService.Interval,
+ })
+ return err
+}
+
+// GetInt returns key value from cache with callback when no key exists in cache
+func GetInt(key string, getFunc func() (int, error)) (int, error) {
+ if conn == nil || setting.CacheService.TTL == 0 {
+ return getFunc()
+ }
+ if !conn.IsExist(key) {
+ var (
+ value int
+ err error
+ )
+ if value, err = getFunc(); err != nil {
+ return value, err
+ }
+ conn.Put(key, value, int64(setting.CacheService.TTL.Seconds()))
+ }
+ return conn.Get(key).(int), nil
+}
+
+// GetInt64 returns key value from cache with callback when no key exists in cache
+func GetInt64(key string, getFunc func() (int64, error)) (int64, error) {
+ if conn == nil || setting.CacheService.TTL == 0 {
+ return getFunc()
+ }
+ if !conn.IsExist(key) {
+ var (
+ value int64
+ err error
+ )
+ if value, err = getFunc(); err != nil {
+ return value, err
+ }
+ conn.Put(key, value, int64(setting.CacheService.TTL.Seconds()))
+ }
+ return conn.Get(key).(int64), nil
+}
+
+// Remove key from cache
+func Remove(key string) {
+ if conn == nil {
+ return
+ }
+ conn.Delete(key)
+}
diff --git a/modules/context/repo.go b/modules/context/repo.go
index c33396c0f9..3aaf1ce64a 100644
--- a/modules/context/repo.go
+++ b/modules/context/repo.go
@@ -13,7 +13,9 @@ import (
"code.gitea.io/git"
"code.gitea.io/gitea/models"
+ "code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/setting"
+
"github.com/Unknwon/com"
"gopkg.in/editorconfig/editorconfig-core-go.v1"
"gopkg.in/macaron.v1"
@@ -100,6 +102,21 @@ func (r *Repository) CanUseTimetracker(issue *models.Issue, user *models.User) b
r.IsWriter() || issue.IsPoster(user.ID) || issue.AssigneeID == user.ID)
}
+// GetCommitsCount returns cached commit count for current view
+func (r *Repository) GetCommitsCount() (int64, error) {
+ var contextName string
+ if r.IsViewBranch {
+ contextName = r.BranchName
+ } else if r.IsViewTag {
+ contextName = r.TagName
+ } else {
+ contextName = r.CommitID
+ }
+ return cache.GetInt64(r.Repository.GetCommitsCountCacheKey(contextName, r.IsViewBranch || r.IsViewTag), func() (int64, error) {
+ return r.Commit.CommitsCount()
+ })
+}
+
// GetEditorconfig returns the .editorconfig definition if found in the
// HEAD of the default repo branch.
func (r *Repository) GetEditorconfig() (*editorconfig.Editorconfig, error) {
@@ -535,9 +552,9 @@ func RepoRef() macaron.Handler {
ctx.Data["IsViewCommit"] = ctx.Repo.IsViewCommit
ctx.Data["CanCreateBranch"] = ctx.Repo.CanCreateBranch()
- ctx.Repo.CommitsCount, err = ctx.Repo.Commit.CommitsCount()
+ ctx.Repo.CommitsCount, err = ctx.Repo.GetCommitsCount()
if err != nil {
- ctx.Handle(500, "CommitsCount", err)
+ ctx.Handle(500, "GetCommitsCount", err)
return
}
ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount
diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index 9787e09280..6c89381f3b 100644
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -325,11 +325,6 @@ var (
// Time settings
TimeFormat string
- // Cache settings
- CacheAdapter string
- CacheInterval int
- CacheConn string
-
// Session settings
SessionConfig session.Options
CSRFCookieName = "_csrf"
@@ -1295,16 +1290,33 @@ func NewXORMLogService(disableConsole bool) {
}
}
+// Cache represents cache settings
+type Cache struct {
+ Adapter string
+ Interval int
+ Conn string
+ TTL time.Duration
+}
+
+var (
+ // CacheService the global cache
+ CacheService *Cache
+)
+
func newCacheService() {
- CacheAdapter = Cfg.Section("cache").Key("ADAPTER").In("memory", []string{"memory", "redis", "memcache"})
- switch CacheAdapter {
- case "memory":
- CacheInterval = Cfg.Section("cache").Key("INTERVAL").MustInt(60)
- case "redis", "memcache":
- CacheConn = strings.Trim(Cfg.Section("cache").Key("HOST").String(), "\" ")
- default:
- log.Fatal(4, "Unknown cache adapter: %s", CacheAdapter)
+ sec := Cfg.Section("cache")
+ CacheService = &Cache{
+ Adapter: sec.Key("ADAPTER").In("memory", []string{"memory", "redis", "memcache"}),
}
+ switch CacheService.Adapter {
+ case "memory":
+ CacheService.Interval = sec.Key("INTERVAL").MustInt(60)
+ case "redis", "memcache":
+ CacheService.Conn = strings.Trim(sec.Key("HOST").String(), "\" ")
+ default:
+ log.Fatal(4, "Unknown cache adapter: %s", CacheService.Adapter)
+ }
+ CacheService.TTL = sec.Key("ITEM_TTL").MustDuration(16 * time.Hour)
log.Info("Cache Service Enabled")
}
diff --git a/routers/admin/admin.go b/routers/admin/admin.go
index 94b88a05c3..39a8f718ca 100644
--- a/routers/admin/admin.go
+++ b/routers/admin/admin.go
@@ -224,9 +224,9 @@ func Config(ctx *context.Context) {
ctx.Data["Mailer"] = setting.MailService
}
- ctx.Data["CacheAdapter"] = setting.CacheAdapter
- ctx.Data["CacheInterval"] = setting.CacheInterval
- ctx.Data["CacheConn"] = setting.CacheConn
+ ctx.Data["CacheAdapter"] = setting.CacheService.Adapter
+ ctx.Data["CacheInterval"] = setting.CacheService.Interval
+ ctx.Data["CacheConn"] = setting.CacheService.Conn
ctx.Data["SessionConfig"] = setting.SessionConfig
diff --git a/routers/init.go b/routers/init.go
index 006f285266..8cfbe39ee5 100644
--- a/routers/init.go
+++ b/routers/init.go
@@ -11,6 +11,7 @@ import (
"code.gitea.io/git"
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/models/migrations"
+ "code.gitea.io/gitea/modules/cache"
"code.gitea.io/gitea/modules/cron"
"code.gitea.io/gitea/modules/highlight"
"code.gitea.io/gitea/modules/log"
@@ -18,6 +19,7 @@ import (
"code.gitea.io/gitea/modules/markup"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/ssh"
+
macaron "gopkg.in/macaron.v1"
)
@@ -37,6 +39,7 @@ func checkRunMode() {
func NewServices() {
setting.NewServices()
mailer.NewContext()
+ cache.NewContext()
}
// GlobalInit is for global configuration reload-able.
diff --git a/routers/repo/commit.go b/routers/repo/commit.go
index 21a0d9dd9f..637a50543a 100644
--- a/routers/repo/commit.go
+++ b/routers/repo/commit.go
@@ -55,7 +55,7 @@ func Commits(ctx *context.Context) {
}
ctx.Data["PageIsViewCode"] = true
- commitsCount, err := ctx.Repo.Commit.CommitsCount()
+ commitsCount, err := ctx.Repo.GetCommitsCount()
if err != nil {
ctx.Handle(500, "GetCommitsCount", err)
return
@@ -91,7 +91,7 @@ func Graph(ctx *context.Context) {
ctx.Data["PageIsCommits"] = true
ctx.Data["PageIsViewCode"] = true
- commitsCount, err := ctx.Repo.Commit.CommitsCount()
+ commitsCount, err := ctx.Repo.GetCommitsCount()
if err != nil {
ctx.Handle(500, "GetCommitsCount", err)
return
diff --git a/routers/routes/routes.go b/routers/routes/routes.go
index 227b4fff9c..703afbb4a7 100644
--- a/routers/routes/routes.go
+++ b/routers/routes/routes.go
@@ -99,9 +99,9 @@ func NewMacaron() *macaron.Macaron {
Redirect: true,
}))
m.Use(cache.Cacher(cache.Options{
- Adapter: setting.CacheAdapter,
- AdapterConfig: setting.CacheConn,
- Interval: setting.CacheInterval,
+ Adapter: setting.CacheService.Adapter,
+ AdapterConfig: setting.CacheService.Conn,
+ Interval: setting.CacheService.Interval,
}))
m.Use(captcha.Captchaer(captcha.Options{
SubURL: setting.AppSubURL,
@@ -576,9 +576,9 @@ func RegisterRoutes(m *macaron.Macaron) {
ctx.Handle(500, "GetBranchCommit", err)
return
}
- ctx.Repo.CommitsCount, err = ctx.Repo.Commit.CommitsCount()
+ ctx.Repo.CommitsCount, err = ctx.Repo.GetCommitsCount()
if err != nil {
- ctx.Handle(500, "CommitsCount", err)
+ ctx.Handle(500, "GetCommitsCount", err)
return
}
ctx.Data["CommitsCount"] = ctx.Repo.CommitsCount