From 9e5d0a09eb7370daedb1cf572d25ffa150eacf17 Mon Sep 17 00:00:00 2001
From: Lunny Xiao <xiaolunwen@gmail.com>
Date: Fri, 16 Mar 2018 22:04:33 +0800
Subject: [PATCH] Global code search support (#3664)
* add global code search on explore
* fix bug when no anyone public repos
* change the icon
* fix typo and add UnitTypeCode check for login non-admin user
* fix ui description when no match
---
models/repo.go | 6 ++
models/repo_list.go | 25 +++++++
modules/indexer/repo.go | 27 ++++++--
modules/search/search.go | 6 +-
options/locale/locale_en-US.ini | 3 +
routers/home.go | 116 ++++++++++++++++++++++++++++++++
routers/repo/search.go | 3 +-
routers/routes/routes.go | 1 +
templates/explore/code.tmpl | 55 +++++++++++++++
templates/explore/navbar.tmpl | 5 ++
10 files changed, 238 insertions(+), 9 deletions(-)
create mode 100644 templates/explore/code.tmpl
diff --git a/models/repo.go b/models/repo.go
index cddd57dc38..e9b3307d9a 100644
--- a/models/repo.go
+++ b/models/repo.go
@@ -1945,6 +1945,12 @@ func GetRepositoryByID(id int64) (*Repository, error) {
return getRepositoryByID(x, id)
}
+// GetRepositoriesMapByIDs returns the repositories by given id slice.
+func GetRepositoriesMapByIDs(ids []int64) (map[int64]*Repository, error) {
+ var repos = make(map[int64]*Repository, len(ids))
+ return repos, x.In("id", ids).Find(&repos)
+}
+
// GetUserRepositories returns a list of repositories of given user.
func GetUserRepositories(userID int64, private bool, page, pageSize int, orderBy string) ([]*Repository, error) {
if len(orderBy) == 0 {
diff --git a/models/repo_list.go b/models/repo_list.go
index bc9b831d30..df6b81cb8d 100644
--- a/models/repo_list.go
+++ b/models/repo_list.go
@@ -249,3 +249,28 @@ func SearchRepositoryByName(opts *SearchRepoOptions) (RepositoryList, int64, err
return repos, count, nil
}
+
+// FindUserAccessibleRepoIDs find all accessible repositories' ID by user's id
+func FindUserAccessibleRepoIDs(userID int64) ([]int64, error) {
+ var accessCond builder.Cond = builder.Eq{"is_private": false}
+
+ if userID > 0 {
+ accessCond = accessCond.Or(
+ builder.Eq{"owner_id": userID},
+ builder.And(
+ builder.Expr("id IN (SELECT repo_id FROM `access` WHERE access.user_id = ?)", userID),
+ builder.Neq{"owner_id": userID},
+ ),
+ )
+ }
+
+ repoIDs := make([]int64, 0, 10)
+ if err := x.
+ Table("repository").
+ Cols("id").
+ Where(accessCond).
+ Find(&repoIDs); err != nil {
+ return nil, fmt.Errorf("FindUserAccesibleRepoIDs: %v", err)
+ }
+ return repoIDs, nil
+}
diff --git a/modules/indexer/repo.go b/modules/indexer/repo.go
index ffb1dc1e62..4d1e792152 100644
--- a/modules/indexer/repo.go
+++ b/modules/indexer/repo.go
@@ -16,6 +16,7 @@ import (
"github.com/blevesearch/bleve/analysis/token/lowercase"
"github.com/blevesearch/bleve/analysis/token/unique"
"github.com/blevesearch/bleve/analysis/tokenizer/unicode"
+ "github.com/blevesearch/bleve/search/query"
"github.com/ethantkoenig/rupture"
)
@@ -158,6 +159,7 @@ func DeleteRepoFromIndexer(repoID int64) error {
// RepoSearchResult result of performing a search in a repo
type RepoSearchResult struct {
+ RepoID int64
StartIndex int
EndIndex int
Filename string
@@ -166,17 +168,29 @@ type RepoSearchResult struct {
// SearchRepoByKeyword searches for files in the specified repo.
// Returns the matching file-paths
-func SearchRepoByKeyword(repoID int64, keyword string, page, pageSize int) (int64, []*RepoSearchResult, error) {
+func SearchRepoByKeyword(repoIDs []int64, keyword string, page, pageSize int) (int64, []*RepoSearchResult, error) {
phraseQuery := bleve.NewMatchPhraseQuery(keyword)
phraseQuery.FieldVal = "Content"
phraseQuery.Analyzer = repoIndexerAnalyzer
- indexerQuery := bleve.NewConjunctionQuery(
- numericEqualityQuery(repoID, "RepoID"),
- phraseQuery,
- )
+
+ var indexerQuery query.Query
+ if len(repoIDs) > 0 {
+ var repoQueries = make([]query.Query, 0, len(repoIDs))
+ for _, repoID := range repoIDs {
+ repoQueries = append(repoQueries, numericEqualityQuery(repoID, "RepoID"))
+ }
+
+ indexerQuery = bleve.NewConjunctionQuery(
+ bleve.NewDisjunctionQuery(repoQueries...),
+ phraseQuery,
+ )
+ } else {
+ indexerQuery = phraseQuery
+ }
+
from := (page - 1) * pageSize
searchRequest := bleve.NewSearchRequestOptions(indexerQuery, pageSize, from, false)
- searchRequest.Fields = []string{"Content"}
+ searchRequest.Fields = []string{"Content", "RepoID"}
searchRequest.IncludeLocations = true
result, err := repoIndexer.Search(searchRequest)
@@ -199,6 +213,7 @@ func SearchRepoByKeyword(repoID int64, keyword string, page, pageSize int) (int6
}
}
searchResults[i] = &RepoSearchResult{
+ RepoID: int64(hit.Fields["RepoID"].(float64)),
StartIndex: startIndex,
EndIndex: endIndex,
Filename: filenameOfIndexerID(hit.ID),
diff --git a/modules/search/search.go b/modules/search/search.go
index db0c0a6168..9b93fe58fb 100644
--- a/modules/search/search.go
+++ b/modules/search/search.go
@@ -17,6 +17,7 @@ import (
// Result a search result to display
type Result struct {
+ RepoID int64
Filename string
HighlightClass string
LineNumbers []int
@@ -98,6 +99,7 @@ func searchResult(result *indexer.RepoSearchResult, startIndex, endIndex int) (*
index += len(line)
}
return &Result{
+ RepoID: result.RepoID,
Filename: result.Filename,
HighlightClass: highlight.FileNameToHighlightClass(result.Filename),
LineNumbers: lineNumbers,
@@ -106,12 +108,12 @@ func searchResult(result *indexer.RepoSearchResult, startIndex, endIndex int) (*
}
// PerformSearch perform a search on a repository
-func PerformSearch(repoID int64, keyword string, page, pageSize int) (int, []*Result, error) {
+func PerformSearch(repoIDs []int64, keyword string, page, pageSize int) (int, []*Result, error) {
if len(keyword) == 0 {
return 0, nil, nil
}
- total, results, err := indexer.SearchRepoByKeyword(repoID, keyword, page, pageSize)
+ total, results, err := indexer.SearchRepoByKeyword(repoIDs, keyword, page, pageSize)
if err != nil {
return 0, nil, err
}
diff --git a/options/locale/locale_en-US.ini b/options/locale/locale_en-US.ini
index 8ff353e410..2e51f68c2a 100644
--- a/options/locale/locale_en-US.ini
+++ b/options/locale/locale_en-US.ini
@@ -169,9 +169,12 @@ repos = Repositories
users = Users
organizations = Organizations
search = Search
+code = Code
repo_no_results = No matching repositories have been found.
user_no_results = No matching users have been found.
org_no_results = No matching organizations have been found.
+code_no_results = No matching codes have been found.
+code_search_results = Search results for "%s"
[auth]
create_new_account = Create Account
diff --git a/routers/home.go b/routers/home.go
index 74125aa2df..4810eb4e6f 100644
--- a/routers/home.go
+++ b/routers/home.go
@@ -11,6 +11,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/base"
"code.gitea.io/gitea/modules/context"
+ "code.gitea.io/gitea/modules/search"
"code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/util"
"code.gitea.io/gitea/routers/user"
@@ -27,6 +28,8 @@ const (
tplExploreUsers base.TplName = "explore/users"
// tplExploreOrganizations explore organizations page template
tplExploreOrganizations base.TplName = "explore/organizations"
+ // tplExploreCode explore code page template
+ tplExploreCode base.TplName = "explore/code"
)
// Home render home page
@@ -49,6 +52,7 @@ func Home(ctx *context.Context) {
}
ctx.Data["PageIsHome"] = true
+ ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
ctx.HTML(200, tplHome)
}
@@ -124,6 +128,7 @@ func RenderRepoSearch(ctx *context.Context, opts *RepoSearchOptions) {
ctx.Data["Total"] = count
ctx.Data["Page"] = paginater.New(int(count), opts.PageSize, page, 5)
ctx.Data["Repos"] = repos
+ ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
ctx.HTML(200, opts.TplName)
}
@@ -133,6 +138,7 @@ func ExploreRepos(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("explore")
ctx.Data["PageIsExplore"] = true
ctx.Data["PageIsExploreRepositories"] = true
+ ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
var ownerID int64
if ctx.User != nil && !ctx.User.IsAdmin {
@@ -194,6 +200,7 @@ func RenderUserSearch(ctx *context.Context, opts *models.SearchUserOptions, tplN
ctx.Data["Page"] = paginater.New(int(count), opts.PageSize, opts.Page, 5)
ctx.Data["Users"] = users
ctx.Data["ShowUserEmail"] = setting.UI.ShowUserEmail
+ ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
ctx.HTML(200, tplName)
}
@@ -203,6 +210,7 @@ func ExploreUsers(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("explore")
ctx.Data["PageIsExplore"] = true
ctx.Data["PageIsExploreUsers"] = true
+ ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
RenderUserSearch(ctx, &models.SearchUserOptions{
Type: models.UserTypeIndividual,
@@ -216,6 +224,7 @@ func ExploreOrganizations(ctx *context.Context) {
ctx.Data["Title"] = ctx.Tr("explore")
ctx.Data["PageIsExplore"] = true
ctx.Data["PageIsExploreOrganizations"] = true
+ ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
RenderUserSearch(ctx, &models.SearchUserOptions{
Type: models.UserTypeOrganization,
@@ -223,6 +232,113 @@ func ExploreOrganizations(ctx *context.Context) {
}, tplExploreOrganizations)
}
+// ExploreCode render explore code page
+func ExploreCode(ctx *context.Context) {
+ if !setting.Indexer.RepoIndexerEnabled {
+ ctx.Redirect("/explore", 302)
+ return
+ }
+
+ ctx.Data["IsRepoIndexerEnabled"] = setting.Indexer.RepoIndexerEnabled
+ ctx.Data["Title"] = ctx.Tr("explore")
+ ctx.Data["PageIsExplore"] = true
+ ctx.Data["PageIsExploreCode"] = true
+
+ keyword := strings.TrimSpace(ctx.Query("q"))
+ page := ctx.QueryInt("page")
+ if page <= 0 {
+ page = 1
+ }
+
+ var (
+ repoIDs []int64
+ err error
+ isAdmin bool
+ userID int64
+ )
+ if ctx.User != nil {
+ userID = ctx.User.ID
+ isAdmin = ctx.User.IsAdmin
+ }
+
+ // guest user or non-admin user
+ if ctx.User == nil || !isAdmin {
+ repoIDs, err = models.FindUserAccessibleRepoIDs(userID)
+ if err != nil {
+ ctx.ServerError("SearchResults", err)
+ return
+ }
+ }
+
+ var (
+ total int
+ searchResults []*search.Result
+ )
+
+ // if non-admin login user, we need check UnitTypeCode at first
+ if ctx.User != nil && len(repoIDs) > 0 {
+ repoMaps, err := models.GetRepositoriesMapByIDs(repoIDs)
+ if err != nil {
+ ctx.ServerError("SearchResults", err)
+ return
+ }
+
+ var rightRepoMap = make(map[int64]*models.Repository, len(repoMaps))
+ repoIDs = make([]int64, 0, len(repoMaps))
+ for id, repo := range repoMaps {
+ if repo.CheckUnitUser(userID, isAdmin, models.UnitTypeCode) {
+ rightRepoMap[id] = repo
+ repoIDs = append(repoIDs, id)
+ }
+ }
+
+ ctx.Data["RepoMaps"] = rightRepoMap
+
+ total, searchResults, err = search.PerformSearch(repoIDs, keyword, page, setting.UI.RepoSearchPagingNum)
+ if err != nil {
+ ctx.ServerError("SearchResults", err)
+ return
+ }
+ // if non-login user or isAdmin, no need to check UnitTypeCode
+ } else if (ctx.User == nil && len(repoIDs) > 0) || isAdmin {
+ total, searchResults, err = search.PerformSearch(repoIDs, keyword, page, setting.UI.RepoSearchPagingNum)
+ if err != nil {
+ ctx.ServerError("SearchResults", err)
+ return
+ }
+
+ var loadRepoIDs = make([]int64, 0, len(searchResults))
+ for _, result := range searchResults {
+ var find bool
+ for _, id := range loadRepoIDs {
+ if id == result.RepoID {
+ find = true
+ break
+ }
+ }
+ if !find {
+ loadRepoIDs = append(loadRepoIDs, result.RepoID)
+ }
+ }
+
+ repoMaps, err := models.GetRepositoriesMapByIDs(loadRepoIDs)
+ if err != nil {
+ ctx.ServerError("SearchResults", err)
+ return
+ }
+
+ ctx.Data["RepoMaps"] = repoMaps
+ }
+
+ ctx.Data["Keyword"] = keyword
+ pager := paginater.New(total, setting.UI.RepoSearchPagingNum, page, 5)
+ ctx.Data["Page"] = pager
+ ctx.Data["SearchResults"] = searchResults
+ ctx.Data["RequireHighlightJS"] = true
+ ctx.Data["PageIsViewCode"] = true
+ ctx.HTML(200, tplExploreCode)
+}
+
// NotFound render 404 page
func NotFound(ctx *context.Context) {
ctx.Data["Title"] = "Page Not Found"
diff --git a/routers/repo/search.go b/routers/repo/search.go
index ed209f2ddc..95715c30c6 100644
--- a/routers/repo/search.go
+++ b/routers/repo/search.go
@@ -29,7 +29,8 @@ func Search(ctx *context.Context) {
if page <= 0 {
page = 1
}
- total, searchResults, err := search.PerformSearch(ctx.Repo.Repository.ID, keyword, page, setting.UI.RepoSearchPagingNum)
+ total, searchResults, err := search.PerformSearch([]int64{ctx.Repo.Repository.ID},
+ keyword, page, setting.UI.RepoSearchPagingNum)
if err != nil {
ctx.ServerError("SearchResults", err)
return
diff --git a/routers/routes/routes.go b/routers/routes/routes.go
index 1d95bb4c76..da7157080a 100644
--- a/routers/routes/routes.go
+++ b/routers/routes/routes.go
@@ -170,6 +170,7 @@ func RegisterRoutes(m *macaron.Macaron) {
m.Get("/repos", routers.ExploreRepos)
m.Get("/users", routers.ExploreUsers)
m.Get("/organizations", routers.ExploreOrganizations)
+ m.Get("/code", routers.ExploreCode)
}, ignSignIn)
m.Combo("/install", routers.InstallInit).Get(routers.Install).
Post(bindIgnErr(auth.InstallForm{}), routers.InstallPost)
diff --git a/templates/explore/code.tmpl b/templates/explore/code.tmpl
new file mode 100644
index 0000000000..6003486213
--- /dev/null
+++ b/templates/explore/code.tmpl
@@ -0,0 +1,55 @@
+{{template "base/head" .}}
+<div class="explore users">
+ {{template "explore/navbar" .}}
+ <div class="ui container">
+ <form class="ui form" style="max-width: 100%">
+ <div class="ui fluid action input">
+ <input name="q" value="{{.Keyword}}" placeholder="{{.i18n.Tr "explore.search"}}..." autofocus>
+ <input type="hidden" name="tab" value="{{$.TabName}}">
+ <button class="ui blue button">{{.i18n.Tr "explore.search"}}</button>
+ </div>
+ </form>
+ <div class="ui divider"></div>
+
+ <div class="ui user list">
+ {{if .SearchResults}}
+ <h3>
+ {{.i18n.Tr "explore.code_search_results" (.Keyword|Escape) | Str2html }}
+ </h3>
+ <div class="repository search">
+ {{range $result := .SearchResults}}
+ {{$repo := (index $.RepoMaps .RepoID)}}
+ <div class="diff-file-box diff-box file-content non-diff-file-content repo-search-result">
+ <h4 class="ui top attached normal header">
+ <span class="file"><a rel="nofollow" href="{{EscapePound $repo.HTMLURL}}">{{$repo.FullName}}</a> - {{.Filename}}</span>
+ <a class="ui basic grey tiny button" rel="nofollow" href="{{EscapePound $repo.HTMLURL}}/src/branch/{{$repo.DefaultBranch}}/{{EscapePound .Filename}}">{{$.i18n.Tr "repo.diff.view_file"}}</a>
+ </h4>
+ <div class="ui attached table segment">
+ <div class="file-body file-code code-view">
+ <table>
+ <tbody>
+ <tr>
+ <td class="lines-num">
+ {{range .LineNumbers}}
+ <a href="{{EscapePound $repo.HTMLURL}}/src/branch/{{$repo.DefaultBranch}}/{{EscapePound $result.Filename}}#L{{.}}"><span>{{.}}</span></a>
+ {{end}}
+ </td>
+ <td class="lines-code"><pre><code class="{{.HighlightClass}}"><ol class="linenums">{{.FormattedLines}}</ol></code></pre></td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </div>
+ </div>
+ {{end}}
+ </div>
+ {{else}}
+ <div>{{$.i18n.Tr "explore.code_no_results"}}</div>
+ {{end}}
+ </div>
+
+ {{template "base/paginate" .}}
+ </div>
+</div>
+{{template "base/footer" .}}
+
diff --git a/templates/explore/navbar.tmpl b/templates/explore/navbar.tmpl
index aa4a21d590..3bd52645e2 100644
--- a/templates/explore/navbar.tmpl
+++ b/templates/explore/navbar.tmpl
@@ -8,4 +8,9 @@
<a class="{{if .PageIsExploreOrganizations}}active{{end}} item" href="{{AppSubUrl}}/explore/organizations">
<span class="octicon octicon-organization"></span> {{.i18n.Tr "explore.organizations"}}
</a>
+ {{if .IsRepoIndexerEnabled}}
+ <a class="{{if .PageIsExploreCode}}active{{end}} item" href="{{AppSubUrl}}/explore/code">
+ <span class="octicon octicon-code"></span> {{.i18n.Tr "explore.code"}}
+ </a>
+ {{end}}
</div>