From f9acad82ca231b2a094879e53134b0d91815ddf0 Mon Sep 17 00:00:00 2001
From: Lunny Xiao <xiaolunwen@gmail.com>
Date: Wed, 18 Aug 2021 21:10:39 +0800
Subject: [PATCH] Add proxy settings and support for migration and webhook
(#16704)
* Add proxy settings and support for migration and webhook
* Fix default value
* Add newline for example ini
* Add lfs proxy support
* Fix lint
* Follow @zeripath's review
* Fix git clone
* Fix test
* missgin http requests for proxy
* use empty
Co-authored-by: zeripath <art27@cantab.net>
Co-authored-by: 6543 <6543@obermui.de>
Co-authored-by: zeripath <art27@cantab.net>
---
custom/conf/app.example.ini | 8 ++
.../doc/advanced/config-cheat-sheet.en-us.md | 18 +++-
.../doc/advanced/config-cheat-sheet.zh-cn.md | 14 ++++
modules/git/command.go | 51 ++++++++----
modules/git/repo.go | 28 ++++++-
modules/lfs/client.go | 4 +-
modules/lfs/client_test.go | 4 +-
modules/lfs/http_client.go | 11 ++-
modules/migrations/gitea_downloader.go | 22 ++++-
modules/migrations/github.go | 14 +++-
modules/migrations/gitlab.go | 20 ++++-
modules/migrations/gogs.go | 6 +-
modules/proxy/proxy.go | 83 +++++++++++++++++++
modules/repository/repo.go | 6 +-
modules/setting/migrations.go | 2 +
modules/setting/proxy.go | 40 +++++++++
modules/setting/setting.go | 1 +
services/mirror/mirror_pull.go | 2 +-
services/mirror/mirror_push.go | 6 +-
services/webhook/deliver.go | 3 +-
20 files changed, 302 insertions(+), 41 deletions(-)
create mode 100644 modules/proxy/proxy.go
create mode 100644 modules/setting/proxy.go
diff --git a/custom/conf/app.example.ini b/custom/conf/app.example.ini
index 95dd8073a4..d0fe6150d6 100644
--- a/custom/conf/app.example.ini
+++ b/custom/conf/app.example.ini
@@ -2127,3 +2127,11 @@ PATH =
;;
;; Minio enabled ssl only available when STORAGE_TYPE is `minio`
;MINIO_USE_SSL = false
+
+;[proxy]
+;; Enable the proxy, all requests to external via HTTP will be affected
+;PROXY_ENABLED = false
+;; Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy/no_proxy
+;PROXY_URL =
+;; Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts.
+;PROXY_HOSTS =
diff --git a/docs/content/doc/advanced/config-cheat-sheet.en-us.md b/docs/content/doc/advanced/config-cheat-sheet.en-us.md
index 572e33af7f..23f125f4da 100644
--- a/docs/content/doc/advanced/config-cheat-sheet.en-us.md
+++ b/docs/content/doc/advanced/config-cheat-sheet.en-us.md
@@ -549,8 +549,8 @@ Define allowed algorithms and their minimum key length (use -1 to disable a type
- `DELIVER_TIMEOUT`: **5**: Delivery timeout (sec) for shooting webhooks.
- `SKIP_TLS_VERIFY`: **false**: Allow insecure certification.
- `PAGING_NUM`: **10**: Number of webhook history events that are shown in one page.
-- `PROXY_URL`: ****: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy
-- `PROXY_HOSTS`: ****: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts.
+- `PROXY_URL`: **\<empty\>**: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy. If not given, will use global proxy setting.
+- `PROXY_HOSTS`: **\<empty\>`**: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts. If not given, will use global proxy setting.
## Mailer (`mailer`)
@@ -950,6 +950,7 @@ Task queue configuration has been moved to `queue.task`. However, the below conf
- `ALLOWED_DOMAINS`: **\<empty\>**: Domains allowlist for migrating repositories, default is blank. It means everything will be allowed. Multiple domains could be separated by commas.
- `BLOCKED_DOMAINS`: **\<empty\>**: Domains blocklist for migrating repositories, default is blank. Multiple domains could be separated by commas. When `ALLOWED_DOMAINS` is not blank, this option will be ignored.
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918, RFC 1122, RFC 4632 and RFC 4291
+- `SKIP_TLS_VERIFY`: **false**: Allow skip tls verify
## Mirror (`mirror`)
@@ -1023,6 +1024,19 @@ is `data/repo-archive` and the default of `MINIO_BASE_PATH` is `repo-archive/`.
- `MINIO_BASE_PATH`: **repo-archive/**: Minio base path on the bucket only available when `STORAGE_TYPE` is `minio`
- `MINIO_USE_SSL`: **false**: Minio enabled ssl only available when `STORAGE_TYPE` is `minio`
+## Proxy (`proxy`)
+
+- `PROXY_ENABLED`: **false**: Enable the proxy if true, all requests to external via HTTP will be affected, if false, no proxy will be used even environment http_proxy/https_proxy
+- `PROXY_URL`: **\<empty\>**: Proxy server URL, support http://, https//, socks://, blank will follow environment http_proxy/https_proxy
+- `PROXY_HOSTS`: **\<empty\>**: Comma separated list of host names requiring proxy. Glob patterns (*) are accepted; use ** to match all hosts.
+
+i.e.
+```ini
+PROXY_ENABLED = true
+PROXY_URL = socks://127.0.0.1:1080
+PROXY_HOSTS = *.github.com
+```
+
## Other (`other`)
- `SHOW_FOOTER_BRANDING`: **false**: Show Gitea branding in the footer.
diff --git a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md
index 31b34db2f7..5c3d69ecfd 100644
--- a/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md
+++ b/docs/content/doc/advanced/config-cheat-sheet.zh-cn.md
@@ -332,6 +332,7 @@ IS_INPUT_FILE = false
- `ALLOWED_DOMAINS`: **\<empty\>**: 迁移仓库的域名白名单，默认为空，表示允许从任意域名迁移仓库，多个域名用逗号分隔。
- `BLOCKED_DOMAINS`: **\<empty\>**: 迁移仓库的域名黑名单，默认为空，多个域名用逗号分隔。如果 `ALLOWED_DOMAINS` 不为空，此选项将会被忽略。
- `ALLOW_LOCALNETWORKS`: **false**: Allow private addresses defined by RFC 1918
+- `SKIP_TLS_VERIFY`: **false**: 允许忽略 TLS 认证
## LFS (`lfs`)
@@ -397,6 +398,19 @@ Repository archive 的存储配置。 如果 `STORAGE_TYPE` 为空，则此配
- `MINIO_BASE_PATH`: **repo-archive/**: Minio base path ，仅当 `STORAGE_TYPE` 为 `minio` 时有效。
- `MINIO_USE_SSL`: **false**: Minio 是否启用 ssl ，仅当 `STORAGE_TYPE` 为 `minio` 时有效。
+## Proxy (`proxy`)
+
+- `PROXY_ENABLED`: **false**: 是否启用全局代理。如果为否，则不使用代理，环境变量中的代理也不使用
+- `PROXY_URL`: **\<empty\>**: 代理服务器地址，支持 http://, https//, socks://，为空则不启用代理而使用环境变量中的 http_proxy/https_proxy
+- `PROXY_HOSTS`: **\<empty\>**: 逗号分隔的多个需要代理的网址，支持 * 号匹配符号， ** 表示匹配所有网站
+
+i.e.
+```ini
+PROXY_ENABLED = true
+PROXY_URL = socks://127.0.0.1:1080
+PROXY_HOSTS = *.github.com
+```
+
## Other (`other`)
- `SHOW_FOOTER_BRANDING`: 为真则在页面底部显示Gitea的字样。
diff --git a/modules/git/command.go b/modules/git/command.go
index d83c42fdc2..e7496f072c 100644
--- a/modules/git/command.go
+++ b/modules/git/command.go
@@ -110,24 +110,47 @@ func (c *Command) RunInDirTimeoutEnvFullPipeline(env []string, timeout time.Dura
// RunInDirTimeoutEnvFullPipelineFunc executes the command in given directory with given timeout,
// it pipes stdout and stderr to given io.Writer and passes in an io.Reader as stdin. Between cmd.Start and cmd.Wait the passed in function is run.
func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.Duration, dir string, stdout, stderr io.Writer, stdin io.Reader, fn func(context.Context, context.CancelFunc) error) error {
- if timeout == -1 {
- timeout = defaultCommandExecutionTimeout
+ return c.RunWithContext(&RunContext{
+ Env: env,
+ Timeout: timeout,
+ Dir: dir,
+ Stdout: stdout,
+ Stderr: stderr,
+ Stdin: stdin,
+ PipelineFunc: fn,
+ })
+}
+
+// RunContext represents parameters to run the command
+type RunContext struct {
+ Env []string
+ Timeout time.Duration
+ Dir string
+ Stdout, Stderr io.Writer
+ Stdin io.Reader
+ PipelineFunc func(context.Context, context.CancelFunc) error
+}
+
+// RunWithContext run the command with context
+func (c *Command) RunWithContext(rc *RunContext) error {
+ if rc.Timeout == -1 {
+ rc.Timeout = defaultCommandExecutionTimeout
}
- if len(dir) == 0 {
+ if len(rc.Dir) == 0 {
log.Debug("%s", c)
} else {
- log.Debug("%s: %v", dir, c)
+ log.Debug("%s: %v", rc.Dir, c)
}
- ctx, cancel := context.WithTimeout(c.parentContext, timeout)
+ ctx, cancel := context.WithTimeout(c.parentContext, rc.Timeout)
defer cancel()
cmd := exec.CommandContext(ctx, c.name, c.args...)
- if env == nil {
+ if rc.Env == nil {
cmd.Env = os.Environ()
} else {
- cmd.Env = env
+ cmd.Env = rc.Env
}
cmd.Env = append(
@@ -141,23 +164,23 @@ func (c *Command) RunInDirTimeoutEnvFullPipelineFunc(env []string, timeout time.
if goVersionLessThan115 {
cmd.Env = append(cmd.Env, "GODEBUG=asyncpreemptoff=1")
}
- cmd.Dir = dir
- cmd.Stdout = stdout
- cmd.Stderr = stderr
- cmd.Stdin = stdin
+ cmd.Dir = rc.Dir
+ cmd.Stdout = rc.Stdout
+ cmd.Stderr = rc.Stderr
+ cmd.Stdin = rc.Stdin
if err := cmd.Start(); err != nil {
return err
}
desc := c.desc
if desc == "" {
- desc = fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), dir)
+ desc = fmt.Sprintf("%s %s %s [repo_path: %s]", GitExecutable, c.name, strings.Join(c.args, " "), rc.Dir)
}
pid := process.GetManager().Add(desc, cancel)
defer process.GetManager().Remove(pid)
- if fn != nil {
- err := fn(ctx, cancel)
+ if rc.PipelineFunc != nil {
+ err := rc.PipelineFunc(ctx, cancel)
if err != nil {
cancel()
_ = cmd.Wait()
diff --git a/modules/git/repo.go b/modules/git/repo.go
index 4e6f90c3ef..f2bbbf4716 100644
--- a/modules/git/repo.go
+++ b/modules/git/repo.go
@@ -9,11 +9,15 @@ import (
"bytes"
"context"
"fmt"
+ "io"
+ "net/url"
"os"
"path"
"strconv"
"strings"
"time"
+
+ "code.gitea.io/gitea/modules/proxy"
)
// GPGSettings represents the default GPG settings for this repository
@@ -99,12 +103,12 @@ type CloneRepoOptions struct {
}
// Clone clones original repository to target path.
-func Clone(from, to string, opts CloneRepoOptions) (err error) {
+func Clone(from, to string, opts CloneRepoOptions) error {
return CloneWithContext(DefaultContext, from, to, opts)
}
// CloneWithContext clones original repository to target path.
-func CloneWithContext(ctx context.Context, from, to string, opts CloneRepoOptions) (err error) {
+func CloneWithContext(ctx context.Context, from, to string, opts CloneRepoOptions) error {
cargs := make([]string, len(GlobalCommandArgs))
copy(cargs, GlobalCommandArgs)
return CloneWithArgs(ctx, from, to, cargs, opts)
@@ -146,8 +150,24 @@ func CloneWithArgs(ctx context.Context, from, to string, args []string, opts Clo
opts.Timeout = -1
}
- _, err = cmd.RunTimeout(opts.Timeout)
- return err
+ var envs = os.Environ()
+ u, err := url.Parse(from)
+ if err == nil && (strings.EqualFold(u.Scheme, "http") || strings.EqualFold(u.Scheme, "https")) {
+ if proxy.Match(u.Host) {
+ envs = append(envs, fmt.Sprintf("https_proxy=%s", proxy.GetProxyURL()))
+ }
+ }
+
+ var stderr = new(bytes.Buffer)
+ if err = cmd.RunWithContext(&RunContext{
+ Timeout: opts.Timeout,
+ Env: envs,
+ Stdout: io.Discard,
+ Stderr: stderr,
+ }); err != nil {
+ return ConcatenateError(err, stderr.String())
+ }
+ return nil
}
// PullRemoteOptions options when pull from remote
diff --git a/modules/lfs/client.go b/modules/lfs/client.go
index 0a21440f73..81b047c5bd 100644
--- a/modules/lfs/client.go
+++ b/modules/lfs/client.go
@@ -24,9 +24,9 @@ type Client interface {
}
// NewClient creates a LFS client
-func NewClient(endpoint *url.URL) Client {
+func NewClient(endpoint *url.URL, skipTLSVerify bool) Client {
if endpoint.Scheme == "file" {
return newFilesystemClient(endpoint)
}
- return newHTTPClient(endpoint)
+ return newHTTPClient(endpoint, skipTLSVerify)
}
diff --git a/modules/lfs/client_test.go b/modules/lfs/client_test.go
index 1040b39925..ee6b7a59fc 100644
--- a/modules/lfs/client_test.go
+++ b/modules/lfs/client_test.go
@@ -13,10 +13,10 @@ import (
func TestNewClient(t *testing.T) {
u, _ := url.Parse("file:///test")
- c := NewClient(u)
+ c := NewClient(u, true)
assert.IsType(t, &FilesystemClient{}, c)
u, _ = url.Parse("https://test.com/lfs")
- c = NewClient(u)
+ c = NewClient(u, true)
assert.IsType(t, &HTTPClient{}, c)
}
diff --git a/modules/lfs/http_client.go b/modules/lfs/http_client.go
index 31c67903a8..5df5ed33a9 100644
--- a/modules/lfs/http_client.go
+++ b/modules/lfs/http_client.go
@@ -7,6 +7,7 @@ package lfs
import (
"bytes"
"context"
+ "crypto/tls"
"errors"
"fmt"
"net/http"
@@ -15,6 +16,7 @@ import (
"code.gitea.io/gitea/modules/json"
"code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/proxy"
)
const batchSize = 20
@@ -32,8 +34,13 @@ func (c *HTTPClient) BatchSize() int {
return batchSize
}
-func newHTTPClient(endpoint *url.URL) *HTTPClient {
- hc := &http.Client{}
+func newHTTPClient(endpoint *url.URL, skipTLSVerify bool) *HTTPClient {
+ hc := &http.Client{
+ Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: skipTLSVerify},
+ Proxy: proxy.Proxy(),
+ },
+ }
client := &HTTPClient{
client: hc,
diff --git a/modules/migrations/gitea_downloader.go b/modules/migrations/gitea_downloader.go
index 2ed6c9113d..23ede93a42 100644
--- a/modules/migrations/gitea_downloader.go
+++ b/modules/migrations/gitea_downloader.go
@@ -6,6 +6,7 @@ package migrations
import (
"context"
+ "crypto/tls"
"errors"
"fmt"
"io"
@@ -17,6 +18,8 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
+ "code.gitea.io/gitea/modules/proxy"
+ "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
gitea_sdk "code.gitea.io/sdk/gitea"
@@ -87,6 +90,12 @@ func NewGiteaDownloader(ctx context.Context, baseURL, repoPath, username, passwo
gitea_sdk.SetToken(token),
gitea_sdk.SetBasicAuth(username, password),
gitea_sdk.SetContext(ctx),
+ gitea_sdk.SetHTTPClient(&http.Client{
+ Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+ Proxy: proxy.Proxy(),
+ },
+ }),
)
if err != nil {
log.Error(fmt.Sprintf("Failed to create NewGiteaDownloader for: %s. Error: %v", baseURL, err))
@@ -266,6 +275,13 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
Created: rel.CreatedAt,
}
+ httpClient := &http.Client{
+ Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+ Proxy: proxy.Proxy(),
+ },
+ }
+
for _, asset := range rel.Attachments {
size := int(asset.Size)
dlCount := int(asset.DownloadCount)
@@ -282,7 +298,11 @@ func (g *GiteaDownloader) convertGiteaRelease(rel *gitea_sdk.Release) *base.Rele
return nil, err
}
// FIXME: for a private download?
- resp, err := http.Get(asset.DownloadURL)
+ req, err := http.NewRequest("GET", asset.DownloadURL, nil)
+ if err != nil {
+ return nil, err
+ }
+ resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
diff --git a/modules/migrations/github.go b/modules/migrations/github.go
index cc5279e38f..f6063b0661 100644
--- a/modules/migrations/github.go
+++ b/modules/migrations/github.go
@@ -7,6 +7,7 @@ package migrations
import (
"context"
+ "crypto/tls"
"fmt"
"io"
"net/http"
@@ -17,6 +18,8 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
+ "code.gitea.io/gitea/modules/proxy"
+ "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"code.gitea.io/gitea/modules/util"
@@ -90,7 +93,7 @@ func NewGithubDownloaderV3(ctx context.Context, baseURL, userName, password, tok
Transport: &http.Transport{
Proxy: func(req *http.Request) (*url.URL, error) {
req.SetBasicAuth(userName, password)
- return nil, nil
+ return proxy.Proxy()(req)
},
},
}
@@ -269,6 +272,13 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
r.Published = rel.PublishedAt.Time
}
+ httpClient := &http.Client{
+ Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+ Proxy: proxy.Proxy(),
+ },
+ }
+
for _, asset := range rel.Assets {
var assetID = *asset.ID // Don't optimize this, for closure we need a local variable
r.Assets = append(r.Assets, &base.ReleaseAsset{
@@ -295,7 +305,7 @@ func (g *GithubDownloaderV3) convertGithubRelease(rel *github.RepositoryRelease)
if err != nil {
return nil, err
}
- resp, err := http.DefaultClient.Do(req)
+ resp, err := httpClient.Do(req)
err1 := g.RefreshRate()
if err1 != nil {
log.Error("g.client.RateLimits: %s", err1)
diff --git a/modules/migrations/gitlab.go b/modules/migrations/gitlab.go
index 1050ffd0c9..28e9eac63c 100644
--- a/modules/migrations/gitlab.go
+++ b/modules/migrations/gitlab.go
@@ -6,6 +6,7 @@ package migrations
import (
"context"
+ "crypto/tls"
"errors"
"fmt"
"io"
@@ -17,6 +18,8 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
+ "code.gitea.io/gitea/modules/proxy"
+ "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"github.com/xanzy/go-gitlab"
@@ -77,7 +80,12 @@ type GitlabDownloader struct {
// Use either a username/password, personal token entered into the username field, or anonymous/public access
// Note: Public access only allows very basic access
func NewGitlabDownloader(ctx context.Context, baseURL, repoPath, username, password, token string) (*GitlabDownloader, error) {
- gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL))
+ gitlabClient, err := gitlab.NewClient(token, gitlab.WithBaseURL(baseURL), gitlab.WithHTTPClient(&http.Client{
+ Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+ Proxy: proxy.Proxy(),
+ },
+ }))
// Only use basic auth if token is blank and password is NOT
// Basic auth will fail with empty strings, but empty token will allow anonymous public API usage
if token == "" && password != "" {
@@ -295,6 +303,13 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
PublisherName: rel.Author.Username,
}
+ httpClient := &http.Client{
+ Transport: &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
+ Proxy: proxy.Proxy(),
+ },
+ }
+
for k, asset := range rel.Assets.Links {
r.Assets = append(r.Assets, &base.ReleaseAsset{
ID: int64(asset.ID),
@@ -313,8 +328,7 @@ func (g *GitlabDownloader) convertGitlabRelease(rel *gitlab.Release) *base.Relea
return nil, err
}
req = req.WithContext(g.ctx)
-
- resp, err := http.DefaultClient.Do(req)
+ resp, err := httpClient.Do(req)
if err != nil {
return nil, err
}
diff --git a/modules/migrations/gogs.go b/modules/migrations/gogs.go
index 388020c88a..2c7fa76146 100644
--- a/modules/migrations/gogs.go
+++ b/modules/migrations/gogs.go
@@ -6,6 +6,7 @@ package migrations
import (
"context"
+ "crypto/tls"
"fmt"
"net/http"
"net/url"
@@ -14,6 +15,8 @@ import (
"code.gitea.io/gitea/modules/log"
"code.gitea.io/gitea/modules/migrations/base"
+ "code.gitea.io/gitea/modules/proxy"
+ "code.gitea.io/gitea/modules/setting"
"code.gitea.io/gitea/modules/structs"
"github.com/gogs/go-gogs-client"
@@ -95,9 +98,10 @@ func NewGogsDownloader(ctx context.Context, baseURL, userName, password, token,
downloader.userName = token
} else {
downloader.transport = &http.Transport{
+ TLSClientConfig: &tls.Config{InsecureSkipVerify: setting.Migrations.SkipTLSVerify},
Proxy: func(req *http.Request) (*url.URL, error) {
req.SetBasicAuth(userName, password)
- return nil, nil
+ return proxy.Proxy()(req)
},
}
diff --git a/modules/proxy/proxy.go b/modules/proxy/proxy.go
new file mode 100644
index 0000000000..0ab6ed3341
--- /dev/null
+++ b/modules/proxy/proxy.go
@@ -0,0 +1,83 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package proxy
+
+import (
+ "net/http"
+ "net/url"
+ "os"
+ "sync"
+
+ "code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/setting"
+
+ "github.com/gobwas/glob"
+)
+
+var (
+ once sync.Once
+ hostMatchers []glob.Glob
+)
+
+// GetProxyURL returns proxy url
+func GetProxyURL() string {
+ if !setting.Proxy.Enabled {
+ return ""
+ }
+
+ if setting.Proxy.ProxyURL == "" {
+ if os.Getenv("http_proxy") != "" {
+ return os.Getenv("http_proxy")
+ }
+ return os.Getenv("https_proxy")
+ }
+ return setting.Proxy.ProxyURL
+}
+
+// Match return true if url needs to be proxied
+func Match(u string) bool {
+ if !setting.Proxy.Enabled {
+ return false
+ }
+
+ // enforce do once
+ Proxy()
+
+ for _, v := range hostMatchers {
+ if v.Match(u) {
+ return true
+ }
+ }
+ return false
+}
+
+// Proxy returns the system proxy
+func Proxy() func(req *http.Request) (*url.URL, error) {
+ if !setting.Proxy.Enabled {
+ return nil
+ }
+ if setting.Proxy.ProxyURL == "" {
+ return http.ProxyFromEnvironment
+ }
+
+ once.Do(func() {
+ for _, h := range setting.Proxy.ProxyHosts {
+ if g, err := glob.Compile(h); err == nil {
+ hostMatchers = append(hostMatchers, g)
+ } else {
+ log.Error("glob.Compile %s failed: %v", h, err)
+ }
+ }
+ })
+
+ return func(req *http.Request) (*url.URL, error) {
+ for _, v := range hostMatchers {
+ if v.Match(req.URL.Host) {
+ return http.ProxyURL(setting.Proxy.ProxyURLFixed)(req)
+ }
+ }
+ return http.ProxyFromEnvironment(req)
+ }
+}
diff --git a/modules/repository/repo.go b/modules/repository/repo.go
index 08531c04ed..6b87039775 100644
--- a/modules/repository/repo.go
+++ b/modules/repository/repo.go
@@ -126,7 +126,7 @@ func MigrateRepositoryGitData(ctx context.Context, u *models.User, repo *models.
if opts.LFS {
ep := lfs.DetermineEndpoint(opts.CloneAddr, opts.LFSEndpoint)
- if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep); err != nil {
+ if err = StoreMissingLfsObjectsInRepository(ctx, repo, gitRepo, ep, setting.Migrations.SkipTLSVerify); err != nil {
log.Error("Failed to store missing LFS objects for repository: %v", err)
}
}
@@ -316,8 +316,8 @@ func PushUpdateAddTag(repo *models.Repository, gitRepo *git.Repository, tagName
}
// StoreMissingLfsObjectsInRepository downloads missing LFS objects
-func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL) error {
- client := lfs.NewClient(endpoint)
+func StoreMissingLfsObjectsInRepository(ctx context.Context, repo *models.Repository, gitRepo *git.Repository, endpoint *url.URL, skipTLSVerify bool) error {
+ client := lfs.NewClient(endpoint, skipTLSVerify)
contentStore := lfs.NewContentStore()
pointerChan := make(chan lfs.PointerBlob)
diff --git a/modules/setting/migrations.go b/modules/setting/migrations.go
index 7808df5280..b663b52f89 100644
--- a/modules/setting/migrations.go
+++ b/modules/setting/migrations.go
@@ -16,6 +16,7 @@ var (
AllowedDomains []string
BlockedDomains []string
AllowLocalNetworks bool
+ SkipTLSVerify bool
}{
MaxAttempts: 3,
RetryBackoff: 3,
@@ -37,4 +38,5 @@ func newMigrationsService() {
}
Migrations.AllowLocalNetworks = sec.Key("ALLOW_LOCALNETWORKS").MustBool(false)
+ Migrations.SkipTLSVerify = sec.Key("SKIP_TLS_VERIFY").MustBool(false)
}
diff --git a/modules/setting/proxy.go b/modules/setting/proxy.go
new file mode 100644
index 0000000000..b99237a398
--- /dev/null
+++ b/modules/setting/proxy.go
@@ -0,0 +1,40 @@
+// Copyright 2021 The Gitea Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package setting
+
+import (
+ "net/url"
+
+ "code.gitea.io/gitea/modules/log"
+)
+
+var (
+ // Proxy settings
+ Proxy = struct {
+ Enabled bool
+ ProxyURL string
+ ProxyURLFixed *url.URL
+ ProxyHosts []string
+ }{
+ Enabled: false,
+ ProxyURL: "",
+ ProxyHosts: []string{},
+ }
+)
+
+func newProxyService() {
+ sec := Cfg.Section("proxy")
+ Proxy.Enabled = sec.Key("PROXY_ENABLED").MustBool(false)
+ Proxy.ProxyURL = sec.Key("PROXY_URL").MustString("")
+ if Proxy.ProxyURL != "" {
+ var err error
+ Proxy.ProxyURLFixed, err = url.Parse(Proxy.ProxyURL)
+ if err != nil {
+ log.Error("Global PROXY_URL is not valid")
+ Proxy.ProxyURL = ""
+ }
+ }
+ Proxy.ProxyHosts = sec.Key("PROXY_HOSTS").Strings(",")
+}
diff --git a/modules/setting/setting.go b/modules/setting/setting.go
index d584ed3d4d..441bceda20 100644
--- a/modules/setting/setting.go
+++ b/modules/setting/setting.go
@@ -1195,6 +1195,7 @@ func NewServices() {
newMailService()
newRegisterMailService()
newNotifyMailService()
+ newProxyService()
newWebhookService()
newMigrationsService()
newIndexerService()
diff --git a/services/mirror/mirror_pull.go b/services/mirror/mirror_pull.go
index 89b5df4638..5bd08fa9bf 100644
--- a/services/mirror/mirror_pull.go
+++ b/services/mirror/mirror_pull.go
@@ -196,7 +196,7 @@ func runSync(ctx context.Context, m *models.Mirror) ([]*mirrorSyncResult, bool)
if m.LFS && setting.LFS.StartServer {
log.Trace("SyncMirrors [repo: %-v]: syncing LFS objects...", m.Repo)
ep := lfs.DetermineEndpoint(remoteAddr.String(), m.LFSEndpoint)
- if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep); err != nil {
+ if err = repo_module.StoreMissingLfsObjectsInRepository(ctx, m.Repo, gitRepo, ep, false); err != nil {
log.Error("Failed to synchronize LFS objects for repository: %v", err)
}
}
diff --git a/services/mirror/mirror_push.go b/services/mirror/mirror_push.go
index de81303689..c1f53196e3 100644
--- a/services/mirror/mirror_push.go
+++ b/services/mirror/mirror_push.go
@@ -134,7 +134,7 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error {
defer gitRepo.Close()
ep := lfs.DetermineEndpoint(remoteAddr.String(), "")
- if err := pushAllLFSObjects(ctx, gitRepo, ep); err != nil {
+ if err := pushAllLFSObjects(ctx, gitRepo, ep, false); err != nil {
return util.NewURLSanitizedError(err, remoteAddr, true)
}
}
@@ -176,8 +176,8 @@ func runPushSync(ctx context.Context, m *models.PushMirror) error {
return nil
}
-func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *url.URL) error {
- client := lfs.NewClient(endpoint)
+func pushAllLFSObjects(ctx context.Context, gitRepo *git.Repository, endpoint *url.URL, skipTLSVerify bool) error {
+ client := lfs.NewClient(endpoint, skipTLSVerify)
contentStore := lfs.NewContentStore()
pointerChan := make(chan lfs.PointerBlob)
diff --git a/services/webhook/deliver.go b/services/webhook/deliver.go
index 8243fde1bb..d0e115b136 100644
--- a/services/webhook/deliver.go
+++ b/services/webhook/deliver.go
@@ -25,6 +25,7 @@ import (
"code.gitea.io/gitea/models"
"code.gitea.io/gitea/modules/graceful"
"code.gitea.io/gitea/modules/log"
+ "code.gitea.io/gitea/modules/proxy"
"code.gitea.io/gitea/modules/setting"
"github.com/gobwas/glob"
)
@@ -260,7 +261,7 @@ var (
func webhookProxy() func(req *http.Request) (*url.URL, error) {
if setting.Webhook.ProxyURL == "" {
- return http.ProxyFromEnvironment
+ return proxy.Proxy()
}
once.Do(func() {