From 5db2f81695c27016c154743929fe727cbfa0c8f9 Mon Sep 17 00:00:00 2001 From: Mohammed Al Sahaf Date: Thu, 6 Jun 2024 11:33:19 +0300 Subject: [PATCH 01/57] ci: add version key for .goreleaser.yml (#6376) Signed-off-by: Mohammed Al Sahaf --- .github/workflows/ci.yml | 2 +- .github/workflows/release.yml | 2 +- .goreleaser.yml | 2 ++ 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index cbd794c1..e6fe6d75 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -175,7 +175,7 @@ jobs: - name: Checkout code uses: actions/checkout@v4 - - uses: goreleaser/goreleaser-action@v5 + - uses: goreleaser/goreleaser-action@v6 with: version: latest args: check diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 514237ff..cb5d750d 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -106,7 +106,7 @@ jobs: run: syft version # GoReleaser will take care of publishing those artifacts into the release - name: Run GoReleaser - uses: goreleaser/goreleaser-action@v5 + uses: goreleaser/goreleaser-action@v6 with: version: latest args: release --clean --timeout 60m diff --git a/.goreleaser.yml b/.goreleaser.yml index 0fba8e2c..22f96b58 100644 --- a/.goreleaser.yml +++ b/.goreleaser.yml @@ -1,3 +1,5 @@ +version: 2 + before: hooks: # The build is done in this particular way to build Caddy in a designated directory named in .gitignore. From 3f1add6c9f5a41500cd6cfc96c5200d2c8291e14 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Thu, 6 Jun 2024 07:11:28 -0600 Subject: [PATCH 02/57] events: Getters for event info (close #6377) --- modules/caddyevents/app.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/caddyevents/app.go b/modules/caddyevents/app.go index 902c6d84..fe76a675 100644 --- a/modules/caddyevents/app.go +++ b/modules/caddyevents/app.go @@ -355,6 +355,11 @@ type Event struct { origin caddy.Module } +func (e Event) ID() uuid.UUID { return e.id } +func (e Event) Timestamp() time.Time { return e.ts } +func (e Event) Name() string { return e.name } +func (e Event) Origin() caddy.Module { return e.origin } + // CloudEvent exports event e as a structure that, when // serialized as JSON, is compatible with the // CloudEvents spec. From 101d3e740783581110340a68f0b0cbe5f1ab6dbb Mon Sep 17 00:00:00 2001 From: Ririsoft Date: Thu, 6 Jun 2024 16:33:34 +0200 Subject: [PATCH 03/57] logging: Customize log file permissions (#6314) Adding a "mode" option to overwrite the default logfile permissions. Default remains "0600" which is the one currently used by lumberjack. --- modules/logging/filewriter.go | 65 ++++- modules/logging/filewriter_test.go | 308 +++++++++++++++++++++ modules/logging/filewriter_test_windows.go | 55 ++++ 3 files changed, 426 insertions(+), 2 deletions(-) create mode 100644 modules/logging/filewriter_test.go create mode 100644 modules/logging/filewriter_test_windows.go diff --git a/modules/logging/filewriter.go b/modules/logging/filewriter.go index 3b1001b7..393228fd 100644 --- a/modules/logging/filewriter.go +++ b/modules/logging/filewriter.go @@ -15,6 +15,7 @@ package logging import ( + "encoding/json" "fmt" "io" "math" @@ -33,6 +34,43 @@ func init() { caddy.RegisterModule(FileWriter{}) } +// fileMode is a string made of 1 to 4 octal digits representing +// a numeric mode as specified with the `chmod` unix command. +// `"0777"` and `"777"` are thus equivalent values. +type fileMode os.FileMode + +// UnmarshalJSON satisfies json.Unmarshaler. +func (m *fileMode) UnmarshalJSON(b []byte) error { + if len(b) == 0 { + return io.EOF + } + + var s string + if err := json.Unmarshal(b, &s); err != nil { + return err + } + + mode, err := parseFileMode(s) + if err != nil { + return err + } + + *m = fileMode(mode) + return err +} + +// parseFileMode parses a file mode string, +// adding support for `chmod` unix command like +// 1 to 4 digital octal values. +func parseFileMode(s string) (os.FileMode, error) { + modeStr := fmt.Sprintf("%04s", s) + mode, err := strconv.ParseUint(modeStr, 8, 32) + if err != nil { + return 0, err + } + return os.FileMode(mode), nil +} + // FileWriter can write logs to files. By default, log files // are rotated ("rolled") when they get large, and old log // files get deleted, to ensure that the process does not @@ -41,6 +79,10 @@ type FileWriter struct { // Filename is the name of the file to write. Filename string `json:"filename,omitempty"` + // The file permissions mode. + // 0600 by default. + Mode fileMode `json:"mode,omitempty"` + // Roll toggles log rolling or rotation, which is // enabled by default. Roll *bool `json:"roll,omitempty"` @@ -100,6 +142,10 @@ func (fw FileWriter) WriterKey() string { // OpenWriter opens a new file writer. func (fw FileWriter) OpenWriter() (io.WriteCloser, error) { + if fw.Mode == 0 { + fw.Mode = 0o600 + } + // roll log files by default if fw.Roll == nil || *fw.Roll { if fw.RollSizeMB == 0 { @@ -116,6 +162,9 @@ func (fw FileWriter) OpenWriter() (io.WriteCloser, error) { fw.RollKeepDays = 90 } + f_tmp, _ := os.OpenFile(fw.Filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.FileMode(fw.Mode)) + f_tmp.Close() + return &lumberjack.Logger{ Filename: fw.Filename, MaxSize: fw.RollSizeMB, @@ -127,12 +176,13 @@ func (fw FileWriter) OpenWriter() (io.WriteCloser, error) { } // otherwise just open a regular file - return os.OpenFile(fw.Filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0o666) + return os.OpenFile(fw.Filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.FileMode(fw.Mode)) } // UnmarshalCaddyfile sets up the module from Caddyfile tokens. Syntax: // // file { +// mode // roll_disabled // roll_size // roll_uncompressed @@ -150,7 +200,7 @@ func (fw FileWriter) OpenWriter() (io.WriteCloser, error) { // The roll_keep_for duration has day resolution. // Fractional values are rounded up to the next whole number of days. // -// If any of the roll_size, roll_keep, or roll_keep_for subdirectives are +// If any of the mode, roll_size, roll_keep, or roll_keep_for subdirectives are // omitted or set to a zero value, then Caddy's default value for that // subdirective is used. func (fw *FileWriter) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { @@ -165,6 +215,17 @@ func (fw *FileWriter) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { for d.NextBlock(0) { switch d.Val() { + case "mode": + var modeStr string + if !d.AllArgs(&modeStr) { + return d.ArgErr() + } + mode, err := parseFileMode(modeStr) + if err != nil { + return d.Errf("parsing mode: %v", err) + } + fw.Mode = fileMode(mode) + case "roll_disabled": var f bool fw.Roll = &f diff --git a/modules/logging/filewriter_test.go b/modules/logging/filewriter_test.go new file mode 100644 index 00000000..2787eeff --- /dev/null +++ b/modules/logging/filewriter_test.go @@ -0,0 +1,308 @@ +// Copyright 2015 Matthew Holt and The Caddy Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !windows + +package logging + +import ( + "encoding/json" + "os" + "path" + "syscall" + "testing" + + "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" +) + +func TestFileCreationMode(t *testing.T) { + on := true + off := false + + tests := []struct { + name string + fw FileWriter + wantMode os.FileMode + }{ + { + name: "default mode no roll", + fw: FileWriter{ + Roll: &off, + }, + wantMode: 0o600, + }, + { + name: "default mode roll", + fw: FileWriter{ + Roll: &on, + }, + wantMode: 0o600, + }, + { + name: "custom mode no roll", + fw: FileWriter{ + Roll: &off, + Mode: 0o666, + }, + wantMode: 0o666, + }, + { + name: "custom mode roll", + fw: FileWriter{ + Roll: &on, + Mode: 0o666, + }, + wantMode: 0o666, + }, + } + + m := syscall.Umask(0o000) + defer syscall.Umask(m) + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir, err := os.MkdirTemp("", "caddytest") + if err != nil { + t.Fatalf("failed to create tempdir: %v", err) + } + defer os.RemoveAll(dir) + fpath := path.Join(dir, "test.log") + tt.fw.Filename = fpath + + logger, err := tt.fw.OpenWriter() + if err != nil { + t.Fatalf("failed to create file: %v", err) + } + defer logger.Close() + + st, err := os.Stat(fpath) + if err != nil { + t.Fatalf("failed to check file permissions: %v", err) + } + + if st.Mode() != tt.wantMode { + t.Errorf("file mode is %v, want %v", st.Mode(), tt.wantMode) + } + }) + } +} + +func TestFileRotationPreserveMode(t *testing.T) { + m := syscall.Umask(0o000) + defer syscall.Umask(m) + + dir, err := os.MkdirTemp("", "caddytest") + if err != nil { + t.Fatalf("failed to create tempdir: %v", err) + } + defer os.RemoveAll(dir) + + fpath := path.Join(dir, "test.log") + + roll := true + mode := fileMode(0o640) + fw := FileWriter{ + Filename: fpath, + Mode: mode, + Roll: &roll, + RollSizeMB: 1, + } + + logger, err := fw.OpenWriter() + if err != nil { + t.Fatalf("failed to create file: %v", err) + } + defer logger.Close() + + b := make([]byte, 1024*1024-1000) + logger.Write(b) + logger.Write(b[0:2000]) + + files, err := os.ReadDir(dir) + if err != nil { + t.Fatalf("failed to read temporary log dir: %v", err) + } + + // We might get 2 or 3 files depending + // on the race between compressed log file generation, + // removal of the non compressed file and reading the directory. + // Ordering of the files are [ test-*.log test-*.log.gz test.log ] + if len(files) < 2 || len(files) > 3 { + t.Log("got files: ", files) + t.Fatalf("got %v files want 2", len(files)) + } + + wantPattern := "test-*-*-*-*-*.*.log" + test_date_log := files[0] + if m, _ := path.Match(wantPattern, test_date_log.Name()); m != true { + t.Fatalf("got %v filename want %v", test_date_log.Name(), wantPattern) + } + + st, err := os.Stat(path.Join(dir, test_date_log.Name())) + if err != nil { + t.Fatalf("failed to check file permissions: %v", err) + } + + if st.Mode() != os.FileMode(mode) { + t.Errorf("file mode is %v, want %v", st.Mode(), mode) + } + + test_dot_log := files[len(files)-1] + if test_dot_log.Name() != "test.log" { + t.Fatalf("got %v filename want test.log", test_dot_log.Name()) + } + + st, err = os.Stat(path.Join(dir, test_dot_log.Name())) + if err != nil { + t.Fatalf("failed to check file permissions: %v", err) + } + + if st.Mode() != os.FileMode(mode) { + t.Errorf("file mode is %v, want %v", st.Mode(), mode) + } +} + +func TestFileModeConfig(t *testing.T) { + tests := []struct { + name string + d *caddyfile.Dispenser + fw FileWriter + wantErr bool + }{ + { + name: "set mode", + d: caddyfile.NewTestDispenser(` +file test.log { + mode 0666 +} +`), + fw: FileWriter{ + Mode: 0o666, + }, + wantErr: false, + }, + { + name: "set mode 3 digits", + d: caddyfile.NewTestDispenser(` +file test.log { + mode 666 +} +`), + fw: FileWriter{ + Mode: 0o666, + }, + wantErr: false, + }, + { + name: "set mode 2 digits", + d: caddyfile.NewTestDispenser(` +file test.log { + mode 66 +} +`), + fw: FileWriter{ + Mode: 0o066, + }, + wantErr: false, + }, + { + name: "set mode 1 digits", + d: caddyfile.NewTestDispenser(` +file test.log { + mode 6 +} +`), + fw: FileWriter{ + Mode: 0o006, + }, + wantErr: false, + }, + { + name: "invalid mode", + d: caddyfile.NewTestDispenser(` +file test.log { + mode foobar +} +`), + fw: FileWriter{}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fw := &FileWriter{} + if err := fw.UnmarshalCaddyfile(tt.d); (err != nil) != tt.wantErr { + t.Fatalf("UnmarshalCaddyfile() error = %v, want %v", err, tt.wantErr) + } + if fw.Mode != tt.fw.Mode { + t.Errorf("got mode %v, want %v", fw.Mode, tt.fw.Mode) + } + }) + } +} + +func TestFileModeJSON(t *testing.T) { + tests := []struct { + name string + config string + fw FileWriter + wantErr bool + }{ + { + name: "set mode", + config: ` +{ + "mode": "0666" +} +`, + fw: FileWriter{ + Mode: 0o666, + }, + wantErr: false, + }, + { + name: "set mode invalid value", + config: ` +{ + "mode": "0x666" +} +`, + fw: FileWriter{}, + wantErr: true, + }, + { + name: "set mode invalid string", + config: ` +{ + "mode": 777 +} +`, + fw: FileWriter{}, + wantErr: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fw := &FileWriter{} + if err := json.Unmarshal([]byte(tt.config), fw); (err != nil) != tt.wantErr { + t.Fatalf("UnmarshalJSON() error = %v, want %v", err, tt.wantErr) + } + if fw.Mode != tt.fw.Mode { + t.Errorf("got mode %v, want %v", fw.Mode, tt.fw.Mode) + } + }) + } +} diff --git a/modules/logging/filewriter_test_windows.go b/modules/logging/filewriter_test_windows.go new file mode 100644 index 00000000..d32a8d2c --- /dev/null +++ b/modules/logging/filewriter_test_windows.go @@ -0,0 +1,55 @@ +// Copyright 2015 Matthew Holt and The Caddy Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build windows + +package logging + +import ( + "os" + "path" + "testing" +) + +// Windows relies on ACLs instead of unix permissions model. +// Go allows to open files with a particular mode put it is limited to read or write. +// See https://cs.opensource.google/go/go/+/refs/tags/go1.22.3:src/syscall/syscall_windows.go;l=708. +// This is pretty restrictive and has few interest for log files and thus we just test that log files are +// opened with R/W permissions by default on Windows too. +func TestFileCreationMode(t *testing.T) { + dir, err := os.MkdirTemp("", "caddytest") + if err != nil { + t.Fatalf("failed to create tempdir: %v", err) + } + defer os.RemoveAll(dir) + + fw := &FileWriter{ + Filename: path.Join(dir, "test.log"), + } + + logger, err := fw.OpenWriter() + if err != nil { + t.Fatalf("failed to create file: %v", err) + } + defer logger.Close() + + st, err := os.Stat(fw.Filename) + if err != nil { + t.Fatalf("failed to check file permissions: %v", err) + } + + if st.Mode().Perm()&0o600 != 0o600 { + t.Fatalf("file mode is %v, want rw for user", st.Mode().Perm()) + } +} From a10117f8bdbfd72fe585b7bb0c4b43ad8f6908bc Mon Sep 17 00:00:00 2001 From: Andreas Kohn Date: Thu, 6 Jun 2024 22:36:06 +0200 Subject: [PATCH 04/57] core: Split `run` into a public `ProvisionContext` and a private method (#6378) * Split `run` into a public `BuildContext` and a private part `BuildContext` can be used to set up a caddy context from a config, but not start any listeners or active components: The returned context has the configured apps provisioned, but otherwise is inert. This is EXPERIMENTAL: Minimally it's missing documentation and the example for how this can be used to run unit tests. * Use the config from the context The config passed into `BuildContext` can be nil, in which case `BuildContext` will just make one up that works. In either case that will end up in the finished context. * Rename `BuildContext` to `ProvisionContext` to better match the function * Hide the `replaceAdminServer` parts The admin server is a global thing, and in the envisioned use case for `ProvisionContext` shouldn't actually exist. Hide this detail in a private `provisionContext` instead, and only expose it publicly with `replaceAdminServer` set to `false`. This should reduce foot-shooting potential further; in addition the documentation comment now clearly spells out that the exact interface and implementation details of `ProvisionContext` are experimental and subject to change. --- caddy.go | 105 ++++++++++++++++++++++++++++++++----------------------- 1 file changed, 62 insertions(+), 43 deletions(-) diff --git a/caddy.go b/caddy.go index 27acabb1..7dd989c9 100644 --- a/caddy.go +++ b/caddy.go @@ -397,6 +397,58 @@ func unsyncedDecodeAndRun(cfgJSON []byte, allowPersist bool) error { // will want to use Run instead, which also // updates the config's raw state. func run(newCfg *Config, start bool) (Context, error) { + ctx, err := provisionContext(newCfg, start) + if err != nil { + return ctx, err + } + + if !start { + return ctx, nil + } + + // Provision any admin routers which may need to access + // some of the other apps at runtime + err = ctx.cfg.Admin.provisionAdminRouters(ctx) + if err != nil { + return ctx, err + } + + // Start + err = func() error { + started := make([]string, 0, len(ctx.cfg.apps)) + for name, a := range ctx.cfg.apps { + err := a.Start() + if err != nil { + // an app failed to start, so we need to stop + // all other apps that were already started + for _, otherAppName := range started { + err2 := ctx.cfg.apps[otherAppName].Stop() + if err2 != nil { + err = fmt.Errorf("%v; additionally, aborting app %s: %v", + err, otherAppName, err2) + } + } + return fmt.Errorf("%s app module: start: %v", name, err) + } + started = append(started, name) + } + return nil + }() + if err != nil { + return ctx, err + } + + // now that the user's config is running, finish setting up anything else, + // such as remote admin endpoint, config loader, etc. + return ctx, finishSettingUp(ctx, ctx.cfg) +} + +// provisionContext creates a new context from the given configuration and provisions +// storage and apps. +// If `newCfg` is nil a new empty configuration will be created. +// If `replaceAdminServer` is true any currently active admin server will be replaced +// with a new admin server based on the provided configuration. +func provisionContext(newCfg *Config, replaceAdminServer bool) (Context, error) { // because we will need to roll back any state // modifications if this function errors, we // keep a single error value and scope all @@ -444,7 +496,7 @@ func run(newCfg *Config, start bool) (Context, error) { } // start the admin endpoint (and stop any prior one) - if start { + if replaceAdminServer { err = replaceLocalAdminServer(newCfg) if err != nil { return ctx, fmt.Errorf("starting caddy administration endpoint: %v", err) @@ -491,49 +543,16 @@ func run(newCfg *Config, start bool) (Context, error) { } return nil }() - if err != nil { - return ctx, err - } + return ctx, err +} - if !start { - return ctx, nil - } - - // Provision any admin routers which may need to access - // some of the other apps at runtime - err = newCfg.Admin.provisionAdminRouters(ctx) - if err != nil { - return ctx, err - } - - // Start - err = func() error { - started := make([]string, 0, len(newCfg.apps)) - for name, a := range newCfg.apps { - err := a.Start() - if err != nil { - // an app failed to start, so we need to stop - // all other apps that were already started - for _, otherAppName := range started { - err2 := newCfg.apps[otherAppName].Stop() - if err2 != nil { - err = fmt.Errorf("%v; additionally, aborting app %s: %v", - err, otherAppName, err2) - } - } - return fmt.Errorf("%s app module: start: %v", name, err) - } - started = append(started, name) - } - return nil - }() - if err != nil { - return ctx, err - } - - // now that the user's config is running, finish setting up anything else, - // such as remote admin endpoint, config loader, etc. - return ctx, finishSettingUp(ctx, newCfg) +// ProvisionContext creates a new context from the configuration and provisions storage +// and app modules. +// The function is intended for testing and advanced use cases only, typically `Run` should be +// use to ensure a fully functional caddy instance. +// EXPERIMENTAL: While this is public the interface and implementation details of this function may change. +func ProvisionContext(newCfg *Config) (Context, error) { + return provisionContext(newCfg, false) } // finishSettingUp should be run after all apps have successfully started. From 9be4f194e036dddd4704a851fd15a9682d9e813d Mon Sep 17 00:00:00 2001 From: Andreas Kohn Date: Fri, 7 Jun 2024 15:25:36 +0200 Subject: [PATCH 05/57] caddyhttp: Write header if needed in responseRecorder.WriteResponse (#6380) --- modules/caddyhttp/responsewriter.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/caddyhttp/responsewriter.go b/modules/caddyhttp/responsewriter.go index d51d37cb..808d2de3 100644 --- a/modules/caddyhttp/responsewriter.go +++ b/modules/caddyhttp/responsewriter.go @@ -219,13 +219,13 @@ func (rr *responseRecorder) Buffered() bool { } func (rr *responseRecorder) WriteResponse() error { - if rr.stream { - return nil - } if rr.statusCode == 0 { // could happen if no handlers actually wrote anything, // and this prevents a panic; status must be > 0 - rr.statusCode = http.StatusOK + rr.WriteHeader(http.StatusOK) + } + if rr.stream { + return nil } rr.ResponseWriterWrapper.WriteHeader(rr.statusCode) _, err := io.Copy(rr.ResponseWriterWrapper, rr.buf) From 0bc27e5fb1252716f82c2b5af56189e8b46ead3c Mon Sep 17 00:00:00 2001 From: Ririsoft Date: Sat, 8 Jun 2024 19:34:18 +0200 Subject: [PATCH 06/57] logging: fix file mode configuration parsing (#6383) Commit 101d3e7 introduced file mode setting, but was missing a JSON Marshaller so that CaddyFile can be converted to JSON safely. --- modules/logging/filewriter.go | 5 ++++ modules/logging/filewriter_test.go | 39 ++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+) diff --git a/modules/logging/filewriter.go b/modules/logging/filewriter.go index 393228fd..09cea1b4 100644 --- a/modules/logging/filewriter.go +++ b/modules/logging/filewriter.go @@ -59,6 +59,11 @@ func (m *fileMode) UnmarshalJSON(b []byte) error { return err } +// MarshalJSON satisfies json.Marshaler. +func (m *fileMode) MarshalJSON() ([]byte, error) { + return []byte(fmt.Sprintf("\"%04o\"", *m)), nil +} + // parseFileMode parses a file mode string, // adding support for `chmod` unix command like // 1 to 4 digital octal values. diff --git a/modules/logging/filewriter_test.go b/modules/logging/filewriter_test.go index 2787eeff..ab403930 100644 --- a/modules/logging/filewriter_test.go +++ b/modules/logging/filewriter_test.go @@ -306,3 +306,42 @@ func TestFileModeJSON(t *testing.T) { }) } } + +func TestFileModeToJSON(t *testing.T) { + tests := []struct { + name string + mode fileMode + want string + wantErr bool + }{ + { + name: "none zero", + mode: 0644, + want: `"0644"`, + wantErr: false, + }, + { + name: "zero mode", + mode: 0, + want: `"0000"`, + wantErr: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var b []byte + var err error + + if b, err = json.Marshal(&tt.mode); (err != nil) != tt.wantErr { + t.Fatalf("MarshalJSON() error = %v, want %v", err, tt.wantErr) + } + + got := string(b[:]) + + if got != tt.want { + t.Errorf("got mode %v, want %v", got, tt.want) + } + }) + } +} From 04fb9fe87ff7406b36a4a7f0a9215ab7a138d345 Mon Sep 17 00:00:00 2001 From: Will Norris Date: Mon, 10 Jun 2024 06:28:30 -0700 Subject: [PATCH 07/57] go.mod: update tscert package (#6384) The latest tscert allows callers to provide a custom http.Transport for calling Tailscale's local API. Updates tailscale/caddy-tailscale#66 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8729be44..f5559a8d 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.9.0 - github.com/tailscale/tscert v0.0.0-20240517230440-bbccfbf48933 + github.com/tailscale/tscert v0.0.0-20240608151842-d3f834017e53 github.com/yuin/goldmark v1.7.1 github.com/yuin/goldmark-highlighting/v2 v2.0.0-20230729083705-37449abec8cc go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 diff --git a/go.sum b/go.sum index 351e449c..63306efc 100644 --- a/go.sum +++ b/go.sum @@ -415,8 +415,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg= github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= -github.com/tailscale/tscert v0.0.0-20240517230440-bbccfbf48933 h1:pV0H+XIvFoP7pl1MRtyPXh5hqoxB5I7snOtTHgrn6HU= -github.com/tailscale/tscert v0.0.0-20240517230440-bbccfbf48933/go.mod h1:kNGUQ3VESx3VZwRwA9MSCUegIl6+saPL8Noq82ozCaU= +github.com/tailscale/tscert v0.0.0-20240608151842-d3f834017e53 h1:uxMgm0C+EjytfAqyfBG55ZONKQ7mvd7x4YYCWsf8QHQ= +github.com/tailscale/tscert v0.0.0-20240608151842-d3f834017e53/go.mod h1:kNGUQ3VESx3VZwRwA9MSCUegIl6+saPL8Noq82ozCaU= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= github.com/urfave/cli v1.22.14 h1:ebbhrRiGK2i4naQJr+1Xj92HXZCrK7MsyTS/ob3HnAk= github.com/urfave/cli v1.22.14/go.mod h1:X0eDS6pD6Exaclxm99NJ3FiCDRED7vIHpx2mDOHLvkA= From d85cc2ec103de72658c55ba74197337c99bd1f74 Mon Sep 17 00:00:00 2001 From: Omar Ramadan Date: Mon, 10 Jun 2024 08:03:24 -0700 Subject: [PATCH 08/57] logging: Customizable zap cores (#6381) --- caddyconfig/httpcaddyfile/builtins.go | 17 ++++++++++ caddyconfig/httpcaddyfile/builtins_test.go | 6 ++-- logging.go | 12 ++++++++ modules/logging/cores.go | 36 ++++++++++++++++++++++ 4 files changed, 69 insertions(+), 2 deletions(-) create mode 100644 modules/logging/cores.go diff --git a/caddyconfig/httpcaddyfile/builtins.go b/caddyconfig/httpcaddyfile/builtins.go index 35a08ef2..e1e95e00 100644 --- a/caddyconfig/httpcaddyfile/builtins.go +++ b/caddyconfig/httpcaddyfile/builtins.go @@ -849,6 +849,7 @@ func parseInvoke(h Helper) (caddyhttp.MiddlewareHandler, error) { // log { // hostnames // output ... +// core ... // format ... // level // } @@ -960,6 +961,22 @@ func parseLogHelper(h Helper, globalLogNames map[string]struct{}) ([]ConfigValue } cl.WriterRaw = caddyconfig.JSONModuleObject(wo, "output", moduleName, h.warnings) + case "core": + if !h.NextArg() { + return nil, h.ArgErr() + } + moduleName := h.Val() + moduleID := "caddy.logging.cores." + moduleName + unm, err := caddyfile.UnmarshalModule(h.Dispenser, moduleID) + if err != nil { + return nil, err + } + core, ok := unm.(zapcore.Core) + if !ok { + return nil, h.Errf("module %s (%T) is not a zapcore.Core", moduleID, unm) + } + cl.CoreRaw = caddyconfig.JSONModuleObject(core, "module", moduleName, h.warnings) + case "format": if !h.NextArg() { return nil, h.ArgErr() diff --git a/caddyconfig/httpcaddyfile/builtins_test.go b/caddyconfig/httpcaddyfile/builtins_test.go index 70f347dd..cf746348 100644 --- a/caddyconfig/httpcaddyfile/builtins_test.go +++ b/caddyconfig/httpcaddyfile/builtins_test.go @@ -25,11 +25,12 @@ func TestLogDirectiveSyntax(t *testing.T) { { input: `:8080 { log { + core mock output file foo.log } } `, - output: `{"logging":{"logs":{"default":{"exclude":["http.log.access.log0"]},"log0":{"writer":{"filename":"foo.log","output":"file"},"include":["http.log.access.log0"]}}},"apps":{"http":{"servers":{"srv0":{"listen":[":8080"],"logs":{"default_logger_name":"log0"}}}}}}`, + output: `{"logging":{"logs":{"default":{"exclude":["http.log.access.log0"]},"log0":{"writer":{"filename":"foo.log","output":"file"},"core":{"module":"mock"},"include":["http.log.access.log0"]}}},"apps":{"http":{"servers":{"srv0":{"listen":[":8080"],"logs":{"default_logger_name":"log0"}}}}}}`, expectError: false, }, { @@ -53,11 +54,12 @@ func TestLogDirectiveSyntax(t *testing.T) { { input: `:8080 { log name-override { + core mock output file foo.log } } `, - output: `{"logging":{"logs":{"default":{"exclude":["http.log.access.name-override"]},"name-override":{"writer":{"filename":"foo.log","output":"file"},"include":["http.log.access.name-override"]}}},"apps":{"http":{"servers":{"srv0":{"listen":[":8080"],"logs":{"default_logger_name":"name-override"}}}}}}`, + output: `{"logging":{"logs":{"default":{"exclude":["http.log.access.name-override"]},"name-override":{"writer":{"filename":"foo.log","output":"file"},"core":{"module":"mock"},"include":["http.log.access.name-override"]}}},"apps":{"http":{"servers":{"srv0":{"listen":[":8080"],"logs":{"default_logger_name":"name-override"}}}}}}`, expectError: false, }, } { diff --git a/logging.go b/logging.go index d3e7bf32..ca10beee 100644 --- a/logging.go +++ b/logging.go @@ -292,6 +292,10 @@ type BaseLog struct { // The encoder is how the log entries are formatted or encoded. EncoderRaw json.RawMessage `json:"encoder,omitempty" caddy:"namespace=caddy.logging.encoders inline_key=format"` + // Tees entries through a zap.Core module which can extract + // log entry metadata and fields for further processing. + CoreRaw json.RawMessage `json:"core,omitempty" caddy:"namespace=caddy.logging.cores inline_key=module"` + // Level is the minimum level to emit, and is inclusive. // Possible levels: DEBUG, INFO, WARN, ERROR, PANIC, and FATAL Level string `json:"level,omitempty"` @@ -366,6 +370,14 @@ func (cl *BaseLog) provisionCommon(ctx Context, logging *Logging) error { cl.encoder = newDefaultProductionLogEncoder(cl.writerOpener) } cl.buildCore() + if cl.CoreRaw != nil { + mod, err := ctx.LoadModule(cl, "CoreRaw") + if err != nil { + return fmt.Errorf("loading log core module: %v", err) + } + core := mod.(zapcore.Core) + cl.core = zapcore.NewTee(cl.core, core) + } return nil } diff --git a/modules/logging/cores.go b/modules/logging/cores.go new file mode 100644 index 00000000..49aa7640 --- /dev/null +++ b/modules/logging/cores.go @@ -0,0 +1,36 @@ +package logging + +import ( + "go.uber.org/zap/zapcore" + + "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" +) + +func init() { + caddy.RegisterModule(MockCore{}) +} + +// MockCore is a no-op module, purely for testing +type MockCore struct { + zapcore.Core `json:"-"` +} + +// CaddyModule returns the Caddy module information. +func (MockCore) CaddyModule() caddy.ModuleInfo { + return caddy.ModuleInfo{ + ID: "caddy.logging.cores.mock", + New: func() caddy.Module { return new(MockCore) }, + } +} + +func (lec *MockCore) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { + return nil +} + +// Interface guards +var ( + _ zapcore.Core = (*MockCore)(nil) + _ caddy.Module = (*MockCore)(nil) + _ caddyfile.Unmarshaler = (*MockCore)(nil) +) From 8e0d3e1ec56cd349f02c9d201234c56373688ddd Mon Sep 17 00:00:00 2001 From: Ririsoft Date: Wed, 12 Jun 2024 23:17:46 +0200 Subject: [PATCH 09/57] logging: set file mode when the file already exist (#6391) 101d3e7 introduced a configuration option to set the log file mode. This option was not taken into account if the file already exists, making users having to delete their logs to have new logs created with the right mode. --- modules/logging/filewriter.go | 12 ++++++++- modules/logging/filewriter_test.go | 39 ++++++++++++++++++++++++++++++ 2 files changed, 50 insertions(+), 1 deletion(-) diff --git a/modules/logging/filewriter.go b/modules/logging/filewriter.go index 09cea1b4..44c0feb6 100644 --- a/modules/logging/filewriter.go +++ b/modules/logging/filewriter.go @@ -167,8 +167,18 @@ func (fw FileWriter) OpenWriter() (io.WriteCloser, error) { fw.RollKeepDays = 90 } - f_tmp, _ := os.OpenFile(fw.Filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.FileMode(fw.Mode)) + // create the file if it does not exist with the right mode. + // lumberjack will reuse the file mode across log rotation. + f_tmp, err := os.OpenFile(fw.Filename, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.FileMode(fw.Mode)) + if err != nil { + return nil, err + } f_tmp.Close() + // ensure already existing files have the right mode, + // since OpenFile will not set the mode in such case. + if err = os.Chmod(fw.Filename, os.FileMode(fw.Mode)); err != nil { + return nil, err + } return &lumberjack.Logger{ Filename: fw.Filename, diff --git a/modules/logging/filewriter_test.go b/modules/logging/filewriter_test.go index ab403930..0c54a659 100644 --- a/modules/logging/filewriter_test.go +++ b/modules/logging/filewriter_test.go @@ -345,3 +345,42 @@ func TestFileModeToJSON(t *testing.T) { }) } } + +func TestFileModeModification(t *testing.T) { + m := syscall.Umask(0o000) + defer syscall.Umask(m) + + dir, err := os.MkdirTemp("", "caddytest") + if err != nil { + t.Fatalf("failed to create tempdir: %v", err) + } + defer os.RemoveAll(dir) + + fpath := path.Join(dir, "test.log") + f_tmp, err := os.OpenFile(fpath, os.O_WRONLY|os.O_APPEND|os.O_CREATE, os.FileMode(0600)) + if err != nil { + t.Fatalf("failed to create test file: %v", err) + } + f_tmp.Close() + + fw := FileWriter{ + Mode: 0o666, + Filename: fpath, + } + + logger, err := fw.OpenWriter() + if err != nil { + t.Fatalf("failed to create file: %v", err) + } + defer logger.Close() + + st, err := os.Stat(fpath) + if err != nil { + t.Fatalf("failed to check file permissions: %v", err) + } + + want := os.FileMode(fw.Mode) + if st.Mode() != want { + t.Errorf("file mode is %v, want %v", st.Mode(), want) + } +} From aca4002fd8ed890f29f74bb7e8d629496f9a6e07 Mon Sep 17 00:00:00 2001 From: a Date: Fri, 14 Jun 2024 12:27:51 -0500 Subject: [PATCH 10/57] caddyfile: Pass blocks to `import` for snippets (#6130) * a * a * a * a * a * a --- caddyconfig/caddyfile/parse.go | 67 ++++++++++++++- .../import_block_snippet.caddyfiletest | 58 +++++++++++++ .../import_block_snippet_args.caddyfiletest | 56 +++++++++++++ .../import_blocks_snippet.caddyfiletest | 76 +++++++++++++++++ ...import_blocks_snippet_nested.caddyfiletest | 82 +++++++++++++++++++ 5 files changed, 337 insertions(+), 2 deletions(-) create mode 100644 caddytest/integration/caddyfile_adapt/import_block_snippet.caddyfiletest create mode 100644 caddytest/integration/caddyfile_adapt/import_block_snippet_args.caddyfiletest create mode 100644 caddytest/integration/caddyfile_adapt/import_blocks_snippet.caddyfiletest create mode 100644 caddytest/integration/caddyfile_adapt/import_blocks_snippet_nested.caddyfiletest diff --git a/caddyconfig/caddyfile/parse.go b/caddyconfig/caddyfile/parse.go index 17b0ca8e..17d824ef 100644 --- a/caddyconfig/caddyfile/parse.go +++ b/caddyconfig/caddyfile/parse.go @@ -364,9 +364,45 @@ func (p *parser) doImport(nesting int) error { // set up a replacer for non-variadic args replacement repl := makeArgsReplacer(args) + // grab all the tokens (if it exists) from within a block that follows the import + var blockTokens []Token + for currentNesting := p.Nesting(); p.NextBlock(currentNesting); { + blockTokens = append(blockTokens, p.Token()) + } + // initialize with size 1 + blockMapping := make(map[string][]Token, 1) + if len(blockTokens) > 0 { + // use such tokens to create a new dispenser, and then use it to parse each block + bd := NewDispenser(blockTokens) + for bd.Next() { + // see if we can grab a key + var currentMappingKey string + if bd.Val() == "{" { + return p.Err("anonymous blocks are not supported") + } + currentMappingKey = bd.Val() + currentMappingTokens := []Token{} + // read all args until end of line / { + if bd.NextArg() { + currentMappingTokens = append(currentMappingTokens, bd.Token()) + for bd.NextArg() { + currentMappingTokens = append(currentMappingTokens, bd.Token()) + } + // TODO(elee1766): we don't enter another mapping here because it's annoying to extract the { and } properly. + // maybe someone can do that in the future + } else { + // attempt to enter a block and add tokens to the currentMappingTokens + for mappingNesting := bd.Nesting(); bd.NextBlock(mappingNesting); { + currentMappingTokens = append(currentMappingTokens, bd.Token()) + } + } + blockMapping[currentMappingKey] = currentMappingTokens + } + } + // splice out the import directive and its arguments // (2 tokens, plus the length of args) - tokensBefore := p.tokens[:p.cursor-1-len(args)] + tokensBefore := p.tokens[:p.cursor-1-len(args)-len(blockTokens)] tokensAfter := p.tokens[p.cursor+1:] var importedTokens []Token var nodes []string @@ -495,6 +531,33 @@ func (p *parser) doImport(nesting int) error { maybeSnippet = false } } + // if it is {block}, we substitute with all tokens in the block + // if it is {blocks.*}, we substitute with the tokens in the mapping for the * + var skip bool + var tokensToAdd []Token + switch { + case token.Text == "{block}": + tokensToAdd = blockTokens + case strings.HasPrefix(token.Text, "{blocks.") && strings.HasSuffix(token.Text, "}"): + // {blocks.foo.bar} will be extracted to key `foo.bar` + blockKey := strings.TrimPrefix(strings.TrimSuffix(token.Text, "}"), "{blocks.") + val, ok := blockMapping[blockKey] + if ok { + tokensToAdd = val + } + default: + skip = true + } + if !skip { + if len(tokensToAdd) == 0 { + // if there is no content in the snippet block, don't do any replacement + // this allows snippets which contained {block}/{block.*} before this change to continue functioning as normal + tokensCopy = append(tokensCopy, token) + } else { + tokensCopy = append(tokensCopy, tokensToAdd...) + } + continue + } if maybeSnippet { tokensCopy = append(tokensCopy, token) @@ -516,7 +579,7 @@ func (p *parser) doImport(nesting int) error { // splice the imported tokens in the place of the import statement // and rewind cursor so Next() will land on first imported token p.tokens = append(tokensBefore, append(tokensCopy, tokensAfter...)...) - p.cursor -= len(args) + 1 + p.cursor -= len(args) + len(blockTokens) + 1 return nil } diff --git a/caddytest/integration/caddyfile_adapt/import_block_snippet.caddyfiletest b/caddytest/integration/caddyfile_adapt/import_block_snippet.caddyfiletest new file mode 100644 index 00000000..a60c238c --- /dev/null +++ b/caddytest/integration/caddyfile_adapt/import_block_snippet.caddyfiletest @@ -0,0 +1,58 @@ +(snippet) { + header { + {block} + } +} + +example.com { + import snippet { + foo bar + } +} +---------- +{ + "apps": { + "http": { + "servers": { + "srv0": { + "listen": [ + ":443" + ], + "routes": [ + { + "match": [ + { + "host": [ + "example.com" + ] + } + ], + "handle": [ + { + "handler": "subroute", + "routes": [ + { + "handle": [ + { + "handler": "headers", + "response": { + "set": { + "Foo": [ + "bar" + ] + } + } + } + ] + } + ] + } + ], + "terminal": true + } + ] + } + } + } + } +} diff --git a/caddytest/integration/caddyfile_adapt/import_block_snippet_args.caddyfiletest b/caddytest/integration/caddyfile_adapt/import_block_snippet_args.caddyfiletest new file mode 100644 index 00000000..7f2e68b7 --- /dev/null +++ b/caddytest/integration/caddyfile_adapt/import_block_snippet_args.caddyfiletest @@ -0,0 +1,56 @@ +(snippet) { + {block} +} + +example.com { + import snippet { + header foo bar + } +} +---------- +{ + "apps": { + "http": { + "servers": { + "srv0": { + "listen": [ + ":443" + ], + "routes": [ + { + "match": [ + { + "host": [ + "example.com" + ] + } + ], + "handle": [ + { + "handler": "subroute", + "routes": [ + { + "handle": [ + { + "handler": "headers", + "response": { + "set": { + "Foo": [ + "bar" + ] + } + } + } + ] + } + ] + } + ], + "terminal": true + } + ] + } + } + } + } +} diff --git a/caddytest/integration/caddyfile_adapt/import_blocks_snippet.caddyfiletest b/caddytest/integration/caddyfile_adapt/import_blocks_snippet.caddyfiletest new file mode 100644 index 00000000..4098f90b --- /dev/null +++ b/caddytest/integration/caddyfile_adapt/import_blocks_snippet.caddyfiletest @@ -0,0 +1,76 @@ +(snippet) { + header { + {blocks.foo} + } + header { + {blocks.bar} + } +} + +example.com { + import snippet { + foo { + foo a + } + bar { + bar b + } + } +} +---------- +{ + "apps": { + "http": { + "servers": { + "srv0": { + "listen": [ + ":443" + ], + "routes": [ + { + "match": [ + { + "host": [ + "example.com" + ] + } + ], + "handle": [ + { + "handler": "subroute", + "routes": [ + { + "handle": [ + { + "handler": "headers", + "response": { + "set": { + "Foo": [ + "a" + ] + } + } + }, + { + "handler": "headers", + "response": { + "set": { + "Bar": [ + "b" + ] + } + } + } + ] + } + ] + } + ], + "terminal": true + } + ] + } + } + } + } +} diff --git a/caddytest/integration/caddyfile_adapt/import_blocks_snippet_nested.caddyfiletest b/caddytest/integration/caddyfile_adapt/import_blocks_snippet_nested.caddyfiletest new file mode 100644 index 00000000..ac1c5226 --- /dev/null +++ b/caddytest/integration/caddyfile_adapt/import_blocks_snippet_nested.caddyfiletest @@ -0,0 +1,82 @@ +(snippet) { + header { + {blocks.bar} + } + import sub_snippet { + bar { + {blocks.foo} + } + } +} +(sub_snippet) { + header { + {blocks.bar} + } +} +example.com { + import snippet { + foo { + foo a + } + bar { + bar b + } + } +} +---------- +{ + "apps": { + "http": { + "servers": { + "srv0": { + "listen": [ + ":443" + ], + "routes": [ + { + "match": [ + { + "host": [ + "example.com" + ] + } + ], + "handle": [ + { + "handler": "subroute", + "routes": [ + { + "handle": [ + { + "handler": "headers", + "response": { + "set": { + "Bar": [ + "b" + ] + } + } + }, + { + "handler": "headers", + "response": { + "set": { + "Foo": [ + "a" + ] + } + } + } + ] + } + ] + } + ], + "terminal": true + } + ] + } + } + } + } +} From fab6375a8bebd952abc80e63fa31b648ae1ebc0b Mon Sep 17 00:00:00 2001 From: Jason Yuan Date: Sat, 15 Jun 2024 09:50:31 -0400 Subject: [PATCH 11/57] reverseproxy: add Max-Age option to sticky cookie (#6398) * reverseproxy: add Max-Age option to sticky cookie * Update selectionpolicies.go Co-authored-by: Francis Lavoie * Update selectionpolicies.go Co-authored-by: Francis Lavoie --------- Co-authored-by: Francis Lavoie --- .../reverseproxy/selectionpolicies.go | 25 +++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/modules/caddyhttp/reverseproxy/selectionpolicies.go b/modules/caddyhttp/reverseproxy/selectionpolicies.go index e61b3e0f..293ff75e 100644 --- a/modules/caddyhttp/reverseproxy/selectionpolicies.go +++ b/modules/caddyhttp/reverseproxy/selectionpolicies.go @@ -26,6 +26,7 @@ import ( "strconv" "strings" "sync/atomic" + "time" "github.com/cespare/xxhash/v2" @@ -613,6 +614,8 @@ type CookieHashSelection struct { Name string `json:"name,omitempty"` // Secret to hash (Hmac256) chosen upstream in cookie Secret string `json:"secret,omitempty"` + // The cookie's Max-Age before it expires. Default is no expiry. + MaxAge caddy.Duration `json:"max_age,omitempty"` // The fallback policy to use if the cookie is not present. Defaults to `random`. FallbackRaw json.RawMessage `json:"fallback,omitempty" caddy:"namespace=http.reverse_proxy.selection_policies inline_key=policy"` @@ -671,6 +674,9 @@ func (s CookieHashSelection) Select(pool UpstreamPool, req *http.Request, w http cookie.Secure = true cookie.SameSite = http.SameSiteNoneMode } + if s.MaxAge > 0 { + cookie.MaxAge = int(time.Duration(s.MaxAge).Seconds()) + } http.SetCookie(w, cookie) return upstream } @@ -699,6 +705,7 @@ func (s CookieHashSelection) Select(pool UpstreamPool, req *http.Request, w http // // lb_policy cookie [ []] { // fallback +// max_age // } // // By default name is `lb` @@ -728,6 +735,24 @@ func (s *CookieHashSelection) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { return err } s.FallbackRaw = mod + case "max_age": + if !d.NextArg() { + return d.ArgErr() + } + if s.MaxAge != 0 { + return d.Err("cookie max_age already specified") + } + maxAge, err := caddy.ParseDuration(d.Val()) + if err != nil { + return d.Errf("invalid duration: %s", d.Val()) + } + if maxAge <= 0 { + return d.Errf("invalid duration: %s, max_age should be non-zero and positive", d.Val()) + } + if d.NextArg() { + return d.ArgErr() + } + s.MaxAge = caddy.Duration(maxAge) default: return d.Errf("unrecognized option '%s'", d.Val()) } From 99dcdf7e426f0dcbdffe510f241ae8a4fd5a56e6 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Tue, 18 Jun 2024 14:43:54 -0600 Subject: [PATCH 12/57] caddyhttp: Convert IDNs to ASCII when provisioning Host matcher --- modules/caddyhttp/matchers.go | 20 ++++++++++++++------ modules/caddyhttp/matchers_test.go | 9 +++++++++ 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/modules/caddyhttp/matchers.go b/modules/caddyhttp/matchers.go index b1da1468..392312b6 100644 --- a/modules/caddyhttp/matchers.go +++ b/modules/caddyhttp/matchers.go @@ -34,6 +34,7 @@ import ( "github.com/google/cel-go/cel" "github.com/google/cel-go/common/types" "github.com/google/cel-go/common/types/ref" + "golang.org/x/net/idna" "github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" @@ -239,13 +240,20 @@ func (m *MatchHost) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { func (m MatchHost) Provision(_ caddy.Context) error { // check for duplicates; they are nonsensical and reduce efficiency // (we could just remove them, but the user should know their config is erroneous) - seen := make(map[string]int) - for i, h := range m { - h = strings.ToLower(h) - if firstI, ok := seen[h]; ok { - return fmt.Errorf("host at index %d is repeated at index %d: %s", firstI, i, h) + seen := make(map[string]int, len(m)) + for i, host := range m { + asciiHost, err := idna.ToASCII(host) + if err != nil { + return fmt.Errorf("converting hostname '%s' to ASCII: %v", host, err) } - seen[h] = i + if asciiHost != host { + m[i] = asciiHost + } + normalizedHost := strings.ToLower(asciiHost) + if firstI, ok := seen[normalizedHost]; ok { + return fmt.Errorf("host at index %d is repeated at index %d: %s", firstI, i, host) + } + seen[normalizedHost] = i } if m.large() { diff --git a/modules/caddyhttp/matchers_test.go b/modules/caddyhttp/matchers_test.go index 5f76a36b..05eaade5 100644 --- a/modules/caddyhttp/matchers_test.go +++ b/modules/caddyhttp/matchers_test.go @@ -78,6 +78,11 @@ func TestHostMatcher(t *testing.T) { input: "bar.example.com", expect: false, }, + { + match: MatchHost{"éxàmplê.com"}, + input: "xn--xmpl-0na6cm.com", + expect: true, + }, { match: MatchHost{"*.example.com"}, input: "example.com", @@ -149,6 +154,10 @@ func TestHostMatcher(t *testing.T) { ctx := context.WithValue(req.Context(), caddy.ReplacerCtxKey, repl) req = req.WithContext(ctx) + if err := tc.match.Provision(caddy.Context{}); err != nil { + t.Errorf("Test %d %v: provisioning failed: %v", i, tc.match, err) + } + actual := tc.match.Match(req) if actual != tc.expect { t.Errorf("Test %d %v: Expected %t, got %t for '%s'", i, tc.match, tc.expect, actual, tc.input) From c2ccf8690f315aa0ebab930c3aadcc6cd11fc9e9 Mon Sep 17 00:00:00 2001 From: Aziz Rmadi <46684200+armadi1809@users.noreply.github.com> Date: Wed, 19 Jun 2024 08:27:10 -0500 Subject: [PATCH 13/57] fileserver: Remove newline characters from precomputed etags (#6394) * Removed newline characters from precomputed etags * Update modules/caddyhttp/fileserver/staticfiles.go --------- Co-authored-by: Matt Holt --- modules/caddyhttp/fileserver/staticfiles.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/caddyhttp/fileserver/staticfiles.go b/modules/caddyhttp/fileserver/staticfiles.go index 5d54742d..3d703280 100644 --- a/modules/caddyhttp/fileserver/staticfiles.go +++ b/modules/caddyhttp/fileserver/staticfiles.go @@ -15,6 +15,7 @@ package fileserver import ( + "bytes" "errors" "fmt" "io" @@ -690,6 +691,10 @@ func (fsrv *FileServer) getEtagFromFile(fileSystem fs.FS, filename string) (stri if err != nil { return "", fmt.Errorf("cannot read etag from file %s: %v", etagFilename, err) } + + // Etags should not contain newline characters + etag = bytes.ReplaceAll(etag, []byte("\n"), []byte{}) + return string(etag), nil } return "", nil From f8861ca16bd475e8519e7dbf5a2b55e81b329874 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Fri, 28 Jun 2024 12:15:41 -0600 Subject: [PATCH 14/57] reverseproxy: Wire up TLS options for H3 transport --- modules/caddyhttp/reverseproxy/httptransport.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/modules/caddyhttp/reverseproxy/httptransport.go b/modules/caddyhttp/reverseproxy/httptransport.go index 80a49806..d4245368 100644 --- a/modules/caddyhttp/reverseproxy/httptransport.go +++ b/modules/caddyhttp/reverseproxy/httptransport.go @@ -363,6 +363,13 @@ func (h *HTTPTransport) NewTransport(caddyCtx caddy.Context) (*http.Transport, e // site owners control the backends), so it must be exclusive if len(h.Versions) == 1 && h.Versions[0] == "3" { h.h3Transport = new(http3.RoundTripper) + if h.TLS != nil { + var err error + h.h3Transport.TLSClientConfig, err = h.TLS.MakeTLSClientConfig(caddyCtx) + if err != nil { + return nil, fmt.Errorf("making TLS client config for HTTP/3 transport: %v", err) + } + } } else if len(h.Versions) > 1 && sliceContains(h.Versions, "3") { return nil, fmt.Errorf("if HTTP/3 is enabled to the upstream, no other HTTP versions are supported") } From 0287009ee5fbe171e7a84f7d5b965992bb5488a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Dunglas?= Date: Wed, 3 Jul 2024 16:43:13 +0200 Subject: [PATCH 15/57] intercept: fix http.intercept.header.* placeholder (#6429) --- caddytest/integration/intercept_test.go | 8 +++++++- modules/caddyhttp/intercept/intercept.go | 3 +-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/caddytest/integration/intercept_test.go b/caddytest/integration/intercept_test.go index 81db6a7d..6f8ffc92 100644 --- a/caddytest/integration/intercept_test.go +++ b/caddytest/integration/intercept_test.go @@ -18,17 +18,23 @@ func TestIntercept(t *testing.T) { localhost:9080 { respond /intercept "I'm a teapot" 408 + header /intercept To-Intercept ok respond /no-intercept "I'm not a teapot" intercept { @teapot status 408 handle_response @teapot { + header /intercept intercepted {resp.header.To-Intercept} respond /intercept "I'm a combined coffee/tea pot that is temporarily out of coffee" 503 } } } `, "caddyfile") - tester.AssertGetResponse("http://localhost:9080/intercept", 503, "I'm a combined coffee/tea pot that is temporarily out of coffee") + r, _ := tester.AssertGetResponse("http://localhost:9080/intercept", 503, "I'm a combined coffee/tea pot that is temporarily out of coffee") + if r.Header.Get("intercepted") != "ok" { + t.Fatalf(`header "intercepted" value is not "ok": %s`, r.Header.Get("intercepted")) + } + tester.AssertGetResponse("http://localhost:9080/no-intercept", 200, "I'm not a teapot") } diff --git a/modules/caddyhttp/intercept/intercept.go b/modules/caddyhttp/intercept/intercept.go index 47d7511f..720a0933 100644 --- a/modules/caddyhttp/intercept/intercept.go +++ b/modules/caddyhttp/intercept/intercept.go @@ -50,7 +50,6 @@ type Intercept struct { // // Three new placeholders are available in this handler chain: // - `{http.intercept.status_code}` The status code from the response - // - `{http.intercept.status_text}` The status text from the response // - `{http.intercept.header.*}` The headers from the response HandleResponse []caddyhttp.ResponseHandler `json:"handle_response,omitempty"` @@ -161,7 +160,7 @@ func (ir Intercept) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddy // set up the replacer so that parts of the original response can be // used for routing decisions - for field, value := range r.Header { + for field, value := range rec.Header() { repl.Set("http.intercept.header."+field, strings.Join(value, ",")) } repl.Set("http.intercept.status_code", rec.Status()) From f350e001b6319dd8833fbdb31ffb0ccadb2aa2e0 Mon Sep 17 00:00:00 2001 From: klaxa Date: Wed, 3 Jul 2024 21:05:52 +0200 Subject: [PATCH 16/57] reverseproxy: Only log host is up status on change (fixes #6415) (#6419) --- modules/caddyhttp/reverseproxy/healthchecks.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/caddyhttp/reverseproxy/healthchecks.go b/modules/caddyhttp/reverseproxy/healthchecks.go index 90db9b34..888dadb7 100644 --- a/modules/caddyhttp/reverseproxy/healthchecks.go +++ b/modules/caddyhttp/reverseproxy/healthchecks.go @@ -426,6 +426,7 @@ func (h *Handler) doActiveHealthCheck(dialInfo DialInfo, hostAddr string, upstre } if upstream.Host.activeHealthPasses() >= h.HealthChecks.Active.Passes { if upstream.setHealthy(true) { + h.HealthChecks.Active.logger.Info("host is up", zap.String("host", hostAddr)) h.events.Emit(h.ctx, "healthy", map[string]any{"host": hostAddr}) upstream.Host.resetHealth() } @@ -492,7 +493,6 @@ func (h *Handler) doActiveHealthCheck(dialInfo DialInfo, hostAddr string, upstre } // passed health check parameters, so mark as healthy - h.HealthChecks.Active.logger.Info("host is up", zap.String("host", hostAddr)) markHealthy() return nil From 15d986e1c9decae4d753d7cbec41275264697b2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?K=C3=A9vin=20Dunglas?= Date: Thu, 4 Jul 2024 22:57:13 +0200 Subject: [PATCH 17/57] encode: Don't compress already-compressed fonts (#6432) * fix: don't compress already compressed fonts * fix: remove WOFF --- modules/caddyhttp/encode/encode.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/caddyhttp/encode/encode.go b/modules/caddyhttp/encode/encode.go index 908e37b3..cf3d17b6 100644 --- a/modules/caddyhttp/encode/encode.go +++ b/modules/caddyhttp/encode/encode.go @@ -112,7 +112,8 @@ func (enc *Encode) Provision(ctx caddy.Context) error { "application/x-ttf*", "application/xhtml+xml*", "application/xml*", - "font/*", + "font/ttf*", + "font/otf*", "image/svg+xml*", "image/vnd.microsoft.icon*", "image/x-icon*", From c3fb5f4d3fb3eed9136f766cb88f2d8ac54de685 Mon Sep 17 00:00:00 2001 From: Matt Holt Date: Fri, 5 Jul 2024 10:46:20 -0600 Subject: [PATCH 18/57] caddyhttp: Reject 0-RTT early data in IP matchers and set Early-Data header when proxying (#6427) * caddyhttp: Reject 0-RTT early data in IP matchers and set Early-Data header when proxying See RFC 8470: https://httpwg.org/specs/rfc8470.html Thanks to Michael Wedl (@MWedl) at the University of Applied Sciences St. Poelten for reporting this. * Don't return value for {remote} placeholder in early data * Add Caddyfile support --- listeners.go | 6 -- modules/caddyhttp/ip_matchers.go | 6 ++ modules/caddyhttp/matchers.go | 64 +++++++++++++++++++ modules/caddyhttp/replacer.go | 8 +++ .../caddyhttp/reverseproxy/reverseproxy.go | 12 ++++ 5 files changed, 90 insertions(+), 6 deletions(-) diff --git a/listeners.go b/listeners.go index bb0e9b69..fa5ac1f5 100644 --- a/listeners.go +++ b/listeners.go @@ -60,8 +60,6 @@ type NetworkAddress struct { // ListenAll calls Listen() for all addresses represented by this struct, i.e. all ports in the range. // (If the address doesn't use ports or has 1 port only, then only 1 listener will be created.) // It returns an error if any listener failed to bind, and closes any listeners opened up to that point. -// -// TODO: Experimental API: subject to change or removal. func (na NetworkAddress) ListenAll(ctx context.Context, config net.ListenConfig) ([]any, error) { var listeners []any var err error @@ -130,8 +128,6 @@ func (na NetworkAddress) ListenAll(ctx context.Context, config net.ListenConfig) // Unix sockets will be unlinked before being created, to ensure we can bind to // it even if the previous program using it exited uncleanly; it will also be // unlinked upon a graceful exit (or when a new config does not use that socket). -// -// TODO: Experimental API: subject to change or removal. func (na NetworkAddress) Listen(ctx context.Context, portOffset uint, config net.ListenConfig) (any, error) { if na.IsUnixNetwork() { unixSocketsMu.Lock() @@ -221,8 +217,6 @@ func (na NetworkAddress) JoinHostPort(offset uint) string { } // Expand returns one NetworkAddress for each port in the port range. -// -// This is EXPERIMENTAL and subject to change or removal. func (na NetworkAddress) Expand() []NetworkAddress { size := na.PortRangeSize() addrs := make([]NetworkAddress, size) diff --git a/modules/caddyhttp/ip_matchers.go b/modules/caddyhttp/ip_matchers.go index baa7c51c..9101a035 100644 --- a/modules/caddyhttp/ip_matchers.go +++ b/modules/caddyhttp/ip_matchers.go @@ -143,6 +143,9 @@ func (m *MatchRemoteIP) Provision(ctx caddy.Context) error { // Match returns true if r matches m. func (m MatchRemoteIP) Match(r *http.Request) bool { + if r.TLS != nil && !r.TLS.HandshakeComplete { + return false // if handshake is not finished, we infer 0-RTT that has not verified remote IP; could be spoofed + } address := r.RemoteAddr clientIP, zoneID, err := parseIPZoneFromString(address) if err != nil { @@ -228,6 +231,9 @@ func (m *MatchClientIP) Provision(ctx caddy.Context) error { // Match returns true if r matches m. func (m MatchClientIP) Match(r *http.Request) bool { + if r.TLS != nil && !r.TLS.HandshakeComplete { + return false // if handshake is not finished, we infer 0-RTT that has not verified remote IP; could be spoofed + } address := GetVar(r.Context(), ClientIPVarKey).(string) clientIP, zoneID, err := parseIPZoneFromString(address) if err != nil { diff --git a/modules/caddyhttp/matchers.go b/modules/caddyhttp/matchers.go index 392312b6..b7952ab6 100644 --- a/modules/caddyhttp/matchers.go +++ b/modules/caddyhttp/matchers.go @@ -178,6 +178,22 @@ type ( // "http/2", "http/3", or minimum versions: "http/2+", etc. MatchProtocol string + // MatchTLS matches HTTP requests based on the underlying + // TLS connection state. If this matcher is specified but + // the request did not come over TLS, it will never match. + // If this matcher is specified but is empty and the request + // did come in over TLS, it will always match. + MatchTLS struct { + // Matches if the TLS handshake has completed. QUIC 0-RTT early + // data may arrive before the handshake completes. Generally, it + // is unsafe to replay these requests if they are not idempotent; + // additionally, the remote IP of early data packets can more + // easily be spoofed. It is conventional to respond with HTTP 425 + // Too Early if the request cannot risk being processed in this + // state. + HandshakeComplete *bool `json:"handshake_complete,omitempty"` + } + // MatchNot matches requests by negating the results of its matcher // sets. A single "not" matcher takes one or more matcher sets. Each // matcher set is OR'ed; in other words, if any matcher set returns @@ -213,6 +229,7 @@ func init() { caddy.RegisterModule(MatchHeader{}) caddy.RegisterModule(MatchHeaderRE{}) caddy.RegisterModule(new(MatchProtocol)) + caddy.RegisterModule(MatchTLS{}) caddy.RegisterModule(MatchNot{}) } @@ -1236,6 +1253,53 @@ func (MatchProtocol) CELLibrary(_ caddy.Context) (cel.Library, error) { ) } +// CaddyModule returns the Caddy module information. +func (MatchTLS) CaddyModule() caddy.ModuleInfo { + return caddy.ModuleInfo{ + ID: "http.matchers.tls", + New: func() caddy.Module { return new(MatchTLS) }, + } +} + +// Match returns true if r matches m. +func (m MatchTLS) Match(r *http.Request) bool { + if r.TLS == nil { + return false + } + if m.HandshakeComplete != nil { + if (!*m.HandshakeComplete && r.TLS.HandshakeComplete) || + (*m.HandshakeComplete && !r.TLS.HandshakeComplete) { + return false + } + } + return true +} + +// UnmarshalCaddyfile parses Caddyfile tokens for this matcher. Syntax: +// +// ... tls [early_data] +// +// EXPERIMENTAL SYNTAX: Subject to change. +func (m *MatchTLS) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { + // iterate to merge multiple matchers into one + for d.Next() { + if d.NextArg() { + switch d.Val() { + case "early_data": + var false bool + m.HandshakeComplete = &false + } + } + if d.NextArg() { + return d.ArgErr() + } + if d.NextBlock(0) { + return d.Err("malformed tls matcher: blocks are not supported yet") + } + } + return nil +} + // CaddyModule returns the Caddy module information. func (MatchNot) CaddyModule() caddy.ModuleInfo { return caddy.ModuleInfo{ diff --git a/modules/caddyhttp/replacer.go b/modules/caddyhttp/replacer.go index 1cf3ec47..2c0f3235 100644 --- a/modules/caddyhttp/replacer.go +++ b/modules/caddyhttp/replacer.go @@ -142,8 +142,16 @@ func addHTTPVarsToReplacer(repl *caddy.Replacer, req *http.Request, w http.Respo } return port, true case "http.request.remote": + if req.TLS != nil && !req.TLS.HandshakeComplete { + // without a complete handshake (QUIC "early data") we can't trust the remote IP address to not be spoofed + return nil, true + } return req.RemoteAddr, true case "http.request.remote.host": + if req.TLS != nil && !req.TLS.HandshakeComplete { + // without a complete handshake (QUIC "early data") we can't trust the remote IP address to not be spoofed + return nil, true + } host, _, err := net.SplitHostPort(req.RemoteAddr) if err != nil { // req.RemoteAddr is host:port for tcp and udp sockets and /unix/socket.path diff --git a/modules/caddyhttp/reverseproxy/reverseproxy.go b/modules/caddyhttp/reverseproxy/reverseproxy.go index 1a559e5d..4f97edea 100644 --- a/modules/caddyhttp/reverseproxy/reverseproxy.go +++ b/modules/caddyhttp/reverseproxy/reverseproxy.go @@ -605,6 +605,18 @@ func (h Handler) prepareRequest(req *http.Request, repl *caddy.Replacer) (*http. req.Header.Set("User-Agent", "") } + // Indicate if request has been conveyed in early data. + // RFC 8470: "An intermediary that forwards a request prior to the + // completion of the TLS handshake with its client MUST send it with + // the Early-Data header field set to “1” (i.e., it adds it if not + // present in the request). An intermediary MUST use the Early-Data + // header field if the request might have been subject to a replay and + // might already have been forwarded by it or another instance + // (see Section 6.2)." + if req.TLS != nil && !req.TLS.HandshakeComplete { + req.Header.Set("Early-Data", "1") + } + reqUpType := upgradeType(req.Header) removeConnectionHeaders(req.Header) From 7142d7c1e43ba2dad8e0118aa29d77dc74b44dda Mon Sep 17 00:00:00 2001 From: Francis Lavoie Date: Sat, 6 Jul 2024 12:43:19 -0400 Subject: [PATCH 19/57] reverseproxy: Add placeholder for host in active health check headers (#6440) --- modules/caddyhttp/reverseproxy/healthchecks.go | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/caddyhttp/reverseproxy/healthchecks.go b/modules/caddyhttp/reverseproxy/healthchecks.go index 888dadb7..ac92604c 100644 --- a/modules/caddyhttp/reverseproxy/healthchecks.go +++ b/modules/caddyhttp/reverseproxy/healthchecks.go @@ -386,6 +386,7 @@ func (h *Handler) doActiveHealthCheck(dialInfo DialInfo, hostAddr string, upstre // set headers, using a replacer with only globals (env vars, system info, etc.) repl := caddy.NewReplacer() + repl.Set("http.reverse_proxy.active.target_host", hostAddr) for key, vals := range h.HealthChecks.Active.Headers { key = repl.ReplaceAll(key, "") if key == "Host" { From 4ef360745dab1023a7d4c04aebca3d05499dd5e1 Mon Sep 17 00:00:00 2001 From: Steffen Busch <37350514+steffenbusch@users.noreply.github.com> Date: Sat, 6 Jul 2024 18:46:08 +0200 Subject: [PATCH 20/57] browse: add Content-Security-Policy w/ nonce (#6425) * browse: add Content-Security-Policy w/ nonce * Add backward-compat values to script-src * Remove dummy "#" href from layout anchors --- modules/caddyhttp/fileserver/browse.html | 73 ++++++++++++++++-------- 1 file changed, 49 insertions(+), 24 deletions(-) diff --git a/modules/caddyhttp/fileserver/browse.html b/modules/caddyhttp/fileserver/browse.html index 7b0df1e5..43d5f451 100644 --- a/modules/caddyhttp/fileserver/browse.html +++ b/modules/caddyhttp/fileserver/browse.html @@ -1,10 +1,17 @@ +{{ $nonce := uuidv4 -}} +{{ $nonceAttribute := print "nonce=" (quote $nonce) -}} +{{ $csp := printf "default-src 'none'; img-src 'self'; object-src 'none'; base-uri 'none'; script-src 'strict-dynamic' 'nonce-%s' 'unsafe-inline' https: http:; style-src 'strict-dynamic' 'nonce-%s'; frame-ancestors 'self'; form-action 'self'; block-all-mixed-content;" $nonce $nonce -}} +{{/* To disable the Content-Security-Policy, set this to false */}}{{ $enableCsp := true -}} +{{ if $enableCsp -}} + {{- .RespHeader.Set "Content-Security-Policy" $csp -}} +{{ end -}} {{- define "icon"}} {{- if .IsDir}} {{- if .IsSymlink}} - + {{- else}} @@ -303,7 +310,7 @@ - {{- if eq .Layout "grid"}} - + {{- end}} - +
Folder Path
@@ -799,7 +810,7 @@ footer { {{- end}}
- + @@ -807,7 +818,7 @@ footer { List - + @@ -886,7 +897,7 @@ footer { - + @@ -1000,70 +1011,70 @@ footer { -