diff options
26 files changed, 679 insertions, 83 deletions
@@ -53,7 +53,7 @@ _**Note:** These steps [will not embed proper version information](https://githu Requirements: -- [Go 1.13 or newer](https://golang.org/dl/) +- [Go 1.14 or newer](https://golang.org/dl/) - Do NOT disable [Go modules](https://github.com/golang/go/wiki/Modules) (`export GO111MODULE=auto`) Download the `v2` source code: diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d1dfb6b..21be1f1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -17,8 +17,6 @@ variables: GOPATH: $(system.defaultWorkingDirectory)/gopath GOBIN: $(GOPATH)/bin modulePath: '$(GOPATH)/src/github.com/$(build.repository.name)' - # TODO: Remove once it's enabled by default - GO111MODULE: on jobs: - job: crossPlatformTest @@ -29,7 +27,7 @@ jobs: imageName: ubuntu-16.04 gorootDir: /usr/local mac: - imageName: macos-10.13 + imageName: macos-10.14 gorootDir: /usr/local windows: imageName: windows-2019 @@ -78,7 +76,7 @@ jobs: condition: eq( variables['Agent.OS'], 'Windows_NT' ) displayName: Install Go on Windows - - bash: curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.22.2 + - bash: curl -sfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.23.6 displayName: Install golangci-lint - script: | @@ -102,6 +100,22 @@ jobs: workingDirectory: '$(modulePath)' displayName: Get dependencies + - bash: CGO_ENABLED=0 go build -trimpath -a -ldflags="-w -s" -v + workingDirectory: '$(modulePath)/cmd/caddy' + displayName: Build Caddy + + - task: PublishBuildArtifacts@1 + condition: eq( variables['Agent.OS'], 'Windows_NT' ) + inputs: + pathtoPublish: '$(modulePath)/cmd/caddy/caddy.exe' + artifactName: caddy_v2.exe + + - task: PublishBuildArtifacts@1 + condition: ne( variables['Agent.OS'], 'Windows_NT' ) + inputs: + pathtoPublish: '$(modulePath)/cmd/caddy/caddy' + artifactName: 'caddy_v2_$(Agent.OS)' + # its behavior is governed by .golangci.yml - script: | (golangci-lint run --out-format junit-xml) > test-results/lint-result.xml @@ -190,7 +190,7 @@ func readConfig(path string, out io.Writer) error { return unsyncedConfigAccess(http.MethodGet, path, nil, out) } -// indexConfigObjects recurisvely searches ptr for object fields named +// indexConfigObjects recursively searches ptr for object fields named // "@id" and maps that ID value to the full configPath in the index. // This function is NOT safe for concurrent access; obtain a write lock // on currentCfgMu. diff --git a/caddyconfig/caddyfile/formatter.go b/caddyconfig/caddyfile/formatter.go new file mode 100644 index 0000000..e937208 --- /dev/null +++ b/caddyconfig/caddyfile/formatter.go @@ -0,0 +1,140 @@ +// Copyright 2015 Matthew Holt and The Caddy Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package caddyfile + +import ( + "bytes" + "io" + "unicode" +) + +// Format formats a Caddyfile to conventional standards. +func Format(body []byte) []byte { + reader := bytes.NewReader(body) + result := new(bytes.Buffer) + + var ( + commented, + quoted, + escaped, + environ, + lineBegin bool + + firstIteration = true + + indentation = 0 + + prev, + curr, + next rune + + err error + ) + + for { + prev = curr + curr = next + + if curr < 0 { + break + } + + next, _, err = reader.ReadRune() + if err != nil { + if err == io.EOF { + next = -1 + } else { + panic(err) + } + } + + if firstIteration { + firstIteration = false + lineBegin = true + continue + } + + if quoted { + if escaped { + escaped = false + } else { + if curr == '\\' { + escaped = true + } + if curr == '"' { + quoted = false + } + } + if curr == '\n' { + quoted = false + } + } else if commented { + if curr == '\n' { + commented = false + } + } else { + if curr == '"' { + quoted = true + } + if curr == '#' { + commented = true + } + if curr == '}' { + if environ { + environ = false + } else if indentation > 0 { + indentation-- + } + } + if curr == '{' { + if unicode.IsSpace(next) { + indentation++ + + if !unicode.IsSpace(prev) { + result.WriteRune(' ') + } + } else { + environ = true + } + } + if lineBegin { + if curr == ' ' || curr == '\t' { + continue + } else { + lineBegin = false + if indentation > 0 { + for tabs := indentation; tabs > 0; tabs-- { + result.WriteRune('\t') + } + } + } + } else { + if prev == '{' && + (curr == ' ' || curr == '\t') && + (next != '\n' && next != '\r') { + curr = '\n' + } + } + } + + if curr == '\n' { + lineBegin = true + } + + result.WriteRune(curr) + } + + return result.Bytes() +} diff --git a/caddyconfig/caddyfile/formatter_test.go b/caddyconfig/caddyfile/formatter_test.go new file mode 100644 index 0000000..76eca00 --- /dev/null +++ b/caddyconfig/caddyfile/formatter_test.go @@ -0,0 +1,195 @@ +// Copyright 2015 Matthew Holt and The Caddy Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package caddyfile + +import ( + "testing" +) + +func TestFormatBasicIndentation(t *testing.T) { + input := []byte(` + a +b + + c { + d +} + +e { f +} + +g { +h { +i +} +} + +j { k { +l +} +} + +m { + n { o + } +} +`) + expected := []byte(` +a +b + +c { + d +} + +e { + f +} + +g { + h { + i + } +} + +j { + k { + l + } +} + +m { + n { + o + } +} +`) + testFormat(t, input, expected) +} + +func TestFormatBasicSpacing(t *testing.T) { + input := []byte(` +a{ + b +} + +c{ d +} +`) + expected := []byte(` +a { + b +} + +c { + d +} +`) + testFormat(t, input, expected) +} + +func TestFormatEnvironmentVariable(t *testing.T) { + input := []byte(` +{$A} + +b { +{$C} +} + +d { {$E} +} +`) + expected := []byte(` +{$A} + +b { + {$C} +} + +d { + {$E} +} +`) + testFormat(t, input, expected) +} + +func TestFormatComments(t *testing.T) { + input := []byte(` +# a "\n" + +# b { + c +} + +d { +e # f +# g +} + +h { # i +} +`) + expected := []byte(` +# a "\n" + +# b { +c +} + +d { + e # f + # g +} + +h { + # i +} +`) + testFormat(t, input, expected) +} + +func TestFormatQuotesAndEscapes(t *testing.T) { + input := []byte(` +"a \"b\" #c + d + +e { +"f" +} + +g { "h" +} +`) + expected := []byte(` +"a \"b\" #c +d + +e { + "f" +} + +g { + "h" +} +`) + testFormat(t, input, expected) +} + +func testFormat(t *testing.T, input, expected []byte) { + output := Format(input) + if string(output) != string(expected) { + t.Errorf("Expected:\n%s\ngot:\n%s", string(output), string(expected)) + } +} diff --git a/caddyconfig/caddyfile/parse.go b/caddyconfig/caddyfile/parse.go index f376033..cdcac26 100755 --- a/caddyconfig/caddyfile/parse.go +++ b/caddyconfig/caddyfile/parse.go @@ -64,7 +64,7 @@ func replaceEnvVars(input []byte) ([]byte, error) { } // get the value of the environment variable - envVarValue := []byte(os.Getenv(string(envVarName))) + envVarValue := []byte(os.ExpandEnv(os.Getenv(string(envVarName)))) // splice in the value input = append(input[:begin], diff --git a/caddyconfig/caddyfile/parse_test.go b/caddyconfig/caddyfile/parse_test.go index 62a3998..e6d0501 100755 --- a/caddyconfig/caddyfile/parse_test.go +++ b/caddyconfig/caddyfile/parse_test.go @@ -256,7 +256,7 @@ func TestRecursiveImport(t *testing.T) { return false } if len(got.Segments[0]) != 1 || len(got.Segments[1]) != 2 { - t.Errorf("got unexpect tokens: %v", got.Segments) + t.Errorf("got unexpected tokens: %v", got.Segments) return false } return true @@ -351,7 +351,7 @@ func TestDirectiveImport(t *testing.T) { return false } if len(got.Segments[0]) != 1 || len(got.Segments[1]) != 8 { - t.Errorf("got unexpect tokens: %v", got.Segments) + t.Errorf("got unexpected tokens: %v", got.Segments) return false } return true diff --git a/caddyconfig/httpcaddyfile/addresses.go b/caddyconfig/httpcaddyfile/addresses.go index 3aedb60..64c5d4f 100644 --- a/caddyconfig/httpcaddyfile/addresses.go +++ b/caddyconfig/httpcaddyfile/addresses.go @@ -45,7 +45,7 @@ import ( // key of its server block (specifying the host part), and each key may have // a different port. And we definitely need to be sure that a site which is // bound to be served on a specific interface is not served on others just -// beceause that is more convenient: it would be a potential security risk +// because that is more convenient: it would be a potential security risk // if the difference between interfaces means private vs. public. // // So what this function does for the example above is iterate each server diff --git a/caddyconfig/httpcaddyfile/builtins.go b/caddyconfig/httpcaddyfile/builtins.go index a085fcb..3b5a4f5 100644 --- a/caddyconfig/httpcaddyfile/builtins.go +++ b/caddyconfig/httpcaddyfile/builtins.go @@ -37,7 +37,7 @@ func init() { RegisterHandlerDirective("redir", parseRedir) RegisterHandlerDirective("respond", parseRespond) RegisterHandlerDirective("route", parseRoute) - RegisterHandlerDirective("handle", parseSegmentAsSubroute) + RegisterHandlerDirective("handle", parseHandle) RegisterDirective("handle_errors", parseHandleErrors) RegisterDirective("log", parseLog) } @@ -152,6 +152,18 @@ func parseTLS(h Helper) ([]ConfigValue, error) { // policy that is looking for any tag but the last one to be // loaded won't find it, and TLS handshakes will fail (see end) // of issue #3004) + + // tlsCertTags maps certificate filenames to their tag. + // This is used to remember which tag is used for each + // certificate files, since we need to avoid loading + // the same certificate files more than once, overwriting + // previous tags + tlsCertTags, ok := h.State["tlsCertTags"].(map[string]string) + if !ok { + tlsCertTags = make(map[string]string) + h.State["tlsCertTags"] = tlsCertTags + } + tag, ok := tlsCertTags[certFilename] if !ok { // haven't seen this cert file yet, let's give it a tag @@ -521,10 +533,15 @@ func parseLog(h Helper) ([]ConfigValue, error) { var val namedCustomLog if !reflect.DeepEqual(cl, new(caddy.CustomLog)) { + logCounter, ok := h.State["logCounter"].(int) + if !ok { + logCounter = 0 + } cl.Include = []string{"http.log.access"} val.name = fmt.Sprintf("log%d", logCounter) val.log = cl logCounter++ + h.State["logCounter"] = logCounter } configValues = append(configValues, ConfigValue{ Class: "custom_log", @@ -533,12 +550,3 @@ func parseLog(h Helper) ([]ConfigValue, error) { } return configValues, nil } - -// tlsCertTags maps certificate filenames to their tag. -// This is used to remember which tag is used for each -// certificate files, since we need to avoid loading -// the same certificate files more than once, overwriting -// previous tags -var tlsCertTags = make(map[string]string) - -var logCounter int diff --git a/caddyconfig/httpcaddyfile/directives.go b/caddyconfig/httpcaddyfile/directives.go index f82e2a8..e7a9686 100644 --- a/caddyconfig/httpcaddyfile/directives.go +++ b/caddyconfig/httpcaddyfile/directives.go @@ -114,6 +114,8 @@ func RegisterHandlerDirective(dir string, setupFunc UnmarshalHandlerFunc) { // Caddyfile tokens. type Helper struct { *caddyfile.Dispenser + // State stores intermediate variables during caddyfile adaptation. + State map[string]interface{} options map[string]interface{} warnings *[]caddyconfig.Warning matcherDefs map[string]caddy.ModuleMap @@ -161,6 +163,23 @@ func (h Helper) MatcherToken() (caddy.ModuleMap, bool, error) { return matcherSetFromMatcherToken(h.Dispenser.Token(), h.matcherDefs, h.warnings) } +// ExtractMatcherSet is like MatcherToken, except this is a higher-level +// method that returns the matcher set described by the matcher token, +// or nil if there is none, and deletes the matcher token from the +// dispenser and resets it as if this look-ahead never happened. Useful +// when wrapping a route (one or more handlers) in a user-defined matcher. +func (h Helper) ExtractMatcherSet() (caddy.ModuleMap, error) { + matcherSet, hasMatcher, err := h.MatcherToken() + if err != nil { + return nil, err + } + if hasMatcher { + h.Dispenser.Delete() // strip matcher token + } + h.Dispenser.Reset() // pretend this lookahead never happened + return matcherSet, nil +} + // NewRoute returns config values relevant to creating a new HTTP route. func (h Helper) NewRoute(matcherSet caddy.ModuleMap, handler caddyhttp.MiddlewareHandler) []ConfigValue { @@ -266,28 +285,31 @@ func sortRoutes(routes []ConfigValue) { return false } - if len(iRoute.MatcherSetsRaw) == 1 && len(jRoute.MatcherSetsRaw) == 1 { - // use already-decoded matcher, or decode if it's the first time seeing it - iPM, jPM := decodedMatchers[i], decodedMatchers[j] - if iPM == nil { - var pathMatcher caddyhttp.MatchPath - _ = json.Unmarshal(iRoute.MatcherSetsRaw[0]["path"], &pathMatcher) - decodedMatchers[i] = pathMatcher - iPM = pathMatcher - } - if jPM == nil { - var pathMatcher caddyhttp.MatchPath - _ = json.Unmarshal(jRoute.MatcherSetsRaw[0]["path"], &pathMatcher) - decodedMatchers[j] = pathMatcher - jPM = pathMatcher - } + // use already-decoded matcher, or decode if it's the first time seeing it + iPM, jPM := decodedMatchers[i], decodedMatchers[j] + if iPM == nil && len(iRoute.MatcherSetsRaw) == 1 { + var pathMatcher caddyhttp.MatchPath + _ = json.Unmarshal(iRoute.MatcherSetsRaw[0]["path"], &pathMatcher) + decodedMatchers[i] = pathMatcher + iPM = pathMatcher + } + if jPM == nil && len(jRoute.MatcherSetsRaw) == 1 { + var pathMatcher caddyhttp.MatchPath + _ = json.Unmarshal(jRoute.MatcherSetsRaw[0]["path"], &pathMatcher) + decodedMatchers[j] = pathMatcher + jPM = pathMatcher + } - // if there is only one path in the matcher, sort by - // longer path (more specific) first - if len(iPM) == 1 && len(jPM) == 1 { - return len(iPM[0]) > len(jPM[0]) - } + // sort by longer path (more specific) first; missing + // path matchers are treated as zero-length paths + var iPathLen, jPathLen int + if iPM != nil { + iPathLen = len(iPM[0]) + } + if jPM != nil { + jPathLen = len(jPM[0]) } + return iPathLen > jPathLen } return dirPositions[iDir] < dirPositions[jDir] diff --git a/caddyconfig/httpcaddyfile/httptype.go b/caddyconfig/httpcaddyfile/httptype.go index aaec2e9..d880d97 100644 --- a/caddyconfig/httpcaddyfile/httptype.go +++ b/caddyconfig/httpcaddyfile/httptype.go @@ -42,6 +42,7 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock, options map[string]interface{}) (*caddy.Config, []caddyconfig.Warning, error) { var warnings []caddyconfig.Warning gc := counter{new(int)} + state := make(map[string]interface{}) // load all the server blocks and associate them with a "pile" // of config values; also prohibit duplicate keys because they @@ -133,14 +134,17 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock, return nil, warnings, fmt.Errorf("%s:%d: unrecognized directive: %s", tkn.File, tkn.Line, dir) } - results, err := dirFunc(Helper{ + h := Helper{ Dispenser: caddyfile.NewDispenser(segment), options: options, warnings: &warnings, matcherDefs: matcherDefs, parentBlock: sb.block, groupCounter: gc, - }) + State: state, + } + + results, err := dirFunc(h) if err != nil { return nil, warnings, fmt.Errorf("parsing caddyfile tokens for '%s': %v", dir, err) } @@ -169,9 +173,10 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock, // now that each server is configured, make the HTTP app httpApp := caddyhttp.App{ - HTTPPort: tryInt(options["http_port"], &warnings), - HTTPSPort: tryInt(options["https_port"], &warnings), - Servers: servers, + HTTPPort: tryInt(options["http_port"], &warnings), + HTTPSPort: tryInt(options["https_port"], &warnings), + DefaultSNI: tryString(options["default_sni"], &warnings), + Servers: servers, } // now for the TLS app! (TODO: refactor into own func) @@ -326,7 +331,23 @@ func (st ServerType) Setup(originalServerBlocks []caddyfile.ServerBlock, &warnings) } if adminConfig, ok := options["admin"].(string); ok && adminConfig != "" { - cfg.Admin = &caddy.AdminConfig{Listen: adminConfig} + if adminConfig == "off" { + cfg.Admin = &caddy.AdminConfig{Disabled: true} + } else { + cfg.Admin = &caddy.AdminConfig{Listen: adminConfig} + } + } + if len(customLogs) > 0 { + if cfg.Logging == nil { + cfg.Logging = &caddy.Logging{ + Logs: make(map[string]*caddy.CustomLog), + } + } + for _, ncl := range customLogs { + if ncl.name != "" { + cfg.Logging.Logs[ncl.name] = ncl.log + } + } } if len(customLogs) > 0 { if cfg.Logging == nil { @@ -985,12 +1006,12 @@ func sliceContains(haystack []string, needle string) bool { return false } -// specifity returns len(s) minus any wildcards (*) and +// specificity returns len(s) minus any wildcards (*) and // placeholders ({...}). Basically, it's a length count // that penalizes the use of wildcards and placeholders. // This is useful for comparing hostnames and paths. // However, wildcards in paths are not a sure answer to -// the question of specificity. For exmaple, +// the question of specificity. For example, // '*.example.com' is clearly less specific than // 'a.example.com', but is '/a' more or less specific // than '/a*'? @@ -1021,17 +1042,12 @@ func (c counter) nextGroup() string { return name } -type matcherSetAndTokens struct { - matcherSet caddy.ModuleMap - tokens []caddyfile.Token -} - type namedCustomLog struct { name string log *caddy.CustomLog } -// sbAddrAssocation is a mapping from a list of +// sbAddrAssociation is a mapping from a list of // addresses to a list of server blocks that are // served on those addresses. type sbAddrAssociation struct { diff --git a/cmd/commandfuncs.go b/cmd/commandfuncs.go index f130c2b..8ab9099 100644 --- a/cmd/commandfuncs.go +++ b/cmd/commandfuncs.go @@ -34,6 +34,7 @@ import ( "github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2/caddyconfig" + "github.com/caddyserver/caddy/v2/caddyconfig/caddyfile" "github.com/caddyserver/certmagic" "go.uber.org/zap" ) @@ -538,6 +539,35 @@ func cmdValidateConfig(fl Flags) (int, error) { return caddy.ExitCodeSuccess, nil } +func cmdFormatConfig(fl Flags) (int, error) { + // Default path of file is Caddyfile + formatCmdConfigFile := fl.Arg(0) + if formatCmdConfigFile == "" { + formatCmdConfigFile = "Caddyfile" + } + + formatCmdWriteFlag := fl.Bool("write") + + input, err := ioutil.ReadFile(formatCmdConfigFile) + if err != nil { + return caddy.ExitCodeFailedStartup, + fmt.Errorf("reading input file: %v", err) + } + + output := caddyfile.Format(input) + + if formatCmdWriteFlag { + err = ioutil.WriteFile(formatCmdConfigFile, output, 0644) + if err != nil { + return caddy.ExitCodeFailedStartup, nil + } + } else { + fmt.Print(string(output)) + } + + return caddy.ExitCodeSuccess, nil +} + func cmdHelp(fl Flags) (int, error) { const fullDocs = `Full documentation is available at: https://caddyserver.com/docs/command-line` diff --git a/cmd/commands.go b/cmd/commands.go index 34154d4..f98fbfa 100644 --- a/cmd/commands.go +++ b/cmd/commands.go @@ -242,6 +242,24 @@ provisioning stages.`, }(), }) + RegisterCommand(Command{ + Name: "fmt", + Func: cmdFormatConfig, + Usage: "[--write] [<path>]", + Short: "Formats a Caddyfile", + Long: ` +Formats the Caddyfile by adding proper indentation and spaces to improve +human readability. It prints the result to stdout. + +If --write is specified, the output will be written to the config file +directly instead of printing it.`, + Flags: func() *flag.FlagSet { + fs := flag.NewFlagSet("format", flag.ExitOnError) + fs.Bool("write", false, "Over-write the output to specified file") + return fs + }(), + }) + } // RegisterCommand registers the command cmd. @@ -211,7 +211,7 @@ func (ctx Context) LoadModule(structPointer interface{}, fieldName string) (inte } // loadModulesFromSomeMap loads modules from val, which must be a type of map[string]interface{}. -// Depending on inlineModuleKey, it will be interpeted as either a ModuleMap (key is the module +// Depending on inlineModuleKey, it will be interpreted as either a ModuleMap (key is the module // name) or as a regular map (key is not the module name, and module name is defined inline). func (ctx Context) loadModulesFromSomeMap(namespace, inlineModuleKey string, val reflect.Value) (map[string]interface{}, error) { // if no inline_key is specified, then val must be a ModuleMap, @@ -14,7 +14,7 @@ require ( github.com/jsternberg/zap-logfmt v1.2.0 github.com/klauspost/compress v1.10.2 github.com/klauspost/cpuid v1.2.3 - github.com/lucas-clemente/quic-go v0.15.1 + github.com/lucas-clemente/quic-go v0.15.2 github.com/manifoldco/promptui v0.7.0 // indirect github.com/miekg/dns v1.1.27 // indirect github.com/muhammadmuzzammil1998/jsonc v0.0.0-20200303171503-1e787b591db7 @@ -402,8 +402,8 @@ github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/linode/linodego v0.10.0/go.mod h1:cziNP7pbvE3mXIPneHj0oRY8L1WtGEIKlZ8LANE4eXA= github.com/liquidweb/liquidweb-go v1.6.0/go.mod h1:UDcVnAMDkZxpw4Y7NOHkqoeiGacVLEIG/i5J9cyixzQ= github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= -github.com/lucas-clemente/quic-go v0.15.1 h1:XB6qeEXGfhveo4t/lClqOcfwravQgyF86DUoVf+YPz0= -github.com/lucas-clemente/quic-go v0.15.1/go.mod h1:qxmO5Y4ZMhdNkunGfxuZnZXnJwYpW9vjQkyrZ7BsgUI= +github.com/lucas-clemente/quic-go v0.15.2 h1:RgxRJ7rPde0Q/uXDeb3/UdblVvxrYGDAG9G9GO78LmI= +github.com/lucas-clemente/quic-go v0.15.2/go.mod h1:qxmO5Y4ZMhdNkunGfxuZnZXnJwYpW9vjQkyrZ7BsgUI= github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a h1:weJVJJRzAJBFRlAiJQROKQs8oC9vOxvm4rZmBBk0ONw= github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lunixbochs/vtclean v1.0.0 h1:xu2sLAri4lGiovBDQKxl5mrXyESr3gUr5m5SM5+LVb8= @@ -954,6 +954,7 @@ golang.org/x/tools v0.0.0-20200103221440-774c71fcf114/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200106190116-7be0a674c9fc h1:MR2F33ipDGog0C4eMhU6u9o3q6c3dvYis2aG6Jl12Wg= golang.org/x/tools v0.0.0-20200106190116-7be0a674c9fc/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898 h1:/atklqdjdhuosWIl6AIbOeHJjicWYPqR9bpxqxYG2pA= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/modules/caddyhttp/caddyhttp.go b/modules/caddyhttp/caddyhttp.go index 94b2eee..6ad70f5 100644 --- a/modules/caddyhttp/caddyhttp.go +++ b/modules/caddyhttp/caddyhttp.go @@ -29,6 +29,7 @@ import ( "github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2/modules/caddytls" + "github.com/caddyserver/certmagic" "github.com/lucas-clemente/quic-go/http3" "go.uber.org/zap" ) @@ -112,6 +113,10 @@ type App struct { // affect functionality. Servers map[string]*Server `json:"servers,omitempty"` + // DefaultSNI if set configures all certificate lookups to fallback to use + // this SNI name if a more specific certificate could not be found + DefaultSNI string `json:"default_sni,omitempty"` + servers []*http.Server h3servers []*http3.Server h3listeners []net.PacketConn @@ -145,8 +150,10 @@ func (app *App) Provision(ctx caddy.Context) error { repl := caddy.NewReplacer() + certmagic.Default.DefaultServerName = app.DefaultSNI + // this provisions the matchers for each route, - // and prepares auto HTTP->HTTP redirects, and + // and prepares auto HTTP->HTTPS redirects, and // is required before we provision each server err = app.automaticHTTPSPhase1(ctx, repl) if err != nil { diff --git a/modules/caddyhttp/encode/encode.go b/modules/caddyhttp/encode/encode.go index c3a1c23..52205aa 100644 --- a/modules/caddyhttp/encode/encode.go +++ b/modules/caddyhttp/encode/encode.go @@ -282,7 +282,7 @@ func acceptedEncodings(r *http.Request) []string { } // encodings with q-factor of 0 are not accepted; - // use a small theshold to account for float precision + // use a small threshold to account for float precision if qFactor < 0.00001 { continue } diff --git a/modules/caddyhttp/fileserver/matcher.go b/modules/caddyhttp/fileserver/matcher.go index ed5c102..1915fb7 100644 --- a/modules/caddyhttp/fileserver/matcher.go +++ b/modules/caddyhttp/fileserver/matcher.go @@ -51,7 +51,7 @@ type MatchFile struct { Root string `json:"root,omitempty"` // The list of files to try. Each path here is - // considered relatice to Root. If nil, the request + // considered related to Root. If nil, the request // URL's path will be assumed. Files and // directories are treated distinctly, so to match // a directory, the filepath MUST end in a forward diff --git a/modules/caddyhttp/matchers.go b/modules/caddyhttp/matchers.go index 3c357c6..6b57ead 100644 --- a/modules/caddyhttp/matchers.go +++ b/modules/caddyhttp/matchers.go @@ -680,7 +680,7 @@ func (m MatchRemoteIP) Match(r *http.Request) bool { return false } -// MatchRegexp is an embeddable type for matching +// MatchRegexp is an embedable type for matching // using regular expressions. It adds placeholders // to the request's replacer. type MatchRegexp struct { diff --git a/modules/caddyhttp/reverseproxy/caddyfile.go b/modules/caddyhttp/reverseproxy/caddyfile.go index d08e7f1..9ff9dce 100644 --- a/modules/caddyhttp/reverseproxy/caddyfile.go +++ b/modules/caddyhttp/reverseproxy/caddyfile.go @@ -15,7 +15,10 @@ package reverseproxy import ( + "net" "net/http" + "net/url" + "reflect" "strconv" "strings" "time" @@ -81,10 +84,106 @@ func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) // } // } // +// Proxy upstream addresses should be network dial addresses such +// as `host:port`, or a URL such as `scheme://host:port`. Scheme +// and port may be inferred from other parts of the address/URL; if +// either are missing, defaults to HTTP. func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { + // currently, all backends must use the same scheme/protocol (the + // underlying JSON does not yet support per-backend transports) + var commonScheme string + + // we'll wait until the very end of parsing before + // validating and encoding the transport + var transport http.RoundTripper + var transportModuleName string + + // TODO: the logic in this function is kind of sensitive, we need + // to write tests before making any more changes to it + upstreamDialAddress := func(upstreamAddr string) (string, error) { + var network, scheme, host, port string + + if strings.Contains(upstreamAddr, "://") { + toURL, err := url.Parse(upstreamAddr) + if err != nil { + return "", d.Errf("parsing upstream URL: %v", err) + } + + // there is currently no way to perform a URL rewrite between choosing + // a backend and proxying to it, so we cannot allow extra components + // in backend URLs + if toURL.Path != "" || toURL.RawQuery != "" || toURL.Fragment != "" { + return "", d.Err("for now, URLs for proxy upstreams only support scheme, host, and port components") + } + + // ensure the port and scheme aren't in conflict + urlPort := toURL.Port() + if toURL.Scheme == "http" && urlPort == "443" { + return "", d.Err("upstream address has conflicting scheme (http://) and port (:443, the HTTPS port)") + } + if toURL.Scheme == "https" && urlPort == "80" { + return "", d.Err("upstream address has conflicting scheme (https://) and port (:80, the HTTP port)") + } + + // if port is missing, attempt to infer from scheme + if toURL.Port() == "" { + var toPort string + switch toURL.Scheme { + case "", "http": + toPort = "80" + case "https": + toPort = "443" + } + toURL.Host = net.JoinHostPort(toURL.Hostname(), toPort) + } + + scheme, host, port = toURL.Scheme, toURL.Hostname(), toURL.Port() + } else { + // extract network manually, since caddy.ParseNetworkAddress() will always add one + if idx := strings.Index(upstreamAddr, "/"); idx >= 0 { + network = strings.ToLower(strings.TrimSpace(upstreamAddr[:idx])) + upstreamAddr = upstreamAddr[idx+1:] + } + var err error + host, port, err = net.SplitHostPort(upstreamAddr) + if err != nil { + host = upstreamAddr + } + } + + // if scheme is not set, we may be able to infer it from a known port + if scheme == "" { + if port == "80" { + scheme = "http" + } else if port == "443" { + scheme = "https" + } + } + + // the underlying JSON does not yet support different + // transports (protocols or schemes) to each backend, + // so we remember the last one we see and compare them + if commonScheme != "" && scheme != commonScheme { + return "", d.Errf("for now, all proxy upstreams must use the same scheme (transport protocol); expecting '%s://' but got '%s://'", + commonScheme, scheme) + } + commonScheme = scheme + + // for simplest possible config, we only need to include + // the network portion if the user specified one + if network != "" { + return caddy.JoinNetworkAddress(network, host, port), nil + } + return net.JoinHostPort(host, port), nil + } + for d.Next() { for _, up := range d.RemainingArgs() { - h.Upstreams = append(h.Upstreams, &Upstream{Dial: up}) + dialAddr, err := upstreamDialAddress(up) + if err != nil { + return err + } + h.Upstreams = append(h.Upstreams, &Upstream{Dial: dialAddr}) } for d.NextBlock(0) { @@ -95,7 +194,11 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { return d.ArgErr() } for _, up := range args { - h.Upstreams = append(h.Upstreams, &Upstream{Dial: up}) + dialAddr, err := upstreamDialAddress(up) + if err != nil { + return err + } + h.Upstreams = append(h.Upstreams, &Upstream{Dial: dialAddr}) } case "lb_policy": @@ -392,8 +495,8 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { if h.TransportRaw != nil { return d.Err("transport already specified") } - name := d.Val() - mod, err := caddy.GetModule("http.reverse_proxy.transport." + name) + transportModuleName = d.Val() + mod, err := caddy.GetModule("http.reverse_proxy.transport." + transportModuleName) if err != nil { return d.Errf("getting transport module '%s': %v", mod, err) } @@ -409,7 +512,7 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { if !ok { return d.Errf("module %s is not a RoundTripper", mod) } - h.TransportRaw = caddyconfig.JSONModuleObject(rt, "protocol", name, nil) + transport = rt default: return d.Errf("unrecognized subdirective %s", d.Val()) @@ -417,6 +520,39 @@ func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { } } + // if the scheme inferred from the backends' addresses is + // HTTPS, we will need a non-nil transport to enable TLS + if commonScheme == "https" && transport == nil { + transport = new(HTTPTransport) + transportModuleName = "http" + } + + // verify transport configuration, and finally encode it + if transport != nil { + // TODO: these two cases are identical, but I don't know how to reuse the code + switch ht := transport.(type) { + case *HTTPTransport: + if commonScheme == "https" && ht.TLS == nil { + ht.TLS = new(TLSConfig) + } + if ht.TLS != nil && commonScheme == "http" { + return d.Errf("upstream address scheme is HTTP but transport is configured for HTTP+TLS (HTTPS)") + } + + case *NTLMTransport: + if commonScheme == "https" && ht.TLS == nil { + ht.TLS = new(TLSConfig) + } + if ht.TLS != nil && commonScheme == "http" { + return d.Errf("upstream address scheme is HTTP but transport is configured for HTTP+TLS (HTTPS)") + } + } + + if !reflect.DeepEqual(transport, new(HTTPTransport)) { + h.TransportRaw = caddyconfig.JSONModuleObject(transport, "protocol", transportModuleName, nil) + } + } + return nil } diff --git a/modules/caddyhttp/reverseproxy/command.go b/modules/caddyhttp/reverseproxy/command.go index 462be1b..6f70d14 100644 --- a/modules/caddyhttp/reverseproxy/command.go +++ b/modules/caddyhttp/reverseproxy/command.go @@ -36,7 +36,7 @@ func init() { caddycmd.RegisterCommand(caddycmd.Command{ Name: "reverse-proxy", Func: cmdReverseProxy, - Usage: "[--from <addr>] [--to <addr>]", + Usage: "[--from <addr>] [--to <addr>] [--change-host-header]", Short: "A quick and production-ready reverse proxy", Long: ` A simple but production-ready reverse proxy. Useful for quick deployments, @@ -46,11 +46,16 @@ Simply shuttles HTTP traffic from the --from address to the --to address. If the --from address has a domain name, Caddy will attempt to serve the proxy over HTTPS with a certificate. + +If --change-host-header is set, the Host header on the request will be modified +from its original incoming value to the address of the upstream. (Otherwise, by +default, all incoming headers are passed through unmodified.) `, Flags: func() *flag.FlagSet { fs := flag.NewFlagSet("file-server", flag.ExitOnError) - fs.String("from", "", "Address to receive traffic on") - fs.String("to", "", "Upstream address to proxy traffic to") + fs.String("from", "", "Address on which to receive traffic") + fs.String("to", "", "Upstream address to which to to proxy traffic") + fs.Bool("change-host-header", false, "Set upstream Host header to address of upstream") return fs }(), }) @@ -59,6 +64,7 @@ proxy over HTTPS with a certificate. func cmdReverseProxy(fs caddycmd.Flags) (int, error) { from := fs.String("from") to := fs.String("to") + changeHost := fs.Bool("change-host-header") if from == "" { from = "localhost:" + httpcaddyfile.DefaultPort @@ -97,13 +103,16 @@ func cmdReverseProxy(fs caddycmd.Flags) (int, error) { handler := Handler{ TransportRaw: caddyconfig.JSONModuleObject(ht, "protocol", "http", nil), Upstreams: UpstreamPool{{Dial: toURL.Host}}, - Headers: &headers.Handler{ + } + + if changeHost { + handler.Headers = &headers.Handler{ Request: &headers.HeaderOps{ Set: http.Header{ - "Host": []string{"{http.reverse_proxy.upstream.host}"}, + "Host": []string{"{http.reverse_proxy.upstream.hostport}"}, }, }, - }, + } } route := caddyhttp.Route{ diff --git a/modules/caddyhttp/reverseproxy/fastcgi/caddyfile.go b/modules/caddyhttp/reverseproxy/fastcgi/caddyfile.go index 8c9fd38..81fd48e 100644 --- a/modules/caddyhttp/reverseproxy/fastcgi/caddyfile.go +++ b/modules/caddyhttp/reverseproxy/fastcgi/caddyfile.go @@ -105,7 +105,7 @@ func (t *Transport) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { // } // // Thus, this directive produces multiple handlers, each with a different -// matcher because multiple consecutive hgandlers are necessary to support +// matcher because multiple consecutive handlers are necessary to support // the common PHP use case. If this "common" config is not compatible // with a user's PHP requirements, they can use a manual approach based // on the example above to configure it precisely as they need. @@ -167,14 +167,10 @@ func parsePHPFastCGI(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error // either way, strip the matcher token and pass // the remaining tokens to the unmarshaler so that // we can gain the rest of the reverse_proxy syntax - userMatcherSet, hasUserMatcher, err := h.MatcherToken() + userMatcherSet, err := h.ExtractMatcherSet() if err != nil { return nil, err } - if hasUserMatcher { - h.Dispenser.Delete() // strip matcher token - } - h.Dispenser.Reset() // pretend this lookahead never happened // set up the transport for FastCGI, and specifically PHP fcgiTransport := Transport{SplitPath: ".php"} @@ -186,6 +182,8 @@ func parsePHPFastCGI(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error // the rest of the config is specified by the user // using the reverse_proxy directive syntax + // TODO: this can overwrite our fcgiTransport that we encoded and + // set on the rpHandler... even with a non-fastcgi transport! err = rpHandler.UnmarshalCaddyfile(h.Dispenser) if err != nil { return nil, err @@ -204,7 +202,7 @@ func parsePHPFastCGI(h httpcaddyfile.Helper) ([]httpcaddyfile.ConfigValue, error // the user's matcher is a prerequisite for ours, so // wrap ours in a subroute and return that - if hasUserMatcher { + if userMatcherSet != nil { return []httpcaddyfile.ConfigValue{ { Class: "route", diff --git a/modules/caddyhttp/reverseproxy/hosts.go b/modules/caddyhttp/reverseproxy/hosts.go index 54de5a8..602aab2 100644 --- a/modules/caddyhttp/reverseproxy/hosts.go +++ b/modules/caddyhttp/reverseproxy/hosts.go @@ -27,7 +27,7 @@ import ( // Host represents a remote host which can be proxied to. // Its methods must be safe for concurrent use. type Host interface { - // NumRequests returns the numnber of requests + // NumRequests returns the number of requests // currently in process with the host. NumRequests() int diff --git a/modules/caddyhttp/server.go b/modules/caddyhttp/server.go index 580449b..1e22790 100644 --- a/modules/caddyhttp/server.go +++ b/modules/caddyhttp/server.go @@ -94,7 +94,9 @@ type Server struct { // client authentication. StrictSNIHost *bool `json:"strict_sni_host,omitempty"` - // Logs customizes how access logs are handled in this server. + // Customizes how access logs are handled in this server. To + // minimally enable access logs, simply set this to a non-null, + // empty struct. Logs *ServerLogConfig `json:"logs,omitempty"` // Enable experimental HTTP/3 support. Note that HTTP/3 is not a diff --git a/modules/caddyhttp/templates/templates.go b/modules/caddyhttp/templates/templates.go index 94764bf..cf0908d 100644 --- a/modules/caddyhttp/templates/templates.go +++ b/modules/caddyhttp/templates/templates.go @@ -254,7 +254,7 @@ func (t *Templates) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddy rec.Header().Del("Accept-Ranges") // we don't know ranges for dynamically-created content rec.Header().Del("Last-Modified") // useless for dynamic content since it's always changing - // we don't know a way to guickly generate etag for dynamic content, + // we don't know a way to quickly generate etag for dynamic content, // and weak etags still cause browsers to rely on it even after a // refresh, so disable them until we find a better way to do this rec.Header().Del("Etag") |