From 8bc05e598da0068358a2876c45f92f2c77455341 Mon Sep 17 00:00:00 2001 From: WeidiDeng Date: Fri, 17 Feb 2023 08:08:36 +0800 Subject: caddyfile: Implement variadics for import args placeholders (#5249) * implement variadic placeholders imported snippets reflect actual lines in file * add import directive line number for imported snippets add tests for parsing * add realfile field to help debug import cycle detection. * use file field to reflect import chain * Switch syntax, deprecate old syntax, refactoring - Moved the import args handling to a separate file - Using {args[0:1]} syntax now - Deprecate {args.*} syntax - Use a replacer map for better control over the parsing - Add plenty of warnings when invalid placeholders are detected - Renaming variables, cleanup comments for readability - More tests to cover edgecases I could think of - Minor cleanup to snippet tracking in tokens, drop a redundant boolean field in tokens --------- Co-authored-by: Francis Lavoie --- caddyconfig/caddyfile/parse.go | 45 ++++++++++++++++++++++++++---------------- 1 file changed, 28 insertions(+), 17 deletions(-) (limited to 'caddyconfig/caddyfile/parse.go') diff --git a/caddyconfig/caddyfile/parse.go b/caddyconfig/caddyfile/parse.go index edc86f2..c65acae 100644 --- a/caddyconfig/caddyfile/parse.go +++ b/caddyconfig/caddyfile/parse.go @@ -20,7 +20,6 @@ import ( "io" "os" "path/filepath" - "strconv" "strings" "github.com/caddyserver/caddy/v2" @@ -173,11 +172,10 @@ func (p *parser) begin() error { return err } // Just as we need to track which file the token comes from, we need to - // keep track of which snippets do the tokens come from. This is helpful - // in tracking import cycles across files/snippets by namespacing them. Without - // this we end up with false-positives in cycle-detection. + // keep track of which snippet the token comes from. This is helpful + // in tracking import cycles across files/snippets by namespacing them. + // Without this, we end up with false-positives in cycle-detection. for k, v := range tokens { - v.inSnippet = true v.snippetName = name tokens[k] = v } @@ -337,11 +335,8 @@ func (p *parser) doImport() error { // grab remaining args as placeholder replacements args := p.RemainingArgs() - // add args to the replacer - repl := caddy.NewEmptyReplacer() - for index, arg := range args { - repl.Set("args."+strconv.Itoa(index), arg) - } + // set up a replacer for non-variadic args replacement + repl := makeArgsReplacer(args) // splice out the import directive and its arguments // (2 tokens, plus the length of args) @@ -416,8 +411,8 @@ func (p *parser) doImport() error { nodes = matches } - nodeName := p.File() - if p.Token().inSnippet { + nodeName := p.Token().originalFile() + if p.Token().snippetName != "" { nodeName += fmt.Sprintf(":%s", p.Token().snippetName) } p.importGraph.addNode(nodeName) @@ -428,13 +423,29 @@ func (p *parser) doImport() error { } // copy the tokens so we don't overwrite p.definedSnippets - tokensCopy := make([]Token, len(importedTokens)) - copy(tokensCopy, importedTokens) + tokensCopy := make([]Token, 0, len(importedTokens)) // run the argument replacer on the tokens - for index, token := range tokensCopy { - token.Text = repl.ReplaceKnown(token.Text, "") - tokensCopy[index] = token + // golang for range slice return a copy of value + // similarly, append also copy value + for _, token := range importedTokens { + // set the token's file to refer to import directive line number and snippet name + if token.snippetName != "" { + token.updateFile(fmt.Sprintf("%s:%d (import %s)", token.File, p.Line(), token.snippetName)) + } else { + token.updateFile(fmt.Sprintf("%s:%d (import)", token.File, p.Line())) + } + + foundVariadic, startIndex, endIndex := parseVariadic(token, len(args)) + if foundVariadic { + for _, arg := range args[startIndex:endIndex] { + token.Text = arg + tokensCopy = append(tokensCopy, token) + } + } else { + token.Text = repl.ReplaceKnown(token.Text, "") + tokensCopy = append(tokensCopy, token) + } } // splice the imported tokens in the place of the import statement -- cgit v1.2.3 From 53b6fab125f3f2f149d59fcfe13b1e8b1735da56 Mon Sep 17 00:00:00 2001 From: Francis Lavoie Date: Thu, 20 Apr 2023 14:43:51 -0400 Subject: caddyfile: Stricter parsing, error for brace on new line (#5505) --- caddyconfig/caddyfile/parse.go | 3 +++ 1 file changed, 3 insertions(+) (limited to 'caddyconfig/caddyfile/parse.go') diff --git a/caddyconfig/caddyfile/parse.go b/caddyconfig/caddyfile/parse.go index c65acae..ab84086 100644 --- a/caddyconfig/caddyfile/parse.go +++ b/caddyconfig/caddyfile/parse.go @@ -520,6 +520,9 @@ func (p *parser) directive() error { if !p.isNextOnNewLine() && p.Token().wasQuoted == 0 { return p.Err("Unexpected next token after '{' on same line") } + if p.isNewLine() { + return p.Err("Unexpected '{' on a new line; did you mean to place the '{' on the previous line?") + } } else if p.Val() == "{}" { if p.isNextOnNewLine() && p.Token().wasQuoted == 0 { return p.Err("Unexpected '{}' at end of line") -- cgit v1.2.3 From cbf16f6d9eb77f37d6eb588ff3e54cfdfddecc21 Mon Sep 17 00:00:00 2001 From: Francis Lavoie Date: Tue, 16 May 2023 11:27:52 -0400 Subject: caddyhttp: Implement named routes, `invoke` directive (#5107) * caddyhttp: Implement named routes, `invoke` directive * gofmt * Add experimental marker * Adjust route compile comments --- caddyconfig/caddyfile/parse.go | 50 +++++++++++++++++++++++++++++++++++------- 1 file changed, 42 insertions(+), 8 deletions(-) (limited to 'caddyconfig/caddyfile/parse.go') diff --git a/caddyconfig/caddyfile/parse.go b/caddyconfig/caddyfile/parse.go index ab84086..64d1062 100644 --- a/caddyconfig/caddyfile/parse.go +++ b/caddyconfig/caddyfile/parse.go @@ -148,7 +148,6 @@ func (p *parser) begin() error { } err := p.addresses() - if err != nil { return err } @@ -159,6 +158,25 @@ func (p *parser) begin() error { return nil } + if ok, name := p.isNamedRoute(); ok { + // named routes only have one key, the route name + p.block.Keys = []string{name} + p.block.IsNamedRoute = true + + // we just need a dummy leading token to ease parsing later + nameToken := p.Token() + nameToken.Text = name + + // get all the tokens from the block, including the braces + tokens, err := p.blockTokens(true) + if err != nil { + return err + } + tokens = append([]Token{nameToken}, tokens...) + p.block.Segments = []Segment{tokens} + return nil + } + if ok, name := p.isSnippet(); ok { if p.definedSnippets == nil { p.definedSnippets = map[string][]Token{} @@ -167,7 +185,7 @@ func (p *parser) begin() error { return p.Errf("redeclaration of previously declared snippet %s", name) } // consume all tokens til matched close brace - tokens, err := p.snippetTokens() + tokens, err := p.blockTokens(false) if err != nil { return err } @@ -576,6 +594,15 @@ func (p *parser) closeCurlyBrace() error { return nil } +func (p *parser) isNamedRoute() (bool, string) { + keys := p.block.Keys + // A named route block is a single key with parens, prefixed with &. + if len(keys) == 1 && strings.HasPrefix(keys[0], "&(") && strings.HasSuffix(keys[0], ")") { + return true, strings.TrimSuffix(keys[0][2:], ")") + } + return false, "" +} + func (p *parser) isSnippet() (bool, string) { keys := p.block.Keys // A snippet block is a single key with parens. Nothing else qualifies. @@ -586,18 +613,24 @@ func (p *parser) isSnippet() (bool, string) { } // read and store everything in a block for later replay. -func (p *parser) snippetTokens() ([]Token, error) { - // snippet must have curlies. +func (p *parser) blockTokens(retainCurlies bool) ([]Token, error) { + // block must have curlies. err := p.openCurlyBrace() if err != nil { return nil, err } - nesting := 1 // count our own nesting in snippets + nesting := 1 // count our own nesting tokens := []Token{} + if retainCurlies { + tokens = append(tokens, p.Token()) + } for p.Next() { if p.Val() == "}" { nesting-- if nesting == 0 { + if retainCurlies { + tokens = append(tokens, p.Token()) + } break } } @@ -617,9 +650,10 @@ func (p *parser) snippetTokens() ([]Token, error) { // head of the server block with tokens, which are // grouped by segments. type ServerBlock struct { - HasBraces bool - Keys []string - Segments []Segment + HasBraces bool + Keys []string + Segments []Segment + IsNamedRoute bool } // DispenseDirective returns a dispenser that contains -- cgit v1.2.3 From cee4441cb1d485b38d728168a315cda5641d84fb Mon Sep 17 00:00:00 2001 From: WeidiDeng Date: Tue, 23 May 2023 05:36:55 +0800 Subject: caddyfile: Do not replace import tokens if they are part of a snippet (#5539) * fix variadic placeholder in imported file which also imports * fix tests. * skip replacing args when imported token may be part of a snippet --- caddyconfig/caddyfile/parse.go | 50 +++++++++++++++++++++++++++++++++++++----- 1 file changed, 45 insertions(+), 5 deletions(-) (limited to 'caddyconfig/caddyfile/parse.go') diff --git a/caddyconfig/caddyfile/parse.go b/caddyconfig/caddyfile/parse.go index 64d1062..5afdc21 100644 --- a/caddyconfig/caddyfile/parse.go +++ b/caddyconfig/caddyfile/parse.go @@ -214,7 +214,7 @@ func (p *parser) addresses() error { // special case: import directive replaces tokens during parse-time if tkn == "import" && p.isNewLine() { - err := p.doImport() + err := p.doImport(0) if err != nil { return err } @@ -314,7 +314,7 @@ func (p *parser) directives() error { // special case: import directive replaces tokens during parse-time if p.Val() == "import" { - err := p.doImport() + err := p.doImport(1) if err != nil { return err } @@ -340,7 +340,7 @@ func (p *parser) directives() error { // is on the token before where the import directive was. In // other words, call Next() to access the first token that was // imported. -func (p *parser) doImport() error { +func (p *parser) doImport(nesting int) error { // syntax checks if !p.NextArg() { return p.ArgErr() @@ -443,10 +443,16 @@ func (p *parser) doImport() error { // copy the tokens so we don't overwrite p.definedSnippets tokensCopy := make([]Token, 0, len(importedTokens)) + var ( + maybeSnippet bool + maybeSnippetId bool + index int + ) + // run the argument replacer on the tokens // golang for range slice return a copy of value // similarly, append also copy value - for _, token := range importedTokens { + for i, token := range importedTokens { // set the token's file to refer to import directive line number and snippet name if token.snippetName != "" { token.updateFile(fmt.Sprintf("%s:%d (import %s)", token.File, p.Line(), token.snippetName)) @@ -454,6 +460,40 @@ func (p *parser) doImport() error { token.updateFile(fmt.Sprintf("%s:%d (import)", token.File, p.Line())) } + // naive way of determine snippets, as snippets definition can only follow name + block + // format, won't check for nesting correctness or any other error, that's what parser does. + if !maybeSnippet && nesting == 0 { + // first of the line + if i == 0 || importedTokens[i-1].originalFile() != token.originalFile() || importedTokens[i-1].Line+importedTokens[i-1].NumLineBreaks() < token.Line { + index = 0 + } else { + index++ + } + + if index == 0 && len(token.Text) >= 3 && strings.HasPrefix(token.Text, "(") && strings.HasSuffix(token.Text, ")") { + maybeSnippetId = true + } + } + + switch token.Text { + case "{": + nesting++ + if index == 1 && maybeSnippetId && nesting == 1 { + maybeSnippet = true + maybeSnippetId = false + } + case "}": + nesting-- + if nesting == 0 && maybeSnippet { + maybeSnippet = false + } + } + + if maybeSnippet { + tokensCopy = append(tokensCopy, token) + continue + } + foundVariadic, startIndex, endIndex := parseVariadic(token, len(args)) if foundVariadic { for _, arg := range args[startIndex:endIndex] { @@ -553,7 +593,7 @@ func (p *parser) directive() error { } else if p.Val() == "}" && p.nesting == 0 { return p.Err("Unexpected '}' because no matching opening brace") } else if p.Val() == "import" && p.isNewLine() { - if err := p.doImport(); err != nil { + if err := p.doImport(1); err != nil { return err } p.cursor-- // cursor is advanced when we continue, so roll back one more -- cgit v1.2.3 From 9cde71552576dcc724eff38dee42879c927b72ec Mon Sep 17 00:00:00 2001 From: WeidiDeng Date: Fri, 26 May 2023 03:05:00 +0800 Subject: caddyfile: Track import name instead of modifying filename (#5540) * Merge branch 'master' into import_file_stack * remove space in log key --- caddyconfig/caddyfile/parse.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'caddyconfig/caddyfile/parse.go') diff --git a/caddyconfig/caddyfile/parse.go b/caddyconfig/caddyfile/parse.go index 5afdc21..81181a4 100644 --- a/caddyconfig/caddyfile/parse.go +++ b/caddyconfig/caddyfile/parse.go @@ -429,7 +429,7 @@ func (p *parser) doImport(nesting int) error { nodes = matches } - nodeName := p.Token().originalFile() + nodeName := p.File() if p.Token().snippetName != "" { nodeName += fmt.Sprintf(":%s", p.Token().snippetName) } @@ -453,18 +453,18 @@ func (p *parser) doImport(nesting int) error { // golang for range slice return a copy of value // similarly, append also copy value for i, token := range importedTokens { - // set the token's file to refer to import directive line number and snippet name + // update the token's imports to refer to import directive filename, line number and snippet name if there is one if token.snippetName != "" { - token.updateFile(fmt.Sprintf("%s:%d (import %s)", token.File, p.Line(), token.snippetName)) + token.imports = append(token.imports, fmt.Sprintf("%s:%d (import %s)", p.File(), p.Line(), token.snippetName)) } else { - token.updateFile(fmt.Sprintf("%s:%d (import)", token.File, p.Line())) + token.imports = append(token.imports, fmt.Sprintf("%s:%d (import)", p.File(), p.Line())) } // naive way of determine snippets, as snippets definition can only follow name + block // format, won't check for nesting correctness or any other error, that's what parser does. if !maybeSnippet && nesting == 0 { // first of the line - if i == 0 || importedTokens[i-1].originalFile() != token.originalFile() || importedTokens[i-1].Line+importedTokens[i-1].NumLineBreaks() < token.Line { + if i == 0 || importedTokens[i-1].File != token.File || importedTokens[i-1].Line+importedTokens[i-1].NumLineBreaks() < token.Line { index = 0 } else { index++ -- cgit v1.2.3 From bbe1952a59a196b7598c6488beb770ec38a70dfc Mon Sep 17 00:00:00 2001 From: WeidiDeng Date: Thu, 13 Jul 2023 04:32:22 +0800 Subject: caddyfile: Fix comparing if two tokens are on the same line (#5626) * fix comparing if two tokens are on the same line * compare tokens from copies when importing --- caddyconfig/caddyfile/parse.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'caddyconfig/caddyfile/parse.go') diff --git a/caddyconfig/caddyfile/parse.go b/caddyconfig/caddyfile/parse.go index 81181a4..0a1a9e7 100644 --- a/caddyconfig/caddyfile/parse.go +++ b/caddyconfig/caddyfile/parse.go @@ -464,7 +464,7 @@ func (p *parser) doImport(nesting int) error { // format, won't check for nesting correctness or any other error, that's what parser does. if !maybeSnippet && nesting == 0 { // first of the line - if i == 0 || importedTokens[i-1].File != token.File || importedTokens[i-1].Line+importedTokens[i-1].NumLineBreaks() < token.Line { + if i == 0 || isNextOnNewLine(tokensCopy[i-1], token) { index = 0 } else { index++ -- cgit v1.2.3 From b32f265ecad60404c3818cc9d42e367a8e4eb7d4 Mon Sep 17 00:00:00 2001 From: Jacob Gadikian Date: Tue, 8 Aug 2023 03:40:31 +0800 Subject: ci: Use gofumpt to format code (#5707) --- caddyconfig/caddyfile/parse.go | 1 - 1 file changed, 1 deletion(-) (limited to 'caddyconfig/caddyfile/parse.go') diff --git a/caddyconfig/caddyfile/parse.go b/caddyconfig/caddyfile/parse.go index 0a1a9e7..620f091 100644 --- a/caddyconfig/caddyfile/parse.go +++ b/caddyconfig/caddyfile/parse.go @@ -565,7 +565,6 @@ func (p *parser) doSingleImport(importFile string) ([]Token, error) { // are loaded into the current server block for later use // by directive setup functions. func (p *parser) directive() error { - // a segment is a list of tokens associated with this directive var segment Segment -- cgit v1.2.3 From d6f86cccf5fa5b4eb30141da390cf2439746c5da Mon Sep 17 00:00:00 2001 From: Jacob Gadikian Date: Mon, 14 Aug 2023 23:41:15 +0800 Subject: ci: use gci linter (#5708) * use gofmput to format code * use gci to format imports * reconfigure gci * linter autofixes * rearrange imports a little * export GOOS=windows golangci-lint run ./... --fix --- caddyconfig/caddyfile/parse.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'caddyconfig/caddyfile/parse.go') diff --git a/caddyconfig/caddyfile/parse.go b/caddyconfig/caddyfile/parse.go index 620f091..65d6ee9 100644 --- a/caddyconfig/caddyfile/parse.go +++ b/caddyconfig/caddyfile/parse.go @@ -22,8 +22,9 @@ import ( "path/filepath" "strings" - "github.com/caddyserver/caddy/v2" "go.uber.org/zap" + + "github.com/caddyserver/caddy/v2" ) // Parse parses the input just enough to group tokens, in -- cgit v1.2.3