From c9980fd3671d873a7197a5ac4d6ac9d6b046abb6 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Wed, 21 Aug 2019 10:46:35 -0600 Subject: Refactor Caddyfile adapter and module registration Use piles from which to draw config values. Module values can return their name, so now we can do two-way mapping from value to name and name to value; whereas before we could only map name to value. This was problematic with the Caddyfile adapter since it receives values and needs to know the name to put in the config. --- caddyconfig/caddyfile/adapter.go | 8 +- caddyconfig/caddyfile/dispenser.go | 30 +++--- caddyconfig/caddyfile/parse.go | 112 ++++++++++++++-------- caddyconfig/caddyfile/parse_test.go | 181 ++++++++++++++---------------------- 4 files changed, 163 insertions(+), 168 deletions(-) (limited to 'caddyconfig/caddyfile') diff --git a/caddyconfig/caddyfile/adapter.go b/caddyconfig/caddyfile/adapter.go index ab4905a..377f77b 100644 --- a/caddyconfig/caddyfile/adapter.go +++ b/caddyconfig/caddyfile/adapter.go @@ -37,14 +37,12 @@ func (a Adapter) Adapt(body []byte, options map[string]string) ([]byte, []caddyc options = make(map[string]string) } - directives := a.ServerType.ValidDirectives() - filename := options["filename"] if filename == "" { filename = "Caddyfile" } - serverBlocks, err := Parse(filename, bytes.NewReader(body), directives) + serverBlocks, err := Parse(filename, bytes.NewReader(body)) if err != nil { return nil, nil, err } @@ -77,10 +75,6 @@ type Unmarshaler interface { // ServerType is a type that can evaluate a Caddyfile and set up a caddy config. type ServerType interface { - // ValidDirectives returns a list of the - // server type's recognized directives. - ValidDirectives() []string - // Setup takes the server blocks which // contain tokens, as well as options // (e.g. CLI flags) and creates a Caddy diff --git a/caddyconfig/caddyfile/dispenser.go b/caddyconfig/caddyfile/dispenser.go index 1cf5d04..0d2c789 100755 --- a/caddyconfig/caddyfile/dispenser.go +++ b/caddyconfig/caddyfile/dispenser.go @@ -31,6 +31,7 @@ type Dispenser struct { } // NewDispenser returns a Dispenser filled with the given tokens. +// TODO: Get rid of the filename argument; it seems pointless here func NewDispenser(filename string, tokens []Token) *Dispenser { return &Dispenser{ filename: filename, @@ -51,15 +52,15 @@ func (d *Dispenser) Next() bool { } // Prev moves to the previous token. It does the inverse -// of Next(). Generally, this should only be used in -// special cases such as deleting a token from the slice -// that d is iterating. In that case, without using Prev(), -// the dispenser would be pointing at the wrong token since -// deleting a token implicitly advances the cursor. +// of Next(), except this function may decrement the cursor +// to -1 so that the next call to Next() points to the +// first token; this allows dispensing to "start over". This +// method returns true if the cursor ends up pointing to a +// valid token. func (d *Dispenser) Prev() bool { - if d.cursor > 0 { + if d.cursor > -1 { d.cursor-- - return true + return d.cursor > -1 } return false } @@ -223,8 +224,7 @@ func (d *Dispenser) RemainingArgs() []string { // "directive" whether that be to the end of the line or // the end of a block that starts at the end of the line. func (d *Dispenser) NewFromNextTokens() *Dispenser { - var tkns []Token - tkns = append(tkns, d.Token()) + tkns := []Token{d.Token()} for d.NextArg() { tkns = append(tkns, d.Token()) } @@ -245,10 +245,14 @@ func (d *Dispenser) NewFromNextTokens() *Dispenser { // Token returns the current token. func (d *Dispenser) Token() Token { - if d.cursor < 0 || d.cursor >= len(d.tokens) { + return d.TokenAt(d.cursor) +} + +func (d *Dispenser) TokenAt(cursor int) Token { + if cursor < 0 || cursor >= len(d.tokens) { return Token{} } - return d.tokens[d.cursor] + return d.tokens[cursor] } // Cursor returns the current cursor (token index). @@ -256,6 +260,10 @@ func (d *Dispenser) Cursor() int { return d.cursor } +func (d *Dispenser) Reset() { + d.cursor = -1 +} + // ArgErr returns an argument error, meaning that another // argument was expected but not found. In other words, // a line break or open curly brace was encountered instead of diff --git a/caddyconfig/caddyfile/parse.go b/caddyconfig/caddyfile/parse.go index cc91e3d..e5b25fc 100755 --- a/caddyconfig/caddyfile/parse.go +++ b/caddyconfig/caddyfile/parse.go @@ -28,12 +28,12 @@ import ( // Directives that do not appear in validDirectives will cause // an error. If you do not want to check for valid directives, // pass in nil instead. -func Parse(filename string, input io.Reader, validDirectives []string) ([]ServerBlock, error) { +func Parse(filename string, input io.Reader) ([]ServerBlock, error) { tokens, err := allTokens(input) if err != nil { return nil, err } - p := parser{Dispenser: NewDispenser(filename, tokens), validDirectives: validDirectives} + p := parser{Dispenser: NewDispenser(filename, tokens)} return p.parseAll() } @@ -56,9 +56,9 @@ func allTokens(input io.Reader) ([]Token, error) { type parser struct { *Dispenser block ServerBlock // current server block being parsed - validDirectives []string // a directive must be valid or it's an error eof bool // if we encounter a valid EOF in a hard place definedSnippets map[string][]Token + nesting int } func (p *parser) parseAll() ([]ServerBlock, error) { @@ -72,14 +72,16 @@ func (p *parser) parseAll() ([]ServerBlock, error) { if len(p.block.Keys) > 0 { blocks = append(blocks, p.block) } + if p.nesting > 0 { + return blocks, p.EOFErr() + } } return blocks, nil } func (p *parser) parseOne() error { - p.block = ServerBlock{Tokens: make(map[string][]Token)} - + p.block = ServerBlock{} return p.begin() } @@ -186,7 +188,7 @@ func (p *parser) blockContents() error { return err } - // Only look for close curly brace if there was an opening + // only look for close curly brace if there was an opening if errOpenCurlyBrace == nil { err = p.closeCurlyBrace() if err != nil { @@ -205,6 +207,7 @@ func (p *parser) directives() error { for p.Next() { // end of server block if p.Val() == "}" { + // p.nesting has already been decremented break } @@ -218,11 +221,15 @@ func (p *parser) directives() error { continue } - // normal case: parse a directive on this line + // normal case: parse a directive as a new segment + // (a "segment" is a line which starts with a directive + // and which ends at the end of the line or at the end of + // the block that is opened at the end of the line) if err := p.directive(); err != nil { return err } } + return nil } @@ -345,25 +352,24 @@ func (p *parser) doSingleImport(importFile string) ([]Token, error) { // are loaded into the current server block for later use // by directive setup functions. func (p *parser) directive() error { - dir := replaceEnvVars(p.Val()) - nesting := 0 + // evaluate any env vars in directive token + p.tokens[p.cursor].Text = replaceEnvVars(p.tokens[p.cursor].Text) - if !p.validDirective(dir) { - return p.Errf("Unknown directive '%s'", dir) - } + // a segment is a list of tokens associated with this directive + var segment Segment - // The directive itself is appended as a relevant token - p.block.Tokens[dir] = append(p.block.Tokens[dir], p.tokens[p.cursor]) + // the directive itself is appended as a relevant token + segment = append(segment, p.Token()) for p.Next() { if p.Val() == "{" { - nesting++ - } else if p.isNewLine() && nesting == 0 { + p.nesting++ + } else if p.isNewLine() && p.nesting == 0 { p.cursor-- // read too far break - } else if p.Val() == "}" && nesting > 0 { - nesting-- - } else if p.Val() == "}" && nesting == 0 { + } else if p.Val() == "}" && p.nesting > 0 { + p.nesting-- + } else if p.Val() == "}" && p.nesting == 0 { return p.Err("Unexpected '}' because no matching opening brace") } else if p.Val() == "import" && p.isNewLine() { if err := p.doImport(); err != nil { @@ -373,12 +379,15 @@ func (p *parser) directive() error { continue } p.tokens[p.cursor].Text = replaceEnvVars(p.tokens[p.cursor].Text) - p.block.Tokens[dir] = append(p.block.Tokens[dir], p.tokens[p.cursor]) + segment = append(segment, p.Token()) } - if nesting > 0 { + p.block.Segments = append(p.block.Segments, segment) + + if p.nesting > 0 { return p.EOFErr() } + return nil } @@ -404,19 +413,6 @@ func (p *parser) closeCurlyBrace() error { return nil } -// validDirective returns true if dir is in p.validDirectives. -func (p *parser) validDirective(dir string) bool { - if p.validDirectives == nil { - return true - } - for _, d := range p.validDirectives { - if d == dir { - return true - } - } - return false -} - // replaceEnvVars replaces environment variables that appear in the token // and understands both the $UNIX and %WINDOWS% syntaxes. func replaceEnvVars(s string) string { @@ -447,13 +443,6 @@ func replaceEnvReferences(s, refStart, refEnd string) string { return s } -// ServerBlock associates any number of keys (usually addresses -// of some sort) with tokens (grouped by directive name). -type ServerBlock struct { - Keys []string - Tokens map[string][]Token -} - func (p *parser) isSnippet() (bool, string) { keys := p.block.Keys // A snippet block is a single key with parens. Nothing else qualifies. @@ -480,6 +469,7 @@ func (p *parser) snippetTokens() ([]Token, error) { } } if p.Val() == "{" { + p.nesting++ count++ } tokens = append(tokens, p.tokens[p.cursor]) @@ -490,3 +480,43 @@ func (p *parser) snippetTokens() ([]Token, error) { } return tokens, nil } + +// ServerBlock associates any number of keys from the +// head of the server block with tokens, which are +// grouped by segments. +type ServerBlock struct { + Keys []string + Segments []Segment +} + +// DispenseDirective returns a dispenser that contains +// all the tokens in the server block. +func (sb ServerBlock) DispenseDirective(dir string) *Dispenser { + var tokens []Token + for _, seg := range sb.Segments { + if len(seg) > 0 && seg[0].Text == dir { + tokens = append(tokens, seg...) + } + } + return NewDispenser("", tokens) +} + +// Segment is a list of tokens which begins with a directive +// and ends at the end of the directive (either at the end of +// the line, or at the end of a block it opens). +type Segment []Token + +// Directive returns the directive name for the segment. +// The directive name is the text of the first token. +func (s Segment) Directive() string { + if len(s) > 0 { + return s[0].Text + } + return "" +} + +// NewDispenser returns a dispenser for this +// segment's tokens. +func (s Segment) NewDispenser() *Dispenser { + return NewDispenser("", s) +} diff --git a/caddyconfig/caddyfile/parse_test.go b/caddyconfig/caddyfile/parse_test.go index 654c68d..19959de 100755 --- a/caddyconfig/caddyfile/parse_test.go +++ b/caddyconfig/caddyfile/parse_test.go @@ -22,6 +22,8 @@ import ( "testing" ) +// TODO: re-enable all tests + func TestAllTokens(t *testing.T) { input := strings.NewReader("a b c\nd e") expected := []string{"a", "b", "c", "d", "e"} @@ -53,84 +55,67 @@ func TestParseOneAndImport(t *testing.T) { input string shouldErr bool keys []string - tokens map[string]int // map of directive name to number of tokens expected + numTokens []int // number of tokens to expect in each segment }{ {`localhost`, false, []string{ "localhost", - }, map[string]int{}}, + }, []int{}}, {`localhost dir1`, false, []string{ "localhost", - }, map[string]int{ - "dir1": 1, - }}, + }, []int{1}}, {`localhost:1234 dir1 foo bar`, false, []string{ "localhost:1234", - }, map[string]int{ - "dir1": 3, - }}, + }, []int{3}, + }, {`localhost { dir1 }`, false, []string{ "localhost", - }, map[string]int{ - "dir1": 1, - }}, + }, []int{1}}, {`localhost:1234 { dir1 foo bar dir2 }`, false, []string{ "localhost:1234", - }, map[string]int{ - "dir1": 3, - "dir2": 1, - }}, + }, []int{3, 1}}, {`http://localhost https://localhost dir1 foo bar`, false, []string{ "http://localhost", "https://localhost", - }, map[string]int{ - "dir1": 3, - }}, + }, []int{3}}, {`http://localhost https://localhost { dir1 foo bar }`, false, []string{ "http://localhost", "https://localhost", - }, map[string]int{ - "dir1": 3, - }}, + }, []int{3}}, {`http://localhost, https://localhost { dir1 foo bar }`, false, []string{ "http://localhost", "https://localhost", - }, map[string]int{ - "dir1": 3, - }}, + }, []int{3}}, {`http://localhost, { }`, true, []string{ "http://localhost", - }, map[string]int{}}, + }, []int{}}, {`host1:80, http://host2.com dir1 foo bar dir2 baz`, false, []string{ "host1:80", "http://host2.com", - }, map[string]int{ - "dir1": 3, - "dir2": 2, - }}, + }, []int{3, 2}}, {`http://host1.com, http://host2.com, @@ -138,7 +123,7 @@ func TestParseOneAndImport(t *testing.T) { "http://host1.com", "http://host2.com", "https://host3.com", - }, map[string]int{}}, + }, []int{}}, {`http://host1.com:1234, https://host2.com dir1 foo { @@ -147,10 +132,7 @@ func TestParseOneAndImport(t *testing.T) { dir2`, false, []string{ "http://host1.com:1234", "https://host2.com", - }, map[string]int{ - "dir1": 6, - "dir2": 1, - }}, + }, []int{6, 1}}, {`127.0.0.1 dir1 { @@ -160,34 +142,25 @@ func TestParseOneAndImport(t *testing.T) { foo bar }`, false, []string{ "127.0.0.1", - }, map[string]int{ - "dir1": 5, - "dir2": 5, - }}, + }, []int{5, 5}}, {`localhost dir1 { foo`, true, []string{ "localhost", - }, map[string]int{ - "dir1": 3, - }}, + }, []int{3}}, {`localhost dir1 { }`, false, []string{ "localhost", - }, map[string]int{ - "dir1": 3, - }}, + }, []int{3}}, {`localhost dir1 { } }`, true, []string{ "localhost", - }, map[string]int{ - "dir1": 3, - }}, + }, []int{}}, {`localhost dir1 { @@ -197,48 +170,38 @@ func TestParseOneAndImport(t *testing.T) { } dir2 foo bar`, false, []string{ "localhost", - }, map[string]int{ - "dir1": 7, - "dir2": 3, - }}, + }, []int{7, 3}}, - {``, false, []string{}, map[string]int{}}, + {``, false, []string{}, []int{}}, {`localhost dir1 arg1 import testdata/import_test1.txt`, false, []string{ "localhost", - }, map[string]int{ - "dir1": 2, - "dir2": 3, - "dir3": 1, - }}, + }, []int{2, 3, 1}}, {`import testdata/import_test2.txt`, false, []string{ "host1", - }, map[string]int{ - "dir1": 1, - "dir2": 2, - }}, + }, []int{1, 2}}, - {`import testdata/import_test1.txt testdata/import_test2.txt`, true, []string{}, map[string]int{}}, + {`import testdata/import_test1.txt testdata/import_test2.txt`, true, []string{}, []int{}}, - {`import testdata/not_found.txt`, true, []string{}, map[string]int{}}, + {`import testdata/not_found.txt`, true, []string{}, []int{}}, - {`""`, false, []string{}, map[string]int{}}, + {`""`, false, []string{}, []int{}}, - {``, false, []string{}, map[string]int{}}, + {``, false, []string{}, []int{}}, // test cases found by fuzzing! - {`import }{$"`, true, []string{}, map[string]int{}}, - {`import /*/*.txt`, true, []string{}, map[string]int{}}, - {`import /???/?*?o`, true, []string{}, map[string]int{}}, - {`import /??`, true, []string{}, map[string]int{}}, - {`import /[a-z]`, true, []string{}, map[string]int{}}, - {`import {$}`, true, []string{}, map[string]int{}}, - {`import {%}`, true, []string{}, map[string]int{}}, - {`import {$$}`, true, []string{}, map[string]int{}}, - {`import {%%}`, true, []string{}, map[string]int{}}, + {`import }{$"`, true, []string{}, []int{}}, + {`import /*/*.txt`, true, []string{}, []int{}}, + {`import /???/?*?o`, true, []string{}, []int{}}, + {`import /??`, true, []string{}, []int{}}, + {`import /[a-z]`, true, []string{}, []int{}}, + {`import {$}`, true, []string{}, []int{}}, + {`import {%}`, true, []string{}, []int{}}, + {`import {$$}`, true, []string{}, []int{}}, + {`import {%%}`, true, []string{}, []int{}}, } { result, err := testParseOne(test.input) @@ -261,15 +224,16 @@ func TestParseOneAndImport(t *testing.T) { } } - if len(result.Tokens) != len(test.tokens) { - t.Errorf("Test %d: Expected %d directives, had %d", - i, len(test.tokens), len(result.Tokens)) + if len(result.Segments) != len(test.numTokens) { + t.Errorf("Test %d: Expected %d segments, had %d", + i, len(test.numTokens), len(result.Segments)) continue } - for directive, tokens := range result.Tokens { - if len(tokens) != test.tokens[directive] { - t.Errorf("Test %d, directive '%s': Expected %d tokens, counted %d", - i, directive, test.tokens[directive], len(tokens)) + + for j, seg := range result.Segments { + if len(seg) != test.numTokens[j] { + t.Errorf("Test %d, segment %d: Expected %d tokens, counted %d", + i, j, test.numTokens[j], len(seg)) continue } } @@ -289,12 +253,12 @@ func TestRecursiveImport(t *testing.T) { t.Errorf("got keys unexpected: expect localhost, got %v", got.Keys) return false } - if len(got.Tokens) != 2 { - t.Errorf("got wrong number of tokens: expect 2, got %d", len(got.Tokens)) + if len(got.Segments) != 2 { + t.Errorf("got wrong number of segments: expect 2, got %d", len(got.Segments)) return false } - if len(got.Tokens["dir1"]) != 1 || len(got.Tokens["dir2"]) != 2 { - t.Errorf("got unexpect tokens: %v", got.Tokens) + if len(got.Segments[0]) != 1 || len(got.Segments[1]) != 2 { + t.Errorf("got unexpect tokens: %v", got.Segments) return false } return true @@ -384,12 +348,12 @@ func TestDirectiveImport(t *testing.T) { t.Errorf("got keys unexpected: expect localhost, got %v", got.Keys) return false } - if len(got.Tokens) != 2 { - t.Errorf("got wrong number of tokens: expect 2, got %d", len(got.Tokens)) + if len(got.Segments) != 2 { + t.Errorf("got wrong number of segments: expect 2, got %d", len(got.Segments)) return false } - if len(got.Tokens["dir1"]) != 1 || len(got.Tokens["proxy"]) != 8 { - t.Errorf("got unexpect tokens: %v", got.Tokens) + if len(got.Segments[0]) != 1 || len(got.Segments[1]) != 8 { + t.Errorf("got unexpect tokens: %v", got.Segments) return false } return true @@ -557,21 +521,21 @@ func TestEnvironmentReplacement(t *testing.T) { if actual, expected := blocks[0].Keys[0], ":8080"; expected != actual { t.Errorf("Expected key to be '%s' but was '%s'", expected, actual) } - if actual, expected := blocks[0].Tokens["dir1"][1].Text, "foobar"; expected != actual { + if actual, expected := blocks[0].Segments[0][1].Text, "foobar"; expected != actual { t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual) } // combined windows env vars in argument p = testParser(":{%PORT%}\ndir1 {%ADDRESS%}/{%FOOBAR%}") blocks, _ = p.parseAll() - if actual, expected := blocks[0].Tokens["dir1"][1].Text, "servername.com/foobar"; expected != actual { + if actual, expected := blocks[0].Segments[0][1].Text, "servername.com/foobar"; expected != actual { t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual) } // malformed env var (windows) p = testParser(":1234\ndir1 {%ADDRESS}") blocks, _ = p.parseAll() - if actual, expected := blocks[0].Tokens["dir1"][1].Text, "{%ADDRESS}"; expected != actual { + if actual, expected := blocks[0].Segments[0][1].Text, "{%ADDRESS}"; expected != actual { t.Errorf("Expected host to be '%s' but was '%s'", expected, actual) } @@ -585,22 +549,18 @@ func TestEnvironmentReplacement(t *testing.T) { // in quoted field p = testParser(":1234\ndir1 \"Test {$FOOBAR} test\"") blocks, _ = p.parseAll() - if actual, expected := blocks[0].Tokens["dir1"][1].Text, "Test foobar test"; expected != actual { + if actual, expected := blocks[0].Segments[0][1].Text, "Test foobar test"; expected != actual { t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual) } // after end token p = testParser(":1234\nanswer \"{{ .Name }} {$FOOBAR}\"") blocks, _ = p.parseAll() - if actual, expected := blocks[0].Tokens["answer"][1].Text, "{{ .Name }} foobar"; expected != actual { + if actual, expected := blocks[0].Segments[0][1].Text, "{{ .Name }} foobar"; expected != actual { t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual) } } -func testParser(input string) parser { - return parser{Dispenser: newTestDispenser(input)} -} - func TestSnippets(t *testing.T) { p := testParser(` (common) { @@ -617,7 +577,7 @@ func TestSnippets(t *testing.T) { } for _, b := range blocks { t.Log(b.Keys) - t.Log(b.Tokens) + t.Log(b.Segments) } if len(blocks) != 1 { t.Fatalf("Expect exactly one server block. Got %d.", len(blocks)) @@ -625,16 +585,15 @@ func TestSnippets(t *testing.T) { if actual, expected := blocks[0].Keys[0], "http://example.com"; expected != actual { t.Errorf("Expected server name to be '%s' but was '%s'", expected, actual) } - if len(blocks[0].Tokens) != 2 { - t.Fatalf("Server block should have tokens from import") + if len(blocks[0].Segments) != 2 { + t.Fatalf("Server block should have tokens from import, got: %+v", blocks[0]) } - if actual, expected := blocks[0].Tokens["gzip"][0].Text, "gzip"; expected != actual { + if actual, expected := blocks[0].Segments[0][0].Text, "gzip"; expected != actual { t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual) } - if actual, expected := blocks[0].Tokens["errors"][1].Text, "stderr"; expected != actual { + if actual, expected := blocks[0].Segments[1][1].Text, "stderr"; expected != actual { t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual) } - } func writeStringToTempFileOrDie(t *testing.T, str string) (pathToFile string) { @@ -666,9 +625,9 @@ func TestImportedFilesIgnoreNonDirectiveImportTokens(t *testing.T) { } for _, b := range blocks { t.Log(b.Keys) - t.Log(b.Tokens) + t.Log(b.Segments) } - auth := blocks[0].Tokens["basicauth"] + auth := blocks[0].Segments[0] line := auth[0].Text + " " + auth[1].Text + " " + auth[2].Text + " " + auth[3].Text if line != "basicauth / import password" { // Previously, it would be changed to: @@ -701,7 +660,7 @@ func TestSnippetAcrossMultipleFiles(t *testing.T) { } for _, b := range blocks { t.Log(b.Keys) - t.Log(b.Tokens) + t.Log(b.Segments) } if len(blocks) != 1 { t.Fatalf("Expect exactly one server block. Got %d.", len(blocks)) @@ -709,10 +668,14 @@ func TestSnippetAcrossMultipleFiles(t *testing.T) { if actual, expected := blocks[0].Keys[0], "http://example.com"; expected != actual { t.Errorf("Expected server name to be '%s' but was '%s'", expected, actual) } - if len(blocks[0].Tokens) != 1 { + if len(blocks[0].Segments) != 1 { t.Fatalf("Server block should have tokens from import") } - if actual, expected := blocks[0].Tokens["gzip"][0].Text, "gzip"; expected != actual { + if actual, expected := blocks[0].Segments[0][0].Text, "gzip"; expected != actual { t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual) } } + +func testParser(input string) parser { + return parser{Dispenser: newTestDispenser(input)} +} -- cgit v1.2.3