diff options
Diffstat (limited to 'caddyconfig/caddyfile')
| -rw-r--r-- | caddyconfig/caddyfile/adapter.go | 8 | ||||
| -rwxr-xr-x | caddyconfig/caddyfile/dispenser.go | 30 | ||||
| -rwxr-xr-x | caddyconfig/caddyfile/parse.go | 112 | ||||
| -rwxr-xr-x | caddyconfig/caddyfile/parse_test.go | 181 | 
4 files changed, 163 insertions, 168 deletions
diff --git a/caddyconfig/caddyfile/adapter.go b/caddyconfig/caddyfile/adapter.go index ab4905a..377f77b 100644 --- a/caddyconfig/caddyfile/adapter.go +++ b/caddyconfig/caddyfile/adapter.go @@ -37,14 +37,12 @@ func (a Adapter) Adapt(body []byte, options map[string]string) ([]byte, []caddyc  		options = make(map[string]string)  	} -	directives := a.ServerType.ValidDirectives() -  	filename := options["filename"]  	if filename == "" {  		filename = "Caddyfile"  	} -	serverBlocks, err := Parse(filename, bytes.NewReader(body), directives) +	serverBlocks, err := Parse(filename, bytes.NewReader(body))  	if err != nil {  		return nil, nil, err  	} @@ -77,10 +75,6 @@ type Unmarshaler interface {  // ServerType is a type that can evaluate a Caddyfile and set up a caddy config.  type ServerType interface { -	// ValidDirectives returns a list of the -	// server type's recognized directives. -	ValidDirectives() []string -  	// Setup takes the server blocks which  	// contain tokens, as well as options  	// (e.g. CLI flags) and creates a Caddy diff --git a/caddyconfig/caddyfile/dispenser.go b/caddyconfig/caddyfile/dispenser.go index 1cf5d04..0d2c789 100755 --- a/caddyconfig/caddyfile/dispenser.go +++ b/caddyconfig/caddyfile/dispenser.go @@ -31,6 +31,7 @@ type Dispenser struct {  }  // NewDispenser returns a Dispenser filled with the given tokens. +// TODO: Get rid of the filename argument; it seems pointless here  func NewDispenser(filename string, tokens []Token) *Dispenser {  	return &Dispenser{  		filename: filename, @@ -51,15 +52,15 @@ func (d *Dispenser) Next() bool {  }  // Prev moves to the previous token. It does the inverse -// of Next(). Generally, this should only be used in -// special cases such as deleting a token from the slice -// that d is iterating. In that case, without using Prev(), -// the dispenser would be pointing at the wrong token since -// deleting a token implicitly advances the cursor. +// of Next(), except this function may decrement the cursor +// to -1 so that the next call to Next() points to the +// first token; this allows dispensing to "start over". This +// method returns true if the cursor ends up pointing to a +// valid token.  func (d *Dispenser) Prev() bool { -	if d.cursor > 0 { +	if d.cursor > -1 {  		d.cursor-- -		return true +		return d.cursor > -1  	}  	return false  } @@ -223,8 +224,7 @@ func (d *Dispenser) RemainingArgs() []string {  // "directive" whether that be to the end of the line or  // the end of a block that starts at the end of the line.  func (d *Dispenser) NewFromNextTokens() *Dispenser { -	var tkns []Token -	tkns = append(tkns, d.Token()) +	tkns := []Token{d.Token()}  	for d.NextArg() {  		tkns = append(tkns, d.Token())  	} @@ -245,10 +245,14 @@ func (d *Dispenser) NewFromNextTokens() *Dispenser {  // Token returns the current token.  func (d *Dispenser) Token() Token { -	if d.cursor < 0 || d.cursor >= len(d.tokens) { +	return d.TokenAt(d.cursor) +} + +func (d *Dispenser) TokenAt(cursor int) Token { +	if cursor < 0 || cursor >= len(d.tokens) {  		return Token{}  	} -	return d.tokens[d.cursor] +	return d.tokens[cursor]  }  // Cursor returns the current cursor (token index). @@ -256,6 +260,10 @@ func (d *Dispenser) Cursor() int {  	return d.cursor  } +func (d *Dispenser) Reset() { +	d.cursor = -1 +} +  // ArgErr returns an argument error, meaning that another  // argument was expected but not found. In other words,  // a line break or open curly brace was encountered instead of diff --git a/caddyconfig/caddyfile/parse.go b/caddyconfig/caddyfile/parse.go index cc91e3d..e5b25fc 100755 --- a/caddyconfig/caddyfile/parse.go +++ b/caddyconfig/caddyfile/parse.go @@ -28,12 +28,12 @@ import (  // Directives that do not appear in validDirectives will cause  // an error. If you do not want to check for valid directives,  // pass in nil instead. -func Parse(filename string, input io.Reader, validDirectives []string) ([]ServerBlock, error) { +func Parse(filename string, input io.Reader) ([]ServerBlock, error) {  	tokens, err := allTokens(input)  	if err != nil {  		return nil, err  	} -	p := parser{Dispenser: NewDispenser(filename, tokens), validDirectives: validDirectives} +	p := parser{Dispenser: NewDispenser(filename, tokens)}  	return p.parseAll()  } @@ -56,9 +56,9 @@ func allTokens(input io.Reader) ([]Token, error) {  type parser struct {  	*Dispenser  	block           ServerBlock // current server block being parsed -	validDirectives []string    // a directive must be valid or it's an error  	eof             bool        // if we encounter a valid EOF in a hard place  	definedSnippets map[string][]Token +	nesting         int  }  func (p *parser) parseAll() ([]ServerBlock, error) { @@ -72,14 +72,16 @@ func (p *parser) parseAll() ([]ServerBlock, error) {  		if len(p.block.Keys) > 0 {  			blocks = append(blocks, p.block)  		} +		if p.nesting > 0 { +			return blocks, p.EOFErr() +		}  	}  	return blocks, nil  }  func (p *parser) parseOne() error { -	p.block = ServerBlock{Tokens: make(map[string][]Token)} - +	p.block = ServerBlock{}  	return p.begin()  } @@ -186,7 +188,7 @@ func (p *parser) blockContents() error {  		return err  	} -	// Only look for close curly brace if there was an opening +	// only look for close curly brace if there was an opening  	if errOpenCurlyBrace == nil {  		err = p.closeCurlyBrace()  		if err != nil { @@ -205,6 +207,7 @@ func (p *parser) directives() error {  	for p.Next() {  		// end of server block  		if p.Val() == "}" { +			// p.nesting has already been decremented  			break  		} @@ -218,11 +221,15 @@ func (p *parser) directives() error {  			continue  		} -		// normal case: parse a directive on this line +		// normal case: parse a directive as a new segment +		// (a "segment" is a line which starts with a directive +		// and which ends at the end of the line or at the end of +		// the block that is opened at the end of the line)  		if err := p.directive(); err != nil {  			return err  		}  	} +  	return nil  } @@ -345,25 +352,24 @@ func (p *parser) doSingleImport(importFile string) ([]Token, error) {  // are loaded into the current server block for later use  // by directive setup functions.  func (p *parser) directive() error { -	dir := replaceEnvVars(p.Val()) -	nesting := 0 +	// evaluate any env vars in directive token +	p.tokens[p.cursor].Text = replaceEnvVars(p.tokens[p.cursor].Text) -	if !p.validDirective(dir) { -		return p.Errf("Unknown directive '%s'", dir) -	} +	// a segment is a list of tokens associated with this directive +	var segment Segment -	// The directive itself is appended as a relevant token -	p.block.Tokens[dir] = append(p.block.Tokens[dir], p.tokens[p.cursor]) +	// the directive itself is appended as a relevant token +	segment = append(segment, p.Token())  	for p.Next() {  		if p.Val() == "{" { -			nesting++ -		} else if p.isNewLine() && nesting == 0 { +			p.nesting++ +		} else if p.isNewLine() && p.nesting == 0 {  			p.cursor-- // read too far  			break -		} else if p.Val() == "}" && nesting > 0 { -			nesting-- -		} else if p.Val() == "}" && nesting == 0 { +		} else if p.Val() == "}" && p.nesting > 0 { +			p.nesting-- +		} else if p.Val() == "}" && p.nesting == 0 {  			return p.Err("Unexpected '}' because no matching opening brace")  		} else if p.Val() == "import" && p.isNewLine() {  			if err := p.doImport(); err != nil { @@ -373,12 +379,15 @@ func (p *parser) directive() error {  			continue  		}  		p.tokens[p.cursor].Text = replaceEnvVars(p.tokens[p.cursor].Text) -		p.block.Tokens[dir] = append(p.block.Tokens[dir], p.tokens[p.cursor]) +		segment = append(segment, p.Token())  	} -	if nesting > 0 { +	p.block.Segments = append(p.block.Segments, segment) + +	if p.nesting > 0 {  		return p.EOFErr()  	} +  	return nil  } @@ -404,19 +413,6 @@ func (p *parser) closeCurlyBrace() error {  	return nil  } -// validDirective returns true if dir is in p.validDirectives. -func (p *parser) validDirective(dir string) bool { -	if p.validDirectives == nil { -		return true -	} -	for _, d := range p.validDirectives { -		if d == dir { -			return true -		} -	} -	return false -} -  // replaceEnvVars replaces environment variables that appear in the token  // and understands both the $UNIX and %WINDOWS% syntaxes.  func replaceEnvVars(s string) string { @@ -447,13 +443,6 @@ func replaceEnvReferences(s, refStart, refEnd string) string {  	return s  } -// ServerBlock associates any number of keys (usually addresses -// of some sort) with tokens (grouped by directive name). -type ServerBlock struct { -	Keys   []string -	Tokens map[string][]Token -} -  func (p *parser) isSnippet() (bool, string) {  	keys := p.block.Keys  	// A snippet block is a single key with parens. Nothing else qualifies. @@ -480,6 +469,7 @@ func (p *parser) snippetTokens() ([]Token, error) {  			}  		}  		if p.Val() == "{" { +			p.nesting++  			count++  		}  		tokens = append(tokens, p.tokens[p.cursor]) @@ -490,3 +480,43 @@ func (p *parser) snippetTokens() ([]Token, error) {  	}  	return tokens, nil  } + +// ServerBlock associates any number of keys from the +// head of the server block with tokens, which are +// grouped by segments. +type ServerBlock struct { +	Keys     []string +	Segments []Segment +} + +// DispenseDirective returns a dispenser that contains +// all the tokens in the server block. +func (sb ServerBlock) DispenseDirective(dir string) *Dispenser { +	var tokens []Token +	for _, seg := range sb.Segments { +		if len(seg) > 0 && seg[0].Text == dir { +			tokens = append(tokens, seg...) +		} +	} +	return NewDispenser("", tokens) +} + +// Segment is a list of tokens which begins with a directive +// and ends at the end of the directive (either at the end of +// the line, or at the end of a block it opens). +type Segment []Token + +// Directive returns the directive name for the segment. +// The directive name is the text of the first token. +func (s Segment) Directive() string { +	if len(s) > 0 { +		return s[0].Text +	} +	return "" +} + +// NewDispenser returns a dispenser for this +// segment's tokens. +func (s Segment) NewDispenser() *Dispenser { +	return NewDispenser("", s) +} diff --git a/caddyconfig/caddyfile/parse_test.go b/caddyconfig/caddyfile/parse_test.go index 654c68d..19959de 100755 --- a/caddyconfig/caddyfile/parse_test.go +++ b/caddyconfig/caddyfile/parse_test.go @@ -22,6 +22,8 @@ import (  	"testing"  ) +// TODO: re-enable all tests +  func TestAllTokens(t *testing.T) {  	input := strings.NewReader("a b c\nd e")  	expected := []string{"a", "b", "c", "d", "e"} @@ -53,84 +55,67 @@ func TestParseOneAndImport(t *testing.T) {  		input     string  		shouldErr bool  		keys      []string -		tokens    map[string]int // map of directive name to number of tokens expected +		numTokens []int // number of tokens to expect in each segment  	}{  		{`localhost`, false, []string{  			"localhost", -		}, map[string]int{}}, +		}, []int{}},  		{`localhost  		  dir1`, false, []string{  			"localhost", -		}, map[string]int{ -			"dir1": 1, -		}}, +		}, []int{1}},  		{`localhost:1234  		  dir1 foo bar`, false, []string{  			"localhost:1234", -		}, map[string]int{ -			"dir1": 3, -		}}, +		}, []int{3}, +		},  		{`localhost {  		    dir1  		  }`, false, []string{  			"localhost", -		}, map[string]int{ -			"dir1": 1, -		}}, +		}, []int{1}},  		{`localhost:1234 {  		    dir1 foo bar  		    dir2  		  }`, false, []string{  			"localhost:1234", -		}, map[string]int{ -			"dir1": 3, -			"dir2": 1, -		}}, +		}, []int{3, 1}},  		{`http://localhost https://localhost  		  dir1 foo bar`, false, []string{  			"http://localhost",  			"https://localhost", -		}, map[string]int{ -			"dir1": 3, -		}}, +		}, []int{3}},  		{`http://localhost https://localhost {  		    dir1 foo bar  		  }`, false, []string{  			"http://localhost",  			"https://localhost", -		}, map[string]int{ -			"dir1": 3, -		}}, +		}, []int{3}},  		{`http://localhost, https://localhost {  		    dir1 foo bar  		  }`, false, []string{  			"http://localhost",  			"https://localhost", -		}, map[string]int{ -			"dir1": 3, -		}}, +		}, []int{3}},  		{`http://localhost, {  		  }`, true, []string{  			"http://localhost", -		}, map[string]int{}}, +		}, []int{}},  		{`host1:80, http://host2.com  		  dir1 foo bar  		  dir2 baz`, false, []string{  			"host1:80",  			"http://host2.com", -		}, map[string]int{ -			"dir1": 3, -			"dir2": 2, -		}}, +		}, []int{3, 2}},  		{`http://host1.com,  		  http://host2.com, @@ -138,7 +123,7 @@ func TestParseOneAndImport(t *testing.T) {  			"http://host1.com",  			"http://host2.com",  			"https://host3.com", -		}, map[string]int{}}, +		}, []int{}},  		{`http://host1.com:1234, https://host2.com  		  dir1 foo { @@ -147,10 +132,7 @@ func TestParseOneAndImport(t *testing.T) {  		  dir2`, false, []string{  			"http://host1.com:1234",  			"https://host2.com", -		}, map[string]int{ -			"dir1": 6, -			"dir2": 1, -		}}, +		}, []int{6, 1}},  		{`127.0.0.1  		  dir1 { @@ -160,34 +142,25 @@ func TestParseOneAndImport(t *testing.T) {  		    foo bar  		  }`, false, []string{  			"127.0.0.1", -		}, map[string]int{ -			"dir1": 5, -			"dir2": 5, -		}}, +		}, []int{5, 5}},  		{`localhost  		  dir1 {  		    foo`, true, []string{  			"localhost", -		}, map[string]int{ -			"dir1": 3, -		}}, +		}, []int{3}},  		{`localhost  		  dir1 {  		  }`, false, []string{  			"localhost", -		}, map[string]int{ -			"dir1": 3, -		}}, +		}, []int{3}},  		{`localhost  		  dir1 {  		  } }`, true, []string{  			"localhost", -		}, map[string]int{ -			"dir1": 3, -		}}, +		}, []int{}},  		{`localhost  		  dir1 { @@ -197,48 +170,38 @@ func TestParseOneAndImport(t *testing.T) {  		  }  		  dir2 foo bar`, false, []string{  			"localhost", -		}, map[string]int{ -			"dir1": 7, -			"dir2": 3, -		}}, +		}, []int{7, 3}}, -		{``, false, []string{}, map[string]int{}}, +		{``, false, []string{}, []int{}},  		{`localhost  		  dir1 arg1  		  import testdata/import_test1.txt`, false, []string{  			"localhost", -		}, map[string]int{ -			"dir1": 2, -			"dir2": 3, -			"dir3": 1, -		}}, +		}, []int{2, 3, 1}},  		{`import testdata/import_test2.txt`, false, []string{  			"host1", -		}, map[string]int{ -			"dir1": 1, -			"dir2": 2, -		}}, +		}, []int{1, 2}}, -		{`import testdata/import_test1.txt testdata/import_test2.txt`, true, []string{}, map[string]int{}}, +		{`import testdata/import_test1.txt testdata/import_test2.txt`, true, []string{}, []int{}}, -		{`import testdata/not_found.txt`, true, []string{}, map[string]int{}}, +		{`import testdata/not_found.txt`, true, []string{}, []int{}}, -		{`""`, false, []string{}, map[string]int{}}, +		{`""`, false, []string{}, []int{}}, -		{``, false, []string{}, map[string]int{}}, +		{``, false, []string{}, []int{}},  		// test cases found by fuzzing! -		{`import }{$"`, true, []string{}, map[string]int{}}, -		{`import /*/*.txt`, true, []string{}, map[string]int{}}, -		{`import /???/?*?o`, true, []string{}, map[string]int{}}, -		{`import /??`, true, []string{}, map[string]int{}}, -		{`import /[a-z]`, true, []string{}, map[string]int{}}, -		{`import {$}`, true, []string{}, map[string]int{}}, -		{`import {%}`, true, []string{}, map[string]int{}}, -		{`import {$$}`, true, []string{}, map[string]int{}}, -		{`import {%%}`, true, []string{}, map[string]int{}}, +		{`import }{$"`, true, []string{}, []int{}}, +		{`import /*/*.txt`, true, []string{}, []int{}}, +		{`import /???/?*?o`, true, []string{}, []int{}}, +		{`import /??`, true, []string{}, []int{}}, +		{`import /[a-z]`, true, []string{}, []int{}}, +		{`import {$}`, true, []string{}, []int{}}, +		{`import {%}`, true, []string{}, []int{}}, +		{`import {$$}`, true, []string{}, []int{}}, +		{`import {%%}`, true, []string{}, []int{}},  	} {  		result, err := testParseOne(test.input) @@ -261,15 +224,16 @@ func TestParseOneAndImport(t *testing.T) {  			}  		} -		if len(result.Tokens) != len(test.tokens) { -			t.Errorf("Test %d: Expected %d directives, had %d", -				i, len(test.tokens), len(result.Tokens)) +		if len(result.Segments) != len(test.numTokens) { +			t.Errorf("Test %d: Expected %d segments, had %d", +				i, len(test.numTokens), len(result.Segments))  			continue  		} -		for directive, tokens := range result.Tokens { -			if len(tokens) != test.tokens[directive] { -				t.Errorf("Test %d, directive '%s': Expected %d tokens, counted %d", -					i, directive, test.tokens[directive], len(tokens)) + +		for j, seg := range result.Segments { +			if len(seg) != test.numTokens[j] { +				t.Errorf("Test %d, segment %d: Expected %d tokens, counted %d", +					i, j, test.numTokens[j], len(seg))  				continue  			}  		} @@ -289,12 +253,12 @@ func TestRecursiveImport(t *testing.T) {  			t.Errorf("got keys unexpected: expect localhost, got %v", got.Keys)  			return false  		} -		if len(got.Tokens) != 2 { -			t.Errorf("got wrong number of tokens: expect 2, got %d", len(got.Tokens)) +		if len(got.Segments) != 2 { +			t.Errorf("got wrong number of segments: expect 2, got %d", len(got.Segments))  			return false  		} -		if len(got.Tokens["dir1"]) != 1 || len(got.Tokens["dir2"]) != 2 { -			t.Errorf("got unexpect tokens: %v", got.Tokens) +		if len(got.Segments[0]) != 1 || len(got.Segments[1]) != 2 { +			t.Errorf("got unexpect tokens: %v", got.Segments)  			return false  		}  		return true @@ -384,12 +348,12 @@ func TestDirectiveImport(t *testing.T) {  			t.Errorf("got keys unexpected: expect localhost, got %v", got.Keys)  			return false  		} -		if len(got.Tokens) != 2 { -			t.Errorf("got wrong number of tokens: expect 2, got %d", len(got.Tokens)) +		if len(got.Segments) != 2 { +			t.Errorf("got wrong number of segments: expect 2, got %d", len(got.Segments))  			return false  		} -		if len(got.Tokens["dir1"]) != 1 || len(got.Tokens["proxy"]) != 8 { -			t.Errorf("got unexpect tokens: %v", got.Tokens) +		if len(got.Segments[0]) != 1 || len(got.Segments[1]) != 8 { +			t.Errorf("got unexpect tokens: %v", got.Segments)  			return false  		}  		return true @@ -557,21 +521,21 @@ func TestEnvironmentReplacement(t *testing.T) {  	if actual, expected := blocks[0].Keys[0], ":8080"; expected != actual {  		t.Errorf("Expected key to be '%s' but was '%s'", expected, actual)  	} -	if actual, expected := blocks[0].Tokens["dir1"][1].Text, "foobar"; expected != actual { +	if actual, expected := blocks[0].Segments[0][1].Text, "foobar"; expected != actual {  		t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual)  	}  	// combined windows env vars in argument  	p = testParser(":{%PORT%}\ndir1 {%ADDRESS%}/{%FOOBAR%}")  	blocks, _ = p.parseAll() -	if actual, expected := blocks[0].Tokens["dir1"][1].Text, "servername.com/foobar"; expected != actual { +	if actual, expected := blocks[0].Segments[0][1].Text, "servername.com/foobar"; expected != actual {  		t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual)  	}  	// malformed env var (windows)  	p = testParser(":1234\ndir1 {%ADDRESS}")  	blocks, _ = p.parseAll() -	if actual, expected := blocks[0].Tokens["dir1"][1].Text, "{%ADDRESS}"; expected != actual { +	if actual, expected := blocks[0].Segments[0][1].Text, "{%ADDRESS}"; expected != actual {  		t.Errorf("Expected host to be '%s' but was '%s'", expected, actual)  	} @@ -585,22 +549,18 @@ func TestEnvironmentReplacement(t *testing.T) {  	// in quoted field  	p = testParser(":1234\ndir1 \"Test {$FOOBAR} test\"")  	blocks, _ = p.parseAll() -	if actual, expected := blocks[0].Tokens["dir1"][1].Text, "Test foobar test"; expected != actual { +	if actual, expected := blocks[0].Segments[0][1].Text, "Test foobar test"; expected != actual {  		t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual)  	}  	// after end token  	p = testParser(":1234\nanswer \"{{ .Name }} {$FOOBAR}\"")  	blocks, _ = p.parseAll() -	if actual, expected := blocks[0].Tokens["answer"][1].Text, "{{ .Name }} foobar"; expected != actual { +	if actual, expected := blocks[0].Segments[0][1].Text, "{{ .Name }} foobar"; expected != actual {  		t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual)  	}  } -func testParser(input string) parser { -	return parser{Dispenser: newTestDispenser(input)} -} -  func TestSnippets(t *testing.T) {  	p := testParser(`  		(common) { @@ -617,7 +577,7 @@ func TestSnippets(t *testing.T) {  	}  	for _, b := range blocks {  		t.Log(b.Keys) -		t.Log(b.Tokens) +		t.Log(b.Segments)  	}  	if len(blocks) != 1 {  		t.Fatalf("Expect exactly one server block. Got %d.", len(blocks)) @@ -625,16 +585,15 @@ func TestSnippets(t *testing.T) {  	if actual, expected := blocks[0].Keys[0], "http://example.com"; expected != actual {  		t.Errorf("Expected server name to be '%s' but was '%s'", expected, actual)  	} -	if len(blocks[0].Tokens) != 2 { -		t.Fatalf("Server block should have tokens from import") +	if len(blocks[0].Segments) != 2 { +		t.Fatalf("Server block should have tokens from import, got: %+v", blocks[0])  	} -	if actual, expected := blocks[0].Tokens["gzip"][0].Text, "gzip"; expected != actual { +	if actual, expected := blocks[0].Segments[0][0].Text, "gzip"; expected != actual {  		t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual)  	} -	if actual, expected := blocks[0].Tokens["errors"][1].Text, "stderr"; expected != actual { +	if actual, expected := blocks[0].Segments[1][1].Text, "stderr"; expected != actual {  		t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual)  	} -  }  func writeStringToTempFileOrDie(t *testing.T, str string) (pathToFile string) { @@ -666,9 +625,9 @@ func TestImportedFilesIgnoreNonDirectiveImportTokens(t *testing.T) {  	}  	for _, b := range blocks {  		t.Log(b.Keys) -		t.Log(b.Tokens) +		t.Log(b.Segments)  	} -	auth := blocks[0].Tokens["basicauth"] +	auth := blocks[0].Segments[0]  	line := auth[0].Text + " " + auth[1].Text + " " + auth[2].Text + " " + auth[3].Text  	if line != "basicauth / import password" {  		// Previously, it would be changed to: @@ -701,7 +660,7 @@ func TestSnippetAcrossMultipleFiles(t *testing.T) {  	}  	for _, b := range blocks {  		t.Log(b.Keys) -		t.Log(b.Tokens) +		t.Log(b.Segments)  	}  	if len(blocks) != 1 {  		t.Fatalf("Expect exactly one server block. Got %d.", len(blocks)) @@ -709,10 +668,14 @@ func TestSnippetAcrossMultipleFiles(t *testing.T) {  	if actual, expected := blocks[0].Keys[0], "http://example.com"; expected != actual {  		t.Errorf("Expected server name to be '%s' but was '%s'", expected, actual)  	} -	if len(blocks[0].Tokens) != 1 { +	if len(blocks[0].Segments) != 1 {  		t.Fatalf("Server block should have tokens from import")  	} -	if actual, expected := blocks[0].Tokens["gzip"][0].Text, "gzip"; expected != actual { +	if actual, expected := blocks[0].Segments[0][0].Text, "gzip"; expected != actual {  		t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual)  	}  } + +func testParser(input string) parser { +	return parser{Dispenser: newTestDispenser(input)} +}  | 
