Refactor Caddyfile adapter and module registration

Use piles from which to draw config values.

Module values can return their name, so now we can do two-way mapping
from value to name and name to value; whereas before we could only map
name to value. This was problematic with the Caddyfile adapter since
it receives values and needs to know the name to put in the config.

4 files changed, 163 insertions, 168 deletions

diff --git a/caddyconfig/caddyfile/adapter.go b/caddyconfig/caddyfile/adapter.go
index ab4905a..377f77b 100644
--- a/caddyconfig/caddyfile/adapter.go
+++ b/caddyconfig/caddyfile/adapter.go
@@ -37,14 +37,12 @@ func (a Adapter) Adapt(body []byte, options map[string]string) ([]byte, []caddyc
 		options = make(map[string]string)
 	}
 
-	directives := a.ServerType.ValidDirectives()
-
 	filename := options["filename"]
 	if filename == "" {
 		filename = "Caddyfile"
 	}
 
-	serverBlocks, err := Parse(filename, bytes.NewReader(body), directives)
+	serverBlocks, err := Parse(filename, bytes.NewReader(body))
 	if err != nil {
 		return nil, nil, err
 	}
@@ -77,10 +75,6 @@ type Unmarshaler interface {
 
 // ServerType is a type that can evaluate a Caddyfile and set up a caddy config.
 type ServerType interface {
-	// ValidDirectives returns a list of the
-	// server type's recognized directives.
-	ValidDirectives() []string
-
 	// Setup takes the server blocks which
 	// contain tokens, as well as options
 	// (e.g. CLI flags) and creates a Caddy
diff --git a/caddyconfig/caddyfile/dispenser.go b/caddyconfig/caddyfile/dispenser.go
index 1cf5d04..0d2c789 100755
--- a/caddyconfig/caddyfile/dispenser.go
+++ b/caddyconfig/caddyfile/dispenser.go
@@ -31,6 +31,7 @@ type Dispenser struct {
 }
 
 // NewDispenser returns a Dispenser filled with the given tokens.
+// TODO: Get rid of the filename argument; it seems pointless here
 func NewDispenser(filename string, tokens []Token) *Dispenser {
 	return &Dispenser{
 		filename: filename,
@@ -51,15 +52,15 @@ func (d *Dispenser) Next() bool {
 }
 
 // Prev moves to the previous token. It does the inverse
-// of Next(). Generally, this should only be used in
-// special cases such as deleting a token from the slice
-// that d is iterating. In that case, without using Prev(),
-// the dispenser would be pointing at the wrong token since
-// deleting a token implicitly advances the cursor.
+// of Next(), except this function may decrement the cursor
+// to -1 so that the next call to Next() points to the
+// first token; this allows dispensing to "start over". This
+// method returns true if the cursor ends up pointing to a
+// valid token.
 func (d *Dispenser) Prev() bool {
-	if d.cursor > 0 {
+	if d.cursor > -1 {
 		d.cursor--
-		return true
+		return d.cursor > -1
 	}
 	return false
 }
@@ -223,8 +224,7 @@ func (d *Dispenser) RemainingArgs() []string {
 // "directive" whether that be to the end of the line or
 // the end of a block that starts at the end of the line.
 func (d *Dispenser) NewFromNextTokens() *Dispenser {
-	var tkns []Token
-	tkns = append(tkns, d.Token())
+	tkns := []Token{d.Token()}
 	for d.NextArg() {
 		tkns = append(tkns, d.Token())
 	}
@@ -245,10 +245,14 @@ func (d *Dispenser) NewFromNextTokens() *Dispenser {
 
 // Token returns the current token.
 func (d *Dispenser) Token() Token {
-	if d.cursor < 0 || d.cursor >= len(d.tokens) {
+	return d.TokenAt(d.cursor)
+}
+
+func (d *Dispenser) TokenAt(cursor int) Token {
+	if cursor < 0 || cursor >= len(d.tokens) {
 		return Token{}
 	}
-	return d.tokens[d.cursor]
+	return d.tokens[cursor]
 }
 
 // Cursor returns the current cursor (token index).
@@ -256,6 +260,10 @@ func (d *Dispenser) Cursor() int {
 	return d.cursor
 }
 
+func (d *Dispenser) Reset() {
+	d.cursor = -1
+}
+
 // ArgErr returns an argument error, meaning that another
 // argument was expected but not found. In other words,
 // a line break or open curly brace was encountered instead of
diff --git a/caddyconfig/caddyfile/parse.go b/caddyconfig/caddyfile/parse.go
index cc91e3d..e5b25fc 100755
--- a/caddyconfig/caddyfile/parse.go
+++ b/caddyconfig/caddyfile/parse.go
@@ -28,12 +28,12 @@ import (
 // Directives that do not appear in validDirectives will cause
 // an error. If you do not want to check for valid directives,
 // pass in nil instead.
-func Parse(filename string, input io.Reader, validDirectives []string) ([]ServerBlock, error) {
+func Parse(filename string, input io.Reader) ([]ServerBlock, error) {
 	tokens, err := allTokens(input)
 	if err != nil {
 		return nil, err
 	}
-	p := parser{Dispenser: NewDispenser(filename, tokens), validDirectives: validDirectives}
+	p := parser{Dispenser: NewDispenser(filename, tokens)}
 	return p.parseAll()
 }
 
@@ -56,9 +56,9 @@ func allTokens(input io.Reader) ([]Token, error) {
 type parser struct {
 	*Dispenser
 	block           ServerBlock // current server block being parsed
-	validDirectives []string    // a directive must be valid or it's an error
 	eof             bool        // if we encounter a valid EOF in a hard place
 	definedSnippets map[string][]Token
+	nesting         int
 }
 
 func (p *parser) parseAll() ([]ServerBlock, error) {
@@ -72,14 +72,16 @@ func (p *parser) parseAll() ([]ServerBlock, error) {
 		if len(p.block.Keys) > 0 {
 			blocks = append(blocks, p.block)
 		}
+		if p.nesting > 0 {
+			return blocks, p.EOFErr()
+		}
 	}
 
 	return blocks, nil
 }
 
 func (p *parser) parseOne() error {
-	p.block = ServerBlock{Tokens: make(map[string][]Token)}
-
+	p.block = ServerBlock{}
 	return p.begin()
 }
 
@@ -186,7 +188,7 @@ func (p *parser) blockContents() error {
 		return err
 	}
 
-	// Only look for close curly brace if there was an opening
+	// only look for close curly brace if there was an opening
 	if errOpenCurlyBrace == nil {
 		err = p.closeCurlyBrace()
 		if err != nil {
@@ -205,6 +207,7 @@ func (p *parser) directives() error {
 	for p.Next() {
 		// end of server block
 		if p.Val() == "}" {
+			// p.nesting has already been decremented
 			break
 		}
 
@@ -218,11 +221,15 @@ func (p *parser) directives() error {
 			continue
 		}
 
-		// normal case: parse a directive on this line
+		// normal case: parse a directive as a new segment
+		// (a "segment" is a line which starts with a directive
+		// and which ends at the end of the line or at the end of
+		// the block that is opened at the end of the line)
 		if err := p.directive(); err != nil {
 			return err
 		}
 	}
+
 	return nil
 }
 
@@ -345,25 +352,24 @@ func (p *parser) doSingleImport(importFile string) ([]Token, error) {
 // are loaded into the current server block for later use
 // by directive setup functions.
 func (p *parser) directive() error {
-	dir := replaceEnvVars(p.Val())
-	nesting := 0
+	// evaluate any env vars in directive token
+	p.tokens[p.cursor].Text = replaceEnvVars(p.tokens[p.cursor].Text)
 
-	if !p.validDirective(dir) {
-		return p.Errf("Unknown directive '%s'", dir)
-	}
+	// a segment is a list of tokens associated with this directive
+	var segment Segment
 
-	// The directive itself is appended as a relevant token
-	p.block.Tokens[dir] = append(p.block.Tokens[dir], p.tokens[p.cursor])
+	// the directive itself is appended as a relevant token
+	segment = append(segment, p.Token())
 
 	for p.Next() {
 		if p.Val() == "{" {
-			nesting++
-		} else if p.isNewLine() && nesting == 0 {
+			p.nesting++
+		} else if p.isNewLine() && p.nesting == 0 {
 			p.cursor-- // read too far
 			break
-		} else if p.Val() == "}" && nesting > 0 {
-			nesting--
-		} else if p.Val() == "}" && nesting == 0 {
+		} else if p.Val() == "}" && p.nesting > 0 {
+			p.nesting--
+		} else if p.Val() == "}" && p.nesting == 0 {
 			return p.Err("Unexpected '}' because no matching opening brace")
 		} else if p.Val() == "import" && p.isNewLine() {
 			if err := p.doImport(); err != nil {
@@ -373,12 +379,15 @@ func (p *parser) directive() error {
 			continue
 		}
 		p.tokens[p.cursor].Text = replaceEnvVars(p.tokens[p.cursor].Text)
-		p.block.Tokens[dir] = append(p.block.Tokens[dir], p.tokens[p.cursor])
+		segment = append(segment, p.Token())
 	}
 
-	if nesting > 0 {
+	p.block.Segments = append(p.block.Segments, segment)
+
+	if p.nesting > 0 {
 		return p.EOFErr()
 	}
+
 	return nil
 }
 
@@ -404,19 +413,6 @@ func (p *parser) closeCurlyBrace() error {
 	return nil
 }
 
-// validDirective returns true if dir is in p.validDirectives.
-func (p *parser) validDirective(dir string) bool {
-	if p.validDirectives == nil {
-		return true
-	}
-	for _, d := range p.validDirectives {
-		if d == dir {
-			return true
-		}
-	}
-	return false
-}
-
 // replaceEnvVars replaces environment variables that appear in the token
 // and understands both the $UNIX and %WINDOWS% syntaxes.
 func replaceEnvVars(s string) string {
@@ -447,13 +443,6 @@ func replaceEnvReferences(s, refStart, refEnd string) string {
 	return s
 }
 
-// ServerBlock associates any number of keys (usually addresses
-// of some sort) with tokens (grouped by directive name).
-type ServerBlock struct {
-	Keys   []string
-	Tokens map[string][]Token
-}
-
 func (p *parser) isSnippet() (bool, string) {
 	keys := p.block.Keys
 	// A snippet block is a single key with parens. Nothing else qualifies.
@@ -480,6 +469,7 @@ func (p *parser) snippetTokens() ([]Token, error) {
 			}
 		}
 		if p.Val() == "{" {
+			p.nesting++
 			count++
 		}
 		tokens = append(tokens, p.tokens[p.cursor])
@@ -490,3 +480,43 @@ func (p *parser) snippetTokens() ([]Token, error) {
 	}
 	return tokens, nil
 }
+
+// ServerBlock associates any number of keys from the
+// head of the server block with tokens, which are
+// grouped by segments.
+type ServerBlock struct {
+	Keys     []string
+	Segments []Segment
+}
+
+// DispenseDirective returns a dispenser that contains
+// all the tokens in the server block.
+func (sb ServerBlock) DispenseDirective(dir string) *Dispenser {
+	var tokens []Token
+	for _, seg := range sb.Segments {
+		if len(seg) > 0 && seg[0].Text == dir {
+			tokens = append(tokens, seg...)
+		}
+	}
+	return NewDispenser("", tokens)
+}
+
+// Segment is a list of tokens which begins with a directive
+// and ends at the end of the directive (either at the end of
+// the line, or at the end of a block it opens).
+type Segment []Token
+
+// Directive returns the directive name for the segment.
+// The directive name is the text of the first token.
+func (s Segment) Directive() string {
+	if len(s) > 0 {
+		return s[0].Text
+	}
+	return ""
+}
+
+// NewDispenser returns a dispenser for this
+// segment's tokens.
+func (s Segment) NewDispenser() *Dispenser {
+	return NewDispenser("", s)
+}
diff --git a/caddyconfig/caddyfile/parse_test.go b/caddyconfig/caddyfile/parse_test.go
index 654c68d..19959de 100755
--- a/caddyconfig/caddyfile/parse_test.go
+++ b/caddyconfig/caddyfile/parse_test.go
@@ -22,6 +22,8 @@ import (
 	"testing"
 )
 
+// TODO: re-enable all tests
+
 func TestAllTokens(t *testing.T) {
 	input := strings.NewReader("a b c\nd e")
 	expected := []string{"a", "b", "c", "d", "e"}
@@ -53,84 +55,67 @@ func TestParseOneAndImport(t *testing.T) {
 		input     string
 		shouldErr bool
 		keys      []string
-		tokens    map[string]int // map of directive name to number of tokens expected
+		numTokens []int // number of tokens to expect in each segment
 	}{
 		{`localhost`, false, []string{
 			"localhost",
-		}, map[string]int{}},
+		}, []int{}},
 
 		{`localhost
 		  dir1`, false, []string{
 			"localhost",
-		}, map[string]int{
-			"dir1": 1,
-		}},
+		}, []int{1}},
 
 		{`localhost:1234
 		  dir1 foo bar`, false, []string{
 			"localhost:1234",
-		}, map[string]int{
-			"dir1": 3,
-		}},
+		}, []int{3},
+		},
 
 		{`localhost {
 		    dir1
 		  }`, false, []string{
 			"localhost",
-		}, map[string]int{
-			"dir1": 1,
-		}},
+		}, []int{1}},
 
 		{`localhost:1234 {
 		    dir1 foo bar
 		    dir2
 		  }`, false, []string{
 			"localhost:1234",
-		}, map[string]int{
-			"dir1": 3,
-			"dir2": 1,
-		}},
+		}, []int{3, 1}},
 
 		{`http://localhost https://localhost
 		  dir1 foo bar`, false, []string{
 			"http://localhost",
 			"https://localhost",
-		}, map[string]int{
-			"dir1": 3,
-		}},
+		}, []int{3}},
 
 		{`http://localhost https://localhost {
 		    dir1 foo bar
 		  }`, false, []string{
 			"http://localhost",
 			"https://localhost",
-		}, map[string]int{
-			"dir1": 3,
-		}},
+		}, []int{3}},
 
 		{`http://localhost, https://localhost {
 		    dir1 foo bar
 		  }`, false, []string{
 			"http://localhost",
 			"https://localhost",
-		}, map[string]int{
-			"dir1": 3,
-		}},
+		}, []int{3}},
 
 		{`http://localhost, {
 		  }`, true, []string{
 			"http://localhost",
-		}, map[string]int{}},
+		}, []int{}},
 
 		{`host1:80, http://host2.com
 		  dir1 foo bar
 		  dir2 baz`, false, []string{
 			"host1:80",
 			"http://host2.com",
-		}, map[string]int{
-			"dir1": 3,
-			"dir2": 2,
-		}},
+		}, []int{3, 2}},
 
 		{`http://host1.com,
 		  http://host2.com,
@@ -138,7 +123,7 @@ func TestParseOneAndImport(t *testing.T) {
 			"http://host1.com",
 			"http://host2.com",
 			"https://host3.com",
-		}, map[string]int{}},
+		}, []int{}},
 
 		{`http://host1.com:1234, https://host2.com
 		  dir1 foo {
@@ -147,10 +132,7 @@ func TestParseOneAndImport(t *testing.T) {
 		  dir2`, false, []string{
 			"http://host1.com:1234",
 			"https://host2.com",
-		}, map[string]int{
-			"dir1": 6,
-			"dir2": 1,
-		}},
+		}, []int{6, 1}},
 
 		{`127.0.0.1
 		  dir1 {
@@ -160,34 +142,25 @@ func TestParseOneAndImport(t *testing.T) {
 		    foo bar
 		  }`, false, []string{
 			"127.0.0.1",
-		}, map[string]int{
-			"dir1": 5,
-			"dir2": 5,
-		}},
+		}, []int{5, 5}},
 
 		{`localhost
 		  dir1 {
 		    foo`, true, []string{
 			"localhost",
-		}, map[string]int{
-			"dir1": 3,
-		}},
+		}, []int{3}},
 
 		{`localhost
 		  dir1 {
 		  }`, false, []string{
 			"localhost",
-		}, map[string]int{
-			"dir1": 3,
-		}},
+		}, []int{3}},
 
 		{`localhost
 		  dir1 {
 		  } }`, true, []string{
 			"localhost",
-		}, map[string]int{
-			"dir1": 3,
-		}},
+		}, []int{}},
 
 		{`localhost
 		  dir1 {
@@ -197,48 +170,38 @@ func TestParseOneAndImport(t *testing.T) {
 		  }
 		  dir2 foo bar`, false, []string{
 			"localhost",
-		}, map[string]int{
-			"dir1": 7,
-			"dir2": 3,
-		}},
+		}, []int{7, 3}},
 
-		{``, false, []string{}, map[string]int{}},
+		{``, false, []string{}, []int{}},
 
 		{`localhost
 		  dir1 arg1
 		  import testdata/import_test1.txt`, false, []string{
 			"localhost",
-		}, map[string]int{
-			"dir1": 2,
-			"dir2": 3,
-			"dir3": 1,
-		}},
+		}, []int{2, 3, 1}},
 
 		{`import testdata/import_test2.txt`, false, []string{
 			"host1",
-		}, map[string]int{
-			"dir1": 1,
-			"dir2": 2,
-		}},
+		}, []int{1, 2}},
 
-		{`import testdata/import_test1.txt testdata/import_test2.txt`, true, []string{}, map[string]int{}},
+		{`import testdata/import_test1.txt testdata/import_test2.txt`, true, []string{}, []int{}},
 
-		{`import testdata/not_found.txt`, true, []string{}, map[string]int{}},
+		{`import testdata/not_found.txt`, true, []string{}, []int{}},
 
-		{`""`, false, []string{}, map[string]int{}},
+		{`""`, false, []string{}, []int{}},
 
-		{``, false, []string{}, map[string]int{}},
+		{``, false, []string{}, []int{}},
 
 		// test cases found by fuzzing!
-		{`import }{$"`, true, []string{}, map[string]int{}},
-		{`import /*/*.txt`, true, []string{}, map[string]int{}},
-		{`import /???/?*?o`, true, []string{}, map[string]int{}},
-		{`import /??`, true, []string{}, map[string]int{}},
-		{`import /[a-z]`, true, []string{}, map[string]int{}},
-		{`import {$}`, true, []string{}, map[string]int{}},
-		{`import {%}`, true, []string{}, map[string]int{}},
-		{`import {$$}`, true, []string{}, map[string]int{}},
-		{`import {%%}`, true, []string{}, map[string]int{}},
+		{`import }{$"`, true, []string{}, []int{}},
+		{`import /*/*.txt`, true, []string{}, []int{}},
+		{`import /???/?*?o`, true, []string{}, []int{}},
+		{`import /??`, true, []string{}, []int{}},
+		{`import /[a-z]`, true, []string{}, []int{}},
+		{`import {$}`, true, []string{}, []int{}},
+		{`import {%}`, true, []string{}, []int{}},
+		{`import {$$}`, true, []string{}, []int{}},
+		{`import {%%}`, true, []string{}, []int{}},
 	} {
 		result, err := testParseOne(test.input)
 
@@ -261,15 +224,16 @@ func TestParseOneAndImport(t *testing.T) {
 			}
 		}
 
-		if len(result.Tokens) != len(test.tokens) {
-			t.Errorf("Test %d: Expected %d directives, had %d",
-				i, len(test.tokens), len(result.Tokens))
+		if len(result.Segments) != len(test.numTokens) {
+			t.Errorf("Test %d: Expected %d segments, had %d",
+				i, len(test.numTokens), len(result.Segments))
 			continue
 		}
-		for directive, tokens := range result.Tokens {
-			if len(tokens) != test.tokens[directive] {
-				t.Errorf("Test %d, directive '%s': Expected %d tokens, counted %d",
-					i, directive, test.tokens[directive], len(tokens))
+
+		for j, seg := range result.Segments {
+			if len(seg) != test.numTokens[j] {
+				t.Errorf("Test %d, segment %d: Expected %d tokens, counted %d",
+					i, j, test.numTokens[j], len(seg))
 				continue
 			}
 		}
@@ -289,12 +253,12 @@ func TestRecursiveImport(t *testing.T) {
 			t.Errorf("got keys unexpected: expect localhost, got %v", got.Keys)
 			return false
 		}
-		if len(got.Tokens) != 2 {
-			t.Errorf("got wrong number of tokens: expect 2, got %d", len(got.Tokens))
+		if len(got.Segments) != 2 {
+			t.Errorf("got wrong number of segments: expect 2, got %d", len(got.Segments))
 			return false
 		}
-		if len(got.Tokens["dir1"]) != 1 || len(got.Tokens["dir2"]) != 2 {
-			t.Errorf("got unexpect tokens: %v", got.Tokens)
+		if len(got.Segments[0]) != 1 || len(got.Segments[1]) != 2 {
+			t.Errorf("got unexpect tokens: %v", got.Segments)
 			return false
 		}
 		return true
@@ -384,12 +348,12 @@ func TestDirectiveImport(t *testing.T) {
 			t.Errorf("got keys unexpected: expect localhost, got %v", got.Keys)
 			return false
 		}
-		if len(got.Tokens) != 2 {
-			t.Errorf("got wrong number of tokens: expect 2, got %d", len(got.Tokens))
+		if len(got.Segments) != 2 {
+			t.Errorf("got wrong number of segments: expect 2, got %d", len(got.Segments))
 			return false
 		}
-		if len(got.Tokens["dir1"]) != 1 || len(got.Tokens["proxy"]) != 8 {
-			t.Errorf("got unexpect tokens: %v", got.Tokens)
+		if len(got.Segments[0]) != 1 || len(got.Segments[1]) != 8 {
+			t.Errorf("got unexpect tokens: %v", got.Segments)
 			return false
 		}
 		return true
@@ -557,21 +521,21 @@ func TestEnvironmentReplacement(t *testing.T) {
 	if actual, expected := blocks[0].Keys[0], ":8080"; expected != actual {
 		t.Errorf("Expected key to be '%s' but was '%s'", expected, actual)
 	}
-	if actual, expected := blocks[0].Tokens["dir1"][1].Text, "foobar"; expected != actual {
+	if actual, expected := blocks[0].Segments[0][1].Text, "foobar"; expected != actual {
 		t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual)
 	}
 
 	// combined windows env vars in argument
 	p = testParser(":{%PORT%}\ndir1 {%ADDRESS%}/{%FOOBAR%}")
 	blocks, _ = p.parseAll()
-	if actual, expected := blocks[0].Tokens["dir1"][1].Text, "servername.com/foobar"; expected != actual {
+	if actual, expected := blocks[0].Segments[0][1].Text, "servername.com/foobar"; expected != actual {
 		t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual)
 	}
 
 	// malformed env var (windows)
 	p = testParser(":1234\ndir1 {%ADDRESS}")
 	blocks, _ = p.parseAll()
-	if actual, expected := blocks[0].Tokens["dir1"][1].Text, "{%ADDRESS}"; expected != actual {
+	if actual, expected := blocks[0].Segments[0][1].Text, "{%ADDRESS}"; expected != actual {
 		t.Errorf("Expected host to be '%s' but was '%s'", expected, actual)
 	}
 
@@ -585,22 +549,18 @@ func TestEnvironmentReplacement(t *testing.T) {
 	// in quoted field
 	p = testParser(":1234\ndir1 \"Test {$FOOBAR} test\"")
 	blocks, _ = p.parseAll()
-	if actual, expected := blocks[0].Tokens["dir1"][1].Text, "Test foobar test"; expected != actual {
+	if actual, expected := blocks[0].Segments[0][1].Text, "Test foobar test"; expected != actual {
 		t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual)
 	}
 
 	// after end token
 	p = testParser(":1234\nanswer \"{{ .Name }} {$FOOBAR}\"")
 	blocks, _ = p.parseAll()
-	if actual, expected := blocks[0].Tokens["answer"][1].Text, "{{ .Name }} foobar"; expected != actual {
+	if actual, expected := blocks[0].Segments[0][1].Text, "{{ .Name }} foobar"; expected != actual {
 		t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual)
 	}
 }
 
-func testParser(input string) parser {
-	return parser{Dispenser: newTestDispenser(input)}
-}
-
 func TestSnippets(t *testing.T) {
 	p := testParser(`
 		(common) {
@@ -617,7 +577,7 @@ func TestSnippets(t *testing.T) {
 	}
 	for _, b := range blocks {
 		t.Log(b.Keys)
-		t.Log(b.Tokens)
+		t.Log(b.Segments)
 	}
 	if len(blocks) != 1 {
 		t.Fatalf("Expect exactly one server block. Got %d.", len(blocks))
@@ -625,16 +585,15 @@ func TestSnippets(t *testing.T) {
 	if actual, expected := blocks[0].Keys[0], "http://example.com"; expected != actual {
 		t.Errorf("Expected server name to be '%s' but was '%s'", expected, actual)
 	}
-	if len(blocks[0].Tokens) != 2 {
-		t.Fatalf("Server block should have tokens from import")
+	if len(blocks[0].Segments) != 2 {
+		t.Fatalf("Server block should have tokens from import, got: %+v", blocks[0])
 	}
-	if actual, expected := blocks[0].Tokens["gzip"][0].Text, "gzip"; expected != actual {
+	if actual, expected := blocks[0].Segments[0][0].Text, "gzip"; expected != actual {
 		t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual)
 	}
-	if actual, expected := blocks[0].Tokens["errors"][1].Text, "stderr"; expected != actual {
+	if actual, expected := blocks[0].Segments[1][1].Text, "stderr"; expected != actual {
 		t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual)
 	}
-
 }
 
 func writeStringToTempFileOrDie(t *testing.T, str string) (pathToFile string) {
@@ -666,9 +625,9 @@ func TestImportedFilesIgnoreNonDirectiveImportTokens(t *testing.T) {
 	}
 	for _, b := range blocks {
 		t.Log(b.Keys)
-		t.Log(b.Tokens)
+		t.Log(b.Segments)
 	}
-	auth := blocks[0].Tokens["basicauth"]
+	auth := blocks[0].Segments[0]
 	line := auth[0].Text + " " + auth[1].Text + " " + auth[2].Text + " " + auth[3].Text
 	if line != "basicauth / import password" {
 		// Previously, it would be changed to:
@@ -701,7 +660,7 @@ func TestSnippetAcrossMultipleFiles(t *testing.T) {
 	}
 	for _, b := range blocks {
 		t.Log(b.Keys)
-		t.Log(b.Tokens)
+		t.Log(b.Segments)
 	}
 	if len(blocks) != 1 {
 		t.Fatalf("Expect exactly one server block. Got %d.", len(blocks))
@@ -709,10 +668,14 @@ func TestSnippetAcrossMultipleFiles(t *testing.T) {
 	if actual, expected := blocks[0].Keys[0], "http://example.com"; expected != actual {
 		t.Errorf("Expected server name to be '%s' but was '%s'", expected, actual)
 	}
-	if len(blocks[0].Tokens) != 1 {
+	if len(blocks[0].Segments) != 1 {
 		t.Fatalf("Server block should have tokens from import")
 	}
-	if actual, expected := blocks[0].Tokens["gzip"][0].Text, "gzip"; expected != actual {
+	if actual, expected := blocks[0].Segments[0][0].Text, "gzip"; expected != actual {
 		t.Errorf("Expected argument to be '%s' but was '%s'", expected, actual)
 	}
 }
+
+func testParser(input string) parser {
+	return parser{Dispenser: newTestDispenser(input)}
+}