v2: Module documentation; refactor LoadModule(); new caddy struct tags (#2924)

This commit goes a long way toward making automated documentation of
Caddy config and Caddy modules possible. It's a broad, sweeping change,
but mostly internal. It allows us to automatically generate docs for all
Caddy modules (including future third-party ones) and make them viewable
on a web page; it also doubles as godoc comments.

As such, this commit makes significant progress in migrating the docs
from our temporary wiki page toward our new website which is still under
construction.

With this change, all host modules will use ctx.LoadModule() and pass in
both the struct pointer and the field name as a string. This allows the
reflect package to read the struct tag from that field so that it can
get the necessary information like the module namespace and the inline
key.

This has the nice side-effect of unifying the code and documentation. It
also simplifies module loading, and handles several variations on field
types for raw module fields (i.e. variations on json.RawMessage, such as
arrays and maps).

I also renamed ModuleInfo.Name -> ModuleInfo.ID, to make it clear that
the ID is the "full name" which includes both the module namespace and
the name. This clarity is helpful when describing module hierarchy.

As of this change, Caddy modules are no longer an experimental design.
I think the architecture is good enough to go forward.

1 files changed, 99 insertions, 44 deletions

diff --git a/modules/caddyhttp/routes.go b/modules/caddyhttp/routes.go
index 550b14e..0dce990 100644
--- a/modules/caddyhttp/routes.go
+++ b/modules/caddyhttp/routes.go
@@ -22,14 +22,73 @@ import (
 	"github.com/caddyserver/caddy/v2"
 )
 
-// Route represents a set of matching rules,
-// middlewares, and a responder for handling HTTP
-// requests.
+// Route consists of a set of rules for matching HTTP requests,
+// a list of handlers to execute, and optional flow control
+// parameters which customize the handling of HTTP requests
+// in a highly flexible and performant manner.
 type Route struct {
-	Group          string            `json:"group,omitempty"`
-	MatcherSetsRaw RawMatcherSets    `json:"match,omitempty"`
-	HandlersRaw    []json.RawMessage `json:"handle,omitempty"`
-	Terminal       bool              `json:"terminal,omitempty"`
+	// Group is an optional name for a group to which this
+	// route belongs. If a route belongs to a group, only
+	// the first matching route in the group will be used.
+	Group string `json:"group,omitempty"`
+
+	// The matcher sets which will be used to qualify this
+	// route for a request. Essentially the "if" statement
+	// of this route. Each matcher set is OR'ed, but matchers
+	// within a set are AND'ed together.
+	MatcherSetsRaw RawMatcherSets `json:"match,omitempty" caddy:"namespace=http.matchers"`
+
+	// The list of handlers for this route. Upon matching a request, they are chained
+	// together in a middleware fashion: requests flow from the first handler to the last
+	// (top of the list to the bottom), with the possibility that any handler could stop
+	// the chain and/or return an error. Responses flow back through the chain (bottom of
+	// the list to the top) as they are written out to the client.
+	//
+	// Not all handlers call the next handler in the chain. For example, the reverse_proxy
+	// handler always sends a request upstream or returns an error. Thus, configuring
+	// handlers after reverse_proxy in the same route is illogical, since they would never
+	// be executed. You will want to put handlers which originate the response at the very
+	// end of your route(s). The documentation for a module should state whether it invokes
+	// the next handler, but sometimes it is common sense.
+	//
+	// Some handlers manipulate the response. Remember that requests flow down the list, and
+	// responses flow up the list.
+	//
+	// For example, if you wanted to use both `templates` and `encode` handlers, you would
+	// need to put `templates` after `encode` in your route, because responses flow up.
+	// Thus, `templates` will be able to parse and execute the plain-text response as a
+	// template, and then return it up to the `encode` handler which will then compress it
+	// into a binary format.
+	//
+	// If `templates` came before `encode`, then `encode` would write a compressed,
+	// binary-encoded response to `templates` which would not be able to parse the response
+	// properly.
+	//
+	// The correct order, then, is this:
+	//
+	//     [
+	//         {"handler": "encode"},
+	//         {"handler": "templates"},
+	//         {"handler": "file_server"}
+	//     ]
+	//
+	// The request flows ⬇️ DOWN (`encode` -> `templates` -> `file_server`).
+	//
+	// 1. First, `encode` will choose how to `encode` the response and wrap the response.
+	// 2. Then, `templates` will wrap the response with a buffer.
+	// 3. Finally, `file_server` will originate the content from a file.
+	//
+	// The response flows ⬆️ UP (`file_server` -> `templates` -> `encode`):
+	//
+	// 1. First, `file_server` will write the file to the response.
+	// 2. That write will be buffered and then executed by `templates`.
+	// 3. Lastly, the write from `templates` will flow into `encode` which will compress the stream.
+	//
+	// If you think of routes in this way, it will be easy and even fun to solve the puzzle of writing correct routes.
+	HandlersRaw []json.RawMessage `json:"handle,omitempty" caddy:"namespace=http.handlers inline_key=handler"`
+
+	// If true, no more routes will be executed after this one, even if they matched.
+	Terminal bool `json:"terminal,omitempty"`
 
 	// decoded values
 	MatcherSets MatcherSets         `json:"-"`
@@ -54,22 +113,23 @@ type RouteList []Route
 func (routes RouteList) Provision(ctx caddy.Context) error {
 	for i, route := range routes {
 		// matchers
-		matcherSets, err := route.MatcherSetsRaw.Setup(ctx)
+		matchersIface, err := ctx.LoadModule(&route, "MatcherSetsRaw")
+		if err != nil {
+			return fmt.Errorf("loadng matchers in route %d: %v", i, err)
+		}
+		err = routes[i].MatcherSets.FromInterface(matchersIface)
 		if err != nil {
-			return err
+			return fmt.Errorf("route %d: %v", i, err)
 		}
-		routes[i].MatcherSets = matcherSets
-		routes[i].MatcherSetsRaw = nil // allow GC to deallocate
 
 		// handlers
-		for j, rawMsg := range route.HandlersRaw {
-			mh, err := ctx.LoadModuleInline("handler", "http.handlers", rawMsg)
-			if err != nil {
-				return fmt.Errorf("loading handler module in position %d: %v", j, err)
-			}
-			routes[i].Handlers = append(routes[i].Handlers, mh.(MiddlewareHandler))
+		handlersIface, err := ctx.LoadModule(&route, "HandlersRaw")
+		if err != nil {
+			return fmt.Errorf("loading handler modules in route %d: %v", i, err)
+		}
+		for _, handler := range handlersIface.([]interface{}) {
+			routes[i].Handlers = append(routes[i].Handlers, handler.(MiddlewareHandler))
 		}
-		routes[i].HandlersRaw = nil // allow GC to deallocate
 	}
 	return nil
 }
@@ -171,28 +231,7 @@ func (mset MatcherSet) Match(r *http.Request) bool {
 
 // RawMatcherSets is a group of matcher sets
 // in their raw, JSON form.
-type RawMatcherSets []map[string]json.RawMessage
-
-// Setup sets up all matcher sets by loading each matcher module
-// and returning the group of provisioned matcher sets.
-func (rm RawMatcherSets) Setup(ctx caddy.Context) (MatcherSets, error) {
-	if rm == nil {
-		return nil, nil
-	}
-	var ms MatcherSets
-	for _, matcherSet := range rm {
-		var matchers MatcherSet
-		for modName, rawMsg := range matcherSet {
-			val, err := ctx.LoadModule("http.matchers."+modName, rawMsg)
-			if err != nil {
-				return nil, fmt.Errorf("loading matcher module '%s': %v", modName, err)
-			}
-			matchers = append(matchers, val.(RequestMatcher))
-		}
-		ms = append(ms, matchers)
-	}
-	return ms, nil
-}
+type RawMatcherSets []caddy.ModuleMap
 
 // MatcherSets is a group of matcher sets capable
 // of checking whether a request matches any of
@@ -202,11 +241,27 @@ type MatcherSets []MatcherSet
 // AnyMatch returns true if req matches any of the
 // matcher sets in mss or if there are no matchers,
 // in which case the request always matches.
-func (mss MatcherSets) AnyMatch(req *http.Request) bool {
-	for _, ms := range mss {
-		if ms.Match(req) {
+func (ms MatcherSets) AnyMatch(req *http.Request) bool {
+	for _, m := range ms {
+		if m.Match(req) {
 			return true
 		}
 	}
-	return len(mss) == 0
+	return len(ms) == 0
+}
+
+// FromInterface fills ms from an interface{} value obtained from LoadModule.
+func (ms *MatcherSets) FromInterface(matcherSets interface{}) error {
+	for _, matcherSetIfaces := range matcherSets.([]map[string]interface{}) {
+		var matcherSet MatcherSet
+		for _, matcher := range matcherSetIfaces {
+			reqMatcher, ok := matcher.(RequestMatcher)
+			if !ok {
+				return fmt.Errorf("decoded module is not a RequestMatcher: %#v", matcher)
+			}
+			matcherSet = append(matcherSet, reqMatcher)
+		}
+		*ms = append(*ms, matcherSet)
+	}
+	return nil
 }