push: Implement HTTP/2 server push (#3573)

* push: Implement HTTP/2 server push (close #3551)

* push: Abstract header ops by embedding into new struct type

This will allow us to add more fields to customize headers in
push-specific ways in the future.

* push: Ensure Link resources are pushed before response is written

* Change header name from X-Caddy-Push to Caddy-Push

6 files changed, 504 insertions, 4 deletions

diff --git a/modules/caddyhttp/headers/headers.go b/modules/caddyhttp/headers/headers.go
index 681c21f..3571dd9 100644
--- a/modules/caddyhttp/headers/headers.go
+++ b/modules/caddyhttp/headers/headers.go
@@ -54,15 +54,15 @@ func (Handler) CaddyModule() caddy.ModuleInfo {
 }
 
 // Provision sets up h's configuration.
-func (h *Handler) Provision(_ caddy.Context) error {
+func (h *Handler) Provision(ctx caddy.Context) error {
 	if h.Request != nil {
-		err := h.Request.provision()
+		err := h.Request.Provision(ctx)
 		if err != nil {
 			return err
 		}
 	}
 	if h.Response != nil {
-		err := h.Response.provision()
+		err := h.Response.Provision(ctx)
 		if err != nil {
 			return err
 		}
@@ -125,7 +125,8 @@ type HeaderOps struct {
 	Replace map[string][]Replacement `json:"replace,omitempty"`
 }
 
-func (ops *HeaderOps) provision() error {
+// Provision sets up the header operations.
+func (ops *HeaderOps) Provision(_ caddy.Context) error {
 	for fieldName, replacements := range ops.Replace {
 		for i, r := range replacements {
 			if r.SearchRegexp != "" {
diff --git a/modules/caddyhttp/push/caddyfile.go b/modules/caddyhttp/push/caddyfile.go
new file mode 100644
index 0000000..a70d5d5
--- /dev/null
+++ b/modules/caddyhttp/push/caddyfile.go
@@ -0,0 +1,99 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package push
+
+import (
+	"github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp/headers"
+)
+
+func init() {
+	httpcaddyfile.RegisterHandlerDirective("push", parseCaddyfile)
+}
+
+// parseCaddyfile sets up the push handler. Syntax:
+//
+//     push [<matcher>] [<resource>] {
+//         [GET|HEAD] <resource>
+//         headers {
+//             [+]<field> [<value|regexp> [<replacement>]]
+//             -<field>
+//         }
+//     }
+//
+// A single resource can be specified inline without opening a
+// block for the most common/simple case. Or, a block can be
+// opened and multiple resources can be specified, one per
+// line, optionally preceded by the method. The headers
+// subdirective can be used to customize the headers that
+// are set on each (synthetic) push request, using the same
+// syntax as the 'header' directive for request headers.
+// Placeholders are accepted in resource and header field
+// name and value and replacement tokens.
+func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) {
+	handler := new(Handler)
+
+	for h.Next() {
+		if h.NextArg() {
+			handler.Resources = append(handler.Resources, Resource{Target: h.Val()})
+		}
+
+		// optional block
+		for outerNesting := h.Nesting(); h.NextBlock(outerNesting); {
+			switch h.Val() {
+			case "headers":
+				if h.NextArg() {
+					return nil, h.ArgErr()
+				}
+				for innerNesting := h.Nesting(); h.NextBlock(innerNesting); {
+					// include current token, which we treat as an argument here
+					args := []string{h.Val()}
+					args = append(args, h.RemainingArgs()...)
+
+					if handler.Headers == nil {
+						handler.Headers = new(HeaderConfig)
+					}
+					switch len(args) {
+					case 1:
+						headers.CaddyfileHeaderOp(&handler.Headers.HeaderOps, args[0], "", "")
+					case 2:
+						headers.CaddyfileHeaderOp(&handler.Headers.HeaderOps, args[0], args[1], "")
+					case 3:
+						headers.CaddyfileHeaderOp(&handler.Headers.HeaderOps, args[0], args[1], args[2])
+					default:
+						return nil, h.ArgErr()
+					}
+				}
+
+			case "GET", "HEAD":
+				method := h.Val()
+				if !h.NextArg() {
+					return nil, h.ArgErr()
+				}
+				target := h.Val()
+				handler.Resources = append(handler.Resources, Resource{
+					Method: method,
+					Target: target,
+				})
+
+			default:
+				handler.Resources = append(handler.Resources, Resource{Target: h.Val()})
+			}
+		}
+	}
+
+	return handler, nil
+}
diff --git a/modules/caddyhttp/push/handler.go b/modules/caddyhttp/push/handler.go
new file mode 100644
index 0000000..a89c0cd
--- /dev/null
+++ b/modules/caddyhttp/push/handler.go
@@ -0,0 +1,236 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package push
+
+import (
+	"fmt"
+	"net/http"
+	"strings"
+
+	"github.com/caddyserver/caddy/v2"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp"
+	"github.com/caddyserver/caddy/v2/modules/caddyhttp/headers"
+	"go.uber.org/zap"
+)
+
+func init() {
+	caddy.RegisterModule(Handler{})
+}
+
+// Handler is a middleware for manipulating the request body.
+type Handler struct {
+	Resources []Resource    `json:"resources,omitempty"`
+	Headers   *HeaderConfig `json:"headers,omitempty"`
+
+	logger *zap.Logger
+}
+
+// CaddyModule returns the Caddy module information.
+func (Handler) CaddyModule() caddy.ModuleInfo {
+	return caddy.ModuleInfo{
+		ID:  "http.handlers.push",
+		New: func() caddy.Module { return new(Handler) },
+	}
+}
+
+// Provision sets up h.
+func (h *Handler) Provision(ctx caddy.Context) error {
+	h.logger = ctx.Logger(h)
+	if h.Headers != nil {
+		err := h.Headers.Provision(ctx)
+		if err != nil {
+			return fmt.Errorf("provisioning header operations: %v", err)
+		}
+	}
+	return nil
+}
+
+func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error {
+	pusher, ok := w.(http.Pusher)
+	if !ok {
+		return next.ServeHTTP(w, r)
+	}
+
+	// short-circuit recursive pushes
+	if _, ok := r.Header[pushHeader]; ok {
+		return next.ServeHTTP(w, r)
+	}
+
+	repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer)
+
+	// create header for push requests
+	hdr := h.initializePushHeaders(r, repl)
+
+	// push first!
+	for _, resource := range h.Resources {
+		h.logger.Debug("pushing resource",
+			zap.String("uri", r.RequestURI),
+			zap.String("push_method", resource.Method),
+			zap.String("push_target", resource.Target),
+			zap.Object("push_headers", caddyhttp.LoggableHTTPHeader(hdr)))
+		err := pusher.Push(repl.ReplaceAll(resource.Target, "."), &http.PushOptions{
+			Method: resource.Method,
+			Header: hdr,
+		})
+		if err != nil {
+			// usually this means either that push is not
+			// supported or concurrent streams are full
+			break
+		}
+	}
+
+	// wrap the response writer so that we can initiate push of any resources
+	// described in Link header fields before the response is written
+	lp := linkPusher{
+		ResponseWriterWrapper: &caddyhttp.ResponseWriterWrapper{ResponseWriter: w},
+		handler:               h,
+		pusher:                pusher,
+		header:                hdr,
+		request:               r,
+	}
+
+	// serve only after pushing!
+	if err := next.ServeHTTP(lp, r); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (h Handler) initializePushHeaders(r *http.Request, repl *caddy.Replacer) http.Header {
+	hdr := make(http.Header)
+
+	// prevent recursive pushes
+	hdr.Set(pushHeader, "1")
+
+	// set initial header fields; since exactly how headers should
+	// be implemented for server push is not well-understood, we
+	// are being conservative for now like httpd is:
+	// https://httpd.apache.org/docs/2.4/en/howto/http2.html#push
+	// we only copy some well-known, safe headers that are likely
+	// crucial when requesting certain kinds of content
+	for _, fieldName := range safeHeaders {
+		if vals, ok := r.Header[fieldName]; ok {
+			hdr[fieldName] = vals
+		}
+	}
+
+	// user can customize the push request headers
+	if h.Headers != nil {
+		h.Headers.ApplyTo(hdr, repl)
+	}
+
+	return hdr
+}
+
+// servePreloadLinks parses Link headers from upstream and pushes
+// resources described by them. If a resource has the "nopush"
+// attribute or describes an external entity (meaning, the resource
+// URI includes a scheme), it will not be pushed.
+func (h Handler) servePreloadLinks(pusher http.Pusher, hdr http.Header, resources []string) {
+	for _, resource := range resources {
+		for _, resource := range parseLinkHeader(resource) {
+			if _, ok := resource.params["nopush"]; ok {
+				continue
+			}
+			if isRemoteResource(resource.uri) {
+				continue
+			}
+			err := pusher.Push(resource.uri, &http.PushOptions{
+				Header: hdr,
+			})
+			if err != nil {
+				return
+			}
+		}
+	}
+}
+
+// Resource represents a request for a resource to push.
+type Resource struct {
+	// Method is the request method, which must be GET or HEAD.
+	// Default is GET.
+	Method string `json:"method,omitempty"`
+
+	// Target is the path to the resource being pushed.
+	Target string `json:"target,omitempty"`
+}
+
+// HeaderConfig configures headers for synthetic push requests.
+type HeaderConfig struct {
+	headers.HeaderOps
+}
+
+// linkPusher is a http.ResponseWriter that intercepts
+// the WriteHeader() call to ensure that any resources
+// described by Link response headers get pushed before
+// the response is allowed to be written.
+type linkPusher struct {
+	*caddyhttp.ResponseWriterWrapper
+	handler Handler
+	pusher  http.Pusher
+	header  http.Header
+	request *http.Request
+}
+
+func (lp linkPusher) WriteHeader(statusCode int) {
+	if links, ok := lp.ResponseWriter.Header()["Link"]; ok {
+		// only initiate these pushes if it hasn't been done yet
+		if val := caddyhttp.GetVar(lp.request.Context(), pushedLink); val == nil {
+			lp.handler.logger.Debug("pushing Link resources", zap.Strings("linked", links))
+			caddyhttp.SetVar(lp.request.Context(), pushedLink, true)
+			lp.handler.servePreloadLinks(lp.pusher, lp.header, links)
+		}
+	}
+	lp.ResponseWriter.WriteHeader(statusCode)
+}
+
+// isRemoteResource returns true if resource starts with
+// a scheme or is a protocol-relative URI.
+func isRemoteResource(resource string) bool {
+	return strings.HasPrefix(resource, "//") ||
+		strings.HasPrefix(resource, "http://") ||
+		strings.HasPrefix(resource, "https://")
+}
+
+// safeHeaders is a list of header fields that are
+// safe to copy to push requests implicitly. It is
+// assumed that requests for certain kinds of content
+// would fail without these fields present.
+var safeHeaders = []string{
+	"Accept-Encoding",
+	"Accept-Language",
+	"Accept",
+	"Cache-Control",
+	"User-Agent",
+}
+
+// pushHeader is a header field that gets added to push requests
+// in order to avoid recursive/infinite pushes.
+const pushHeader = "Caddy-Push"
+
+// pushedLink is the key for the variable on the request
+// context that we use to remember whether we have already
+// pushed resources from Link headers yet; otherwise, if
+// multiple push handlers are invoked, it would repeat the
+// pushing of Link headers.
+const pushedLink = "http.handlers.push.pushed_link"
+
+// Interface guards
+var (
+	_ caddy.Provisioner           = (*Handler)(nil)
+	_ caddyhttp.MiddlewareHandler = (*Handler)(nil)
+	_ caddyhttp.HTTPInterfaces    = (*linkPusher)(nil)
+)
diff --git a/modules/caddyhttp/push/link.go b/modules/caddyhttp/push/link.go
new file mode 100644
index 0000000..16b0e7d
--- /dev/null
+++ b/modules/caddyhttp/push/link.go
@@ -0,0 +1,78 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package push
+
+import (
+	"strings"
+)
+
+// linkResource contains the results of a parsed Link header.
+type linkResource struct {
+	uri    string
+	params map[string]string
+}
+
+// parseLinkHeader is responsible for parsing Link header
+// and returning list of found resources.
+//
+// Accepted formats are:
+//
+//     Link: <resource>; as=script
+//     Link: <resource>; as=script,<resource>; as=style
+//     Link: <resource>;<resource2>
+//
+// where <resource> begins with a forward slash (/).
+func parseLinkHeader(header string) []linkResource {
+	resources := []linkResource{}
+
+	if header == "" {
+		return resources
+	}
+
+	for _, link := range strings.Split(header, comma) {
+		l := linkResource{params: make(map[string]string)}
+
+		li, ri := strings.Index(link, "<"), strings.Index(link, ">")
+		if li == -1 || ri == -1 {
+			continue
+		}
+
+		l.uri = strings.TrimSpace(link[li+1 : ri])
+
+		for _, param := range strings.Split(strings.TrimSpace(link[ri+1:]), semicolon) {
+			parts := strings.SplitN(strings.TrimSpace(param), equal, 2)
+			key := strings.TrimSpace(parts[0])
+			if key == "" {
+				continue
+			}
+			if len(parts) == 1 {
+				l.params[key] = key
+			}
+			if len(parts) == 2 {
+				l.params[key] = strings.TrimSpace(parts[1])
+			}
+		}
+
+		resources = append(resources, l)
+	}
+
+	return resources
+}
+
+const (
+	comma     = ","
+	semicolon = ";"
+	equal     = "="
+)
diff --git a/modules/caddyhttp/push/link_test.go b/modules/caddyhttp/push/link_test.go
new file mode 100644
index 0000000..238e284
--- /dev/null
+++ b/modules/caddyhttp/push/link_test.go
@@ -0,0 +1,85 @@
+// Copyright 2015 Matthew Holt and The Caddy Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+package push
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestParseLinkHeader(t *testing.T) {
+	testCases := []struct {
+		header            string
+		expectedResources []linkResource
+	}{
+		{
+			header:            "</resource>; as=script",
+			expectedResources: []linkResource{{uri: "/resource", params: map[string]string{"as": "script"}}},
+		},
+		{
+			header:            "</resource>",
+			expectedResources: []linkResource{{uri: "/resource", params: map[string]string{}}},
+		},
+		{
+			header:            "</resource>; nopush",
+			expectedResources: []linkResource{{uri: "/resource", params: map[string]string{"nopush": "nopush"}}},
+		},
+		{
+			header:            "</resource>;nopush;rel=next",
+			expectedResources: []linkResource{{uri: "/resource", params: map[string]string{"nopush": "nopush", "rel": "next"}}},
+		},
+		{
+			header: "</resource>;nopush;rel=next,</resource2>;nopush",
+			expectedResources: []linkResource{
+				{uri: "/resource", params: map[string]string{"nopush": "nopush", "rel": "next"}},
+				{uri: "/resource2", params: map[string]string{"nopush": "nopush"}},
+			},
+		},
+		{
+			header: "</resource>,</resource2>",
+			expectedResources: []linkResource{
+				{uri: "/resource", params: map[string]string{}},
+				{uri: "/resource2", params: map[string]string{}},
+			},
+		},
+		{
+			header:            "malformed",
+			expectedResources: []linkResource{},
+		},
+		{
+			header:            "<malformed",
+			expectedResources: []linkResource{},
+		},
+		{
+			header:            ",",
+			expectedResources: []linkResource{},
+		},
+		{
+			header:            ";",
+			expectedResources: []linkResource{},
+		},
+		{
+			header:            "</resource> ; ",
+			expectedResources: []linkResource{{uri: "/resource", params: map[string]string{}}},
+		},
+	}
+
+	for i, test := range testCases {
+		actualResources := parseLinkHeader(test.header)
+		if !reflect.DeepEqual(actualResources, test.expectedResources) {
+			t.Errorf("Test %d (header: %s) - expected resources %v, got %v",
+				i, test.header, test.expectedResources, actualResources)
+		}
+	}
+}
diff --git a/modules/caddyhttp/standard/imports.go b/modules/caddyhttp/standard/imports.go
index dabec81..0aeef84 100644
--- a/modules/caddyhttp/standard/imports.go
+++ b/modules/caddyhttp/standard/imports.go
@@ -10,6 +10,7 @@ import (
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/fileserver"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/headers"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/map"
+	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/push"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/requestbody"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy"
 	_ "github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/fastcgi"