From 6cea1f239d01fc065bc6f4b22d765d89b6db0152 Mon Sep 17 00:00:00 2001 From: Matt Holt Date: Mon, 20 Jul 2020 12:28:40 -0600 Subject: push: Implement HTTP/2 server push (#3573) * push: Implement HTTP/2 server push (close #3551) * push: Abstract header ops by embedding into new struct type This will allow us to add more fields to customize headers in push-specific ways in the future. * push: Ensure Link resources are pushed before response is written * Change header name from X-Caddy-Push to Caddy-Push --- caddyconfig/httpcaddyfile/directives.go | 3 +- modules/caddyhttp/headers/headers.go | 9 +- modules/caddyhttp/push/caddyfile.go | 99 ++++++++++++++ modules/caddyhttp/push/handler.go | 236 ++++++++++++++++++++++++++++++++ modules/caddyhttp/push/link.go | 78 +++++++++++ modules/caddyhttp/push/link_test.go | 85 ++++++++++++ modules/caddyhttp/standard/imports.go | 1 + 7 files changed, 506 insertions(+), 5 deletions(-) create mode 100644 modules/caddyhttp/push/caddyfile.go create mode 100644 modules/caddyhttp/push/handler.go create mode 100644 modules/caddyhttp/push/link.go create mode 100644 modules/caddyhttp/push/link_test.go diff --git a/caddyconfig/httpcaddyfile/directives.go b/caddyconfig/httpcaddyfile/directives.go index ee73078..c31bdd3 100644 --- a/caddyconfig/httpcaddyfile/directives.go +++ b/caddyconfig/httpcaddyfile/directives.go @@ -55,10 +55,11 @@ var directiveOrder = []string{ "encode", "templates", - // special routing directives + // special routing & dispatching directives "handle", "handle_path", "route", + "push", // handlers that typically respond to requests "respond", diff --git a/modules/caddyhttp/headers/headers.go b/modules/caddyhttp/headers/headers.go index 681c21f..3571dd9 100644 --- a/modules/caddyhttp/headers/headers.go +++ b/modules/caddyhttp/headers/headers.go @@ -54,15 +54,15 @@ func (Handler) CaddyModule() caddy.ModuleInfo { } // Provision sets up h's configuration. -func (h *Handler) Provision(_ caddy.Context) error { +func (h *Handler) Provision(ctx caddy.Context) error { if h.Request != nil { - err := h.Request.provision() + err := h.Request.Provision(ctx) if err != nil { return err } } if h.Response != nil { - err := h.Response.provision() + err := h.Response.Provision(ctx) if err != nil { return err } @@ -125,7 +125,8 @@ type HeaderOps struct { Replace map[string][]Replacement `json:"replace,omitempty"` } -func (ops *HeaderOps) provision() error { +// Provision sets up the header operations. +func (ops *HeaderOps) Provision(_ caddy.Context) error { for fieldName, replacements := range ops.Replace { for i, r := range replacements { if r.SearchRegexp != "" { diff --git a/modules/caddyhttp/push/caddyfile.go b/modules/caddyhttp/push/caddyfile.go new file mode 100644 index 0000000..a70d5d5 --- /dev/null +++ b/modules/caddyhttp/push/caddyfile.go @@ -0,0 +1,99 @@ +// Copyright 2015 Matthew Holt and The Caddy Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package push + +import ( + "github.com/caddyserver/caddy/v2/caddyconfig/httpcaddyfile" + "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "github.com/caddyserver/caddy/v2/modules/caddyhttp/headers" +) + +func init() { + httpcaddyfile.RegisterHandlerDirective("push", parseCaddyfile) +} + +// parseCaddyfile sets up the push handler. Syntax: +// +// push [] [] { +// [GET|HEAD] +// headers { +// [+] [ []] +// - +// } +// } +// +// A single resource can be specified inline without opening a +// block for the most common/simple case. Or, a block can be +// opened and multiple resources can be specified, one per +// line, optionally preceded by the method. The headers +// subdirective can be used to customize the headers that +// are set on each (synthetic) push request, using the same +// syntax as the 'header' directive for request headers. +// Placeholders are accepted in resource and header field +// name and value and replacement tokens. +func parseCaddyfile(h httpcaddyfile.Helper) (caddyhttp.MiddlewareHandler, error) { + handler := new(Handler) + + for h.Next() { + if h.NextArg() { + handler.Resources = append(handler.Resources, Resource{Target: h.Val()}) + } + + // optional block + for outerNesting := h.Nesting(); h.NextBlock(outerNesting); { + switch h.Val() { + case "headers": + if h.NextArg() { + return nil, h.ArgErr() + } + for innerNesting := h.Nesting(); h.NextBlock(innerNesting); { + // include current token, which we treat as an argument here + args := []string{h.Val()} + args = append(args, h.RemainingArgs()...) + + if handler.Headers == nil { + handler.Headers = new(HeaderConfig) + } + switch len(args) { + case 1: + headers.CaddyfileHeaderOp(&handler.Headers.HeaderOps, args[0], "", "") + case 2: + headers.CaddyfileHeaderOp(&handler.Headers.HeaderOps, args[0], args[1], "") + case 3: + headers.CaddyfileHeaderOp(&handler.Headers.HeaderOps, args[0], args[1], args[2]) + default: + return nil, h.ArgErr() + } + } + + case "GET", "HEAD": + method := h.Val() + if !h.NextArg() { + return nil, h.ArgErr() + } + target := h.Val() + handler.Resources = append(handler.Resources, Resource{ + Method: method, + Target: target, + }) + + default: + handler.Resources = append(handler.Resources, Resource{Target: h.Val()}) + } + } + } + + return handler, nil +} diff --git a/modules/caddyhttp/push/handler.go b/modules/caddyhttp/push/handler.go new file mode 100644 index 0000000..a89c0cd --- /dev/null +++ b/modules/caddyhttp/push/handler.go @@ -0,0 +1,236 @@ +// Copyright 2015 Matthew Holt and The Caddy Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package push + +import ( + "fmt" + "net/http" + "strings" + + "github.com/caddyserver/caddy/v2" + "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "github.com/caddyserver/caddy/v2/modules/caddyhttp/headers" + "go.uber.org/zap" +) + +func init() { + caddy.RegisterModule(Handler{}) +} + +// Handler is a middleware for manipulating the request body. +type Handler struct { + Resources []Resource `json:"resources,omitempty"` + Headers *HeaderConfig `json:"headers,omitempty"` + + logger *zap.Logger +} + +// CaddyModule returns the Caddy module information. +func (Handler) CaddyModule() caddy.ModuleInfo { + return caddy.ModuleInfo{ + ID: "http.handlers.push", + New: func() caddy.Module { return new(Handler) }, + } +} + +// Provision sets up h. +func (h *Handler) Provision(ctx caddy.Context) error { + h.logger = ctx.Logger(h) + if h.Headers != nil { + err := h.Headers.Provision(ctx) + if err != nil { + return fmt.Errorf("provisioning header operations: %v", err) + } + } + return nil +} + +func (h Handler) ServeHTTP(w http.ResponseWriter, r *http.Request, next caddyhttp.Handler) error { + pusher, ok := w.(http.Pusher) + if !ok { + return next.ServeHTTP(w, r) + } + + // short-circuit recursive pushes + if _, ok := r.Header[pushHeader]; ok { + return next.ServeHTTP(w, r) + } + + repl := r.Context().Value(caddy.ReplacerCtxKey).(*caddy.Replacer) + + // create header for push requests + hdr := h.initializePushHeaders(r, repl) + + // push first! + for _, resource := range h.Resources { + h.logger.Debug("pushing resource", + zap.String("uri", r.RequestURI), + zap.String("push_method", resource.Method), + zap.String("push_target", resource.Target), + zap.Object("push_headers", caddyhttp.LoggableHTTPHeader(hdr))) + err := pusher.Push(repl.ReplaceAll(resource.Target, "."), &http.PushOptions{ + Method: resource.Method, + Header: hdr, + }) + if err != nil { + // usually this means either that push is not + // supported or concurrent streams are full + break + } + } + + // wrap the response writer so that we can initiate push of any resources + // described in Link header fields before the response is written + lp := linkPusher{ + ResponseWriterWrapper: &caddyhttp.ResponseWriterWrapper{ResponseWriter: w}, + handler: h, + pusher: pusher, + header: hdr, + request: r, + } + + // serve only after pushing! + if err := next.ServeHTTP(lp, r); err != nil { + return err + } + + return nil +} + +func (h Handler) initializePushHeaders(r *http.Request, repl *caddy.Replacer) http.Header { + hdr := make(http.Header) + + // prevent recursive pushes + hdr.Set(pushHeader, "1") + + // set initial header fields; since exactly how headers should + // be implemented for server push is not well-understood, we + // are being conservative for now like httpd is: + // https://httpd.apache.org/docs/2.4/en/howto/http2.html#push + // we only copy some well-known, safe headers that are likely + // crucial when requesting certain kinds of content + for _, fieldName := range safeHeaders { + if vals, ok := r.Header[fieldName]; ok { + hdr[fieldName] = vals + } + } + + // user can customize the push request headers + if h.Headers != nil { + h.Headers.ApplyTo(hdr, repl) + } + + return hdr +} + +// servePreloadLinks parses Link headers from upstream and pushes +// resources described by them. If a resource has the "nopush" +// attribute or describes an external entity (meaning, the resource +// URI includes a scheme), it will not be pushed. +func (h Handler) servePreloadLinks(pusher http.Pusher, hdr http.Header, resources []string) { + for _, resource := range resources { + for _, resource := range parseLinkHeader(resource) { + if _, ok := resource.params["nopush"]; ok { + continue + } + if isRemoteResource(resource.uri) { + continue + } + err := pusher.Push(resource.uri, &http.PushOptions{ + Header: hdr, + }) + if err != nil { + return + } + } + } +} + +// Resource represents a request for a resource to push. +type Resource struct { + // Method is the request method, which must be GET or HEAD. + // Default is GET. + Method string `json:"method,omitempty"` + + // Target is the path to the resource being pushed. + Target string `json:"target,omitempty"` +} + +// HeaderConfig configures headers for synthetic push requests. +type HeaderConfig struct { + headers.HeaderOps +} + +// linkPusher is a http.ResponseWriter that intercepts +// the WriteHeader() call to ensure that any resources +// described by Link response headers get pushed before +// the response is allowed to be written. +type linkPusher struct { + *caddyhttp.ResponseWriterWrapper + handler Handler + pusher http.Pusher + header http.Header + request *http.Request +} + +func (lp linkPusher) WriteHeader(statusCode int) { + if links, ok := lp.ResponseWriter.Header()["Link"]; ok { + // only initiate these pushes if it hasn't been done yet + if val := caddyhttp.GetVar(lp.request.Context(), pushedLink); val == nil { + lp.handler.logger.Debug("pushing Link resources", zap.Strings("linked", links)) + caddyhttp.SetVar(lp.request.Context(), pushedLink, true) + lp.handler.servePreloadLinks(lp.pusher, lp.header, links) + } + } + lp.ResponseWriter.WriteHeader(statusCode) +} + +// isRemoteResource returns true if resource starts with +// a scheme or is a protocol-relative URI. +func isRemoteResource(resource string) bool { + return strings.HasPrefix(resource, "//") || + strings.HasPrefix(resource, "http://") || + strings.HasPrefix(resource, "https://") +} + +// safeHeaders is a list of header fields that are +// safe to copy to push requests implicitly. It is +// assumed that requests for certain kinds of content +// would fail without these fields present. +var safeHeaders = []string{ + "Accept-Encoding", + "Accept-Language", + "Accept", + "Cache-Control", + "User-Agent", +} + +// pushHeader is a header field that gets added to push requests +// in order to avoid recursive/infinite pushes. +const pushHeader = "Caddy-Push" + +// pushedLink is the key for the variable on the request +// context that we use to remember whether we have already +// pushed resources from Link headers yet; otherwise, if +// multiple push handlers are invoked, it would repeat the +// pushing of Link headers. +const pushedLink = "http.handlers.push.pushed_link" + +// Interface guards +var ( + _ caddy.Provisioner = (*Handler)(nil) + _ caddyhttp.MiddlewareHandler = (*Handler)(nil) + _ caddyhttp.HTTPInterfaces = (*linkPusher)(nil) +) diff --git a/modules/caddyhttp/push/link.go b/modules/caddyhttp/push/link.go new file mode 100644 index 0000000..16b0e7d --- /dev/null +++ b/modules/caddyhttp/push/link.go @@ -0,0 +1,78 @@ +// Copyright 2015 Matthew Holt and The Caddy Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package push + +import ( + "strings" +) + +// linkResource contains the results of a parsed Link header. +type linkResource struct { + uri string + params map[string]string +} + +// parseLinkHeader is responsible for parsing Link header +// and returning list of found resources. +// +// Accepted formats are: +// +// Link: ; as=script +// Link: ; as=script,; as=style +// Link: ; +// +// where begins with a forward slash (/). +func parseLinkHeader(header string) []linkResource { + resources := []linkResource{} + + if header == "" { + return resources + } + + for _, link := range strings.Split(header, comma) { + l := linkResource{params: make(map[string]string)} + + li, ri := strings.Index(link, "<"), strings.Index(link, ">") + if li == -1 || ri == -1 { + continue + } + + l.uri = strings.TrimSpace(link[li+1 : ri]) + + for _, param := range strings.Split(strings.TrimSpace(link[ri+1:]), semicolon) { + parts := strings.SplitN(strings.TrimSpace(param), equal, 2) + key := strings.TrimSpace(parts[0]) + if key == "" { + continue + } + if len(parts) == 1 { + l.params[key] = key + } + if len(parts) == 2 { + l.params[key] = strings.TrimSpace(parts[1]) + } + } + + resources = append(resources, l) + } + + return resources +} + +const ( + comma = "," + semicolon = ";" + equal = "=" +) diff --git a/modules/caddyhttp/push/link_test.go b/modules/caddyhttp/push/link_test.go new file mode 100644 index 0000000..238e284 --- /dev/null +++ b/modules/caddyhttp/push/link_test.go @@ -0,0 +1,85 @@ +// Copyright 2015 Matthew Holt and The Caddy Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +package push + +import ( + "reflect" + "testing" +) + +func TestParseLinkHeader(t *testing.T) { + testCases := []struct { + header string + expectedResources []linkResource + }{ + { + header: "; as=script", + expectedResources: []linkResource{{uri: "/resource", params: map[string]string{"as": "script"}}}, + }, + { + header: "", + expectedResources: []linkResource{{uri: "/resource", params: map[string]string{}}}, + }, + { + header: "; nopush", + expectedResources: []linkResource{{uri: "/resource", params: map[string]string{"nopush": "nopush"}}}, + }, + { + header: ";nopush;rel=next", + expectedResources: []linkResource{{uri: "/resource", params: map[string]string{"nopush": "nopush", "rel": "next"}}}, + }, + { + header: ";nopush;rel=next,;nopush", + expectedResources: []linkResource{ + {uri: "/resource", params: map[string]string{"nopush": "nopush", "rel": "next"}}, + {uri: "/resource2", params: map[string]string{"nopush": "nopush"}}, + }, + }, + { + header: ",", + expectedResources: []linkResource{ + {uri: "/resource", params: map[string]string{}}, + {uri: "/resource2", params: map[string]string{}}, + }, + }, + { + header: "malformed", + expectedResources: []linkResource{}, + }, + { + header: " ; ", + expectedResources: []linkResource{{uri: "/resource", params: map[string]string{}}}, + }, + } + + for i, test := range testCases { + actualResources := parseLinkHeader(test.header) + if !reflect.DeepEqual(actualResources, test.expectedResources) { + t.Errorf("Test %d (header: %s) - expected resources %v, got %v", + i, test.header, test.expectedResources, actualResources) + } + } +} diff --git a/modules/caddyhttp/standard/imports.go b/modules/caddyhttp/standard/imports.go index dabec81..0aeef84 100644 --- a/modules/caddyhttp/standard/imports.go +++ b/modules/caddyhttp/standard/imports.go @@ -10,6 +10,7 @@ import ( _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/fileserver" _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/headers" _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/map" + _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/push" _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/requestbody" _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy" _ "github.com/caddyserver/caddy/v2/modules/caddyhttp/reverseproxy/fastcgi" -- cgit v1.2.3