From fec7fa8bfda713e8042b9bbf9a480c7792b78c41 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Mon, 20 May 2019 10:59:20 -0600 Subject: Implement most of static file server; refactor and improve Replacer --- modules/caddyhttp/caddyhttp.go | 117 ++--------- modules/caddyhttp/caddyhttp_test.go | 11 - modules/caddyhttp/matchers.go | 68 +++--- modules/caddyhttp/matchers_test.go | 34 ++- modules/caddyhttp/replacer.go | 164 ++++++--------- modules/caddyhttp/routes.go | 53 ++++- modules/caddyhttp/server.go | 91 ++++++++ modules/caddyhttp/staticfiles/browse.go | 205 ++++++++++++++++++ modules/caddyhttp/staticfiles/matcher.go | 54 +++++ modules/caddyhttp/staticfiles/staticfiles.go | 299 ++++++++++++++++++++++++++- modules/caddyhttp/staticresp.go | 8 +- modules/caddyhttp/table.go | 41 ++++ 12 files changed, 889 insertions(+), 256 deletions(-) create mode 100644 modules/caddyhttp/server.go create mode 100644 modules/caddyhttp/staticfiles/browse.go create mode 100644 modules/caddyhttp/staticfiles/matcher.go create mode 100644 modules/caddyhttp/table.go (limited to 'modules') diff --git a/modules/caddyhttp/caddyhttp.go b/modules/caddyhttp/caddyhttp.go index 0fe9c98..449d07f 100644 --- a/modules/caddyhttp/caddyhttp.go +++ b/modules/caddyhttp/caddyhttp.go @@ -45,7 +45,13 @@ type App struct { func (app *App) Provision(ctx caddy2.Context) error { app.ctx = ctx + repl := caddy2.NewReplacer() + for _, srv := range app.Servers { + // TODO: Test this function to ensure these replacements are performed + for i := range srv.Listen { + srv.Listen[i] = repl.ReplaceAll(srv.Listen[i], "") + } err := srv.Routes.Provision(ctx) if err != nil { return fmt.Errorf("setting up server routes: %v", err) @@ -78,6 +84,13 @@ func (app *App) Validate() error { } } + // each server's max rehandle value must be valid + for srvName, srv := range app.Servers { + if srv.MaxRehandles < 0 { + return fmt.Errorf("%s: invalid max_rehandles value: %d", srvName, srv.MaxRehandles) + } + } + return nil } @@ -231,7 +244,7 @@ func (app *App) automaticHTTPS() error { redirTo += "{request.uri}" redirRoutes = append(redirRoutes, ServerRoute{ - matchers: []RouteMatcher{ + matchers: []RequestMatcher{ matchProtocol("http"), matchHost(domains), }, @@ -292,84 +305,9 @@ func (app *App) listenerTaken(network, address string) bool { var defaultALPN = []string{"h2", "http/1.1"} -// Server is an HTTP server. -type Server struct { - Listen []string `json:"listen"` - ReadTimeout caddy2.Duration `json:"read_timeout"` - ReadHeaderTimeout caddy2.Duration `json:"read_header_timeout"` - HiddenFiles []string `json:"hidden_files"` // TODO:... experimenting with shared/common state - Routes RouteList `json:"routes"` - Errors httpErrorConfig `json:"errors"` - TLSConnPolicies caddytls.ConnectionPolicies `json:"tls_connection_policies"` - DisableAutoHTTPS bool `json:"disable_auto_https"` - DisableAutoHTTPSRedir bool `json:"disable_auto_https_redir"` - - tlsApp *caddytls.TLS -} - -type httpErrorConfig struct { - Routes RouteList `json:"routes"` - // TODO: some way to configure the logging of errors, probably? standardize - // the logging configuration first. -} - -// ServeHTTP is the entry point for all HTTP requests. -func (s Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { - if s.tlsApp.HandleHTTPChallenge(w, r) { - return - } - - // set up the replacer - repl := NewReplacer(r, w) - ctx := context.WithValue(r.Context(), ReplacerCtxKey, repl) - r = r.WithContext(ctx) - - // build and execute the main middleware chain - stack := s.Routes.BuildHandlerChain(w, r) - err := executeMiddlewareChain(w, r, stack) - if err != nil { - // add the error value to the request context so - // it can be accessed by error handlers - c := context.WithValue(r.Context(), ErrorCtxKey, err) - r = r.WithContext(c) - // TODO: add error values to Replacer - - if len(s.Errors.Routes) == 0 { - // TODO: implement a default error handler? - log.Printf("[ERROR] %s", err) - } else { - errStack := s.Errors.Routes.BuildHandlerChain(w, r) - err := executeMiddlewareChain(w, r, errStack) - if err != nil { - // TODO: what should we do if the error handler has an error? - log.Printf("[ERROR] handling error: %v", err) - } - } - } -} - -// executeMiddlewareChain executes stack with w and r. This function handles -// the special ErrRehandle error value, which reprocesses requests through -// the stack again. Any error value returned from this function would be an -// actual error that needs to be handled. -func executeMiddlewareChain(w http.ResponseWriter, r *http.Request, stack Handler) error { - const maxRehandles = 3 - var err error - for i := 0; i < maxRehandles; i++ { - err = stack.ServeHTTP(w, r) - if err != ErrRehandle { - break - } - if i == maxRehandles-1 { - return fmt.Errorf("too many rehandles") - } - } - return err -} - -// RouteMatcher is a type that can match to a request. +// RequestMatcher is a type that can match to a request. // A route matcher MUST NOT modify the request. -type RouteMatcher interface { +type RequestMatcher interface { Match(*http.Request) bool } @@ -421,7 +359,6 @@ func parseListenAddr(a string) (network string, addrs []string, err error) { if err != nil { return } - host = NewReplacer(nil, nil).Replace(host, "") ports := strings.SplitN(port, "-", 2) if len(ports) == 1 { ports = append(ports, ports[0]) @@ -466,25 +403,6 @@ func joinListenAddr(network, host, port string) string { return a } -type middlewareResponseWriter struct { - *ResponseWriterWrapper - allowWrites bool -} - -func (mrw middlewareResponseWriter) WriteHeader(statusCode int) { - if !mrw.allowWrites { - panic("WriteHeader: middleware cannot write to the response") - } - mrw.ResponseWriterWrapper.WriteHeader(statusCode) -} - -func (mrw middlewareResponseWriter) Write(b []byte) (int, error) { - if !mrw.allowWrites { - panic("Write: middleware cannot write to the response") - } - return mrw.ResponseWriterWrapper.Write(b) -} - const ( // DefaultHTTPPort is the default port for HTTP. DefaultHTTPPort = 80 @@ -493,6 +411,5 @@ const ( DefaultHTTPSPort = 443 ) -// Interface guards -var _ HTTPInterfaces = middlewareResponseWriter{} +// Interface guard var _ caddy2.App = (*App)(nil) diff --git a/modules/caddyhttp/caddyhttp_test.go b/modules/caddyhttp/caddyhttp_test.go index dee3977..8d25332 100644 --- a/modules/caddyhttp/caddyhttp_test.go +++ b/modules/caddyhttp/caddyhttp_test.go @@ -1,7 +1,6 @@ package caddyhttp import ( - "os" "reflect" "testing" ) @@ -115,11 +114,6 @@ func TestJoinListenerAddr(t *testing.T) { } func TestParseListenerAddr(t *testing.T) { - hostname, err := os.Hostname() - if err != nil { - t.Fatalf("Cannot ascertain system hostname: %v", err) - } - for i, tc := range []struct { input string expectNetwork string @@ -176,11 +170,6 @@ func TestParseListenerAddr(t *testing.T) { expectNetwork: "tcp", expectAddrs: []string{"localhost:0"}, }, - { - input: "{system.hostname}:0", - expectNetwork: "tcp", - expectAddrs: []string{hostname + ":0"}, - }, } { actualNetwork, actualAddrs, err := parseListenAddr(tc.input) if tc.expectErr && err == nil { diff --git a/modules/caddyhttp/matchers.go b/modules/caddyhttp/matchers.go index 7df4d8f..e467c84 100644 --- a/modules/caddyhttp/matchers.go +++ b/modules/caddyhttp/matchers.go @@ -6,6 +6,8 @@ import ( "net/http" "net/textproto" "net/url" + "path" + "path/filepath" "regexp" "strings" @@ -15,15 +17,16 @@ import ( ) type ( - matchHost []string - matchPath []string - matchPathRE struct{ matchRegexp } - matchMethod []string - matchQuery url.Values - matchHeader http.Header - matchHeaderRE map[string]*matchRegexp - matchProtocol string - matchStarlark string + matchHost []string + matchPath []string + matchPathRE struct{ matchRegexp } + matchMethod []string + matchQuery url.Values + matchHeader http.Header + matchHeaderRE map[string]*matchRegexp + matchProtocol string + matchStarlarkExpr string + matchTable string ) func init() { @@ -60,8 +63,8 @@ func init() { New: func() (interface{}, error) { return new(matchProtocol), nil }, }) caddy2.RegisterModule(caddy2.Module{ - Name: "http.matchers.caddyscript", - New: func() (interface{}, error) { return new(matchStarlark), nil }, + Name: "http.matchers.starlark_expr", + New: func() (interface{}, error) { return new(matchStarlarkExpr), nil }, }) } @@ -91,8 +94,17 @@ outer: } func (m matchPath) Match(r *http.Request) bool { - for _, path := range m { - if strings.HasPrefix(r.URL.Path, path) { + for _, matchPath := range m { + compare := r.URL.Path + if strings.HasPrefix(matchPath, "*") { + compare = path.Base(compare) + } + // can ignore error here because we can't handle it anyway + matches, _ := filepath.Match(matchPath, compare) + if matches { + return true + } + if strings.HasPrefix(r.URL.Path, matchPath) { return true } } @@ -100,7 +112,7 @@ func (m matchPath) Match(r *http.Request) bool { } func (m matchPathRE) Match(r *http.Request) bool { - repl := r.Context().Value(ReplacerCtxKey).(*Replacer) + repl := r.Context().Value(caddy2.ReplacerCtxKey).(caddy2.Replacer) return m.match(r.URL.Path, repl, "path_regexp") } @@ -147,7 +159,7 @@ func (m matchHeader) Match(r *http.Request) bool { func (m matchHeaderRE) Match(r *http.Request) bool { for field, rm := range m { - repl := r.Context().Value(ReplacerCtxKey).(*Replacer) + repl := r.Context().Value(caddy2.ReplacerCtxKey).(caddy2.Replacer) match := rm.match(r.Header.Get(field), repl, "header_regexp") if !match { return false @@ -188,7 +200,7 @@ func (m matchProtocol) Match(r *http.Request) bool { return false } -func (m matchStarlark) Match(r *http.Request) bool { +func (m matchStarlarkExpr) Match(r *http.Request) bool { input := string(m) thread := new(starlark.Thread) env := caddyscript.MatcherEnv(r) @@ -225,7 +237,7 @@ func (mre *matchRegexp) Validate() error { return nil } -func (mre *matchRegexp) match(input string, repl *Replacer, scope string) bool { +func (mre *matchRegexp) match(input string, repl caddy2.Replacer, scope string) bool { matches := mre.compiled.FindStringSubmatch(input) if matches == nil { return false @@ -234,14 +246,14 @@ func (mre *matchRegexp) match(input string, repl *Replacer, scope string) bool { // save all capture groups, first by index for i, match := range matches { key := fmt.Sprintf("matchers.%s.%s.%d", scope, mre.Name, i) - repl.Map(key, match) + repl.Set(key, match) } // then by name for i, name := range mre.compiled.SubexpNames() { if i != 0 && name != "" { key := fmt.Sprintf("matchers.%s.%s.%s", scope, mre.Name, name) - repl.Map(key, matches[i]) + repl.Set(key, matches[i]) } } @@ -252,13 +264,13 @@ var wordRE = regexp.MustCompile(`\w+`) // Interface guards var ( - _ RouteMatcher = (*matchHost)(nil) - _ RouteMatcher = (*matchPath)(nil) - _ RouteMatcher = (*matchPathRE)(nil) - _ RouteMatcher = (*matchMethod)(nil) - _ RouteMatcher = (*matchQuery)(nil) - _ RouteMatcher = (*matchHeader)(nil) - _ RouteMatcher = (*matchHeaderRE)(nil) - _ RouteMatcher = (*matchProtocol)(nil) - _ RouteMatcher = (*matchStarlark)(nil) + _ RequestMatcher = (*matchHost)(nil) + _ RequestMatcher = (*matchPath)(nil) + _ RequestMatcher = (*matchPathRE)(nil) + _ RequestMatcher = (*matchMethod)(nil) + _ RequestMatcher = (*matchQuery)(nil) + _ RequestMatcher = (*matchHeader)(nil) + _ RequestMatcher = (*matchHeaderRE)(nil) + _ RequestMatcher = (*matchProtocol)(nil) + _ RequestMatcher = (*matchStarlarkExpr)(nil) ) diff --git a/modules/caddyhttp/matchers_test.go b/modules/caddyhttp/matchers_test.go index f23efab..c279bad 100644 --- a/modules/caddyhttp/matchers_test.go +++ b/modules/caddyhttp/matchers_test.go @@ -7,6 +7,8 @@ import ( "net/http/httptest" "net/url" "testing" + + "bitbucket.org/lightcodelabs/caddy2" ) func TestHostMatcher(t *testing.T) { @@ -131,6 +133,26 @@ func TestPathMatcher(t *testing.T) { input: "/other/", expect: true, }, + { + match: matchPath{"*.ext"}, + input: "foo.ext", + expect: true, + }, + { + match: matchPath{"*.ext"}, + input: "/foo/bar.ext", + expect: true, + }, + { + match: matchPath{"/foo/*/baz"}, + input: "/foo/bar/baz", + expect: true, + }, + { + match: matchPath{"/foo/*/baz/bam"}, + input: "/foo/bar/bam", + expect: false, + }, } { req := &http.Request{URL: &url.URL{Path: tc.input}} actual := tc.match.Match(req) @@ -205,8 +227,8 @@ func TestPathREMatcher(t *testing.T) { // set up the fake request and its Replacer req := &http.Request{URL: &url.URL{Path: tc.input}} - repl := NewReplacer(req, httptest.NewRecorder()) - ctx := context.WithValue(req.Context(), ReplacerCtxKey, repl) + repl := newReplacer(req, httptest.NewRecorder()) + ctx := context.WithValue(req.Context(), caddy2.ReplacerCtxKey, repl) req = req.WithContext(ctx) actual := tc.match.Match(req) @@ -218,7 +240,7 @@ func TestPathREMatcher(t *testing.T) { for key, expectVal := range tc.expectRepl { placeholder := fmt.Sprintf("{matchers.path_regexp.%s}", key) - actualVal := repl.Replace(placeholder, "") + actualVal := repl.ReplaceAll(placeholder, "") if actualVal != expectVal { t.Errorf("Test %d [%v]: Expected placeholder {matchers.path_regexp.%s} to be '%s' but got '%s'", i, tc.match.Pattern, key, expectVal, actualVal) @@ -322,8 +344,8 @@ func TestHeaderREMatcher(t *testing.T) { // set up the fake request and its Replacer req := &http.Request{Header: tc.input, URL: new(url.URL)} - repl := NewReplacer(req, httptest.NewRecorder()) - ctx := context.WithValue(req.Context(), ReplacerCtxKey, repl) + repl := newReplacer(req, httptest.NewRecorder()) + ctx := context.WithValue(req.Context(), caddy2.ReplacerCtxKey, repl) req = req.WithContext(ctx) actual := tc.match.Match(req) @@ -335,7 +357,7 @@ func TestHeaderREMatcher(t *testing.T) { for key, expectVal := range tc.expectRepl { placeholder := fmt.Sprintf("{matchers.header_regexp.%s}", key) - actualVal := repl.Replace(placeholder, "") + actualVal := repl.ReplaceAll(placeholder, "") if actualVal != expectVal { t.Errorf("Test %d [%v]: Expected placeholder {matchers.header_regexp.%s} to be '%s' but got '%s'", i, tc.match, key, expectVal, actualVal) diff --git a/modules/caddyhttp/replacer.go b/modules/caddyhttp/replacer.go index e7aa250..6feb143 100644 --- a/modules/caddyhttp/replacer.go +++ b/modules/caddyhttp/replacer.go @@ -1,119 +1,83 @@ package caddyhttp import ( + "fmt" "net" "net/http" - "os" + "path" "strings" "bitbucket.org/lightcodelabs/caddy2" ) -// Replacer can replace values in strings based -// on a request and/or response writer. The zero -// Replacer is not valid; use NewReplacer() to -// initialize one. -type Replacer struct { - req *http.Request - resp http.ResponseWriter - custom map[string]string -} - -// NewReplacer makes a new Replacer, initializing all necessary -// fields. The request and response writer are optional, but -// necessary for most replacements to work. -func NewReplacer(req *http.Request, rw http.ResponseWriter) *Replacer { - return &Replacer{ - req: req, - resp: rw, - custom: make(map[string]string), - } -} - -// Map sets a custom variable mapping to a value. -func (r *Replacer) Map(variable, value string) { - r.custom[variable] = value -} - -// Replace replaces placeholders in input with the value. If -// the value is empty string, the placeholder is substituted -// with the value empty. -func (r *Replacer) Replace(input, empty string) string { - if !strings.Contains(input, phOpen) { - return input - } - - input = r.replaceAll(input, empty, r.defaults()) - input = r.replaceAll(input, empty, r.custom) - - return input -} - -func (r *Replacer) replaceAll(input, empty string, mapping map[string]string) string { - for key, val := range mapping { - if val == "" { - val = empty - } - input = strings.ReplaceAll(input, phOpen+key+phClose, val) - } - return input -} - -func (r *Replacer) defaults() map[string]string { - m := map[string]string{ - "system.hostname": func() string { - // OK if there is an error; just return empty string - name, _ := os.Hostname() - return name - }(), - } - - if r.req != nil { - m["request.host"] = func() string { - host, _, err := net.SplitHostPort(r.req.Host) - if err != nil { - return r.req.Host // OK; there probably was no port +// TODO: A simple way to format or escape or encode each value would be nice +// ... TODO: Should we just use templates? :-/ yeesh... + +func newReplacer(req *http.Request, w http.ResponseWriter) caddy2.Replacer { + repl := caddy2.NewReplacer() + + httpVars := func() map[string]string { + m := make(map[string]string) + if req != nil { + m["http.request.host"] = func() string { + host, _, err := net.SplitHostPort(req.Host) + if err != nil { + return req.Host // OK; there probably was no port + } + return host + }() + m["http.request.hostport"] = req.Host // may include both host and port + m["http.request.method"] = req.Method + m["http.request.port"] = func() string { + // if there is no port, there will be an error; in + // that case, port is the empty string anyway + _, port, _ := net.SplitHostPort(req.Host) + return port + }() + m["http.request.scheme"] = func() string { + if req.TLS != nil { + return "https" + } + return "http" + }() + m["http.request.uri"] = req.URL.RequestURI() + m["http.request.uri.path"] = req.URL.Path + m["http.request.uri.path.file"] = func() string { + _, file := path.Split(req.URL.Path) + return file + }() + m["http.request.uri.path.dir"] = func() string { + dir, _ := path.Split(req.URL.Path) + return dir + }() + + for field, vals := range req.Header { + m["http.request.header."+strings.ToLower(field)] = strings.Join(vals, ",") } - return host - }() - m["request.hostport"] = r.req.Host // may include both host and port - m["request.method"] = r.req.Method - m["request.port"] = func() string { - // if there is no port, there will be an error; in - // that case, port is the empty string anyway - _, port, _ := net.SplitHostPort(r.req.Host) - return port - }() - m["request.scheme"] = func() string { - if r.req.TLS != nil { - return "https" + for _, cookie := range req.Cookies() { + m["http.request.cookie."+cookie.Name] = cookie.Value + } + for param, vals := range req.URL.Query() { + m["http.request.uri.query."+param] = strings.Join(vals, ",") } - return "http" - }() - m["request.uri"] = r.req.URL.RequestURI() - m["request.uri.path"] = r.req.URL.Path - for field, vals := range r.req.Header { - m["request.header."+strings.ToLower(field)] = strings.Join(vals, ",") - } - for _, cookie := range r.req.Cookies() { - m["request.cookie."+cookie.Name] = cookie.Value - } - for param, vals := range r.req.URL.Query() { - m["request.uri.query."+param] = strings.Join(vals, ",") + hostLabels := strings.Split(req.Host, ".") + for i, label := range hostLabels { + key := fmt.Sprintf("http.request.host.labels.%d", len(hostLabels)-i-1) + m[key] = label + } } - } - if r.resp != nil { - for field, vals := range r.resp.Header() { - m["response.header."+strings.ToLower(field)] = strings.Join(vals, ",") + if w != nil { + for field, vals := range w.Header() { + m["http.response.header."+strings.ToLower(field)] = strings.Join(vals, ",") + } } - } - return m -} + return m + } -const phOpen, phClose = "{", "}" + repl.Map(httpVars) -// ReplacerCtxKey is the context key for the request's replacer. -const ReplacerCtxKey caddy2.CtxKey = "replacer" + return repl +} diff --git a/modules/caddyhttp/routes.go b/modules/caddyhttp/routes.go index d204939..daae080 100644 --- a/modules/caddyhttp/routes.go +++ b/modules/caddyhttp/routes.go @@ -12,6 +12,7 @@ import ( // middlewares, and a responder for handling HTTP // requests. type ServerRoute struct { + Group string `json:"group"` Matchers map[string]json.RawMessage `json:"match"` Apply []json.RawMessage `json:"apply"` Respond json.RawMessage `json:"respond"` @@ -19,7 +20,7 @@ type ServerRoute struct { Terminal bool `json:"terminal"` // decoded values - matchers []RouteMatcher + matchers []RequestMatcher middleware []MiddlewareHandler responder Handler } @@ -37,7 +38,7 @@ func (routes RouteList) Provision(ctx caddy2.Context) error { if err != nil { return fmt.Errorf("loading matcher module '%s': %v", modName, err) } - routes[i].matchers = append(routes[i].matchers, val.(RouteMatcher)) + routes[i].matchers = append(routes[i].matchers, val.(RequestMatcher)) } routes[i].Matchers = nil // allow GC to deallocate - TODO: Does this help? @@ -64,9 +65,9 @@ func (routes RouteList) Provision(ctx caddy2.Context) error { return nil } -// BuildHandlerChain creates a chain of handlers by +// BuildCompositeRoute creates a chain of handlers by // applying all the matching routes. -func (routes RouteList) BuildHandlerChain(w http.ResponseWriter, r *http.Request) Handler { +func (routes RouteList) BuildCompositeRoute(w http.ResponseWriter, r *http.Request) Handler { if len(routes) == 0 { return emptyHandler } @@ -74,17 +75,39 @@ func (routes RouteList) BuildHandlerChain(w http.ResponseWriter, r *http.Request var mid []Middleware var responder Handler mrw := &middlewareResponseWriter{ResponseWriterWrapper: &ResponseWriterWrapper{w}} + groups := make(map[string]struct{}) routeLoop: for _, route := range routes { + // see if route matches for _, m := range route.matchers { if !m.Match(r) { continue routeLoop } } + + // if route is part of a group, ensure only + // the first matching route in the group is + // applied + if route.Group != "" { + _, ok := groups[route.Group] + if ok { + // this group has already been satisfied + // by a matching route + continue + } + // this matching route satisfies the group + groups[route.Group] = struct{}{} + } + + // apply the rest of the route for _, m := range route.middleware { mid = append(mid, func(next HandlerFunc) HandlerFunc { return func(w http.ResponseWriter, r *http.Request) error { + // TODO: This is where request tracing could be implemented; also + // see below to trace the responder as well + // TODO: Trace a diff of the request, would be cool too! see what changed since the last middleware (host, headers, URI...) + // TODO: see what the std lib gives us in terms of stack trracing too return m.ServeHTTP(mrw, r, next) } }) @@ -111,3 +134,25 @@ routeLoop: return stack } + +type middlewareResponseWriter struct { + *ResponseWriterWrapper + allowWrites bool +} + +func (mrw middlewareResponseWriter) WriteHeader(statusCode int) { + if !mrw.allowWrites { + panic("WriteHeader: middleware cannot write to the response") + } + mrw.ResponseWriterWrapper.WriteHeader(statusCode) +} + +func (mrw middlewareResponseWriter) Write(b []byte) (int, error) { + if !mrw.allowWrites { + panic("Write: middleware cannot write to the response") + } + return mrw.ResponseWriterWrapper.Write(b) +} + +// Interface guard +var _ HTTPInterfaces = middlewareResponseWriter{} diff --git a/modules/caddyhttp/server.go b/modules/caddyhttp/server.go new file mode 100644 index 0000000..5ab7693 --- /dev/null +++ b/modules/caddyhttp/server.go @@ -0,0 +1,91 @@ +package caddyhttp + +import ( + "context" + "fmt" + "log" + "net/http" + + "bitbucket.org/lightcodelabs/caddy2" + "bitbucket.org/lightcodelabs/caddy2/modules/caddytls" +) + +// Server is an HTTP server. +type Server struct { + Listen []string `json:"listen"` + ReadTimeout caddy2.Duration `json:"read_timeout"` + ReadHeaderTimeout caddy2.Duration `json:"read_header_timeout"` + Routes RouteList `json:"routes"` + Errors httpErrorConfig `json:"errors"` + TLSConnPolicies caddytls.ConnectionPolicies `json:"tls_connection_policies"` + DisableAutoHTTPS bool `json:"disable_auto_https"` + DisableAutoHTTPSRedir bool `json:"disable_auto_https_redir"` + MaxRehandles int `json:"max_rehandles"` + + tlsApp *caddytls.TLS +} + +// ServeHTTP is the entry point for all HTTP requests. +func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) { + if s.tlsApp.HandleHTTPChallenge(w, r) { + return + } + + // set up the replacer + repl := newReplacer(r, w) + ctx := context.WithValue(r.Context(), caddy2.ReplacerCtxKey, repl) + ctx = context.WithValue(ctx, TableCtxKey, make(map[string]interface{})) // TODO: Implement this + r = r.WithContext(ctx) + + // build and execute the main handler chain + stack := s.Routes.BuildCompositeRoute(w, r) + err := s.executeCompositeRoute(w, r, stack) + if err != nil { + // add the error value to the request context so + // it can be accessed by error handlers + c := context.WithValue(r.Context(), ErrorCtxKey, err) + r = r.WithContext(c) + // TODO: add error values to Replacer + + if len(s.Errors.Routes) == 0 { + // TODO: implement a default error handler? + log.Printf("[ERROR] %s", err) + } else { + errStack := s.Errors.Routes.BuildCompositeRoute(w, r) + err := s.executeCompositeRoute(w, r, errStack) + if err != nil { + // TODO: what should we do if the error handler has an error? + log.Printf("[ERROR] handling error: %v", err) + } + } + } +} + +// executeCompositeRoute executes stack with w and r. This function handles +// the special ErrRehandle error value, which reprocesses requests through +// the stack again. Any error value returned from this function would be an +// actual error that needs to be handled. +func (s *Server) executeCompositeRoute(w http.ResponseWriter, r *http.Request, stack Handler) error { + var err error + for i := -1; i <= s.MaxRehandles; i++ { + // we started the counter at -1 because we + // always want to run this at least once + err = stack.ServeHTTP(w, r) + if err != ErrRehandle { + break + } + if i >= s.MaxRehandles-1 { + return fmt.Errorf("too many rehandles") + } + } + return err +} + +type httpErrorConfig struct { + Routes RouteList `json:"routes"` + // TODO: some way to configure the logging of errors, probably? standardize + // the logging configuration first. +} + +// TableCtxKey is the context key for the request's variable table. +const TableCtxKey caddy2.CtxKey = "table" diff --git a/modules/caddyhttp/staticfiles/browse.go b/modules/caddyhttp/staticfiles/browse.go new file mode 100644 index 0000000..15ff105 --- /dev/null +++ b/modules/caddyhttp/staticfiles/browse.go @@ -0,0 +1,205 @@ +package staticfiles + +import ( + "net/http" +) + +// Browse configures directory browsing. +type Browse struct { +} + +// ServeHTTP determines if the request is for this plugin, and if all prerequisites are met. +// If so, control is handed over to ServeListing. +func (b Browse) ServeHTTP(w http.ResponseWriter, r *http.Request) error { + // TODO: convert this handler + return nil + + // // Browse works on existing directories; delegate everything else + // requestedFilepath, err := bc.Fs.Root.Open(r.URL.Path) + // if err != nil { + // switch { + // case os.IsPermission(err): + // return http.StatusForbidden, err + // case os.IsExist(err): + // return http.StatusNotFound, err + // default: + // return b.Next.ServeHTTP(w, r) + // } + // } + // defer requestedFilepath.Close() + + // info, err := requestedFilepath.Stat() + // if err != nil { + // switch { + // case os.IsPermission(err): + // return http.StatusForbidden, err + // case os.IsExist(err): + // return http.StatusGone, err + // default: + // return b.Next.ServeHTTP(w, r) + // } + // } + // if !info.IsDir() { + // return b.Next.ServeHTTP(w, r) + // } + + // // Do not reply to anything else because it might be nonsensical + // switch r.Method { + // case http.MethodGet, http.MethodHead: + // // proceed, noop + // case "PROPFIND", http.MethodOptions: + // return http.StatusNotImplemented, nil + // default: + // return b.Next.ServeHTTP(w, r) + // } + + // // Browsing navigation gets messed up if browsing a directory + // // that doesn't end in "/" (which it should, anyway) + // u := *r.URL + // if u.Path == "" { + // u.Path = "/" + // } + // if u.Path[len(u.Path)-1] != '/' { + // u.Path += "/" + // http.Redirect(w, r, u.String(), http.StatusMovedPermanently) + // return http.StatusMovedPermanently, nil + // } + + // return b.ServeListing(w, r, requestedFilepath, bc) +} + +// func (b Browse) loadDirectoryContents(requestedFilepath http.File, urlPath string, config *Config) (*Listing, bool, error) { +// files, err := requestedFilepath.Readdir(-1) +// if err != nil { +// return nil, false, err +// } + +// // Determine if user can browse up another folder +// var canGoUp bool +// curPathDir := path.Dir(strings.TrimSuffix(urlPath, "/")) +// for _, other := range b.Configs { +// if strings.HasPrefix(curPathDir, other.PathScope) { +// canGoUp = true +// break +// } +// } + +// // Assemble listing of directory contents +// listing, hasIndex := directoryListing(files, canGoUp, urlPath, config) + +// return &listing, hasIndex, nil +// } + +// // handleSortOrder gets and stores for a Listing the 'sort' and 'order', +// // and reads 'limit' if given. The latter is 0 if not given. +// // +// // This sets Cookies. +// func (b Browse) handleSortOrder(w http.ResponseWriter, r *http.Request, scope string) (sort string, order string, limit int, err error) { +// sort, order, limitQuery := r.URL.Query().Get("sort"), r.URL.Query().Get("order"), r.URL.Query().Get("limit") + +// // If the query 'sort' or 'order' is empty, use defaults or any values previously saved in Cookies +// switch sort { +// case "": +// sort = sortByNameDirFirst +// if sortCookie, sortErr := r.Cookie("sort"); sortErr == nil { +// sort = sortCookie.Value +// } +// case sortByName, sortByNameDirFirst, sortBySize, sortByTime: +// http.SetCookie(w, &http.Cookie{Name: "sort", Value: sort, Path: scope, Secure: r.TLS != nil}) +// } + +// switch order { +// case "": +// order = "asc" +// if orderCookie, orderErr := r.Cookie("order"); orderErr == nil { +// order = orderCookie.Value +// } +// case "asc", "desc": +// http.SetCookie(w, &http.Cookie{Name: "order", Value: order, Path: scope, Secure: r.TLS != nil}) +// } + +// if limitQuery != "" { +// limit, err = strconv.Atoi(limitQuery) +// if err != nil { // if the 'limit' query can't be interpreted as a number, return err +// return +// } +// } + +// return +// } + +// // ServeListing returns a formatted view of 'requestedFilepath' contents'. +// func (b Browse) ServeListing(w http.ResponseWriter, r *http.Request, requestedFilepath http.File, bc *Config) (int, error) { +// listing, containsIndex, err := b.loadDirectoryContents(requestedFilepath, r.URL.Path, bc) +// if err != nil { +// switch { +// case os.IsPermission(err): +// return http.StatusForbidden, err +// case os.IsExist(err): +// return http.StatusGone, err +// default: +// return http.StatusInternalServerError, err +// } +// } +// if containsIndex && !b.IgnoreIndexes { // directory isn't browsable +// return b.Next.ServeHTTP(w, r) +// } +// listing.Context = httpserver.Context{ +// Root: bc.Fs.Root, +// Req: r, +// URL: r.URL, +// } +// listing.User = bc.Variables + +// // Copy the query values into the Listing struct +// var limit int +// listing.Sort, listing.Order, limit, err = b.handleSortOrder(w, r, bc.PathScope) +// if err != nil { +// return http.StatusBadRequest, err +// } + +// listing.applySort() + +// if limit > 0 && limit <= len(listing.Items) { +// listing.Items = listing.Items[:limit] +// listing.ItemsLimitedTo = limit +// } + +// var buf *bytes.Buffer +// acceptHeader := strings.ToLower(strings.Join(r.Header["Accept"], ",")) +// switch { +// case strings.Contains(acceptHeader, "application/json"): +// if buf, err = b.formatAsJSON(listing, bc); err != nil { +// return http.StatusInternalServerError, err +// } +// w.Header().Set("Content-Type", "application/json; charset=utf-8") + +// default: // There's no 'application/json' in the 'Accept' header; browse normally +// if buf, err = b.formatAsHTML(listing, bc); err != nil { +// return http.StatusInternalServerError, err +// } +// w.Header().Set("Content-Type", "text/html; charset=utf-8") + +// } + +// _, _ = buf.WriteTo(w) + +// return http.StatusOK, nil +// } + +// func (b Browse) formatAsJSON(listing *Listing, bc *Config) (*bytes.Buffer, error) { +// marsh, err := json.Marshal(listing.Items) +// if err != nil { +// return nil, err +// } + +// buf := new(bytes.Buffer) +// _, err = buf.Write(marsh) +// return buf, err +// } + +// func (b Browse) formatAsHTML(listing *Listing, bc *Config) (*bytes.Buffer, error) { +// buf := new(bytes.Buffer) +// err := bc.Template.Execute(buf, listing) +// return buf, err +// } diff --git a/modules/caddyhttp/staticfiles/matcher.go b/modules/caddyhttp/staticfiles/matcher.go new file mode 100644 index 0000000..cccf54b --- /dev/null +++ b/modules/caddyhttp/staticfiles/matcher.go @@ -0,0 +1,54 @@ +package staticfiles + +import ( + "net/http" + "os" + "path/filepath" + + "bitbucket.org/lightcodelabs/caddy2" + "bitbucket.org/lightcodelabs/caddy2/modules/caddyhttp" +) + +func init() { + caddy2.RegisterModule(caddy2.Module{ + Name: "http.matchers.file", + New: func() (interface{}, error) { return new(FileMatcher), nil }, + }) +} + +// TODO: Not sure how to do this well; we'd need the ability to +// hide files, etc... +// TODO: Also consider a feature to match directory that +// contains a certain filename (use filepath.Glob), useful +// if wanting to map directory-URI requests where the dir +// has index.php to PHP backends, for example (although this +// can effectively be done with rehandling already) +type FileMatcher struct { + Root string `json:"root"` + Path string `json:"path"` + Flags []string `json:"flags"` +} + +func (m FileMatcher) Match(r *http.Request) bool { + // TODO: sanitize path + fullPath := filepath.Join(m.Root, m.Path) + var match bool + if len(m.Flags) > 0 { + match = true + fi, err := os.Stat(fullPath) + for _, f := range m.Flags { + switch f { + case "EXIST": + match = match && os.IsNotExist(err) + case "DIR": + match = match && err == nil && fi.IsDir() + default: + match = false + } + } + } + return match +} + +// Interface guard +var _ caddyhttp.RequestMatcher = (*FileMatcher)(nil) diff --git a/modules/caddyhttp/staticfiles/staticfiles.go b/modules/caddyhttp/staticfiles/staticfiles.go index 2a6fe37..0ef3c63 100644 --- a/modules/caddyhttp/staticfiles/staticfiles.go +++ b/modules/caddyhttp/staticfiles/staticfiles.go @@ -1,7 +1,15 @@ package staticfiles import ( + "fmt" + weakrand "math/rand" "net/http" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "time" "bitbucket.org/lightcodelabs/caddy2" "bitbucket.org/lightcodelabs/caddy2/modules/caddyhttp" @@ -16,13 +24,298 @@ func init() { // StaticFiles implements a static file server responder for Caddy. type StaticFiles struct { - Root string + Root string `json:"root"` // default is current directory + IndexNames []string `json:"index_names"` + Files []string `json:"files"` // all relative to the root; default is request URI path + SelectionPolicy string `json:"selection_policy"` + Fallback caddyhttp.RouteList `json:"fallback"` + Browse *Browse `json:"browse"` + Hide []string `json:"hide"` + Rehandle bool `json:"rehandle"` // issue a rehandle (internal redirect) if request is rewritten + // TODO: Etag + // TODO: Content negotiation } -func (sf StaticFiles) ServeHTTP(w http.ResponseWriter, r *http.Request) error { - http.FileServer(http.Dir(sf.Root)).ServeHTTP(w, r) +// Provision sets up the static files responder. +func (sf *StaticFiles) Provision(ctx caddy2.Context) error { + if sf.Fallback != nil { + err := sf.Fallback.Provision(ctx) + if err != nil { + return fmt.Errorf("setting up fallback routes: %v", err) + } + } + if sf.IndexNames == nil { + sf.IndexNames = defaultIndexNames + } return nil } +// Validate ensures that sf has a valid configuration. +func (sf *StaticFiles) Validate() error { + switch sf.SelectionPolicy { + case "", + "first_existing", + "largest_size", + "smallest_size", + "most_recently_modified": + default: + return fmt.Errorf("unknown selection policy %s", sf.SelectionPolicy) + } + return nil +} + +func (sf *StaticFiles) ServeHTTP(w http.ResponseWriter, r *http.Request) error { + // TODO: Prevent directory traversal, see https://play.golang.org/p/oh77BiVQFti + + // http.FileServer(http.Dir(sf.Directory)).ServeHTTP(w, r) + + ////////////// + + // TODO: Still needed? + // // Prevent absolute path access on Windows, e.g.: http://localhost:5000/C:\Windows\notepad.exe + // // TODO: does stdlib http.Dir handle this? see first check of http.Dir.Open()... + // if runtime.GOOS == "windows" && len(reqPath) > 0 && filepath.IsAbs(reqPath[1:]) { + // return caddyhttp.Error(http.StatusNotFound, fmt.Errorf("request path was absolute")) + // } + + repl := r.Context().Value(caddy2.ReplacerCtxKey).(caddy2.Replacer) + + // map the request to a filename + pathBefore := r.URL.Path + filename := sf.selectFile(r, repl) + if filename == "" { + // no files worked, so resort to fallback + if sf.Fallback != nil { + fallback := sf.Fallback.BuildCompositeRoute(w, r) + return fallback.ServeHTTP(w, r) + } + return caddyhttp.Error(http.StatusNotFound, nil) + } + + // if the ultimate destination has changed, submit + // this request for a rehandling (internal redirect) + // if configured to do so + // TODO: double check this against https://docs.nginx.com/nginx/admin-guide/web-server/serving-static-content/ + if r.URL.Path != pathBefore && sf.Rehandle { + return caddyhttp.ErrRehandle + } + + // get information about the file + info, err := os.Stat(filename) + if err != nil { + if os.IsNotExist(err) { + return caddyhttp.Error(http.StatusNotFound, err) + } else if os.IsPermission(err) { + return caddyhttp.Error(http.StatusForbidden, err) + } + // TODO: treat this as resource exhaustion like with os.Open? Or unnecessary here? + return caddyhttp.Error(http.StatusInternalServerError, err) + } + + // if the request mapped to a directory, see if + // there is an index file we can serve + if info.IsDir() && len(sf.IndexNames) > 0 { + filesToHide := sf.transformHidePaths(repl) + + for _, indexPage := range sf.IndexNames { + indexPath := path.Join(filename, indexPage) + if fileIsHidden(indexPath, filesToHide) { + // pretend this file doesn't exist + continue + } + + indexInfo, err := os.Stat(indexPath) + if err != nil { + continue + } + + // we found an index file that might work, + // so rewrite the request path and, if + // configured, do an internal redirect + // TODO: I don't know if the logic for rewriting + // the URL here is the right logic + r.URL.Path = path.Join(r.URL.Path, indexPage) + if sf.Rehandle { + return caddyhttp.ErrRehandle + } + + info = indexInfo + break + } + } + + // if still referencing a directory, delegate + // to browse or return an error + if info.IsDir() { + if sf.Browse != nil { + return sf.Browse.ServeHTTP(w, r) + } + return caddyhttp.Error(http.StatusNotFound, nil) + } + + // open the file + file, err := os.Open(info.Name()) + if err != nil { + if os.IsNotExist(err) { + return caddyhttp.Error(http.StatusNotFound, err) + } else if os.IsPermission(err) { + return caddyhttp.Error(http.StatusForbidden, err) + } + // maybe the server is under load and ran out of file descriptors? + // have client wait arbitrary seconds to help prevent a stampede + backoff := weakrand.Intn(maxBackoff-minBackoff) + minBackoff + w.Header().Set("Retry-After", strconv.Itoa(backoff)) + return caddyhttp.Error(http.StatusServiceUnavailable, err) + } + defer file.Close() + + // TODO: Right now we return an invalid response if the + // request is for a directory and there is no index file + // or dir browsing; we should return a 404 I think... + + // TODO: Etag? + + // TODO: content negotiation? (brotli sidecar files, etc...) + + // let the standard library do what it does best; note, however, + // that errors generated by ServeContent are written immediately + // to the response, so we cannot handle them (but errors here are rare) + http.ServeContent(w, r, info.Name(), info.ModTime(), file) + + return nil +} + +func (sf *StaticFiles) transformHidePaths(repl caddy2.Replacer) []string { + hide := make([]string, len(sf.Hide)) + for i := range sf.Hide { + hide[i] = repl.ReplaceAll(sf.Hide[i], "") + } + return hide +} + +func (sf *StaticFiles) selectFile(r *http.Request, repl caddy2.Replacer) string { + root := repl.ReplaceAll(sf.Root, "") + if root == "" { + root = "." + } + + if sf.Files == nil { + return filepath.Join(root, r.URL.Path) + } + + switch sf.SelectionPolicy { + // TODO: Make these policy names constants + case "", "first_existing": + filesToHide := sf.transformHidePaths(repl) + for _, f := range sf.Files { + suffix := repl.ReplaceAll(f, "") + // TODO: sanitize path + fullpath := filepath.Join(root, suffix) + if !fileIsHidden(fullpath, filesToHide) && fileExists(fullpath) { + r.URL.Path = suffix + return fullpath + } + } + + case "largest_size": + var largestSize int64 + var largestFilename string + var largestSuffix string + for _, f := range sf.Files { + suffix := repl.ReplaceAll(f, "") + // TODO: sanitize path + fullpath := filepath.Join(root, suffix) + info, err := os.Stat(fullpath) + if err == nil && info.Size() > largestSize { + largestSize = info.Size() + largestFilename = fullpath + largestSuffix = suffix + } + } + r.URL.Path = largestSuffix + return largestFilename + + case "smallest_size": + var smallestSize int64 + var smallestFilename string + var smallestSuffix string + for _, f := range sf.Files { + suffix := repl.ReplaceAll(f, "") + // TODO: sanitize path + fullpath := filepath.Join(root, suffix) + info, err := os.Stat(fullpath) + if err == nil && (smallestSize == 0 || info.Size() < smallestSize) { + smallestSize = info.Size() + smallestFilename = fullpath + smallestSuffix = suffix + } + } + r.URL.Path = smallestSuffix + return smallestFilename + + case "most_recently_modified": + var recentDate time.Time + var recentFilename string + var recentSuffix string + for _, f := range sf.Files { + suffix := repl.ReplaceAll(f, "") + // TODO: sanitize path + fullpath := filepath.Join(root, suffix) + info, err := os.Stat(fullpath) + if err == nil && + (recentDate.IsZero() || info.ModTime().After(recentDate)) { + recentDate = info.ModTime() + recentFilename = fullpath + recentSuffix = suffix + } + } + r.URL.Path = recentSuffix + return recentFilename + } + + return "" +} + +// fileExists returns true if file exists. +func fileExists(file string) bool { + _, err := os.Stat(file) + return !os.IsNotExist(err) +} + +func fileIsHidden(filename string, hide []string) bool { + nameOnly := filepath.Base(filename) + sep := string(filepath.Separator) + + // see if file is hidden + for _, h := range hide { + // assuming h is a glob/shell-like pattern, + // use it to compare the whole file path; + // but if there is no separator in h, then + // just compare against the file's name + compare := filename + if !strings.Contains(h, sep) { + compare = nameOnly + } + + hidden, err := filepath.Match(h, compare) + if err != nil { + // malformed pattern; fallback by checking prefix + if strings.HasPrefix(filename, h) { + return true + } + } + if hidden { + // file name or path matches hide pattern + return true + } + } + + return false +} + +var defaultIndexNames = []string{"index.html"} + +const minBackoff, maxBackoff = 2, 5 + // Interface guard var _ caddyhttp.Handler = (*StaticFiles)(nil) diff --git a/modules/caddyhttp/staticresp.go b/modules/caddyhttp/staticresp.go index 506689a..69ec45b 100644 --- a/modules/caddyhttp/staticresp.go +++ b/modules/caddyhttp/staticresp.go @@ -23,16 +23,16 @@ type Static struct { } func (s Static) ServeHTTP(w http.ResponseWriter, r *http.Request) error { - repl := r.Context().Value(ReplacerCtxKey).(*Replacer) + repl := r.Context().Value(caddy2.ReplacerCtxKey).(caddy2.Replacer) // close the connection after responding r.Close = s.Close // set all headers, with replacements for field, vals := range s.Headers { - field = repl.Replace(field, "") + field = repl.ReplaceAll(field, "") for i := range vals { - vals[i] = repl.Replace(vals[i], "") + vals[i] = repl.ReplaceAll(vals[i], "") } w.Header()[field] = vals } @@ -46,7 +46,7 @@ func (s Static) ServeHTTP(w http.ResponseWriter, r *http.Request) error { // write the response body, with replacements if s.Body != "" { - fmt.Fprint(w, repl.Replace(s.Body, "")) + fmt.Fprint(w, repl.ReplaceAll(s.Body, "")) } return nil diff --git a/modules/caddyhttp/table.go b/modules/caddyhttp/table.go new file mode 100644 index 0000000..8c3ebe0 --- /dev/null +++ b/modules/caddyhttp/table.go @@ -0,0 +1,41 @@ +package caddyhttp + +import ( + "net/http" + + "bitbucket.org/lightcodelabs/caddy2" +) + +func init() { + caddy2.RegisterModule(caddy2.Module{ + Name: "http.middleware.table", + New: func() (interface{}, error) { return new(tableMiddleware), nil }, + }) + + caddy2.RegisterModule(caddy2.Module{ + Name: "http.matchers.table", + New: func() (interface{}, error) { return new(tableMatcher), nil }, + }) +} + +type tableMiddleware struct { +} + +func (t tableMiddleware) ServeHTTP(w http.ResponseWriter, r *http.Request, next Handler) error { + // tbl := r.Context().Value(TableCtxKey).(map[string]interface{}) + + // TODO: implement this... + + return nil +} + +type tableMatcher struct { +} + +func (m tableMatcher) Match(r *http.Request) bool { + return false // TODO: implement +} + +// Interface guards +var _ MiddlewareHandler = (*tableMiddleware)(nil) +var _ RequestMatcher = (*tableMatcher)(nil) -- cgit v1.2.3