From fec7fa8bfda713e8042b9bbf9a480c7792b78c41 Mon Sep 17 00:00:00 2001 From: Matthew Holt Date: Mon, 20 May 2019 10:59:20 -0600 Subject: Implement most of static file server; refactor and improve Replacer --- modules/caddyhttp/staticfiles/browse.go | 205 ++++++++++++++++++ modules/caddyhttp/staticfiles/matcher.go | 54 +++++ modules/caddyhttp/staticfiles/staticfiles.go | 299 ++++++++++++++++++++++++++- 3 files changed, 555 insertions(+), 3 deletions(-) create mode 100644 modules/caddyhttp/staticfiles/browse.go create mode 100644 modules/caddyhttp/staticfiles/matcher.go (limited to 'modules/caddyhttp/staticfiles') diff --git a/modules/caddyhttp/staticfiles/browse.go b/modules/caddyhttp/staticfiles/browse.go new file mode 100644 index 0000000..15ff105 --- /dev/null +++ b/modules/caddyhttp/staticfiles/browse.go @@ -0,0 +1,205 @@ +package staticfiles + +import ( + "net/http" +) + +// Browse configures directory browsing. +type Browse struct { +} + +// ServeHTTP determines if the request is for this plugin, and if all prerequisites are met. +// If so, control is handed over to ServeListing. +func (b Browse) ServeHTTP(w http.ResponseWriter, r *http.Request) error { + // TODO: convert this handler + return nil + + // // Browse works on existing directories; delegate everything else + // requestedFilepath, err := bc.Fs.Root.Open(r.URL.Path) + // if err != nil { + // switch { + // case os.IsPermission(err): + // return http.StatusForbidden, err + // case os.IsExist(err): + // return http.StatusNotFound, err + // default: + // return b.Next.ServeHTTP(w, r) + // } + // } + // defer requestedFilepath.Close() + + // info, err := requestedFilepath.Stat() + // if err != nil { + // switch { + // case os.IsPermission(err): + // return http.StatusForbidden, err + // case os.IsExist(err): + // return http.StatusGone, err + // default: + // return b.Next.ServeHTTP(w, r) + // } + // } + // if !info.IsDir() { + // return b.Next.ServeHTTP(w, r) + // } + + // // Do not reply to anything else because it might be nonsensical + // switch r.Method { + // case http.MethodGet, http.MethodHead: + // // proceed, noop + // case "PROPFIND", http.MethodOptions: + // return http.StatusNotImplemented, nil + // default: + // return b.Next.ServeHTTP(w, r) + // } + + // // Browsing navigation gets messed up if browsing a directory + // // that doesn't end in "/" (which it should, anyway) + // u := *r.URL + // if u.Path == "" { + // u.Path = "/" + // } + // if u.Path[len(u.Path)-1] != '/' { + // u.Path += "/" + // http.Redirect(w, r, u.String(), http.StatusMovedPermanently) + // return http.StatusMovedPermanently, nil + // } + + // return b.ServeListing(w, r, requestedFilepath, bc) +} + +// func (b Browse) loadDirectoryContents(requestedFilepath http.File, urlPath string, config *Config) (*Listing, bool, error) { +// files, err := requestedFilepath.Readdir(-1) +// if err != nil { +// return nil, false, err +// } + +// // Determine if user can browse up another folder +// var canGoUp bool +// curPathDir := path.Dir(strings.TrimSuffix(urlPath, "/")) +// for _, other := range b.Configs { +// if strings.HasPrefix(curPathDir, other.PathScope) { +// canGoUp = true +// break +// } +// } + +// // Assemble listing of directory contents +// listing, hasIndex := directoryListing(files, canGoUp, urlPath, config) + +// return &listing, hasIndex, nil +// } + +// // handleSortOrder gets and stores for a Listing the 'sort' and 'order', +// // and reads 'limit' if given. The latter is 0 if not given. +// // +// // This sets Cookies. +// func (b Browse) handleSortOrder(w http.ResponseWriter, r *http.Request, scope string) (sort string, order string, limit int, err error) { +// sort, order, limitQuery := r.URL.Query().Get("sort"), r.URL.Query().Get("order"), r.URL.Query().Get("limit") + +// // If the query 'sort' or 'order' is empty, use defaults or any values previously saved in Cookies +// switch sort { +// case "": +// sort = sortByNameDirFirst +// if sortCookie, sortErr := r.Cookie("sort"); sortErr == nil { +// sort = sortCookie.Value +// } +// case sortByName, sortByNameDirFirst, sortBySize, sortByTime: +// http.SetCookie(w, &http.Cookie{Name: "sort", Value: sort, Path: scope, Secure: r.TLS != nil}) +// } + +// switch order { +// case "": +// order = "asc" +// if orderCookie, orderErr := r.Cookie("order"); orderErr == nil { +// order = orderCookie.Value +// } +// case "asc", "desc": +// http.SetCookie(w, &http.Cookie{Name: "order", Value: order, Path: scope, Secure: r.TLS != nil}) +// } + +// if limitQuery != "" { +// limit, err = strconv.Atoi(limitQuery) +// if err != nil { // if the 'limit' query can't be interpreted as a number, return err +// return +// } +// } + +// return +// } + +// // ServeListing returns a formatted view of 'requestedFilepath' contents'. +// func (b Browse) ServeListing(w http.ResponseWriter, r *http.Request, requestedFilepath http.File, bc *Config) (int, error) { +// listing, containsIndex, err := b.loadDirectoryContents(requestedFilepath, r.URL.Path, bc) +// if err != nil { +// switch { +// case os.IsPermission(err): +// return http.StatusForbidden, err +// case os.IsExist(err): +// return http.StatusGone, err +// default: +// return http.StatusInternalServerError, err +// } +// } +// if containsIndex && !b.IgnoreIndexes { // directory isn't browsable +// return b.Next.ServeHTTP(w, r) +// } +// listing.Context = httpserver.Context{ +// Root: bc.Fs.Root, +// Req: r, +// URL: r.URL, +// } +// listing.User = bc.Variables + +// // Copy the query values into the Listing struct +// var limit int +// listing.Sort, listing.Order, limit, err = b.handleSortOrder(w, r, bc.PathScope) +// if err != nil { +// return http.StatusBadRequest, err +// } + +// listing.applySort() + +// if limit > 0 && limit <= len(listing.Items) { +// listing.Items = listing.Items[:limit] +// listing.ItemsLimitedTo = limit +// } + +// var buf *bytes.Buffer +// acceptHeader := strings.ToLower(strings.Join(r.Header["Accept"], ",")) +// switch { +// case strings.Contains(acceptHeader, "application/json"): +// if buf, err = b.formatAsJSON(listing, bc); err != nil { +// return http.StatusInternalServerError, err +// } +// w.Header().Set("Content-Type", "application/json; charset=utf-8") + +// default: // There's no 'application/json' in the 'Accept' header; browse normally +// if buf, err = b.formatAsHTML(listing, bc); err != nil { +// return http.StatusInternalServerError, err +// } +// w.Header().Set("Content-Type", "text/html; charset=utf-8") + +// } + +// _, _ = buf.WriteTo(w) + +// return http.StatusOK, nil +// } + +// func (b Browse) formatAsJSON(listing *Listing, bc *Config) (*bytes.Buffer, error) { +// marsh, err := json.Marshal(listing.Items) +// if err != nil { +// return nil, err +// } + +// buf := new(bytes.Buffer) +// _, err = buf.Write(marsh) +// return buf, err +// } + +// func (b Browse) formatAsHTML(listing *Listing, bc *Config) (*bytes.Buffer, error) { +// buf := new(bytes.Buffer) +// err := bc.Template.Execute(buf, listing) +// return buf, err +// } diff --git a/modules/caddyhttp/staticfiles/matcher.go b/modules/caddyhttp/staticfiles/matcher.go new file mode 100644 index 0000000..cccf54b --- /dev/null +++ b/modules/caddyhttp/staticfiles/matcher.go @@ -0,0 +1,54 @@ +package staticfiles + +import ( + "net/http" + "os" + "path/filepath" + + "bitbucket.org/lightcodelabs/caddy2" + "bitbucket.org/lightcodelabs/caddy2/modules/caddyhttp" +) + +func init() { + caddy2.RegisterModule(caddy2.Module{ + Name: "http.matchers.file", + New: func() (interface{}, error) { return new(FileMatcher), nil }, + }) +} + +// TODO: Not sure how to do this well; we'd need the ability to +// hide files, etc... +// TODO: Also consider a feature to match directory that +// contains a certain filename (use filepath.Glob), useful +// if wanting to map directory-URI requests where the dir +// has index.php to PHP backends, for example (although this +// can effectively be done with rehandling already) +type FileMatcher struct { + Root string `json:"root"` + Path string `json:"path"` + Flags []string `json:"flags"` +} + +func (m FileMatcher) Match(r *http.Request) bool { + // TODO: sanitize path + fullPath := filepath.Join(m.Root, m.Path) + var match bool + if len(m.Flags) > 0 { + match = true + fi, err := os.Stat(fullPath) + for _, f := range m.Flags { + switch f { + case "EXIST": + match = match && os.IsNotExist(err) + case "DIR": + match = match && err == nil && fi.IsDir() + default: + match = false + } + } + } + return match +} + +// Interface guard +var _ caddyhttp.RequestMatcher = (*FileMatcher)(nil) diff --git a/modules/caddyhttp/staticfiles/staticfiles.go b/modules/caddyhttp/staticfiles/staticfiles.go index 2a6fe37..0ef3c63 100644 --- a/modules/caddyhttp/staticfiles/staticfiles.go +++ b/modules/caddyhttp/staticfiles/staticfiles.go @@ -1,7 +1,15 @@ package staticfiles import ( + "fmt" + weakrand "math/rand" "net/http" + "os" + "path" + "path/filepath" + "strconv" + "strings" + "time" "bitbucket.org/lightcodelabs/caddy2" "bitbucket.org/lightcodelabs/caddy2/modules/caddyhttp" @@ -16,13 +24,298 @@ func init() { // StaticFiles implements a static file server responder for Caddy. type StaticFiles struct { - Root string + Root string `json:"root"` // default is current directory + IndexNames []string `json:"index_names"` + Files []string `json:"files"` // all relative to the root; default is request URI path + SelectionPolicy string `json:"selection_policy"` + Fallback caddyhttp.RouteList `json:"fallback"` + Browse *Browse `json:"browse"` + Hide []string `json:"hide"` + Rehandle bool `json:"rehandle"` // issue a rehandle (internal redirect) if request is rewritten + // TODO: Etag + // TODO: Content negotiation } -func (sf StaticFiles) ServeHTTP(w http.ResponseWriter, r *http.Request) error { - http.FileServer(http.Dir(sf.Root)).ServeHTTP(w, r) +// Provision sets up the static files responder. +func (sf *StaticFiles) Provision(ctx caddy2.Context) error { + if sf.Fallback != nil { + err := sf.Fallback.Provision(ctx) + if err != nil { + return fmt.Errorf("setting up fallback routes: %v", err) + } + } + if sf.IndexNames == nil { + sf.IndexNames = defaultIndexNames + } return nil } +// Validate ensures that sf has a valid configuration. +func (sf *StaticFiles) Validate() error { + switch sf.SelectionPolicy { + case "", + "first_existing", + "largest_size", + "smallest_size", + "most_recently_modified": + default: + return fmt.Errorf("unknown selection policy %s", sf.SelectionPolicy) + } + return nil +} + +func (sf *StaticFiles) ServeHTTP(w http.ResponseWriter, r *http.Request) error { + // TODO: Prevent directory traversal, see https://play.golang.org/p/oh77BiVQFti + + // http.FileServer(http.Dir(sf.Directory)).ServeHTTP(w, r) + + ////////////// + + // TODO: Still needed? + // // Prevent absolute path access on Windows, e.g.: http://localhost:5000/C:\Windows\notepad.exe + // // TODO: does stdlib http.Dir handle this? see first check of http.Dir.Open()... + // if runtime.GOOS == "windows" && len(reqPath) > 0 && filepath.IsAbs(reqPath[1:]) { + // return caddyhttp.Error(http.StatusNotFound, fmt.Errorf("request path was absolute")) + // } + + repl := r.Context().Value(caddy2.ReplacerCtxKey).(caddy2.Replacer) + + // map the request to a filename + pathBefore := r.URL.Path + filename := sf.selectFile(r, repl) + if filename == "" { + // no files worked, so resort to fallback + if sf.Fallback != nil { + fallback := sf.Fallback.BuildCompositeRoute(w, r) + return fallback.ServeHTTP(w, r) + } + return caddyhttp.Error(http.StatusNotFound, nil) + } + + // if the ultimate destination has changed, submit + // this request for a rehandling (internal redirect) + // if configured to do so + // TODO: double check this against https://docs.nginx.com/nginx/admin-guide/web-server/serving-static-content/ + if r.URL.Path != pathBefore && sf.Rehandle { + return caddyhttp.ErrRehandle + } + + // get information about the file + info, err := os.Stat(filename) + if err != nil { + if os.IsNotExist(err) { + return caddyhttp.Error(http.StatusNotFound, err) + } else if os.IsPermission(err) { + return caddyhttp.Error(http.StatusForbidden, err) + } + // TODO: treat this as resource exhaustion like with os.Open? Or unnecessary here? + return caddyhttp.Error(http.StatusInternalServerError, err) + } + + // if the request mapped to a directory, see if + // there is an index file we can serve + if info.IsDir() && len(sf.IndexNames) > 0 { + filesToHide := sf.transformHidePaths(repl) + + for _, indexPage := range sf.IndexNames { + indexPath := path.Join(filename, indexPage) + if fileIsHidden(indexPath, filesToHide) { + // pretend this file doesn't exist + continue + } + + indexInfo, err := os.Stat(indexPath) + if err != nil { + continue + } + + // we found an index file that might work, + // so rewrite the request path and, if + // configured, do an internal redirect + // TODO: I don't know if the logic for rewriting + // the URL here is the right logic + r.URL.Path = path.Join(r.URL.Path, indexPage) + if sf.Rehandle { + return caddyhttp.ErrRehandle + } + + info = indexInfo + break + } + } + + // if still referencing a directory, delegate + // to browse or return an error + if info.IsDir() { + if sf.Browse != nil { + return sf.Browse.ServeHTTP(w, r) + } + return caddyhttp.Error(http.StatusNotFound, nil) + } + + // open the file + file, err := os.Open(info.Name()) + if err != nil { + if os.IsNotExist(err) { + return caddyhttp.Error(http.StatusNotFound, err) + } else if os.IsPermission(err) { + return caddyhttp.Error(http.StatusForbidden, err) + } + // maybe the server is under load and ran out of file descriptors? + // have client wait arbitrary seconds to help prevent a stampede + backoff := weakrand.Intn(maxBackoff-minBackoff) + minBackoff + w.Header().Set("Retry-After", strconv.Itoa(backoff)) + return caddyhttp.Error(http.StatusServiceUnavailable, err) + } + defer file.Close() + + // TODO: Right now we return an invalid response if the + // request is for a directory and there is no index file + // or dir browsing; we should return a 404 I think... + + // TODO: Etag? + + // TODO: content negotiation? (brotli sidecar files, etc...) + + // let the standard library do what it does best; note, however, + // that errors generated by ServeContent are written immediately + // to the response, so we cannot handle them (but errors here are rare) + http.ServeContent(w, r, info.Name(), info.ModTime(), file) + + return nil +} + +func (sf *StaticFiles) transformHidePaths(repl caddy2.Replacer) []string { + hide := make([]string, len(sf.Hide)) + for i := range sf.Hide { + hide[i] = repl.ReplaceAll(sf.Hide[i], "") + } + return hide +} + +func (sf *StaticFiles) selectFile(r *http.Request, repl caddy2.Replacer) string { + root := repl.ReplaceAll(sf.Root, "") + if root == "" { + root = "." + } + + if sf.Files == nil { + return filepath.Join(root, r.URL.Path) + } + + switch sf.SelectionPolicy { + // TODO: Make these policy names constants + case "", "first_existing": + filesToHide := sf.transformHidePaths(repl) + for _, f := range sf.Files { + suffix := repl.ReplaceAll(f, "") + // TODO: sanitize path + fullpath := filepath.Join(root, suffix) + if !fileIsHidden(fullpath, filesToHide) && fileExists(fullpath) { + r.URL.Path = suffix + return fullpath + } + } + + case "largest_size": + var largestSize int64 + var largestFilename string + var largestSuffix string + for _, f := range sf.Files { + suffix := repl.ReplaceAll(f, "") + // TODO: sanitize path + fullpath := filepath.Join(root, suffix) + info, err := os.Stat(fullpath) + if err == nil && info.Size() > largestSize { + largestSize = info.Size() + largestFilename = fullpath + largestSuffix = suffix + } + } + r.URL.Path = largestSuffix + return largestFilename + + case "smallest_size": + var smallestSize int64 + var smallestFilename string + var smallestSuffix string + for _, f := range sf.Files { + suffix := repl.ReplaceAll(f, "") + // TODO: sanitize path + fullpath := filepath.Join(root, suffix) + info, err := os.Stat(fullpath) + if err == nil && (smallestSize == 0 || info.Size() < smallestSize) { + smallestSize = info.Size() + smallestFilename = fullpath + smallestSuffix = suffix + } + } + r.URL.Path = smallestSuffix + return smallestFilename + + case "most_recently_modified": + var recentDate time.Time + var recentFilename string + var recentSuffix string + for _, f := range sf.Files { + suffix := repl.ReplaceAll(f, "") + // TODO: sanitize path + fullpath := filepath.Join(root, suffix) + info, err := os.Stat(fullpath) + if err == nil && + (recentDate.IsZero() || info.ModTime().After(recentDate)) { + recentDate = info.ModTime() + recentFilename = fullpath + recentSuffix = suffix + } + } + r.URL.Path = recentSuffix + return recentFilename + } + + return "" +} + +// fileExists returns true if file exists. +func fileExists(file string) bool { + _, err := os.Stat(file) + return !os.IsNotExist(err) +} + +func fileIsHidden(filename string, hide []string) bool { + nameOnly := filepath.Base(filename) + sep := string(filepath.Separator) + + // see if file is hidden + for _, h := range hide { + // assuming h is a glob/shell-like pattern, + // use it to compare the whole file path; + // but if there is no separator in h, then + // just compare against the file's name + compare := filename + if !strings.Contains(h, sep) { + compare = nameOnly + } + + hidden, err := filepath.Match(h, compare) + if err != nil { + // malformed pattern; fallback by checking prefix + if strings.HasPrefix(filename, h) { + return true + } + } + if hidden { + // file name or path matches hide pattern + return true + } + } + + return false +} + +var defaultIndexNames = []string{"index.html"} + +const minBackoff, maxBackoff = 2, 5 + // Interface guard var _ caddyhttp.Handler = (*StaticFiles)(nil) -- cgit v1.2.3