Implement most of static file server; refactor and improve Replacer

3 files changed, 555 insertions, 3 deletions

diff --git a/modules/caddyhttp/staticfiles/browse.go b/modules/caddyhttp/staticfiles/browse.go
new file mode 100644
index 0000000..15ff105
--- /dev/null
+++ b/modules/caddyhttp/staticfiles/browse.go
@@ -0,0 +1,205 @@
+package staticfiles
+
+import (
+	"net/http"
+)
+
+// Browse configures directory browsing.
+type Browse struct {
+}
+
+// ServeHTTP determines if the request is for this plugin, and if all prerequisites are met.
+// If so, control is handed over to ServeListing.
+func (b Browse) ServeHTTP(w http.ResponseWriter, r *http.Request) error {
+	// TODO: convert this handler
+	return nil
+
+	// // Browse works on existing directories; delegate everything else
+	// requestedFilepath, err := bc.Fs.Root.Open(r.URL.Path)
+	// if err != nil {
+	// 	switch {
+	// 	case os.IsPermission(err):
+	// 		return http.StatusForbidden, err
+	// 	case os.IsExist(err):
+	// 		return http.StatusNotFound, err
+	// 	default:
+	// 		return b.Next.ServeHTTP(w, r)
+	// 	}
+	// }
+	// defer requestedFilepath.Close()
+
+	// info, err := requestedFilepath.Stat()
+	// if err != nil {
+	// 	switch {
+	// 	case os.IsPermission(err):
+	// 		return http.StatusForbidden, err
+	// 	case os.IsExist(err):
+	// 		return http.StatusGone, err
+	// 	default:
+	// 		return b.Next.ServeHTTP(w, r)
+	// 	}
+	// }
+	// if !info.IsDir() {
+	// 	return b.Next.ServeHTTP(w, r)
+	// }
+
+	// // Do not reply to anything else because it might be nonsensical
+	// switch r.Method {
+	// case http.MethodGet, http.MethodHead:
+	// 	// proceed, noop
+	// case "PROPFIND", http.MethodOptions:
+	// 	return http.StatusNotImplemented, nil
+	// default:
+	// 	return b.Next.ServeHTTP(w, r)
+	// }
+
+	// // Browsing navigation gets messed up if browsing a directory
+	// // that doesn't end in "/" (which it should, anyway)
+	// u := *r.URL
+	// if u.Path == "" {
+	// 	u.Path = "/"
+	// }
+	// if u.Path[len(u.Path)-1] != '/' {
+	// 	u.Path += "/"
+	// 	http.Redirect(w, r, u.String(), http.StatusMovedPermanently)
+	// 	return http.StatusMovedPermanently, nil
+	// }
+
+	// return b.ServeListing(w, r, requestedFilepath, bc)
+}
+
+// func (b Browse) loadDirectoryContents(requestedFilepath http.File, urlPath string, config *Config) (*Listing, bool, error) {
+// 	files, err := requestedFilepath.Readdir(-1)
+// 	if err != nil {
+// 		return nil, false, err
+// 	}
+
+// 	// Determine if user can browse up another folder
+// 	var canGoUp bool
+// 	curPathDir := path.Dir(strings.TrimSuffix(urlPath, "/"))
+// 	for _, other := range b.Configs {
+// 		if strings.HasPrefix(curPathDir, other.PathScope) {
+// 			canGoUp = true
+// 			break
+// 		}
+// 	}
+
+// 	// Assemble listing of directory contents
+// 	listing, hasIndex := directoryListing(files, canGoUp, urlPath, config)
+
+// 	return &listing, hasIndex, nil
+// }
+
+// // handleSortOrder gets and stores for a Listing the 'sort' and 'order',
+// // and reads 'limit' if given. The latter is 0 if not given.
+// //
+// // This sets Cookies.
+// func (b Browse) handleSortOrder(w http.ResponseWriter, r *http.Request, scope string) (sort string, order string, limit int, err error) {
+// 	sort, order, limitQuery := r.URL.Query().Get("sort"), r.URL.Query().Get("order"), r.URL.Query().Get("limit")
+
+// 	// If the query 'sort' or 'order' is empty, use defaults or any values previously saved in Cookies
+// 	switch sort {
+// 	case "":
+// 		sort = sortByNameDirFirst
+// 		if sortCookie, sortErr := r.Cookie("sort"); sortErr == nil {
+// 			sort = sortCookie.Value
+// 		}
+// 	case sortByName, sortByNameDirFirst, sortBySize, sortByTime:
+// 		http.SetCookie(w, &http.Cookie{Name: "sort", Value: sort, Path: scope, Secure: r.TLS != nil})
+// 	}
+
+// 	switch order {
+// 	case "":
+// 		order = "asc"
+// 		if orderCookie, orderErr := r.Cookie("order"); orderErr == nil {
+// 			order = orderCookie.Value
+// 		}
+// 	case "asc", "desc":
+// 		http.SetCookie(w, &http.Cookie{Name: "order", Value: order, Path: scope, Secure: r.TLS != nil})
+// 	}
+
+// 	if limitQuery != "" {
+// 		limit, err = strconv.Atoi(limitQuery)
+// 		if err != nil { // if the 'limit' query can't be interpreted as a number, return err
+// 			return
+// 		}
+// 	}
+
+// 	return
+// }
+
+// // ServeListing returns a formatted view of 'requestedFilepath' contents'.
+// func (b Browse) ServeListing(w http.ResponseWriter, r *http.Request, requestedFilepath http.File, bc *Config) (int, error) {
+// 	listing, containsIndex, err := b.loadDirectoryContents(requestedFilepath, r.URL.Path, bc)
+// 	if err != nil {
+// 		switch {
+// 		case os.IsPermission(err):
+// 			return http.StatusForbidden, err
+// 		case os.IsExist(err):
+// 			return http.StatusGone, err
+// 		default:
+// 			return http.StatusInternalServerError, err
+// 		}
+// 	}
+// 	if containsIndex && !b.IgnoreIndexes { // directory isn't browsable
+// 		return b.Next.ServeHTTP(w, r)
+// 	}
+// 	listing.Context = httpserver.Context{
+// 		Root: bc.Fs.Root,
+// 		Req:  r,
+// 		URL:  r.URL,
+// 	}
+// 	listing.User = bc.Variables
+
+// 	// Copy the query values into the Listing struct
+// 	var limit int
+// 	listing.Sort, listing.Order, limit, err = b.handleSortOrder(w, r, bc.PathScope)
+// 	if err != nil {
+// 		return http.StatusBadRequest, err
+// 	}
+
+// 	listing.applySort()
+
+// 	if limit > 0 && limit <= len(listing.Items) {
+// 		listing.Items = listing.Items[:limit]
+// 		listing.ItemsLimitedTo = limit
+// 	}
+
+// 	var buf *bytes.Buffer
+// 	acceptHeader := strings.ToLower(strings.Join(r.Header["Accept"], ","))
+// 	switch {
+// 	case strings.Contains(acceptHeader, "application/json"):
+// 		if buf, err = b.formatAsJSON(listing, bc); err != nil {
+// 			return http.StatusInternalServerError, err
+// 		}
+// 		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+
+// 	default: // There's no 'application/json' in the 'Accept' header; browse normally
+// 		if buf, err = b.formatAsHTML(listing, bc); err != nil {
+// 			return http.StatusInternalServerError, err
+// 		}
+// 		w.Header().Set("Content-Type", "text/html; charset=utf-8")
+
+// 	}
+
+// 	_, _ = buf.WriteTo(w)
+
+// 	return http.StatusOK, nil
+// }
+
+// func (b Browse) formatAsJSON(listing *Listing, bc *Config) (*bytes.Buffer, error) {
+// 	marsh, err := json.Marshal(listing.Items)
+// 	if err != nil {
+// 		return nil, err
+// 	}
+
+// 	buf := new(bytes.Buffer)
+// 	_, err = buf.Write(marsh)
+// 	return buf, err
+// }
+
+// func (b Browse) formatAsHTML(listing *Listing, bc *Config) (*bytes.Buffer, error) {
+// 	buf := new(bytes.Buffer)
+// 	err := bc.Template.Execute(buf, listing)
+// 	return buf, err
+// }
diff --git a/modules/caddyhttp/staticfiles/matcher.go b/modules/caddyhttp/staticfiles/matcher.go
new file mode 100644
index 0000000..cccf54b
--- /dev/null
+++ b/modules/caddyhttp/staticfiles/matcher.go
@@ -0,0 +1,54 @@
+package staticfiles
+
+import (
+	"net/http"
+	"os"
+	"path/filepath"
+
+	"bitbucket.org/lightcodelabs/caddy2"
+	"bitbucket.org/lightcodelabs/caddy2/modules/caddyhttp"
+)
+
+func init() {
+	caddy2.RegisterModule(caddy2.Module{
+		Name: "http.matchers.file",
+		New:  func() (interface{}, error) { return new(FileMatcher), nil },
+	})
+}
+
+// TODO: Not sure how to do this well; we'd need the ability to
+// hide files, etc...
+// TODO: Also consider a feature to match directory that
+// contains a certain filename (use filepath.Glob), useful
+// if wanting to map directory-URI requests where the dir
+// has index.php to PHP backends, for example (although this
+// can effectively be done with rehandling already)
+type FileMatcher struct {
+	Root  string   `json:"root"`
+	Path  string   `json:"path"`
+	Flags []string `json:"flags"`
+}
+
+func (m FileMatcher) Match(r *http.Request) bool {
+	// TODO: sanitize path
+	fullPath := filepath.Join(m.Root, m.Path)
+	var match bool
+	if len(m.Flags) > 0 {
+		match = true
+		fi, err := os.Stat(fullPath)
+		for _, f := range m.Flags {
+			switch f {
+			case "EXIST":
+				match = match && os.IsNotExist(err)
+			case "DIR":
+				match = match && err == nil && fi.IsDir()
+			default:
+				match = false
+			}
+		}
+	}
+	return match
+}
+
+// Interface guard
+var _ caddyhttp.RequestMatcher = (*FileMatcher)(nil)
diff --git a/modules/caddyhttp/staticfiles/staticfiles.go b/modules/caddyhttp/staticfiles/staticfiles.go
index 2a6fe37..0ef3c63 100644
--- a/modules/caddyhttp/staticfiles/staticfiles.go
+++ b/modules/caddyhttp/staticfiles/staticfiles.go
@@ -1,7 +1,15 @@
 package staticfiles
 
 import (
+	"fmt"
+	weakrand "math/rand"
 	"net/http"
+	"os"
+	"path"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
 
 	"bitbucket.org/lightcodelabs/caddy2"
 	"bitbucket.org/lightcodelabs/caddy2/modules/caddyhttp"
@@ -16,13 +24,298 @@ func init() {
 
 // StaticFiles implements a static file server responder for Caddy.
 type StaticFiles struct {
-	Root string
+	Root            string              `json:"root"` // default is current directory
+	IndexNames      []string            `json:"index_names"`
+	Files           []string            `json:"files"` // all relative to the root; default is request URI path
+	SelectionPolicy string              `json:"selection_policy"`
+	Fallback        caddyhttp.RouteList `json:"fallback"`
+	Browse          *Browse             `json:"browse"`
+	Hide            []string            `json:"hide"`
+	Rehandle        bool                `json:"rehandle"` // issue a rehandle (internal redirect) if request is rewritten
+	// TODO: Etag
+	// TODO: Content negotiation
 }
 
-func (sf StaticFiles) ServeHTTP(w http.ResponseWriter, r *http.Request) error {
-	http.FileServer(http.Dir(sf.Root)).ServeHTTP(w, r)
+// Provision sets up the static files responder.
+func (sf *StaticFiles) Provision(ctx caddy2.Context) error {
+	if sf.Fallback != nil {
+		err := sf.Fallback.Provision(ctx)
+		if err != nil {
+			return fmt.Errorf("setting up fallback routes: %v", err)
+		}
+	}
+	if sf.IndexNames == nil {
+		sf.IndexNames = defaultIndexNames
+	}
 	return nil
 }
 
+// Validate ensures that sf has a valid configuration.
+func (sf *StaticFiles) Validate() error {
+	switch sf.SelectionPolicy {
+	case "",
+		"first_existing",
+		"largest_size",
+		"smallest_size",
+		"most_recently_modified":
+	default:
+		return fmt.Errorf("unknown selection policy %s", sf.SelectionPolicy)
+	}
+	return nil
+}
+
+func (sf *StaticFiles) ServeHTTP(w http.ResponseWriter, r *http.Request) error {
+	// TODO: Prevent directory traversal, see https://play.golang.org/p/oh77BiVQFti
+
+	// http.FileServer(http.Dir(sf.Directory)).ServeHTTP(w, r)
+
+	//////////////
+
+	// TODO: Still needed?
+	// // Prevent absolute path access on Windows, e.g.: http://localhost:5000/C:\Windows\notepad.exe
+	// // TODO: does stdlib http.Dir handle this? see first check of http.Dir.Open()...
+	// if runtime.GOOS == "windows" && len(reqPath) > 0 && filepath.IsAbs(reqPath[1:]) {
+	// 	return caddyhttp.Error(http.StatusNotFound, fmt.Errorf("request path was absolute"))
+	// }
+
+	repl := r.Context().Value(caddy2.ReplacerCtxKey).(caddy2.Replacer)
+
+	// map the request to a filename
+	pathBefore := r.URL.Path
+	filename := sf.selectFile(r, repl)
+	if filename == "" {
+		// no files worked, so resort to fallback
+		if sf.Fallback != nil {
+			fallback := sf.Fallback.BuildCompositeRoute(w, r)
+			return fallback.ServeHTTP(w, r)
+		}
+		return caddyhttp.Error(http.StatusNotFound, nil)
+	}
+
+	// if the ultimate destination has changed, submit
+	// this request for a rehandling (internal redirect)
+	// if configured to do so
+	// TODO: double check this against https://docs.nginx.com/nginx/admin-guide/web-server/serving-static-content/
+	if r.URL.Path != pathBefore && sf.Rehandle {
+		return caddyhttp.ErrRehandle
+	}
+
+	// get information about the file
+	info, err := os.Stat(filename)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return caddyhttp.Error(http.StatusNotFound, err)
+		} else if os.IsPermission(err) {
+			return caddyhttp.Error(http.StatusForbidden, err)
+		}
+		// TODO: treat this as resource exhaustion like with os.Open? Or unnecessary here?
+		return caddyhttp.Error(http.StatusInternalServerError, err)
+	}
+
+	// if the request mapped to a directory, see if
+	// there is an index file we can serve
+	if info.IsDir() && len(sf.IndexNames) > 0 {
+		filesToHide := sf.transformHidePaths(repl)
+
+		for _, indexPage := range sf.IndexNames {
+			indexPath := path.Join(filename, indexPage)
+			if fileIsHidden(indexPath, filesToHide) {
+				// pretend this file doesn't exist
+				continue
+			}
+
+			indexInfo, err := os.Stat(indexPath)
+			if err != nil {
+				continue
+			}
+
+			// we found an index file that might work,
+			// so rewrite the request path and, if
+			// configured, do an internal redirect
+			// TODO: I don't know if the logic for rewriting
+			// the URL here is the right logic
+			r.URL.Path = path.Join(r.URL.Path, indexPage)
+			if sf.Rehandle {
+				return caddyhttp.ErrRehandle
+			}
+
+			info = indexInfo
+			break
+		}
+	}
+
+	// if still referencing a directory, delegate
+	// to browse or return an error
+	if info.IsDir() {
+		if sf.Browse != nil {
+			return sf.Browse.ServeHTTP(w, r)
+		}
+		return caddyhttp.Error(http.StatusNotFound, nil)
+	}
+
+	// open the file
+	file, err := os.Open(info.Name())
+	if err != nil {
+		if os.IsNotExist(err) {
+			return caddyhttp.Error(http.StatusNotFound, err)
+		} else if os.IsPermission(err) {
+			return caddyhttp.Error(http.StatusForbidden, err)
+		}
+		// maybe the server is under load and ran out of file descriptors?
+		// have client wait arbitrary seconds to help prevent a stampede
+		backoff := weakrand.Intn(maxBackoff-minBackoff) + minBackoff
+		w.Header().Set("Retry-After", strconv.Itoa(backoff))
+		return caddyhttp.Error(http.StatusServiceUnavailable, err)
+	}
+	defer file.Close()
+
+	// TODO: Right now we return an invalid response if the
+	// request is for a directory and there is no index file
+	// or dir browsing; we should return a 404 I think...
+
+	// TODO: Etag?
+
+	// TODO: content negotiation? (brotli sidecar files, etc...)
+
+	// let the standard library do what it does best; note, however,
+	// that errors generated by ServeContent are written immediately
+	// to the response, so we cannot handle them (but errors here are rare)
+	http.ServeContent(w, r, info.Name(), info.ModTime(), file)
+
+	return nil
+}
+
+func (sf *StaticFiles) transformHidePaths(repl caddy2.Replacer) []string {
+	hide := make([]string, len(sf.Hide))
+	for i := range sf.Hide {
+		hide[i] = repl.ReplaceAll(sf.Hide[i], "")
+	}
+	return hide
+}
+
+func (sf *StaticFiles) selectFile(r *http.Request, repl caddy2.Replacer) string {
+	root := repl.ReplaceAll(sf.Root, "")
+	if root == "" {
+		root = "."
+	}
+
+	if sf.Files == nil {
+		return filepath.Join(root, r.URL.Path)
+	}
+
+	switch sf.SelectionPolicy {
+	// TODO: Make these policy names constants
+	case "", "first_existing":
+		filesToHide := sf.transformHidePaths(repl)
+		for _, f := range sf.Files {
+			suffix := repl.ReplaceAll(f, "")
+			// TODO: sanitize path
+			fullpath := filepath.Join(root, suffix)
+			if !fileIsHidden(fullpath, filesToHide) && fileExists(fullpath) {
+				r.URL.Path = suffix
+				return fullpath
+			}
+		}
+
+	case "largest_size":
+		var largestSize int64
+		var largestFilename string
+		var largestSuffix string
+		for _, f := range sf.Files {
+			suffix := repl.ReplaceAll(f, "")
+			// TODO: sanitize path
+			fullpath := filepath.Join(root, suffix)
+			info, err := os.Stat(fullpath)
+			if err == nil && info.Size() > largestSize {
+				largestSize = info.Size()
+				largestFilename = fullpath
+				largestSuffix = suffix
+			}
+		}
+		r.URL.Path = largestSuffix
+		return largestFilename
+
+	case "smallest_size":
+		var smallestSize int64
+		var smallestFilename string
+		var smallestSuffix string
+		for _, f := range sf.Files {
+			suffix := repl.ReplaceAll(f, "")
+			// TODO: sanitize path
+			fullpath := filepath.Join(root, suffix)
+			info, err := os.Stat(fullpath)
+			if err == nil && (smallestSize == 0 || info.Size() < smallestSize) {
+				smallestSize = info.Size()
+				smallestFilename = fullpath
+				smallestSuffix = suffix
+			}
+		}
+		r.URL.Path = smallestSuffix
+		return smallestFilename
+
+	case "most_recently_modified":
+		var recentDate time.Time
+		var recentFilename string
+		var recentSuffix string
+		for _, f := range sf.Files {
+			suffix := repl.ReplaceAll(f, "")
+			// TODO: sanitize path
+			fullpath := filepath.Join(root, suffix)
+			info, err := os.Stat(fullpath)
+			if err == nil &&
+				(recentDate.IsZero() || info.ModTime().After(recentDate)) {
+				recentDate = info.ModTime()
+				recentFilename = fullpath
+				recentSuffix = suffix
+			}
+		}
+		r.URL.Path = recentSuffix
+		return recentFilename
+	}
+
+	return ""
+}
+
+// fileExists returns true if file exists.
+func fileExists(file string) bool {
+	_, err := os.Stat(file)
+	return !os.IsNotExist(err)
+}
+
+func fileIsHidden(filename string, hide []string) bool {
+	nameOnly := filepath.Base(filename)
+	sep := string(filepath.Separator)
+
+	// see if file is hidden
+	for _, h := range hide {
+		// assuming h is a glob/shell-like pattern,
+		// use it to compare the whole file path;
+		// but if there is no separator in h, then
+		// just compare against the file's name
+		compare := filename
+		if !strings.Contains(h, sep) {
+			compare = nameOnly
+		}
+
+		hidden, err := filepath.Match(h, compare)
+		if err != nil {
+			// malformed pattern; fallback by checking prefix
+			if strings.HasPrefix(filename, h) {
+				return true
+			}
+		}
+		if hidden {
+			// file name or path matches hide pattern
+			return true
+		}
+	}
+
+	return false
+}
+
+var defaultIndexNames = []string{"index.html"}
+
+const minBackoff, maxBackoff = 2, 5
+
 // Interface guard
 var _ caddyhttp.Handler = (*StaticFiles)(nil)