From f35a7fa466ffb06c38dcb3216e30c13aa8e14ce5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steffen=20Br=C3=BCheim?= Date: Tue, 30 Mar 2021 02:47:19 +0200 Subject: encode,staticfiles: Content negotiation, precompressed files (#4045) * encode: implement prefer setting * encode: minimum_length configurable via caddyfile * encode: configurable content-types which to encode * file_server: support precompressed files * encode: use ReponseMatcher for conditional encoding of content * linting error & documentation of encode.PrecompressedOrder * encode: allow just one response matcher also change the namespace of the encoders back, I accidently changed to precompressed >.> default matchers include a * to match to any charset, that may be appended * rounding of the PR * added integration tests for new caddyfile directives * improved various doc strings (punctuation and typos) * added json tag for file_server precompress order and encode matcher * file_server: add vary header, remove accept-ranges when serving precompressed files * encode: move Suffix implementation to precompressed modules --- modules/caddyhttp/fileserver/staticfiles.go | 90 +++++++++++++++++++++++++---- 1 file changed, 79 insertions(+), 11 deletions(-) (limited to 'modules/caddyhttp/fileserver/staticfiles.go') diff --git a/modules/caddyhttp/fileserver/staticfiles.go b/modules/caddyhttp/fileserver/staticfiles.go index f58dfe0..c670788 100644 --- a/modules/caddyhttp/fileserver/staticfiles.go +++ b/modules/caddyhttp/fileserver/staticfiles.go @@ -31,6 +31,7 @@ import ( "github.com/caddyserver/caddy/v2" "github.com/caddyserver/caddy/v2/modules/caddyhttp" + "github.com/caddyserver/caddy/v2/modules/caddyhttp/encode" "go.uber.org/zap" ) @@ -79,6 +80,16 @@ type FileServer struct { // a 404 error. By default, this is false (disabled). PassThru bool `json:"pass_thru,omitempty"` + // Selection of encoders to use to check for precompressed files. + PrecompressedRaw caddy.ModuleMap `json:"precompressed,omitempty" caddy:"namespace=http.precompressed"` + + // If the client has no strong preference (q-factor), choose these encodings in order. + // If no order specified here, the first encoding from the Accept-Encoding header + // that both client and server support is used + PrecompressedOrder []string `json:"precompressed_order,omitempty"` + + precompressors map[string]encode.Precompressed + logger *zap.Logger } @@ -129,6 +140,32 @@ func (fsrv *FileServer) Provision(ctx caddy.Context) error { } } + mods, err := ctx.LoadModule(fsrv, "PrecompressedRaw") + if err != nil { + return fmt.Errorf("loading encoder modules: %v", err) + } + for modName, modIface := range mods.(map[string]interface{}) { + p, ok := modIface.(encode.Precompressed) + if !ok { + return fmt.Errorf("module %s is not precompressor", modName) + } + ae := p.AcceptEncoding() + if ae == "" { + return fmt.Errorf("precompressor does not specify an Accept-Encoding value") + } + suffix := p.Suffix() + if suffix == "" { + return fmt.Errorf("precompressor does not specify a Suffix value") + } + if _, ok := fsrv.precompressors[ae]; ok { + return fmt.Errorf("precompressor already added: %s", ae) + } + if fsrv.precompressors == nil { + fsrv.precompressors = make(map[string]encode.Precompressed) + } + fsrv.precompressors[ae] = p + } + return nil } @@ -205,8 +242,6 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c return fsrv.notFound(w, r, next) } - // TODO: content negotiation (brotli sidecar files, etc...) - // one last check to ensure the file isn't hidden (we might // have changed the filename from when we last checked) if fileHidden(filename, filesToHide) { @@ -230,18 +265,51 @@ func (fsrv *FileServer) ServeHTTP(w http.ResponseWriter, r *http.Request, next c } } - fsrv.logger.Debug("opening file", zap.String("filename", filename)) + var file *os.File - // open the file - file, err := fsrv.openFile(filename, w) - if err != nil { - if herr, ok := err.(caddyhttp.HandlerError); ok && - herr.StatusCode == http.StatusNotFound { - return fsrv.notFound(w, r, next) + // check for precompressed files + for _, ae := range encode.AcceptedEncodings(r, fsrv.PrecompressedOrder) { + precompress, ok := fsrv.precompressors[ae] + if !ok { + continue + } + compressedFilename := filename + precompress.Suffix() + compressedInfo, err := os.Stat(compressedFilename) + if err != nil || compressedInfo.IsDir() { + fsrv.logger.Debug("precompressed file not accessible", zap.String("filename", compressedFilename), zap.Error(err)) + continue + } + fsrv.logger.Debug("opening compressed sidecar file", zap.String("filename", compressedFilename), zap.Error(err)) + file, err = fsrv.openFile(compressedFilename, w) + if err != nil { + fsrv.logger.Warn("opening precompressed file failed", zap.String("filename", compressedFilename), zap.Error(err)) + if caddyErr, ok := err.(caddyhttp.HandlerError); ok && caddyErr.StatusCode == http.StatusServiceUnavailable { + return err + } + continue + } + defer file.Close() + w.Header().Set("Content-Encoding", ae) + w.Header().Del("Accept-Ranges") + w.Header().Add("Vary", "Accept-Encoding") + break + } + + // no precompressed file found, use the actual file + if file == nil { + fsrv.logger.Debug("opening file", zap.String("filename", filename)) + + // open the file + file, err = fsrv.openFile(filename, w) + if err != nil { + if herr, ok := err.(caddyhttp.HandlerError); ok && + herr.StatusCode == http.StatusNotFound { + return fsrv.notFound(w, r, next) + } + return err // error is already structured } - return err // error is already structured + defer file.Close() } - defer file.Close() // set the ETag - note that a conditional If-None-Match request is handled // by http.ServeContent below, which checks against this ETag value -- cgit v1.2.3