Documentation ¶
Overview ¶
Package diskcache provides a http.RoundTripper implementation that can minify, compress, and cache HTTP responses retrieved using a standard http.Client on disk. Provides ability to define custom retention and storage policies depending on the host, path, or other URL components.
Package diskcache does not aim to work as a on-disk HTTP proxy -- see github.com/gregjones/httpcache for a HTTP transport (http.RoundTripper) implementation that provides a RFC 7234 compliant cache.
See _example/example.go for a more complete example.
Example ¶
Example demonstrates setting up a simple diskcache for use with a http.Client.
package main import ( "bytes" "fmt" "log" "net/http" "net/http/httptest" "net/http/httputil" "time" "github.com/kenshaw/diskcache" ) func main() { // set up simple test server for demonstration s := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { res.Header().Set("content-type", "text/html") res.Header().Set("X-Header", "test") fmt.Fprintf(res, `<!doctype html> <html lang="en"> <head> </head> <body attribute="value"> <p> hello %s! </p> <div> something </div> <a href="http://example.com/full/path">a link!</a> </body> </html> `, req.URL.Query().Get("name")) })) defer s.Close() // create disk cache d, err := diskcache.New( // diskcache.WithBasePathFs("/path/to/cacheDir"), diskcache.WithAppCacheDir("diskcache-test"), diskcache.WithHeaderBlacklist("Set-Cookie", "Date"), diskcache.WithMinifier(), diskcache.WithErrorTruncator(), diskcache.WithGzipCompression(), diskcache.WithTTL(365*24*time.Hour), ) if err != nil { log.Fatal(err) } // build and execute request cl := &http.Client{Transport: d} req, err := http.NewRequest("GET", s.URL+"/hello?name=ken", nil) if err != nil { log.Fatal(err) } res, err := cl.Do(req) if err != nil { log.Fatal(err) } defer res.Body.Close() // dump buf, err := httputil.DumpResponse(res, true) if err != nil { log.Fatal(err) } fmt.Println(string(bytes.ReplaceAll(buf, []byte("\r\n"), []byte("\n")))) }
Output: HTTP/1.1 200 OK Connection: close Content-Type: text/html X-Header: test <!doctype html><html lang=en><body attribute=value><p>hello ken!<div>something</div><a href=//example.com/full/path>a link!</a>
Index ¶
- func TTL(ctx context.Context) (time.Duration, bool)
- func UserCacheDir(paths ...string) (string, error)
- func WithContextTTL(parent context.Context, ttl time.Duration) context.Context
- type Base64Decoder
- type BodyTransformer
- type Cache
- func (c *Cache) Cached(req *http.Request) (bool, error)
- func (c *Cache) Evict(req *http.Request) error
- func (c *Cache) EvictKey(key string) error
- func (c *Cache) Exec(key string, p Policy, req *http.Request) (*http.Response, error)
- func (c *Cache) Fetch(key string, p Policy, req *http.Request, force bool) (bool, time.Time, *http.Response, error)
- func (c *Cache) Load(key string, p Policy, req *http.Request) (*http.Response, error)
- func (c *Cache) Match(req *http.Request) (string, Policy, error)
- func (c *Cache) Mod(key string) (time.Time, error)
- func (c *Cache) RoundTrip(req *http.Request) (*http.Response, error)
- func (c *Cache) Stale(ctx context.Context, key string, ttl time.Duration) (bool, time.Time, error)
- type FlatMarshalUnmarshaler
- type GzipMarshalUnmarshaler
- type HeaderTransformer
- type HeaderTransformerFunc
- type MarshalUnmarshaler
- type Matcher
- type Minifier
- type Option
- func WithAppCacheDir(app string, paths ...string) Option
- func WithBase64Decoder(contentTypes ...string) Option
- func WithBasePathFs(basePath string) Option
- func WithBodyTransformers(bodyTransformers ...BodyTransformer) Option
- func WithContentTypeTTL(ttl time.Duration, contentTypes ...string) Option
- func WithDefaultMatcher(method, host, path, key string, opts ...Option) Option
- func WithErrorTruncator() Option
- func WithFlatChain(marshalUnmarshaler MarshalUnmarshaler) Option
- func WithFlatGzipCompression() Option
- func WithFlatStorage() Option
- func WithFlatZlibCompression() Option
- func WithFs(fs afero.Fs) Option
- func WithGzipCompression() Option
- func WithHeaderBlacklist(blacklist ...string) Option
- func WithHeaderTransform(pairs ...string) Option
- func WithHeaderTransformers(headerTransformers ...HeaderTransformer) Option
- func WithHeaderWhitelist(whitelist ...string) Option
- func WithIndexPath(indexPath string) Option
- func WithLongPathHandler(longPathHandler func(string) string) Option
- func WithMarshalUnmarshaler(marshalUnmarshaler MarshalUnmarshaler) Option
- func WithMatchers(matchers ...Matcher) Option
- func WithMethod(method ...string) Option
- func WithMinifier() Option
- func WithMode(dirMode, fileMode os.FileMode) Option
- func WithNoDefault() Option
- func WithPrefixStripper(prefix []byte, contentTypes ...string) Option
- func WithQueryEncoder(queryEncoder func(url.Values) string) Option
- func WithQueryPrefix(prefix string, fields ...string) Option
- func WithRetryStatusCode(retries int, expected ...int) Option
- func WithStatusCodeTruncator(statusCodes ...int) Option
- func WithTTL(ttl time.Duration) Option
- func WithTransport(transport http.RoundTripper) Option
- func WithTruncator(priority TransformPriority, match func(string, int, string) bool) Option
- func WithValidator(validator Validator) Option
- func WithValidatorFunc(f ValidatorFunc) Option
- func WithZlibCompression() Option
- type Policy
- type PrefixStripper
- type RegexpHeaderTransformer
- type SimpleMatcher
- type SimpleValidator
- type TransformPriority
- type Truncator
- type Validator
- type ValidatorFunc
- type Validity
- type ZlibMarshalUnmarshaler
Examples ¶
Constants ¶
This section is empty.
Variables ¶
This section is empty.
Functions ¶
func UserCacheDir ¶ added in v0.3.0
UserCacheDir returns the user's system cache dir, adding paths to the end.
Example usage:
dir, err := diskcache.UserCacheDir("my-app-name") cache, err := diskcache.New(diskcache.WithBasePathFs(dir))
Note: WithAppCacheDir is easier.
Types ¶
type Base64Decoder ¶
type Base64Decoder struct { Priority TransformPriority ContentTypes []string Encoding *base64.Encoding }
Base64Decoder is a body transformer that base64 decodes the body.
func (Base64Decoder) BodyTransform ¶
func (t Base64Decoder) BodyTransform(w io.Writer, r io.Reader, urlstr string, code int, contentType string) (bool, error)
BodyTransform satisfies the BodyTransformer interface.
func (Base64Decoder) TransformPriority ¶
func (t Base64Decoder) TransformPriority() TransformPriority
TransformPriority satisfies the BodyTransformer interface.
type BodyTransformer ¶
type BodyTransformer interface { // TransformPriority returns the order for the transformer. TransformPriority() TransformPriority // BodyTransform transforms data read from r to w for the provided URL, // status code, and content type. A return of false prevents further // passing the stream to lower priority body transformers. BodyTransform(w io.Writer, r io.Reader, urlstr string, code int, contentType string) (bool, error) }
BodyTransformer is the shared interface for mangling body content prior to storage in the fs.
type Cache ¶
type Cache struct {
// contains filtered or unexported fields
}
Cache is a http.RoundTripper compatible disk cache.
func New ¶
New creates a new disk cache.
By default, the cache path will be <working directory>/cache. Change location using options.
func (*Cache) Exec ¶
Exec executes the request, storing the response using the key and cache policy. Applies header and body transformers, before marshaling and the response.
func (*Cache) Fetch ¶ added in v0.6.0
func (c *Cache) Fetch(key string, p Policy, req *http.Request, force bool) (bool, time.Time, *http.Response, error)
Fetch retrieves the key from the cache based on the policy TTL. When forced, or if the cached response is stale the request will be executed and the response cached.
type FlatMarshalUnmarshaler ¶
type FlatMarshalUnmarshaler struct { // Chain is an additional MarshalUnmarshaler that the data can be sent to // prior to storage on disk, but after the header has been stripped. Chain MarshalUnmarshaler }
FlatMarshalUnmarshaler is a flat file marshaler/unmarshaler, dropping original response header when marshaling.
type GzipMarshalUnmarshaler ¶
type GzipMarshalUnmarshaler struct { // Level is the compression level. Level int }
GzipMarshalUnmarshaler is a gzip mashaler/unmarshaler.
type HeaderTransformer ¶
HeaderTransformer is the shared interface for modifying/altering headers prior to storage on disk.
type HeaderTransformerFunc ¶
HeaderTransformerFunc is a header rewriter func.
func (HeaderTransformerFunc) HeaderTransform ¶
func (f HeaderTransformerFunc) HeaderTransform(buf []byte) []byte
HeaderTransformer satisfies the HeaderTransformer interface.
type MarshalUnmarshaler ¶
type MarshalUnmarshaler interface { Marshal(w io.Writer, r io.Reader) error Unmarshal(w io.Writer, r io.Reader) error }
MarshalUnmarshaler is the shared interface for marshaling/unmarshaling.
type Matcher ¶
type Matcher interface { // Match matches the passed request, returning the key and ttl. Match(*http.Request) (string, Policy, error) }
Matcher is the shared interface for retrivieving a disk cache policy for requests.
type Minifier ¶
type Minifier struct {
Priority TransformPriority
}
Minifier is a body transformer that minifies HTML, XML, SVG, JavaScript, JSON, and CSS content.
See: https://github.com/tdewolff/minify
func (Minifier) BodyTransform ¶
func (t Minifier) BodyTransform(w io.Writer, r io.Reader, urlstr string, code int, contentType string) (bool, error)
BodyTransform satisfies the BodyTransformer interface.
func (Minifier) TransformPriority ¶
func (t Minifier) TransformPriority() TransformPriority
TransformPriority satisfies the BodyTransformer interface.
type Option ¶
type Option interface {
// contains filtered or unexported methods
}
Option is a disk cache option.
func WithAppCacheDir ¶ added in v0.3.0
WithAppCacheDir is a disk cache option to set the afero fs used locked to the user's cache directory joined with the app name and any passed paths.
The afero base fs directory will typically be $HOME/.cache/<app>/paths...
func WithBase64Decoder ¶
WithBase64Decoder is a disk cache option to add a body transformer that does base64 decoding of responses for specific content types.
func WithBasePathFs ¶
WithBasePathFs is a disk cache option to set the afero fs used locked to a base directory.
func WithBodyTransformers ¶
func WithBodyTransformers(bodyTransformers ...BodyTransformer) Option
WithBodyTransformers is a disk cache option to set the body transformers.
func WithContentTypeTTL ¶ added in v0.6.0
WithContentTypeTTL is a disk cache option to set the cache policy TTL for matching content types.
func WithDefaultMatcher ¶
WithDefaultMatcher is a disk cache option to set the default matcher.
func WithErrorTruncator ¶
func WithErrorTruncator() Option
WithErrorTruncator is a disk cache option to add a body transformer that truncates responses when the HTTP status code != OK (200).
func WithFlatChain ¶
func WithFlatChain(marshalUnmarshaler MarshalUnmarshaler) Option
WithFlatChain is a disk cache option that marshals/unmarshals responses, removing headers from responses, and chaining marshaling/unmarshaling to a provided marshaler/unmarshaler.
Note: cached responses will not have original headers.
func WithFlatGzipCompression ¶
func WithFlatGzipCompression() Option
WithFlatGzipCompression is a disk cache option that marshals/unmarshals responses, with headers removed from responses, and with gzip compression.
Note: cached responses will not have original headers.
func WithFlatStorage ¶
func WithFlatStorage() Option
WithFlatStorage is a disk cache option to set a flat marshaler/unmarshaler removing headers from responses.
Note: cached responses will not have original headers.
func WithFlatZlibCompression ¶
func WithFlatZlibCompression() Option
WithFlatZlibCompression is a disk cache option that marshals/unmarshals responses, with headers removed from responses, and with zlib compression.
Note: cached responses will not have original headers.
func WithGzipCompression ¶
func WithGzipCompression() Option
WithGzipCompression is a disk cache option to set a gzip marshaler/unmarshaler.
func WithHeaderBlacklist ¶
WithHeaderBlacklist is a disk cache option to add a header transformer that removes any header in the blacklist.
func WithHeaderTransform ¶
WithHeaderTransform is a disk cache option to add a header transformer that transforms headers matching the provided regexp pairs and replacements.
func WithHeaderTransformers ¶
func WithHeaderTransformers(headerTransformers ...HeaderTransformer) Option
WithHeaderTransformers is a disk cache option to set the header transformers.
func WithHeaderWhitelist ¶
WithHeaderWhitelist is a disk cache option to add a header transformer that removes any header not in the whitelist.
func WithIndexPath ¶
WithIndexPath is a disk cache option to set the index path name.
func WithLongPathHandler ¶
WithLongPathHandler is a disk cache option to set a long path handler.
func WithMarshalUnmarshaler ¶
func WithMarshalUnmarshaler(marshalUnmarshaler MarshalUnmarshaler) Option
WithMarshalUnmarshaler is a disk cache option to set a marshaler/unmarshaler.
func WithMatchers ¶
WithMatchers is a disk cache option to set matchers.
func WithMethod ¶ added in v0.9.0
WithMethod is a disk cache option to set matching request method(s).
func WithMinifier ¶
func WithMinifier() Option
WithMinifier is a disk cache option to add a body transformer that does content minification of HTML, XML, SVG, JavaScript, JSON, and CSS data. Useful for reducing disk storage sizes.
func WithMode ¶
WithMode is a disk cache option to set the file mode used when creating files and directories on disk.
func WithNoDefault ¶
func WithNoDefault() Option
WithNoDefault is a disk cache option to disable the default matcher.
Prevents propagating settings from default matcher.
func WithPrefixStripper ¶
WithPrefixStripper is a disk cache option to add a body transformer that strips a specific XSS prefix for a specified content type.
Useful for removing XSS prefixes added to JavaScript or JSON content.
func WithQueryEncoder ¶
WithQueryEncoder is a disk cache option to set the query encoder.
func WithQueryPrefix ¶
WithQueryPrefix is a disk cache option that sets a query encoder, that adds the supplied prefix to non-empty and canonical encoding
The query string encoder can be limited to only the passed fields.
func WithRetryStatusCode ¶ added in v0.6.0
WithRetryStatusCode is a disk cache option to add a validator to the cache policy that retries when the response status is not the expected status.
func WithStatusCodeTruncator ¶
WithStatusCodeTrunactor is a disk cache option to add a body transformer that truncates responses when the status code is not in the provided list.
func WithTransport ¶
func WithTransport(transport http.RoundTripper) Option
WithTransport is a disk cache option to set the underlying HTTP transport.
func WithTruncator ¶
WithTruncator is a disk cache option to add a body transformer that truncates responses based on match criteria.
func WithValidator ¶ added in v0.6.0
WithValidator is a disk cache option to set the cache policy validator.
func WithValidatorFunc ¶ added in v0.6.0
func WithValidatorFunc(f ValidatorFunc) Option
WithValidatorFunc is a disk cache option to set the cache policy validator.
func WithZlibCompression ¶
func WithZlibCompression() Option
WithZlibCompression is a disk cache option to set a zlib marshaler/unmarshaler.
type Policy ¶
type Policy struct { // TTL is the time-to-live. TTL time.Duration // HeaderTransformers are the set of header transformers. HeaderTransformers []HeaderTransformer // BodyTransformers are the set of body tranformers. BodyTransformers []BodyTransformer // MarshalUnmarshaler is the marshal/unmarshaler responsible for storage on // disk. MarshalUnmarshaler MarshalUnmarshaler // Validator validates responses. Validator Validator }
Policy is a disk cache policy.
type PrefixStripper ¶
type PrefixStripper struct { Priority TransformPriority ContentTypes []string Prefix []byte }
PrefixStripper is a body transformer that strips a prefix.
Useful for munging content that may have had a preventative XSS prefix attached to it, such as some JavaScript or JSON content.
func (PrefixStripper) BodyTransform ¶
func (t PrefixStripper) BodyTransform(w io.Writer, r io.Reader, urlstr string, code int, contentType string) (bool, error)
BodyTransform satisfies the BodyTransformer interface.
func (PrefixStripper) TransformPriority ¶
func (t PrefixStripper) TransformPriority() TransformPriority
TransformPriority satisfies the BodyTransformer interface.
type RegexpHeaderTransformer ¶
RegexpHeaderTransformer transforms headers matching regexps and replacements.
func NewHeaderTransformer ¶
func NewHeaderTransformer(pairs ...string) (*RegexpHeaderTransformer, error)
NewHeaderTransformer creates a new header transformer from the passed matching regexp and replacement pairs.
func (*RegexpHeaderTransformer) HeaderTransform ¶
func (t *RegexpHeaderTransformer) HeaderTransform(buf []byte) []byte
HeaderTransform satisfies the HeaderTransformer interface.
type SimpleMatcher ¶
type SimpleMatcher struct {
// contains filtered or unexported fields
}
SimpleMatcher handles matching caching policies to requests.
func NewSimpleMatcher ¶ added in v0.5.2
func NewSimpleMatcher(method, host, path, key string, opts ...Option) (*SimpleMatcher, error)
NewSimpleMatcher creates a simple matcher for the provided method, host and path regular expressions, substitution key string, and other options.
Example:
m, err := NewSimpleMatcher( `GET`, `^(?P<proto>https?)://(?P<host>[^:]+)(?P<port>:[0-9]+)?$`, `^/?(?P<path>.*)$`, `{{proto}}/{{host}}{{port}}/{{path}}{{query}}`, WithIndexPath("?index"), WithQueryPrefix("_"), WithLongPathHandler(func(key string) string { if len(key) > 128 { return fmt.Sprintf("?long/%x", sha256.Sum256([]byte(key))) } return key }), ) if err != nil { return nil, err }
type SimpleValidator ¶ added in v0.6.0
type SimpleValidator struct {
// contains filtered or unexported fields
}
SimpleValidator is a simple response validator.
func NewSimpleValidator ¶ added in v0.6.0
func NewSimpleValidator(validator ValidatorFunc) *SimpleValidator
NewSimpleValidator creates a simple validator.
type TransformPriority ¶
type TransformPriority int
TransformPriority is the body transform priority.
const ( TransformPriorityFirst TransformPriority = 10 TransformPriorityDecode TransformPriority = 50 TransformPriorityModify TransformPriority = 60 TransformPriorityMinify TransformPriority = 80 TransformPriorityLast TransformPriority = 90 )
Transform priorities.
type Truncator ¶
type Truncator struct { Priority TransformPriority Match func(string, int, string) bool }
Truncator is a body transformer that truncates responses based on match criteria.
func (Truncator) BodyTransform ¶
func (t Truncator) BodyTransform(w io.Writer, r io.Reader, urlstr string, code int, contentType string) (bool, error)
BodyTransform satisfies the BodyTransformer interface.
func (Truncator) TransformPriority ¶
func (t Truncator) TransformPriority() TransformPriority
TransformPriority satisfies the BodyTransformer interface.
type Validator ¶ added in v0.6.0
type Validator interface { // Validate validates the response based on Validate(*http.Request, *http.Response, time.Time, bool) (Validity, error) }
Validator is the shared interface for validating responses.
type ValidatorFunc ¶ added in v0.6.0
ValidatorFunc is a response validator func.
type ZlibMarshalUnmarshaler ¶
type ZlibMarshalUnmarshaler struct { // Level is the compression level. Level int // Dict is the compression dictionary. Dict []byte }
ZlibMarshalUnmarshaler is a zlib mashaler/unmarshaler.