fetcher

package
v0.6.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 16, 2024 License: BSD-3-Clause Imports: 18 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type ArchiveFetcher 

type ArchiveFetcher struct {
	// contains filtered or unexported fields
}

Provides access to entries of an archive.

func NewArchiveFetcher 

func NewArchiveFetcher(a archive.Archive) *ArchiveFetcher

func NewArchiveFetcherFromPath 

func NewArchiveFetcherFromPath(filepath string) (*ArchiveFetcher, error)

func NewArchiveFetcherFromPathWithFactory 

func NewArchiveFetcherFromPathWithFactory(path string, factory archive.ArchiveFactory) (*ArchiveFetcher, error)

func (*ArchiveFetcher) Close 

func (f *ArchiveFetcher) Close()

Close implements Fetcher

func (*ArchiveFetcher) Get 

func (f *ArchiveFetcher) Get(link manifest.Link) Resource

Get implements Fetcher 

func (f *ArchiveFetcher) Links() (manifest.LinkList, error)

Links implements Fetcher

type BytesResource 

type BytesResource struct {
	// contains filtered or unexported fields
}

BytesResource is a Resource serving a lazy-loaded bytes buffer.

func NewBytesResource 

func NewBytesResource(link manifest.Link, loader func() []byte) *BytesResource

NewBytesResource creates a new BytesResources from a lazy loader callback.

func (*BytesResource) Close 

func (r *BytesResource) Close()

Close implements Resource

func (*BytesResource) File 

func (r *BytesResource) File() string

File implements Resource

func (*BytesResource) Length 

func (r *BytesResource) Length() (int64, *ResourceError)

Length implements Resource 

func (r *BytesResource) Link() manifest.Link

Link implements Resource

func (*BytesResource) Properties 

func (r *BytesResource) Properties() manifest.Properties

Properties implements Resource

func (*BytesResource) Read 

func (r *BytesResource) Read(start int64, end int64) ([]byte, *ResourceError)

Read implements Resource

func (*BytesResource) ReadAsJSON 

func (r *BytesResource) ReadAsJSON() (map[string]interface{}, *ResourceError)

ReadAsJSON implements Resource

func (*BytesResource) ReadAsString 

func (r *BytesResource) ReadAsString() (string, *ResourceError)

ReadAsString implements Resource

func (*BytesResource) ReadAsXML 

func (r *BytesResource) ReadAsXML(prefixes map[string]string) (*xmlquery.Node, *ResourceError)

ReadAsXML implements Resource

func (*BytesResource) Stream 

func (r *BytesResource) Stream(w io.Writer, start int64, end int64) (int64, *ResourceError)

Stream implements Resource

type CompressedResource added in v0.5.0 

type CompressedResource interface {
	CompressedAs(compressionMethod archive.CompressionMethod) bool
	CompressedLength() int64
	StreamCompressed(w io.Writer) (int64, *ResourceError)
	StreamCompressedGzip(w io.Writer) (int64, *ResourceError)
	ReadCompressed() ([]byte, *ResourceError)
	ReadCompressedGzip() ([]byte, *ResourceError)
}

type EmptyFetcher 

type EmptyFetcher struct{}

A Fetcher providing no resources at all.

func (EmptyFetcher) Close 

func (f EmptyFetcher) Close()

func (EmptyFetcher) Get 

func (f EmptyFetcher) Get(link manifest.Link) Resource
func (f EmptyFetcher) Links() (manifest.LinkList, error)

type FailureResource 

type FailureResource struct {
	// contains filtered or unexported fields
}

Creates a Resource that will always return the given [error].

func NewFailureResource 

func NewFailureResource(link manifest.Link, ex *ResourceError) FailureResource

func (FailureResource) Close 

func (r FailureResource) Close()

Close implements Resource

func (FailureResource) File 

func (r FailureResource) File() string

File implements Resource

func (FailureResource) Length 

func (r FailureResource) Length() (int64, *ResourceError)

Length implements Resource 

func (r FailureResource) Link() manifest.Link

Link implements Resource

func (FailureResource) Properties 

func (r FailureResource) Properties() manifest.Properties

func (FailureResource) Read 

func (r FailureResource) Read(start int64, end int64) ([]byte, *ResourceError)

Read implements Resource

func (FailureResource) ReadAsJSON 

func (r FailureResource) ReadAsJSON() (map[string]interface{}, *ResourceError)

ReadAsJSON implements Resource

func (FailureResource) ReadAsString 

func (r FailureResource) ReadAsString() (string, *ResourceError)

ReadAsString implements Resource

func (FailureResource) ReadAsXML 

func (r FailureResource) ReadAsXML(prefixes map[string]string) (*xmlquery.Node, *ResourceError)

ReadAsXML implements Resource

func (FailureResource) Stream 

func (r FailureResource) Stream(w io.Writer, start int64, end int64) (int64, *ResourceError)

Stream implements Resource

type Fetcher 

type Fetcher interface {

	/**
	 * Known resources available in the medium, such as file paths on the file system
	 * or entries in a ZIP archive. This list is not exhaustive, and additional
	 * unknown resources might be reachable.
	 *
	 * If the medium has an inherent resource order, it should be followed.
	 * Otherwise, HREFs are sorted alphabetically.
	 */
	Links() (manifest.LinkList, error)

	/**
	 * Returns the [Resource] at the given [link]'s HREF.
	 *
	 * A [Resource] is always returned, since for some cases we can't know if it exists before
	 * actually fetching it, such as HTTP. Therefore, errors are handled at the Resource level.
	 */
	Get(link manifest.Link) Resource

	// Closes this object and releases any resources associated with it.
	// If the object is already closed then invoking this method has no effect.
	Close()
}

Fetcher provides access to a Resource from a Link.

type FileFetcher 

type FileFetcher struct {
	// contains filtered or unexported fields
}

Provides access to resources on the local file system.

func NewFileFetcher 

func NewFileFetcher(href string, fpath string) *FileFetcher

func (*FileFetcher) Close 

func (f *FileFetcher) Close()

Close implements Fetcher

func (*FileFetcher) Get 

func (f *FileFetcher) Get(link manifest.Link) Resource

Get implements Fetcher 

func (f *FileFetcher) Links() (manifest.LinkList, error)

Links implements Fetcher

type FileResource 

type FileResource struct {
	// contains filtered or unexported fields
}

func NewFileResource 

func NewFileResource(link manifest.Link, abspath string) *FileResource

func (*FileResource) Close 

func (r *FileResource) Close()

Close implements Resource

func (*FileResource) File 

func (r *FileResource) File() string

File implements Resource

func (*FileResource) Length 

func (r *FileResource) Length() (int64, *ResourceError)

Length implements Resource 

func (r *FileResource) Link() manifest.Link

Link implements Resource

func (*FileResource) Properties 

func (r *FileResource) Properties() manifest.Properties

func (*FileResource) Read 

func (r *FileResource) Read(start int64, end int64) ([]byte, *ResourceError)

Read implements Resource

func (*FileResource) ReadAsJSON 

func (r *FileResource) ReadAsJSON() (map[string]interface{}, *ResourceError)

ReadAsJSON implements Resource

func (*FileResource) ReadAsString 

func (r *FileResource) ReadAsString() (string, *ResourceError)

ReadAsString implements Resource

func (*FileResource) ReadAsXML 

func (r *FileResource) ReadAsXML(prefixes map[string]string) (*xmlquery.Node, *ResourceError)

ReadAsXML implements Resource

func (*FileResource) Stream 

func (r *FileResource) Stream(w io.Writer, start int64, end int64) (int64, *ResourceError)

Stream implements Resource

type ProxyResource 

type ProxyResource struct {
	Res Resource
}

A base class for a Resource which acts as a proxy to another one. Every function is delegating to the proxied resource, and subclasses should override some of them.

func (ProxyResource) Close 

func (r ProxyResource) Close()

Close implements Resource

func (ProxyResource) CompressedAs added in v0.5.0 

func (r ProxyResource) CompressedAs(compressionMethod archive.CompressionMethod) bool

CompressedAs implements CompressedResource

func (ProxyResource) CompressedLength added in v0.5.0 

func (r ProxyResource) CompressedLength() int64

CompressedLength implements CompressedResource

func (ProxyResource) File 

func (r ProxyResource) File() string

File implements Resource

func (ProxyResource) Length 

func (r ProxyResource) Length() (int64, *ResourceError)

Length implements Resource 

func (r ProxyResource) Link() manifest.Link

Link implements Resource

func (ProxyResource) Properties 

func (r ProxyResource) Properties() manifest.Properties

func (ProxyResource) Read 

func (r ProxyResource) Read(start int64, end int64) ([]byte, *ResourceError)

Read implements Resource

func (ProxyResource) ReadAsJSON 

func (r ProxyResource) ReadAsJSON() (map[string]interface{}, *ResourceError)

ReadAsJSON implements Resource

func (ProxyResource) ReadAsString 

func (r ProxyResource) ReadAsString() (string, *ResourceError)

ReadAsString implements Resource

func (ProxyResource) ReadAsXML 

func (r ProxyResource) ReadAsXML(prefixes map[string]string) (*xmlquery.Node, *ResourceError)

ReadAsXML implements Resource

func (ProxyResource) ReadCompressed added in v0.5.0 

func (r ProxyResource) ReadCompressed() ([]byte, *ResourceError)

ReadCompressed implements CompressedResource

func (ProxyResource) ReadCompressedGzip added in v0.5.0 

func (r ProxyResource) ReadCompressedGzip() ([]byte, *ResourceError)

ReadCompressedGzip implements CompressedResource

func (ProxyResource) Stream 

func (r ProxyResource) Stream(w io.Writer, start int64, end int64) (int64, *ResourceError)

Stream implements Resource

func (ProxyResource) StreamCompressed added in v0.5.0 

func (r ProxyResource) StreamCompressed(w io.Writer) (int64, *ResourceError)

StreamCompressed implements CompressedResource

func (ProxyResource) StreamCompressedGzip added in v0.5.0 

func (r ProxyResource) StreamCompressedGzip(w io.Writer) (int64, *ResourceError)

StreamCompressedGzip implements CompressedResource

type Resource 

type Resource interface {

	// Direct filepath for this resource, when available.
	// Not guaranteed to be set, for example if the resource underwent transformations or is being read from an archive.
	File() string

	// Closes this object and releases any resources associated with it.
	// If the object is already closed then invoking this method has no effect.
	Close()

	// Returns the link from which the resource was retrieved.
	// It might be modified by the [Resource] to include additional metadata, e.g. the `Content-Type` HTTP header in [Link.Type].
	Link() manifest.Link

	// Returns the properties associated with the resource.
	// This is opened for extensions.
	Properties() manifest.Properties

	// Returns data length from metadata if available, or calculated from reading the bytes otherwise.
	// This value must be treated as a hint, as it might not reflect the actual bytes length. To get the real length, you need to read the whole resource.
	Length() (int64, *ResourceError)

	// Reads the bytes at the given range.
	// When start and end are null, the whole content is returned. Out-of-range indexes are clamped to the available length automatically.
	Read(start int64, end int64) ([]byte, *ResourceError)

	// Stream the bytes at the given range to a writer.
	// When start and end are null, the whole content is returned. Out-of-range indexes are clamped to the available length automatically.
	Stream(w io.Writer, start int64, end int64) (int64, *ResourceError)

	// Reads the full content as a string.
	// Assumes UTF-8 encoding if no Link charset is given
	ReadAsString() (string, *ResourceError)

	// Reads the full content as a JSON object.
	ReadAsJSON() (map[string]interface{}, *ResourceError)

	// Reads the full content as a generic XML document.
	ReadAsXML(prefixes map[string]string) (*xmlquery.Node, *ResourceError)
}

Acts as a proxy to an actual resource by handling read access.

type ResourceError 

type ResourceError struct {
	Cause error
	Code  ResourceErrorCode
}

Errors occurring while accessing a resource.

func BadRequest 

func BadRequest(cause error) *ResourceError

Equivalent to a 400 HTTP error.

func Forbidden 

func Forbidden(cause error) *ResourceError

Equivalent to a 403 HTTP error. This can be returned when trying to read a resource protected with a DRM that is not unlocked.

func NewResourceError 

func NewResourceError(code ResourceErrorCode) *ResourceError

func NewResourceErrorWithCause 

func NewResourceErrorWithCause(code ResourceErrorCode, cause error) *ResourceError

func NotFound 

func NotFound(cause error) *ResourceError

Equivalent to a 404 HTTP error.

func OsErrorToException 

func OsErrorToException(err error) *ResourceError

Convert a Go os error to an exception

func Other 

func Other(cause error) *ResourceError

For any other error, such as HTTP 500.

func OutOfMemory 

func OutOfMemory(cause error) *ResourceError

Equivalent to a 507 HTTP error. Used when the requested range is too large to be read in memory.

func RangeNotSatisfiable 

func RangeNotSatisfiable(cause error) *ResourceError

Equivalent to a 416 HTTP error. Used when the requested range is not satisfiable (invalid)

func ReadResourceAsJSON 

func ReadResourceAsJSON(r Resource) (map[string]interface{}, *ResourceError)

func ReadResourceAsString 

func ReadResourceAsString(r Resource) (string, *ResourceError)

func ReadResourceAsXML 

func ReadResourceAsXML(r Resource, prefixes map[string]string) (*xmlquery.Node, *ResourceError)

func Timeout 

func Timeout(cause error) *ResourceError

Equivalent to a 504 HTTP error. Used when a request for a file times out (e.g. when fetching from remote storage)

func Unavailable 

func Unavailable(cause error) *ResourceError

Equivalent to a 503 HTTP error. Used when the source can't be reached, e.g. no Internet connection, or an issue with the file system. Usually this is a temporary error.

func (*ResourceError) Error 

func (ex *ResourceError) Error() string

func (*ResourceError) HTTPStatus 

func (ex *ResourceError) HTTPStatus() int

type ResourceErrorCode 

type ResourceErrorCode uint16
const (
	CodeBadRequest                   ResourceErrorCode = http.StatusBadGateway
	CodeNotFound                     ResourceErrorCode = http.StatusNotFound
	CodeForbidden                    ResourceErrorCode = http.StatusForbidden
	CodeServiceUnavailable           ResourceErrorCode = http.StatusServiceUnavailable
	CodeInsufficientStorage          ResourceErrorCode = http.StatusInsufficientStorage
	CodeRequestedRangeNotSatisfiable ResourceErrorCode = http.StatusRequestedRangeNotSatisfiable
	CodeGatewayTimeout               ResourceErrorCode = http.StatusGatewayTimeout
	CodeInternalServerError          ResourceErrorCode = http.StatusInternalServerError
)

Error codes with HTTP equivalents 

const (
	Offline ResourceErrorCode
	Cancelled
)

The rest of the codes

type ResourceReadSeeker 

type ResourceReadSeeker struct {
	// contains filtered or unexported fields
}

For opening a fetcher.Resource as a io.ReadSeeker

func NewResourceReadSeeker 

func NewResourceReadSeeker(r Resource) *ResourceReadSeeker

func (*ResourceReadSeeker) Read 

func (rs *ResourceReadSeeker) Read(p []byte) (n int, err error)

Seek implements io.ReadSeeker

func (*ResourceReadSeeker) Seek 

func (rs *ResourceReadSeeker) Seek(offset int64, whence int) (int64, error)

Seek implements io.ReadSeeker

type ResourceTransformer 

type ResourceTransformer func(Resource) Resource

*

  • Implements the transformation of a Resource. It can be used, for example, to decrypt,
  • deobfuscate, inject CSS or JavaScript, correct content – e.g. adding a missing dir="rtl" in an
  • HTML document, pre-process – e.g. before indexing a publication's content, etc. *
  • If the transformation doesn't apply, simply return resource unchanged.

type TransformingFetcher 

type TransformingFetcher struct {
	// contains filtered or unexported fields
}

Transforms the resources' content of a child fetcher using a list of ResourceTransformer functions.

func NewTransformingFetcher 

func NewTransformingFetcher(fetcher Fetcher, transformers ...ResourceTransformer) *TransformingFetcher

func (*TransformingFetcher) Close 

func (f *TransformingFetcher) Close()

Close implements Fetcher

func (*TransformingFetcher) Get 

func (f *TransformingFetcher) Get(link manifest.Link) Resource

Get implements Fetcher 

func (f *TransformingFetcher) Links() (manifest.LinkList, error)

Links implements Fetcher

type TransformingResource 

type TransformingResource struct {
	// contains filtered or unexported fields
}

*

  • Transforms the bytes of [resource] on-the-fly. *
  • Warning: The transformation runs on the full content of [resource], so it's not appropriate for
  • large resources which can't be held in memory. Pass [cacheBytes] = true to cache the result of
  • the transformation. This may be useful if multiple ranges will be read.

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.