gowarc

May 6, 2024

Changes in this version

+ const WarcJSONMetadata

+ const WarcPageID

+ const WarcResourceType

Feb 14, 2024

Changes in this version

type WarcFileWriterOption

+ func WithAfterFileCreationHook(f func(fileName string, size int64, warcInfoId string) error) WarcFileWriterOption

+ func WithBeforeFileCreationHook(f func(fileName string) error) WarcFileWriterOption

Feb 9, 2024

Changes in this version

type WarcFileReader

+ func NewWarcFileReaderFromStream(r io.Reader, offset int64, opts ...WarcRecordOption) (*WarcFileReader, error)

Jan 24, 2024

Changes in this version

type WarcRecordOption

+ func WithBlockErrorPolicy(policy errorPolicy) WarcRecordOption

Jun 30, 2023

Changes in this version

type WarcRecordOption

+ func WithFixWarcFieldsBlockErrors(fixWarcFieldsBlockErrors bool) WarcRecordOption

Jan 12, 2023

Changes in this version

+ const ApplicationHttp

+ const ApplicationWarcFields

+ const Base16

+ const Base32

+ const Base64

+ const ContentLength

+ const ContentType

+ const ErrFail

+ const ErrIgnore

+ const ErrWarn

+ const ProfileIdenticalPayloadDigestV1_0

+ const ProfileIdenticalPayloadDigestV1_1

+ const ProfileServerNotModifiedV1_0

+ const ProfileServerNotModifiedV1_1

+ const WarcBlockDigest

+ const WarcConcurrentTo

+ const WarcDate

+ const WarcFilename

+ const WarcIPAddress

+ const WarcIdentifiedPayloadType

+ const WarcPayloadDigest

+ const WarcProfile

+ const WarcRecordID

+ const WarcRefersTo

+ const WarcRefersToDate

+ const WarcRefersToTargetURI

+ const WarcSegmentNumber

+ const WarcSegmentOriginID

+ const WarcSegmentTotalLength

+ const WarcTargetURI

+ const WarcTruncated

+ const WarcType

+ const WarcWarcinfoID

+ var V1_0 = &WarcVersion

+ var V1_1 = &WarcVersion

+ type Block interface

+ BlockDigest func() string

+ Cache func() error

+ IsCached func() bool

+ RawBytes func() (io.Reader, error)

+ Size func() int64

+ type HeaderFieldError struct

+ func (e *HeaderFieldError) Error() string

+ type HttpRequestBlock interface

+ HttpHeader func() *http.Header

+ HttpRequestLine func() string

+ type HttpResponseBlock interface

+ HttpHeader func() *http.Header

+ HttpStatusCode func() int

+ HttpStatusLine func() string

+ type Marshaler interface

+ Marshal func(w io.Writer, record WarcRecord, maxSize int64) (WarcRecord, int64, error)

+ func NewMarshaler() Marshaler

+ type PatternNameGenerator struct

+ Directory string

+ Extension string

+ Pattern string

+ Prefix string

+ Serial int32

+ func (g *PatternNameGenerator) NewWarcfileName() (string, string)

+ type PayloadBlock interface

+ PayloadBytes func() (io.Reader, error)

+ PayloadDigest func() string

+ type ProtocolHeaderBlock interface

+ ProtocolHeaderBytes func() []byte

+ type RecordType uint16

+ const Continuation

+ const Conversion

+ const Metadata

+ const Request

+ const Resource

+ const Response

+ const Revisit

+ const Warcinfo

+ func (rt RecordType) String() string

+ type RevisitRef struct

+ Profile string

+ TargetDate string

+ TargetRecordId string

+ TargetUri string

+ type SyntaxError struct

+ func (e *SyntaxError) Error() string

+ func (e *SyntaxError) Unwrap() error

+ type Unmarshaler interface

+ Unmarshal func(b *bufio.Reader) (WarcRecord, int64, *Validation, error)

+ func NewUnmarshaler(opts ...WarcRecordOption) Unmarshaler

+ type Validation []error

+ func (v *Validation) Error() string

+ func (v *Validation) String() string

+ func (v *Validation) Valid() bool

+ type WarcFields []*nameValue

+ func (wf *WarcFields) Add(name string, value string)

+ func (wf *WarcFields) AddId(name, value string)

+ func (wf *WarcFields) AddInt(name string, value int)

+ func (wf *WarcFields) AddInt64(name string, value int64)

+ func (wf *WarcFields) AddTime(name string, value time.Time)

+ func (wf *WarcFields) CanonicalHeaderKey(s string) string

+ func (wf *WarcFields) Delete(key string)

+ func (wf *WarcFields) Get(key string) string

+ func (wf *WarcFields) GetAll(name string) []string

+ func (wf *WarcFields) GetId(name string) string

+ func (wf *WarcFields) GetInt(key string) (int, error)

+ func (wf *WarcFields) GetInt64(name string) (int64, error)

+ func (wf *WarcFields) GetTime(name string) (time.Time, error)

+ func (wf *WarcFields) Has(name string) bool

+ func (wf *WarcFields) Set(name string, value string)

+ func (wf *WarcFields) SetId(name, value string)

+ func (wf *WarcFields) SetInt(name string, value int)

+ func (wf *WarcFields) SetInt64(name string, value int64)

+ func (wf *WarcFields) SetTime(name string, value time.Time)

+ func (wf *WarcFields) Sort()

+ func (wf *WarcFields) String() string

+ func (wf *WarcFields) Write(w io.Writer) (bytesWritten int64, err error)

+ type WarcFieldsBlock interface

+ WarcFields func() *WarcFields

+ type WarcFileNameGenerator interface

+ NewWarcfileName func() (string, string)

+ type WarcFileReader struct

+ func NewWarcFileReader(filename string, offset int64, opts ...WarcRecordOption) (*WarcFileReader, error)

+ func (wf *WarcFileReader) Close() error

+ func (wf *WarcFileReader) Next() (WarcRecord, int64, *Validation, error)

+ type WarcFileWriter struct

+ func NewWarcFileWriter(opts ...WarcFileWriterOption) *WarcFileWriter

+ func (w *WarcFileWriter) Close() error

+ func (w *WarcFileWriter) Rotate() error

+ func (w *WarcFileWriter) String() string

+ func (w *WarcFileWriter) Write(record ...WarcRecord) []WriteResponse

+ type WarcFileWriterOption interface

+ func WithAddWarcConcurrentToHeader(addConcurrentHeader bool) WarcFileWriterOption

+ func WithCompressedFileSuffix(suffix string) WarcFileWriterOption

+ func WithCompression(compress bool) WarcFileWriterOption

+ func WithCompressionLevel(gzipLevel int) WarcFileWriterOption

+ func WithExpectedCompressionRatio(ratio float64) WarcFileWriterOption

+ func WithFileNameGenerator(generator WarcFileNameGenerator) WarcFileWriterOption

+ func WithFlush(flush bool) WarcFileWriterOption

+ func WithMarshaler(marshaler Marshaler) WarcFileWriterOption

+ func WithMaxConcurrentWriters(count int) WarcFileWriterOption

+ func WithMaxFileSize(size int64) WarcFileWriterOption

+ func WithOpenFileSuffix(suffix string) WarcFileWriterOption

+ func WithRecordOptions(opts ...WarcRecordOption) WarcFileWriterOption

+ func WithSegmentation() WarcFileWriterOption

+ func WithWarcInfoFunc(f func(recordBuilder WarcRecordBuilder) error) WarcFileWriterOption

+ type WarcRecord interface

+ Block func() Block

+ ContentLength func() (int64, error)

+ CreateRevisitRef func(profile string) (*RevisitRef, error)

+ Date func() (time.Time, error)

+ Merge func(record ...WarcRecord) (WarcRecord, error)

+ RecordId func() string

+ RevisitRef func() (*RevisitRef, error)

+ String func() string

+ ToRevisitRecord func(ref *RevisitRef) (WarcRecord, error)

+ Type func() RecordType

+ ValidateDigest func(validation *Validation) error

+ Version func() *WarcVersion

+ WarcHeader func() *WarcFields

+ type WarcRecordBuilder interface

+ AddWarcHeader func(name string, value string)

+ AddWarcHeaderInt func(name string, value int)

+ AddWarcHeaderInt64 func(name string, value int64)

+ AddWarcHeaderTime func(name string, value time.Time)

+ Build func() (WarcRecord, *Validation, error)

+ SetRecordType func(recordType RecordType)

+ Size func() int64

+ func NewRecordBuilder(recordType RecordType, opts ...WarcRecordOption) WarcRecordBuilder

+ type WarcRecordOption interface

+ func WithAddMissingContentLength(addMissingContentLength bool) WarcRecordOption

+ func WithAddMissingDigest(addMissingDigest bool) WarcRecordOption

+ func WithAddMissingRecordId(addMissingRecordId bool) WarcRecordOption

+ func WithBufferMaxMemBytes(size int64) WarcRecordOption

+ func WithBufferTmpDir(dir string) WarcRecordOption

+ func WithDefaultDigestAlgorithm(defaultDigestAlgorithm string) WarcRecordOption

+ func WithDefaultDigestEncoding(defaultDigestEncoding digestEncoding) WarcRecordOption

+ func WithFixContentLength(fixContentLength bool) WarcRecordOption

+ func WithFixDigest(fixDigest bool) WarcRecordOption

+ func WithFixSyntaxErrors(fixSyntaxErrors bool) WarcRecordOption

+ func WithNoValidation() WarcRecordOption

+ func WithRecordIdFunc(recordIdFunc func() (string, error)) WarcRecordOption

+ func WithSkipParseBlock() WarcRecordOption

+ func WithSpecViolationPolicy(policy errorPolicy) WarcRecordOption

+ func WithStrictValidation() WarcRecordOption

+ func WithSyntaxErrorPolicy(policy errorPolicy) WarcRecordOption

+ func WithUnknownRecordTypePolicy(policy errorPolicy) WarcRecordOption

+ func WithVersion(version *WarcVersion) WarcRecordOption

+ type WarcVersion struct

+ func (v *WarcVersion) Major() uint8

+ func (v *WarcVersion) Minor() uint8

+ func (v *WarcVersion) String() string

+ type WriteResponse struct

+ BytesWritten int64

+ Err error

+ FileName string

+ FileOffset int64