warc

package module

Go to main page

Versions in this module

v0

v0.8.39

May 8, 2024

Changes in this version

+ var CDXHTTPClient = http.Client

v0.8.38

Apr 25, 2024

Changes in this version

+ var IPv4 *availableIPs

+ var IPv6 *availableIPs

type HTTPClientSettings

+ RandomLocalIP bool

v0.8.37

Apr 5, 2024

Changes in this version

+ func GetSHA256(r io.Reader) string

+ func GetSHA256Base16(r io.Reader) string

v0.8.36

Feb 21, 2024

Changes in this version

type CustomHTTPClient

+ func (c *CustomHTTPClient) WriteMetadataRecord(WARCTargetURI, contentType, payload string)

v0.8.35

Feb 7, 2024

Changes in this version

type DedupeOptions

+ CDXCookie string

v0.8.34

Sep 14, 2023

v0.8.33

Aug 23, 2023

v0.8.32

Jun 23, 2023

v0.8.31

Jun 16, 2023

Changes in this version

type HTTPClientSettings

+ TCPTimeout time.Duration

+ TLSHandshakeTimeout time.Duration

v0.8.30

Jun 8, 2023

Changes in this version

type HTTPClientSettings

+ FollowRedirects bool

v0.8.29

May 17, 2023

Changes in this version

type CustomHTTPClient

+ ErrChan chan *Error

+ type Error struct

+ Err error

+ Func string

v0.8.28

Apr 4, 2023

v0.8.27

Apr 4, 2023

Changes in this version

type DedupeOptions

+ SizeThreshold int

v0.8.26

Mar 1, 2023

Changes in this version

+ func GenerateWarcFileName(prefix string, compression string, serial int) (fileName string)

v0.8.25

Nov 23, 2022

v0.8.24

Oct 26, 2022

Changes in this version

+ var DataTotal *ratecounter.Counter

type CustomHTTPClient

+ DataTotal *ratecounter.Counter

+ MaxReadBeforeTruncate int

type HTTPClientSettings

+ MaxReadBeforeTruncate int

v0.8.23

Sep 8, 2022

v0.8.22

Jul 25, 2022

v0.8.21

Jul 13, 2022

v0.8.20

Jul 4, 2022

Changes in this version

type CustomHTTPClient

+ WARCWriterDoneChannels []chan bool

type RotatorSettings

+ WARCWriterPoolSize int

type Writer

+ PGZIPWriter *pgzip.Writer

+ ParallelGZIP bool

+ func (w *Writer) CloseCompressedWriter()

v0.8.19

Jul 3, 2022

v0.8.18

Jul 3, 2022

Changes in this version

+ type WaitGroupWithCount struct

+ func (wg *WaitGroupWithCount) Add(delta int)

+ func (wg *WaitGroupWithCount) Done()

+ func (wg *WaitGroupWithCount) Size() int

v0.8.17

Jul 1, 2022

v0.8.16

Jul 1, 2022

v0.8.15

Jul 1, 2022

v0.8.14

Jul 1, 2022

Changes in this version

type CustomHTTPClient

+ FullOnDisk bool

+ TempDir string

+ type HTTPClientSettings struct

+ DecompressBody bool

+ DedupeOptions DedupeOptions

+ FullOnDisk bool

+ Proxy string

+ RotatorSettings *RotatorSettings

+ SkipHTTPStatusCodes []int

+ TempDir string

+ VerifyCerts bool

v0.8.13

Jun 30, 2022

v0.8.12

Jun 30, 2022

v0.8.11

Jun 30, 2022

v0.8.10

Jun 29, 2022

v0.8.9

Jun 28, 2022

Changes in this version

+ var MaxInMemorySize = 2097152

+ type ReadSeekCloser interface

+ FileName func() string

+ type ReadWriteSeekCloser interface

+ func NewSpooledTempFile(filePrefix string) ReadWriteSeekCloser

+ type ReaderAt interface

+ ReadAt func(p []byte, off int64) (n int, err error)

v0.8.8

Jun 27, 2022

v0.8.7

Jun 21, 2022

v0.8.6

Jun 20, 2022

v0.8.5

Jun 19, 2022

v0.8.4

May 19, 2022

Changes in this version

type CustomHTTPClient

+ WARCTempDir string

v0.8.3

May 4, 2022

v0.8.2

Apr 21, 2022

v0.8.1

Apr 7, 2022

v0.8.0

Apr 7, 2022

Changes in this version

+ type DedupeOptions struct

+ CDXDedupe bool

+ CDXURL string

+ LocalDedupe bool

v0.7.0

Mar 26, 2022

Changes in this version

+ func GetSHA1FromReader(r io.Reader) string

+ type CustomHTTPClient struct

+ WARCWriter chan *RecordBatch

+ WARCWriterFinish chan bool

+ WaitGroup *sync.WaitGroup

+ func NewWARCWritingHTTPClient(rotatorSettings *RotatorSettings, proxy string, decompressBody bool) (httpClient *CustomHTTPClient, err error)

+ func (c *CustomHTTPClient) Close() error

v0.6.0

Mar 19, 2022

?	: This menu
/	: Search site
f or F	: Jump to
y or Y	: Canonical URL