archive

README

archive

Archive is a golang package for creating/working with warc & cdxj archives.

License & Copyright

Copyright (C) 2017 Data Together

This program is free software: you can redistribute it and/or modify it under the terms of the GNU AFFERO General Public License as published by the Free Software Foundation, version 3.0.

This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.

See the LICENSE file for details.

Getting Involved

We would love involvement from more people! If you notice any errors or would like to submit changes, please see our Contributing Guidelines.

We use GitHub issues for tracking bugs and feature requests and Pull Requests (PRs) for submitting changes

Installation

Use in any golang package with:

import "github.com/datatogether/archive"

Documentation

Index

• func ArchiveUrl(req *http.Request, rw *rewrite.WarcRecordRewriter, records warc.Records) (addRecords warc.Records, err error)
• func DoRequest(req *http.Request, records warc.Records) (reqr, resr *warc.Record, err error)
• func HttpResponseRecord(res *http.Response) (*warc.Record, error)
• func PackagePathName(rawurl string) string
• func PackageRecords(urls []string, records warc.Records) (*memfs.Memdir, error)
• func RenderIndexTemplate(w io.Writer, urls []string, records warc.Records) error
• func RequestRecord(req *http.Request) *warc.Record
• func SanitizeResponse(res *http.Response) (raw, sanitized []byte, mimetype string, err error)
• type TemplateData
  • func NewTemplateData(title string, urls []string, records warc.Records) *TemplateData
• type TemplateRecord

Functions

func ArchiveUrl

func ArchiveUrl(req *http.Request, rw *rewrite.WarcRecordRewriter, records warc.Records) (addRecords warc.Records, err error)

func DoRequest

func DoRequest(req *http.Request, records warc.Records) (reqr, resr *warc.Record, err error)

DoRequest performs an archival HTTP request, adding request & response records to the given records pointer, returning the response record & a list of all dependant resources

func HttpResponseRecord

func HttpResponseRecord(res *http.Response) (*warc.Record, error)

HttpResponseRecord creates a record from an HTTP response

func PackagePathName

func PackagePathName(rawurl string) string

func PackageRecords

func PackageRecords(urls []string, records warc.Records) (*memfs.Memdir, error)

func RenderIndexTemplate

func RenderIndexTemplate(w io.Writer, urls []string, records warc.Records) error

func RequestRecord

func RequestRecord(req *http.Request) *warc.Record

func SanitizeResponse

func SanitizeResponse(res *http.Response) (raw, sanitized []byte, mimetype string, err error)

Types

type TemplateData

type TemplateData struct {
	Title     string
	Datestamp string
	Records   []*TemplateRecord
}

func NewTemplateData

func NewTemplateData(title string, urls []string, records warc.Records) *TemplateData

type TemplateRecord

type TemplateRecord struct {
	Path      string
	Url       string
	Datestamp time.Time
}