annotation

Documentation

Index

• type HopProcessor
  • func NewHopProcessor(client *storage.Client, outBucket string, rv4, rv6, asnames *url.URL) *HopProcessor
  • func (p *HopProcessor) File(h *tar.Header, b []byte) ([]byte, error)
  • func (p *HopProcessor) Finish(ctx context.Context, out *archive.Target) error
  • func (p *HopProcessor) Init(ctx context.Context, date string)
  • func (p *HopProcessor) Source(ctx context.Context, row Result) *archive.Source
• type Processor
  • func NewProcessor(client *storage.Client, outBucket string, rv4, rv6, asnames *url.URL) *Processor
  • func (p *Processor) File(h *tar.Header, b []byte) ([]byte, error)
  • func (p *Processor) Finish(ctx context.Context, out *archive.Target) error
  • func (p *Processor) Init(ctx context.Context, date string)
  • func (p *Processor) Source(ctx context.Context, row Result) *archive.Source
• type Renamer
  • func NewRenamer(client *storage.Client, bucket, experiment, fromDatatype, newDatatype string) *Renamer
  • func (r *Renamer) List(ctx context.Context, date string) ([]string, error)
  • func (r *Renamer) Rename(ctx context.Context, url string) (string, error)
• type Result

Types

type HopProcessor

type HopProcessor struct {
	// contains filtered or unexported fields
}

Processor maintains state for reprocessing annotation archives.

func NewHopProcessor

func NewHopProcessor(client *storage.Client, outBucket string, rv4, rv6, asnames *url.URL) *HopProcessor

NewHopProcessor creates a new annotation processor.

func (*HopProcessor) File

func (p *HopProcessor) File(h *tar.Header, b []byte) ([]byte, error)

File processes the given file header and file contents. File returns the new file content or process.ErrCorrupt.

func (*HopProcessor) Finish

func (p *HopProcessor) Finish(ctx context.Context, out *archive.Target) error

Finish completes processing of the given output archive by uploading to GCS to an alternate bucket and object name.

func (*HopProcessor) Init

func (p *HopProcessor) Init(ctx context.Context, date string)

Init downloads the routeview datasets for the given date and initializes the ASN annotator.

func (*HopProcessor) Source

func (p *HopProcessor) Source(ctx context.Context, row Result) *archive.Source

Source generates a new archive.Reader for the result row.ArchiveURL.

type Processor

type Processor struct {
	// contains filtered or unexported fields
}

Processor maintains state for reprocessing annotation archives.

func NewProcessor

func NewProcessor(client *storage.Client, outBucket string, rv4, rv6, asnames *url.URL) *Processor

NewProcessor creates a new annotation processor.

func (*Processor) File

func (p *Processor) File(h *tar.Header, b []byte) ([]byte, error)

File processes the given file header and file contents. File returns the new file content or process.ErrCorrupt.

func (*Processor) Finish

func (p *Processor) Finish(ctx context.Context, out *archive.Target) error

Finish completes processing of the given output archive by uploading to GCS to an alternate bucket and object name.

func (*Processor) Init

func (p *Processor) Init(ctx context.Context, date string)

Init downloads the routeview datasets for the given date and initializes the ASN annotator.

func (*Processor) Source

func (p *Processor) Source(ctx context.Context, row Result) *archive.Source

Source generates a new archive.Reader for the result row.ArchiveURL.

type Renamer

type Renamer struct {
	// contains filtered or unexported fields
}

Renamer manages GCS operations to rename objects from one datatype to a new datatype.

func NewRenamer

func NewRenamer(client *storage.Client, bucket, experiment, fromDatatype, newDatatype string) *Renamer

NewRenamer creates a new Renamer. Objects are listed from bucket and written to bucket.

func (*Renamer) List

func (r *Renamer) List(ctx context.Context, date string) ([]string, error)

List returns GCS URLs for every fromDatatype object under the given date prefix.

func (*Renamer) Rename

func (r *Renamer) Rename(ctx context.Context, url string) (string, error)

Rename copies the named URL to a new object, replacing fromDatatype with newDatatype in the object name.

type Result

type Result struct {
	// ArchiveURL is the GCS URL of the archive that contains Files.
	ArchiveURL string
	// Files is an array of tuples for annotation Filename and client DstIP addresses.
	Files []struct {
		DstIP    string
		Filename string
	}
}

Result is a structure to read BigQuery result rows.