pointers

Documentation

Index

• func ComplexBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Complex) <-chan error
• func NestedBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Nested) <-chan error
• func SimpleBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Simple) <-chan error
• func TimesBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Times) <-chan error
• type Complex
  • func (z *Complex) WriteAvroCF(writer io.Writer, writeTime time.Time) error
  • func (z *Complex) WriteAvroDeletedCF(writer io.Writer, writeTime time.Time) error
• type ComplexCrops
• type ComplexOriginalSize
• type ComplexURL
• type ComplexURLMeta
• type Nested
  • func (z *Nested) WriteAvroCF(writer io.Writer, writeTime time.Time) error
  • func (z *Nested) WriteAvroDeletedCF(writer io.Writer, writeTime time.Time) error
• type NestedFactCheckClaims
• type NestedFactCheckClaimsAppearanceURLs
• type Simple
  • func (z *Simple) WriteAvroCF(writer io.Writer, writeTime time.Time) error
  • func (z *Simple) WriteAvroDeletedCF(writer io.Writer, writeTime time.Time) error
• type SimpleContributors
• type SimpleSomeDateObj
• type Times
  • func (z *Times) WriteAvroCF(writer io.Writer, writeTime time.Time) error
  • func (z *Times) WriteAvroDeletedCF(writer io.Writer, writeTime time.Time) error

Examples

• ComplexBulkAvroWriter
• NestedBulkAvroWriter
• SimpleBulkAvroWriter
• TimesBulkAvroWriter

Functions

func ComplexBulkAvroWriter

func ComplexBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Complex) <-chan error

ComplexBulkAvroWriter will begin a go routine writing an Avro Container File to the writer and add each item from the request channel. If an error is encountered it will be sent on the returned error channel. The given writeTime will be used for all data items written by this function. When the returned request channel is closed this function will finalize the Container File and exit. The returned error channel will be closed just before the go routine exits. Note: That though a nil item will be written as delete it will also be written without an ID or other identifying field and so this is of limited value. In general deletes should be done using WriteAvroDeletedCF.

Example

input := []*Complex{{}, {}, {}}
inputChan := make(chan *Complex)

devnull, _ := os.Open("/dev/null")
defer devnull.Close()

errChan := ComplexBulkAvroWriter(devnull, time.Now(), inputChan)

for _, item := range input {
	select {
	case err := <-errChan:
		fmt.Print(err)
		return
	case inputChan <- item:
	}
}

// Check for any final errors, the errorChan should be closed when the BulkWriter is finished processing
for err := range errChan {
	if err != nil {
		fmt.Print(err)
		return
	}
}

func NestedBulkAvroWriter

func NestedBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Nested) <-chan error

NestedBulkAvroWriter will begin a go routine writing an Avro Container File to the writer and add each item from the request channel. If an error is encountered it will be sent on the returned error channel. The given writeTime will be used for all data items written by this function. When the returned request channel is closed this function will finalize the Container File and exit. The returned error channel will be closed just before the go routine exits. Note: That though a nil item will be written as delete it will also be written without an ID or other identifying field and so this is of limited value. In general deletes should be done using WriteAvroDeletedCF.

Example

input := []*Nested{{}, {}, {}}
inputChan := make(chan *Nested)

devnull, _ := os.Open("/dev/null")
defer devnull.Close()

errChan := NestedBulkAvroWriter(devnull, time.Now(), inputChan)

for _, item := range input {
	select {
	case err := <-errChan:
		fmt.Print(err)
		return
	case inputChan <- item:
	}
}

// Check for any final errors, the errorChan should be closed when the BulkWriter is finished processing
for err := range errChan {
	if err != nil {
		fmt.Print(err)
		return
	}
}

func SimpleBulkAvroWriter

func SimpleBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Simple) <-chan error

SimpleBulkAvroWriter will begin a go routine writing an Avro Container File to the writer and add each item from the request channel. If an error is encountered it will be sent on the returned error channel. The given writeTime will be used for all data items written by this function. When the returned request channel is closed this function will finalize the Container File and exit. The returned error channel will be closed just before the go routine exits. Note: That though a nil item will be written as delete it will also be written without an ID or other identifying field and so this is of limited value. In general deletes should be done using WriteAvroDeletedCF.

Example

input := []*Simple{{}, {}, {}}
inputChan := make(chan *Simple)

devnull, _ := os.Open("/dev/null")
defer devnull.Close()

errChan := SimpleBulkAvroWriter(devnull, time.Now(), inputChan)

for _, item := range input {
	select {
	case err := <-errChan:
		fmt.Print(err)
		return
	case inputChan <- item:
	}
}

// Check for any final errors, the errorChan should be closed when the BulkWriter is finished processing
for err := range errChan {
	if err != nil {
		fmt.Print(err)
		return
	}
}

func TimesBulkAvroWriter

func TimesBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Times) <-chan error

TimesBulkAvroWriter will begin a go routine writing an Avro Container File to the writer and add each item from the request channel. If an error is encountered it will be sent on the returned error channel. The given writeTime will be used for all data items written by this function. When the returned request channel is closed this function will finalize the Container File and exit. The returned error channel will be closed just before the go routine exits. Note: That though a nil item will be written as delete it will also be written without an ID or other identifying field and so this is of limited value. In general deletes should be done using WriteAvroDeletedCF.

Example

input := []*Times{{}, {}, {}}
inputChan := make(chan *Times)

devnull, _ := os.Open("/dev/null")
defer devnull.Close()

errChan := TimesBulkAvroWriter(devnull, time.Now(), inputChan)

for _, item := range input {
	select {
	case err := <-errChan:
		fmt.Print(err)
		return
	case inputChan <- item:
	}
}

// Check for any final errors, the errorChan should be closed when the BulkWriter is finished processing
for err := range errChan {
	if err != nil {
		fmt.Print(err)
		return
	}
}

Types

type Complex

type Complex struct {
	Simple

	Caption string `json:"caption"`
	Credit  string `json:"credit"`
	// The available cropped images
	Crops          []ComplexCrops      `json:"crops"`
	Cutline        string              `json:"cutline,omitempty"`
	DatePhotoTaken time.Time           `json:"datePhotoTaken"`
	Orientation    string              `json:"orientation"`
	OriginalSize   ComplexOriginalSize `json:"originalSize"`
	// a type
	Type string `json:"type"`
	// Universal Resource Locator
	URL ComplexURL `json:"URL"`
}

func (*Complex) WriteAvroCF

func (z *Complex) WriteAvroCF(writer io.Writer, writeTime time.Time) error

WriteAvroCF writes an Avro Containter File to the given io.Writer using snappy compression for the data. The time is used as the AvroWriteTime, if the time is the Zero value then the current time is used. NOTE: If the type has a field in an embedded struct with the same name as a field not in the embedded struct the value will be pulled from the field not in the embedded struct.

func (*Complex) WriteAvroDeletedCF

func (z *Complex) WriteAvroDeletedCF(writer io.Writer, writeTime time.Time) error

WriteAvroDeletedCF works nearly identically to WriteAvroCF but sets the AvroDeleted metadata field to true.

type ComplexCrops

type ComplexCrops struct {
	Height float64 `json:"height"`
	Name   string  `json:"name"`
	// full path to the cropped image file
	Path string `json:"path"`
	// a long
	// multi-line description
	RelativePath string  `json:"relativePath"`
	Width        float64 `json:"width"`
}

type ComplexOriginalSize

type ComplexOriginalSize struct {
	Height float64 `json:"height"`
	Width  float64 `json:"width"`
}

type ComplexURL

type ComplexURL struct {
	// The full Canonical URL
	Absolute string          `json:"absolute"`
	Meta     *ComplexURLMeta `json:"meta,omitempty"`
	Publish  string          `json:"publish"`
}

type ComplexURLMeta

type ComplexURLMeta struct {
	Description string `json:"description"`
	SiteName    string `json:"siteName"`
}

type Nested

type Nested struct {
	// Information related to the International Fact-Checking Network (IFCN) program
	FactCheckClaims []NestedFactCheckClaims `json:"factCheckClaims"`
}

func (*Nested) WriteAvroCF

func (z *Nested) WriteAvroCF(writer io.Writer, writeTime time.Time) error

WriteAvroCF writes an Avro Containter File to the given io.Writer using snappy compression for the data. The time is used as the AvroWriteTime, if the time is the Zero value then the current time is used. NOTE: If the type has a field in an embedded struct with the same name as a field not in the embedded struct the value will be pulled from the field not in the embedded struct.

func (*Nested) WriteAvroDeletedCF

func (z *Nested) WriteAvroDeletedCF(writer io.Writer, writeTime time.Time) error

WriteAvroDeletedCF works nearly identically to WriteAvroCF but sets the AvroDeleted metadata field to true.

type NestedFactCheckClaims

type NestedFactCheckClaims struct {
	// The factCheck appearanceURLs
	AppearanceURLs []*NestedFactCheckClaimsAppearanceURLs `json:"appearanceURLs,omitempty"`
	// The factCheck author
	Author string `json:"author,omitempty"`
	// The factCheck claim
	Claim string `json:"claim,omitempty"`
	// The date of the factCheck
	Date string `json:"date,omitempty"`
	// The factCheck rating
	Rating string `json:"rating,omitempty"`
}

type NestedFactCheckClaimsAppearanceURLs

type NestedFactCheckClaimsAppearanceURLs struct {
	// The original flag
	Original bool `json:"original,omitempty"`
	// The appearance url
	Url string `json:"url"`
}

type Simple

type Simple struct {
	Contributors []*SimpleContributors `json:"contributors,omitempty"`
	Height       int64                 `json:"height,omitempty"`
	SomeDateObj  *SimpleSomeDateObj    `json:"someDateObj,omitempty"`
	Type         string                `json:"type"`
	Visible      bool                  `json:"visible,omitempty"`
	Width        float64               `json:"width,omitempty"`
}

func (*Simple) WriteAvroCF

func (z *Simple) WriteAvroCF(writer io.Writer, writeTime time.Time) error

WriteAvroCF writes an Avro Containter File to the given io.Writer using snappy compression for the data. The time is used as the AvroWriteTime, if the time is the Zero value then the current time is used. NOTE: If the type has a field in an embedded struct with the same name as a field not in the embedded struct the value will be pulled from the field not in the embedded struct.

func (*Simple) WriteAvroDeletedCF

func (z *Simple) WriteAvroDeletedCF(writer io.Writer, writeTime time.Time) error

WriteAvroDeletedCF works nearly identically to WriteAvroCF but sets the AvroDeleted metadata field to true.

type SimpleContributors

type SimpleContributors struct {
	ContributorId string `json:"contributorId,omitempty"`
	Id            string `json:"id"`
	Name          string `json:"name"`
}

type SimpleSomeDateObj

type SimpleSomeDateObj struct {
	Dates []*time.Time `json:"dates,omitempty"`
}

type Times

type Times struct {
	NonRequiredDate *time.Time `json:"nonRequiredDate,omitempty"`
	RequiredDate    time.Time  `json:"requiredDate"`
}

func (*Times) WriteAvroCF

func (z *Times) WriteAvroCF(writer io.Writer, writeTime time.Time) error

WriteAvroCF writes an Avro Containter File to the given io.Writer using snappy compression for the data. The time is used as the AvroWriteTime, if the time is the Zero value then the current time is used. NOTE: If the type has a field in an embedded struct with the same name as a field not in the embedded struct the value will be pulled from the field not in the embedded struct.

func (*Times) WriteAvroDeletedCF

func (z *Times) WriteAvroDeletedCF(writer io.Writer, writeTime time.Time) error

WriteAvroDeletedCF works nearly identically to WriteAvroCF but sets the AvroDeleted metadata field to true.