nonest

Documentation

Index

• func ComplexBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Complex) <-chan error
• func SimpleBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Simple) <-chan error
• type Complex
  • func (z *Complex) WriteAvroCF(writer io.Writer, writeTime time.Time) error
  • func (z *Complex) WriteAvroDeletedCF(writer io.Writer, writeTime time.Time) error
• type ComplexCrops
• type ComplexOriginalSize
• type ComplexURL
• type ComplexURLMeta
• type Simple
  • func (z *Simple) WriteAvroCF(writer io.Writer, writeTime time.Time) error
  • func (z *Simple) WriteAvroDeletedCF(writer io.Writer, writeTime time.Time) error
• type SimpleContributors
• type SimpleSomeDateObj

Examples

• ComplexBulkAvroWriter
• SimpleBulkAvroWriter

Functions

func ComplexBulkAvroWriter

func ComplexBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Complex) <-chan error

ComplexBulkAvroWriter will begin a go routine writing an Avro Container File to the writer and add each item from the request channel. If an error is encountered it will be sent on the returned error channel. The given writeTime will be used for all data items written by this function. When the returned request channel is closed this function will finalize the Container File and exit. The returned error channel will be closed just before the go routine exits. Note: That though a nil item will be written as delete it will also be written without an ID or other identifying field and so this is of limited value. In general deletes should be done using WriteAvroDeletedCF.

Example

input := []*Complex{{}, {}, {}}
inputChan := make(chan *Complex)

devnull, _ := os.Open("/dev/null")
defer devnull.Close()

errChan := ComplexBulkAvroWriter(devnull, time.Now(), inputChan)

for _, item := range input {
	select {
	case err := <-errChan:
		fmt.Print(err)
		return
	case inputChan <- item:
	}
}

// Check for any final errors, the errorChan should be closed when the BulkWriter is finished processing
for err := range errChan {
	if err != nil {
		fmt.Print(err)
		return
	}
}

func SimpleBulkAvroWriter

func SimpleBulkAvroWriter(writer io.Writer, writeTime time.Time, request <-chan *Simple) <-chan error

SimpleBulkAvroWriter will begin a go routine writing an Avro Container File to the writer and add each item from the request channel. If an error is encountered it will be sent on the returned error channel. The given writeTime will be used for all data items written by this function. When the returned request channel is closed this function will finalize the Container File and exit. The returned error channel will be closed just before the go routine exits. Note: That though a nil item will be written as delete it will also be written without an ID or other identifying field and so this is of limited value. In general deletes should be done using WriteAvroDeletedCF.

Example

input := []*Simple{{}, {}, {}}
inputChan := make(chan *Simple)

devnull, _ := os.Open("/dev/null")
defer devnull.Close()

errChan := SimpleBulkAvroWriter(devnull, time.Now(), inputChan)

for _, item := range input {
	select {
	case err := <-errChan:
		fmt.Print(err)
		return
	case inputChan <- item:
	}
}

// Check for any final errors, the errorChan should be closed when the BulkWriter is finished processing
for err := range errChan {
	if err != nil {
		fmt.Print(err)
		return
	}
}

Types

type Complex

type Complex struct {
	Simple

	Caption string `json:"caption"`
	Credit  string `json:"credit"`
	// The available cropped images
	Crops          []ComplexCrops      `json:"crops"`
	Cutline        string              `json:"cutline,omitempty"`
	DatePhotoTaken time.Time           `json:"datePhotoTaken"`
	Orientation    string              `json:"orientation"`
	OriginalSize   ComplexOriginalSize `json:"originalSize"`
	// a type
	Type string `json:"type"`
	// Universal Resource Locator
	URL ComplexURL `json:"URL"`
}

func (*Complex) WriteAvroCF

func (z *Complex) WriteAvroCF(writer io.Writer, writeTime time.Time) error

WriteAvroCF writes an Avro Containter File to the given io.Writer using snappy compression for the data. The time is used as the AvroWriteTime, if the time is the Zero value then the current time is used. NOTE: If the type has a field in an embedded struct with the same name as a field not in the embedded struct the value will be pulled from the field not in the embedded struct.

func (*Complex) WriteAvroDeletedCF

func (z *Complex) WriteAvroDeletedCF(writer io.Writer, writeTime time.Time) error

WriteAvroDeletedCF works nearly identically to WriteAvroCF but sets the AvroDeleted metadata field to true.

type ComplexCrops

type ComplexCrops struct {
	Height float64 `json:"height"`
	Name   string  `json:"name"`
	// full path to the cropped image file
	Path string `json:"path"`
	// a long
	// multi-line description
	RelativePath string  `json:"relativePath"`
	Width        float64 `json:"width"`
}

type ComplexOriginalSize

type ComplexOriginalSize struct {
	Height float64 `json:"height"`
	Width  float64 `json:"width"`
}

type ComplexURL

type ComplexURL struct {
	// The full Canonical URL
	Absolute string          `json:"absolute"`
	Meta     *ComplexURLMeta `json:"meta,omitempty"`
	Publish  string          `json:"publish"`
}

type ComplexURLMeta

type ComplexURLMeta struct {
	Description string `json:"description"`
	SiteName    string `json:"siteName"`
}

type Simple

type Simple struct {
	Contributors []*SimpleContributors `json:"contributors,omitempty"`
	Height       int64                 `json:"height,omitempty"`
	SomeDateObj  *SimpleSomeDateObj    `json:"someDateObj,omitempty"`
	Type         string                `json:"type"`
	Visible      bool                  `json:"visible,omitempty"`
	Width        float64               `json:"width,omitempty"`
}

func (*Simple) WriteAvroCF

func (z *Simple) WriteAvroCF(writer io.Writer, writeTime time.Time) error

WriteAvroCF writes an Avro Containter File to the given io.Writer using snappy compression for the data. The time is used as the AvroWriteTime, if the time is the Zero value then the current time is used. NOTE: If the type has a field in an embedded struct with the same name as a field not in the embedded struct the value will be pulled from the field not in the embedded struct.

func (*Simple) WriteAvroDeletedCF

func (z *Simple) WriteAvroDeletedCF(writer io.Writer, writeTime time.Time) error

WriteAvroDeletedCF works nearly identically to WriteAvroCF but sets the AvroDeleted metadata field to true.

type SimpleContributors

type SimpleContributors struct {
	ContributorId string `json:"contributorId,omitempty"`
	Id            string `json:"id"`
	Name          string `json:"name"`
}

type SimpleSomeDateObj

type SimpleSomeDateObj struct {
	Dates []time.Time `json:"dates,omitempty"`
}