iterator

Documentation

Overview

Package iterator provides support for standard Google API iterators. See https://github.com/GoogleCloudPlatform/gcloud-golang/wiki/Iterator-Guidelines.

Example (PageLoop)

This example demonstrates how to use a Pager to page through an iterator in a loop.

package main

import (
	"fmt"
	"log"
	"math"
	"sort"
	"strconv"

	"google.golang.org/api/iterator"
)

func main() {
	// Find all primes up to 42, in pages of size 5.
	const max = 42
	const pageSize = 5
	p := iterator.NewPager(Primes(max), pageSize, "" /* start from the beginning */)
	for page := 0; ; page++ {
		var items []int
		pageToken, err := p.NextPage(&items)
		if err != nil {
			log.Fatalf("Iterator paging failed: %v", err)
		}
		fmt.Printf("Page %d: %v\n", page, items)
		if pageToken == "" {
			break
		}
	}
}

// Primes returns a iterator which returns a sequence of prime numbers.
// If non-zero, max specifies the maximum number which could possibly be
// returned.
func Primes(max int) *SieveIterator {
	it := &SieveIterator{pos: 2, max: max}
	it.pageInfo, it.nextFunc = iterator.NewPageInfo(
		it.fetch,
		func() int { return len(it.items) },
		func() interface{} { b := it.items; it.items = nil; return b })
	return it
}

// SieveIterator is an iterator that returns primes using the sieve of
// Eratosthenes. It is a demonstration of how an iterator might work.
// Internally, it uses "page size" as the number of ints to consider,
// and "page token" as the first number to consider (defaults to 2).
type SieveIterator struct {
	pageInfo *iterator.PageInfo
	nextFunc func() error
	max      int
	p        []int
	pos      int
	items    []int
}

// PageInfo returns a PageInfo, which supports pagination.
func (it *SieveIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }

func (it *SieveIterator) fetch(pageSize int, pageToken string) (string, error) {
	start := 2
	if pageToken != "" {
		s, err := strconv.Atoi(pageToken)
		if err != nil || s < 2 {
			return "", fmt.Errorf("invalid token %q", pageToken)
		}
		start = s
	}
	if pageSize == 0 {
		pageSize = 20
	}

	it.calc(start + pageSize)

	items := it.p[sort.SearchInts(it.p, start):]
	items = items[:sort.SearchInts(items, start+pageSize)]
	it.items = append(it.items, items...)

	if it.max > 0 && start+pageSize > it.max {
		return "", nil
	}

	return strconv.Itoa(start + pageSize), nil
}

// calc populates p with all primes up to, but not including, max.
func (it *SieveIterator) calc(max int) {
	if it.max > 0 && max > it.max+1 {
		max = it.max + 1
	}
outer:
	for x := it.pos; x < max; x++ {
		sqrt := int(math.Sqrt(float64(x)))
		for _, p := range it.p {
			switch {
			case x%p == 0:

				continue outer
			case p > sqrt:

				break
			}
		}
		it.p = append(it.p, x)
	}
	it.pos = max
}

func (it *SieveIterator) Next() (int, error) {
	if err := it.nextFunc(); err != nil {
		return 0, err
	}
	item := it.items[0]
	it.items = it.items[1:]
	return item, nil
}

Output:

Page 0: [2 3 5 7 11]
Page 1: [13 17 19 23 29]
Page 2: [31 37 41]

Example (PageToken)

The example demonstrates how to use a Pager to request a page from a given token.

package main

import (
	"fmt"
	"log"
	"math"
	"sort"
	"strconv"

	"google.golang.org/api/iterator"
)

func main() {
	const pageSize = 5
	const pageToken = "1337"
	p := iterator.NewPager(Primes(0), pageSize, pageToken)

	var items []int
	nextPage, err := p.NextPage(&items)
	if err != nil {
		log.Fatalf("Iterator paging failed: %v", err)
	}
	fmt.Printf("Primes: %v\nToken:  %q\n", items, nextPage)
}

// Primes returns a iterator which returns a sequence of prime numbers.
// If non-zero, max specifies the maximum number which could possibly be
// returned.
func Primes(max int) *SieveIterator {
	it := &SieveIterator{pos: 2, max: max}
	it.pageInfo, it.nextFunc = iterator.NewPageInfo(
		it.fetch,
		func() int { return len(it.items) },
		func() interface{} { b := it.items; it.items = nil; return b })
	return it
}

// SieveIterator is an iterator that returns primes using the sieve of
// Eratosthenes. It is a demonstration of how an iterator might work.
// Internally, it uses "page size" as the number of ints to consider,
// and "page token" as the first number to consider (defaults to 2).
type SieveIterator struct {
	pageInfo *iterator.PageInfo
	nextFunc func() error
	max      int
	p        []int
	pos      int
	items    []int
}

// PageInfo returns a PageInfo, which supports pagination.
func (it *SieveIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }

func (it *SieveIterator) fetch(pageSize int, pageToken string) (string, error) {
	start := 2
	if pageToken != "" {
		s, err := strconv.Atoi(pageToken)
		if err != nil || s < 2 {
			return "", fmt.Errorf("invalid token %q", pageToken)
		}
		start = s
	}
	if pageSize == 0 {
		pageSize = 20
	}

	it.calc(start + pageSize)

	items := it.p[sort.SearchInts(it.p, start):]
	items = items[:sort.SearchInts(items, start+pageSize)]
	it.items = append(it.items, items...)

	if it.max > 0 && start+pageSize > it.max {
		return "", nil
	}

	return strconv.Itoa(start + pageSize), nil
}

// calc populates p with all primes up to, but not including, max.
func (it *SieveIterator) calc(max int) {
	if it.max > 0 && max > it.max+1 {
		max = it.max + 1
	}
outer:
	for x := it.pos; x < max; x++ {
		sqrt := int(math.Sqrt(float64(x)))
		for _, p := range it.p {
			switch {
			case x%p == 0:

				continue outer
			case p > sqrt:

				break
			}
		}
		it.p = append(it.p, x)
	}
	it.pos = max
}

func (it *SieveIterator) Next() (int, error) {
	if err := it.nextFunc(); err != nil {
		return 0, err
	}
	item := it.items[0]
	it.items = it.items[1:]
	return item, nil
}

Output:

Primes: [1361 1367 1373 1381 1399]
Token:  "1400"

Example (ServerPages)

This example demonstrates how to get exactly the items in the buffer, without triggering an extra RPC.

package main

import (
	"fmt"
	"log"
	"math"
	"sort"
	"strconv"

	"google.golang.org/api/iterator"
)

func main() {
	// The iterator returned by Primes has a default page size of 20, which means
	// it will return all the primes in the range [2, 21).
	it := Primes(0)
	var items []int
	for {
		item, err := it.Next()
		if err != nil && err != iterator.Done {
			log.Fatal(err)
		}
		if err == iterator.Done {
			break
		}
		items = append(items, item)
		if it.PageInfo().Remaining() == 0 {
			break
		}
	}
	fmt.Println(items)
}

// Primes returns a iterator which returns a sequence of prime numbers.
// If non-zero, max specifies the maximum number which could possibly be
// returned.
func Primes(max int) *SieveIterator {
	it := &SieveIterator{pos: 2, max: max}
	it.pageInfo, it.nextFunc = iterator.NewPageInfo(
		it.fetch,
		func() int { return len(it.items) },
		func() interface{} { b := it.items; it.items = nil; return b })
	return it
}

// SieveIterator is an iterator that returns primes using the sieve of
// Eratosthenes. It is a demonstration of how an iterator might work.
// Internally, it uses "page size" as the number of ints to consider,
// and "page token" as the first number to consider (defaults to 2).
type SieveIterator struct {
	pageInfo *iterator.PageInfo
	nextFunc func() error
	max      int
	p        []int
	pos      int
	items    []int
}

// PageInfo returns a PageInfo, which supports pagination.
func (it *SieveIterator) PageInfo() *iterator.PageInfo { return it.pageInfo }

func (it *SieveIterator) fetch(pageSize int, pageToken string) (string, error) {
	start := 2
	if pageToken != "" {
		s, err := strconv.Atoi(pageToken)
		if err != nil || s < 2 {
			return "", fmt.Errorf("invalid token %q", pageToken)
		}
		start = s
	}
	if pageSize == 0 {
		pageSize = 20
	}

	it.calc(start + pageSize)

	items := it.p[sort.SearchInts(it.p, start):]
	items = items[:sort.SearchInts(items, start+pageSize)]
	it.items = append(it.items, items...)

	if it.max > 0 && start+pageSize > it.max {
		return "", nil
	}

	return strconv.Itoa(start + pageSize), nil
}

// calc populates p with all primes up to, but not including, max.
func (it *SieveIterator) calc(max int) {
	if it.max > 0 && max > it.max+1 {
		max = it.max + 1
	}
outer:
	for x := it.pos; x < max; x++ {
		sqrt := int(math.Sqrt(float64(x)))
		for _, p := range it.p {
			switch {
			case x%p == 0:

				continue outer
			case p > sqrt:

				break
			}
		}
		it.p = append(it.p, x)
	}
	it.pos = max
}

func (it *SieveIterator) Next() (int, error) {
	if err := it.nextFunc(); err != nil {
		return 0, err
	}
	item := it.items[0]
	it.items = it.items[1:]
	return item, nil
}

Output:

[2 3 5 7 11 13 17 19]

Index

• Variables
• type PageInfo
  • func (pi *PageInfo) Remaining() int
• type Pageable
• type Pager
  • func NewPager(iter Pageable, pageSize int, pageToken string) *Pager
  • func (p *Pager) NextPage(slicep interface{}) (nextPageToken string, err error)

Examples

• Package (PageLoop)
• Package (PageToken)
• Package (ServerPages)

Variables

var Done = errors.New("no more items in iterator")

Done is returned by an iterator's Next method when the iteration is complete; when there are no more items to return.

var NewPageInfo = newPageInfo

NewPageInfo exposes internals for iterator implementations. It is not a stable interface.

Types

type PageInfo

type PageInfo struct {
	// Token is the token used to retrieve the next page of items from the
	// API. You may set Token immediately after creating an iterator to
	// begin iteration at a particular point. If Token is the empty string,
	// the iterator will begin with the first eligible item.
	//
	// The result of setting Token after the first call to Next is undefined.
	//
	// After the underlying API method is called to retrieve a page of items,
	// Token is set to the next-page token in the response.
	Token string

	// MaxSize is the maximum number of items returned by a call to the API.
	// Set MaxSize as a hint to optimize the buffering behavior of the iterator.
	// If zero, the page size is determined by the underlying service.
	//
	// Use Pager to retrieve a page of a specific, exact size.
	MaxSize int
	// contains filtered or unexported fields
}

PageInfo contains information about an iterator's paging state.

func (*PageInfo) Remaining

func (pi *PageInfo) Remaining() int

Remaining returns the number of items available before the iterator makes another API call.

type Pageable

type Pageable interface {
	// PageInfo returns paging information associated with the iterator.
	PageInfo() *PageInfo
}

Pageable is implemented by iterators that support paging.

type Pager

type Pager struct {
	// contains filtered or unexported fields
}

Pager supports retrieving iterator items a page at a time.

func NewPager

func NewPager(iter Pageable, pageSize int, pageToken string) *Pager

NewPager returns a pager that uses iter. Calls to its NextPage method will obtain exactly pageSize items, unless fewer remain. The pageToken argument indicates where to start the iteration. Pass the empty string to start at the beginning, or pass a token retrieved from a call to Pager.NextPage.

If you use an iterator with a Pager, you must not call Next on the iterator.

func (*Pager) NextPage

func (p *Pager) NextPage(slicep interface{}) (nextPageToken string, err error)

NextPage retrieves a sequence of items from the iterator and appends them to slicep, which must be a pointer to a slice of the iterator's item type. Exactly p.pageSize items will be appended, unless fewer remain.

The first return value is the page token to use for the next page of items. If empty, there are no more pages. Aside from checking for the end of the iteration, the returned page token is only needed if the iteration is to be resumed a later time, in another context (possibly another process).

The second return value is non-nil if an error occurred. It will never be the special iterator sentinel value Done. To recognize the end of the iteration, compare nextPageToken to the empty string.

It is possible for NextPage to return a single zero-length page along with an empty page token when there are no more items in the iteration.