downloader

Documentation

Overview

Example (GenerateURLs)

urls, urlSymbolMap := GenerateURLs(testSymbols, testURL)
for _, url := range urls {
	fmt.Printf("%s\n", url)
}
fmt.Printf("urlSymbolMap: %+v", urlSymbolMap)

Output:

https://query1.finance.yahoo.com/v7/finance/download/dia?period1=631152000&period2=1672531200&interval=1d&events=history&includeAdjustedClose=true
https://query1.finance.yahoo.com/v7/finance/download/qqq?period1=631152000&period2=1672531200&interval=1d&events=history&includeAdjustedClose=true
urlSymbolMap: map[https://query1.finance.yahoo.com/v7/finance/download/dia:dia https://query1.finance.yahoo.com/v7/finance/download/qqq:qqq]

Index

• Constants
• Variables
• func BaseURL(url string) string
• func GenerateURLs(symbols []string, url string) (urls []string, urlSymbolMap map[string]string)
• func Init(appNameInit string)
• func LoadURLCollectionDataFromFile(dataFilePath string) (urlData []httph.URLCollectionData, err error)
• func StringRecordToFloat64Record(record []string, skipIndices []int, symbol string) (out []float64, nulls int, err error)
• func URLCollectionDataDateOrder(records [][]string, urlCollectionDataHeaderIndicesMap map[string]int) (orderAsc bool, dateFirst time.Time, dateLast time.Time)
• type Data
  • func (dt Data) String() string
• type DatasetAsColumns
  • func (dac DatasetAsColumns) String() string
• type Downloader
• type Group
  • func NewGroup(liveData bool, dataFilePath string, name string, symbols []string, url string, ...) (*Group, error)
  • func (grp Group) SaveCSV(dataFilePath string) error
  • func (grp Group) String() string
• type Issue
  • func (iss Issue) String() string
  • func (iss Issue) ToDatasetAsColumns() DatasetAsColumns
• type URLCollectionDataToGroup

Examples

• Package (GenerateURLs)
• BaseURL

Constants

const (
	BinaryExtension = ".bin"
	CSVExtension    = ".csv"
	DateFormat      = "2006-01-02"

	// Floats are rounded to this number of decimal points. Yahoo will slightly alter some values
	// with each call, but only at very high numbers of decimal places. That makes debugging
	// difficult.
	InputPrecision = 4

	// err on the side of caution; if yahoo sees to much traffic from the same user, it may block.
	Threads = 1
)

Variables

var (

	// earliest date for which data is fetched.
	EarliestDate = time.Date(1990, time.January, 1, 0, 0, 0, 0, time.UTC).Unix()
	LatestDate   = time.Now().AddDate(0, 0, 1).Unix()

	URLCollectionTimeout = time.Duration(10 * time.Second)
)

Functions

func BaseURL

func BaseURL(url string) string

Example

urls, _ := GenerateURLs([]string{"qqq"}, testURL)
fmt.Printf("%s\n", urls[0])
fmt.Printf("%s\n", BaseURL(urls[0]))

Output:

https://query1.finance.yahoo.com/v7/finance/download/qqq?period1=631152000&period2=1672531200&interval=1d&events=history&includeAdjustedClose=true
https://query1.finance.yahoo.com/v7/finance/download/qqq

func GenerateURLs

func GenerateURLs(symbols []string, url string) (urls []string, urlSymbolMap map[string]string)

GenerateURLs generates URLs for the symbols as well as a symbol to URL map for use with data returned from collectGroup

func Init

func Init(appNameInit string)

func LoadURLCollectionDataFromFile

func LoadURLCollectionDataFromFile(dataFilePath string) (urlData []httph.URLCollectionData, err error)

func StringRecordToFloat64Record

func StringRecordToFloat64Record(record []string, skipIndices []int, symbol string) (out []float64, nulls int, err error)

func URLCollectionDataDateOrder

func URLCollectionDataDateOrder(records [][]string, urlCollectionDataHeaderIndicesMap map[string]int) (orderAsc bool,
	dateFirst time.Time, dateLast time.Time)

Types

type Data

type Data struct {
	Date      time.Time
	Open      float64
	High      float64
	Low       float64
	Close     float64
	Volume    float64
	AdjOpen   float64
	AdjHigh   float64
	AdjLow    float64
	AdjClose  float64
	AdjVolume float64
}

Data is used to Unmarshal data. This structure must match the JSON returned from finance.yahoo. Returned data is always sorted in Date ascending order.

func (Data) String

func (dt Data) String() string

type DatasetAsColumns

type DatasetAsColumns struct {
	Date      []time.Time
	Open      []float64
	High      []float64
	Low       []float64
	Close     []float64
	Volume    []float64
	AdjOpen   []float64
	AdjHigh   []float64
	AdjLow    []float64
	AdjClose  []float64
	AdjVolume []float64
}

func (DatasetAsColumns) String

func (dac DatasetAsColumns) String() string

type Downloader

type Downloader interface {
	NewGroup(liveData bool, dataFilePath string, name string, symbols []string) (*Group, error)
}

type Group

type Group struct {
	Name   string
	Issues []Issue
}

Group is a collection of Issues (stocks, ETFs, etc.)

func NewGroup

func NewGroup(liveData bool, dataFilePath string, name string, symbols []string, url string,
	callbackURLCollectionDataToGroup URLCollectionDataToGroup) (*Group, error)

NewGroup is a factory for Group. liveData == true, data is downloaded from Yahoo; otherwise it is loaded from a file saved from the prior call.

func (Group) SaveCSV

func (grp Group) SaveCSV(dataFilePath string) error

func (Group) String

func (grp Group) String() string

type Issue

type Issue struct {
	Symbol string
	URL    string
	// Dataset is row based data, as source data is row based.
	Dataset []Data
	// DatasetAsColumns is column based data as that is sometimes easier to work with.
	DatasetAsColumns DatasetAsColumns
}

Issue is an item for which Data is collected. I.E. a stock, ETF, etc.

func (Issue) String

func (iss Issue) String() string

func (Issue) ToDatasetAsColumns

func (iss Issue) ToDatasetAsColumns() DatasetAsColumns

ToDatasetAsColumns converts an issue from a row format to column format

type URLCollectionDataToGroup

type URLCollectionDataToGroup func(urlData []httph.URLCollectionData, urlSymbolMap map[string]string, name string) (group *Group, err error)