scrapper

Documentation

Index

• Constants
• func LoadFile(filename string, object any) error
• func QueryHTML(url string) *goquery.Document
• type DateTime
  • func (s DateTime) MarshalJSON() ([]byte, error)
  • func (s *DateTime) UnmarshalJSON(bytes []byte) error
• type MetadataFileManager
  • func NewMetadataFileManager(schemas []byte, locator fileconv.FileLocator) *MetadataFileManager
  • func (m MetadataFileManager) LoadIndex() (*ModelURLRegistry, error)
  • func (m MetadataFileManager) SaveIndex(index *ModelURLRegistry) error
  • func (m MetadataFileManager) SaveQueryParamStats(stats *QueryParamStats) error
• type ModelDocLink
• type ModelDocLinks
  • func (l ModelDocLinks) FindByName(name string) (ModelDocLink, bool)
• type ModelURLRegistry
  • func NewModelURLRegistry() *ModelURLRegistry
  • func (r *ModelURLRegistry) Add(displayName, url string)
  • func (r *ModelURLRegistry) Sort()
• type QueryParamStats
  • func CalculateQueryParamStats(queryParamRegistry datautils.NamedLists[string]) *QueryParamStats
  • func NewQueryParamStats(totalObjects int) *QueryParamStats
  • func (s *QueryParamStats) SaveParameterStats(queryParamName string, objectNames []string)

Constants

const (
	IndexFile           = "index.json"
	QueryParamStatsFile = "queryParamStats.json"
)

Functions

func LoadFile

func LoadFile(filename string, object any) error

func QueryHTML

func QueryHTML(url string) *goquery.Document

QueryHTML this method can panic.

Types

type DateTime

type DateTime struct {
	Time time.Time
}

func (DateTime) MarshalJSON

func (s DateTime) MarshalJSON() ([]byte, error)

func (*DateTime) UnmarshalJSON

func (s *DateTime) UnmarshalJSON(bytes []byte) error

type MetadataFileManager

type MetadataFileManager struct {
	staticschema.FileManager
	// contains filtered or unexported fields
}

func NewMetadataFileManager

func NewMetadataFileManager(schemas []byte, locator fileconv.FileLocator) *MetadataFileManager

func (MetadataFileManager) LoadIndex

func (m MetadataFileManager) LoadIndex() (*ModelURLRegistry, error)

func (MetadataFileManager) SaveIndex

func (m MetadataFileManager) SaveIndex(index *ModelURLRegistry) error

func (MetadataFileManager) SaveQueryParamStats

func (m MetadataFileManager) SaveQueryParamStats(stats *QueryParamStats) error

type ModelDocLink

type ModelDocLink struct {
	Name        string `json:"name"`
	DisplayName string `json:"displayName"`
	URL         string `json:"url"`
}

type ModelDocLinks

type ModelDocLinks []ModelDocLink

func (ModelDocLinks) FindByName

func (l ModelDocLinks) FindByName(name string) (ModelDocLink, bool)

type ModelURLRegistry

type ModelURLRegistry struct {
	ModelDocs ModelDocLinks `json:"data"`
}

func NewModelURLRegistry

func NewModelURLRegistry() *ModelURLRegistry

func (*ModelURLRegistry) Add

func (r *ModelURLRegistry) Add(displayName, url string)

func (*ModelURLRegistry) Sort

func (r *ModelURLRegistry) Sort()

type QueryParamStats

type QueryParamStats struct {
	Meta queryParamStatsMeta     `json:"meta"`
	Data []queryParamObjectStats `json:"queryParams"`
}

func CalculateQueryParamStats

func CalculateQueryParamStats(queryParamRegistry datautils.NamedLists[string]) *QueryParamStats

CalculateQueryParamStats produces statistics on objects and their query parameters. queryParamRegistry - holds query parameter name to the list of object names that use it.

func NewQueryParamStats

func NewQueryParamStats(totalObjects int) *QueryParamStats

func (*QueryParamStats) SaveParameterStats

func (s *QueryParamStats) SaveParameterStats(queryParamName string, objectNames []string)