crawler

Documentation

Overview

*

• Copyright (c) 2019-present Future Corporation *
• Licensed under the Apache License, Version 2.0 (the "License");
• you may not use this file except in compliance with the License.
• You may obtain a copy of the License at *
• http://www.apache.org/licenses/LICENSE-2.0 *
• Unless required by applicable law or agreed to in writing, software
• distributed under the License is distributed on an "AS IS" BASIS,
• WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
• See the License for the specific language governing permissions and
• limitations under the License.

*

• Copyright (c) 2019-present Future Corporation *
• Licensed under the Apache License, Version 2.0 (the "License");
• you may not use this file except in compliance with the License.
• You may obtain a copy of the License at *
• http://www.apache.org/licenses/LICENSE-2.0 *
• Unless required by applicable law or agreed to in writing, software
• distributed under the License is distributed on an "AS IS" BASIS,
• WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
• See the License for the specific language governing permissions and
• limitations under the License.

Index

• Constants
• func Contains(arr []int, e int) bool
• type Crawler
  • func NewGitHubCrawler(token string) Crawler
• type File
• type Files
  • func (fs Files) Index(file File) int
  • func (fs Files) Merge(f File) Files
• type Repositories
  • func (rs Repositories) Exclude(targetIndexes []int) Repositories
  • func (rs Repositories) Index(r Repository) (index int)
  • func (rs Repositories) Merge(r Repository) Repositories
• type Repository

Constants

const MaxPageSize = 100

Functions

func Contains

func Contains(arr []int, e int) bool

Types

type Crawler

type Crawler interface {
	Search(ctx context.Context, words []string) (Repositories, error)
	FulfillForkSource(ctx context.Context, repos Repositories) (Repositories, error)
}

func NewGitHubCrawler

func NewGitHubCrawler(token string) Crawler

type File

type File struct {
	URL       string
	Fragments []string
}

Fragments is represents github api result https://developer.github.com/v3/search/#text-match-metadata

type Files

type Files []File

func (Files) Index

func (fs Files) Index(file File) int

func (Files) Merge

func (fs Files) Merge(f File) Files

type Repositories

type Repositories []Repository

func (Repositories) Exclude

func (rs Repositories) Exclude(targetIndexes []int) Repositories

func (Repositories) Index

func (rs Repositories) Index(r Repository) (index int)

func (Repositories) Merge

func (rs Repositories) Merge(r Repository) Repositories

type Repository

type Repository struct {
	URL        string
	Owner      string
	Name       string
	HitFiles   Files
	ForkSource string // parent is the repository this repository was forked from, source is the ultimate source for the network. https://developer.github.com/v3/repos/#response-4
}