imhashdb

package module
v0.0.0-...-997f03f Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Mar 28, 2021 License: GPL-3.0 Imports: 21 Imported by: 0

README

imhashdb

Setup (development):
  1. Install PostgreSQL 12
  2. Install pg_hamming
  3. Create a database called imhashdb (su postgres; createdb imhashdb)
  4. Install redis-server
  5. Install fastimagehash
  6. Start an instance of imhashdb hasher (See ./imhashdb --help for CLI options)
  7. Start an image crawler (e.g. reddit_feed), with ARC_LIST=imhash,...
  8. Start an instance of imhashdb web
  9. Start the development web server imhashdb-frontend
  • Run npm run start

Documentation

Index

Constants

View Source 
const MaxDistance = 100
View Source 
const MaxLimit = 1000
View Source 
const UserAgent = "imhashdb/v1.0"

Variables

View Source 
var HashTypes = []HashType{
	DHash8, DHash16, DHash32,
	MHash8, MHash16, MHash32,
	PHash8, PHash16, PHash32,
	WHash8Haar, WHash16Haar, WHash32Haar,
}
View Source 
var ImageBlackList = []string{}
View Source 
var ImageSuffixes = []string{
	".jpeg", ".jpg", ".png",
	".jpeg:orig", ".jpg:orig", ".png:orig",
	".bmp", ".webp",
}
View Source 
var Logger *zap.Logger
View Source 
var MatchTriggers = []MatchTrigger{
	{
		HashType:    PHash16,
		MinDistance: 25,
		Id:          2,
	},
}
View Source 
var Pgdb *pgx.ConnPool
View Source 
var Rdb *redis.Client
View Source 
var ReImgurAlbum = regexp.MustCompile("(?:https?://)?(?:www\\.|[im]\\.)?imgur\\.com/a/(\\w{7}|\\w{5})")
View Source 
var ReImgurImg = regexp.MustCompile("(?:https?://)?(?:www\\.|[im]\\.)?imgur\\.com/(\\w{7}|\\w{5})[sbtmlh]?")

Functions

func DbConnect 

func DbConnect(host string, port int, user, password, database string) *pgx.ConnPool

func DbInit 

func DbInit(pool *pgx.ConnPool)

func Fetch 

func Fetch(link string, headers ...[]string) ([]byte, error)

func FetchJson 

func FetchJson(link string, v interface{}, raw *[]byte, headers ...[]string) error

func FindImagesByHash 

func FindImagesByHash(ctx context.Context, hash []byte, hashType HashType, distance, limit, offset uint) ([]byte, error)

func Init 

func Init()
func IsImageLink(link string) bool

func Store 

func Store(entry *Entry)
func TransformLink(link string, meta *[]Meta) []string

Types

type Config 

type Config struct {
	PgUser     string
	PgPassword string
	PgDb       string
	PgHost     string
	PgPort     int

	RedisAddr     string
	RedisPassword string
	RedisDb       int

	ApiAddr string

	HasherConcurrency int
	QueryConcurrency  int

	Store string

	ImgurClientId string
	HasherPattern string
}
var Conf Config

type Entry 

type Entry struct {
	H      *Hashes
	Size   int
	Sha1   [sha1.Size]byte
	Md5    [md5.Size]byte
	Sha256 [sha256.Size]byte
	Crc32  uint32
	Meta   []Meta
	Url    string
}

type HashReq 

type HashReq struct {
	Data []byte `json:"data"`
}

func (HashReq) MarshalEasyJSON 

func (v HashReq) MarshalEasyJSON(w *jwriter.Writer)

MarshalEasyJSON supports easyjson.Marshaler interface

func (HashReq) MarshalJSON 

func (v HashReq) MarshalJSON() ([]byte, error)

MarshalJSON supports json.Marshaler interface

func (*HashReq) UnmarshalEasyJSON 

func (v *HashReq) UnmarshalEasyJSON(l *jlexer.Lexer)

UnmarshalEasyJSON supports easyjson.Unmarshaler interface

func (*HashReq) UnmarshalJSON 

func (v *HashReq) UnmarshalJSON(data []byte) error

UnmarshalJSON supports json.Unmarshaler interface

type HashType 

type HashType string
const (
	DHash8  HashType = "dhash8"
	DHash16 HashType = "dhash16"
	DHash32 HashType = "dhash32"

	MHash8  HashType = "mhash8"
	MHash16 HashType = "mhash16"
	MHash32 HashType = "mhash32"

	PHash8  HashType = "phash8"
	PHash16 HashType = "phash16"
	PHash32 HashType = "phash32"

	WHash8Haar  HashType = "whash8haar"
	WHash16Haar HashType = "whash16haar"
	WHash32Haar HashType = "whash32haar"
)

func (HashType) HashLength 

func (h HashType) HashLength() int

type Hashes 

type Hashes struct {
	DHash8  *fastimagehash.Hash `json:"dhash8"`
	DHash16 *fastimagehash.Hash `json:"dhash16"`
	DHash32 *fastimagehash.Hash `json:"dhash32"`

	MHash8  *fastimagehash.Hash `json:"mhash8"`
	MHash16 *fastimagehash.Hash `json:"mhash16"`
	MHash32 *fastimagehash.Hash `json:"mhash32"`

	PHash8  *fastimagehash.Hash `json:"phash8"`
	PHash16 *fastimagehash.Hash `json:"phash16"`
	PHash32 *fastimagehash.Hash `json:"phash32"`

	WHash8  *fastimagehash.Hash `json:"whash8haar"`
	WHash16 *fastimagehash.Hash `json:"whash16haar"`
	WHash32 *fastimagehash.Hash `json:"whash32haar"`
}

func ComputeHash 

func ComputeHash(data []byte) (*Hashes, error)

func (Hashes) MarshalEasyJSON 

func (v Hashes) MarshalEasyJSON(w *jwriter.Writer)

MarshalEasyJSON supports easyjson.Marshaler interface

func (Hashes) MarshalJSON 

func (v Hashes) MarshalJSON() ([]byte, error)

MarshalJSON supports json.Marshaler interface

func (*Hashes) UnmarshalEasyJSON 

func (v *Hashes) UnmarshalEasyJSON(l *jlexer.Lexer)

UnmarshalEasyJSON supports easyjson.Unmarshaler interface

func (*Hashes) UnmarshalJSON 

func (v *Hashes) UnmarshalJSON(data []byte) error

UnmarshalJSON supports json.Unmarshaler interface

type Image 

type Image struct {
	Size   int    `json:"size"`
	Sha1   []byte `json:"sha1"`
	Md5    []byte `json:"md5"`
	Sha256 []byte `json:"sha256"`
	Crc32  uint32 `json:"crc32"`

	Meta []ImageHasMeta `json:"meta"`
	// contains filtered or unexported fields
}

func (Image) MarshalEasyJSON 

func (v Image) MarshalEasyJSON(w *jwriter.Writer)

MarshalEasyJSON supports easyjson.Marshaler interface

func (Image) MarshalJSON 

func (v Image) MarshalJSON() ([]byte, error)

MarshalJSON supports json.Marshaler interface

func (*Image) UnmarshalEasyJSON 

func (v *Image) UnmarshalEasyJSON(l *jlexer.Lexer)

UnmarshalEasyJSON supports easyjson.Unmarshaler interface

func (*Image) UnmarshalJSON 

func (v *Image) UnmarshalJSON(data []byte) error

UnmarshalJSON supports json.Unmarshaler interface

type ImageHasMeta 

type ImageHasMeta struct {
	Url  string `json:"url"`
	Meta Meta   `json:"meta"`
}

func (ImageHasMeta) MarshalEasyJSON 

func (v ImageHasMeta) MarshalEasyJSON(w *jwriter.Writer)

MarshalEasyJSON supports easyjson.Marshaler interface

func (ImageHasMeta) MarshalJSON 

func (v ImageHasMeta) MarshalJSON() ([]byte, error)

MarshalJSON supports json.Marshaler interface

func (*ImageHasMeta) UnmarshalEasyJSON 

func (v *ImageHasMeta) UnmarshalEasyJSON(l *jlexer.Lexer)

UnmarshalEasyJSON supports easyjson.Unmarshaler interface

func (*ImageHasMeta) UnmarshalJSON 

func (v *ImageHasMeta) UnmarshalJSON(data []byte) error

UnmarshalJSON supports json.Unmarshaler interface

type ImageList 

type ImageList struct {
	Images []*Image `json:"images"`
}

func (ImageList) MarshalEasyJSON 

func (v ImageList) MarshalEasyJSON(w *jwriter.Writer)

MarshalEasyJSON supports easyjson.Marshaler interface

func (ImageList) MarshalJSON 

func (v ImageList) MarshalJSON() ([]byte, error)

MarshalJSON supports json.Marshaler interface

func (*ImageList) UnmarshalEasyJSON 

func (v *ImageList) UnmarshalEasyJSON(l *jlexer.Lexer)

UnmarshalEasyJSON supports easyjson.Unmarshaler interface

func (*ImageList) UnmarshalJSON 

func (v *ImageList) UnmarshalJSON(data []byte) error

UnmarshalJSON supports json.Unmarshaler interface

type ImgurAlbumResp 

type ImgurAlbumResp struct {
	Data struct {
		Images []ImgurImg `json:"images"`
	} `json:"data"`
}

type ImgurImg 

type ImgurImg struct {
	Link string `json:"link"`
}

type ImgurImgResp 

type ImgurImgResp struct {
	Data ImgurImg `json:"data"`
}

type MatchTrigger 

type MatchTrigger struct {
	HashType    HashType
	MinDistance int
	Id          int
}

type Meta 

type Meta struct {
	RetrievedAt int64       `json:"retrieved_at"`
	Id          string      `json:"id"`
	Meta        interface{} `json:"meta"`
}

func (Meta) MarshalEasyJSON 

func (v Meta) MarshalEasyJSON(w *jwriter.Writer)

MarshalEasyJSON supports easyjson.Marshaler interface

func (Meta) MarshalJSON 

func (v Meta) MarshalJSON() ([]byte, error)

MarshalJSON supports json.Marshaler interface

func (*Meta) UnmarshalEasyJSON 

func (v *Meta) UnmarshalEasyJSON(l *jlexer.Lexer)

UnmarshalEasyJSON supports easyjson.Unmarshaler interface

func (*Meta) UnmarshalJSON 

func (v *Meta) UnmarshalJSON(data []byte) error

UnmarshalJSON supports json.Unmarshaler interface

type QueryReq 

type QueryReq struct {
	HashType HashType `json:"type"`
	Hash     []byte   `json:"hash"`
	Distance uint     `json:"distance"`
	Limit    uint     `json:"limit"`
	Offset   uint     `json:"offset"`
}

func (QueryReq) MarshalEasyJSON 

func (v QueryReq) MarshalEasyJSON(w *jwriter.Writer)

MarshalEasyJSON supports easyjson.Marshaler interface

func (QueryReq) MarshalJSON 

func (v QueryReq) MarshalJSON() ([]byte, error)

MarshalJSON supports json.Marshaler interface

func (*QueryReq) UnmarshalEasyJSON 

func (v *QueryReq) UnmarshalEasyJSON(l *jlexer.Lexer)

UnmarshalEasyJSON supports easyjson.Unmarshaler interface

func (*QueryReq) UnmarshalJSON 

func (v *QueryReq) UnmarshalJSON(data []byte) error

UnmarshalJSON supports json.Unmarshaler interface

type QueryResp 

type QueryResp struct {
	Err string `json:"err,omitempty"`
}

func (QueryResp) MarshalEasyJSON 

func (v QueryResp) MarshalEasyJSON(w *jwriter.Writer)

MarshalEasyJSON supports easyjson.Marshaler interface

func (QueryResp) MarshalJSON 

func (v QueryResp) MarshalJSON() ([]byte, error)

MarshalJSON supports json.Marshaler interface

func (*QueryResp) UnmarshalEasyJSON 

func (v *QueryResp) UnmarshalEasyJSON(l *jlexer.Lexer)

UnmarshalEasyJSON supports easyjson.Unmarshaler interface

func (*QueryResp) UnmarshalJSON 

func (v *QueryResp) UnmarshalJSON(data []byte) error

UnmarshalJSON supports json.Unmarshaler interface

Source Files

View all Source files

Directories

Path Synopsis

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.