billtitles

README

# billtitles

An application to load bill title data into a sql repository. This repository works in conjunction with `github.com/aih/bills`, a Go repository to process bill metadata files to create an index of `title: bills`.

This repository provides a module that converts the json output from `bills` into sqlite3. It also provides a (crude) Go API to query for bills and titles.

To test and build, install `cmake` and run `make`. This will initially take some time to load data, then run the tests and build three executables into the `cmd/bin` directory:

billquery:: a Go http service to retrieve bill metadata by bill number, and retrieve related bills (superceded by the API at `billtitles-py`) 

related:: collects the 'esSimilarCategory.json' files in the directory provided at `-p`, and adds them to the sql database

titlesjson:: loads title data from files `data/titles.json` and `data/maintitles.json` into a sql repository (`billtitles.db`)

Documentation

Index

• Constants
• Variables
• func AddBillNumbersToTitle(titleMap *sync.Map, title string, billnumbers []string) (*sync.Map, error)
• func AddBillStructDb(db *gorm.DB, billStruct *Bill)
• func AddBillnumberversionsDb(db *gorm.DB, billnumberversions []string)
• func AddTitle(titleMap *sync.Map, title string) (*sync.Map, error)
• func AddTitleDb(db *gorm.DB, title string) *gorm.DB
• func AddTitleStructDb(db *gorm.DB, title *Title)
• func BillNumberVersionToBillNumber(billNumberVersion string) string
• func BillNumberVersionsToBillNumbers(billNumberVersions []string) (billNumbers []string)
• func GetBillnumbersByTitle(titleMap *sync.Map, title string) (billnumbers []string, err error)
• func GetDb(dbname string) *gorm.DB
• func GetDb_old(dbname string) *gorm.DB
• func GetRelatedDb(dbname string) *gorm.DB
• func GetSyncMapKeys(m *sync.Map) (s string)
• func LoadBillTitleToDBFromJson(db *gorm.DB, jsonPath string, isForWhole bool)
• func LoadBillsRelatedToDBFromJson(db *gorm.DB, parentPath string)
• func LoadEnv() (err error)
• func LoadTitlesMap(titlePath string) (*sync.Map, error)
• func LoadTitlesToDBFromJson(db *gorm.DB, jsonPath string)
• func MakeSampleTitlesFile(titleMap *sync.Map)
• func MarshalJSONStringArray(m *sync.Map) ([]byte, error)
• func RemoveDuplicates(elements []string) []string
• func RemoveTitle(titleMap *sync.Map, title string) (*sync.Map, error)
• func RemoveTitleDb(db *gorm.DB, title string)
• func SaveTitlesMap(titleMap *sync.Map, titlePath string) (err error)
• func UnmarshalJson(data []byte) (*sync.Map, error)
• func UnmarshalJsonFile(jpath string) (*sync.Map, error)
• func UnmarshalTitlesJson(data []byte) (*sync.Map, error)
• func UnmarshalTitlesJsonFile(jpath string) (*sync.Map, error)
• type Bill
  • func GetBillsByTitleDb(db *gorm.DB, title string) (bills []*Bill)
  • func GetBillsWithSameTitleDb(db *gorm.DB, billnumber string) (bills, bills_whole []*Bill, err error)
• type BillTitle
• type BillToBill
• type LogLevel
• type LogLevels
• type RelatedBillItem
• type RelatedBillMap
  • func GetRelatedBills(titleMap *sync.Map, titleWholeBillMap *sync.Map, ...) (relatedBills RelatedBillMap, err error)
• type Title
  • func GetTitlesByBillnumberDb(db *gorm.DB, billnumber string) (titles []*Title)
  • func GetTitlesByBillnumberVersionDb(db *gorm.DB, billnumberversion string) (titles []*Title)
  • func GetTitlesWholeByBillnumberDb(db *gorm.DB, billnumber string) (titles []*Title)
  • func GetTitlesWholeByBillnumberVersionDb(db *gorm.DB, billnumberversion string) (titles []*Title)

Constants

const BILLSRELATED_DB = "billsrelated.db"

const BILLTITLES_DB = "billtitles.db"

Variables

var (
	TitleIndex              = "titles.json"
	MainTitleIndex          = "maintitles.json"
	SampleTitleIndex        = "sampletitles.json"
	PathToDataDir           = "data"
	TitlesPath              = path.Join(PathToDataDir, TitleIndex)
	MainTitlesPath          = path.Join(PathToDataDir, MainTitleIndex)
	SampleTitlesPath        = path.Join(PathToDataDir, SampleTitleIndex)
	BillnumberRegexCompiled = regexp.MustCompile(`(?P<congress>[1-9][0-9]*)(?P<stage>[a-z]{1,8})(?P<billnumber>[1-9][0-9]*)(?P<version>[a-z]+)?`)
	BillVersionsOrdered     = billVersions{"ih": 0, "rh": 1, "rfs": 2, "eh": 3, "es": 4, "enr": 5}
	ZLogLevels              = LogLevels{"Debug": zerolog.DebugLevel, "Info": zerolog.InfoLevel, "Error": zerolog.ErrorLevel}
)

Constants for this package

Functions

func AddBillNumbersToTitle

func AddBillNumbersToTitle(titleMap *sync.Map, title string, billnumbers []string) (*sync.Map, error)

func AddBillStructDb

func AddBillStructDb(db *gorm.DB, billStruct *Bill)

func AddBillnumberversionsDb

func AddBillnumberversionsDb(db *gorm.DB, billnumberversions []string)

func AddTitle

func AddTitle(titleMap *sync.Map, title string) (*sync.Map, error)

func AddTitleDb

func AddTitleDb(db *gorm.DB, title string) *gorm.DB

func AddTitleStructDb

func AddTitleStructDb(db *gorm.DB, title *Title)

func BillNumberVersionToBillNumber

func BillNumberVersionToBillNumber(billNumberVersion string) string

func BillNumberVersionsToBillNumbers

func BillNumberVersionsToBillNumbers(billNumberVersions []string) (billNumbers []string)

func GetBillnumbersByTitle

func GetBillnumbersByTitle(titleMap *sync.Map, title string) (billnumbers []string, err error)

func GetDb

func GetDb(dbname string) *gorm.DB

func GetDb_old

func GetDb_old(dbname string) *gorm.DB

func GetRelatedDb

func GetRelatedDb(dbname string) *gorm.DB

func GetSyncMapKeys

func GetSyncMapKeys(m *sync.Map) (s string)

Gets keys of a sync.Map

func LoadBillTitleToDBFromJson

func LoadBillTitleToDBFromJson(db *gorm.DB, jsonPath string, isForWhole bool)

func LoadBillsRelatedToDBFromJson

func LoadBillsRelatedToDBFromJson(db *gorm.DB, parentPath string)

jsonPath := RelatedJsonPath db := GetDb(BILLSRELATED_DB)

func LoadEnv

func LoadEnv() (err error)

func LoadTitlesMap

func LoadTitlesMap(titlePath string) (*sync.Map, error)

func LoadTitlesToDBFromJson

func LoadTitlesToDBFromJson(db *gorm.DB, jsonPath string)

jsonPath := TitlesPath db := GetDb(BILLTITLES_DB) NOTE: This adds all bills at the 'ih' version, since we currently only have the json map with billnumbers, not billnumberversions

func MakeSampleTitlesFile

func MakeSampleTitlesFile(titleMap *sync.Map)

func MarshalJSONStringArray

func MarshalJSONStringArray(m *sync.Map) ([]byte, error)

Marshals a sync.Map object of the type map[string][]string see https://stackoverflow.com/a/46390611/628748 and https://stackoverflow.com/a/65442862/628748

func RemoveDuplicates

func RemoveDuplicates(elements []string) []string

Removes duplicates in a list of strings Returns the deduplicated list Trims leading and trailing space for each element

func RemoveTitle

func RemoveTitle(titleMap *sync.Map, title string) (*sync.Map, error)

func RemoveTitleDb

func RemoveTitleDb(db *gorm.DB, title string)

func SaveTitlesMap

func SaveTitlesMap(titleMap *sync.Map, titlePath string) (err error)

func UnmarshalJson

func UnmarshalJson(data []byte) (*sync.Map, error)

Unmarshals from JSON to a syncMap See https://stackoverflow.com/a/65442862/628748

func UnmarshalJsonFile

func UnmarshalJsonFile(jpath string) (*sync.Map, error)

func UnmarshalTitlesJson

func UnmarshalTitlesJson(data []byte) (*sync.Map, error)

Unmarshals from JSON in the form of map[string][]string to a syncMap See https://stackoverflow.com/a/65442862/628748

func UnmarshalTitlesJsonFile

func UnmarshalTitlesJsonFile(jpath string) (*sync.Map, error)

Types

type Bill

type Bill struct {
	gorm.Model
	Billnumber        string   `gorm:"not null" json:"billnumber"`
	Billnumberversion string   `gorm:"index:,unique" json:"billnumberversion"`
	Titles            []*Title `gorm:"many2many:bill_titles;" json:"titles"`
	TitlesWhole       []*Title `gorm:"many2many:bill_titleswhole" json:"titles_whole"`
}

func GetBillsByTitleDb

func GetBillsByTitleDb(db *gorm.DB, title string) (bills []*Bill)

func GetBillsWithSameTitleDb

func GetBillsWithSameTitleDb(db *gorm.DB, billnumber string) (bills, bills_whole []*Bill, err error)

type BillTitle

type BillTitle struct {
	gorm.Model
	Billnumber     string `gorm:"index:,not null" json:"billnumber"`
	Title          string `gorm:"index:,not null" json:"title"`
	IsForWholeBill bool   `gorm:"not null" json:"is_for_whole_bill"`
}

type BillToBill

type BillToBill struct {
	gorm.Model
	Billnumber    string  `gorm:"UNIQUE_INDEX:compositeindex;index:,not null" json:"billnumber"`
	Billnumber_to string  `gorm:"UNIQUE_INDEX:compositeindex;index:,not null" json:"billnumber_to"`
	Reason        string  `gorm:"UNIQUE_INDEX:compositeindex;not null" json:"reason"`
	Score         float64 `json:"score"`
	ScoreOther    float64 `json:"score_other"` // score for other bill
	Identified_by string  `gorm:"index:,not null" json:"identified_by"`
}

type LogLevel

type LogLevel int8

type LogLevels

type LogLevels map[string]zerolog.Level

type RelatedBillItem

type RelatedBillItem struct {
	BillId                        string   `json:"bill_id"`
	IdentifiedBy                  string   `json:"identified_by"`
	Reason                        string   `json:"reason"`
	Type                          string   `json:"type"`
	BillCongressTypeNumber        string   `json:"bill_congress_type_number"`
	BillCongressTypeNumberVersion string   `json:"bill_congress_type_number_version"`
	Titles                        []string `json:"titles"`
	TitlesWholeBill               []string `json:"titles_whole_bill"`
}

type RelatedBillMap

type RelatedBillMap map[string]RelatedBillItem

func GetRelatedBills

func GetRelatedBills(titleMap *sync.Map, titleWholeBillMap *sync.Map, relatedBillItem RelatedBillItem) (relatedBills RelatedBillMap, err error)

type Title

type Title struct {
	gorm.Model
	Title string  `gorm:"index:,unique"`
	Bills []*Bill `gorm:"many2many:bill_titles;"`
}

func GetTitlesByBillnumberDb

func GetTitlesByBillnumberDb(db *gorm.DB, billnumber string) (titles []*Title)

func GetTitlesByBillnumberVersionDb

func GetTitlesByBillnumberVersionDb(db *gorm.DB, billnumberversion string) (titles []*Title)

func GetTitlesWholeByBillnumberDb

func GetTitlesWholeByBillnumberDb(db *gorm.DB, billnumber string) (titles []*Title)

func GetTitlesWholeByBillnumberVersionDb

func GetTitlesWholeByBillnumberVersionDb(db *gorm.DB, billnumberversion string) (titles []*Title)