internal

package
v0.0.26 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Apr 23, 2024 License: GPL-3.0 Imports: 23 Imported by: 0

Documentation

Index

Constants

View Source 
const (
	DESC_PROMPT = `` /* 1277-byte string literal not displayed */

	TESTS_PROMPT = `` /* 1059-byte string literal not displayed */

)
View Source 
const VERSION = "0.0.25"

Variables

This section is empty.

Functions

func CreateTempDbtProfiles 

func CreateTempDbtProfiles(t *testing.T) string

func CreateTempSourceTables 

func CreateTempSourceTables() shared.SourceTables

func Deduplicate 

func Deduplicate(elements []string) []string

func InferColumnFields 

func InferColumnFields(llm Llm, ts shared.SourceTables)

func PrepBuildDir 

func PrepBuildDir(bd string) error

func SetConnectionDetails 

func SetConnectionDetails(fr FormResponse, ps DbtProfiles) shared.ConnectionDetails

func WriteFiles 

func WriteFiles(ts shared.SourceTables, bd string, prefix string) error

func WriteProfile 

func WriteProfile(cd shared.ConnectionDetails, bd string)

func WriteScaffoldProject 

func WriteScaffoldProject(cd shared.ConnectionDetails, bd string, pn string) (string, error)

func WriteStagingModels 

func WriteStagingModels(ts shared.SourceTables, buildDir string, prefix string)

func WriteYAML 

func WriteYAML(tables shared.SourceTables, buildDir string)

Types

type Anthropic 

type Anthropic struct {
	Type     string
	ApiKey   string
	Model    string
	Url      string
	Response struct {
		Content []struct {
			Type string `json:"type"`
			Text string `json:"text"`
		} `json:"content"`
		Id           string   `json:"id"`
		Type         string   `json:"type"`
		Role         string   `json:"role"`
		StopReason   string   `json:"stop_reason"`
		StopSequence []string `json:"stop_sequence"`
		Usage        struct {
			InputTokens int `json:"input_tokens"`
			OutputToken int `json:"output_tokens"`
		} `json:"usage"`
	}
}

func (*Anthropic) GetRateLimiter 

func (a *Anthropic) GetRateLimiter() (semaphore chan struct{}, limiter *time.Ticker)

func (*Anthropic) GetResponse 

func (a *Anthropic) GetResponse(prompt string) error

func (*Anthropic) SetDescription 

func (a *Anthropic) SetDescription(descPrompt string, ts shared.SourceTables, i, j int) error

func (*Anthropic) SetTests 

func (a *Anthropic) SetTests(testsPrompt string, ts shared.SourceTables, i, j int) error

type DbtProfile 

type DbtProfile struct {
	Outputs map[string]struct {
		Settings map[string]struct {
			S3Region          string `yaml:"s3_region"`
			S3AccessKeyID     string `yaml:"s3_access_key_id"`
			S3SecretAccessKey string `yaml:"s3_secret_access_key"`
		} `yaml:"settings"`
		DataprocBatch map[string]interface{} `yaml:"dataproc_batch"`
		KeyfileJson   map[string]struct {
			Type                    string `yaml:"type"`
			ProjectId               string `yaml:"project_id"`
			PrivateKeyId            string `yaml:"private_key_id"`
			PrivateKey              string `yaml:"private_key"`
			ClientEmail             string `yaml:"client_email"`
			ClientId                string `yaml:"client_id"`
			AuthURI                 string `yaml:"auth_uri"`
			TokenURI                string `yaml:"token_uri"`
			AuthProviderX509CertUrl string `yaml:"auth_provider_x509_cert_url"`
			ClientX509CertUrl       string `yaml:"client_x509_cert_url"`
		} `yaml:"keyfile_json"`
		PrivateKeyPath             string   `yaml:"private_key_path"`
		DbName                     string   `yaml:"dbname"`
		Database                   string   `yaml:"database"`
		Account                    string   `yaml:"account"`
		Schema                     string   `yaml:"schema"`
		QueryTag                   string   `yaml:"query_tag"`
		Dataset                    string   `yaml:"dataset"`
		Path                       string   `yaml:"path"`
		Role                       string   `yaml:"role"`
		Password                   string   `yaml:"password"`
		User                       string   `yaml:"user"`
		Warehouse                  string   `yaml:"warehouse"`
		Method                     string   `yaml:"method"`
		Host                       string   `yaml:"host"`
		PrivateKey                 string   `yaml:"private_key"`
		Location                   string   `yaml:"location"`
		ConnType                   string   `yaml:"type"`
		Authenticator              string   `yaml:"authenticator"`
		Project                    string   `yaml:"project"`
		SslMode                    string   `yaml:"sslmode"`
		DataprocClusterName        string   `yaml:"dataproc_cluster_name"`
		DataprocRegion             string   `yaml:"dataproc_region"`
		GcsBucket                  string   `yaml:"gcs_bucket"`
		ExecutionProject           string   `yaml:"execution_project"`
		PrivateKeyPassphrase       string   `yaml:"private_key_passphrase"`
		RefreshToken               string   `yaml:"refresh_token"`
		ClientID                   string   `yaml:"client_id"`
		ClientSecret               string   `yaml:"client_secret"`
		TokenURI                   string   `yaml:"token_uri"`
		Token                      string   `yaml:"token"`
		Priority                   string   `yaml:"priority"`
		Keyfile                    string   `yaml:"keyfile"`
		ImpersonateServiceAccount  string   `yaml:"impersonate_service_account"`
		HttpPath                   string   `yaml:"http_path"`
		Extensions                 []string `yaml:"extensions"`
		Scopes                     []string `yaml:"scopes"`
		JobCreationTimeoutSeconds  int      `yaml:"job_creation_timeout_seconds"`
		MaximumBytesBilled         int      `yaml:"maximum_bytes_billed"`
		JobRetryDeadlineSeconds    int      `yaml:"job_retry_deadline_seconds"`
		JobExecutionTimeoutSeconds int      `yaml:"job_execution_timeout_seconds"`
		ConnectTimeout             int      `yaml:"connect_timeout"`
		ConnectRetries             int      `yaml:"connect_retries"`
		Port                       int      `yaml:"port"`
		Threads                    int      `yaml:"threads"`
		ReuseConnections           bool     `yaml:"reuse_connections"`
		RetryAll                   bool     `yaml:"retry_all"`
		RetryOnDatabaseErrors      bool     `yaml:"retry_on_database_errors"`
		ClientSessionKeepAlive     bool     `yaml:"client_session_keep_alive"`
	} `yaml:"outputs"`
	Target string `yaml:"target"`
}

func GetDbtProfile 

func GetDbtProfile(pn string, ps DbtProfiles) (DbtProfile, error)

type DbtProfiles 

type DbtProfiles map[string]DbtProfile

func FetchDbtProfiles 

func FetchDbtProfiles() (DbtProfiles, error)

type FormResponse 

type FormResponse struct {
	Password             string
	Host                 string
	BuildDir             string
	SslMode              string
	Database             string
	Schema               string
	Project              string
	Dataset              string
	ProjectName          string
	Warehouse            string
	Account              string
	LlmKeyEnvVar         string
	DbtProfileOutput     string
	DbtProfileName       string
	Path                 string
	Port                 string
	TokenEnvVar          string
	HttpPath             string
	Username             string
	Prefix               string
	Llm                  string
	GenerateDescriptions bool
	ScaffoldProject      bool
	CreateProfile        bool
	UseDbtProfile        bool
	Confirm              bool
}

func Forms 

func Forms(ps DbtProfiles) (FormResponse, error)

type Groq 

type Groq struct {
	Type     string
	ApiKey   string
	Model    string
	Url      string
	Response struct {
		SystemFingerprint interface{} `json:"system_fingerprint"`
		Id                string      `json:"id"`
		Object            string      `json:"object"`
		Model             string      `json:"model"`
		Choices           []struct {
			Logprobs     interface{} `json:"logprobs"`
			Message      Message     `json:"message"`
			FinishReason string      `json:"finish_reason"`
			Index        int         `json:"index"`
		} `json:"choices"`
		Usage struct {
			PromptTokens     int     `json:"prompt_tokens"`
			PromptTime       float64 `json:"prompt_time"`
			CompletionTokens int     `json:"completion_tokens"`
			CompletionTime   float64 `json:"completion_time"`
			TotalTokens      int     `json:"total_tokens"`
			TotalTime        float64 `json:"total_time"`
		} `json:"usage"`
		Created int `json:"created"`
	}
}

func (*Groq) GetRateLimiter 

func (g *Groq) GetRateLimiter() (semaphore chan struct{}, limiter *time.Ticker)

func (*Groq) GetResponse 

func (o *Groq) GetResponse(prompt string) error

func (*Groq) SetDescription 

func (g *Groq) SetDescription(descPrompt string, ts shared.SourceTables, i, j int) error

func (*Groq) SetTests 

func (g *Groq) SetTests(testsPrompt string, ts shared.SourceTables, i, j int) error

type Llm 

type Llm interface {
	GetResponse(prompt string) error
	SetDescription(descPrompt string, ts shared.SourceTables, i, j int) error
	SetTests(descPrompt string, ts shared.SourceTables, i, j int) error
	GetRateLimiter() (chan struct{}, *time.Ticker)
}

func GetLlm 

func GetLlm(fr FormResponse) (Llm, error)

type Message 

type Message struct {
	Role    string `json:"role"`
	Content string `json:"content"`
}

type OpenAI 

type OpenAI struct {
	Type     string
	ApiKey   string
	Model    string
	Url      string
	Response struct {
		SystemFingerprint interface{} `json:"system_fingerprint"`
		Id                string      `json:"id"`
		Object            string      `json:"object"`
		Model             string      `json:"model"`
		Choices           []struct {
			Logprobs     interface{} `json:"logprobs"`
			Message      Message     `json:"message"`
			FinishReason string      `json:"finish_reason"`
			Index        int         `json:"index"`
		} `json:"choices"`
		Usage struct {
			PromptTokens     int     `json:"prompt_tokens"`
			PromptTime       float64 `json:"prompt_time"`
			CompletionTokens int     `json:"completion_tokens"`
			CompletionTime   float64 `json:"completion_time"`
			TotalTokens      int     `json:"total_tokens"`
			TotalTime        float64 `json:"total_time"`
		} `json:"usage"`
		Created int `json:"created"`
	}
}

func (*OpenAI) GetRateLimiter 

func (o *OpenAI) GetRateLimiter() (semaphore chan struct{}, limiter *time.Ticker)

func (*OpenAI) GetResponse 

func (o *OpenAI) GetResponse(prompt string) error

func (*OpenAI) SetDescription 

func (o *OpenAI) SetDescription(descPrompt string, ts shared.SourceTables, i, j int) error

func (*OpenAI) SetTests 

func (o *OpenAI) SetTests(testsPrompt string, ts shared.SourceTables, i, j int) error

type Payload 

type Payload struct {
	Model     string    `json:"model"`
	Messages  []Message `json:"messages"`
	Temp      float64   `json:"temperature"`
	MaxTokens int       `json:"max_tokens"`
	TopP      int       `json:"top_p"`
	Stream    bool      `json:"stream"`
}

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.