hive

Documentation

Index

• Constants
• Variables
• func CheckDataType(inputModel reflect.StructField, inputVal interface{}, dateFormat string) (output string)
• func DetectDataType(in string, dateFormat string) (res string)
• func InspectJson(ins []string) (out []string)
• func Parse(header []string, in interface{}, m interface{}, outputType string, ...) (e error)
• func QueryBuilder(clause, tablename, input string, TableModel interface{}) (retVal string)
• type DuplexTerm
  • func (d *DuplexTerm) Close()
  • func (d *DuplexTerm) Open() (e error)
  • func (d *DuplexTerm) SendInput(input string) (res HiveResult, err error)
  • func (d *DuplexTerm) TestConnection() (e error)
• type FieldMismatch
  • func (e *FieldMismatch) Error() string
• type FnHiveReceive
• type Hive
  • func HiveConfig(server, dbName, userid, password, path string, delimiter ...string) *Hive
  • func (h *Hive) CheckDataStructure(Tablename string, TableModel interface{}) (isMatch bool, err error)
  • func (h *Hive) Exec(query string, fn FnHiveReceive) (e error)
  • func (h *Hive) Load(TableName, dateFormat string, TableModel interface{}) (retVal string, err error)
  • func (h *Hive) LoadFile(FilePath, TableName, fileType, dateFormat string, TableModel interface{}) (retVal string, err error)
  • func (h *Hive) LoadFileWithWorker(FilePath, TableName, fileType string, dateFormat string, ...) (retVal string, err error)
  • func (h *Hive) Populate(query string, m interface{}) (e error)
• type HiveManager
  • func NewHiveManager(numWorkers int) HiveManager
  • func (m *HiveManager) AssignTask(task string, wg *sync.WaitGroup)
  • func (m *HiveManager) DoMonitor(wg *sync.WaitGroup)
  • func (m *HiveManager) EndWorker()
  • func (m *HiveManager) InProgress(result int64, wg *sync.WaitGroup)
  • func (m *HiveManager) Timeout(seconds int, wg *sync.WaitGroup)
• type HiveResult
• type HiveWorker
  • func (w *HiveWorker) Work(task string, wg *sync.WaitGroup)
• type UnsupportedType
  • func (e *UnsupportedType) Error() string

Constants

const (
	BEE_CLI_STR      = "jdbc:hive2:"
	CLOSE_SCRIPT     = "!quit"
	BEE_CLOSED       = "(closed)>"
	TEST_CONN_SCRIPT = "show tables;"
)

const (
	BEE_TEMPLATE  = "%sbeeline -u jdbc:hive2://%s/%s"
	BEE_USER      = " -n %s"
	BEE_PASSWORD  = " -p %s"
	BEE_QUERY     = " -e \"%s\""
	PACKAGENAME   = "Hive"
	CSV_FORMAT    = " --outputFormat=csv"
	TSV_FORMAT    = " --outputFormat=tsv"
	DSV_FORMAT    = " --outputFormat=dsv --delimiterForDSV=|\t"
	DSV_DELIMITER = "|\t"
	TSV           = "tsv"
	CSV           = "csv"
	JSON          = "json"
)

Variables

var JsonPart string

Functions

func CheckDataType

func CheckDataType(inputModel reflect.StructField, inputVal interface{}, dateFormat string) (output string)

func DetectDataType

func DetectDataType(in string, dateFormat string) (res string)

func InspectJson

func InspectJson(ins []string) (out []string)

func Parse

func Parse(header []string, in interface{}, m interface{}, outputType string, dateFormat string) (e error)

func QueryBuilder

func QueryBuilder(clause, tablename, input string, TableModel interface{}) (retVal string)

Types

type DuplexTerm

type DuplexTerm struct {
	Writer     *bufio.Writer
	Reader     *bufio.Reader
	Cmd        *exec.Cmd
	CmdStr     string
	Stdin      io.WriteCloser
	Stdout     io.ReadCloser
	FnReceive  FnHiveReceive
	OutputType string
	DateFormat string
}

func (*DuplexTerm) Close

func (d *DuplexTerm) Close()

func (*DuplexTerm) Open

func (d *DuplexTerm) Open() (e error)

func (*DuplexTerm) SendInput

func (d *DuplexTerm) SendInput(input string) (res HiveResult, err error)

func (*DuplexTerm) TestConnection

func (d *DuplexTerm) TestConnection() (e error)

type FieldMismatch

type FieldMismatch struct {
	// contains filtered or unexported fields
}

func (*FieldMismatch) Error

func (e *FieldMismatch) Error() string

type FnHiveReceive

type FnHiveReceive func(HiveResult) error

type Hive

type Hive struct {
	BeePath    string
	Server     string
	User       string
	Password   string
	DBName     string
	Conn       *DuplexTerm
	OutputType string
	DateFormat string
}

func HiveConfig

func HiveConfig(server, dbName, userid, password, path string, delimiter ...string) *Hive

func (*Hive) CheckDataStructure

func (h *Hive) CheckDataStructure(Tablename string, TableModel interface{}) (isMatch bool, err error)

func (*Hive) Exec

func (h *Hive) Exec(query string, fn FnHiveReceive) (e error)

func (*Hive) Load

func (h *Hive) Load(TableName, dateFormat string, TableModel interface{}) (retVal string, err error)

func (*Hive) LoadFile

func (h *Hive) LoadFile(FilePath, TableName, fileType, dateFormat string, TableModel interface{}) (retVal string, err error)

func (*Hive) LoadFileWithWorker

func (h *Hive) LoadFileWithWorker(FilePath, TableName, fileType string, dateFormat string, TableModel interface{}, TotalWorker int) (retVal string, err error)

loading file with worker

func (*Hive) Populate

func (h *Hive) Populate(query string, m interface{}) (e error)

type HiveManager

type HiveManager struct {
	FreeWorkers  chan *HiveWorker
	Tasks        chan string
	Done         chan bool
	TimeProcess  chan int64
	LastProcess  int64
	TotalTimeOut int64
}

manager model

func NewHiveManager

func NewHiveManager(numWorkers int) HiveManager

initiate new manager

func (*HiveManager) AssignTask

func (m *HiveManager) AssignTask(task string, wg *sync.WaitGroup)

assign task to free worker

func (*HiveManager) DoMonitor

func (m *HiveManager) DoMonitor(wg *sync.WaitGroup)

do monitoring worker thats free or not

func (*HiveManager) EndWorker

func (m *HiveManager) EndWorker()

func (*HiveManager) InProgress

func (m *HiveManager) InProgress(result int64, wg *sync.WaitGroup)

check if a task still in progress to wait it till finish

func (*HiveManager) Timeout

func (m *HiveManager) Timeout(seconds int, wg *sync.WaitGroup)

set the timeout to waiting for tasks execution

type HiveResult

type HiveResult struct {
	Header     []string
	Result     []string
	ResultObj  interface{}
	DateFormat string
}

type HiveWorker

type HiveWorker struct {
	WorkerId    int
	TimeProcess chan int64
	FreeWorkers chan *HiveWorker
	Context     *Hive
	IsConnOpen  bool
}

worker model

func (*HiveWorker) Work

func (w *HiveWorker) Work(task string, wg *sync.WaitGroup)

do a task for worker

type UnsupportedType

type UnsupportedType struct {
	Type string
}

func (*UnsupportedType) Error

func (e *UnsupportedType) Error() string