loadbalancer

package
v0.6.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 13, 2024 License: MIT Imports: 14 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	ErrorNoSlotsAvailable   = errors.New("no slots available")
	ErrorNoTargetsAvailable = errors.New("no targets available")
)

Functions

This section is empty.

Types

type BalancingAttemptStatus added in v0.3.0 

type BalancingAttemptStatus struct {
	Error          error
	LlamaCppTarget *LlamaCppTarget
}

type BufferedRequest added in v0.3.0 

type BufferedRequest struct {
	DoneChannel         chan bool
	IsDoneChannelClosed bool
	IsDone              bool
	Response            http.ResponseWriter
	Request             *http.Request
	Timeout             time.Time
}

func (*BufferedRequest) Close added in v0.3.0 

func (self *BufferedRequest) Close()

func (*BufferedRequest) SendError added in v0.3.0 

func (self *BufferedRequest) SendError(message string, errorCode int)

func (*BufferedRequest) SetDone added in v0.3.0 

func (self *BufferedRequest) SetDone()

type BufferedRequestsStats added in v0.3.0 

type BufferedRequestsStats struct {
	RequestsBuffered int
}

type LlamaCppHealthStatusAggregate added in v0.3.0 

type LlamaCppHealthStatusAggregate struct {
	AggregatedHealthStatus *llamacpp.LlamaCppHealthStatus
	RBMutex                xsync.RBMutex
}

func (*LlamaCppHealthStatusAggregate) AddSlotsFrom added in v0.3.0 

func (self *LlamaCppHealthStatusAggregate) AddSlotsFrom(llamaCppTarget *LlamaCppTarget)

func (*LlamaCppHealthStatusAggregate) IncreaseBy added in v0.3.0 

func (self *LlamaCppHealthStatusAggregate) IncreaseBy(slotsIdle int, slotsProcessing int)

func (*LlamaCppHealthStatusAggregate) RemoveSlotsFrom added in v0.3.0 

func (self *LlamaCppHealthStatusAggregate) RemoveSlotsFrom(llamaCppTarget *LlamaCppTarget)

func (*LlamaCppHealthStatusAggregate) SetTo added in v0.3.0 

func (self *LlamaCppHealthStatusAggregate) SetTo(slotsIdle int, slotsProcessing int)

func (*LlamaCppHealthStatusAggregate) UseSlot added in v0.3.0 

func (self *LlamaCppHealthStatusAggregate) UseSlot()

type LlamaCppPickedTarget 

type LlamaCppPickedTarget struct {
	Element        *list.Element
	LlamaCppTarget *LlamaCppTarget
}

type LlamaCppTarget 

type LlamaCppTarget struct {
	LastUpdate                  time.Time
	LlamaCppTargetConfiguration *LlamaCppTargetConfiguration
	LlamaCppClient              *llamacpp.LlamaCppClient
	LlamaCppHealthStatus        *llamacpp.LlamaCppHealthStatus
	RemainingTicksUntilRemoved  int
	RBMutex                     xsync.RBMutex
	TotalUpdates                int
	ReverseProxy                *httputil.ReverseProxy
}

func (*LlamaCppTarget) DecrementIdleSlots added in v0.5.0 

func (self *LlamaCppTarget) DecrementIdleSlots()

func (*LlamaCppTarget) DecrementRemainingTicks added in v0.5.0 

func (self *LlamaCppTarget) DecrementRemainingTicks()

func (*LlamaCppTarget) GetSlotsStatus added in v0.5.0 

func (self *LlamaCppTarget) GetSlotsStatus() (int, int)

func (*LlamaCppTarget) HasLessSlotsThan 

func (self *LlamaCppTarget) HasLessSlotsThan(other *LlamaCppTarget) bool

func (*LlamaCppTarget) HasRemainingTicks added in v0.5.0 

func (self *LlamaCppTarget) HasRemainingTicks() bool

func (*LlamaCppTarget) SetTickStatus added in v0.5.0 

func (self *LlamaCppTarget) SetTickStatus(
	lastUpdate time.Time,
	llamaCppHealthStatus *llamacpp.LlamaCppHealthStatus,
	remainingTicksUntilRemoved int,
) (int, int)

type LlamaCppTargetConfiguration 

type LlamaCppTargetConfiguration struct {
	Id                    string
	Name                  string
	LlamaCppConfiguration *llamacpp.LlamaCppConfiguration `json:"llama_cpp_configuration"`
}

type LoadBalancer 

type LoadBalancer struct {
	LoadBalancerTargetCollection *LoadBalancerTargetCollection
	Logger                       hclog.Logger
}

func (*LoadBalancer) Balance 

func (self *LoadBalancer) Balance(
	balancingAttemptStatusChannel chan<- *BalancingAttemptStatus,
	request *LoadBalancerRequest,
)

func (*LoadBalancer) GetStatus 

func (self *LoadBalancer) GetStatus() *LoadBalancerStatus

type LoadBalancerConfiguration added in v0.3.0 

type LoadBalancerConfiguration struct {
	RequestBufferSize    uint
	RequestBufferTimeout time.Duration
}

type LoadBalancerRequest 

type LoadBalancerRequest struct {
	HttpRequest *http.Request
}

type LoadBalancerStatus 

type LoadBalancerStatus struct {
	RegisteredTargets int `json:"registred_targets"`
}

type LoadBalancerTargetCollection 

type LoadBalancerTargetCollection struct {
	LlamaCppHealthStatusAggregate *LlamaCppHealthStatusAggregate
	Targets                       *list.List
	TargetsRBMutex                xsync.RBMutex
	// contains filtered or unexported fields
}

func NewLoadBalancerTargetCollection added in v0.3.0 

func NewLoadBalancerTargetCollection(
	llamaCppHealthStatusAggregate *LlamaCppHealthStatusAggregate,
) *LoadBalancerTargetCollection

func (*LoadBalancerTargetCollection) FixTargetOrder 

func (self *LoadBalancerTargetCollection) FixTargetOrder(target *LlamaCppTarget)

func (*LoadBalancerTargetCollection) GetHeadTarget 

func (self *LoadBalancerTargetCollection) GetHeadTarget() *LlamaCppTarget

func (*LoadBalancerTargetCollection) GetTargetByConfiguration 

func (self *LoadBalancerTargetCollection) GetTargetByConfiguration(
	targetConfiguration *LlamaCppTargetConfiguration,
) *LlamaCppTarget

func (*LoadBalancerTargetCollection) Len 

func (self *LoadBalancerTargetCollection) Len() int

func (*LoadBalancerTargetCollection) RegisterTarget 

func (self *LoadBalancerTargetCollection) RegisterTarget(llamaCppTarget *LlamaCppTarget)

func (*LoadBalancerTargetCollection) RemoveTarget 

func (self *LoadBalancerTargetCollection) RemoveTarget(llamaCppTarget *LlamaCppTarget)

func (*LoadBalancerTargetCollection) UpdateTargetWithLlamaCppHealthStatus added in v0.1.0 

func (self *LoadBalancerTargetCollection) UpdateTargetWithLlamaCppHealthStatus(
	llamaCppTarget *LlamaCppTarget,
	llamaCppHealthStatus *llamacpp.LlamaCppHealthStatus,
)

func (*LoadBalancerTargetCollection) UseSlot added in v0.3.0 

func (self *LoadBalancerTargetCollection) UseSlot(llamaCppTarget *LlamaCppTarget)

type LoadBalancerTargetRegistrar added in v0.3.0 

type LoadBalancerTargetRegistrar struct {
	HttpClient                   *http.Client
	LoadBalancerTargetCollection *LoadBalancerTargetCollection
	Logger                       hclog.Logger
}

func (*LoadBalancerTargetRegistrar) RegisterOrUpdateTarget added in v0.3.0 

func (self *LoadBalancerTargetRegistrar) RegisterOrUpdateTarget(
	targetConfiguration *LlamaCppTargetConfiguration,
	llamaCppHealthStatus *llamacpp.LlamaCppHealthStatus,
)

type LoadBalancerTemporalManager added in v0.3.0 

type LoadBalancerTemporalManager struct {
	BufferedRequestsStats         *BufferedRequestsStats
	LlamaCppHealthStatusAggregate *LlamaCppHealthStatusAggregate
	LoadBalancerTargetCollection  *LoadBalancerTargetCollection
	Logger                        hclog.Logger
	StatsdReporter                StatsdReporterInterface
}

func (*LoadBalancerTemporalManager) OnApplicationTick added in v0.3.0 

func (self *LoadBalancerTemporalManager) OnApplicationTick()

func (*LoadBalancerTemporalManager) ReduceTargetCollectionRemainingTicks added in v0.3.0 

func (self *LoadBalancerTemporalManager) ReduceTargetCollectionRemainingTicks()

func (*LoadBalancerTemporalManager) ReportStats added in v0.3.0 

func (self *LoadBalancerTemporalManager) ReportStats()

func (*LoadBalancerTemporalManager) RunTickerInterval added in v0.3.0 

func (self *LoadBalancerTemporalManager) RunTickerInterval()

type RespondToAggregatedHealth added in v0.1.0 

type RespondToAggregatedHealth struct {
	LlamaCppHealthStatusAggregate *LlamaCppHealthStatusAggregate
	ServerEventsChannel           chan<- goroutine.ResultMessage
}

func (*RespondToAggregatedHealth) ServeHTTP added in v0.1.0 

func (self *RespondToAggregatedHealth) ServeHTTP(response http.ResponseWriter, request *http.Request)

type RespondToCompletion added in v0.3.0 

type RespondToCompletion struct {
	BufferChannel             chan *BufferedRequest
	BufferedRequestsStats     *BufferedRequestsStats
	LoadBalancer              *LoadBalancer
	LoadBalancerConfiguration *LoadBalancerConfiguration
	Logger                    hclog.Logger
	ServerEventsChannel       chan<- goroutine.ResultMessage
}

func (*RespondToCompletion) ServeHTTP added in v0.3.0 

func (self *RespondToCompletion) ServeHTTP(response http.ResponseWriter, request *http.Request)

func (*RespondToCompletion) StartBufferProcessor added in v0.3.0 

func (self *RespondToCompletion) StartBufferProcessor()

type RespondToFallback added in v0.3.0 

type RespondToFallback struct {
	LoadBalancerTargetCollection *LoadBalancerTargetCollection
}

func (*RespondToFallback) ServeHTTP added in v0.3.0 

func (self *RespondToFallback) ServeHTTP(response http.ResponseWriter, request *http.Request)

type RespondToFavicon added in v0.1.0 

type RespondToFavicon struct {
	ServerEventsChannel chan<- goroutine.ResultMessage
}

func (*RespondToFavicon) ServeHTTP added in v0.1.0 

func (self *RespondToFavicon) ServeHTTP(response http.ResponseWriter, request *http.Request)

type ReverseProxyServer 

type ReverseProxyServer struct {
	Logger                    hclog.Logger
	RespondToAggregatedHealth *RespondToAggregatedHealth
	RespondToCompletion       *RespondToCompletion
	RespondToFavicon          *RespondToFavicon
	RespondToFallback         *RespondToFallback
	ReverseProxyConfiguration *reverseproxy.ReverseProxyConfiguration
}

func (*ReverseProxyServer) Serve 

func (self *ReverseProxyServer) Serve(serverEventsChannel chan<- goroutine.ResultMessage)

type StatsdConfiguration added in v0.3.0 

type StatsdConfiguration struct {
	EnableStatsdReporter bool
	HttpAddress          *netcfg.HttpAddressConfiguration
}

type StatsdReporter added in v0.3.0 

type StatsdReporter struct {
	StatsdClient statsd.Client
}

func (*StatsdReporter) ReportAggregatedHealthStatus added in v0.3.0 

func (self *StatsdReporter) ReportAggregatedHealthStatus(
	bufferedRequestsStats *BufferedRequestsStats,
	llamaCppHealthStatusAggregate *LlamaCppHealthStatusAggregate,
) error

type StatsdReporterInterface added in v0.3.0 

type StatsdReporterInterface interface {
	ReportAggregatedHealthStatus(
		bufferedRequestsStats *BufferedRequestsStats,
		llamaCppHealthStatusAggregate *LlamaCppHealthStatusAggregate,
	) error
}

type StatsdReporterVoid added in v0.3.0 

type StatsdReporterVoid struct{}

func (*StatsdReporterVoid) ReportAggregatedHealthStatus added in v0.3.0 

func (self *StatsdReporterVoid) ReportAggregatedHealthStatus(
	bufferedRequestsStats *BufferedRequestsStats,
	llamaCppHealthStatusAggregate *LlamaCppHealthStatusAggregate,
) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.