loadbalancer

package
v0.10.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Nov 14, 2024 License: MIT Imports: 15 Imported by: 0

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	ErrorNoSlotsAvailable   = errors.New("no slots available")
	ErrorNoTargetsAvailable = errors.New("no targets available")
)

Functions

func CreateLlamaCppTargetReverseProxy added in v0.8.0 

func CreateLlamaCppTargetReverseProxy(
	logger hclog.Logger,
	loadBalancerConfiguration *LoadBalancerConfiguration,
	targetConfiguration *LlamaCppTargetConfiguration,
) *httputil.ReverseProxy

func LlamaCppTargetBalancingCompare added in v0.7.1 

func LlamaCppTargetBalancingCompare(a, b *LlamaCppTarget) int

Types

type BalancingAttemptStatus added in v0.3.0 

type BalancingAttemptStatus struct {
	Error          error
	LlamaCppTarget *LlamaCppTarget
}

type BufferedRequest added in v0.3.0 

type BufferedRequest struct {
	DoneChannel         chan bool
	IsDoneChannelClosed bool
	IsDone              bool
	Response            http.ResponseWriter
	Request             *http.Request
	Timeout             time.Time
}

func (*BufferedRequest) Close added in v0.3.0 

func (self *BufferedRequest) Close()

func (*BufferedRequest) SendError added in v0.3.0 

func (self *BufferedRequest) SendError(message string, errorCode int)

func (*BufferedRequest) SetDone added in v0.3.0 

func (self *BufferedRequest) SetDone()

type BufferedRequestsStats added in v0.3.0 

type BufferedRequestsStats struct {
	RequestsBuffered int
}

type LlamaCppHealthStatusAggregate added in v0.3.0 

type LlamaCppHealthStatusAggregate struct {
	AggregatedHealthStatus *llamacpp.LlamaCppSlotsAggregatedStatus
	RBMutex                xsync.RBMutex
}

func (*LlamaCppHealthStatusAggregate) AddSlotsFrom added in v0.3.0 

func (self *LlamaCppHealthStatusAggregate) AddSlotsFrom(llamaCppTarget *LlamaCppTarget)

func (*LlamaCppHealthStatusAggregate) IncreaseBy added in v0.3.0 

func (self *LlamaCppHealthStatusAggregate) IncreaseBy(slotsIdle int, slotsProcessing int)

func (*LlamaCppHealthStatusAggregate) RemoveSlotsFrom added in v0.3.0 

func (self *LlamaCppHealthStatusAggregate) RemoveSlotsFrom(llamaCppTarget *LlamaCppTarget)

func (*LlamaCppHealthStatusAggregate) SetTo added in v0.3.0 

func (self *LlamaCppHealthStatusAggregate) SetTo(slotsIdle int, slotsProcessing int)

func (*LlamaCppHealthStatusAggregate) UseSlot added in v0.3.0 

func (self *LlamaCppHealthStatusAggregate) UseSlot()

type LlamaCppPickedTarget 

type LlamaCppPickedTarget struct {
	Element        *list.Element
	LlamaCppTarget *LlamaCppTarget
}

type LlamaCppTarget 

type LlamaCppTarget struct {
	LastUpdate                    time.Time                               `json:"last_update"`
	LlamaCppClient                *llamacpp.LlamaCppClient                `json:"-"`
	LlamaCppSlotsAggregatedStatus *llamacpp.LlamaCppSlotsAggregatedStatus `json:"llamacpp_slots_aggregated_status"`
	LlamaCppTargetConfiguration   *LlamaCppTargetConfiguration            `json:"llamacpp_target_configuration"`
	RBMutex                       xsync.RBMutex                           `json:"-"`
	RemainingTicksUntilRemoved    int                                     `json:"remaining_ticks_until_removed"`
	ReverseProxy                  *httputil.ReverseProxy                  `json:"-"`
	TotalUpdates                  int                                     `json:"total_updates"`
}

func (*LlamaCppTarget) DecrementIdleSlots added in v0.5.0 

func (self *LlamaCppTarget) DecrementIdleSlots()

func (*LlamaCppTarget) DecrementRemainingTicks added in v0.5.0 

func (self *LlamaCppTarget) DecrementRemainingTicks()

func (*LlamaCppTarget) GetSlotsStatus added in v0.5.0 

func (self *LlamaCppTarget) GetSlotsStatus() (int, int)

func (*LlamaCppTarget) HasLessSlotsThan 

func (self *LlamaCppTarget) HasLessSlotsThan(other *LlamaCppTarget) bool

func (*LlamaCppTarget) HasRemainingTicks added in v0.5.0 

func (self *LlamaCppTarget) HasRemainingTicks() bool

func (*LlamaCppTarget) SetTickStatus added in v0.5.0 

func (self *LlamaCppTarget) SetTickStatus(
	lastUpdate time.Time,
	llamaCppSlotsAggregatedStatus *llamacpp.LlamaCppSlotsAggregatedStatus,
	remainingTicksUntilRemoved int,
) (int, int)

type LlamaCppTargetConfiguration 

type LlamaCppTargetConfiguration struct {
	Id                    string
	Name                  string
	LlamaCppConfiguration *llamacpp.LlamaCppConfiguration `json:"llama_cpp_configuration"`
}

type LoadBalancer 

type LoadBalancer struct {
	LoadBalancerTargetCollection *LoadBalancerTargetCollection
	Logger                       hclog.Logger
}

func (*LoadBalancer) Balance 

func (self *LoadBalancer) Balance(
	balancingAttemptStatusChannel chan<- *BalancingAttemptStatus,
	request *LoadBalancerRequest,
)

func (*LoadBalancer) GetStatus 

func (self *LoadBalancer) GetStatus() *LoadBalancerStatus

type LoadBalancerConfiguration added in v0.3.0 

type LoadBalancerConfiguration struct {
	RequestBufferSize    uint
	RequestBufferTimeout time.Duration
	RewriteHostHeader    bool
}

type LoadBalancerRequest 

type LoadBalancerRequest struct {
	HttpRequest *http.Request
}

type LoadBalancerStatus 

type LoadBalancerStatus struct {
	RegisteredTargets int `json:"registered_targets"`
}

type LoadBalancerTargetCollection 

type LoadBalancerTargetCollection struct {
	LlamaCppHealthStatusAggregate *LlamaCppHealthStatusAggregate `json:"-"`
	Targets                       []*LlamaCppTarget              `json:"targets"`

	RBMutex xsync.RBMutex `json:"-"`
	// contains filtered or unexported fields
}

func NewLoadBalancerTargetCollection added in v0.3.0 

func NewLoadBalancerTargetCollection(
	llamaCppHealthStatusAggregate *LlamaCppHealthStatusAggregate,
) *LoadBalancerTargetCollection

func (*LoadBalancerTargetCollection) GetHeadTarget 

func (self *LoadBalancerTargetCollection) GetHeadTarget() *LlamaCppTarget

func (*LoadBalancerTargetCollection) GetTargetById added in v0.7.1 

func (self *LoadBalancerTargetCollection) GetTargetById(targetId string) *LlamaCppTarget

func (*LoadBalancerTargetCollection) Len 

func (self *LoadBalancerTargetCollection) Len() int

func (*LoadBalancerTargetCollection) RegisterTarget 

func (self *LoadBalancerTargetCollection) RegisterTarget(llamaCppTarget *LlamaCppTarget)

func (*LoadBalancerTargetCollection) RemoveTarget 

func (self *LoadBalancerTargetCollection) RemoveTarget(llamaCppTarget *LlamaCppTarget)

func (*LoadBalancerTargetCollection) UpdateTargetWithLlamaCppSlotsAggregatedStatus added in v0.7.0 

func (self *LoadBalancerTargetCollection) UpdateTargetWithLlamaCppSlotsAggregatedStatus(
	llamaCppTarget *LlamaCppTarget,
	llamaCppSlotsAggregatedStatus *llamacpp.LlamaCppSlotsAggregatedStatus,
)

func (*LoadBalancerTargetCollection) UseSlot added in v0.3.0 

func (self *LoadBalancerTargetCollection) UseSlot(llamaCppTarget *LlamaCppTarget)

type LoadBalancerTargetRegistrar added in v0.3.0 

type LoadBalancerTargetRegistrar struct {
	HttpClient                   *http.Client
	LoadBalancerConfiguration    *LoadBalancerConfiguration
	LoadBalancerTargetCollection *LoadBalancerTargetCollection
	Logger                       hclog.Logger
}

func (*LoadBalancerTargetRegistrar) RegisterOrUpdateTarget added in v0.3.0 

func (self *LoadBalancerTargetRegistrar) RegisterOrUpdateTarget(
	targetConfiguration *LlamaCppTargetConfiguration,
	llamaCppSlotsAggregatedStatus *llamacpp.LlamaCppSlotsAggregatedStatus,
)

type LoadBalancerTemporalManager added in v0.3.0 

type LoadBalancerTemporalManager struct {
	BufferedRequestsStats         *BufferedRequestsStats
	LlamaCppHealthStatusAggregate *LlamaCppHealthStatusAggregate
	LoadBalancerTargetCollection  *LoadBalancerTargetCollection
	Logger                        hclog.Logger
	StatsdReporter                StatsdReporterInterface
}

func (*LoadBalancerTemporalManager) OnApplicationTick added in v0.3.0 

func (self *LoadBalancerTemporalManager) OnApplicationTick()

func (*LoadBalancerTemporalManager) ReduceTargetCollectionRemainingTicks added in v0.3.0 

func (self *LoadBalancerTemporalManager) ReduceTargetCollectionRemainingTicks()

func (*LoadBalancerTemporalManager) ReportStats added in v0.3.0 

func (self *LoadBalancerTemporalManager) ReportStats()

func (*LoadBalancerTemporalManager) RunTickerInterval added in v0.3.0 

func (self *LoadBalancerTemporalManager) RunTickerInterval()

type RespondToAggregatedHealth added in v0.1.0 

type RespondToAggregatedHealth struct {
	LlamaCppHealthStatusAggregate *LlamaCppHealthStatusAggregate
	ServerEventsChannel           chan<- goroutine.ResultMessage
}

func (*RespondToAggregatedHealth) ServeHTTP added in v0.1.0 

func (self *RespondToAggregatedHealth) ServeHTTP(response http.ResponseWriter, request *http.Request)

type RespondToCompletion added in v0.3.0 

type RespondToCompletion struct {
	BufferChannel             chan *BufferedRequest
	BufferedRequestsStats     *BufferedRequestsStats
	LoadBalancer              *LoadBalancer
	LoadBalancerConfiguration *LoadBalancerConfiguration
	Logger                    hclog.Logger
	ServerEventsChannel       chan<- goroutine.ResultMessage
}

func (*RespondToCompletion) ServeHTTP added in v0.3.0 

func (self *RespondToCompletion) ServeHTTP(response http.ResponseWriter, request *http.Request)

func (*RespondToCompletion) StartBufferProcessor added in v0.3.0 

func (self *RespondToCompletion) StartBufferProcessor()

type RespondToFallback added in v0.3.0 

type RespondToFallback struct {
	LoadBalancerTargetCollection *LoadBalancerTargetCollection
}

func (*RespondToFallback) ServeHTTP added in v0.3.0 

func (self *RespondToFallback) ServeHTTP(response http.ResponseWriter, request *http.Request)

type RespondToFavicon added in v0.1.0 

type RespondToFavicon struct {
	ServerEventsChannel chan<- goroutine.ResultMessage
}

func (*RespondToFavicon) ServeHTTP added in v0.1.0 

func (self *RespondToFavicon) ServeHTTP(response http.ResponseWriter, request *http.Request)

type ReverseProxyServer 

type ReverseProxyServer struct {
	Logger                    hclog.Logger
	RespondToAggregatedHealth *RespondToAggregatedHealth
	RespondToCompletion       *RespondToCompletion
	RespondToFavicon          *RespondToFavicon
	RespondToFallback         *RespondToFallback
	ReverseProxyConfiguration *reverseproxy.ReverseProxyConfiguration
}

func (*ReverseProxyServer) Serve 

func (self *ReverseProxyServer) Serve(serverEventsChannel chan<- goroutine.ResultMessage)

type StatsdConfiguration added in v0.3.0 

type StatsdConfiguration struct {
	EnableStatsdReporter bool
	HttpAddress          *netcfg.HttpAddressConfiguration
}

type StatsdReporter added in v0.3.0 

type StatsdReporter struct {
	StatsdClient statsd.Client
}

func (*StatsdReporter) ReportAggregatedHealthStatus added in v0.3.0 

func (self *StatsdReporter) ReportAggregatedHealthStatus(
	bufferedRequestsStats *BufferedRequestsStats,
	llamaCppHealthStatusAggregate *LlamaCppHealthStatusAggregate,
) error

type StatsdReporterInterface added in v0.3.0 

type StatsdReporterInterface interface {
	ReportAggregatedHealthStatus(
		bufferedRequestsStats *BufferedRequestsStats,
		llamaCppHealthStatusAggregate *LlamaCppHealthStatusAggregate,
	) error
}

type StatsdReporterVoid added in v0.3.0 

type StatsdReporterVoid struct{}

func (*StatsdReporterVoid) ReportAggregatedHealthStatus added in v0.3.0 

func (self *StatsdReporterVoid) ReportAggregatedHealthStatus(
	bufferedRequestsStats *BufferedRequestsStats,
	llamaCppHealthStatusAggregate *LlamaCppHealthStatusAggregate,
) error

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.