deepcfr

Documentation

Index

• type AdvantageModel
  • func (m AdvantageModel) Predict(infoSet cfr.InfoSet, nActions int) []float32
• type Buffer
• type CircularBuffer
  • func NewCircularBuffer(maxSize int) *CircularBuffer
  • func (b *CircularBuffer) AddSample(sample Sample)
  • func (b *CircularBuffer) Close() error
  • func (b *CircularBuffer) GetSample(idx int) Sample
  • func (b *CircularBuffer) GetSamples() []Sample
  • func (b *CircularBuffer) Len() int
  • func (b *CircularBuffer) MarshalBinary() ([]byte, error)
  • func (b *CircularBuffer) UnmarshalBinary(buf []byte) error
• type ExperienceTuple
  • func NewExperienceTuple(node cfr.GameTreeNode, weight float32, action int, value float32) *ExperienceTuple
  • func (t *ExperienceTuple) MarshalBinary() ([]byte, error)
  • func (t *ExperienceTuple) UnmarshalBinary(buf []byte) error
• type Model
• type RegretSample
  • func NewRegretSample(node cfr.GameTreeNode, advantages []float32, weight float32) *RegretSample
  • func (s *RegretSample) MarshalBinary() ([]byte, error)
  • func (s *RegretSample) UnmarshalBinary(buf []byte) error
• type ReservoirBuffer
  • func NewReservoirBuffer(maxSize, maxParallel int) *ReservoirBuffer
  • func (b *ReservoirBuffer) AddSample(sample Sample)
  • func (b *ReservoirBuffer) Close() error
  • func (b *ReservoirBuffer) GetSample(idx int) Sample
  • func (b *ReservoirBuffer) GetSamples() []Sample
  • func (b *ReservoirBuffer) Len() int
  • func (b *ReservoirBuffer) MarshalBinary() ([]byte, error)
  • func (b *ReservoirBuffer) UnmarshalBinary(buf []byte) error
• type Sample
• type SingleDeepCFR
  • func NewSingleDeepCFR(model Model, buffers []Buffer) *SingleDeepCFR
  • func (d *SingleDeepCFR) Close() error
  • func (d *SingleDeepCFR) GetBuffer(player int) Buffer
  • func (d *SingleDeepCFR) GetPolicy(node cfr.GameTreeNode) cfr.NodePolicy
  • func (d *SingleDeepCFR) Iter() int
  • func (d *SingleDeepCFR) MarshalBinary() ([]byte, error)
  • func (d *SingleDeepCFR) UnmarshalBinary(buf []byte) error
  • func (d *SingleDeepCFR) Update()
• type TrainedModel
• type VRSingleDeepCFR
  • func NewVRSingleDeepCFR(model Model, buffers, baselineBuffers []Buffer) *VRSingleDeepCFR
  • func (d *VRSingleDeepCFR) Close() error
  • func (d *VRSingleDeepCFR) GetBuffer(player int) Buffer
  • func (d *VRSingleDeepCFR) GetPolicy(node cfr.GameTreeNode) cfr.NodePolicy
  • func (d *VRSingleDeepCFR) Iter() int
  • func (d *VRSingleDeepCFR) MarshalBinary() ([]byte, error)
  • func (d *VRSingleDeepCFR) UnmarshalBinary(buf []byte) error
  • func (d *VRSingleDeepCFR) Update()

Types

type AdvantageModel

type AdvantageModel struct {
	Model TrainedModel
}

func (AdvantageModel) Predict

func (m AdvantageModel) Predict(infoSet cfr.InfoSet, nActions int) []float32

type Buffer

type Buffer interface {
	AddSample(Sample)
	GetSample(idx int) Sample
	GetSamples() []Sample
	Len() int
	io.Closer
}

Buffer collects samples of infoset action advantages to train a Model.

type CircularBuffer

type CircularBuffer struct {
	// contains filtered or unexported fields
}

func NewCircularBuffer

func NewCircularBuffer(maxSize int) *CircularBuffer

func (*CircularBuffer) AddSample

func (b *CircularBuffer) AddSample(sample Sample)

func (*CircularBuffer) Close

func (b *CircularBuffer) Close() error

func (*CircularBuffer) GetSample

func (b *CircularBuffer) GetSample(idx int) Sample

func (*CircularBuffer) GetSamples

func (b *CircularBuffer) GetSamples() []Sample

func (*CircularBuffer) Len

func (b *CircularBuffer) Len() int

func (*CircularBuffer) MarshalBinary

func (b *CircularBuffer) MarshalBinary() ([]byte, error)

func (*CircularBuffer) UnmarshalBinary

func (b *CircularBuffer) UnmarshalBinary(buf []byte) error

type ExperienceTuple

type ExperienceTuple struct {
	Weight  float32
	InfoSet []byte
	Action  uint16
	Value   float32
}

ExperienceTuple is a single sample of action-result collected for training.

func NewExperienceTuple

func NewExperienceTuple(node cfr.GameTreeNode, weight float32, action int, value float32) *ExperienceTuple

func (*ExperienceTuple) MarshalBinary

func (t *ExperienceTuple) MarshalBinary() ([]byte, error)

MarshalBinary implements encoding.BinaryMarshaler.

func (*ExperienceTuple) UnmarshalBinary

func (t *ExperienceTuple) UnmarshalBinary(buf []byte) error

UnmarshalBinary implements encoding.BinaryUnmarshaler.

type Model

type Model interface {
	Train(buffer Buffer) TrainedModel
}

Model is a regression model that can be used to fit the given samples.

type RegretSample

type RegretSample struct {
	Weight     float32
	InfoSet    []byte
	Advantages []float32
}

RegretSample is a single sample of instantaneous advantages collected for training.

func NewRegretSample

func NewRegretSample(node cfr.GameTreeNode, advantages []float32, weight float32) *RegretSample

func (*RegretSample) MarshalBinary

func (s *RegretSample) MarshalBinary() ([]byte, error)

MarshalBinary implements encoding.BinaryMarshaler.

func (*RegretSample) UnmarshalBinary

func (s *RegretSample) UnmarshalBinary(buf []byte) error

UnmarshalBinary implements encoding.BinaryUnmarshaler.

type ReservoirBuffer

type ReservoirBuffer struct {
	// contains filtered or unexported fields
}

ReservoirBuffer is a collection of samples held in memory. One the buffer's max size is reached, additional samples are added via reservoir sampling, maintaining a uniform distribution over all previous values.

It is safe to call AddSample concurrently from multiple goroutines. GetSamples does not copy the underlying slice of samples, and therefore is not safe to call concurrently with AddSample.

func NewReservoirBuffer

func NewReservoirBuffer(maxSize, maxParallel int) *ReservoirBuffer

NewBuffer returns an empty Buffer with the given max size.

func (*ReservoirBuffer) AddSample

func (b *ReservoirBuffer) AddSample(sample Sample)

AddSample implements Buffer.

func (*ReservoirBuffer) Close

func (b *ReservoirBuffer) Close() error

func (*ReservoirBuffer) GetSample

func (b *ReservoirBuffer) GetSample(idx int) Sample

GetSample implements Buffer.

func (*ReservoirBuffer) GetSamples

func (b *ReservoirBuffer) GetSamples() []Sample

GetSamples implements Buffer.

func (*ReservoirBuffer) Len

func (b *ReservoirBuffer) Len() int

func (*ReservoirBuffer) MarshalBinary

func (b *ReservoirBuffer) MarshalBinary() ([]byte, error)

MarshalBinary implements encoding.BinaryMarshaler.

func (*ReservoirBuffer) UnmarshalBinary

func (b *ReservoirBuffer) UnmarshalBinary(buf []byte) error

UnmarshalBinary implements encoding.BinaryUnmarshaler.

type Sample

type Sample interface{}

Samples must be binary marshalable, but embedding that interface breaks gob decoding. Ref: https://stackoverflow.com/questions/43324919/gob-panics-decoding-an-interface

type SingleDeepCFR

type SingleDeepCFR struct {
	// contains filtered or unexported fields
}

SingleDeepCFR implements cfr.StrategyProfile, and uses function approximation to estimate strategies rather than accumulation of regrets for all infosets. This can be more tractable for large games where storing all of the regrets for all infosets is impractical.

During CFR iterations, samples are added to the given buffer. When Update is called, the model is retrained.

func NewSingleDeepCFR

func NewSingleDeepCFR(model Model, buffers []Buffer) *SingleDeepCFR

New returns a new SingleDeepCFR policy with the given model and sample buffer.

func (*SingleDeepCFR) Close

func (d *SingleDeepCFR) Close() error

func (*SingleDeepCFR) GetBuffer

func (d *SingleDeepCFR) GetBuffer(player int) Buffer

func (*SingleDeepCFR) GetPolicy

func (d *SingleDeepCFR) GetPolicy(node cfr.GameTreeNode) cfr.NodePolicy

func (*SingleDeepCFR) Iter

func (d *SingleDeepCFR) Iter() int

Iter implements cfr.StrategyProfile.

func (*SingleDeepCFR) MarshalBinary

func (d *SingleDeepCFR) MarshalBinary() ([]byte, error)

MarshalBinary implements encoding.BinaryMarshaler. Note that to be able to use this method, the concrete types implementing the Model, TrainedModel, and Buffers must be registered with gob.

func (*SingleDeepCFR) UnmarshalBinary

func (d *SingleDeepCFR) UnmarshalBinary(buf []byte) error

UnmarshalBinary implements encoding.BinaryUnmarshaler.

func (*SingleDeepCFR) Update

func (d *SingleDeepCFR) Update()

Update implements cfr.StrategyProfile.

type TrainedModel

type TrainedModel interface {
	Predict(infoSet cfr.InfoSet, nActions int) (advantages []float32)
}

TrainedModel is a regression model to use in DeepCFR that predicts a vector of advantages for a given InfoSet.

type VRSingleDeepCFR

type VRSingleDeepCFR struct {
	// contains filtered or unexported fields
}

VRSingleDeepCFR implements cfr.StrategyProfile, and uses function approximation to estimate strategies rather than accumulation of regrets for all infosets. This can be more tractable for large games where storing all of the regrets for all infosets is impractical.

During CFR iterations, samples are added to the given buffer. When Update is called, the model is retrained.

func NewVRSingleDeepCFR

func NewVRSingleDeepCFR(model Model, buffers, baselineBuffers []Buffer) *VRSingleDeepCFR

New returns a new VRSingleDeepCFR policy with the given model and sample buffer.

func (*VRSingleDeepCFR) Close

func (d *VRSingleDeepCFR) Close() error

func (*VRSingleDeepCFR) GetBuffer

func (d *VRSingleDeepCFR) GetBuffer(player int) Buffer

func (*VRSingleDeepCFR) GetPolicy

func (d *VRSingleDeepCFR) GetPolicy(node cfr.GameTreeNode) cfr.NodePolicy

func (*VRSingleDeepCFR) Iter

func (d *VRSingleDeepCFR) Iter() int

Iter implements cfr.StrategyProfile.

func (*VRSingleDeepCFR) MarshalBinary

func (d *VRSingleDeepCFR) MarshalBinary() ([]byte, error)

MarshalBinary implements encoding.BinaryMarshaler. Note that to be able to use this method, the concrete types implementing the Model, TrainedModel, and Buffers must be registered with gob.

func (*VRSingleDeepCFR) UnmarshalBinary

func (d *VRSingleDeepCFR) UnmarshalBinary(buf []byte) error

UnmarshalBinary implements encoding.BinaryUnmarshaler.

func (*VRSingleDeepCFR) Update

func (d *VRSingleDeepCFR) Update()

Update implements cfr.StrategyProfile.