urkel

README

Urkel

Urkel is a gRPC service and client for the injection of controlled infrastructure failures within a Kubernetes envrionment.

An urkel server runs as a priviledged DaemonSet across all Kubernetes nodes. Or, if running MicroK8s, running urkel on the host outside of Kubernetes is sufficient. urkel uses its priviledge to run commands within the networking namespace of Pod containers running on that same node.

Faults are initiated on behalf of a client gRPC stream which specifies faults to install, and are kept alive so long as the gRPC client stream is still active, after which faults are fully unwound. This design gives clients precise control over a set of faults applied across an entire Kubernetes cluster, while ensuring faults are removed when clients exit (or crash). This model is well suited for crafting "chaos" tests implemented as, for example, regular test cases of the go test tool.

On the client side, the urkel package performs configuration of a Kubernetes client, provides helpers to identify sets of Pods to place under test, and includes urkel.FaultSet for dispatching faults to appropriate urkel servers within the cluster.

Example

By way of example, this Go test partitions members of an Etcd cluster in half, leaves the partition in place for a minute, and then exits.

func TestPartitionWithinEtcdCluster(t *testing.T) {
	var pods = urkel.FetchPods(t, "default", "app=etcd")

	var fs = urkel.NewFaultSet(t)
	defer fs.RemoveAll()

	fs.Partition(pods[:len(pods)/2], pods[len(pods)/2:], drop)
	time.Sleep(time.Minute)
}

Documentation

Index

• func FetchPods(t require.TestingT, namespace, selector string) []v1.Pod
• func RegisterChaosServer(s *grpc.Server, srv ChaosServer)
• type ChaosClient
  • func NewChaosClient(cc *grpc.ClientConn) ChaosClient
• type ChaosServer
• type Chaos_InjectFaultClient
• type Chaos_InjectFaultServer
• type Fault
  • func (*Fault) Descriptor() ([]byte, []int)
  • func (m *Fault) GetPartition() *Partition
  • func (*Fault) ProtoMessage()
  • func (m *Fault) Reset()
  • func (m *Fault) String() string
  • func (m *Fault) XXX_DiscardUnknown()
  • func (m *Fault) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
  • func (m *Fault) XXX_Merge(src proto.Message)
  • func (m *Fault) XXX_Size() int
  • func (m *Fault) XXX_Unmarshal(b []byte) error
• type FaultSet
  • func NewFaultSet(t require.TestingT) *FaultSet
  • func (fs *FaultSet) Crash(pods ...v1.Pod)
  • func (fs *FaultSet) Delete(pods ...v1.Pod)
  • func (fs *FaultSet) Partition(partA, partB []v1.Pod, mode PartitionMode)
  • func (fs *FaultSet) RemoveAll()
• type Partition
  • func (*Partition) Descriptor() ([]byte, []int)
  • func (m *Partition) GetFromIpRange() string
  • func (m *Partition) GetInterfaceIndex() string
  • func (m *Partition) GetMode() string
  • func (*Partition) ProtoMessage()
  • func (m *Partition) Reset()
  • func (m *Partition) String() string
  • func (m *Partition) XXX_DiscardUnknown()
  • func (m *Partition) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
  • func (m *Partition) XXX_Merge(src proto.Message)
  • func (m *Partition) XXX_Size() int
  • func (m *Partition) XXX_Unmarshal(b []byte) error
• type PartitionMode

Functions

func FetchPods

func FetchPods(t require.TestingT, namespace, selector string) []v1.Pod

FetchPods from Kubernetes using the given namespace and ListOptions. Pods are returned in randomly shuffled order.

func RegisterChaosServer

func RegisterChaosServer(s *grpc.Server, srv ChaosServer)

Types

type ChaosClient

type ChaosClient interface {
	InjectFault(ctx context.Context, opts ...grpc.CallOption) (Chaos_InjectFaultClient, error)
}

ChaosClient is the client API for Chaos service.

For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.

func NewChaosClient

func NewChaosClient(cc *grpc.ClientConn) ChaosClient

type ChaosServer

type ChaosServer interface {
	InjectFault(Chaos_InjectFaultServer) error
}

ChaosServer is the server API for Chaos service.

type Chaos_InjectFaultClient

type Chaos_InjectFaultClient interface {
	Send(*Fault) error
	Recv() (*empty.Empty, error)
	grpc.ClientStream
}

type Chaos_InjectFaultServer

type Chaos_InjectFaultServer interface {
	Send(*empty.Empty) error
	Recv() (*Fault, error)
	grpc.ServerStream
}

type Fault

type Fault struct {
	Partition            *Partition `protobuf:"bytes,1,opt,name=partition,proto3" json:"partition,omitempty"`
	XXX_NoUnkeyedLiteral struct{}   `json:"-"`
	XXX_unrecognized     []byte     `json:"-"`
	XXX_sizecache        int32      `json:"-"`
}

Fault is a union type indicating the variety of Fault to be applied.

func (*Fault) Descriptor

func (*Fault) Descriptor() ([]byte, []int)

func (*Fault) GetPartition

func (m *Fault) GetPartition() *Partition

func (*Fault) ProtoMessage

func (*Fault) ProtoMessage()

func (*Fault) Reset

func (m *Fault) Reset()

func (*Fault) String

func (m *Fault) String() string

func (*Fault) XXX_DiscardUnknown

func (m *Fault) XXX_DiscardUnknown()

func (*Fault) XXX_Marshal

func (m *Fault) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Fault) XXX_Merge

func (m *Fault) XXX_Merge(src proto.Message)

func (*Fault) XXX_Size

func (m *Fault) XXX_Size() int

func (*Fault) XXX_Unmarshal

func (m *Fault) XXX_Unmarshal(b []byte) error

type FaultSet

type FaultSet struct {
	// contains filtered or unexported fields
}

FaultSet dispatches a set of removable faults to apply to a collection of Pods.

func NewFaultSet

func NewFaultSet(t require.TestingT) *FaultSet

NewFaultSet returns an empty FaultSet.

func (*FaultSet) Crash

func (fs *FaultSet) Crash(pods ...v1.Pod)

Crash Pods immediately, with no grace period. Crash faults are not remove-able.

func (*FaultSet) Delete

func (fs *FaultSet) Delete(pods ...v1.Pod)

Delete pods using their default GracePeriodSeconds policy. Deletion faults are not remove-able.

func (*FaultSet) Partition

func (fs *FaultSet) Partition(partA, partB []v1.Pod, mode PartitionMode)

Partition Pods in |partA| from Pods in |partB|. The PartitionMode determines whether connections are actively reset (REJECT) or packets are passively ignored (DROP).

func (*FaultSet) RemoveAll

func (fs *FaultSet) RemoveAll()

RemoveAll previously installed faults.

type Partition

type Partition struct {
	// Index of the interface to partition. Can be determined by
	// querying "/sys/class/net/eth0/iflink" within the target container.
	InterfaceIndex string `protobuf:"bytes,1,opt,name=interface_index,json=interfaceIndex,proto3" json:"interface_index,omitempty"`
	// Network IP range to partition from (eg "1.2.3.4/16").
	FromIpRange string `protobuf:"bytes,2,opt,name=from_ip_range,json=fromIpRange,proto3" json:"from_ip_range,omitempty"`
	// Partition mode. REJECT to actively reset connections, DROP to silently
	// drop packet flows.
	Mode                 string   `protobuf:"bytes,3,opt,name=mode,proto3" json:"mode,omitempty"`
	XXX_NoUnkeyedLiteral struct{} `json:"-"`
	XXX_unrecognized     []byte   `json:"-"`
	XXX_sizecache        int32    `json:"-"`
}

Partition traffic flows of a specific interface from a source IP range, using the given mode (either REJECT or DROP).

func (*Partition) Descriptor

func (*Partition) Descriptor() ([]byte, []int)

func (*Partition) GetFromIpRange

func (m *Partition) GetFromIpRange() string

func (*Partition) GetInterfaceIndex

func (m *Partition) GetInterfaceIndex() string

func (*Partition) GetMode

func (m *Partition) GetMode() string

func (*Partition) ProtoMessage

func (*Partition) ProtoMessage()

func (*Partition) Reset

func (m *Partition) Reset()

func (*Partition) String

func (m *Partition) String() string

func (*Partition) XXX_DiscardUnknown

func (m *Partition) XXX_DiscardUnknown()

func (*Partition) XXX_Marshal

func (m *Partition) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)

func (*Partition) XXX_Merge

func (m *Partition) XXX_Merge(src proto.Message)

func (*Partition) XXX_Size

func (m *Partition) XXX_Size() int

func (*Partition) XXX_Unmarshal

func (m *Partition) XXX_Unmarshal(b []byte) error

type PartitionMode

type PartitionMode string

PartitionMode to use with `iptables` command; REJECT or DROP.

var (
	Drop   PartitionMode = "DROP"
	Reject PartitionMode = "REJECT"
)