Documentation ¶
Index ¶
- Variables
- func ExtractShardSpecMetaLabels(spec *ShardSpec, out pb.LabelSet) pb.LabelSet
- func NewKeySpace(prefix string) *keyspace.KeySpace
- func RegisterShardServer(s *grpc.Server, srv ShardServer)
- type Application
- type ApplyRequest
- func (*ApplyRequest) Descriptor() ([]byte, []int)
- func (m *ApplyRequest) Marshal() (dAtA []byte, err error)
- func (m *ApplyRequest) MarshalTo(dAtA []byte) (int, error)
- func (*ApplyRequest) ProtoMessage()
- func (m *ApplyRequest) ProtoSize() (n int)
- func (m *ApplyRequest) Reset()
- func (m *ApplyRequest) String() string
- func (m *ApplyRequest) Unmarshal(dAtA []byte) error
- func (m *ApplyRequest) Validate() error
- func (m *ApplyRequest) XXX_DiscardUnknown()
- func (m *ApplyRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (dst *ApplyRequest) XXX_Merge(src proto.Message)
- func (m *ApplyRequest) XXX_Size() int
- func (m *ApplyRequest) XXX_Unmarshal(b []byte) error
- type ApplyRequest_Change
- func (*ApplyRequest_Change) Descriptor() ([]byte, []int)
- func (m *ApplyRequest_Change) Marshal() (dAtA []byte, err error)
- func (m *ApplyRequest_Change) MarshalTo(dAtA []byte) (int, error)
- func (*ApplyRequest_Change) ProtoMessage()
- func (m *ApplyRequest_Change) ProtoSize() (n int)
- func (m *ApplyRequest_Change) Reset()
- func (m *ApplyRequest_Change) String() string
- func (m *ApplyRequest_Change) Unmarshal(dAtA []byte) error
- func (m *ApplyRequest_Change) Validate() error
- func (m *ApplyRequest_Change) XXX_DiscardUnknown()
- func (m *ApplyRequest_Change) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (dst *ApplyRequest_Change) XXX_Merge(src proto.Message)
- func (m *ApplyRequest_Change) XXX_Size() int
- func (m *ApplyRequest_Change) XXX_Unmarshal(b []byte) error
- type ApplyResponse
- func (*ApplyResponse) Descriptor() ([]byte, []int)
- func (m *ApplyResponse) Marshal() (dAtA []byte, err error)
- func (m *ApplyResponse) MarshalTo(dAtA []byte) (int, error)
- func (*ApplyResponse) ProtoMessage()
- func (m *ApplyResponse) ProtoSize() (n int)
- func (m *ApplyResponse) Reset()
- func (m *ApplyResponse) String() string
- func (m *ApplyResponse) Unmarshal(dAtA []byte) error
- func (m *ApplyResponse) Validate() error
- func (m *ApplyResponse) XXX_DiscardUnknown()
- func (m *ApplyResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (dst *ApplyResponse) XXX_Merge(src proto.Message)
- func (m *ApplyResponse) XXX_Size() int
- func (m *ApplyResponse) XXX_Unmarshal(b []byte) error
- type BeginFinisher
- type ConsumerSpec
- func (*ConsumerSpec) Descriptor() ([]byte, []int)
- func (m *ConsumerSpec) ItemLimit() int
- func (m *ConsumerSpec) Marshal() (dAtA []byte, err error)
- func (m *ConsumerSpec) MarshalString() string
- func (m *ConsumerSpec) MarshalTo(dAtA []byte) (int, error)
- func (*ConsumerSpec) ProtoMessage()
- func (m *ConsumerSpec) ProtoSize() (n int)
- func (m *ConsumerSpec) Reset()
- func (m *ConsumerSpec) String() string
- func (m *ConsumerSpec) Unmarshal(dAtA []byte) error
- func (m *ConsumerSpec) Validate() error
- func (m *ConsumerSpec) XXX_DiscardUnknown()
- func (m *ConsumerSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (dst *ConsumerSpec) XXX_Merge(src proto.Message)
- func (m *ConsumerSpec) XXX_Size() int
- func (m *ConsumerSpec) XXX_Unmarshal(b []byte) error
- func (m *ConsumerSpec) ZeroLimit()
- type GetHintsRequest
- func (*GetHintsRequest) Descriptor() ([]byte, []int)
- func (m *GetHintsRequest) Marshal() (dAtA []byte, err error)
- func (m *GetHintsRequest) MarshalTo(dAtA []byte) (int, error)
- func (*GetHintsRequest) ProtoMessage()
- func (m *GetHintsRequest) ProtoSize() (n int)
- func (m *GetHintsRequest) Reset()
- func (m *GetHintsRequest) String() string
- func (m *GetHintsRequest) Unmarshal(dAtA []byte) error
- func (m *GetHintsRequest) Validate() error
- func (m *GetHintsRequest) XXX_DiscardUnknown()
- func (m *GetHintsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (dst *GetHintsRequest) XXX_Merge(src proto.Message)
- func (m *GetHintsRequest) XXX_Size() int
- func (m *GetHintsRequest) XXX_Unmarshal(b []byte) error
- type GetHintsResponse
- func (*GetHintsResponse) Descriptor() ([]byte, []int)
- func (m *GetHintsResponse) Marshal() (dAtA []byte, err error)
- func (m *GetHintsResponse) MarshalTo(dAtA []byte) (int, error)
- func (*GetHintsResponse) ProtoMessage()
- func (m *GetHintsResponse) ProtoSize() (n int)
- func (m *GetHintsResponse) Reset()
- func (m *GetHintsResponse) String() string
- func (m *GetHintsResponse) Unmarshal(dAtA []byte) error
- func (m *GetHintsResponse) Validate() error
- func (m *GetHintsResponse) XXX_DiscardUnknown()
- func (m *GetHintsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (dst *GetHintsResponse) XXX_Merge(src proto.Message)
- func (m *GetHintsResponse) XXX_Size() int
- func (m *GetHintsResponse) XXX_Unmarshal(b []byte) error
- type GetHintsResponse_ResponseHints
- func (*GetHintsResponse_ResponseHints) Descriptor() ([]byte, []int)
- func (m *GetHintsResponse_ResponseHints) Marshal() (dAtA []byte, err error)
- func (m *GetHintsResponse_ResponseHints) MarshalTo(dAtA []byte) (int, error)
- func (*GetHintsResponse_ResponseHints) ProtoMessage()
- func (m *GetHintsResponse_ResponseHints) ProtoSize() (n int)
- func (m *GetHintsResponse_ResponseHints) Reset()
- func (m *GetHintsResponse_ResponseHints) String() string
- func (m *GetHintsResponse_ResponseHints) Unmarshal(dAtA []byte) error
- func (m GetHintsResponse_ResponseHints) Validate() error
- func (m *GetHintsResponse_ResponseHints) XXX_DiscardUnknown()
- func (m *GetHintsResponse_ResponseHints) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (dst *GetHintsResponse_ResponseHints) XXX_Merge(src proto.Message)
- func (m *GetHintsResponse_ResponseHints) XXX_Size() int
- func (m *GetHintsResponse_ResponseHints) XXX_Unmarshal(b []byte) error
- type JSONFileStore
- type ListRequest
- func (*ListRequest) Descriptor() ([]byte, []int)
- func (m *ListRequest) Marshal() (dAtA []byte, err error)
- func (m *ListRequest) MarshalTo(dAtA []byte) (int, error)
- func (*ListRequest) ProtoMessage()
- func (m *ListRequest) ProtoSize() (n int)
- func (m *ListRequest) Reset()
- func (m *ListRequest) String() string
- func (m *ListRequest) Unmarshal(dAtA []byte) error
- func (m *ListRequest) Validate() error
- func (m *ListRequest) XXX_DiscardUnknown()
- func (m *ListRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (dst *ListRequest) XXX_Merge(src proto.Message)
- func (m *ListRequest) XXX_Size() int
- func (m *ListRequest) XXX_Unmarshal(b []byte) error
- type ListResponse
- func (*ListResponse) Descriptor() ([]byte, []int)
- func (m *ListResponse) Marshal() (dAtA []byte, err error)
- func (m *ListResponse) MarshalTo(dAtA []byte) (int, error)
- func (*ListResponse) ProtoMessage()
- func (m *ListResponse) ProtoSize() (n int)
- func (m *ListResponse) Reset()
- func (m *ListResponse) String() string
- func (m *ListResponse) Unmarshal(dAtA []byte) error
- func (m *ListResponse) Validate() error
- func (m *ListResponse) XXX_DiscardUnknown()
- func (m *ListResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (dst *ListResponse) XXX_Merge(src proto.Message)
- func (m *ListResponse) XXX_Size() int
- func (m *ListResponse) XXX_Unmarshal(b []byte) error
- type ListResponse_Shard
- func (*ListResponse_Shard) Descriptor() ([]byte, []int)
- func (m *ListResponse_Shard) Marshal() (dAtA []byte, err error)
- func (m *ListResponse_Shard) MarshalTo(dAtA []byte) (int, error)
- func (*ListResponse_Shard) ProtoMessage()
- func (m *ListResponse_Shard) ProtoSize() (n int)
- func (m *ListResponse_Shard) Reset()
- func (m *ListResponse_Shard) String() string
- func (m *ListResponse_Shard) Unmarshal(dAtA []byte) error
- func (m *ListResponse_Shard) Validate() error
- func (m *ListResponse_Shard) XXX_DiscardUnknown()
- func (m *ListResponse_Shard) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (dst *ListResponse_Shard) XXX_Merge(src proto.Message)
- func (m *ListResponse_Shard) XXX_Size() int
- func (m *ListResponse_Shard) XXX_Unmarshal(b []byte) error
- type Replica
- type ReplicaStatus
- func (*ReplicaStatus) Descriptor() ([]byte, []int)
- func (m *ReplicaStatus) Marshal() (dAtA []byte, err error)
- func (m *ReplicaStatus) MarshalString() string
- func (m *ReplicaStatus) MarshalTo(dAtA []byte) (int, error)
- func (*ReplicaStatus) ProtoMessage()
- func (m *ReplicaStatus) ProtoSize() (n int)
- func (m *ReplicaStatus) Reduce(other *ReplicaStatus)
- func (m *ReplicaStatus) Reset()
- func (m *ReplicaStatus) String() string
- func (m *ReplicaStatus) Unmarshal(dAtA []byte) error
- func (m *ReplicaStatus) Validate() error
- func (m *ReplicaStatus) XXX_DiscardUnknown()
- func (m *ReplicaStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (dst *ReplicaStatus) XXX_Merge(src proto.Message)
- func (m *ReplicaStatus) XXX_Size() int
- func (m *ReplicaStatus) XXX_Unmarshal(b []byte) error
- type ReplicaStatus_Code
- type Resolution
- type ResolveArgs
- type Resolver
- type RoutedShardClient
- type Service
- func (srv *Service) Apply(ctx context.Context, req *ApplyRequest) (*ApplyResponse, error)
- func (srv *Service) GetHints(ctx context.Context, req *GetHintsRequest) (*GetHintsResponse, error)
- func (srv *Service) List(ctx context.Context, req *ListRequest) (*ListResponse, error)
- func (srv *Service) Stat(ctx context.Context, req *StatRequest) (*StatResponse, error)
- func (svc *Service) Watch(ctx context.Context) error
- type Shard
- type ShardClient
- type ShardID
- type ShardServer
- type ShardSpec
- func (*ShardSpec) Descriptor() ([]byte, []int)
- func (m *ShardSpec) DesiredReplication() int
- func (m *ShardSpec) HintBackupKeys() []string
- func (m *ShardSpec) HintPrimaryKey() string
- func (m *ShardSpec) IsConsistent(assignment keyspace.KeyValue, _ keyspace.KeyValues) bool
- func (m *ShardSpec) Marshal() (dAtA []byte, err error)
- func (m *ShardSpec) MarshalString() string
- func (m *ShardSpec) MarshalTo(dAtA []byte) (int, error)
- func (*ShardSpec) ProtoMessage()
- func (m *ShardSpec) ProtoSize() (n int)
- func (m *ShardSpec) RecoveryLog() pb.Journal
- func (m *ShardSpec) Reset()
- func (m *ShardSpec) String() string
- func (m *ShardSpec) Unmarshal(dAtA []byte) error
- func (m *ShardSpec) Validate() error
- func (m *ShardSpec) XXX_DiscardUnknown()
- func (m *ShardSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (dst *ShardSpec) XXX_Merge(src proto.Message)
- func (m *ShardSpec) XXX_Size() int
- func (m *ShardSpec) XXX_Unmarshal(b []byte) error
- type ShardSpec_Source
- func (*ShardSpec_Source) Descriptor() ([]byte, []int)
- func (m *ShardSpec_Source) Marshal() (dAtA []byte, err error)
- func (m *ShardSpec_Source) MarshalTo(dAtA []byte) (int, error)
- func (*ShardSpec_Source) ProtoMessage()
- func (m *ShardSpec_Source) ProtoSize() (n int)
- func (m *ShardSpec_Source) Reset()
- func (m *ShardSpec_Source) String() string
- func (m *ShardSpec_Source) Unmarshal(dAtA []byte) error
- func (m *ShardSpec_Source) Validate() error
- func (m *ShardSpec_Source) XXX_DiscardUnknown()
- func (m *ShardSpec_Source) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (dst *ShardSpec_Source) XXX_Merge(src proto.Message)
- func (m *ShardSpec_Source) XXX_Size() int
- func (m *ShardSpec_Source) XXX_Unmarshal(b []byte) error
- type StatRequest
- func (*StatRequest) Descriptor() ([]byte, []int)
- func (m *StatRequest) Marshal() (dAtA []byte, err error)
- func (m *StatRequest) MarshalTo(dAtA []byte) (int, error)
- func (*StatRequest) ProtoMessage()
- func (m *StatRequest) ProtoSize() (n int)
- func (m *StatRequest) Reset()
- func (m *StatRequest) String() string
- func (m *StatRequest) Unmarshal(dAtA []byte) error
- func (m *StatRequest) Validate() error
- func (m *StatRequest) XXX_DiscardUnknown()
- func (m *StatRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (dst *StatRequest) XXX_Merge(src proto.Message)
- func (m *StatRequest) XXX_Size() int
- func (m *StatRequest) XXX_Unmarshal(b []byte) error
- type StatResponse
- func (*StatResponse) Descriptor() ([]byte, []int)
- func (m *StatResponse) Marshal() (dAtA []byte, err error)
- func (m *StatResponse) MarshalTo(dAtA []byte) (int, error)
- func (*StatResponse) ProtoMessage()
- func (m *StatResponse) ProtoSize() (n int)
- func (m *StatResponse) Reset()
- func (m *StatResponse) String() string
- func (m *StatResponse) Unmarshal(dAtA []byte) error
- func (m *StatResponse) Validate() error
- func (m *StatResponse) XXX_DiscardUnknown()
- func (m *StatResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
- func (dst *StatResponse) XXX_Merge(src proto.Message)
- func (m *StatResponse) XXX_Size() int
- func (m *StatResponse) XXX_Unmarshal(b []byte) error
- type Status
- type Store
Constants ¶
This section is empty.
Variables ¶
var ( ErrInvalidLengthConsumer = fmt.Errorf("proto: negative length found during unmarshaling") ErrIntOverflowConsumer = fmt.Errorf("proto: integer overflow") )
var ReplicaStatus_Code_name = map[int32]string{
0: "IDLE",
100: "BACKFILL",
200: "TAILING",
300: "PRIMARY",
400: "FAILED",
}
var ReplicaStatus_Code_value = map[string]int32{
"IDLE": 0,
"BACKFILL": 100,
"TAILING": 200,
"PRIMARY": 300,
"FAILED": 400,
}
var Status_name = map[int32]string{
0: "OK",
1: "SHARD_NOT_FOUND",
2: "NO_SHARD_PRIMARY",
3: "NOT_SHARD_PRIMARY",
4: "ETCD_TRANSACTION_FAILED",
}
var Status_value = map[string]int32{
"OK": 0,
"SHARD_NOT_FOUND": 1,
"NO_SHARD_PRIMARY": 2,
"NOT_SHARD_PRIMARY": 3,
"ETCD_TRANSACTION_FAILED": 4,
}
Functions ¶
func ExtractShardSpecMetaLabels ¶
ExtractShardSpecMetaLabels returns meta-labels of the ShardSpec, using |out| as a buffer.
func NewKeySpace ¶
NewKeySpace returns a KeySpace suitable for use with an Allocator. It decodes allocator Items as ShardSpec messages, Members as ConsumerSpecs, and Assignments as RecoveryStatus enums.
func RegisterShardServer ¶
func RegisterShardServer(s *grpc.Server, srv ShardServer)
Types ¶
type Application ¶
type Application interface { // NewStore constructs a Store for |shard| around the recovered local directory // |dir| and initialized Recorder |rec|. NewStore(shard Shard, dir string, rec *recoverylog.Recorder) (Store, error) // NewMessage returns a zero-valued Message of an appropriate representation // for the JournalSpec. NewMessage(*pb.JournalSpec) (message.Message, error) // ConsumeMessage consumes a message within the scope of a transaction. ConsumeMessage(Shard, Store, message.Envelope) error // FinalizeTxn indicates a consumer transaction is ending, and that the // Application must flush any in-memory transaction state or caches, and // issued any relevant journal writes. At completion all writes must have // been published to the Shard AsyncJournalClient, and all state must be captured // by the Store. FinalizeTxn(shard Shard, store Store) error }
Application is the interface provided by domain applications running as Gazette consumers. Only unrecoverable errors should be returned by Application. A returned error will abort processing of an assigned Shard, and will update the assignment's ReplicaStatus to FAILED.
Gazette consumers process messages within pipelined transactions. A transaction begins upon receipt of a new message from a read journal at its present read offset. The message is consumed, and in the course of the transaction many more messages may be read & consumed. When consuming a message, the Application is free to consult the Store, to publish other messages to other journals via the Shard AsyncJournalClient, or to simply keep in-memory-only aggregates such as counts.
Eventually, either because of a read stall or maximum duration, the transaction will be finalized, at which point the Application must at least _start_ any related journal writes, and must reflect any in-memory caches or aggregates back into the Store.
The offsets which have been read through are also flushed to the Store at this time, and to disk, often as an atomic update mixed with Application state updates (when atomic writes are used, the framework provides an exactly-once guarantee over message state updates).
This final flush to the Store (and its recoverylog) is made dependent upon all other writes issued to the AsyncJournalClient, ensuring that read offsets are persisted only after related writes have completed, and thereby providing an at-least once guarantee for journal writes driven by the transaction.
Transactions are pipelined, which means that while asynchronous writes from the finalized transaction run to completion, another consumer transaction may begin processing concurrently. That new transaction is constrained only in that it may not itself finalize until all writes of its predecessor have fully committed to the brokers. If that takes so long that the successor transaction reaches its maximum duration, then that successor will stall without processing further messages until all predecessor writes commit.
type ApplyRequest ¶
type ApplyRequest struct {
Changes []ApplyRequest_Change `protobuf:"bytes,1,rep,name=changes" json:"changes"`
}
func (*ApplyRequest) Descriptor ¶
func (*ApplyRequest) Descriptor() ([]byte, []int)
func (*ApplyRequest) Marshal ¶
func (m *ApplyRequest) Marshal() (dAtA []byte, err error)
func (*ApplyRequest) ProtoMessage ¶
func (*ApplyRequest) ProtoMessage()
func (*ApplyRequest) ProtoSize ¶
func (m *ApplyRequest) ProtoSize() (n int)
func (*ApplyRequest) Reset ¶
func (m *ApplyRequest) Reset()
func (*ApplyRequest) String ¶
func (m *ApplyRequest) String() string
func (*ApplyRequest) Unmarshal ¶
func (m *ApplyRequest) Unmarshal(dAtA []byte) error
func (*ApplyRequest) Validate ¶
func (m *ApplyRequest) Validate() error
Validate returns an error if the ApplyRequest is not well-formed.
func (*ApplyRequest) XXX_DiscardUnknown ¶
func (m *ApplyRequest) XXX_DiscardUnknown()
func (*ApplyRequest) XXX_Marshal ¶
func (m *ApplyRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
func (*ApplyRequest) XXX_Merge ¶
func (dst *ApplyRequest) XXX_Merge(src proto.Message)
func (*ApplyRequest) XXX_Size ¶
func (m *ApplyRequest) XXX_Size() int
func (*ApplyRequest) XXX_Unmarshal ¶
func (m *ApplyRequest) XXX_Unmarshal(b []byte) error
type ApplyRequest_Change ¶
type ApplyRequest_Change struct { // Expected ModRevision of the current ShardSpec. If the shard is being // created, expect_mod_revision is zero. ExpectModRevision int64 `protobuf:"varint,1,opt,name=expect_mod_revision,json=expectModRevision,proto3" json:"expect_mod_revision,omitempty"` // ShardSpec to be updated (if expect_mod_revision > 0) or created // (if expect_mod_revision == 0). Upsert *ShardSpec `protobuf:"bytes,2,opt,name=upsert" json:"upsert,omitempty"` // Shard to be deleted. expect_mod_revision must not be zero. Delete ShardID `protobuf:"bytes,3,opt,name=delete,proto3,casttype=ShardID" json:"delete,omitempty"` }
Change defines an insertion, update, or deletion to be applied to the set of ShardSpecs. Exactly one of |upsert| or |delete| must be set.
func (*ApplyRequest_Change) Descriptor ¶
func (*ApplyRequest_Change) Descriptor() ([]byte, []int)
func (*ApplyRequest_Change) Marshal ¶
func (m *ApplyRequest_Change) Marshal() (dAtA []byte, err error)
func (*ApplyRequest_Change) MarshalTo ¶
func (m *ApplyRequest_Change) MarshalTo(dAtA []byte) (int, error)
func (*ApplyRequest_Change) ProtoMessage ¶
func (*ApplyRequest_Change) ProtoMessage()
func (*ApplyRequest_Change) ProtoSize ¶
func (m *ApplyRequest_Change) ProtoSize() (n int)
func (*ApplyRequest_Change) Reset ¶
func (m *ApplyRequest_Change) Reset()
func (*ApplyRequest_Change) String ¶
func (m *ApplyRequest_Change) String() string
func (*ApplyRequest_Change) Unmarshal ¶
func (m *ApplyRequest_Change) Unmarshal(dAtA []byte) error
func (*ApplyRequest_Change) Validate ¶
func (m *ApplyRequest_Change) Validate() error
Validate returns an error if the ApplyRequest_Change is not well-formed.
func (*ApplyRequest_Change) XXX_DiscardUnknown ¶
func (m *ApplyRequest_Change) XXX_DiscardUnknown()
func (*ApplyRequest_Change) XXX_Marshal ¶
func (m *ApplyRequest_Change) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
func (*ApplyRequest_Change) XXX_Merge ¶
func (dst *ApplyRequest_Change) XXX_Merge(src proto.Message)
func (*ApplyRequest_Change) XXX_Size ¶
func (m *ApplyRequest_Change) XXX_Size() int
func (*ApplyRequest_Change) XXX_Unmarshal ¶
func (m *ApplyRequest_Change) XXX_Unmarshal(b []byte) error
type ApplyResponse ¶
type ApplyResponse struct { // Status of the Apply RPC. Status Status `protobuf:"varint,1,opt,name=status,proto3,enum=consumer.Status" json:"status,omitempty"` // Header of the response. Header protocol.Header `protobuf:"bytes,2,opt,name=header" json:"header"` }
func ApplyShards ¶
func ApplyShards(ctx context.Context, sc ShardClient, req *ApplyRequest) (*ApplyResponse, error)
ApplyShards invokes the Apply RPC.
func ApplyShardsInBatches ¶
func ApplyShardsInBatches(ctx context.Context, sc ShardClient, req *ApplyRequest, size int) (*ApplyResponse, error)
ApplyShardsInBatches applies changes to shards which may be larger than the configured etcd transaction size size. The changes in |req| will be sent serially in batches of size |size|. If |size| is 0 all changes will be attempted as part of a single transaction. This function will return the response of the final ShardClient.Apply call. Response validation or !OK status from Apply RPC are mapped to error.
func (*ApplyResponse) Descriptor ¶
func (*ApplyResponse) Descriptor() ([]byte, []int)
func (*ApplyResponse) Marshal ¶
func (m *ApplyResponse) Marshal() (dAtA []byte, err error)
func (*ApplyResponse) ProtoMessage ¶
func (*ApplyResponse) ProtoMessage()
func (*ApplyResponse) ProtoSize ¶
func (m *ApplyResponse) ProtoSize() (n int)
func (*ApplyResponse) Reset ¶
func (m *ApplyResponse) Reset()
func (*ApplyResponse) String ¶
func (m *ApplyResponse) String() string
func (*ApplyResponse) Unmarshal ¶
func (m *ApplyResponse) Unmarshal(dAtA []byte) error
func (*ApplyResponse) Validate ¶
func (m *ApplyResponse) Validate() error
Validate returns an error if the ApplyResponse is not well-formed.
func (*ApplyResponse) XXX_DiscardUnknown ¶
func (m *ApplyResponse) XXX_DiscardUnknown()
func (*ApplyResponse) XXX_Marshal ¶
func (m *ApplyResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
func (*ApplyResponse) XXX_Merge ¶
func (dst *ApplyResponse) XXX_Merge(src proto.Message)
func (*ApplyResponse) XXX_Size ¶
func (m *ApplyResponse) XXX_Size() int
func (*ApplyResponse) XXX_Unmarshal ¶
func (m *ApplyResponse) XXX_Unmarshal(b []byte) error
type BeginFinisher ¶
type BeginFinisher interface { // BeginTxn is called to notify the Application that a transaction is beginning // (and a call to ConsumeMessage will be immediately forthcoming), allowing // the Application to perform any preparatory work. For consumers doing // extensive aggregation, it may be beneficial to focus available compute // resource on a small number of transactions while completely stalling // others: this can be accomplished by blocking in BeginTxn until a semaphore // is acquired. A call to BeginTx is always paired with a call to FinishTxn. BeginTxn(Shard, Store) error // FinishTxn is notified that a previously begun transaction has completed // or errored. It allows the Application to perform related cleanup (eg, // releasing a previously acquired semaphore), implement transactional / // exactly-once semantics (via two-phase commit), etc. Note that, because // transactions are pipelined, underlying journal writes of the transaction // may still be ongoing. The Application can use a WeakBarrier() of the // Recorder to condition further writes (eg, a "commit acknowledgement" // message in 2PC) on the consumer transaction having fully committed. // If the argument error is non-nil, the transaction failed and the Shard // is entering a "failed" state. It is not necessary to pass-through or // return an error just because argument error is non-nil. FinishTxn(Shard, Store, error) error }
BeginFinisher is an optional interface of Application which is informed when consumer transactions begin or finish.
type ConsumerSpec ¶
type ConsumerSpec struct { // ProcessSpec of the consumer. protocol.ProcessSpec `protobuf:"bytes,1,opt,name=process_spec,json=processSpec,embedded=process_spec" json:"process_spec" yaml:",inline"` // Maximum number of assigned Shards. ShardLimit uint32 `protobuf:"varint,2,opt,name=shard_limit,json=shardLimit,proto3" json:"shard_limit,omitempty"` }
ConsumerSpec describes a Consumer process instance and its configuration. It serves as a allocator MemberValue.
func (*ConsumerSpec) Descriptor ¶
func (*ConsumerSpec) Descriptor() ([]byte, []int)
func (*ConsumerSpec) ItemLimit ¶
func (m *ConsumerSpec) ItemLimit() int
ItemLimit is the maximum number of shards this consumer may process. allocator.MemberValue implementation.
func (*ConsumerSpec) Marshal ¶
func (m *ConsumerSpec) Marshal() (dAtA []byte, err error)
func (*ConsumerSpec) MarshalString ¶
func (m *ConsumerSpec) MarshalString() string
MarshalString returns the marshaled encoding of the ConsumerSpec as a string.
func (*ConsumerSpec) ProtoMessage ¶
func (*ConsumerSpec) ProtoMessage()
func (*ConsumerSpec) ProtoSize ¶
func (m *ConsumerSpec) ProtoSize() (n int)
func (*ConsumerSpec) Reset ¶
func (m *ConsumerSpec) Reset()
func (*ConsumerSpec) String ¶
func (m *ConsumerSpec) String() string
func (*ConsumerSpec) Unmarshal ¶
func (m *ConsumerSpec) Unmarshal(dAtA []byte) error
func (*ConsumerSpec) Validate ¶
func (m *ConsumerSpec) Validate() error
Validate returns an error if the ConsumerSpec is not well-formed.
func (*ConsumerSpec) XXX_DiscardUnknown ¶
func (m *ConsumerSpec) XXX_DiscardUnknown()
func (*ConsumerSpec) XXX_Marshal ¶
func (m *ConsumerSpec) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
func (*ConsumerSpec) XXX_Merge ¶
func (dst *ConsumerSpec) XXX_Merge(src proto.Message)
func (*ConsumerSpec) XXX_Size ¶
func (m *ConsumerSpec) XXX_Size() int
func (*ConsumerSpec) XXX_Unmarshal ¶
func (m *ConsumerSpec) XXX_Unmarshal(b []byte) error
func (*ConsumerSpec) ZeroLimit ¶
func (m *ConsumerSpec) ZeroLimit()
ZeroLimit zeros the ConsumerSpec ShardLimit.
type GetHintsRequest ¶
type GetHintsRequest struct { // Shard to fetch hints for. Shard ShardID `protobuf:"bytes,1,opt,name=shard,proto3,casttype=ShardID" json:"shard,omitempty"` }
func (*GetHintsRequest) Descriptor ¶
func (*GetHintsRequest) Descriptor() ([]byte, []int)
func (*GetHintsRequest) Marshal ¶
func (m *GetHintsRequest) Marshal() (dAtA []byte, err error)
func (*GetHintsRequest) ProtoMessage ¶
func (*GetHintsRequest) ProtoMessage()
func (*GetHintsRequest) ProtoSize ¶
func (m *GetHintsRequest) ProtoSize() (n int)
func (*GetHintsRequest) Reset ¶
func (m *GetHintsRequest) Reset()
func (*GetHintsRequest) String ¶
func (m *GetHintsRequest) String() string
func (*GetHintsRequest) Unmarshal ¶
func (m *GetHintsRequest) Unmarshal(dAtA []byte) error
func (*GetHintsRequest) Validate ¶
func (m *GetHintsRequest) Validate() error
Validate returns an error if the HintsRequest is not well-formed.
func (*GetHintsRequest) XXX_DiscardUnknown ¶
func (m *GetHintsRequest) XXX_DiscardUnknown()
func (*GetHintsRequest) XXX_Marshal ¶
func (m *GetHintsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
func (*GetHintsRequest) XXX_Merge ¶
func (dst *GetHintsRequest) XXX_Merge(src proto.Message)
func (*GetHintsRequest) XXX_Size ¶
func (m *GetHintsRequest) XXX_Size() int
func (*GetHintsRequest) XXX_Unmarshal ¶
func (m *GetHintsRequest) XXX_Unmarshal(b []byte) error
type GetHintsResponse ¶
type GetHintsResponse struct { // Status of the Hints RPC. Status Status `protobuf:"varint,1,opt,name=status,proto3,enum=consumer.Status" json:"status,omitempty"` // Header of the response. Header protocol.Header `protobuf:"bytes,2,opt,name=header" json:"header"` // Primary hints for the shard. PrimaryHints GetHintsResponse_ResponseHints `protobuf:"bytes,3,opt,name=primary_hints,json=primaryHints" json:"primary_hints"` // List of backup hints for a shard. The most recent recovery log hints will be // first, any subsequent hints are for historical backup. If there is no value // for a hint key the value corresponding hints will be nil. BackupHints []GetHintsResponse_ResponseHints `protobuf:"bytes,4,rep,name=backup_hints,json=backupHints" json:"backup_hints"` }
func FetchHints ¶
func FetchHints(ctx context.Context, sc ShardClient, req *GetHintsRequest) (*GetHintsResponse, error)
FetchHints invokes the Hints RPC, and maps a validation or !OK status to an error.
func (*GetHintsResponse) Descriptor ¶
func (*GetHintsResponse) Descriptor() ([]byte, []int)
func (*GetHintsResponse) Marshal ¶
func (m *GetHintsResponse) Marshal() (dAtA []byte, err error)
func (*GetHintsResponse) ProtoMessage ¶
func (*GetHintsResponse) ProtoMessage()
func (*GetHintsResponse) ProtoSize ¶
func (m *GetHintsResponse) ProtoSize() (n int)
func (*GetHintsResponse) Reset ¶
func (m *GetHintsResponse) Reset()
func (*GetHintsResponse) String ¶
func (m *GetHintsResponse) String() string
func (*GetHintsResponse) Unmarshal ¶
func (m *GetHintsResponse) Unmarshal(dAtA []byte) error
func (*GetHintsResponse) Validate ¶
func (m *GetHintsResponse) Validate() error
Validate returns an error if the HintsResponse is not well-formed.
func (*GetHintsResponse) XXX_DiscardUnknown ¶
func (m *GetHintsResponse) XXX_DiscardUnknown()
func (*GetHintsResponse) XXX_Marshal ¶
func (m *GetHintsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
func (*GetHintsResponse) XXX_Merge ¶
func (dst *GetHintsResponse) XXX_Merge(src proto.Message)
func (*GetHintsResponse) XXX_Size ¶
func (m *GetHintsResponse) XXX_Size() int
func (*GetHintsResponse) XXX_Unmarshal ¶
func (m *GetHintsResponse) XXX_Unmarshal(b []byte) error
type GetHintsResponse_ResponseHints ¶
type GetHintsResponse_ResponseHints struct { // If the hints value does not exist Hints will be nill. Hints *recoverylog.FSMHints `protobuf:"bytes,1,opt,name=hints" json:"hints,omitempty"` }
func (*GetHintsResponse_ResponseHints) Descriptor ¶
func (*GetHintsResponse_ResponseHints) Descriptor() ([]byte, []int)
func (*GetHintsResponse_ResponseHints) Marshal ¶
func (m *GetHintsResponse_ResponseHints) Marshal() (dAtA []byte, err error)
func (*GetHintsResponse_ResponseHints) MarshalTo ¶
func (m *GetHintsResponse_ResponseHints) MarshalTo(dAtA []byte) (int, error)
func (*GetHintsResponse_ResponseHints) ProtoMessage ¶
func (*GetHintsResponse_ResponseHints) ProtoMessage()
func (*GetHintsResponse_ResponseHints) ProtoSize ¶
func (m *GetHintsResponse_ResponseHints) ProtoSize() (n int)
func (*GetHintsResponse_ResponseHints) Reset ¶
func (m *GetHintsResponse_ResponseHints) Reset()
func (*GetHintsResponse_ResponseHints) String ¶
func (m *GetHintsResponse_ResponseHints) String() string
func (*GetHintsResponse_ResponseHints) Unmarshal ¶
func (m *GetHintsResponse_ResponseHints) Unmarshal(dAtA []byte) error
func (GetHintsResponse_ResponseHints) Validate ¶
func (m GetHintsResponse_ResponseHints) Validate() error
Validate returns an error if the GetHintsResponse_ResponseHints is not well-formed.
func (*GetHintsResponse_ResponseHints) XXX_DiscardUnknown ¶
func (m *GetHintsResponse_ResponseHints) XXX_DiscardUnknown()
func (*GetHintsResponse_ResponseHints) XXX_Marshal ¶
func (m *GetHintsResponse_ResponseHints) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
func (*GetHintsResponse_ResponseHints) XXX_Merge ¶
func (dst *GetHintsResponse_ResponseHints) XXX_Merge(src proto.Message)
func (*GetHintsResponse_ResponseHints) XXX_Size ¶
func (m *GetHintsResponse_ResponseHints) XXX_Size() int
func (*GetHintsResponse_ResponseHints) XXX_Unmarshal ¶
func (m *GetHintsResponse_ResponseHints) XXX_Unmarshal(b []byte) error
type JSONFileStore ¶
type JSONFileStore struct { // State is a user-provided instance which is un/marshal-able to JSON. State interface{} // contains filtered or unexported fields }
JSONFileStore is a simple Store which materializes itself as a JSON-encoded file. The store is careful to flush to a new temporary file which is then moved to the well-known location: eg, a process failure cannot result in a recovery of a partially written JSON file.
func NewJSONFileStore ¶
func NewJSONFileStore(rec *recoverylog.Recorder, dir string, state interface{}) (*JSONFileStore, error)
NewJSONFileStore returns a new JSONFileStore. |state| is the runtime instance of the Store's state, which is decoded into, encoded from, and retained as JSONFileState.State.
func (*JSONFileStore) Destroy ¶
func (s *JSONFileStore) Destroy()
Destroy the JSONFileStore directory and state file.
func (*JSONFileStore) FetchJournalOffsets ¶
func (s *JSONFileStore) FetchJournalOffsets() (map[pb.Journal]int64, error)
FetchJournalOffsets returns offsets encoded by the JSONFileStore.
func (*JSONFileStore) Flush ¶
func (s *JSONFileStore) Flush(offsets map[pb.Journal]int64) error
Flush the store's JSONFileState to disk.
func (*JSONFileStore) Recorder ¶
func (s *JSONFileStore) Recorder() *recoverylog.Recorder
Recorder of the JSONFileStore.
type ListRequest ¶
type ListRequest struct { // Selector optionally refines the set of shards which will be enumerated. // If zero-valued, all shards are returned. Otherwise, only ShardSpecs // matching the LabelSelector will be returned. One meta-label "id" is // additionally supported by the selector, where "id=example-shard-ID" // will match a ShardSpec with ID "example-shard-ID". Selector protocol.LabelSelector `protobuf:"bytes,1,opt,name=selector" json:"selector"` }
func (*ListRequest) Descriptor ¶
func (*ListRequest) Descriptor() ([]byte, []int)
func (*ListRequest) Marshal ¶
func (m *ListRequest) Marshal() (dAtA []byte, err error)
func (*ListRequest) ProtoMessage ¶
func (*ListRequest) ProtoMessage()
func (*ListRequest) ProtoSize ¶
func (m *ListRequest) ProtoSize() (n int)
func (*ListRequest) Reset ¶
func (m *ListRequest) Reset()
func (*ListRequest) String ¶
func (m *ListRequest) String() string
func (*ListRequest) Unmarshal ¶
func (m *ListRequest) Unmarshal(dAtA []byte) error
func (*ListRequest) Validate ¶
func (m *ListRequest) Validate() error
Validate returns an error if the ListRequest is not well-formed.
func (*ListRequest) XXX_DiscardUnknown ¶
func (m *ListRequest) XXX_DiscardUnknown()
func (*ListRequest) XXX_Marshal ¶
func (m *ListRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
func (*ListRequest) XXX_Merge ¶
func (dst *ListRequest) XXX_Merge(src proto.Message)
func (*ListRequest) XXX_Size ¶
func (m *ListRequest) XXX_Size() int
func (*ListRequest) XXX_Unmarshal ¶
func (m *ListRequest) XXX_Unmarshal(b []byte) error
type ListResponse ¶
type ListResponse struct { // Status of the List RPC. Status Status `protobuf:"varint,1,opt,name=status,proto3,enum=consumer.Status" json:"status,omitempty"` // Header of the response. Header protocol.Header `protobuf:"bytes,2,opt,name=header" json:"header"` Shards []ListResponse_Shard `protobuf:"bytes,3,rep,name=shards" json:"shards"` }
func ListShards ¶
func ListShards(ctx context.Context, sc ShardClient, req *ListRequest) (*ListResponse, error)
ListShards invokes the List RPC, and maps a validation or !OK status to an error.
func (*ListResponse) Descriptor ¶
func (*ListResponse) Descriptor() ([]byte, []int)
func (*ListResponse) Marshal ¶
func (m *ListResponse) Marshal() (dAtA []byte, err error)
func (*ListResponse) ProtoMessage ¶
func (*ListResponse) ProtoMessage()
func (*ListResponse) ProtoSize ¶
func (m *ListResponse) ProtoSize() (n int)
func (*ListResponse) Reset ¶
func (m *ListResponse) Reset()
func (*ListResponse) String ¶
func (m *ListResponse) String() string
func (*ListResponse) Unmarshal ¶
func (m *ListResponse) Unmarshal(dAtA []byte) error
func (*ListResponse) Validate ¶
func (m *ListResponse) Validate() error
Validate returns an error if the ListResponse is not well-formed.
func (*ListResponse) XXX_DiscardUnknown ¶
func (m *ListResponse) XXX_DiscardUnknown()
func (*ListResponse) XXX_Marshal ¶
func (m *ListResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
func (*ListResponse) XXX_Merge ¶
func (dst *ListResponse) XXX_Merge(src proto.Message)
func (*ListResponse) XXX_Size ¶
func (m *ListResponse) XXX_Size() int
func (*ListResponse) XXX_Unmarshal ¶
func (m *ListResponse) XXX_Unmarshal(b []byte) error
type ListResponse_Shard ¶
type ListResponse_Shard struct { Spec ShardSpec `protobuf:"bytes,1,opt,name=spec" json:"spec"` // Current ModRevision of the ShardSpec. ModRevision int64 `protobuf:"varint,2,opt,name=mod_revision,json=modRevision,proto3" json:"mod_revision,omitempty"` // Route of the shard, including endpoints. Route protocol.Route `protobuf:"bytes,3,opt,name=route" json:"route"` // Status of each replica. Cardinality and ordering matches |route|. Status []ReplicaStatus `protobuf:"bytes,4,rep,name=status" json:"status"` }
Shards of the response.
func (*ListResponse_Shard) Descriptor ¶
func (*ListResponse_Shard) Descriptor() ([]byte, []int)
func (*ListResponse_Shard) Marshal ¶
func (m *ListResponse_Shard) Marshal() (dAtA []byte, err error)
func (*ListResponse_Shard) MarshalTo ¶
func (m *ListResponse_Shard) MarshalTo(dAtA []byte) (int, error)
func (*ListResponse_Shard) ProtoMessage ¶
func (*ListResponse_Shard) ProtoMessage()
func (*ListResponse_Shard) ProtoSize ¶
func (m *ListResponse_Shard) ProtoSize() (n int)
func (*ListResponse_Shard) Reset ¶
func (m *ListResponse_Shard) Reset()
func (*ListResponse_Shard) String ¶
func (m *ListResponse_Shard) String() string
func (*ListResponse_Shard) Unmarshal ¶
func (m *ListResponse_Shard) Unmarshal(dAtA []byte) error
func (*ListResponse_Shard) Validate ¶
func (m *ListResponse_Shard) Validate() error
Validate returns an error if the ListResponse_Shard is not well-formed.
func (*ListResponse_Shard) XXX_DiscardUnknown ¶
func (m *ListResponse_Shard) XXX_DiscardUnknown()
func (*ListResponse_Shard) XXX_Marshal ¶
func (m *ListResponse_Shard) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
func (*ListResponse_Shard) XXX_Merge ¶
func (dst *ListResponse_Shard) XXX_Merge(src proto.Message)
func (*ListResponse_Shard) XXX_Size ¶
func (m *ListResponse_Shard) XXX_Size() int
func (*ListResponse_Shard) XXX_Unmarshal ¶
func (m *ListResponse_Shard) XXX_Unmarshal(b []byte) error
type Replica ¶
type Replica struct {
// contains filtered or unexported fields
}
Replica of a shard which is processed locally.
func NewReplica ¶
func NewReplica(app Application, ks *keyspace.KeySpace, etcd *clientv3.Client, rjc pb.RoutedJournalClient) *Replica
NewReplica returns a Replica in its initial state. The Replica must be transitioned by Resolver to begin processing of a shard.
func (*Replica) Assignment ¶
Assignment of the ShardSpec to the local ConsumerSpec, which motivates this Replica.
func (*Replica) JournalClient ¶
func (r *Replica) JournalClient() client.AsyncJournalClient
JournalClient for broker operations performed in the course of processing this Replica.
type ReplicaStatus ¶
type ReplicaStatus struct { Code ReplicaStatus_Code `protobuf:"varint,1,opt,name=code,proto3,enum=consumer.ReplicaStatus_Code" json:"code,omitempty"` // Errors encountered during replica processing. Set iff |code| is FAILED. Errors []string `protobuf:"bytes,2,rep,name=errors" json:"errors,omitempty"` }
ReplicaStatus is the status of a ShardSpec assigned to a ConsumerSpec. It serves as an allocator AssignmentValue. ReplicaStatus is reduced by taking the maximum enum value among statuses. Eg, if a primary is PRIMARY, one replica is BACKFILL and the other TAILING, then the status is PRIMARY. If one of the replicas transitioned to FAILED, than the status is FAILED. This reduction behavior is used to summarize status across all replicas.
func (*ReplicaStatus) Descriptor ¶
func (*ReplicaStatus) Descriptor() ([]byte, []int)
func (*ReplicaStatus) Marshal ¶
func (m *ReplicaStatus) Marshal() (dAtA []byte, err error)
func (*ReplicaStatus) MarshalString ¶
func (m *ReplicaStatus) MarshalString() string
MarshalString returns the marshaled encoding of the ReplicaStatus as a string.
func (*ReplicaStatus) ProtoMessage ¶
func (*ReplicaStatus) ProtoMessage()
func (*ReplicaStatus) ProtoSize ¶
func (m *ReplicaStatus) ProtoSize() (n int)
func (*ReplicaStatus) Reduce ¶
func (m *ReplicaStatus) Reduce(other *ReplicaStatus)
Reduce folds another ReplicaStatus into this one.
func (*ReplicaStatus) Reset ¶
func (m *ReplicaStatus) Reset()
func (*ReplicaStatus) String ¶
func (m *ReplicaStatus) String() string
func (*ReplicaStatus) Unmarshal ¶
func (m *ReplicaStatus) Unmarshal(dAtA []byte) error
func (*ReplicaStatus) Validate ¶
func (m *ReplicaStatus) Validate() error
Validate returns an error if the ReplicaStatus is not well-formed.
func (*ReplicaStatus) XXX_DiscardUnknown ¶
func (m *ReplicaStatus) XXX_DiscardUnknown()
func (*ReplicaStatus) XXX_Marshal ¶
func (m *ReplicaStatus) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
func (*ReplicaStatus) XXX_Merge ¶
func (dst *ReplicaStatus) XXX_Merge(src proto.Message)
func (*ReplicaStatus) XXX_Size ¶
func (m *ReplicaStatus) XXX_Size() int
func (*ReplicaStatus) XXX_Unmarshal ¶
func (m *ReplicaStatus) XXX_Unmarshal(b []byte) error
type ReplicaStatus_Code ¶
type ReplicaStatus_Code int32
const ( ReplicaStatus_IDLE ReplicaStatus_Code = 0 // The replica is actively playing the historical log. ReplicaStatus_BACKFILL ReplicaStatus_Code = 100 // The replica has finished playing the historical log, and is tailing the // live log to locally mirror recorded operations as they are produced. It // is a "hot standby" and can take over as primary at any time. ReplicaStatus_TAILING ReplicaStatus_Code = 200 // The replica is actively serving as primary. ReplicaStatus_PRIMARY ReplicaStatus_Code = 300 // The replica has encountered an unrecoverable error. ReplicaStatus_FAILED ReplicaStatus_Code = 400 )
func (ReplicaStatus_Code) EnumDescriptor ¶
func (ReplicaStatus_Code) EnumDescriptor() ([]byte, []int)
func (ReplicaStatus_Code) String ¶
func (x ReplicaStatus_Code) String() string
func (ReplicaStatus_Code) Validate ¶
func (x ReplicaStatus_Code) Validate() error
Validate returns an error if the ReplicaStatus_Code is not well-formed.
type Resolution ¶
type Resolution struct { Status Status // Header captures the resolved consumer ProcessId, effective Etcd Revision, // and Route of the shard resolution. Header pb.Header // Spec of the Shard at the current Etcd revision. Spec *ShardSpec // Shard processing context, or nil if this process is not primary for the ShardID. Shard Shard // Store of the Shard, or nil if this process is not primary for the ShardID. Store Store // Done releases Shard & Store, and must be called when no longer needed. // Iff Shard & Store are nil, so is Done. Done func() }
Resolution result of a ShardID.
type ResolveArgs ¶
type ResolveArgs struct { Context context.Context // ShardID to be resolved. ShardID ShardID // Whether we may resolve to another consumer peer. MayProxy bool // Optional Header attached to the request from a proxy-ing peer. ProxyHeader *pb.Header }
ResolveArgs which parametrize resolution of ShardIDs.
type Resolver ¶
type Resolver struct {
// contains filtered or unexported fields
}
Resolver maps shards to responsible consumer processes, and manages the set of local shard Replicas which are assigned to the local ConsumerSpec.
func NewResolver ¶
NewResolver returns a Resolver derived from the allocator.State, which will use the |newReplica| closure to instantiate Replicas for Assignments of the local ConsumerSpec.
func (*Resolver) Resolve ¶
func (r *Resolver) Resolve(args ResolveArgs) (res Resolution, err error)
Resolve a ShardID to its Resolution.
type RoutedShardClient ¶
type RoutedShardClient interface { ShardClient pb.DispatchRouter }
RoutedShardClient composes a ShardClient and DispatchRouter.
func NewRoutedShardClient ¶
func NewRoutedShardClient(sc ShardClient, dr pb.DispatchRouter) RoutedShardClient
NewRoutedShardClient composes a ShardClient and DispatchRouter.
type Service ¶
type Service struct { // Resolver of Service shards. Resolver *Resolver // Distributed allocator state of the service. State *allocator.State // Loopback connection which defaults to the local server, but is wired with // a protocol.DispatchBalancer. Consumer applications may use Loopback to // proxy application-specific RPCs to peer consumer instances, after // performing shard resolution. Loopback *grpc.ClientConn // Journal client for use by consumer applications. Journals pb.RoutedJournalClient // Etcd client for use by consumer applications. Etcd *clientv3.Client }
Service is the top-level runtime concern of a Gazette Consumer process. It drives local shard processing in response to allocator.State, powers shard resolution, and is also an implementation of ShardServer.
func NewService ¶
func NewService(app Application, state *allocator.State, rjc pb.RoutedJournalClient, lo *grpc.ClientConn, etcd *clientv3.Client) *Service
NewService constructs a new Service of the Application, driven by allocator.State.
func (*Service) Apply ¶
func (srv *Service) Apply(ctx context.Context, req *ApplyRequest) (*ApplyResponse, error)
Apply dispatches the ShardServer.Apply API.
func (*Service) GetHints ¶
func (srv *Service) GetHints(ctx context.Context, req *GetHintsRequest) (*GetHintsResponse, error)
GetHints dispatches the ShardServer.Hints API.
func (*Service) List ¶
func (srv *Service) List(ctx context.Context, req *ListRequest) (*ListResponse, error)
List dispatches the ShardServer.List API.
func (*Service) Stat ¶
func (srv *Service) Stat(ctx context.Context, req *StatRequest) (*StatResponse, error)
Stat dispatches the ShardServer.Stat API.
type Shard ¶
type Shard interface { // Context of this shard. Notably, the Context will be cancelled when this // process is no longer responsible for the shard. Context() context.Context // Spec of the shard. Spec() *ShardSpec // Assignment of the shard to this process. Assignment() keyspace.KeyValue // JournalClient for broker operations performed in the course of processing // this Shard. Applications should use this AsyncJournalClient to allow // consumer transactions to track and appropriately sync on written journals. JournalClient() client.AsyncJournalClient }
Shard is the processing context of a ShardSpec which is assigned to the local consumer process.
type ShardClient ¶
type ShardClient interface { // Stat returns detailed status of a given Shard. Stat(ctx context.Context, in *StatRequest, opts ...grpc.CallOption) (*StatResponse, error) // List Shards, their ShardSpecs and their processing status. List(ctx context.Context, in *ListRequest, opts ...grpc.CallOption) (*ListResponse, error) // Apply changes to the collection of Shards managed by the consumer. Apply(ctx context.Context, in *ApplyRequest, opts ...grpc.CallOption) (*ApplyResponse, error) // GetHints fetchs hints for a shard. GetHints(ctx context.Context, in *GetHintsRequest, opts ...grpc.CallOption) (*GetHintsResponse, error) }
ShardClient is the client API for Shard service.
For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
func NewShardClient ¶
func NewShardClient(cc *grpc.ClientConn) ShardClient
type ShardID ¶
type ShardID string
ShardID uniquely identifies a shard processed by a Gazette consumer.
type ShardServer ¶
type ShardServer interface { // Stat returns detailed status of a given Shard. Stat(context.Context, *StatRequest) (*StatResponse, error) // List Shards, their ShardSpecs and their processing status. List(context.Context, *ListRequest) (*ListResponse, error) // Apply changes to the collection of Shards managed by the consumer. Apply(context.Context, *ApplyRequest) (*ApplyResponse, error) // GetHints fetchs hints for a shard. GetHints(context.Context, *GetHintsRequest) (*GetHintsResponse, error) }
ShardServer is the server API for Shard service.
type ShardSpec ¶
type ShardSpec struct { // ID of the Shard. Id ShardID `protobuf:"bytes,1,opt,name=id,proto3,casttype=ShardID" json:"id,omitempty" yaml:",omitempty"` // Sources of the shard, uniquely ordered on Source journal. Sources []ShardSpec_Source `protobuf:"bytes,2,rep,name=sources" json:"sources" yaml:",omitempty"` // Prefix of the Journal into which the Shard's `recoverylog` will be recorded. // The complete Journal name is built as "{recovery_log_prefix}/{shard_id}". RecoveryLogPrefix string `` /* 151-byte string literal not displayed */ // Prefix of Etcd keys into which `recoverylog` FSMHints are written to and // read from. FSMHints allow readers of the `recoverylog` to efficiently // determine the minimum fragments of log which must be read to fully recover // local store state. The complete hint key written by the Shard primary is: // // "{hint_prefix}/{shard_id}.primary" // // The primary will regularly produce updated hints into this key, and // players of the log will similarly utilize hints from this key. HintPrefix string `protobuf:"bytes,4,opt,name=hint_prefix,json=hintPrefix,proto3" json:"hint_prefix,omitempty" yaml:"hint_prefix,omitempty"` // Backups of verified `recoverylog` FSMHints, retained as a disaster-recovery // mechanism. On completing playback, a player will write recovered hints to: // // "{hints_prefix}/{shard_id}.backup.0". // // It also move hints previously stored under // "{hints_prefix/{shard_id}.backup.0" to // "{hints_prefix/{shard_id}.backup.1", and so on, keeping at most // |hint_backups| distinct sets of FSMHints. // // In the case of disaster or data-loss, these copied hints can be an important // fallback for recovering a consistent albeit older version of the Shard // store, with each relying on only progressively older portions of the // recoverylog. // // When pruning the recoverylog, log fragments which are older than (and no // longer required by) the *oldest* backup are discarded, ensuring that // all hints remain valid for playback. HintBackups int32 `protobuf:"varint,5,opt,name=hint_backups,json=hintBackups,proto3" json:"hint_backups,omitempty" yaml:"hint_backups,omitempty"` // Max duration of shard transactions. This duration upper-bounds the amount // of time during which a transaction may process messages before it must flush // and commit. It may run for less time if an input message stall occurs (eg, // no decoded journal message is ready without blocking). A typical value // would be `1s`: applications which perform extensive aggregation over message // streams exhibiting locality of "hot" keys may benefit from larger values. MaxTxnDuration time.Duration `` /* 134-byte string literal not displayed */ // Min duration of shard transactions. This duration lower-bounds the amount // of time during which a transaction must process messages before it may flush // and commit. It may run for more time if additional messages are available // (eg, decoded journal messages are ready without blocking). Note also that // transactions are pipelined: a current transaction may process messages while // a prior transaction's recoverylog writes flush to Gazette, but it cannot // begin to commit until the prior transaction writes complete. In other words // even if |min_txn_quantum| is zero, some degree of message batching is // expected due to the network delay inherent in Gazette writes. A typical // value of would be `0s`: applications which perform extensive aggregation // may benefit from larger values. MinTxnDuration time.Duration `` /* 134-byte string literal not displayed */ // Disable processing of the Shard. Disable bool `protobuf:"varint,8,opt,name=disable,proto3" json:"disable,omitempty" yaml:",omitempty"` // Hot standbys is the desired number of consumer processes which should be // replicating the primary consumer's recovery log. Standbys are allocated in // a separate availability zone of the current primary, and tail the live log // to continuously mirror the primary's on-disk DB file structure. Should the // primary experience failure, one of the hot standbys will be assigned to take // over as the new shard primary, which is accomplished by simply opening its // local copy of the RocksDB. // // Note that under regular operation, Shard hand-off is zero downtime even if // standbys are zero, as the current primary will not cede ownership until the // replacement process has completed log playback. However, a process failure // will leave the Shard without an owner until log playback can complete. HotStandbys uint32 `protobuf:"varint,9,opt,name=hot_standbys,json=hotStandbys,proto3" json:"hot_standbys,omitempty" yaml:"hot_standbys,omitempty"` // User-defined Labels of this ShardSpec. The label "id" is reserved and may // not be used with a ShardSpec's labels. protocol.LabelSet `protobuf:"bytes,10,opt,name=labels,embedded=labels" json:"labels" yaml:",omitempty,inline"` }
ShardSpec describes a shard and its configuration. Shards represent the combination of a consumer application, a specific journal selected for consumption, and a recoverylog journal which replicates the stateful consumption of that journal by the consumer. ShardSpec serves as a allocator ItemValue.
func IntersectShardSpecs ¶
IntersectShardSpecs returns a ShardSpec having a non-zero-valued field for each field value which is shared between |a| and |b|.
func SubtractShardSpecs ¶
SubtractShardSpecs returns a ShardSpec derived from |a| but having a zero-valued field for each field which is matched by |b|.
func UnionShardSpecs ¶
UnionShardSpecs returns a ShardSpec combining all non-zero-valued fields across |a| and |b|. Where both |a| and |b| provide a non-zero value for a field, the value of |a| is retained.
func (*ShardSpec) Descriptor ¶
func (*ShardSpec) DesiredReplication ¶
DesiredReplication is the desired number of shard replicas. allocator.ItemValue implementation.
func (*ShardSpec) HintBackupKeys ¶
HintBackupKeys returns Etcd keys to which verified, disaster-recovery hints are written.
func (*ShardSpec) HintPrimaryKey ¶
HintPrimaryKey returns the Etcd key to which recorded, primary hints are written.
func (*ShardSpec) IsConsistent ¶
IsConsistent is whether the shard assignment is consistent. allocator.ItemValue implementation.
func (*ShardSpec) MarshalString ¶
MarshalString returns the marshaled encoding of the ShardSpec as a string.
func (*ShardSpec) ProtoMessage ¶
func (*ShardSpec) ProtoMessage()
func (*ShardSpec) RecoveryLog ¶
RecoveryLog returns the Journal to which the Shard's recoverylog is recorded.
func (*ShardSpec) XXX_DiscardUnknown ¶
func (m *ShardSpec) XXX_DiscardUnknown()
func (*ShardSpec) XXX_Marshal ¶
func (*ShardSpec) XXX_Unmarshal ¶
type ShardSpec_Source ¶
type ShardSpec_Source struct { // Journal which this Shard is consuming. Journal github_com_LiveRamp_gazette_v2_pkg_protocol.Journal `` /* 128-byte string literal not displayed */ // Minimum journal byte offset the shard should begin reading from. Typically // this should be zero, as read offsets are persisted to and recovered from // the shard store as the journal is processed. |min_offset| can be useful // for shard initialization, directing it to skip over undesired historical // sections of the journal. MinOffset int64 `protobuf:"varint,3,opt,name=min_offset,json=minOffset,proto3" json:"min_offset,omitempty" yaml:"min_offset,omitempty"` }
Sources define the set of Journals which this Shard consumes. At least one Source must be specified, and in many use cases only one will be needed. For advanced use cases which can benefit, multiple sources may be specified to represent a "join" over messages of distinct journals. Note the effective mapping of messages to each of the joined journals should align (eg, joining a journal of customer updates with one of orders, where both are mapped on customer ID). Another powerful pattern is to join each partition of a high-volume event stream with a low-volume journal of queries, obtaining a reliable distributed scatter/gather query engine.
func (*ShardSpec_Source) Descriptor ¶
func (*ShardSpec_Source) Descriptor() ([]byte, []int)
func (*ShardSpec_Source) Marshal ¶
func (m *ShardSpec_Source) Marshal() (dAtA []byte, err error)
func (*ShardSpec_Source) ProtoMessage ¶
func (*ShardSpec_Source) ProtoMessage()
func (*ShardSpec_Source) ProtoSize ¶
func (m *ShardSpec_Source) ProtoSize() (n int)
func (*ShardSpec_Source) Reset ¶
func (m *ShardSpec_Source) Reset()
func (*ShardSpec_Source) String ¶
func (m *ShardSpec_Source) String() string
func (*ShardSpec_Source) Unmarshal ¶
func (m *ShardSpec_Source) Unmarshal(dAtA []byte) error
func (*ShardSpec_Source) Validate ¶
func (m *ShardSpec_Source) Validate() error
Validate returns an error if the ShardSpec_Source is not well-formed.
func (*ShardSpec_Source) XXX_DiscardUnknown ¶
func (m *ShardSpec_Source) XXX_DiscardUnknown()
func (*ShardSpec_Source) XXX_Marshal ¶
func (m *ShardSpec_Source) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
func (*ShardSpec_Source) XXX_Merge ¶
func (dst *ShardSpec_Source) XXX_Merge(src proto.Message)
func (*ShardSpec_Source) XXX_Size ¶
func (m *ShardSpec_Source) XXX_Size() int
func (*ShardSpec_Source) XXX_Unmarshal ¶
func (m *ShardSpec_Source) XXX_Unmarshal(b []byte) error
type StatRequest ¶
type StatRequest struct { // Header may be attached by a proxying consumer peer. Header *protocol.Header `protobuf:"bytes,1,opt,name=header" json:"header,omitempty"` // Shard to Stat. Shard ShardID `protobuf:"bytes,2,opt,name=shard,proto3,casttype=ShardID" json:"shard,omitempty"` }
func (*StatRequest) Descriptor ¶
func (*StatRequest) Descriptor() ([]byte, []int)
func (*StatRequest) Marshal ¶
func (m *StatRequest) Marshal() (dAtA []byte, err error)
func (*StatRequest) ProtoMessage ¶
func (*StatRequest) ProtoMessage()
func (*StatRequest) ProtoSize ¶
func (m *StatRequest) ProtoSize() (n int)
func (*StatRequest) Reset ¶
func (m *StatRequest) Reset()
func (*StatRequest) String ¶
func (m *StatRequest) String() string
func (*StatRequest) Unmarshal ¶
func (m *StatRequest) Unmarshal(dAtA []byte) error
func (*StatRequest) Validate ¶
func (m *StatRequest) Validate() error
Validate returns an error if the StatRequest is not well-formed.
func (*StatRequest) XXX_DiscardUnknown ¶
func (m *StatRequest) XXX_DiscardUnknown()
func (*StatRequest) XXX_Marshal ¶
func (m *StatRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
func (*StatRequest) XXX_Merge ¶
func (dst *StatRequest) XXX_Merge(src proto.Message)
func (*StatRequest) XXX_Size ¶
func (m *StatRequest) XXX_Size() int
func (*StatRequest) XXX_Unmarshal ¶
func (m *StatRequest) XXX_Unmarshal(b []byte) error
type StatResponse ¶
type StatResponse struct { // Status of the Stat RPC. Status Status `protobuf:"varint,1,opt,name=status,proto3,enum=consumer.Status" json:"status,omitempty"` // Header of the response. Header protocol.Header `protobuf:"bytes,2,opt,name=header" json:"header"` // Offsets of journals being read by the shard. Offsets map[github_com_LiveRamp_gazette_v2_pkg_protocol.Journal]int64 `` /* 209-byte string literal not displayed */ }
func StatShard ¶
func StatShard(ctx context.Context, rc RoutedShardClient, req *StatRequest) (*StatResponse, error)
StatShard invokes the Stat RPC, and maps a validation or !OK status to an error.
func (*StatResponse) Descriptor ¶
func (*StatResponse) Descriptor() ([]byte, []int)
func (*StatResponse) Marshal ¶
func (m *StatResponse) Marshal() (dAtA []byte, err error)
func (*StatResponse) ProtoMessage ¶
func (*StatResponse) ProtoMessage()
func (*StatResponse) ProtoSize ¶
func (m *StatResponse) ProtoSize() (n int)
func (*StatResponse) Reset ¶
func (m *StatResponse) Reset()
func (*StatResponse) String ¶
func (m *StatResponse) String() string
func (*StatResponse) Unmarshal ¶
func (m *StatResponse) Unmarshal(dAtA []byte) error
func (*StatResponse) Validate ¶
func (m *StatResponse) Validate() error
Validate returns an error if the StatResponse is not well-formed.
func (*StatResponse) XXX_DiscardUnknown ¶
func (m *StatResponse) XXX_DiscardUnknown()
func (*StatResponse) XXX_Marshal ¶
func (m *StatResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error)
func (*StatResponse) XXX_Merge ¶
func (dst *StatResponse) XXX_Merge(src proto.Message)
func (*StatResponse) XXX_Size ¶
func (m *StatResponse) XXX_Size() int
func (*StatResponse) XXX_Unmarshal ¶
func (m *StatResponse) XXX_Unmarshal(b []byte) error
type Status ¶
type Status int32
Status is a response status code, used across Gazette Consumer RPC APIs.
const ( Status_OK Status = 0 // The named shard does not exist. Status_SHARD_NOT_FOUND Status = 1 // There is no current primary consumer process for the shard. This is a // temporary condition which should quickly resolve, assuming sufficient // consumer capacity. Status_NO_SHARD_PRIMARY Status = 2 // The present consumer process is not the assigned primary for the shard, // and was not instructed to proxy the request. Status_NOT_SHARD_PRIMARY Status = 3 // The Etcd transaction failed. Returned by Update RPC when an // expect_mod_revision of the UpdateRequest differs from the current // ModRevision of the ShardSpec within the store. Status_ETCD_TRANSACTION_FAILED Status = 4 )
func (Status) EnumDescriptor ¶
type Store ¶
type Store interface { // Recorder which this Store wraps, into which all Store state is recorded. Recorder() *recoverylog.Recorder // FetchJournalOffsets returns the collection of Journals and corresponding // offsets represented within the Store. FetchJournalOffsets() (map[pb.Journal]int64, error) // Flush |offsets| and any in-memory Store state to disk. Flush(offsets map[pb.Journal]int64) error // Destroy releases all resources and removes all local files associated // with the Store. It is guaranteed that the Store is no longer being used or // referenced at the onset of this call. Destroy() }
Store is a stateful storage backend which is minimally able to record its file operations into a provided recoverylog.Recorder, and is able to represent a collection of Journal offsets. Application implementations control the selection, initialization, and usage of an appropriate Store backend for their use case.
Source Files ¶
Directories ¶
Path | Synopsis |
---|---|
Package shardspace provides mechanisms for mapping a collection of ShardSpecs into a minimally-described, semi hierarchical structure, and for mapping back again.
|
Package shardspace provides mechanisms for mapping a collection of ShardSpecs into a minimally-described, semi hierarchical structure, and for mapping back again. |