contentwriter

package
v1.0.0 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Aug 21, 2023 License: Apache-2.0 Imports: 10 Imported by: 10

Documentation

Index

Constants

This section is empty.

Variables

View Source 
var (
	RecordType_name = map[int32]string{
		0: "WARCINFO",
		1: "RESPONSE",
		2: "RESOURCE",
		3: "REQUEST",
		4: "METADATA",
		5: "REVISIT",
		6: "CONVERSION",
		7: "CONTINUATION",
	}
	RecordType_value = map[string]int32{
		"WARCINFO":     0,
		"RESPONSE":     1,
		"RESOURCE":     2,
		"REQUEST":      3,
		"METADATA":     4,
		"REVISIT":      5,
		"CONVERSION":   6,
		"CONTINUATION": 7,
	}
)

Enum value maps for RecordType.

View Source 
var ContentWriter_ServiceDesc = grpc.ServiceDesc{
	ServiceName: "veidemann.api.contentwriter.v1.ContentWriter",
	HandlerType: (*ContentWriterServer)(nil),
	Methods:     []grpc.MethodDesc{},
	Streams: []grpc.StreamDesc{
		{
			StreamName:    "write",
			Handler:       _ContentWriter_Write_Handler,
			ClientStreams: true,
		},
	},
	Metadata: "contentwriter/v1/contentwriter.proto",
}

ContentWriter_ServiceDesc is the grpc.ServiceDesc for ContentWriter service. It's only intended for direct use with grpc.RegisterService, and not to be introspected or modified (even as a copy)

View Source 
var File_contentwriter_v1_contentwriter_proto protoreflect.FileDescriptor
View Source 
var File_contentwriter_v1_resources_proto protoreflect.FileDescriptor

Functions

func RegisterContentWriterServer 

func RegisterContentWriterServer(s grpc.ServiceRegistrar, srv ContentWriterServer)

Types

type ContentWriterClient 

type ContentWriterClient interface {
	Write(ctx context.Context, opts ...grpc.CallOption) (ContentWriter_WriteClient, error)
}

ContentWriterClient is the client API for ContentWriter service.

For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream.

func NewContentWriterClient 

func NewContentWriterClient(cc grpc.ClientConnInterface) ContentWriterClient

type ContentWriterServer 

type ContentWriterServer interface {
	Write(ContentWriter_WriteServer) error
	// contains filtered or unexported methods
}

ContentWriterServer is the server API for ContentWriter service. All implementations must embed UnimplementedContentWriterServer for forward compatibility

type ContentWriter_WriteClient 

type ContentWriter_WriteClient interface {
	Send(*WriteRequest) error
	CloseAndRecv() (*WriteReply, error)
	grpc.ClientStream
}

type ContentWriter_WriteServer 

type ContentWriter_WriteServer interface {
	SendAndClose(*WriteReply) error
	Recv() (*WriteRequest, error)
	grpc.ServerStream
}

type CrawledContent 

type CrawledContent struct {
	Digest    string                 `protobuf:"bytes,1,opt,name=digest,proto3" json:"digest,omitempty"`
	WarcId    string                 `protobuf:"bytes,2,opt,name=warc_id,json=warcId,proto3" json:"warc_id,omitempty"`
	TargetUri string                 `protobuf:"bytes,3,opt,name=target_uri,json=targetUri,proto3" json:"target_uri,omitempty"`
	Date      *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=date,proto3" json:"date,omitempty"`
	// contains filtered or unexported fields
}

func (*CrawledContent) Descriptor deprecated 

func (*CrawledContent) Descriptor() ([]byte, []int)

Deprecated: Use CrawledContent.ProtoReflect.Descriptor instead.

func (*CrawledContent) GetDate 

func (x *CrawledContent) GetDate() *timestamppb.Timestamp

func (*CrawledContent) GetDigest 

func (x *CrawledContent) GetDigest() string

func (*CrawledContent) GetTargetUri 

func (x *CrawledContent) GetTargetUri() string

func (*CrawledContent) GetWarcId 

func (x *CrawledContent) GetWarcId() string

func (*CrawledContent) ProtoMessage 

func (*CrawledContent) ProtoMessage()

func (*CrawledContent) ProtoReflect 

func (x *CrawledContent) ProtoReflect() protoreflect.Message

func (*CrawledContent) Reset 

func (x *CrawledContent) Reset()

func (*CrawledContent) String 

func (x *CrawledContent) String() string

type Data 

type Data struct {
	RecordNum int32  `protobuf:"varint,1,opt,name=record_num,json=recordNum,proto3" json:"record_num,omitempty"`
	Data      []byte `protobuf:"bytes,2,opt,name=data,proto3" json:"data,omitempty"`
	// contains filtered or unexported fields
}

func (*Data) Descriptor deprecated 

func (*Data) Descriptor() ([]byte, []int)

Deprecated: Use Data.ProtoReflect.Descriptor instead.

func (*Data) GetData 

func (x *Data) GetData() []byte

func (*Data) GetRecordNum 

func (x *Data) GetRecordNum() int32

func (*Data) ProtoMessage 

func (*Data) ProtoMessage()

func (*Data) ProtoReflect 

func (x *Data) ProtoReflect() protoreflect.Message

func (*Data) Reset 

func (x *Data) Reset()

func (*Data) String 

func (x *Data) String() string

type RecordType 

type RecordType int32
const (
	RecordType_WARCINFO     RecordType = 0
	RecordType_RESPONSE     RecordType = 1
	RecordType_RESOURCE     RecordType = 2
	RecordType_REQUEST      RecordType = 3
	RecordType_METADATA     RecordType = 4
	RecordType_REVISIT      RecordType = 5
	RecordType_CONVERSION   RecordType = 6
	RecordType_CONTINUATION RecordType = 7
)

func (RecordType) Descriptor 

func (RecordType) Descriptor() protoreflect.EnumDescriptor

func (RecordType) Enum 

func (x RecordType) Enum() *RecordType

func (RecordType) EnumDescriptor deprecated 

func (RecordType) EnumDescriptor() ([]byte, []int)

Deprecated: Use RecordType.Descriptor instead.

func (RecordType) Number 

func (x RecordType) Number() protoreflect.EnumNumber

func (RecordType) String 

func (x RecordType) String() string

func (RecordType) Type 

func (RecordType) Type() protoreflect.EnumType

type StorageRef 

type StorageRef struct {
	WarcId     string     `protobuf:"bytes,1,opt,name=warc_id,json=warcId,proto3" json:"warc_id,omitempty"`
	RecordType RecordType `` /* 139-byte string literal not displayed */
	StorageRef string     `protobuf:"bytes,3,opt,name=storage_ref,json=storageRef,proto3" json:"storage_ref,omitempty"`
	// contains filtered or unexported fields
}

func (*StorageRef) Descriptor deprecated 

func (*StorageRef) Descriptor() ([]byte, []int)

Deprecated: Use StorageRef.ProtoReflect.Descriptor instead.

func (*StorageRef) GetRecordType 

func (x *StorageRef) GetRecordType() RecordType

func (*StorageRef) GetStorageRef 

func (x *StorageRef) GetStorageRef() string

func (*StorageRef) GetWarcId 

func (x *StorageRef) GetWarcId() string

func (*StorageRef) ProtoMessage 

func (*StorageRef) ProtoMessage()

func (*StorageRef) ProtoReflect 

func (x *StorageRef) ProtoReflect() protoreflect.Message

func (*StorageRef) Reset 

func (x *StorageRef) Reset()

func (*StorageRef) String 

func (x *StorageRef) String() string

type UnimplementedContentWriterServer 

type UnimplementedContentWriterServer struct {
}

UnimplementedContentWriterServer must be embedded to have forward compatible implementations.

func (UnimplementedContentWriterServer) Write 

func (UnimplementedContentWriterServer) Write(ContentWriter_WriteServer) error

type UnsafeContentWriterServer 

type UnsafeContentWriterServer interface {
	// contains filtered or unexported methods
}

UnsafeContentWriterServer may be embedded to opt out of forward compatibility for this service. Use of this interface is not recommended, as added methods to ContentWriterServer will result in compilation errors.

type WriteReply 

type WriteReply struct {
	Meta *WriteResponseMeta `protobuf:"bytes,1,opt,name=meta,proto3" json:"meta,omitempty"`
	// contains filtered or unexported fields
}

func (*WriteReply) Descriptor deprecated 

func (*WriteReply) Descriptor() ([]byte, []int)

Deprecated: Use WriteReply.ProtoReflect.Descriptor instead.

func (*WriteReply) GetMeta 

func (x *WriteReply) GetMeta() *WriteResponseMeta

func (*WriteReply) ProtoMessage 

func (*WriteReply) ProtoMessage()

func (*WriteReply) ProtoReflect 

func (x *WriteReply) ProtoReflect() protoreflect.Message

func (*WriteReply) Reset 

func (x *WriteReply) Reset()

func (*WriteReply) String 

func (x *WriteReply) String() string

type WriteRequest 

type WriteRequest struct {

	// Types that are assignable to Value:
	//
	//	*WriteRequest_Meta
	//	*WriteRequest_ProtocolHeader
	//	*WriteRequest_Payload
	//	*WriteRequest_Cancel
	Value isWriteRequest_Value `protobuf_oneof:"value"`
	// contains filtered or unexported fields
}

func (*WriteRequest) Descriptor deprecated 

func (*WriteRequest) Descriptor() ([]byte, []int)

Deprecated: Use WriteRequest.ProtoReflect.Descriptor instead.

func (*WriteRequest) GetCancel 

func (x *WriteRequest) GetCancel() string

func (*WriteRequest) GetMeta 

func (x *WriteRequest) GetMeta() *WriteRequestMeta

func (*WriteRequest) GetPayload 

func (x *WriteRequest) GetPayload() *Data

func (*WriteRequest) GetProtocolHeader 

func (x *WriteRequest) GetProtocolHeader() *Data

func (*WriteRequest) GetValue 

func (m *WriteRequest) GetValue() isWriteRequest_Value

func (*WriteRequest) ProtoMessage 

func (*WriteRequest) ProtoMessage()

func (*WriteRequest) ProtoReflect 

func (x *WriteRequest) ProtoReflect() protoreflect.Message

func (*WriteRequest) Reset 

func (x *WriteRequest) Reset()

func (*WriteRequest) String 

func (x *WriteRequest) String() string

type WriteRequestMeta 

type WriteRequestMeta struct {
	ExecutionId    string                                 `protobuf:"bytes,1,opt,name=execution_id,json=executionId,proto3" json:"execution_id,omitempty"`
	TargetUri      string                                 `protobuf:"bytes,2,opt,name=target_uri,json=targetUri,proto3" json:"target_uri,omitempty"`
	RecordMeta     map[int32]*WriteRequestMeta_RecordMeta `` /* 180-byte string literal not displayed */
	FetchTimeStamp *timestamppb.Timestamp                 `protobuf:"bytes,4,opt,name=fetch_time_stamp,json=fetchTimeStamp,proto3" json:"fetch_time_stamp,omitempty"`
	IpAddress      string                                 `protobuf:"bytes,5,opt,name=ip_address,json=ipAddress,proto3" json:"ip_address,omitempty"`
	CollectionRef  *v1.ConfigRef                          `protobuf:"bytes,7,opt,name=collection_ref,json=collectionRef,proto3" json:"collection_ref,omitempty"`
	// contains filtered or unexported fields
}

func (*WriteRequestMeta) Descriptor deprecated 

func (*WriteRequestMeta) Descriptor() ([]byte, []int)

Deprecated: Use WriteRequestMeta.ProtoReflect.Descriptor instead.

func (*WriteRequestMeta) GetCollectionRef 

func (x *WriteRequestMeta) GetCollectionRef() *v1.ConfigRef

func (*WriteRequestMeta) GetExecutionId 

func (x *WriteRequestMeta) GetExecutionId() string

func (*WriteRequestMeta) GetFetchTimeStamp 

func (x *WriteRequestMeta) GetFetchTimeStamp() *timestamppb.Timestamp

func (*WriteRequestMeta) GetIpAddress 

func (x *WriteRequestMeta) GetIpAddress() string

func (*WriteRequestMeta) GetRecordMeta 

func (x *WriteRequestMeta) GetRecordMeta() map[int32]*WriteRequestMeta_RecordMeta

func (*WriteRequestMeta) GetTargetUri 

func (x *WriteRequestMeta) GetTargetUri() string

func (*WriteRequestMeta) ProtoMessage 

func (*WriteRequestMeta) ProtoMessage()

func (*WriteRequestMeta) ProtoReflect 

func (x *WriteRequestMeta) ProtoReflect() protoreflect.Message

func (*WriteRequestMeta) Reset 

func (x *WriteRequestMeta) Reset()

func (*WriteRequestMeta) String 

func (x *WriteRequestMeta) String() string

type WriteRequestMeta_RecordMeta 

type WriteRequestMeta_RecordMeta struct {
	RecordNum         int32                           `protobuf:"varint,1,opt,name=record_num,json=recordNum,proto3" json:"record_num,omitempty"`
	Type              RecordType                      `protobuf:"varint,2,opt,name=type,proto3,enum=veidemann.api.contentwriter.v1.RecordType" json:"type,omitempty"`
	RecordContentType string                          `protobuf:"bytes,3,opt,name=record_content_type,json=recordContentType,proto3" json:"record_content_type,omitempty"`
	BlockDigest       string                          `protobuf:"bytes,4,opt,name=block_digest,json=blockDigest,proto3" json:"block_digest,omitempty"`
	PayloadDigest     string                          `protobuf:"bytes,5,opt,name=payload_digest,json=payloadDigest,proto3" json:"payload_digest,omitempty"`
	Size              int64                           `protobuf:"varint,6,opt,name=size,proto3" json:"size,omitempty"`
	SubCollection     v1.Collection_SubCollectionType `` /* 159-byte string literal not displayed */
	WarcConcurrentTo  []string                        `protobuf:"bytes,8,rep,name=warc_concurrent_to,json=warcConcurrentTo,proto3" json:"warc_concurrent_to,omitempty"`
	// contains filtered or unexported fields
}

func (*WriteRequestMeta_RecordMeta) Descriptor deprecated 

func (*WriteRequestMeta_RecordMeta) Descriptor() ([]byte, []int)

Deprecated: Use WriteRequestMeta_RecordMeta.ProtoReflect.Descriptor instead.

func (*WriteRequestMeta_RecordMeta) GetBlockDigest 

func (x *WriteRequestMeta_RecordMeta) GetBlockDigest() string

func (*WriteRequestMeta_RecordMeta) GetPayloadDigest 

func (x *WriteRequestMeta_RecordMeta) GetPayloadDigest() string

func (*WriteRequestMeta_RecordMeta) GetRecordContentType 

func (x *WriteRequestMeta_RecordMeta) GetRecordContentType() string

func (*WriteRequestMeta_RecordMeta) GetRecordNum 

func (x *WriteRequestMeta_RecordMeta) GetRecordNum() int32

func (*WriteRequestMeta_RecordMeta) GetSize 

func (x *WriteRequestMeta_RecordMeta) GetSize() int64

func (*WriteRequestMeta_RecordMeta) GetSubCollection 

func (x *WriteRequestMeta_RecordMeta) GetSubCollection() v1.Collection_SubCollectionType

func (*WriteRequestMeta_RecordMeta) GetType 

func (x *WriteRequestMeta_RecordMeta) GetType() RecordType

func (*WriteRequestMeta_RecordMeta) GetWarcConcurrentTo 

func (x *WriteRequestMeta_RecordMeta) GetWarcConcurrentTo() []string

func (*WriteRequestMeta_RecordMeta) ProtoMessage 

func (*WriteRequestMeta_RecordMeta) ProtoMessage()

func (*WriteRequestMeta_RecordMeta) ProtoReflect 

func (x *WriteRequestMeta_RecordMeta) ProtoReflect() protoreflect.Message

func (*WriteRequestMeta_RecordMeta) Reset 

func (x *WriteRequestMeta_RecordMeta) Reset()

func (*WriteRequestMeta_RecordMeta) String 

func (x *WriteRequestMeta_RecordMeta) String() string

type WriteRequest_Cancel 

type WriteRequest_Cancel struct {
	Cancel string `protobuf:"bytes,4,opt,name=cancel,proto3,oneof"` // Roll back the request. The message should contain the reason for canceling the request.
}

type WriteRequest_Meta 

type WriteRequest_Meta struct {
	Meta *WriteRequestMeta `protobuf:"bytes,1,opt,name=meta,proto3,oneof"`
}

type WriteRequest_Payload 

type WriteRequest_Payload struct {
	Payload *Data `protobuf:"bytes,3,opt,name=payload,proto3,oneof"` // Send part of the payload for the request. Repeat until complete payload is sent.
}

type WriteRequest_ProtocolHeader 

type WriteRequest_ProtocolHeader struct {
	ProtocolHeader *Data `protobuf:"bytes,2,opt,name=protocol_header,json=protocolHeader,proto3,oneof"` // Send protocol headers for the payload, e.g. http headers.
}

type WriteResponseMeta 

type WriteResponseMeta struct {
	RecordMeta map[int32]*WriteResponseMeta_RecordMeta `` /* 180-byte string literal not displayed */
	// contains filtered or unexported fields
}

func (*WriteResponseMeta) Descriptor deprecated 

func (*WriteResponseMeta) Descriptor() ([]byte, []int)

Deprecated: Use WriteResponseMeta.ProtoReflect.Descriptor instead.

func (*WriteResponseMeta) GetRecordMeta 

func (x *WriteResponseMeta) GetRecordMeta() map[int32]*WriteResponseMeta_RecordMeta

func (*WriteResponseMeta) ProtoMessage 

func (*WriteResponseMeta) ProtoMessage()

func (*WriteResponseMeta) ProtoReflect 

func (x *WriteResponseMeta) ProtoReflect() protoreflect.Message

func (*WriteResponseMeta) Reset 

func (x *WriteResponseMeta) Reset()

func (*WriteResponseMeta) String 

func (x *WriteResponseMeta) String() string

type WriteResponseMeta_RecordMeta 

type WriteResponseMeta_RecordMeta struct {
	RecordNum     int32      `protobuf:"varint,1,opt,name=record_num,json=recordNum,proto3" json:"record_num,omitempty"`
	Type          RecordType `protobuf:"varint,2,opt,name=type,proto3,enum=veidemann.api.contentwriter.v1.RecordType" json:"type,omitempty"`
	WarcId        string     `protobuf:"bytes,3,opt,name=warc_id,json=warcId,proto3" json:"warc_id,omitempty"`
	StorageRef    string     `protobuf:"bytes,4,opt,name=storage_ref,json=storageRef,proto3" json:"storage_ref,omitempty"`
	BlockDigest   string     `protobuf:"bytes,5,opt,name=block_digest,json=blockDigest,proto3" json:"block_digest,omitempty"`
	PayloadDigest string     `protobuf:"bytes,6,opt,name=payload_digest,json=payloadDigest,proto3" json:"payload_digest,omitempty"`
	// If the stored record was detected as a revisit, this field contains the id of the revisited record.
	RevisitReferenceId  string `protobuf:"bytes,7,opt,name=revisit_reference_id,json=revisitReferenceId,proto3" json:"revisit_reference_id,omitempty"`
	CollectionFinalName string `protobuf:"bytes,8,opt,name=collection_final_name,json=collectionFinalName,proto3" json:"collection_final_name,omitempty"`
	// contains filtered or unexported fields
}

func (*WriteResponseMeta_RecordMeta) Descriptor deprecated 

func (*WriteResponseMeta_RecordMeta) Descriptor() ([]byte, []int)

Deprecated: Use WriteResponseMeta_RecordMeta.ProtoReflect.Descriptor instead.

func (*WriteResponseMeta_RecordMeta) GetBlockDigest 

func (x *WriteResponseMeta_RecordMeta) GetBlockDigest() string

func (*WriteResponseMeta_RecordMeta) GetCollectionFinalName 

func (x *WriteResponseMeta_RecordMeta) GetCollectionFinalName() string

func (*WriteResponseMeta_RecordMeta) GetPayloadDigest 

func (x *WriteResponseMeta_RecordMeta) GetPayloadDigest() string

func (*WriteResponseMeta_RecordMeta) GetRecordNum 

func (x *WriteResponseMeta_RecordMeta) GetRecordNum() int32

func (*WriteResponseMeta_RecordMeta) GetRevisitReferenceId 

func (x *WriteResponseMeta_RecordMeta) GetRevisitReferenceId() string

func (*WriteResponseMeta_RecordMeta) GetStorageRef 

func (x *WriteResponseMeta_RecordMeta) GetStorageRef() string

func (*WriteResponseMeta_RecordMeta) GetType 

func (x *WriteResponseMeta_RecordMeta) GetType() RecordType

func (*WriteResponseMeta_RecordMeta) GetWarcId 

func (x *WriteResponseMeta_RecordMeta) GetWarcId() string

func (*WriteResponseMeta_RecordMeta) ProtoMessage 

func (*WriteResponseMeta_RecordMeta) ProtoMessage()

func (*WriteResponseMeta_RecordMeta) ProtoReflect 

func (x *WriteResponseMeta_RecordMeta) ProtoReflect() protoreflect.Message

func (*WriteResponseMeta_RecordMeta) Reset 

func (x *WriteResponseMeta_RecordMeta) Reset()

func (*WriteResponseMeta_RecordMeta) String 

func (x *WriteResponseMeta_RecordMeta) String() string

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.