s3s

package module
v0.3.3 Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Sep 16, 2022 License: MIT Imports: 11 Imported by: 0

README

S3S

S3S is a go binary instead of vast-engineering/s3select.

Feature

  • Input JSON to Output JSON

Usage

$ s3s --help
NAME:
   s3s - Easy S3 select like searching in directories

USAGE:
   s3s [global options] command [command options] [arguments...]

VERSION:
   v0.3.0

COMMANDS:
   help, h  Shows a list of commands or help for one command

GLOBAL OPTIONS:
   --alb-logs, --alb_logs          (default: false)
   --cf-logs, --cf_logs            (default: false)
   --count, -c                     max number of results from each key to return (default: false)
   --csv                           (default: false)
   --delve                         like directory move before querying (default: false)
   --help, -h                      show help (default: false)
   --limit value, -l value         max number of results from each key to return (default: 0)
   --max-retries value, -M value   max number of api requests to retry (default: 20)
   --query value, -q value         a query for S3 Select
   --region value                  region of target s3 bucket exist (default: ENV["AWS_REGION"])
   --thread-count value, -t value  max number of api requests to concurrently (default: 150)
   --version, -v                   print the version (default: false)
   --where value, -w value         WHERE part of the query

$ s3s s3://bucket/prefix
{"time":1654848930,"type":"speak"}
{"time":1654848969,"type":"sleep"}

// $ s3s s3://bucket/prefix_A s3://bucket/prefix_B s3://bucket/prefix_C

$ s3s -q 'SELECT * FROM S3Object s WHERE s.type = "speak"' s3://bucket/prefix
{"time":1654848930,"type":"speak"}

// alternate
// $ s3s -w 's.type = "speak"' s3://bucket/prefix
CSV support

--csv option is no header csv only

// 122, hello
$ s3s s3://bucket/prefix
{"_1":122,"_2":"hello"}

--alb-logs or --cf-logs option is tagging available instead of _1, _2, etc

-delve, like directory move before querying

search from prefix

$ s3s -delve s3://bucket/prefix

search from bucket list

$ s3s -delve

  bucket/prefix/C/
  bucket/prefix/B/
  bucket/prefix/A/        # delve more lower path than this prefix
  Query↵ (s3://bucket/prefix/) # choose and execute s3select this prefix
> ←Back upper path        # back to parent prefix
5/5
>

Querying after Enter.

{"time":1654848930,"type":"speak"}
{"time":1654848969,"type":"sleep"}

...

bucket/prefix/A/ (print path to stderr at end)

Documentation

Index

Constants

This section is empty.

Variables

This section is empty.

Functions

This section is empty.

Types

type ALBLogsSchema added in v0.3.0 

type ALBLogsSchema struct {
	Type                   interface{} `json:"type"`
	Time                   interface{} `json:"time"`
	Elb                    interface{} `json:"elb"`
	ClientPort             interface{} `json:"client:port"`
	TargetPort             interface{} `json:"target:port"`
	RequestProcessingTime  interface{} `json:"request_processing_time"`
	TargetProcessingTime   interface{} `json:"target_processing_time"`
	ResponseProcessingTime interface{} `json:"response_processing_time"`
	ElbStatusCode          interface{} `json:"elb_status_code"`
	TargetStatusCode       interface{} `json:"target_status_code"`
	ReceivedBytes          interface{} `json:"received_bytes"`
	SentBytes              interface{} `json:"sent_bytes"`
	Request                interface{} `json:"request"`
	UserAgent              interface{} `json:"user_agent"`
	SslCipher              interface{} `json:"ssl_cipher"`
	SslProtocol            interface{} `json:"ssl_protocol"`
	TargetGroupArn         interface{} `json:"target_group_arn"`
	TraceId                interface{} `json:"trace_id"`
	DomainName             interface{} `json:"domain_name"`
	ChosenCertArn          interface{} `json:"chosen_cert_arn"`
	MatchedRulePriority    interface{} `json:"matched_rule_priority"`
	RequestCreationTime    interface{} `json:"request_creation_time"`
	ActionsExecuted        interface{} `json:"actions_executed"`
	RedirectUrl            interface{} `json:"redirect_url"`
	ErrorReason            interface{} `json:"error_reason"`
	TargetPortList         interface{} `json:"target:port_list"`
	TargetStatusCodeList   interface{} `json:"target_status_code_list"`
	Classification         interface{} `json:"classification"`
	ClassificationReason   interface{} `json:"classification_reason"`
}

func (*ALBLogsSchema) UnmarshalJSON added in v0.3.0 

func (schema *ALBLogsSchema) UnmarshalJSON(b []byte) error

type App 

type App struct {
	// contains filtered or unexported fields
}

func NewApp 

func NewApp(ctx context.Context, region string, maxRetries int, threadCount int) (*App, error)

func (*App) GetS3Bucket added in v0.3.0 

func (app *App) GetS3Bucket(ctx context.Context) ([]string, error)

func (*App) GetS3Dir added in v0.3.0 

func (app *App) GetS3Dir(ctx context.Context, bucket string, prefix string) ([]string, error)

func (*App) GetS3Keys added in v0.3.0 

func (app *App) GetS3Keys(ctx context.Context, bucket string, prefix string) ([]string, error)

func (*App) GetS3KeysWithChannel added in v0.3.0 

func (app *App) GetS3KeysWithChannel(ctx context.Context, sender chan<- Path, bucket string, prefix string) error

func (*App) Run added in v0.3.0 

func (app *App) Run(ctx context.Context, paths []string, queryStr string, queryInfo *QueryInfo) error

func (*App) S3Select added in v0.3.0 

func (app *App) S3Select(ctx context.Context, input Querying, info *QueryInfo) (*Result, error)

type CFLogsSchema added in v0.3.0 

type CFLogsSchema struct {
	Date                    interface{} `json:"date"`
	Time                    interface{} `json:"time"`
	XEdgeLocation           interface{} `json:"x-edge-location"`
	ScBytes                 interface{} `json:"sc-bytes"`
	CIp                     interface{} `json:"c-ip"`
	CsMethod                interface{} `json:"cs-method"`
	CsHost                  interface{} `json:"cs(Host)"`
	CsUriStem               interface{} `json:"cs-uri-stem"`
	ScStatus                interface{} `json:"sc-status"`
	CsReferer               interface{} `json:"cs(Referer)"`
	CsUserAgent             interface{} `json:"cs(User-Agent)"`
	CsUriQuery              interface{} `json:"cs-uri-query"`
	CsCookie                interface{} `json:"cs(Cookie)"`
	XEdgeResultType         interface{} `json:"x-edge-result-type"`
	XEdgeRequestId          interface{} `json:"x-edge-request-id"`
	XHostHeader             interface{} `json:"x-host-header"`
	CsProtocol              interface{} `json:"cs-protocol"`
	CsBytes                 interface{} `json:"cs-bytes"`
	TimeTaken               interface{} `json:"time-taken"`
	XForwardedFor           interface{} `json:"x-forwarded-for"`
	SslProtocol             interface{} `json:"ssl-protocol"`
	SslCipher               interface{} `json:"ssl-cipher"`
	XEdgeResponseResultType interface{} `json:"x-edge-response-result-type"`
	CsProtocolVersion       interface{} `json:"cs-protocol-version"`
	FleStatus               interface{} `json:"fle-status"`
	FleEncryptedFields      interface{} `json:"fle-encrypted-fields"`
	CPort                   interface{} `json:"c-port"`
	TimeToFirstByte         interface{} `json:"time-to-first-byte"`
	XEdgeDetailedResultType interface{} `json:"x-edge-detailed-result-type"`
	ScContentType           interface{} `json:"sc-content-type"`
	ScContentLen            interface{} `json:"sc-content-len"`
	ScRangeStart            interface{} `json:"sc-range-start"`
	ScRangeEnd              interface{} `json:"sc-range-end"`
}

func (*CFLogsSchema) UnmarshalJSON added in v0.3.0 

func (schema *CFLogsSchema) UnmarshalJSON(b []byte) error

type CSVInput added in v0.3.0 

type CSVInput struct {
	Bucket          string
	Key             string
	Query           string
	FieldDelimiter  string
	RecordDelimiter string
}

type CountOnlySchema added in v0.3.1 

type CountOnlySchema struct {
	Count int `json:"_1"`
}

type FormatType added in v0.3.0 

type FormatType int
const (
	FormatTypeJSON FormatType = iota
	FormatTypeCSV
	FormatTypeALBLogs
	FormatTypeCFLogs
)

type JSONInput added in v0.3.0 

type JSONInput struct {
	Bucket string
	Key    string
	Query  string
}

type Path added in v0.3.0 

type Path struct {
	Bucket string
	Key    string
}

type QueryInfo added in v0.3.0 

type QueryInfo struct {
	FormatType      FormatType
	FieldDelimiter  string
	RecordDelimiter string
	IsCountMode     bool
}

type Querying added in v0.3.0 

type Querying interface {
	// contains filtered or unexported methods
}

type Result added in v0.3.1 

type Result struct {
	Count int
}

Source Files

View all Source files

Directories

Path Synopsis
cmd
s3s

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.