carves

package module
v0.0.0-...-78c4e6d Latest Latest
Warning

This package is not in the latest version of its module.

 Go to latest Published: Dec 12, 2024 License: MIT Imports: 22 Imported by: 3

Documentation

Index

Constants

View Source 
const (
	// StatusQueried for queried carves that did not hit nodes yet
	StatusQueried string = "QUERIED"
	// StatusScheduled for initialized carves
	StatusScheduled string = "SCHEDULED"
	// StatusInProgress for carves that are on-going
	StatusInProgress string = "IN PROGRESS"
	// StatusCompleted for carves that finalized
	StatusCompleted string = "COMPLETED"
	// TarFileExtension to identify Tar files extension
	TarFileExtension string = ".tar"
	// ZstFileExtension to identify ZST compressed files
	ZstFileExtension string = ".zst"
)
View Source 
const (
	// MaxUploadRetries to define how many times retry to upload
	MaxUploadRetries = 3
	// MaxChunkSize to define max size for each part. AWS defines 5MB max per part
	MaxChunkSize = int64(5 * 1024 * 1024)
	// DownloadLinkExpiration in minutes to expire download links
	DownloadLinkExpiration = 5
)
View Source 
const (
	// S3proto to be used as s3 URL
	S3proto = "s3://"
	// S3URL to format the s3 URL
	S3URL = S3proto + "%s/%s"
	// S3Key to format the s3 key for a block
	S3Key = "%s:%s:%s:%d"
	// S3File to format the s3 key for a reconstructed file
	S3File = "%s:%s:%s:%s" + TarFileExtension
	// LocalFile to format the local file name
	LocalFile = "%s_%s_%s" + TarFileExtension
)

Variables

View Source 
var (
	// CompressionHeader to detect the usage of compressed carves (zstd header)
	// https://github.com/facebook/zstd
	CompressionHeader = []byte{0x28, 0xb5, 0x2f, 0xfd}
)

Functions

func CheckCompressionBlock 

func CheckCompressionBlock(block CarvedBlock) (bool, error)

Function to check if a block data is compressed using zstd https://github.com/facebook/zstd

func CheckCompressionRaw 

func CheckCompressionRaw(data []byte) bool

Function to check if data is compressed using zstd https://github.com/facebook/zstd

func GenCarveName 

func GenCarveName() string

Helper to generate a random carve name

func GenCarveQuery 

func GenCarveQuery(file string, glob bool) string

Helper to generate the carve query

func GenerateArchiveName 

func GenerateArchiveName(carve CarvedFile) string

Function to generate a local file for carve archives

func GenerateS3Archive 

func GenerateS3Archive(bucket, env, uuid, sessionid, path string) string

Function to generate a carve archived filename for s3

func GenerateS3Data 

func GenerateS3Data(bucket, env, uuid, sessionid string, blockid int) string

Function to generate a carve block filename for s3

func GenerateS3File 

func GenerateS3File(env, uuid, sessionid, path string) string

Function to generate the s3 file reconstructed from blocks

func GenerateS3Key 

func GenerateS3Key(env, uuid, sessionid string, blockid int) string

Function to generate the s3 key for a carve block

func LoadS3 

func LoadS3(file string) (types.S3Configuration, error)

LoadS3 - Function to load the S3 configuration from JSON file

func S3URLtoKey 

func S3URLtoKey(s3url, bucket string) string

Function to translate from a s3:// URL to just the key

Types

type CarveResult 

type CarveResult struct {
	Size int64
	File string
}

CarveResult holds metadata related to a carve

type CarvedBlock 

type CarvedBlock struct {
	gorm.Model
	RequestID     string `gorm:"index"`
	SessionID     string `gorm:"index"`
	Environment   string
	BlockID       int
	Data          string
	Size          int
	Carver        string
	EnvironmentID uint
}

CarvedBlock to store each block from a carve

type CarvedFile 

type CarvedFile struct {
	gorm.Model
	CarveID         string `gorm:"unique;index"`
	RequestID       string
	SessionID       string
	QueryName       string
	UUID            string `gorm:"index"`
	NodeID          uint
	Environment     string
	Path            string
	CarveSize       int
	BlockSize       int
	TotalBlocks     int
	CompletedBlocks int
	Status          string
	CompletedAt     time.Time
	Carver          string
	Archived        bool
	ArchivePath     string
	EnvironmentID   uint
}

CarvedFile to keep track of carved files from nodes

type CarverS3 

type CarverS3 struct {
	S3Config  types.S3Configuration
	AWSConfig aws.Config
	Client    *s3.Client
	Uploader  *manager.Uploader
	Enabled   bool
	Debug     bool
}

CarverS3 will be used to carve files using S3 as destination

func CreateCarverS3 

func CreateCarverS3(s3Config types.S3Configuration) (*CarverS3, error)

CreateCarverS3 to initialize the carver

func CreateCarverS3File 

func CreateCarverS3File(s3File string) (*CarverS3, error)

CreateCarverS3File to initialize the carver

func (*CarverS3) Archive 

func (carveS3 *CarverS3) Archive(carve CarvedFile, blocks []CarvedBlock) (*CarveResult, error)

Archive - Function to convert finalize a completed carve and create a file ready to download

func (*CarverS3) Concatenate 

func (carveS3 *CarverS3) Concatenate(key string, destKey string, part int, uploadid *string) (*string, error)

Concatenate - Function to concatenate a file that have been already uploaded in s3

func (*CarverS3) Download 

func (carveS3 *CarverS3) Download(carve CarvedFile) (io.WriterAt, error)

Download - Function to download an archived carve from s3 

func (carveS3 *CarverS3) GetDownloadLink(carve CarvedFile) (string, error)

GetDownloadLink - Function to generate a pre-signed link to download directly from s3

func (*CarverS3) Settings 

func (carveS3 *CarverS3) Settings(mgr *settings.Settings)

Settings - Function to prepare settings for the logger

func (*CarverS3) Upload 

func (carveS3 *CarverS3) Upload(block CarvedBlock, uuid, data string) error

Upload - Function that sends data from carves to S3

type CarverType 

type CarverType int

CarveType as abstraction of storage type for the carver

type Carves 

type Carves struct {
	DB     *gorm.DB
	S3     *CarverS3
	Carver string
}

Carves to handle file carves from nodes

func CreateFileCarves 

func CreateFileCarves(backend *gorm.DB, carverType string, s3 *CarverS3) *Carves

CreateFileCarves to initialize the carves struct and tables

func (*Carves) Archive 

func (c *Carves) Archive(sessionid, destPath string) (*CarveResult, error)

Archive to convert finalize a completed carve and create a file ready to download

func (*Carves) ArchiveCarve 

func (c *Carves) ArchiveCarve(sessionid, archive string) error

ArchiveCarve to mark one carve as archived and set the received file

func (*Carves) ArchiveLocal 

func (c *Carves) ArchiveLocal(destPath string, carve CarvedFile, blocks []CarvedBlock) (*CarveResult, error)

Archive to convert finalize a completed carve and create a file ready to download

func (*Carves) ChangeStatus 

func (c *Carves) ChangeStatus(status, sessionid string) error

ChangeStatus to change the status of a carve

func (*Carves) CheckCarve 

func (c *Carves) CheckCarve(sessionid, requestid string) bool

CheckCarve to verify a session belong to a carve

func (*Carves) CompleteBlock 

func (c *Carves) CompleteBlock(sessionid string) error

CompleteBlock to increase one block for a carve

func (*Carves) Completed 

func (c *Carves) Completed(sessionid string) bool

Completed to check if a carve is completed FIXME return error maybe?

func (*Carves) CreateBlock 

func (c *Carves) CreateBlock(block CarvedBlock, uuid, data string) error

CreateBlock to create a new block for a carve

func (*Carves) CreateCarve 

func (c *Carves) CreateCarve(carve CarvedFile) error

CreateCarve to create a new carved file for a node

func (*Carves) Delete 

func (c *Carves) Delete(carveid string) error

Delete to delete a carve by id

func (*Carves) DeleteBlocks 

func (c *Carves) DeleteBlocks(sessionid string) error

DeleteBlocks to delete all blocks by session id

func (*Carves) GetBlocks 

func (c *Carves) GetBlocks(sessionid string) ([]CarvedBlock, error)

GetBlocks to get a carve by session_id and ordered by block_id

func (*Carves) GetByCarve 

func (c *Carves) GetByCarve(carveid string) (CarvedFile, error)

GetByCarve to get a carve by carve id

func (*Carves) GetByEnv 

func (c *Carves) GetByEnv(env uint) ([]CarvedFile, error)

GetByEnv to get carves by environment

func (*Carves) GetByQuery 

func (c *Carves) GetByQuery(name string, env uint) ([]CarvedFile, error)

GetByQuery to get a carve by query name

func (*Carves) GetByRequest 

func (c *Carves) GetByRequest(requestid string) ([]CarvedFile, error)

GetByRequest to get a carve by request_id

func (*Carves) GetBySession 

func (c *Carves) GetBySession(sessionid string) (CarvedFile, error)

GetBySession to get a carve by session_id

func (*Carves) GetCheckCarve 

func (c *Carves) GetCheckCarve(sessionid, requestid string) (CarvedFile, error)

GetCarve to verify a session belong to a carve

func (*Carves) GetNodeCarves 

func (c *Carves) GetNodeCarves(uuid string) ([]CarvedFile, error)

GetNodeCarves to get all the carves for a given node

func (*Carves) InitCarve 

func (c *Carves) InitCarve(req types.CarveInitRequest, sessionid string) error

InitCarve to initialize an scheduled carve

func (*Carves) InitateBlock 

func (c *Carves) InitateBlock(env, uuid, requestid, sessionid, data string, blockid int, envid uint) CarvedBlock

InitateBlock to initiate a block based on the configured carver

type MappedCarves 

type MappedCarves map[string][]CarvedFile

MappedCarves to pass carves by query name / Request ID

type QueriedCarve 

type QueriedCarve struct {
	Name    string
	Path    string
	Status  string
	Creator string
}

QueriedCarve to be used to display the carves in a table

Source Files

View all Source files

Jump to

Keyboard shortcuts

? : This menu
/ : Search site
f or F : Jump to
y or Y : Canonical URL
go.dev uses cookies from Google to deliver and enhance the quality of its services and to analyze traffic. Learn more.