silk

README

silk flows

https://tools.netsa.cert.org/silk/docs.html

Description

This package makes it easy to read common silk files without using C Go.

Go Doc

https://godoc.org/github.com/chrispassas/silk

What is silk

Source https://tools.netsa.cert.org/silk/faq.html#what-silk

SiLK is a suite of network traffic collection and analysis tools developed and maintained by the CERT Network Situational >Awareness Team (CERT NetSA) at Carnegie Mellon University to facilitate security analysis of large networks. The SiLK tool >suite supports the efficient collection, storage, and analysis of network flow data, enabling network security analysts to >rapidly query large historical traffic data sets.

Supported File Formats

Record Size	Record Version	Compression	Supported
88	RWIPV6ROUTING VERSION 1	None (0)	✅
88	RWIPV6ROUTING VERSION 1	Zlib (1)	✅
88	RWIPV6ROUTING VERSION 1	Lzo (2)	✅
88	RWIPV6ROUTING VERSION 1	Snappy (3)	✅
68	RWIPV6 VERSION 1	None (0)	✅
68	RWIPV6 VERSION 1	Zlib (1)	✅
68	RWIPV6 VERSION 1	Lzo (2)	✅
68	RWIPV6 VERSION 1	Snappy (3)	✅
56	RWIPV6 VERSION 2	None (0)	✅
56	RWIPV6 VERSION 2	Zlib (1)	✅
56	RWIPV6 VERSION 2	Lzo (2)	✅
56	RWIPV6 VERSION 2	Snappy (3)	✅
52	RWGENERIC VERSION 5	None (0)	✅
52	RWGENERIC VERSION 5	Zlib (1)	✅
52	RWGENERIC VERSION 5	Lzo (2)	✅
52	RWGENERIC VERSION 5	Snappy (3)	✅

Example

Parse Whole File

package main

import (
    "fmt"
    "log"
    "github.com/chrispassas/silk"
)

func main() {

    var testFile = "testdata/FT_RWIPV6-v2-c0-L.dat"
    var err error
    var sf silk.File

    if sf, err = silk.OpenFile(testFile); err != nil {
        log.Fatalf("OpenFile() error:%s", err)
    }

    log.Printf("Compression:%d", sf.Header.Compression)
    log.Printf("FileFlags:%d", sf.Header.FileFlags)
    log.Printf("FileVersion:%d", sf.Header.FileVersion)
    log.Printf("HeaderLength:%d", sf.Header.HeaderLength)
    log.Printf("MagicNumber:%x", sf.Header.MagicNumber)
    log.Printf("RecordFormat:%d", sf.Header.RecordFormat)
    log.Printf("RecordSize:%d", sf.Header.RecordSize)
    log.Printf("RecordVersion:%d", sf.Header.RecordVersion)
    log.Printf("SilkVersion:%d", sf.Header.SilkVersion)

    log.Printf("File record count:%d\n", len(sf.Flows))

    fmt.Printf("start_time_ms,src_ip,dst_ip,src_port,dst_port\n")
    for _, flow := range sf.Flows {
        fmt.Printf("%d,%s,%s,%d,%d\n",
            flow.StartTimeMS,
            flow.SrcIP.String(),
            flow.DstIP.String(),
            flow.SrcPort,
            flow.DstPort,
        )
        //Etc... for other silk.Flow values
    }
}

Channel Based Parsing

package main

import (
    "fmt"
    "log"
    "os"
    "github.com/chrispassas/silk"
)

func main() {
    var testFile = "testdata/FT_RWIPV6-v2-c0-L.dat"
    var err error
    
    flows := make([]silk.Flow, 0, 245340)
    receiver := silk.NewChannelFlowReceiver(0)

    reader, err := os.Open(testFile)
    if err != nil {
        log.Fatal(err)
    }
    defer reader.Close()

    go func() {
        if err = silk.Parse(reader, receiver); err != nil {
            log.Fatal(err)
        }
    }()

    for flow := range receiver.Read() {
        /*
            Pulling all data into an in memory array. That really isn't the point of the channel based
            parser. You would want to stream it somewhere else to keep memory usage low. This is for example
            purposes only.
        */
        flows = append(flows, flow)
    }

    log.Printf("Compression:%d", receiver.Header.Compression)
    log.Printf("FileFlags:%d", receiver.Header.FileFlags)
    log.Printf("FileVersion:%d", receiver.Header.FileVersion)
    log.Printf("HeaderLength:%d", receiver.Header.HeaderLength)
    log.Printf("MagicNumber:%x", receiver.Header.MagicNumber)
    log.Printf("RecordFormat:%d", receiver.Header.RecordFormat)
    log.Printf("RecordSize:%d", receiver.Header.RecordSize)
    log.Printf("RecordVersion:%d", receiver.Header.RecordVersion)
    log.Printf("SilkVersion:%d", receiver.Header.SilkVersion)

    log.Printf("File record count:%d\n", len(flows))

    fmt.Printf("start_time_ms,src_ip,dst_ip,src_port,dst_port\n")
    for _, flow := range flows {
        fmt.Printf("%d,%s,%s,%d,%d\n",
            flow.StartTimeMS,
            flow.SrcIP.String(),
            flow.DstIP.String(),
            flow.SrcPort,
            flow.DstPort,
        )
        //Etc... for other silk.Flow values
    }
}

Documentation

Overview

Package silk is written without cgo to read common silk file formats.

What is silk?

https://tools.netsa.cert.org/silk/faq.html#what-silk

"SiLK is a suite of network traffic collection and analysis tools developed and
maintained by the CERT Network Situational Awareness Team (CERT NetSA) at
Carnegie Mellon University to facilitate security analysis of large networks.
The SiLK tool suite supports the efficient collection, storage, and analysis
of network flow data, enabling network security analysts to rapidly query
large historical traffic data sets."

Example:

import (
	"fmt"
	"log"
	"silk"
)

func main() {

	var testFile = "testdata/FT_RWIPV6-v2-c0-L.dat"
	var err error
	var sf silk.File

	if sf, err = silk.OpenFile(testFile); err != nil {
		log.Fatalf("OpenFile() error:%s", err)
	}

	log.Printf("Compression:%d", sf.Header.Compression)
	log.Printf("FileFlags:%d", sf.Header.FileFlags)
	log.Printf("FileVersion:%d", sf.Header.FileVersion)
	log.Printf("HeaderLength:%d", sf.Header.HeaderLength)
	log.Printf("MagicNumber:%x", sf.Header.MagicNumber)
	log.Printf("RecordFormat:%d", sf.Header.RecordFormat)
	log.Printf("RecordSize:%d", sf.Header.RecordSize)
	log.Printf("RecordVersion:%d", sf.Header.RecordVersion)
	log.Printf("SilkVersion:%d", sf.Header.SilkVersion)

	log.Printf("File record count:%d\n", len(sf.Flows))

	fmt.Printf("start_time_ms,src_ip,dst_ip,src_port,dst_port\n")
	for _, flow := range sf.Flows {
		fmt.Printf("%d,%s,%s,%d,%d\n",
			flow.StartTimeMS,
			flow.SrcIP.String(),
			flow.DstIP.String(),
			flow.SrcPort,
			flow.DstPort,
		)
		//Etc... for other silk.Flow values
	}
}

Index

• Constants
• Variables
• func Parse(reader io.Reader, receiver FlowReceiver) (err error)
• type ChannelFlowReceiver
  • func NewChannelFlowReceiver(channelBufferSize int) *ChannelFlowReceiver
  • func (c *ChannelFlowReceiver) Close()
  • func (c *ChannelFlowReceiver) HandleFlow(f Flow)
  • func (c *ChannelFlowReceiver) HandleHeader(h Header)
  • func (c ChannelFlowReceiver) Read() <-chan Flow
• type File
  • func OpenFile(filePath string) (sf File, err error)
  • func ParseReader(r io.Reader, receiver FlowReceiver) (sf File, err error)
• type Flow
• type FlowReceiver
• type Header
• type SliceFlowReceiver
  • func NewSliceFlowReceiver(initialSize int) *SliceFlowReceiver
  • func (a *SliceFlowReceiver) Close()
  • func (a *SliceFlowReceiver) HandleFlow(f Flow)
  • func (a *SliceFlowReceiver) HandleHeader(h Header)
• type VarLenHeader

Constants

const SilkTCPStateExpanded uint8 = 0x01

SilkTCPStateExpanded constant value defined in silk code

Variables

var ErrUnsupportedCompression = fmt.Errorf("Unsupported compression")

ErrUnsupportedCompression unknown compression type. Currently supported

0 = no compression
1 = zlib
2 = lzo
3 = snappy

var ErrUnsupportedPartialRead = fmt.Errorf("Reader failed to read expected length")

Functions

func Parse

func Parse(reader io.Reader, receiver FlowReceiver) (err error)

Types

type ChannelFlowReceiver

type ChannelFlowReceiver struct {
	Header Header
	// contains filtered or unexported fields
}

func NewChannelFlowReceiver

func NewChannelFlowReceiver(channelBufferSize int) *ChannelFlowReceiver

func (*ChannelFlowReceiver) Close

func (c *ChannelFlowReceiver) Close()

func (*ChannelFlowReceiver) HandleFlow

func (c *ChannelFlowReceiver) HandleFlow(f Flow)

func (*ChannelFlowReceiver) HandleHeader

func (c *ChannelFlowReceiver) HandleHeader(h Header)

func (ChannelFlowReceiver) Read

func (c ChannelFlowReceiver) Read() <-chan Flow

type File

type File struct {
	Header Header
	Flows  []Flow
}

File contains header and silk slice of flow records

func OpenFile

func OpenFile(filePath string) (sf File, err error)

OpenFile opens and parses silk file returning silk File struct and Error

func ParseReader

func ParseReader(r io.Reader, receiver FlowReceiver) (sf File, err error)

type Flow

type Flow struct {
	StartTimeMS  uint64
	Duration     uint32
	SrcIP        net.IP
	DstIP        net.IP
	SrcPort      uint16
	DstPort      uint16
	Proto        uint8
	Flags        uint8
	Packets      uint32
	Bytes        uint32
	ClassType    uint8
	Sensor       uint16
	InitalFlags  uint8
	SessionFlags uint8
	Attributes   uint8
	Application  uint16
	SNMPIn       uint16
	SNMPOut      uint16
	NextHopIP    net.IP
	// contains filtered or unexported fields
}

Flow represents a silk flow row of data Depending on type of silk record not all fields are used More details on the Flow stuct fields can be found here:

https://tools.netsa.cert.org/silk/faq.html#file-formats

type FlowReceiver

type FlowReceiver interface {
	HandleHeader(h Header)
	HandleFlow(f Flow)
	Close()
}

type Header

type Header struct {
	MagicNumber   []byte
	FileFlags     uint8
	RecordFormat  uint8
	FileVersion   uint8
	Compression   uint8
	SilkVersion   uint32
	RecordSize    uint16
	RecordVersion uint16
	VarLenHeaders []VarLenHeader
	HeaderLength  int
	// contains filtered or unexported fields
}

Header is documented here:

https://tools.netsa.cert.org/silk/faq.html#file-header

type SliceFlowReceiver

type SliceFlowReceiver struct {
	File
}

func NewSliceFlowReceiver

func NewSliceFlowReceiver(initialSize int) *SliceFlowReceiver

func (*SliceFlowReceiver) Close

func (a *SliceFlowReceiver) Close()

func (*SliceFlowReceiver) HandleFlow

func (a *SliceFlowReceiver) HandleFlow(f Flow)

func (*SliceFlowReceiver) HandleHeader

func (a *SliceFlowReceiver) HandleHeader(h Header)

type VarLenHeader

type VarLenHeader struct {
	ID      uint32
	Length  uint32
	Content []byte
}

VarLenHeader is part of the silk header. They contain different things like the cli command used to create the file. For some file types the variable length header also contains the year/month/day/hour of the file.