Styx

Install

go get -u gitlab.dcso.lolcat/LABS/styx
cd $GOPATH/src/gitlab.dcso.lolcat/LABS/styx
go build
./styx

Example configuration:

certstream:
  activated: true

pastebin:
  activated: true

shodan:
  activated: true
  key: "SHODAN_KEY"
  ports:
    - 80
    - 443

kafka:
  activated: true
  protocol: "tcp"
  host: "localhost"
  port: 9092
  topic: "styx"
  partition: 0

balboa:
  url: http://127.0.0.1:8030
  activated: true

elasticsearch:
  activated: true
  url: http://localhost:9200
  index: "pastebin"

Datastructure

Meta

Node --[Edge]-- Node

type Node struct {
	ID       string `json:"id"`
	Type     string `json:"type"`
	Data     string `json:"data"` // For plain Node, the data is the ID of another typed node or a unique value like a domain or a host name.
	Created  string `json:"created"`
	Modified string `json:"modified"`
}

// Edge defines a relation between two nodes.
type Edge struct {
	ID        string `json:"id"`
	NodeOneID string `json:"nodeOneID"`
	NodeTwoID string `json:"nodeTwoID"`
	Timestamp string `json:"timestamp"`
	Source    string `json:"source"`
}

Certstream

Node --[Edge]-- CertNode --[Edge]-- CertStreamRaw Node(domain) --[Edge]-- CertNode

// CertStreamRaw is a wrapper around the stream function to unmarshall the
// data receive in a Go structure.
type CertStreamRaw struct {
	ID       string           `json:"id"`
	Type     string           `json:"type"`
	Data     CertStreamStruct `json:"data"`
	Created  string           `json:"created"`
	Modified string           `json:"modified"`
}

// CertNode represents our custom struct of data extraction from CertStream.
type CertNode struct {
	ID               string     `json:"id"`
	Fingerprint      string     `json:"fingerprint"`
	NotBefore        string     `json:"notBefore"`
	NotAfter         string     `json:"notAfter"`
	CN               string     `json:"cn"`
	SourceName       string     `json:"sourceName"`
	SerialNumber     string     `json:"serialNumber"`
	BasicConstraints string     `json:"basicConstraints"`
	RawUUID          string     `json:"rawUUID"`
	Chain            []CertNode `json:"chainedTo"`
}

Pastebin

Node --[Edge]-- PasteNode --[Edge]-- FullPaste

// PasteNode is a node from PasteBin.
type PasteNode struct {
	ID       string    `json:"id"`
	Type     string    `json:"type"`
	Data     FullPaste `json:"data"`
	Created  string    `json:"create"`
	Modified string    `json:"modified"`
}

// FullPaste wrapes meta and information from Pastebin.
type FullPaste struct {
	Meta PasteMeta `json:"meta"`
	Full string    `json:"full"`
}

Shodan

Node --[Edge]-- ShodanNode --[Edge]-- Node(s) (hostnames and domains)

type ShodanNode struct {
	ID       string           `json:"id"`
	Type     string           `json:"type"`
	Data     *shodan.HostData `json:"data"`
	Created  string           `json:"created"`
	Modified string           `json:"modified"`
}

Balboa

Balboa enrichment happens on domains and hostnames extracted from Certstream and Shodan streams and the node is created only if Balboa returns data.

Node --[Edge]-- ShodanNode --[Edge]-- Node (domain) --[Edge]-- BalboaNode

type BalboaNode struct {
	ID       string           `json:"id"`
	Type     string           `json:"type"`
	Data     []balboa.Entries `json:"data"`
	Created  string           `json:"created"`
	Modified string           `json:"modified"`
}