2020-01-15 14:40:47 +01:00
|
|
|
# Styx
|
|
|
|
## Install
|
|
|
|
|
|
|
|
```sh
|
|
|
|
go get -u gitlab.dcso.lolcat/LABS/styx
|
|
|
|
cd $GOPATH/src/gitlab.dcso.lolcat/LABS/styx
|
|
|
|
go build
|
|
|
|
./styx
|
|
|
|
```
|
2020-02-07 15:49:42 +01:00
|
|
|
|
2020-02-10 14:40:33 +01:00
|
|
|
### Example configuration:
|
|
|
|
```
|
|
|
|
// config.yml
|
2020-02-14 11:30:59 +01:00
|
|
|
certstream:
|
|
|
|
activated: true (Boolean)
|
|
|
|
|
|
|
|
pastebin:
|
|
|
|
activated: true (Boolean)
|
|
|
|
|
2020-02-10 14:40:33 +01:00
|
|
|
shodan:
|
2020-02-14 11:30:59 +01:00
|
|
|
activated: true (Boolean)
|
2020-02-10 16:11:25 +01:00
|
|
|
key: String (Required)
|
2020-02-12 16:54:14 +01:00
|
|
|
ports:
|
|
|
|
- 80
|
|
|
|
- 443
|
2020-02-10 14:40:33 +01:00
|
|
|
|
|
|
|
// do not forget to set up kafka and create the topic
|
|
|
|
kafka:
|
2020-02-14 11:30:59 +01:00
|
|
|
activated: true (Boolean)
|
2020-02-10 16:11:25 +01:00
|
|
|
protocol: "tcp" (String)
|
|
|
|
host: "localhost" (String)
|
|
|
|
port: 9092 (Int)
|
|
|
|
topic: "styx" (String)
|
|
|
|
partition: 0 (Int)
|
2020-02-10 14:40:33 +01:00
|
|
|
|
|
|
|
balboa:
|
2020-02-10 16:11:25 +01:00
|
|
|
url: String (Required)
|
2020-02-14 11:30:59 +01:00
|
|
|
activated: true (Boolean)
|
2020-02-10 14:40:33 +01:00
|
|
|
|
|
|
|
```
|
|
|
|
|
2020-02-07 15:49:42 +01:00
|
|
|
## Datastructure
|
|
|
|
|
|
|
|
### Meta
|
|
|
|
|
2020-02-10 10:36:36 +01:00
|
|
|
Node --[Edge]-- Node
|
2020-02-07 17:50:07 +01:00
|
|
|
|
2020-02-07 15:49:42 +01:00
|
|
|
```go
|
|
|
|
type Node struct {
|
|
|
|
ID string `json:"id"`
|
|
|
|
Type string `json:"type"`
|
|
|
|
Data string `json:"data"` // For plain Node, the data is the ID of another typed node or a unique value like a domain or a host name.
|
|
|
|
Created string `json:"created"`
|
|
|
|
Modified string `json:"modified"`
|
|
|
|
}
|
|
|
|
|
|
|
|
// Edge defines a relation between two nodes.
|
|
|
|
type Edge struct {
|
|
|
|
ID string `json:"id"`
|
|
|
|
NodeOneID string `json:"nodeOneID"`
|
|
|
|
NodeTwoID string `json:"nodeTwoID"`
|
|
|
|
Timestamp string `json:"timestamp"`
|
|
|
|
Source string `json:"source"`
|
|
|
|
}
|
|
|
|
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
|
|
### Certstream
|
|
|
|
|
2020-02-10 10:36:36 +01:00
|
|
|
Node --[Edge]-- CertNode --[Edge]-- CertStreamRaw
|
|
|
|
Node(domain) --[Edge]-- CertNode
|
2020-02-07 17:50:07 +01:00
|
|
|
|
2020-02-07 15:49:42 +01:00
|
|
|
```go
|
|
|
|
|
|
|
|
// CertStreamRaw is a wrapper around the stream function to unmarshall the
|
|
|
|
// data receive in a Go structure.
|
|
|
|
type CertStreamRaw struct {
|
|
|
|
ID string `json:"id"`
|
|
|
|
Type string `json:"type"`
|
|
|
|
Data CertStreamStruct `json:"data"`
|
|
|
|
Created string `json:"created"`
|
|
|
|
Modified string `json:"modified"`
|
|
|
|
}
|
|
|
|
|
|
|
|
// CertNode represents our custom struct of data extraction from CertStream.
|
|
|
|
type CertNode struct {
|
|
|
|
ID string `json:"id"`
|
|
|
|
Fingerprint string `json:"fingerprint"`
|
|
|
|
NotBefore string `json:"notBefore"`
|
|
|
|
NotAfter string `json:"notAfter"`
|
|
|
|
CN string `json:"cn"`
|
|
|
|
SourceName string `json:"sourceName"`
|
|
|
|
SerialNumber string `json:"serialNumber"`
|
|
|
|
BasicConstraints string `json:"basicConstraints"`
|
|
|
|
RawUUID string `json:"rawUUID"`
|
|
|
|
Chain []CertNode `json:"chainedTo"`
|
|
|
|
}
|
|
|
|
|
|
|
|
```
|
|
|
|
|
|
|
|
### Pastebin
|
|
|
|
|
2020-02-10 10:36:36 +01:00
|
|
|
Node --[Edge]-- PasteNode --[Edge]-- FullPaste
|
2020-02-07 17:50:07 +01:00
|
|
|
|
2020-02-07 15:49:42 +01:00
|
|
|
```go
|
|
|
|
// PasteNode is a node from PasteBin.
|
|
|
|
type PasteNode struct {
|
|
|
|
ID string `json:"id"`
|
|
|
|
Type string `json:"type"`
|
|
|
|
Data FullPaste `json:"data"`
|
|
|
|
Created string `json:"create"`
|
|
|
|
Modified string `json:"modified"`
|
|
|
|
}
|
|
|
|
|
|
|
|
// FullPaste wrapes meta and information from Pastebin.
|
|
|
|
type FullPaste struct {
|
|
|
|
Meta PasteMeta `json:"meta"`
|
|
|
|
Full string `json:"full"`
|
|
|
|
}
|
|
|
|
```
|
|
|
|
|
|
|
|
### Shodan
|
2020-02-07 17:50:07 +01:00
|
|
|
|
2020-02-10 10:36:36 +01:00
|
|
|
Node --[Edge]-- ShodanNode --[Edge]-- Node(s) (hostnames and domains)
|
2020-02-07 17:50:07 +01:00
|
|
|
|
2020-02-07 15:49:42 +01:00
|
|
|
```go
|
|
|
|
type ShodanNode struct {
|
|
|
|
ID string `json:"id"`
|
|
|
|
Type string `json:"type"`
|
|
|
|
Data *shodan.HostData `json:"data"`
|
|
|
|
Created string `json:"created"`
|
|
|
|
Modified string `json:"modified"`
|
|
|
|
}
|
|
|
|
```
|
2020-02-07 17:39:33 +01:00
|
|
|
|
2020-02-07 17:45:37 +01:00
|
|
|
### Balboa
|
|
|
|
|
|
|
|
Balboa enrichment happens on domains and hostnames extracted from Certstream
|
|
|
|
and Shodan streams and the node is created only if Balboa returns data.
|
|
|
|
|
2020-02-10 10:36:36 +01:00
|
|
|
Node --[Edge]-- ShodanNode --[Edge]-- Node (domain) --[Edge]-- BalboaNode
|
2020-02-07 17:45:37 +01:00
|
|
|
|
|
|
|
```go
|
|
|
|
type BalboaNode struct {
|
|
|
|
ID string `json:"id"`
|
|
|
|
Type string `json:"type"`
|
|
|
|
Data []balboa.Entries `json:"data"`
|
|
|
|
Created string `json:"created"`
|
|
|
|
Modified string `json:"modified"`
|
|
|
|
}
|
|
|
|
```
|