2020-01-15 14:40:47 +01:00
|
|
|
# Styx
|
2020-05-13 11:51:54 +02:00
|
|
|
|
|
|
|
## Prerequisites
|
|
|
|
|
|
|
|
Styx uses a couple of other services to run:
|
|
|
|
|
|
|
|
* Kafka for messaging (not implemented yet in the docker, but currently not
|
|
|
|
necessary)
|
|
|
|
* Dgraph for graph representation of results
|
|
|
|
|
|
|
|
For that purposes, there is a `docker-compose.yml` file that you can spin up
|
|
|
|
with the following command when in the directory:
|
|
|
|
|
|
|
|
```sh
|
|
|
|
docker-compose up -d
|
|
|
|
```
|
|
|
|
|
|
|
|
*Note*: for some reasons, OpenVPN blocks the establishment of the docker
|
|
|
|
compose, you can alternatively run Dgraph manually as such:
|
|
|
|
|
|
|
|
```sh
|
|
|
|
docker run --rm -it -p 8080:8080 -p 9080:9080 -p 8000:8000 -v ~/dgraph:/dgraph dgraph/standalone:v20.03.0
|
|
|
|
```
|
|
|
|
|
2020-01-15 14:40:47 +01:00
|
|
|
## Install
|
|
|
|
|
|
|
|
```sh
|
|
|
|
go get -u gitlab.dcso.lolcat/LABS/styx
|
|
|
|
cd $GOPATH/src/gitlab.dcso.lolcat/LABS/styx
|
|
|
|
go build
|
2020-05-13 11:51:54 +02:00
|
|
|
docker-compose up -d # or the other command if you're connected with OpenVPN
|
2020-01-15 14:40:47 +01:00
|
|
|
./styx
|
|
|
|
```
|
2020-02-07 15:49:42 +01:00
|
|
|
|
2020-02-10 14:40:33 +01:00
|
|
|
### Example configuration:
|
|
|
|
```
|
2020-02-14 11:30:59 +01:00
|
|
|
certstream:
|
2020-02-19 10:03:49 +01:00
|
|
|
activated: true
|
2020-02-14 11:30:59 +01:00
|
|
|
|
|
|
|
pastebin:
|
2020-02-19 10:03:49 +01:00
|
|
|
activated: true
|
2020-02-14 11:30:59 +01:00
|
|
|
|
2020-02-10 14:40:33 +01:00
|
|
|
shodan:
|
2020-02-19 10:03:49 +01:00
|
|
|
activated: true
|
|
|
|
key: "SHODAN_KEY"
|
2020-02-12 16:54:14 +01:00
|
|
|
ports:
|
|
|
|
- 80
|
|
|
|
- 443
|
2020-02-10 14:40:33 +01:00
|
|
|
|
|
|
|
kafka:
|
2020-02-19 10:03:49 +01:00
|
|
|
activated: true
|
|
|
|
protocol: "tcp"
|
|
|
|
host: "localhost"
|
|
|
|
port: 9092
|
|
|
|
topic: "styx"
|
|
|
|
partition: 0
|
2020-02-10 14:40:33 +01:00
|
|
|
|
|
|
|
balboa:
|
2020-02-19 10:03:49 +01:00
|
|
|
url: http://127.0.0.1:8030
|
|
|
|
activated: true
|
2020-02-10 14:40:33 +01:00
|
|
|
|
2020-02-17 12:10:27 +01:00
|
|
|
elasticsearch:
|
2020-02-19 10:03:49 +01:00
|
|
|
activated: true
|
|
|
|
url: http://localhost:9200
|
|
|
|
index: "pastebin"
|
2020-02-17 12:10:27 +01:00
|
|
|
|
2020-02-10 14:40:33 +01:00
|
|
|
```
|
|
|
|
|
2020-05-13 14:52:38 +02:00
|
|
|
## Dgraph Interface
|
|
|
|
|
|
|
|
You can connect to the Dgraph interface at this default address: localhost:8000.
|
|
|
|
There you would be able to run GraphQL+ queries, here to query a node.
|
|
|
|
|
|
|
|
```graphql
|
|
|
|
query {
|
|
|
|
Node(func: eq(id, "node--cde8decb-0a8b-4d19-bd77-c2decb6dab9c")) {
|
|
|
|
uid
|
|
|
|
ndata
|
|
|
|
modified
|
|
|
|
type
|
|
|
|
id
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
```
|
|
|
|
|
2020-02-07 15:49:42 +01:00
|
|
|
## Datastructure
|
|
|
|
|
|
|
|
### Meta
|
|
|
|
|
2020-02-10 10:36:36 +01:00
|
|
|
Node --[Edge]-- Node
|
2020-02-07 17:50:07 +01:00
|
|
|
|
2020-02-07 15:49:42 +01:00
|
|
|
```go
|
|
|
|
type Node struct {
|
|
|
|
ID string `json:"id"`
|
|
|
|
Type string `json:"type"`
|
|
|
|
Data string `json:"data"` // For plain Node, the data is the ID of another typed node or a unique value like a domain or a host name.
|
|
|
|
Created string `json:"created"`
|
|
|
|
Modified string `json:"modified"`
|
|
|
|
}
|
|
|
|
|
|
|
|
// Edge defines a relation between two nodes.
|
|
|
|
type Edge struct {
|
|
|
|
ID string `json:"id"`
|
|
|
|
NodeOneID string `json:"nodeOneID"`
|
|
|
|
NodeTwoID string `json:"nodeTwoID"`
|
|
|
|
Timestamp string `json:"timestamp"`
|
|
|
|
Source string `json:"source"`
|
|
|
|
}
|
|
|
|
|
|
|
|
```
|
|
|
|
|
|
|
|
### Certstream
|
|
|
|
|
2020-02-10 10:36:36 +01:00
|
|
|
Node --[Edge]-- CertNode --[Edge]-- CertStreamRaw
|
|
|
|
Node(domain) --[Edge]-- CertNode
|
2020-02-07 17:50:07 +01:00
|
|
|
|
2020-02-07 15:49:42 +01:00
|
|
|
```go
|
|
|
|
|
|
|
|
// CertStreamRaw is a wrapper around the stream function to unmarshall the
|
|
|
|
// data receive in a Go structure.
|
|
|
|
type CertStreamRaw struct {
|
|
|
|
ID string `json:"id"`
|
|
|
|
Type string `json:"type"`
|
|
|
|
Data CertStreamStruct `json:"data"`
|
|
|
|
Created string `json:"created"`
|
|
|
|
Modified string `json:"modified"`
|
|
|
|
}
|
|
|
|
|
|
|
|
// CertNode represents our custom struct of data extraction from CertStream.
|
|
|
|
type CertNode struct {
|
|
|
|
ID string `json:"id"`
|
|
|
|
Fingerprint string `json:"fingerprint"`
|
|
|
|
NotBefore string `json:"notBefore"`
|
|
|
|
NotAfter string `json:"notAfter"`
|
|
|
|
CN string `json:"cn"`
|
|
|
|
SourceName string `json:"sourceName"`
|
|
|
|
SerialNumber string `json:"serialNumber"`
|
|
|
|
BasicConstraints string `json:"basicConstraints"`
|
|
|
|
RawUUID string `json:"rawUUID"`
|
|
|
|
Chain []CertNode `json:"chainedTo"`
|
|
|
|
}
|
|
|
|
|
|
|
|
```
|
|
|
|
|
|
|
|
### Pastebin
|
|
|
|
|
2020-02-10 10:36:36 +01:00
|
|
|
Node --[Edge]-- PasteNode --[Edge]-- FullPaste
|
2020-02-07 17:50:07 +01:00
|
|
|
|
2020-02-07 15:49:42 +01:00
|
|
|
```go
|
|
|
|
// PasteNode is a node from PasteBin.
|
|
|
|
type PasteNode struct {
|
|
|
|
ID string `json:"id"`
|
|
|
|
Type string `json:"type"`
|
|
|
|
Data FullPaste `json:"data"`
|
|
|
|
Created string `json:"create"`
|
|
|
|
Modified string `json:"modified"`
|
|
|
|
}
|
|
|
|
|
|
|
|
// FullPaste wrapes meta and information from Pastebin.
|
|
|
|
type FullPaste struct {
|
|
|
|
Meta PasteMeta `json:"meta"`
|
|
|
|
Full string `json:"full"`
|
|
|
|
}
|
|
|
|
```
|
|
|
|
|
|
|
|
### Shodan
|
2020-02-07 17:50:07 +01:00
|
|
|
|
2020-02-10 10:36:36 +01:00
|
|
|
Node --[Edge]-- ShodanNode --[Edge]-- Node(s) (hostnames and domains)
|
2020-02-07 17:50:07 +01:00
|
|
|
|
2020-02-07 15:49:42 +01:00
|
|
|
```go
|
|
|
|
type ShodanNode struct {
|
|
|
|
ID string `json:"id"`
|
|
|
|
Type string `json:"type"`
|
|
|
|
Data *shodan.HostData `json:"data"`
|
|
|
|
Created string `json:"created"`
|
|
|
|
Modified string `json:"modified"`
|
|
|
|
}
|
|
|
|
```
|
2020-02-07 17:39:33 +01:00
|
|
|
|
2020-02-07 17:45:37 +01:00
|
|
|
### Balboa
|
|
|
|
|
|
|
|
Balboa enrichment happens on domains and hostnames extracted from Certstream
|
|
|
|
and Shodan streams and the node is created only if Balboa returns data.
|
|
|
|
|
2020-02-10 10:36:36 +01:00
|
|
|
Node --[Edge]-- ShodanNode --[Edge]-- Node (domain) --[Edge]-- BalboaNode
|
2020-02-07 17:45:37 +01:00
|
|
|
|
|
|
|
```go
|
|
|
|
type BalboaNode struct {
|
|
|
|
ID string `json:"id"`
|
|
|
|
Type string `json:"type"`
|
|
|
|
Data []balboa.Entries `json:"data"`
|
|
|
|
Created string `json:"created"`
|
|
|
|
Modified string `json:"modified"`
|
|
|
|
}
|
|
|
|
```
|