chris/styx
chris/styx
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Shodan in Dgraph, first part

Browse source 
Implementing first version for shodan node, missing yet some models, but
the overal approach works and can be queried in Ratel.
This commit is contained in:
Christopher Talib 2020-05-18 16:09:04 +02:00
parent cbdca52ab2
commit b1ca4b3c5f
7 changed files with 240 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

79
README.md
View file

@ -11,7 +11,7 @@ easier. Just so you know.
Styx uses a couple of other services to run:
* Kafka for messaging (not implemented yet in the docker, but currently not
necessary)
necessary)
* Dgraph for graph representation of results
* Docker-compose to launch everything
@ -45,35 +45,35 @@ same subnet. Check [this](https://serverfault.com/questions/916941/configuring-d
### Example configuration:
```
certstream:
activated: true
activated: true
pastebin:
activated: true
activated: true
shodan:
activated: true
key: "SHODAN_KEY"
ports:
- 80
- 443
activated: true
key: "SHODAN_KEY"
ports:
- 80
- 443
kafka:
activated: true
protocol: "tcp"
host: "localhost"
port: 9092
topic: "styx"
partition: 0
activated: true
protocol: "tcp"
host: "localhost"
port: 9092
topic: "styx"
partition: 0
balboa:
# the url you tunneled to Balboa
url: http://127.0.0.1:8030
activated: true
# the url you tunneled to Balboa
url: http://127.0.0.1:8030
activated: true
elasticsearch:
activated: true
url: http://localhost:9200
index: "pastebin"
activated: true
url: http://localhost:9200
index: "pastebin"
```
## Dgraph Interface
@ -83,13 +83,13 @@ There you would be able to run GraphQL+ queries, here to query a node.
```graphql
query {
Node(func: eq(id, "node--cde8decb-0a8b-4d19-bd77-c2decb6dab9c")) {
uid
ndata
modified
type
id
}
Node(func: eq(id, "node--cde8decb-0a8b-4d19-bd77-c2decb6dab9c")) {
uid
ndata
modified
type
id
}
}
```
@ -98,13 +98,13 @@ Or filter node by type, this example works for certstream nodes:
```graphql
query {
Node(func: eq(type, "certstream")) {
uid
created
modified
type
ndata
cert_node {
Node(func: eq(type, "certstream")) {
uid
created
modified
type
ndata
certNode {
uid
fingerprint
cn
@ -122,6 +122,17 @@ query {
notBefore
notAfter
}
shodanNode {
uid
hostData {
product
ip
version
hostnames
port
html
}
}
}
}
```

19
broker/main.go
View file

@ -79,18 +79,31 @@ func ReadEventFromKafka() {
if len(node.ID) != 0 {
// TODO: refactor this context
ctx := context.Background()
entries, err := c.GetAllEntries(ctx, node.Data, "", "", int32(1))
entries, err := c.GetAllEntries(ctx, node.NData, "", "", int32(1))
if err != nil {
logrus.Error("error from balboa", err)
}
if len(entries) != 0 {
balboaNode := models.BuildBalboaNode(entries)
models.SaveBalboaNode("bnodes.json", balboaNode)
edge := models.BuildEdge("balboa", node.ID, balboaNode.ID)
models.SaveEdge(edge)
// edge := models.BuildEdge("balboa", node.ID, balboaNode.ID)
// models.SaveEdge(edge)
}
}
}
}
}
// Helpers
// func SaveSingleValues(brokerConn *kafka.Conn, source string, datatype string, originNodeID string, values []string) {
// for _, value := range values {
// domainNode := models.BuildNode(source, datatype, value)
// models.SaveNode("nodes.json", domainNode)
// if domainNode.Type == "domain" || domainNode.Type == "hostname" {
// broker.SendEventToKafka(brokerConn, *domainNode)
// }
// edge := models.BuildEdge(source, originNodeID, domainNode.ID)
// models.SaveEdge(edge)
// }
// }

3
filters/main.go
View file

@ -18,7 +18,8 @@ var (
)
// RunIPFilters runs the battery of filters for an IP.
func RunIPFilters(ip net.IP) bool {
func RunIPFilters(InputIP string) bool {
ip := net.ParseIP(InputIP)
if ip.To4() != nil {
path := basepath + "/data/ipv4/"
sliceIPv4, err := ioutil.ReadDir(path)

65
graph/main.go
View file

@ -44,7 +44,8 @@ sourceName: string @index(term) .
timestamp: string .
created: string .
modified: string .
cert_node: uid .
certNode: uid .
shodanNode: uid .
type Node {
id: string
@ -52,7 +53,8 @@ type: string
ndata: string
created: string
modified: string
cert_node: CertNode
certNode: CertNode
shodanNode: ShodanNode
}
type Edge {
@ -95,37 +97,52 @@ modified: string
csdata: string
}
type PasteNode {
id: string
type: string
created: string
modified: string
ndata: uid
}
meta: uid .
full: string .
type FullPaste {
meta: PasteNode
full: string
}
hostData: uid .
type ShodanNode {
id: string
type: string
ndata: string
created: string
modified: string
hostData: uid
}
type BalboaNode {
id: string
type: string
ndata: string
created: string
modified: string
product: string .
hostnames: [string] .
version: string .
title: string .
ip: string .
os: string .
organization: string .
isp: string .
cpe: [string] .
asn: string .
port: int .
html: string .
banner: string .
transport: string .
domains: [string] .
timestamp: string .
type Hostdata {
product: string
hostnames: [string]
version: string
title: string
ip: string
os: string
organization: string
isp: string
cpe: [string]
asn: string
port: int
html: string
banner: string
transport: string
domains: [string]
timestamp: string
}
`})
if err != nil {
return err

2
main.go
View file

@ -68,7 +68,7 @@ func main() {
if ok := s.Initialize(); !ok {
logrus.Info("shodan plugin not activated")
} else {
p.Run(&wg)
s.Run(&wg, dgraphClient)
}
go func() {

83
models/main.go
View file

@ -25,13 +25,14 @@ Structure of this file:
// Styx terminology
// (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx)
type Node struct {
ID string `json:"id,omiempty"`
Type string `json:"type,omiempty"`
NData string `json:"ndata,omiempty"`
Created string `json:"created,omiempty"`
Modified string `json:"modified,omiempty"`
DType []string `json:"dgraph.type,omiempty"`
CertNode CertNode `json:"cert_node,omiempty"`
ID string `json:"id,omiempty"`
Type string `json:"type,omiempty"`
NData string `json:"ndata,omiempty"`
Created string `json:"created,omiempty"`
Modified string `json:"modified,omiempty"`
DType []string `json:"dgraph.type,omiempty"`
CertNode CertNode `json:"certNode,omiempty"`
ShodanNode ShodanNode `json:"shodanNode,omiempty"`
}
// BuildNode builds a node to send to MQ instance.
@ -151,7 +152,7 @@ type CertNode struct {
SerialNumber string `json:"serialNumber,omiempty"`
BasicConstraints string `json:"basicConstraints,omiempty"`
Raw CertStreamRaw `json:"raw,omiempty"`
Chain []CertNode `json:"chainedTo,omiempty"`
Chain []CertNode `json:"chain,omiempty"`
}
// WrapCertStreamData is a wrapper around CertStreamStruct.
@ -316,11 +317,39 @@ func SavePaste(filename string, data *PasteNode) {
// ShodanNode is node around the shodan.HostData struct.
type ShodanNode struct {
ID string `json:"id"`
Type string `json:"type"`
Data *shodan.HostData `json:"data"`
Created string `json:"created"`
Modified string `json:"modified"`
ID string `json:"id,omiempty"`
Type string `json:"type,omiempty"`
HostData ShodanHostData `json:"hostData,omiempty"`
Created string `json:"created,omiempty"`
Modified string `json:"modified,omiempty"`
}
// ShodanHostData is a copy of the structure in the go shodan library. It's a
// workaround to have more control on the data send.
type ShodanHostData struct {
Product string `json:"product,omiempty"`
Hostnames []string `json:"hostnames,omiempty"`
Version string `json:"version,omiempty"`
Title string `json:"title,omiempty"`
// SSL *HostSSL `json:"ssl"`
IP string `json:"ip_str,omiempty"`
OS string `json:"os,omiempty"`
Organization string `json:"org,omiempty"`
ISP string `json:"isp,omiempty"`
CPE []string `json:"cpe,omiempty"`
// Data string `json:"data,omiempty"`
ASN string `json:"asn,omiempty"`
Port int `json:"port,omiempty"`
HTML string `json:"html,omiempty"`
Banner string `json:"banner,omiempty"`
Link string `json:"link,omiempty"`
Transport string `json:"transport,omiempty"`
Domains []string `json:"domains,omiempty"`
Timestamp string `json:"timestamp,omiempty"`
DeviceType string `json:"devicetype,omiempty"`
// Location *HostLocation `json:"location"`
ShodanData map[string]interface{} `json:"_shodan,omiempty"`
Opts map[string]interface{} `json:"opts,omiempty"`
}
// BuildShodanNode builds a wrapper node around shodan.HostData.
@ -328,9 +357,31 @@ func BuildShodanNode(data *shodan.HostData) *ShodanNode {
t := time.Now()
rfc3339time := t.Format(time.RFC3339)
return &ShodanNode{
ID: "shodan--" + uuid.New().String(),
Type: "shodan_stream",
Data: data,
ID: "shodan--" + uuid.New().String(),
Type: "shodan_stream",
HostData: ShodanHostData{
Product: data.Product,
Hostnames: data.Hostnames,
Version: data.Version.String(),
Title: data.Title,
IP: data.IP.String(),
OS: data.OS,
Organization: data.Organization,
ISP: data.ISP,
CPE: data.CPE,
// Data: data.Data,
ASN: data.ASN,
Port: data.Port,
HTML: data.HTML,
Banner: data.Banner,
Link: data.Link,
Transport: data.Transport,
Domains: data.Domains,
Timestamp: data.Timestamp,
DeviceType: data.DeviceType,
ShodanData: data.ShodanData,
Opts: data.Opts,
},
Created: rfc3339time,
Modified: rfc3339time,
}

99
plugins/shodan.go
View file

@ -1,10 +1,12 @@
package plugins
import (
"fmt"
"context"
"encoding/json"
"sync"
"github.com/christalib/structs"
"github.com/dgraph-io/dgo/v2"
"github.com/dgraph-io/dgo/v2/protos/api"
"github.com/ns3777k/go-shodan/v4/shodan"
"github.com/sirupsen/logrus"
"github.com/spf13/viper"
@ -27,15 +29,16 @@ func (s *ShodanPlugin) Initialize() bool {
return false
}
logrus.Info("shodan plugin is activated")
s.ShodanChan = make(chan *shodan.HostData)
return true
}
// Run runs the Shodan plugin.
func (s *ShodanPlugin) Run(wg *sync.WaitGroup) {
func (s *ShodanPlugin) Run(wg *sync.WaitGroup, dgraphClient *dgo.Dgraph) {
if !s.Running {
s.StoppedChan = make(chan bool)
wg.Add(1)
go s.doRun()
go s.doRun(dgraphClient)
s.Running = true
}
}
@ -51,7 +54,13 @@ func (s *ShodanPlugin) Stop(wg *sync.WaitGroup) {
}
}
func (s *ShodanPlugin) doRun() {
func (s *ShodanPlugin) doRun(graphClient *dgo.Dgraph) {
client := shodan.NewEnvClient(nil)
err := client.GetBannersByPorts(context.Background(), viper.GetIntSlice("shodan.ports"), s.ShodanChan)
if err != nil {
logrus.Panic(err)
}
for {
select {
default:
@ -63,36 +72,64 @@ func (s *ShodanPlugin) doRun() {
shodanNode := models.BuildShodanNode(banner)
// first filter poc
if shodanNode.Data.HTML != "" {
if !filters.RunIPFilters(shodanNode.Data.IP) {
hostnames := shodanNode.Data.Hostnames
var hostNotInFilters, domainNotInFilters bool
if len(hostnames) != 0 {
for _, hostname := range hostnames {
hostNotInFilters = filters.RunDomainFilters(hostname)
if hostNotInFilters {
// saveSingleValues(conn, "shodan_stream", "hostname", shodanNode.ID, hostname)
}
// if shodanNode.Data.HTML != "" {
if !filters.RunIPFilters(shodanNode.HostData.IP) {
hostnames := shodanNode.HostData.Hostnames
var hostNotInFilters, domainNotInFilters bool
if len(hostnames) != 0 {
for _, hostname := range hostnames {
hostNotInFilters = filters.RunDomainFilters(hostname)
if hostNotInFilters {
logrus.Info("host", hostname, "not in filters")
// saveSingleValues(conn, "shodan_stream", "hostname", shodanNode.ID, hostname)
}
}
domains := shodanNode.Data.Domains
if len(domains) != 0 {
for _, domain := range domains {
domainNotInFilters = filters.RunDomainFilters(domain)
// saveSingleValues(conn, "shodan_stream", "domain", shodanNode.ID, domain)
}
}
if domainNotInFilters && hostNotInFilters {
models.SaveShodanNode("raw_shodan.json", shodanNode)
node := models.BuildNode("shodan", "shodan_stream", shodanNode.ID)
models.SaveNode("nodes.json", node)
edge := models.BuildEdge("shodan", structs.Map(shodanNode), structs.Map(node))
models.SaveEdge(edge)
}
} else {
fmt.Println("is akamai", shodanNode.Data.IP)
}
domains := shodanNode.HostData.Domains
if len(domains) != 0 {
for _, domain := range domains {
domainNotInFilters = filters.RunDomainFilters(domain)
logrus.Info("domain", domain, "not in filters")
// saveSingleValues(conn, "shodan_stream", "domain", shodanNode.ID, domain)
}
}
if domainNotInFilters && hostNotInFilters {
models.SaveShodanNode("raw_shodan.json", shodanNode)
mainNode := models.BuildNode("shodan", "shodan_stream", shodanNode.ID)
// models.SaveNode("nodes.json", mainNode)
// edge := models.BuildEdge("shodan", structs.Map(shodanNode), structs.Map(mainNode))
// models.SaveEdge(edge)
e := models.Node{
ID: mainNode.ID,
Type: mainNode.Type,
NData: mainNode.NData,
Created: mainNode.Created,
Modified: mainNode.Modified,
ShodanNode: *shodanNode,
}
ctx := context.Background()
mu := &api.Mutation{
CommitNow: true,
}
pb, err := json.Marshal(e)
if err != nil {
logrus.Fatal(err)
}
mu.SetJson = pb
_, err = graphClient.NewTxn().Mutate(ctx, mu)
if err != nil {
logrus.Fatal(err)
}
}
} else {
logrus.Info(shodanNode.HostData.IP, "is akamain")
}
}
// }
}
}