chris/styx
chris/styx
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Shodan in Dgraph, first part

Browse source 
Implementing first version for shodan node, missing yet some models, but
the overal approach works and can be queried in Ratel.
This commit is contained in:
Christopher Talib 2020-05-18 16:09:04 +02:00
parent cbdca52ab2
commit b1ca4b3c5f
7 changed files with 240 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

79
README.md
View file

@ -11,7 +11,7 @@ easier. Just so you know.
Styx uses a couple of other services to run: Styx uses a couple of other services to run:
* Kafka for messaging (not implemented yet in the docker, but currently not * Kafka for messaging (not implemented yet in the docker, but currently not
necessary) necessary)
* Dgraph for graph representation of results * Dgraph for graph representation of results
* Docker-compose to launch everything * Docker-compose to launch everything
@ -45,35 +45,35 @@ same subnet. Check [this](https://serverfault.com/questions/916941/configuring-d
### Example configuration: ### Example configuration:
``` ```
certstream: certstream:
activated: true activated: true
pastebin: pastebin:
activated: true activated: true
shodan: shodan:
activated: true activated: true
key: "SHODAN_KEY" key: "SHODAN_KEY"
ports: ports:
- 80 - 80
- 443 - 443
kafka: kafka:
activated: true activated: true
protocol: "tcp" protocol: "tcp"
host: "localhost" host: "localhost"
port: 9092 port: 9092
topic: "styx" topic: "styx"
partition: 0 partition: 0
balboa: balboa:
# the url you tunneled to Balboa # the url you tunneled to Balboa
url: http://127.0.0.1:8030 url: http://127.0.0.1:8030
activated: true activated: true
elasticsearch: elasticsearch:
activated: true activated: true
url: http://localhost:9200 url: http://localhost:9200
index: "pastebin" index: "pastebin"
``` ```
## Dgraph Interface ## Dgraph Interface
@ -83,13 +83,13 @@ There you would be able to run GraphQL+ queries, here to query a node.
```graphql ```graphql
query { query {
Node(func: eq(id, "node--cde8decb-0a8b-4d19-bd77-c2decb6dab9c")) { Node(func: eq(id, "node--cde8decb-0a8b-4d19-bd77-c2decb6dab9c")) {
uid uid
ndata ndata
modified modified
type type
id id
} }
} }
``` ```
@ -98,13 +98,13 @@ Or filter node by type, this example works for certstream nodes:
```graphql ```graphql
query { query {
Node(func: eq(type, "certstream")) { Node(func: eq(type, "certstream")) {
uid uid
created created
modified modified
type type
ndata ndata
cert_node { certNode {
uid uid
fingerprint fingerprint
cn cn
@ -122,6 +122,17 @@ query {
notBefore notBefore
notAfter notAfter
} }
shodanNode {
uid
hostData {
product
ip
version
hostnames
port
html
}
}
} }
} }
``` ```

19
broker/main.go
View file

@ -79,18 +79,31 @@ func ReadEventFromKafka() {
if len(node.ID) != 0 { if len(node.ID) != 0 {
// TODO: refactor this context // TODO: refactor this context
ctx := context.Background() ctx := context.Background()
entries, err := c.GetAllEntries(ctx, node.Data, "", "", int32(1)) entries, err := c.GetAllEntries(ctx, node.NData, "", "", int32(1))
if err != nil { if err != nil {
logrus.Error("error from balboa", err) logrus.Error("error from balboa", err)
} }
if len(entries) != 0 { if len(entries) != 0 {
balboaNode := models.BuildBalboaNode(entries) balboaNode := models.BuildBalboaNode(entries)
models.SaveBalboaNode("bnodes.json", balboaNode) models.SaveBalboaNode("bnodes.json", balboaNode)
edge := models.BuildEdge("balboa", node.ID, balboaNode.ID) // edge := models.BuildEdge("balboa", node.ID, balboaNode.ID)
models.SaveEdge(edge) // models.SaveEdge(edge)
} }
} }
} }
} }
} }
// Helpers
// func SaveSingleValues(brokerConn *kafka.Conn, source string, datatype string, originNodeID string, values []string) {
// for _, value := range values {
// domainNode := models.BuildNode(source, datatype, value)
// models.SaveNode("nodes.json", domainNode)
// if domainNode.Type == "domain" || domainNode.Type == "hostname" {
// broker.SendEventToKafka(brokerConn, *domainNode)
// }
// edge := models.BuildEdge(source, originNodeID, domainNode.ID)
// models.SaveEdge(edge)
// }
// }

3
filters/main.go
View file

@ -18,7 +18,8 @@ var (
) )
// RunIPFilters runs the battery of filters for an IP. // RunIPFilters runs the battery of filters for an IP.
func RunIPFilters(ip net.IP) bool { func RunIPFilters(InputIP string) bool {
ip := net.ParseIP(InputIP)
if ip.To4() != nil { if ip.To4() != nil {
path := basepath + "/data/ipv4/" path := basepath + "/data/ipv4/"
sliceIPv4, err := ioutil.ReadDir(path) sliceIPv4, err := ioutil.ReadDir(path)

65
graph/main.go
View file

@ -44,7 +44,8 @@ sourceName: string @index(term) .
timestamp: string . timestamp: string .
created: string . created: string .
modified: string . modified: string .
cert_node: uid . certNode: uid .
shodanNode: uid .
type Node { type Node {
id: string id: string
@ -52,7 +53,8 @@ type: string
ndata: string ndata: string
created: string created: string
modified: string modified: string
cert_node: CertNode certNode: CertNode
shodanNode: ShodanNode
} }
type Edge { type Edge {
@ -95,37 +97,52 @@ modified: string
csdata: string csdata: string
} }
type PasteNode { hostData: uid .
id: string
type: string
created: string
modified: string
ndata: uid
}
meta: uid .
full: string .
type FullPaste {
meta: PasteNode
full: string
}
type ShodanNode { type ShodanNode {
id: string id: string
type: string type: string
ndata: string
created: string created: string
modified: string modified: string
hostData: uid
} }
type BalboaNode { product: string .
id: string hostnames: [string] .
type: string version: string .
ndata: string title: string .
created: string ip: string .
modified: string os: string .
organization: string .
isp: string .
cpe: [string] .
asn: string .
port: int .
html: string .
banner: string .
transport: string .
domains: [string] .
timestamp: string .
type Hostdata {
product: string
hostnames: [string]
version: string
title: string
ip: string
os: string
organization: string
isp: string
cpe: [string]
asn: string
port: int
html: string
banner: string
transport: string
domains: [string]
timestamp: string
} }
`}) `})
if err != nil { if err != nil {
return err return err

2
main.go
View file

@ -68,7 +68,7 @@ func main() {
if ok := s.Initialize(); !ok { if ok := s.Initialize(); !ok {
logrus.Info("shodan plugin not activated") logrus.Info("shodan plugin not activated")
} else { } else {
p.Run(&wg) s.Run(&wg, dgraphClient)
} }
go func() { go func() {

83
models/main.go
View file

@ -25,13 +25,14 @@ Structure of this file:
// Styx terminology // Styx terminology
// (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx) // (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx)
type Node struct { type Node struct {
ID string `json:"id,omiempty"` ID string `json:"id,omiempty"`
Type string `json:"type,omiempty"` Type string `json:"type,omiempty"`
NData string `json:"ndata,omiempty"` NData string `json:"ndata,omiempty"`
Created string `json:"created,omiempty"` Created string `json:"created,omiempty"`
Modified string `json:"modified,omiempty"` Modified string `json:"modified,omiempty"`
DType []string `json:"dgraph.type,omiempty"` DType []string `json:"dgraph.type,omiempty"`
CertNode CertNode `json:"cert_node,omiempty"` CertNode CertNode `json:"certNode,omiempty"`
ShodanNode ShodanNode `json:"shodanNode,omiempty"`
} }
// BuildNode builds a node to send to MQ instance. // BuildNode builds a node to send to MQ instance.
@ -151,7 +152,7 @@ type CertNode struct {
SerialNumber string `json:"serialNumber,omiempty"` SerialNumber string `json:"serialNumber,omiempty"`
BasicConstraints string `json:"basicConstraints,omiempty"` BasicConstraints string `json:"basicConstraints,omiempty"`
Raw CertStreamRaw `json:"raw,omiempty"` Raw CertStreamRaw `json:"raw,omiempty"`
Chain []CertNode `json:"chainedTo,omiempty"` Chain []CertNode `json:"chain,omiempty"`
} }
// WrapCertStreamData is a wrapper around CertStreamStruct. // WrapCertStreamData is a wrapper around CertStreamStruct.
@ -316,11 +317,39 @@ func SavePaste(filename string, data *PasteNode) {
// ShodanNode is node around the shodan.HostData struct. // ShodanNode is node around the shodan.HostData struct.
type ShodanNode struct { type ShodanNode struct {
ID string `json:"id"` ID string `json:"id,omiempty"`
Type string `json:"type"` Type string `json:"type,omiempty"`
Data *shodan.HostData `json:"data"` HostData ShodanHostData `json:"hostData,omiempty"`
Created string `json:"created"` Created string `json:"created,omiempty"`
Modified string `json:"modified"` Modified string `json:"modified,omiempty"`
}
// ShodanHostData is a copy of the structure in the go shodan library. It's a
// workaround to have more control on the data send.
type ShodanHostData struct {
Product string `json:"product,omiempty"`
Hostnames []string `json:"hostnames,omiempty"`
Version string `json:"version,omiempty"`
Title string `json:"title,omiempty"`
// SSL *HostSSL `json:"ssl"`
IP string `json:"ip_str,omiempty"`
OS string `json:"os,omiempty"`
Organization string `json:"org,omiempty"`
ISP string `json:"isp,omiempty"`
CPE []string `json:"cpe,omiempty"`
// Data string `json:"data,omiempty"`
ASN string `json:"asn,omiempty"`
Port int `json:"port,omiempty"`
HTML string `json:"html,omiempty"`
Banner string `json:"banner,omiempty"`
Link string `json:"link,omiempty"`
Transport string `json:"transport,omiempty"`
Domains []string `json:"domains,omiempty"`
Timestamp string `json:"timestamp,omiempty"`
DeviceType string `json:"devicetype,omiempty"`
// Location *HostLocation `json:"location"`
ShodanData map[string]interface{} `json:"_shodan,omiempty"`
Opts map[string]interface{} `json:"opts,omiempty"`
} }
// BuildShodanNode builds a wrapper node around shodan.HostData. // BuildShodanNode builds a wrapper node around shodan.HostData.
@ -328,9 +357,31 @@ func BuildShodanNode(data *shodan.HostData) *ShodanNode {
t := time.Now() t := time.Now()
rfc3339time := t.Format(time.RFC3339) rfc3339time := t.Format(time.RFC3339)
return &ShodanNode{ return &ShodanNode{
ID: "shodan--" + uuid.New().String(), ID: "shodan--" + uuid.New().String(),
Type: "shodan_stream", Type: "shodan_stream",
Data: data, HostData: ShodanHostData{
Product: data.Product,
Hostnames: data.Hostnames,
Version: data.Version.String(),
Title: data.Title,
IP: data.IP.String(),
OS: data.OS,
Organization: data.Organization,
ISP: data.ISP,
CPE: data.CPE,
// Data: data.Data,
ASN: data.ASN,
Port: data.Port,
HTML: data.HTML,
Banner: data.Banner,
Link: data.Link,
Transport: data.Transport,
Domains: data.Domains,
Timestamp: data.Timestamp,
DeviceType: data.DeviceType,
ShodanData: data.ShodanData,
Opts: data.Opts,
},
Created: rfc3339time, Created: rfc3339time,
Modified: rfc3339time, Modified: rfc3339time,
} }

99
plugins/shodan.go
View file

@ -1,10 +1,12 @@
package plugins package plugins
import ( import (
"fmt" "context"
"encoding/json"
"sync" "sync"
"github.com/christalib/structs" "github.com/dgraph-io/dgo/v2"
"github.com/dgraph-io/dgo/v2/protos/api"
"github.com/ns3777k/go-shodan/v4/shodan" "github.com/ns3777k/go-shodan/v4/shodan"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/spf13/viper" "github.com/spf13/viper"
@ -27,15 +29,16 @@ func (s *ShodanPlugin) Initialize() bool {
return false return false
} }
logrus.Info("shodan plugin is activated") logrus.Info("shodan plugin is activated")
s.ShodanChan = make(chan *shodan.HostData)
return true return true
} }
// Run runs the Shodan plugin. // Run runs the Shodan plugin.
func (s *ShodanPlugin) Run(wg *sync.WaitGroup) { func (s *ShodanPlugin) Run(wg *sync.WaitGroup, dgraphClient *dgo.Dgraph) {
if !s.Running { if !s.Running {
s.StoppedChan = make(chan bool) s.StoppedChan = make(chan bool)
wg.Add(1) wg.Add(1)
go s.doRun() go s.doRun(dgraphClient)
s.Running = true s.Running = true
} }
} }
@ -51,7 +54,13 @@ func (s *ShodanPlugin) Stop(wg *sync.WaitGroup) {
} }
} }
func (s *ShodanPlugin) doRun() { func (s *ShodanPlugin) doRun(graphClient *dgo.Dgraph) {
client := shodan.NewEnvClient(nil)
err := client.GetBannersByPorts(context.Background(), viper.GetIntSlice("shodan.ports"), s.ShodanChan)
if err != nil {
logrus.Panic(err)
}
for { for {
select { select {
default: default:
@ -63,36 +72,64 @@ func (s *ShodanPlugin) doRun() {
shodanNode := models.BuildShodanNode(banner) shodanNode := models.BuildShodanNode(banner)
// first filter poc // first filter poc
if shodanNode.Data.HTML != "" { // if shodanNode.Data.HTML != "" {
if !filters.RunIPFilters(shodanNode.Data.IP) { if !filters.RunIPFilters(shodanNode.HostData.IP) {
hostnames := shodanNode.Data.Hostnames hostnames := shodanNode.HostData.Hostnames
var hostNotInFilters, domainNotInFilters bool var hostNotInFilters, domainNotInFilters bool
if len(hostnames) != 0 { if len(hostnames) != 0 {
for _, hostname := range hostnames { for _, hostname := range hostnames {
hostNotInFilters = filters.RunDomainFilters(hostname) hostNotInFilters = filters.RunDomainFilters(hostname)
if hostNotInFilters { if hostNotInFilters {
// saveSingleValues(conn, "shodan_stream", "hostname", shodanNode.ID, hostname) logrus.Info("host", hostname, "not in filters")
} // saveSingleValues(conn, "shodan_stream", "hostname", shodanNode.ID, hostname)
} }
} }
domains := shodanNode.Data.Domains
if len(domains) != 0 {
for _, domain := range domains {
domainNotInFilters = filters.RunDomainFilters(domain)
// saveSingleValues(conn, "shodan_stream", "domain", shodanNode.ID, domain)
}
}
if domainNotInFilters && hostNotInFilters {
models.SaveShodanNode("raw_shodan.json", shodanNode)
node := models.BuildNode("shodan", "shodan_stream", shodanNode.ID)
models.SaveNode("nodes.json", node)
edge := models.BuildEdge("shodan", structs.Map(shodanNode), structs.Map(node))
models.SaveEdge(edge)
}
} else {
fmt.Println("is akamai", shodanNode.Data.IP)
} }
domains := shodanNode.HostData.Domains
if len(domains) != 0 {
for _, domain := range domains {
domainNotInFilters = filters.RunDomainFilters(domain)
logrus.Info("domain", domain, "not in filters")
// saveSingleValues(conn, "shodan_stream", "domain", shodanNode.ID, domain)
}
}
if domainNotInFilters && hostNotInFilters {
models.SaveShodanNode("raw_shodan.json", shodanNode)
mainNode := models.BuildNode("shodan", "shodan_stream", shodanNode.ID)
// models.SaveNode("nodes.json", mainNode)
// edge := models.BuildEdge("shodan", structs.Map(shodanNode), structs.Map(mainNode))
// models.SaveEdge(edge)
e := models.Node{
ID: mainNode.ID,
Type: mainNode.Type,
NData: mainNode.NData,
Created: mainNode.Created,
Modified: mainNode.Modified,
ShodanNode: *shodanNode,
}
ctx := context.Background()
mu := &api.Mutation{
CommitNow: true,
}
pb, err := json.Marshal(e)
if err != nil {
logrus.Fatal(err)
}
mu.SetJson = pb
_, err = graphClient.NewTxn().Mutate(ctx, mu)
if err != nil {
logrus.Fatal(err)
}
}
} else {
logrus.Info(shodanNode.HostData.IP, "is akamain")
} }
} }
// }
} }
} }