From b1ca4b3c5fe8c042fe20d88159d4af06d2f8c7e0 Mon Sep 17 00:00:00 2001 From: Christopher Talib Date: Mon, 18 May 2020 16:09:04 +0200 Subject: [PATCH] Shodan in Dgraph, first part Implementing first version for shodan node, missing yet some models, but the overal approach works and can be queried in Ratel. --- README.md | 79 +++++++++++++++++++++---------------- broker/main.go | 19 +++++++-- filters/main.go | 3 +- graph/main.go | 65 +++++++++++++++++++------------ main.go | 2 +- models/main.go | 83 +++++++++++++++++++++++++++++++-------- plugins/shodan.go | 99 ++++++++++++++++++++++++++++++++--------------- 7 files changed, 240 insertions(+), 110 deletions(-) diff --git a/README.md b/README.md index abcb228..53ac2c5 100644 --- a/README.md +++ b/README.md @@ -11,7 +11,7 @@ easier. Just so you know. Styx uses a couple of other services to run: * Kafka for messaging (not implemented yet in the docker, but currently not - necessary) + necessary) * Dgraph for graph representation of results * Docker-compose to launch everything @@ -45,35 +45,35 @@ same subnet. Check [this](https://serverfault.com/questions/916941/configuring-d ### Example configuration: ``` certstream: - activated: true +activated: true pastebin: - activated: true +activated: true shodan: - activated: true - key: "SHODAN_KEY" - ports: - - 80 - - 443 +activated: true +key: "SHODAN_KEY" +ports: +- 80 +- 443 kafka: - activated: true - protocol: "tcp" - host: "localhost" - port: 9092 - topic: "styx" - partition: 0 +activated: true +protocol: "tcp" +host: "localhost" +port: 9092 +topic: "styx" +partition: 0 balboa: - # the url you tunneled to Balboa - url: http://127.0.0.1:8030 - activated: true +# the url you tunneled to Balboa +url: http://127.0.0.1:8030 +activated: true elasticsearch: - activated: true - url: http://localhost:9200 - index: "pastebin" +activated: true +url: http://localhost:9200 +index: "pastebin" ``` ## Dgraph Interface @@ -83,13 +83,13 @@ There you would be able to run GraphQL+ queries, here to query a node. ```graphql query { - Node(func: eq(id, "node--cde8decb-0a8b-4d19-bd77-c2decb6dab9c")) { - uid - ndata - modified - type - id - } + Node(func: eq(id, "node--cde8decb-0a8b-4d19-bd77-c2decb6dab9c")) { + uid + ndata + modified + type + id + } } ``` @@ -98,13 +98,13 @@ Or filter node by type, this example works for certstream nodes: ```graphql query { - Node(func: eq(type, "certstream")) { - uid - created - modified - type - ndata - cert_node { + Node(func: eq(type, "certstream")) { + uid + created + modified + type + ndata + certNode { uid fingerprint cn @@ -122,6 +122,17 @@ query { notBefore notAfter } + shodanNode { + uid + hostData { + product + ip + version + hostnames + port + html + } + } } } ``` diff --git a/broker/main.go b/broker/main.go index 2af6f70..606d48b 100644 --- a/broker/main.go +++ b/broker/main.go @@ -79,18 +79,31 @@ func ReadEventFromKafka() { if len(node.ID) != 0 { // TODO: refactor this context ctx := context.Background() - entries, err := c.GetAllEntries(ctx, node.Data, "", "", int32(1)) + entries, err := c.GetAllEntries(ctx, node.NData, "", "", int32(1)) if err != nil { logrus.Error("error from balboa", err) } if len(entries) != 0 { balboaNode := models.BuildBalboaNode(entries) models.SaveBalboaNode("bnodes.json", balboaNode) - edge := models.BuildEdge("balboa", node.ID, balboaNode.ID) - models.SaveEdge(edge) + // edge := models.BuildEdge("balboa", node.ID, balboaNode.ID) + // models.SaveEdge(edge) } } } } } + +// Helpers +// func SaveSingleValues(brokerConn *kafka.Conn, source string, datatype string, originNodeID string, values []string) { +// for _, value := range values { +// domainNode := models.BuildNode(source, datatype, value) +// models.SaveNode("nodes.json", domainNode) +// if domainNode.Type == "domain" || domainNode.Type == "hostname" { +// broker.SendEventToKafka(brokerConn, *domainNode) +// } +// edge := models.BuildEdge(source, originNodeID, domainNode.ID) +// models.SaveEdge(edge) +// } +// } diff --git a/filters/main.go b/filters/main.go index ae1adea..70cd05f 100644 --- a/filters/main.go +++ b/filters/main.go @@ -18,7 +18,8 @@ var ( ) // RunIPFilters runs the battery of filters for an IP. -func RunIPFilters(ip net.IP) bool { +func RunIPFilters(InputIP string) bool { + ip := net.ParseIP(InputIP) if ip.To4() != nil { path := basepath + "/data/ipv4/" sliceIPv4, err := ioutil.ReadDir(path) diff --git a/graph/main.go b/graph/main.go index 8f7ca82..fba0227 100644 --- a/graph/main.go +++ b/graph/main.go @@ -44,7 +44,8 @@ sourceName: string @index(term) . timestamp: string . created: string . modified: string . -cert_node: uid . +certNode: uid . +shodanNode: uid . type Node { id: string @@ -52,7 +53,8 @@ type: string ndata: string created: string modified: string -cert_node: CertNode +certNode: CertNode +shodanNode: ShodanNode } type Edge { @@ -95,37 +97,52 @@ modified: string csdata: string } -type PasteNode { -id: string -type: string -created: string -modified: string -ndata: uid -} - -meta: uid . -full: string . - -type FullPaste { -meta: PasteNode -full: string -} +hostData: uid . type ShodanNode { id: string type: string -ndata: string created: string modified: string +hostData: uid } -type BalboaNode { -id: string -type: string -ndata: string -created: string -modified: string +product: string . +hostnames: [string] . +version: string . +title: string . +ip: string . +os: string . +organization: string . +isp: string . +cpe: [string] . +asn: string . +port: int . +html: string . +banner: string . +transport: string . +domains: [string] . +timestamp: string . + +type Hostdata { +product: string +hostnames: [string] +version: string +title: string +ip: string +os: string +organization: string +isp: string +cpe: [string] +asn: string +port: int +html: string +banner: string +transport: string +domains: [string] +timestamp: string } + `}) if err != nil { return err diff --git a/main.go b/main.go index 370559f..c1476ae 100644 --- a/main.go +++ b/main.go @@ -68,7 +68,7 @@ func main() { if ok := s.Initialize(); !ok { logrus.Info("shodan plugin not activated") } else { - p.Run(&wg) + s.Run(&wg, dgraphClient) } go func() { diff --git a/models/main.go b/models/main.go index 650de83..0479e35 100644 --- a/models/main.go +++ b/models/main.go @@ -25,13 +25,14 @@ Structure of this file: // Styx terminology // (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx) type Node struct { - ID string `json:"id,omiempty"` - Type string `json:"type,omiempty"` - NData string `json:"ndata,omiempty"` - Created string `json:"created,omiempty"` - Modified string `json:"modified,omiempty"` - DType []string `json:"dgraph.type,omiempty"` - CertNode CertNode `json:"cert_node,omiempty"` + ID string `json:"id,omiempty"` + Type string `json:"type,omiempty"` + NData string `json:"ndata,omiempty"` + Created string `json:"created,omiempty"` + Modified string `json:"modified,omiempty"` + DType []string `json:"dgraph.type,omiempty"` + CertNode CertNode `json:"certNode,omiempty"` + ShodanNode ShodanNode `json:"shodanNode,omiempty"` } // BuildNode builds a node to send to MQ instance. @@ -151,7 +152,7 @@ type CertNode struct { SerialNumber string `json:"serialNumber,omiempty"` BasicConstraints string `json:"basicConstraints,omiempty"` Raw CertStreamRaw `json:"raw,omiempty"` - Chain []CertNode `json:"chainedTo,omiempty"` + Chain []CertNode `json:"chain,omiempty"` } // WrapCertStreamData is a wrapper around CertStreamStruct. @@ -316,11 +317,39 @@ func SavePaste(filename string, data *PasteNode) { // ShodanNode is node around the shodan.HostData struct. type ShodanNode struct { - ID string `json:"id"` - Type string `json:"type"` - Data *shodan.HostData `json:"data"` - Created string `json:"created"` - Modified string `json:"modified"` + ID string `json:"id,omiempty"` + Type string `json:"type,omiempty"` + HostData ShodanHostData `json:"hostData,omiempty"` + Created string `json:"created,omiempty"` + Modified string `json:"modified,omiempty"` +} + +// ShodanHostData is a copy of the structure in the go shodan library. It's a +// workaround to have more control on the data send. +type ShodanHostData struct { + Product string `json:"product,omiempty"` + Hostnames []string `json:"hostnames,omiempty"` + Version string `json:"version,omiempty"` + Title string `json:"title,omiempty"` + // SSL *HostSSL `json:"ssl"` + IP string `json:"ip_str,omiempty"` + OS string `json:"os,omiempty"` + Organization string `json:"org,omiempty"` + ISP string `json:"isp,omiempty"` + CPE []string `json:"cpe,omiempty"` + // Data string `json:"data,omiempty"` + ASN string `json:"asn,omiempty"` + Port int `json:"port,omiempty"` + HTML string `json:"html,omiempty"` + Banner string `json:"banner,omiempty"` + Link string `json:"link,omiempty"` + Transport string `json:"transport,omiempty"` + Domains []string `json:"domains,omiempty"` + Timestamp string `json:"timestamp,omiempty"` + DeviceType string `json:"devicetype,omiempty"` + // Location *HostLocation `json:"location"` + ShodanData map[string]interface{} `json:"_shodan,omiempty"` + Opts map[string]interface{} `json:"opts,omiempty"` } // BuildShodanNode builds a wrapper node around shodan.HostData. @@ -328,9 +357,31 @@ func BuildShodanNode(data *shodan.HostData) *ShodanNode { t := time.Now() rfc3339time := t.Format(time.RFC3339) return &ShodanNode{ - ID: "shodan--" + uuid.New().String(), - Type: "shodan_stream", - Data: data, + ID: "shodan--" + uuid.New().String(), + Type: "shodan_stream", + HostData: ShodanHostData{ + Product: data.Product, + Hostnames: data.Hostnames, + Version: data.Version.String(), + Title: data.Title, + IP: data.IP.String(), + OS: data.OS, + Organization: data.Organization, + ISP: data.ISP, + CPE: data.CPE, + // Data: data.Data, + ASN: data.ASN, + Port: data.Port, + HTML: data.HTML, + Banner: data.Banner, + Link: data.Link, + Transport: data.Transport, + Domains: data.Domains, + Timestamp: data.Timestamp, + DeviceType: data.DeviceType, + ShodanData: data.ShodanData, + Opts: data.Opts, + }, Created: rfc3339time, Modified: rfc3339time, } diff --git a/plugins/shodan.go b/plugins/shodan.go index de4e5cf..8a8cea5 100644 --- a/plugins/shodan.go +++ b/plugins/shodan.go @@ -1,10 +1,12 @@ package plugins import ( - "fmt" + "context" + "encoding/json" "sync" - "github.com/christalib/structs" + "github.com/dgraph-io/dgo/v2" + "github.com/dgraph-io/dgo/v2/protos/api" "github.com/ns3777k/go-shodan/v4/shodan" "github.com/sirupsen/logrus" "github.com/spf13/viper" @@ -27,15 +29,16 @@ func (s *ShodanPlugin) Initialize() bool { return false } logrus.Info("shodan plugin is activated") + s.ShodanChan = make(chan *shodan.HostData) return true } // Run runs the Shodan plugin. -func (s *ShodanPlugin) Run(wg *sync.WaitGroup) { +func (s *ShodanPlugin) Run(wg *sync.WaitGroup, dgraphClient *dgo.Dgraph) { if !s.Running { s.StoppedChan = make(chan bool) wg.Add(1) - go s.doRun() + go s.doRun(dgraphClient) s.Running = true } } @@ -51,7 +54,13 @@ func (s *ShodanPlugin) Stop(wg *sync.WaitGroup) { } } -func (s *ShodanPlugin) doRun() { +func (s *ShodanPlugin) doRun(graphClient *dgo.Dgraph) { + client := shodan.NewEnvClient(nil) + err := client.GetBannersByPorts(context.Background(), viper.GetIntSlice("shodan.ports"), s.ShodanChan) + if err != nil { + logrus.Panic(err) + } + for { select { default: @@ -63,36 +72,64 @@ func (s *ShodanPlugin) doRun() { shodanNode := models.BuildShodanNode(banner) // first filter poc - if shodanNode.Data.HTML != "" { - if !filters.RunIPFilters(shodanNode.Data.IP) { - hostnames := shodanNode.Data.Hostnames - var hostNotInFilters, domainNotInFilters bool - if len(hostnames) != 0 { - for _, hostname := range hostnames { - hostNotInFilters = filters.RunDomainFilters(hostname) - if hostNotInFilters { - // saveSingleValues(conn, "shodan_stream", "hostname", shodanNode.ID, hostname) - } + // if shodanNode.Data.HTML != "" { + if !filters.RunIPFilters(shodanNode.HostData.IP) { + hostnames := shodanNode.HostData.Hostnames + var hostNotInFilters, domainNotInFilters bool + if len(hostnames) != 0 { + for _, hostname := range hostnames { + hostNotInFilters = filters.RunDomainFilters(hostname) + if hostNotInFilters { + logrus.Info("host", hostname, "not in filters") + // saveSingleValues(conn, "shodan_stream", "hostname", shodanNode.ID, hostname) } } - domains := shodanNode.Data.Domains - if len(domains) != 0 { - for _, domain := range domains { - domainNotInFilters = filters.RunDomainFilters(domain) - // saveSingleValues(conn, "shodan_stream", "domain", shodanNode.ID, domain) - } - } - if domainNotInFilters && hostNotInFilters { - models.SaveShodanNode("raw_shodan.json", shodanNode) - node := models.BuildNode("shodan", "shodan_stream", shodanNode.ID) - models.SaveNode("nodes.json", node) - edge := models.BuildEdge("shodan", structs.Map(shodanNode), structs.Map(node)) - models.SaveEdge(edge) - } - } else { - fmt.Println("is akamai", shodanNode.Data.IP) } + domains := shodanNode.HostData.Domains + if len(domains) != 0 { + for _, domain := range domains { + domainNotInFilters = filters.RunDomainFilters(domain) + logrus.Info("domain", domain, "not in filters") + // saveSingleValues(conn, "shodan_stream", "domain", shodanNode.ID, domain) + } + } + if domainNotInFilters && hostNotInFilters { + models.SaveShodanNode("raw_shodan.json", shodanNode) + mainNode := models.BuildNode("shodan", "shodan_stream", shodanNode.ID) + // models.SaveNode("nodes.json", mainNode) + // edge := models.BuildEdge("shodan", structs.Map(shodanNode), structs.Map(mainNode)) + // models.SaveEdge(edge) + e := models.Node{ + ID: mainNode.ID, + Type: mainNode.Type, + NData: mainNode.NData, + Created: mainNode.Created, + Modified: mainNode.Modified, + ShodanNode: *shodanNode, + } + + ctx := context.Background() + mu := &api.Mutation{ + CommitNow: true, + } + + pb, err := json.Marshal(e) + if err != nil { + logrus.Fatal(err) + } + + mu.SetJson = pb + + _, err = graphClient.NewTxn().Mutate(ctx, mu) + if err != nil { + logrus.Fatal(err) + } + } + } else { + logrus.Info(shodanNode.HostData.IP, "is akamain") } } + // } + } }