chris/styx
chris/styx
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Shodan in Dgraph, first part

Browse source 
Implementing first version for shodan node, missing yet some models, but
the overal approach works and can be queried in Ratel.
This commit is contained in:
Christopher Talib 2020-05-18 16:09:04 +02:00
parent cbdca52ab2
commit b1ca4b3c5f
7 changed files with 240 additions and 110 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
README.md
View file

@ -104,7 +104,7 @@ query {
modified
type
ndata
cert_node {
certNode {
uid
fingerprint
cn
@ -122,6 +122,17 @@ query {
notBefore
notAfter
}
shodanNode {
uid
hostData {
product
ip
version
hostnames
port
html
}
}
}
}
```

19
broker/main.go
View file

@ -79,18 +79,31 @@ func ReadEventFromKafka() {
if len(node.ID) != 0 {
// TODO: refactor this context
ctx := context.Background()
entries, err := c.GetAllEntries(ctx, node.Data, "", "", int32(1))
entries, err := c.GetAllEntries(ctx, node.NData, "", "", int32(1))
if err != nil {
logrus.Error("error from balboa", err)
}
if len(entries) != 0 {
balboaNode := models.BuildBalboaNode(entries)
models.SaveBalboaNode("bnodes.json", balboaNode)
edge := models.BuildEdge("balboa", node.ID, balboaNode.ID)
models.SaveEdge(edge)
// edge := models.BuildEdge("balboa", node.ID, balboaNode.ID)
// models.SaveEdge(edge)
}
}
}
}
}
// Helpers
// func SaveSingleValues(brokerConn *kafka.Conn, source string, datatype string, originNodeID string, values []string) {
// for _, value := range values {
// domainNode := models.BuildNode(source, datatype, value)
// models.SaveNode("nodes.json", domainNode)
// if domainNode.Type == "domain" || domainNode.Type == "hostname" {
// broker.SendEventToKafka(brokerConn, *domainNode)
// }
// edge := models.BuildEdge(source, originNodeID, domainNode.ID)
// models.SaveEdge(edge)
// }
// }

3
filters/main.go
View file

@ -18,7 +18,8 @@ var (
)
// RunIPFilters runs the battery of filters for an IP.
func RunIPFilters(ip net.IP) bool {
func RunIPFilters(InputIP string) bool {
ip := net.ParseIP(InputIP)
if ip.To4() != nil {
path := basepath + "/data/ipv4/"
sliceIPv4, err := ioutil.ReadDir(path)

65
graph/main.go
View file

@ -44,7 +44,8 @@ sourceName: string @index(term) .
timestamp: string .
created: string .
modified: string .
cert_node: uid .
certNode: uid .
shodanNode: uid .
type Node {
id: string
@ -52,7 +53,8 @@ type: string
ndata: string
created: string
modified: string
cert_node: CertNode
certNode: CertNode
shodanNode: ShodanNode
}
type Edge {
@ -95,37 +97,52 @@ modified: string
csdata: string
}
type PasteNode {
id: string
type: string
created: string
modified: string
ndata: uid
}
meta: uid .
full: string .
type FullPaste {
meta: PasteNode
full: string
}
hostData: uid .
type ShodanNode {
id: string
type: string
ndata: string
created: string
modified: string
hostData: uid
}
type BalboaNode {
id: string
type: string
ndata: string
created: string
modified: string
product: string .
hostnames: [string] .
version: string .
title: string .
ip: string .
os: string .
organization: string .
isp: string .
cpe: [string] .
asn: string .
port: int .
html: string .
banner: string .
transport: string .
domains: [string] .
timestamp: string .
type Hostdata {
product: string
hostnames: [string]
version: string
title: string
ip: string
os: string
organization: string
isp: string
cpe: [string]
asn: string
port: int
html: string
banner: string
transport: string
domains: [string]
timestamp: string
}
`})
if err != nil {
return err

2
main.go
View file

@ -68,7 +68,7 @@ func main() {
if ok := s.Initialize(); !ok {
logrus.Info("shodan plugin not activated")
} else {
p.Run(&wg)
s.Run(&wg, dgraphClient)
}
go func() {

67
models/main.go
View file

@ -31,7 +31,8 @@ type Node struct {
Created string `json:"created,omiempty"`
Modified string `json:"modified,omiempty"`
DType []string `json:"dgraph.type,omiempty"`
CertNode CertNode `json:"cert_node,omiempty"`
CertNode CertNode `json:"certNode,omiempty"`
ShodanNode ShodanNode `json:"shodanNode,omiempty"`
}
// BuildNode builds a node to send to MQ instance.
@ -151,7 +152,7 @@ type CertNode struct {
SerialNumber string `json:"serialNumber,omiempty"`
BasicConstraints string `json:"basicConstraints,omiempty"`
Raw CertStreamRaw `json:"raw,omiempty"`
Chain []CertNode `json:"chainedTo,omiempty"`
Chain []CertNode `json:"chain,omiempty"`
}
// WrapCertStreamData is a wrapper around CertStreamStruct.
@ -316,11 +317,39 @@ func SavePaste(filename string, data *PasteNode) {
// ShodanNode is node around the shodan.HostData struct.
type ShodanNode struct {
ID string `json:"id"`
Type string `json:"type"`
Data *shodan.HostData `json:"data"`
Created string `json:"created"`
Modified string `json:"modified"`
ID string `json:"id,omiempty"`
Type string `json:"type,omiempty"`
HostData ShodanHostData `json:"hostData,omiempty"`
Created string `json:"created,omiempty"`
Modified string `json:"modified,omiempty"`
}
// ShodanHostData is a copy of the structure in the go shodan library. It's a
// workaround to have more control on the data send.
type ShodanHostData struct {
Product string `json:"product,omiempty"`
Hostnames []string `json:"hostnames,omiempty"`
Version string `json:"version,omiempty"`
Title string `json:"title,omiempty"`
// SSL *HostSSL `json:"ssl"`
IP string `json:"ip_str,omiempty"`
OS string `json:"os,omiempty"`
Organization string `json:"org,omiempty"`
ISP string `json:"isp,omiempty"`
CPE []string `json:"cpe,omiempty"`
// Data string `json:"data,omiempty"`
ASN string `json:"asn,omiempty"`
Port int `json:"port,omiempty"`
HTML string `json:"html,omiempty"`
Banner string `json:"banner,omiempty"`
Link string `json:"link,omiempty"`
Transport string `json:"transport,omiempty"`
Domains []string `json:"domains,omiempty"`
Timestamp string `json:"timestamp,omiempty"`
DeviceType string `json:"devicetype,omiempty"`
// Location *HostLocation `json:"location"`
ShodanData map[string]interface{} `json:"_shodan,omiempty"`
Opts map[string]interface{} `json:"opts,omiempty"`
}
// BuildShodanNode builds a wrapper node around shodan.HostData.
@ -330,7 +359,29 @@ func BuildShodanNode(data *shodan.HostData) *ShodanNode {
return &ShodanNode{
ID: "shodan--" + uuid.New().String(),
Type: "shodan_stream",
Data: data,
HostData: ShodanHostData{
Product: data.Product,
Hostnames: data.Hostnames,
Version: data.Version.String(),
Title: data.Title,
IP: data.IP.String(),
OS: data.OS,
Organization: data.Organization,
ISP: data.ISP,
CPE: data.CPE,
// Data: data.Data,
ASN: data.ASN,
Port: data.Port,
HTML: data.HTML,
Banner: data.Banner,
Link: data.Link,
Transport: data.Transport,
Domains: data.Domains,
Timestamp: data.Timestamp,
DeviceType: data.DeviceType,
ShodanData: data.ShodanData,
Opts: data.Opts,
},
Created: rfc3339time,
Modified: rfc3339time,
}

67
plugins/shodan.go
View file

@ -1,10 +1,12 @@
package plugins
import (
"fmt"
"context"
"encoding/json"
"sync"
"github.com/christalib/structs"
"github.com/dgraph-io/dgo/v2"
"github.com/dgraph-io/dgo/v2/protos/api"
"github.com/ns3777k/go-shodan/v4/shodan"
"github.com/sirupsen/logrus"
"github.com/spf13/viper"
@ -27,15 +29,16 @@ func (s *ShodanPlugin) Initialize() bool {
return false
}
logrus.Info("shodan plugin is activated")
s.ShodanChan = make(chan *shodan.HostData)
return true
}
// Run runs the Shodan plugin.
func (s *ShodanPlugin) Run(wg *sync.WaitGroup) {
func (s *ShodanPlugin) Run(wg *sync.WaitGroup, dgraphClient *dgo.Dgraph) {
if !s.Running {
s.StoppedChan = make(chan bool)
wg.Add(1)
go s.doRun()
go s.doRun(dgraphClient)
s.Running = true
}
}
@ -51,7 +54,13 @@ func (s *ShodanPlugin) Stop(wg *sync.WaitGroup) {
}
}
func (s *ShodanPlugin) doRun() {
func (s *ShodanPlugin) doRun(graphClient *dgo.Dgraph) {
client := shodan.NewEnvClient(nil)
err := client.GetBannersByPorts(context.Background(), viper.GetIntSlice("shodan.ports"), s.ShodanChan)
if err != nil {
logrus.Panic(err)
}
for {
select {
default:
@ -63,36 +72,64 @@ func (s *ShodanPlugin) doRun() {
shodanNode := models.BuildShodanNode(banner)
// first filter poc
if shodanNode.Data.HTML != "" {
if !filters.RunIPFilters(shodanNode.Data.IP) {
hostnames := shodanNode.Data.Hostnames
// if shodanNode.Data.HTML != "" {
if !filters.RunIPFilters(shodanNode.HostData.IP) {
hostnames := shodanNode.HostData.Hostnames
var hostNotInFilters, domainNotInFilters bool
if len(hostnames) != 0 {
for _, hostname := range hostnames {
hostNotInFilters = filters.RunDomainFilters(hostname)
if hostNotInFilters {
logrus.Info("host", hostname, "not in filters")
// saveSingleValues(conn, "shodan_stream", "hostname", shodanNode.ID, hostname)
}
}
}
domains := shodanNode.Data.Domains
domains := shodanNode.HostData.Domains
if len(domains) != 0 {
for _, domain := range domains {
domainNotInFilters = filters.RunDomainFilters(domain)
logrus.Info("domain", domain, "not in filters")
// saveSingleValues(conn, "shodan_stream", "domain", shodanNode.ID, domain)
}
}
if domainNotInFilters && hostNotInFilters {
models.SaveShodanNode("raw_shodan.json", shodanNode)
node := models.BuildNode("shodan", "shodan_stream", shodanNode.ID)
models.SaveNode("nodes.json", node)
edge := models.BuildEdge("shodan", structs.Map(shodanNode), structs.Map(node))
models.SaveEdge(edge)
mainNode := models.BuildNode("shodan", "shodan_stream", shodanNode.ID)
// models.SaveNode("nodes.json", mainNode)
// edge := models.BuildEdge("shodan", structs.Map(shodanNode), structs.Map(mainNode))
// models.SaveEdge(edge)
e := models.Node{
ID: mainNode.ID,
Type: mainNode.Type,
NData: mainNode.NData,
Created: mainNode.Created,
Modified: mainNode.Modified,
ShodanNode: *shodanNode,
}
ctx := context.Background()
mu := &api.Mutation{
CommitNow: true,
}
pb, err := json.Marshal(e)
if err != nil {
logrus.Fatal(err)
}
mu.SetJson = pb
_, err = graphClient.NewTxn().Mutate(ctx, mu)
if err != nil {
logrus.Fatal(err)
}
}
} else {
fmt.Println("is akamai", shodanNode.Data.IP)
}
logrus.Info(shodanNode.HostData.IP, "is akamain")
}
}
// }
}
}