Shodan in Dgraph, first part
Implementing first version for shodan node, missing yet some models, but the overal approach works and can be queried in Ratel.
This commit is contained in:
parent
cbdca52ab2
commit
b1ca4b3c5f
51
README.md
51
README.md
|
@ -45,35 +45,35 @@ same subnet. Check [this](https://serverfault.com/questions/916941/configuring-d
|
|||
### Example configuration:
|
||||
```
|
||||
certstream:
|
||||
activated: true
|
||||
activated: true
|
||||
|
||||
pastebin:
|
||||
activated: true
|
||||
activated: true
|
||||
|
||||
shodan:
|
||||
activated: true
|
||||
key: "SHODAN_KEY"
|
||||
ports:
|
||||
- 80
|
||||
- 443
|
||||
activated: true
|
||||
key: "SHODAN_KEY"
|
||||
ports:
|
||||
- 80
|
||||
- 443
|
||||
|
||||
kafka:
|
||||
activated: true
|
||||
protocol: "tcp"
|
||||
host: "localhost"
|
||||
port: 9092
|
||||
topic: "styx"
|
||||
partition: 0
|
||||
activated: true
|
||||
protocol: "tcp"
|
||||
host: "localhost"
|
||||
port: 9092
|
||||
topic: "styx"
|
||||
partition: 0
|
||||
|
||||
balboa:
|
||||
# the url you tunneled to Balboa
|
||||
url: http://127.0.0.1:8030
|
||||
activated: true
|
||||
# the url you tunneled to Balboa
|
||||
url: http://127.0.0.1:8030
|
||||
activated: true
|
||||
|
||||
elasticsearch:
|
||||
activated: true
|
||||
url: http://localhost:9200
|
||||
index: "pastebin"
|
||||
activated: true
|
||||
url: http://localhost:9200
|
||||
index: "pastebin"
|
||||
```
|
||||
|
||||
## Dgraph Interface
|
||||
|
@ -104,7 +104,7 @@ query {
|
|||
modified
|
||||
type
|
||||
ndata
|
||||
cert_node {
|
||||
certNode {
|
||||
uid
|
||||
fingerprint
|
||||
cn
|
||||
|
@ -122,6 +122,17 @@ query {
|
|||
notBefore
|
||||
notAfter
|
||||
}
|
||||
shodanNode {
|
||||
uid
|
||||
hostData {
|
||||
product
|
||||
ip
|
||||
version
|
||||
hostnames
|
||||
port
|
||||
html
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
|
|
@ -79,18 +79,31 @@ func ReadEventFromKafka() {
|
|||
if len(node.ID) != 0 {
|
||||
// TODO: refactor this context
|
||||
ctx := context.Background()
|
||||
entries, err := c.GetAllEntries(ctx, node.Data, "", "", int32(1))
|
||||
entries, err := c.GetAllEntries(ctx, node.NData, "", "", int32(1))
|
||||
if err != nil {
|
||||
logrus.Error("error from balboa", err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
balboaNode := models.BuildBalboaNode(entries)
|
||||
models.SaveBalboaNode("bnodes.json", balboaNode)
|
||||
edge := models.BuildEdge("balboa", node.ID, balboaNode.ID)
|
||||
models.SaveEdge(edge)
|
||||
// edge := models.BuildEdge("balboa", node.ID, balboaNode.ID)
|
||||
// models.SaveEdge(edge)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Helpers
|
||||
// func SaveSingleValues(brokerConn *kafka.Conn, source string, datatype string, originNodeID string, values []string) {
|
||||
// for _, value := range values {
|
||||
// domainNode := models.BuildNode(source, datatype, value)
|
||||
// models.SaveNode("nodes.json", domainNode)
|
||||
// if domainNode.Type == "domain" || domainNode.Type == "hostname" {
|
||||
// broker.SendEventToKafka(brokerConn, *domainNode)
|
||||
// }
|
||||
// edge := models.BuildEdge(source, originNodeID, domainNode.ID)
|
||||
// models.SaveEdge(edge)
|
||||
// }
|
||||
// }
|
||||
|
|
|
@ -18,7 +18,8 @@ var (
|
|||
)
|
||||
|
||||
// RunIPFilters runs the battery of filters for an IP.
|
||||
func RunIPFilters(ip net.IP) bool {
|
||||
func RunIPFilters(InputIP string) bool {
|
||||
ip := net.ParseIP(InputIP)
|
||||
if ip.To4() != nil {
|
||||
path := basepath + "/data/ipv4/"
|
||||
sliceIPv4, err := ioutil.ReadDir(path)
|
||||
|
|
|
@ -44,7 +44,8 @@ sourceName: string @index(term) .
|
|||
timestamp: string .
|
||||
created: string .
|
||||
modified: string .
|
||||
cert_node: uid .
|
||||
certNode: uid .
|
||||
shodanNode: uid .
|
||||
|
||||
type Node {
|
||||
id: string
|
||||
|
@ -52,7 +53,8 @@ type: string
|
|||
ndata: string
|
||||
created: string
|
||||
modified: string
|
||||
cert_node: CertNode
|
||||
certNode: CertNode
|
||||
shodanNode: ShodanNode
|
||||
}
|
||||
|
||||
type Edge {
|
||||
|
@ -95,37 +97,52 @@ modified: string
|
|||
csdata: string
|
||||
}
|
||||
|
||||
type PasteNode {
|
||||
id: string
|
||||
type: string
|
||||
created: string
|
||||
modified: string
|
||||
ndata: uid
|
||||
}
|
||||
|
||||
meta: uid .
|
||||
full: string .
|
||||
|
||||
type FullPaste {
|
||||
meta: PasteNode
|
||||
full: string
|
||||
}
|
||||
hostData: uid .
|
||||
|
||||
type ShodanNode {
|
||||
id: string
|
||||
type: string
|
||||
ndata: string
|
||||
created: string
|
||||
modified: string
|
||||
hostData: uid
|
||||
}
|
||||
|
||||
type BalboaNode {
|
||||
id: string
|
||||
type: string
|
||||
ndata: string
|
||||
created: string
|
||||
modified: string
|
||||
product: string .
|
||||
hostnames: [string] .
|
||||
version: string .
|
||||
title: string .
|
||||
ip: string .
|
||||
os: string .
|
||||
organization: string .
|
||||
isp: string .
|
||||
cpe: [string] .
|
||||
asn: string .
|
||||
port: int .
|
||||
html: string .
|
||||
banner: string .
|
||||
transport: string .
|
||||
domains: [string] .
|
||||
timestamp: string .
|
||||
|
||||
type Hostdata {
|
||||
product: string
|
||||
hostnames: [string]
|
||||
version: string
|
||||
title: string
|
||||
ip: string
|
||||
os: string
|
||||
organization: string
|
||||
isp: string
|
||||
cpe: [string]
|
||||
asn: string
|
||||
port: int
|
||||
html: string
|
||||
banner: string
|
||||
transport: string
|
||||
domains: [string]
|
||||
timestamp: string
|
||||
}
|
||||
|
||||
`})
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
2
main.go
2
main.go
|
@ -68,7 +68,7 @@ func main() {
|
|||
if ok := s.Initialize(); !ok {
|
||||
logrus.Info("shodan plugin not activated")
|
||||
} else {
|
||||
p.Run(&wg)
|
||||
s.Run(&wg, dgraphClient)
|
||||
}
|
||||
|
||||
go func() {
|
||||
|
|
|
@ -31,7 +31,8 @@ type Node struct {
|
|||
Created string `json:"created,omiempty"`
|
||||
Modified string `json:"modified,omiempty"`
|
||||
DType []string `json:"dgraph.type,omiempty"`
|
||||
CertNode CertNode `json:"cert_node,omiempty"`
|
||||
CertNode CertNode `json:"certNode,omiempty"`
|
||||
ShodanNode ShodanNode `json:"shodanNode,omiempty"`
|
||||
}
|
||||
|
||||
// BuildNode builds a node to send to MQ instance.
|
||||
|
@ -151,7 +152,7 @@ type CertNode struct {
|
|||
SerialNumber string `json:"serialNumber,omiempty"`
|
||||
BasicConstraints string `json:"basicConstraints,omiempty"`
|
||||
Raw CertStreamRaw `json:"raw,omiempty"`
|
||||
Chain []CertNode `json:"chainedTo,omiempty"`
|
||||
Chain []CertNode `json:"chain,omiempty"`
|
||||
}
|
||||
|
||||
// WrapCertStreamData is a wrapper around CertStreamStruct.
|
||||
|
@ -316,11 +317,39 @@ func SavePaste(filename string, data *PasteNode) {
|
|||
|
||||
// ShodanNode is node around the shodan.HostData struct.
|
||||
type ShodanNode struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Data *shodan.HostData `json:"data"`
|
||||
Created string `json:"created"`
|
||||
Modified string `json:"modified"`
|
||||
ID string `json:"id,omiempty"`
|
||||
Type string `json:"type,omiempty"`
|
||||
HostData ShodanHostData `json:"hostData,omiempty"`
|
||||
Created string `json:"created,omiempty"`
|
||||
Modified string `json:"modified,omiempty"`
|
||||
}
|
||||
|
||||
// ShodanHostData is a copy of the structure in the go shodan library. It's a
|
||||
// workaround to have more control on the data send.
|
||||
type ShodanHostData struct {
|
||||
Product string `json:"product,omiempty"`
|
||||
Hostnames []string `json:"hostnames,omiempty"`
|
||||
Version string `json:"version,omiempty"`
|
||||
Title string `json:"title,omiempty"`
|
||||
// SSL *HostSSL `json:"ssl"`
|
||||
IP string `json:"ip_str,omiempty"`
|
||||
OS string `json:"os,omiempty"`
|
||||
Organization string `json:"org,omiempty"`
|
||||
ISP string `json:"isp,omiempty"`
|
||||
CPE []string `json:"cpe,omiempty"`
|
||||
// Data string `json:"data,omiempty"`
|
||||
ASN string `json:"asn,omiempty"`
|
||||
Port int `json:"port,omiempty"`
|
||||
HTML string `json:"html,omiempty"`
|
||||
Banner string `json:"banner,omiempty"`
|
||||
Link string `json:"link,omiempty"`
|
||||
Transport string `json:"transport,omiempty"`
|
||||
Domains []string `json:"domains,omiempty"`
|
||||
Timestamp string `json:"timestamp,omiempty"`
|
||||
DeviceType string `json:"devicetype,omiempty"`
|
||||
// Location *HostLocation `json:"location"`
|
||||
ShodanData map[string]interface{} `json:"_shodan,omiempty"`
|
||||
Opts map[string]interface{} `json:"opts,omiempty"`
|
||||
}
|
||||
|
||||
// BuildShodanNode builds a wrapper node around shodan.HostData.
|
||||
|
@ -330,7 +359,29 @@ func BuildShodanNode(data *shodan.HostData) *ShodanNode {
|
|||
return &ShodanNode{
|
||||
ID: "shodan--" + uuid.New().String(),
|
||||
Type: "shodan_stream",
|
||||
Data: data,
|
||||
HostData: ShodanHostData{
|
||||
Product: data.Product,
|
||||
Hostnames: data.Hostnames,
|
||||
Version: data.Version.String(),
|
||||
Title: data.Title,
|
||||
IP: data.IP.String(),
|
||||
OS: data.OS,
|
||||
Organization: data.Organization,
|
||||
ISP: data.ISP,
|
||||
CPE: data.CPE,
|
||||
// Data: data.Data,
|
||||
ASN: data.ASN,
|
||||
Port: data.Port,
|
||||
HTML: data.HTML,
|
||||
Banner: data.Banner,
|
||||
Link: data.Link,
|
||||
Transport: data.Transport,
|
||||
Domains: data.Domains,
|
||||
Timestamp: data.Timestamp,
|
||||
DeviceType: data.DeviceType,
|
||||
ShodanData: data.ShodanData,
|
||||
Opts: data.Opts,
|
||||
},
|
||||
Created: rfc3339time,
|
||||
Modified: rfc3339time,
|
||||
}
|
||||
|
|
|
@ -1,10 +1,12 @@
|
|||
package plugins
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
|
||||
"github.com/christalib/structs"
|
||||
"github.com/dgraph-io/dgo/v2"
|
||||
"github.com/dgraph-io/dgo/v2/protos/api"
|
||||
"github.com/ns3777k/go-shodan/v4/shodan"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/viper"
|
||||
|
@ -27,15 +29,16 @@ func (s *ShodanPlugin) Initialize() bool {
|
|||
return false
|
||||
}
|
||||
logrus.Info("shodan plugin is activated")
|
||||
s.ShodanChan = make(chan *shodan.HostData)
|
||||
return true
|
||||
}
|
||||
|
||||
// Run runs the Shodan plugin.
|
||||
func (s *ShodanPlugin) Run(wg *sync.WaitGroup) {
|
||||
func (s *ShodanPlugin) Run(wg *sync.WaitGroup, dgraphClient *dgo.Dgraph) {
|
||||
if !s.Running {
|
||||
s.StoppedChan = make(chan bool)
|
||||
wg.Add(1)
|
||||
go s.doRun()
|
||||
go s.doRun(dgraphClient)
|
||||
s.Running = true
|
||||
}
|
||||
}
|
||||
|
@ -51,7 +54,13 @@ func (s *ShodanPlugin) Stop(wg *sync.WaitGroup) {
|
|||
}
|
||||
}
|
||||
|
||||
func (s *ShodanPlugin) doRun() {
|
||||
func (s *ShodanPlugin) doRun(graphClient *dgo.Dgraph) {
|
||||
client := shodan.NewEnvClient(nil)
|
||||
err := client.GetBannersByPorts(context.Background(), viper.GetIntSlice("shodan.ports"), s.ShodanChan)
|
||||
if err != nil {
|
||||
logrus.Panic(err)
|
||||
}
|
||||
|
||||
for {
|
||||
select {
|
||||
default:
|
||||
|
@ -63,36 +72,64 @@ func (s *ShodanPlugin) doRun() {
|
|||
|
||||
shodanNode := models.BuildShodanNode(banner)
|
||||
// first filter poc
|
||||
if shodanNode.Data.HTML != "" {
|
||||
if !filters.RunIPFilters(shodanNode.Data.IP) {
|
||||
hostnames := shodanNode.Data.Hostnames
|
||||
// if shodanNode.Data.HTML != "" {
|
||||
if !filters.RunIPFilters(shodanNode.HostData.IP) {
|
||||
hostnames := shodanNode.HostData.Hostnames
|
||||
var hostNotInFilters, domainNotInFilters bool
|
||||
if len(hostnames) != 0 {
|
||||
for _, hostname := range hostnames {
|
||||
hostNotInFilters = filters.RunDomainFilters(hostname)
|
||||
if hostNotInFilters {
|
||||
logrus.Info("host", hostname, "not in filters")
|
||||
// saveSingleValues(conn, "shodan_stream", "hostname", shodanNode.ID, hostname)
|
||||
}
|
||||
}
|
||||
}
|
||||
domains := shodanNode.Data.Domains
|
||||
domains := shodanNode.HostData.Domains
|
||||
if len(domains) != 0 {
|
||||
for _, domain := range domains {
|
||||
domainNotInFilters = filters.RunDomainFilters(domain)
|
||||
logrus.Info("domain", domain, "not in filters")
|
||||
// saveSingleValues(conn, "shodan_stream", "domain", shodanNode.ID, domain)
|
||||
}
|
||||
}
|
||||
if domainNotInFilters && hostNotInFilters {
|
||||
models.SaveShodanNode("raw_shodan.json", shodanNode)
|
||||
node := models.BuildNode("shodan", "shodan_stream", shodanNode.ID)
|
||||
models.SaveNode("nodes.json", node)
|
||||
edge := models.BuildEdge("shodan", structs.Map(shodanNode), structs.Map(node))
|
||||
models.SaveEdge(edge)
|
||||
mainNode := models.BuildNode("shodan", "shodan_stream", shodanNode.ID)
|
||||
// models.SaveNode("nodes.json", mainNode)
|
||||
// edge := models.BuildEdge("shodan", structs.Map(shodanNode), structs.Map(mainNode))
|
||||
// models.SaveEdge(edge)
|
||||
e := models.Node{
|
||||
ID: mainNode.ID,
|
||||
Type: mainNode.Type,
|
||||
NData: mainNode.NData,
|
||||
Created: mainNode.Created,
|
||||
Modified: mainNode.Modified,
|
||||
ShodanNode: *shodanNode,
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
mu := &api.Mutation{
|
||||
CommitNow: true,
|
||||
}
|
||||
|
||||
pb, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
|
||||
mu.SetJson = pb
|
||||
|
||||
_, err = graphClient.NewTxn().Mutate(ctx, mu)
|
||||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
fmt.Println("is akamai", shodanNode.Data.IP)
|
||||
}
|
||||
logrus.Info(shodanNode.HostData.IP, "is akamain")
|
||||
}
|
||||
}
|
||||
// }
|
||||
|
||||
}
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue