saving
This commit is contained in:
Christopher Talib
parent
cd43194873
commit
b72e82071d
1
go.sum
1
go.sum
|
|
@ -226,6 +226,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
|
|||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
|
||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
|
||||
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
|
||||
|
|
|
|||
|
|
@ -1,8 +1,7 @@
|
|||
package main
|
||||
package graph
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"github.com/dgraph-io/dgo"
|
||||
"github.com/dgraph-io/dgo/protos/api"
|
||||
|
|
@ -18,7 +17,6 @@ func ConnectToDgraph() error {
|
|||
|
||||
dgraphClient := dgo.NewDgraphClient(api.NewDgraphClient(conn))
|
||||
|
||||
fmt.Println("pouet")
|
||||
err = setupDgraphSchema(dgraphClient)
|
||||
if err != nil {
|
||||
return err
|
||||
104
main.go
104
main.go
|
|
@ -1,19 +1,12 @@
|
|||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ns3777k/go-shodan/v4/shodan"
|
||||
"github.com/segmentio/kafka-go"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/viper"
|
||||
"gitlab.dcso.lolcat/LABS/styx/broker"
|
||||
"gitlab.dcso.lolcat/LABS/styx/elasticsearch"
|
||||
"gitlab.dcso.lolcat/LABS/styx/filters"
|
||||
"gitlab.dcso.lolcat/LABS/styx/models"
|
||||
"gitlab.dcso.lolcat/LABS/styx/graph"
|
||||
"gitlab.dcso.lolcat/LABS/styx/plugins"
|
||||
)
|
||||
|
||||
|
|
@ -35,7 +28,7 @@ func main() {
|
|||
}
|
||||
|
||||
os.Setenv("SHODAN_KEY", viper.GetString("shodan.key"))
|
||||
fmt.Println("Starting to get data from the Internet...")
|
||||
logrus.Info("Starting to get data from the Internet...")
|
||||
|
||||
// The false flag specifies that we want heartbeat messages.
|
||||
// conn, err := broker.SetUpKafkaConnecter()
|
||||
|
|
@ -43,7 +36,7 @@ func main() {
|
|||
// panic(err)
|
||||
// }
|
||||
|
||||
err = ConnectToDgraph()
|
||||
err = graph.ConnectToDgraph()
|
||||
if err != nil {
|
||||
logrus.WithField("err", err).Error("error initialising the graph database")
|
||||
}
|
||||
|
|
@ -57,14 +50,14 @@ func main() {
|
|||
// certstream
|
||||
c := plugins.CertStreamPlugin{}
|
||||
if ok := c.Initialize(); !ok {
|
||||
logrus.Info("certstream module not activated")
|
||||
logrus.Info("certstream plugin not activated")
|
||||
}
|
||||
c.Run(&wg)
|
||||
|
||||
// pastebin
|
||||
p := plugins.PastebinPlugin{}
|
||||
if ok := p.Initialize(); !ok {
|
||||
logrus.Info("pastebin module not activated")
|
||||
logrus.Info("pastebin plugin not activated")
|
||||
}
|
||||
p.Run(&wg)
|
||||
|
||||
|
|
@ -77,96 +70,13 @@ func main() {
|
|||
|
||||
go func() {
|
||||
<-stopChan
|
||||
logrus.Println("Shutting down...")
|
||||
logrus.Info("Shutting down...")
|
||||
c.Stop(&wg)
|
||||
p.Stop(&wg)
|
||||
s.Stop(&wg)
|
||||
}()
|
||||
|
||||
wg.Wait()
|
||||
fmt.Println("done")
|
||||
logrus.Info("done")
|
||||
|
||||
}
|
||||
|
||||
// routines
|
||||
|
||||
func pastebinRoutine(stopChan chan os.Signal, wg *sync.WaitGroup) {
|
||||
fmt.Println("pastebin is activated")
|
||||
|
||||
elastic := viper.GetBool("elasticsearch.activated")
|
||||
var e *elasticsearch.ElasticStorageModule
|
||||
if elastic {
|
||||
fmt.Println("elasticsearch is activated")
|
||||
e = &elasticsearch.ElasticStorageModule{
|
||||
ElasticURL: viper.GetString("elasticsearch.url"),
|
||||
DailyIndexes: true,
|
||||
UseIndex: "pastebin",
|
||||
LastChk: time.Now(),
|
||||
}
|
||||
err := e.Initialize()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func shodanRoutine(client *shodan.Client, shodanChan chan *shodan.HostData, conn *kafka.Conn, stopChan chan os.Signal, wg *sync.WaitGroup) {
|
||||
fmt.Println("shodan is activated")
|
||||
for {
|
||||
select {
|
||||
default:
|
||||
banner, ok := <-shodanChan
|
||||
if !ok {
|
||||
logrus.Error("channel is closed")
|
||||
break
|
||||
}
|
||||
|
||||
shodanNode := models.BuildShodanNode(banner)
|
||||
// first filter poc
|
||||
if shodanNode.Data.HTML != "" {
|
||||
if !filters.RunIPFilters(shodanNode.Data.IP) {
|
||||
hostnames := shodanNode.Data.Hostnames
|
||||
var hostNotInFilters, domainNotInFilters bool
|
||||
if len(hostnames) != 0 {
|
||||
for _, hostname := range hostnames {
|
||||
hostNotInFilters = filters.RunDomainFilters(hostname)
|
||||
if hostNotInFilters {
|
||||
saveSingleValues(conn, "shodan_stream", "hostname", shodanNode.ID, hostname)
|
||||
}
|
||||
}
|
||||
}
|
||||
domains := shodanNode.Data.Domains
|
||||
if len(domains) != 0 {
|
||||
for _, domain := range domains {
|
||||
domainNotInFilters = filters.RunDomainFilters(domain)
|
||||
saveSingleValues(conn, "shodan_stream", "domain", shodanNode.ID, domain)
|
||||
}
|
||||
}
|
||||
if domainNotInFilters && hostNotInFilters {
|
||||
models.SaveShodanNode("raw_shodan.json", shodanNode)
|
||||
node := models.BuildNode("shodan", "shodan_stream", shodanNode.ID)
|
||||
models.SaveNode("nodes.json", node)
|
||||
edge := models.BuildEdge("shodan", shodanNode.ID, node.ID)
|
||||
models.SaveEdge(edge)
|
||||
}
|
||||
} else {
|
||||
fmt.Println("is akamai", shodanNode.Data.IP)
|
||||
}
|
||||
}
|
||||
case <-stopChan:
|
||||
wg.Done()
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// helpers
|
||||
func saveSingleValues(brokerConn *kafka.Conn, source string, datatype string, originNodeID string, values string) {
|
||||
domainNode := models.BuildNode(source, datatype, values)
|
||||
models.SaveNode("nodes.json", domainNode)
|
||||
if domainNode.Type == "domain" || domainNode.Type == "hostname" {
|
||||
broker.SendEventToKafka(brokerConn, *domainNode)
|
||||
}
|
||||
edge := models.BuildEdge(source, originNodeID, domainNode.ID)
|
||||
models.SaveEdge(edge)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -25,11 +25,12 @@ Structure of this file:
|
|||
// Styx terminology
|
||||
// (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx)
|
||||
type Node struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Data string `json:"data"`
|
||||
Created string `json:"created"`
|
||||
Modified string `json:"modified"`
|
||||
Uid string `json:"uid,omiempty"`
|
||||
ID string `json:"id,omiempty"`
|
||||
Type string `json:"type,omiempty"`
|
||||
Data string `json:"data,omiempty"`
|
||||
Created string `json:"created,omiempty"`
|
||||
Modified string `json:"modified,omiempty"`
|
||||
}
|
||||
|
||||
// BuildNode builds a node to send to MQ instance.
|
||||
|
|
@ -37,6 +38,7 @@ func BuildNode(flag string, dataType string, data string) *Node {
|
|||
t := time.Now()
|
||||
rfc3339time := t.Format(time.RFC3339)
|
||||
return &Node{
|
||||
Uid: "_:" + flag + "--" + uuid.New().String(),
|
||||
ID: flag + "--" + uuid.New().String(),
|
||||
Type: dataType,
|
||||
Data: data,
|
||||
|
|
@ -76,11 +78,12 @@ func SaveNode(filename string, node *Node) {
|
|||
|
||||
// Edge defines a relation between two nodes.
|
||||
type Edge struct {
|
||||
ID string `json:"id"`
|
||||
NodeOneID string `json:"nodeOneID"`
|
||||
NodeTwoID string `json:"nodeTwoID"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
Source string `json:"source"`
|
||||
Uid string `json:"uid,omiempty"`
|
||||
ID string `json:"id,omiempty"`
|
||||
NodeOneID string `json:"nodeOneID,omiempty"`
|
||||
NodeTwoID string `json:"nodeTwoID,omiempty"`
|
||||
Timestamp string `json:"timestamp,omiempty"`
|
||||
Source string `json:"source,omiempty"`
|
||||
}
|
||||
|
||||
// BuildEdge build a send from two nodes with a given source type.
|
||||
|
|
@ -88,6 +91,7 @@ func BuildEdge(source string, nodeOneUUID string, nodeTwoUUID string) *Edge {
|
|||
t := time.Now()
|
||||
rfc3339time := t.Format(time.RFC3339)
|
||||
return &Edge{
|
||||
Uid: "_:" + "edge--" + uuid.New().String(),
|
||||
ID: "edge--" + uuid.New().String(),
|
||||
Source: source,
|
||||
NodeOneID: nodeOneUUID,
|
||||
|
|
@ -137,16 +141,17 @@ type CertStreamRaw struct {
|
|||
|
||||
// CertNode represents our custom struct of data extraction from CertStream.
|
||||
type CertNode struct {
|
||||
ID string `json:"id"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
NotBefore string `json:"notBefore"`
|
||||
NotAfter string `json:"notAfter"`
|
||||
CN string `json:"cn"`
|
||||
SourceName string `json:"sourceName"`
|
||||
SerialNumber string `json:"serialNumber"`
|
||||
BasicConstraints string `json:"basicConstraints"`
|
||||
RawUUID string `json:"rawUUID"`
|
||||
Chain []CertNode `json:"chainedTo"`
|
||||
Uid string `json:"uid,omiempty"`
|
||||
ID string `json:"id,omiempty"`
|
||||
Fingerprint string `json:"fingerprint,omiempty"`
|
||||
NotBefore string `json:"notBefore,omiempty"`
|
||||
NotAfter string `json:"notAfter,omiempty"`
|
||||
CN string `json:"cn,omiempty"`
|
||||
SourceName string `json:"sourceName,omiempty"`
|
||||
SerialNumber string `json:"serialNumber,omiempty"`
|
||||
BasicConstraints string `json:"basicConstraints,omiempty"`
|
||||
RawUUID string `json:"rawUUID,omiempty"`
|
||||
Chain []CertNode `json:"chainedTo,omiempty"`
|
||||
}
|
||||
|
||||
// WrapCertStreamData is a wrapper around CertStreamStruct.
|
||||
|
|
|
|||
Loading…
Reference in a new issue