From b72e82071d705ea181fc3db9c751e1c21ac81121 Mon Sep 17 00:00:00 2001 From: Christopher Talib Date: Mon, 2 Mar 2020 17:06:28 +0100 Subject: [PATCH] saving --- go.sum | 1 + schema.go => graph/main.go | 4 +- main.go | 104 +++---------------------------------- models/main.go | 45 +++++++++------- 4 files changed, 34 insertions(+), 120 deletions(-) rename schema.go => graph/main.go (97%) diff --git a/go.sum b/go.sum index d5e6187..b04447a 100644 --- a/go.sum +++ b/go.sum @@ -226,6 +226,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= diff --git a/schema.go b/graph/main.go similarity index 97% rename from schema.go rename to graph/main.go index edd2aa6..1116cf2 100644 --- a/schema.go +++ b/graph/main.go @@ -1,8 +1,7 @@ -package main +package graph import ( "context" - "fmt" "github.com/dgraph-io/dgo" "github.com/dgraph-io/dgo/protos/api" @@ -18,7 +17,6 @@ func ConnectToDgraph() error { dgraphClient := dgo.NewDgraphClient(api.NewDgraphClient(conn)) - fmt.Println("pouet") err = setupDgraphSchema(dgraphClient) if err != nil { return err diff --git a/main.go b/main.go index 6362274..487e6aa 100644 --- a/main.go +++ b/main.go @@ -1,19 +1,12 @@ package main import ( - "fmt" "os" "sync" - "time" - "github.com/ns3777k/go-shodan/v4/shodan" - "github.com/segmentio/kafka-go" "github.com/sirupsen/logrus" "github.com/spf13/viper" - "gitlab.dcso.lolcat/LABS/styx/broker" - "gitlab.dcso.lolcat/LABS/styx/elasticsearch" - "gitlab.dcso.lolcat/LABS/styx/filters" - "gitlab.dcso.lolcat/LABS/styx/models" + "gitlab.dcso.lolcat/LABS/styx/graph" "gitlab.dcso.lolcat/LABS/styx/plugins" ) @@ -35,7 +28,7 @@ func main() { } os.Setenv("SHODAN_KEY", viper.GetString("shodan.key")) - fmt.Println("Starting to get data from the Internet...") + logrus.Info("Starting to get data from the Internet...") // The false flag specifies that we want heartbeat messages. // conn, err := broker.SetUpKafkaConnecter() @@ -43,7 +36,7 @@ func main() { // panic(err) // } - err = ConnectToDgraph() + err = graph.ConnectToDgraph() if err != nil { logrus.WithField("err", err).Error("error initialising the graph database") } @@ -57,14 +50,14 @@ func main() { // certstream c := plugins.CertStreamPlugin{} if ok := c.Initialize(); !ok { - logrus.Info("certstream module not activated") + logrus.Info("certstream plugin not activated") } c.Run(&wg) // pastebin p := plugins.PastebinPlugin{} if ok := p.Initialize(); !ok { - logrus.Info("pastebin module not activated") + logrus.Info("pastebin plugin not activated") } p.Run(&wg) @@ -77,96 +70,13 @@ func main() { go func() { <-stopChan - logrus.Println("Shutting down...") + logrus.Info("Shutting down...") c.Stop(&wg) p.Stop(&wg) s.Stop(&wg) }() wg.Wait() - fmt.Println("done") + logrus.Info("done") } - -// routines - -func pastebinRoutine(stopChan chan os.Signal, wg *sync.WaitGroup) { - fmt.Println("pastebin is activated") - - elastic := viper.GetBool("elasticsearch.activated") - var e *elasticsearch.ElasticStorageModule - if elastic { - fmt.Println("elasticsearch is activated") - e = &elasticsearch.ElasticStorageModule{ - ElasticURL: viper.GetString("elasticsearch.url"), - DailyIndexes: true, - UseIndex: "pastebin", - LastChk: time.Now(), - } - err := e.Initialize() - if err != nil { - panic(err) - } - } -} - -func shodanRoutine(client *shodan.Client, shodanChan chan *shodan.HostData, conn *kafka.Conn, stopChan chan os.Signal, wg *sync.WaitGroup) { - fmt.Println("shodan is activated") - for { - select { - default: - banner, ok := <-shodanChan - if !ok { - logrus.Error("channel is closed") - break - } - - shodanNode := models.BuildShodanNode(banner) - // first filter poc - if shodanNode.Data.HTML != "" { - if !filters.RunIPFilters(shodanNode.Data.IP) { - hostnames := shodanNode.Data.Hostnames - var hostNotInFilters, domainNotInFilters bool - if len(hostnames) != 0 { - for _, hostname := range hostnames { - hostNotInFilters = filters.RunDomainFilters(hostname) - if hostNotInFilters { - saveSingleValues(conn, "shodan_stream", "hostname", shodanNode.ID, hostname) - } - } - } - domains := shodanNode.Data.Domains - if len(domains) != 0 { - for _, domain := range domains { - domainNotInFilters = filters.RunDomainFilters(domain) - saveSingleValues(conn, "shodan_stream", "domain", shodanNode.ID, domain) - } - } - if domainNotInFilters && hostNotInFilters { - models.SaveShodanNode("raw_shodan.json", shodanNode) - node := models.BuildNode("shodan", "shodan_stream", shodanNode.ID) - models.SaveNode("nodes.json", node) - edge := models.BuildEdge("shodan", shodanNode.ID, node.ID) - models.SaveEdge(edge) - } - } else { - fmt.Println("is akamai", shodanNode.Data.IP) - } - } - case <-stopChan: - wg.Done() - return - } - } -} - -// helpers -func saveSingleValues(brokerConn *kafka.Conn, source string, datatype string, originNodeID string, values string) { - domainNode := models.BuildNode(source, datatype, values) - models.SaveNode("nodes.json", domainNode) - if domainNode.Type == "domain" || domainNode.Type == "hostname" { - broker.SendEventToKafka(brokerConn, *domainNode) - } - edge := models.BuildEdge(source, originNodeID, domainNode.ID) - models.SaveEdge(edge) -} diff --git a/models/main.go b/models/main.go index c6a9f5b..32494b0 100644 --- a/models/main.go +++ b/models/main.go @@ -25,11 +25,12 @@ Structure of this file: // Styx terminology // (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx) type Node struct { - ID string `json:"id"` - Type string `json:"type"` - Data string `json:"data"` - Created string `json:"created"` - Modified string `json:"modified"` + Uid string `json:"uid,omiempty"` + ID string `json:"id,omiempty"` + Type string `json:"type,omiempty"` + Data string `json:"data,omiempty"` + Created string `json:"created,omiempty"` + Modified string `json:"modified,omiempty"` } // BuildNode builds a node to send to MQ instance. @@ -37,6 +38,7 @@ func BuildNode(flag string, dataType string, data string) *Node { t := time.Now() rfc3339time := t.Format(time.RFC3339) return &Node{ + Uid: "_:" + flag + "--" + uuid.New().String(), ID: flag + "--" + uuid.New().String(), Type: dataType, Data: data, @@ -76,11 +78,12 @@ func SaveNode(filename string, node *Node) { // Edge defines a relation between two nodes. type Edge struct { - ID string `json:"id"` - NodeOneID string `json:"nodeOneID"` - NodeTwoID string `json:"nodeTwoID"` - Timestamp string `json:"timestamp"` - Source string `json:"source"` + Uid string `json:"uid,omiempty"` + ID string `json:"id,omiempty"` + NodeOneID string `json:"nodeOneID,omiempty"` + NodeTwoID string `json:"nodeTwoID,omiempty"` + Timestamp string `json:"timestamp,omiempty"` + Source string `json:"source,omiempty"` } // BuildEdge build a send from two nodes with a given source type. @@ -88,6 +91,7 @@ func BuildEdge(source string, nodeOneUUID string, nodeTwoUUID string) *Edge { t := time.Now() rfc3339time := t.Format(time.RFC3339) return &Edge{ + Uid: "_:" + "edge--" + uuid.New().String(), ID: "edge--" + uuid.New().String(), Source: source, NodeOneID: nodeOneUUID, @@ -137,16 +141,17 @@ type CertStreamRaw struct { // CertNode represents our custom struct of data extraction from CertStream. type CertNode struct { - ID string `json:"id"` - Fingerprint string `json:"fingerprint"` - NotBefore string `json:"notBefore"` - NotAfter string `json:"notAfter"` - CN string `json:"cn"` - SourceName string `json:"sourceName"` - SerialNumber string `json:"serialNumber"` - BasicConstraints string `json:"basicConstraints"` - RawUUID string `json:"rawUUID"` - Chain []CertNode `json:"chainedTo"` + Uid string `json:"uid,omiempty"` + ID string `json:"id,omiempty"` + Fingerprint string `json:"fingerprint,omiempty"` + NotBefore string `json:"notBefore,omiempty"` + NotAfter string `json:"notAfter,omiempty"` + CN string `json:"cn,omiempty"` + SourceName string `json:"sourceName,omiempty"` + SerialNumber string `json:"serialNumber,omiempty"` + BasicConstraints string `json:"basicConstraints,omiempty"` + RawUUID string `json:"rawUUID,omiempty"` + Chain []CertNode `json:"chainedTo,omiempty"` } // WrapCertStreamData is a wrapper around CertStreamStruct.