saving

go.mod

@@ -7,6 +7,7 @@ require (
 	dmitri.shuralyov.com/gpu/mtl v0.0.0-20191203043605-d42048ed14fd // indirect
 	github.com/CaliDog/certstream-go v0.0.0-20180219203951-6016c5462366
 	github.com/DataDog/zstd v1.4.4 // indirect
+	github.com/christalib/structs v1.1.0
 	github.com/cncf/udpa/go v0.0.0-20200124205748-db4b343e48c1 // indirect
 	github.com/dgraph-io/dgo v1.0.0
 	github.com/dgraph-io/dgo/v2 v2.2.0

go.sum

@@ -38,6 +38,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
 github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
+github.com/christalib/structs v1.1.0 h1:vWXpO00DCoGj5jYo+HV2pvNmoDsKHAAHPdkK2aTZjXE=
+github.com/christalib/structs v1.1.0/go.mod h1:sKgAVbcnUvqeaSB3E5Y8eRgaTyo1DFz7lG2EKiSy7SU=
 github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
 github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
 github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
@@ -112,6 +114,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
+github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=

graph/main.go

@@ -26,35 +26,38 @@ func ConnectToDgraph() (*dgo.Dgraph, error) {
 
 func setupDgraphSchema(c *dgo.Dgraph) error {
 	err := c.Alter(context.Background(), &api.Operation{
-		DropAll: true,
+		DropOp: api.Operation_ALL,
 	})
+	if err != nil {
+		return err
+	}
 
 	err = c.Alter(context.Background(), &api.Operation{
 		Schema: `
 id: string @index(term) .
 type: string @index(term) .
-data: string .
+ndata: string .
-nodeOneID: uid .
+nodeOne: string @index(term) .
-nodeTwoID: uid .
+nodeTwo: string @index(term) .
-source: string @index(term) .
+sourceName: string @index(term) .
 timestamp: string .
-created: dateTime .
+created: string .
-modified: dateTime .
+modified: string .
 
 type Node {
 id
 type
-data
+ndata
 created
 modified
 }
 
 type Edge {
 id
-nodeOneID
+nodeOne
-nodeTwoID
+nodeTwo
 timestamp
-source
+sourceName
 }
 
 fingerprint: string .
@@ -65,6 +68,7 @@ sourceName: string .
 serialNumber: string .
 basicConstraints: string .
 chain: uid .
+csdata: uid .
 
 type CertNode {
 id
@@ -83,7 +87,7 @@ id
 type
 created
 modified
-data
+csdata
 }
 
 type PasteNode {
@@ -91,7 +95,7 @@ id
 type
 created
 modified
-data
+ndata
 }
 
 meta: uid .
@@ -105,7 +109,7 @@ full
 type ShodanNode {
 id
 type
-data
+ndata
 created
 modified
 }
@@ -113,7 +117,7 @@ modified
 type BalboaNode {
 id
 type
-data
+ndata
 created
 modified
 }

main.go

@@ -51,22 +51,25 @@ func main() {
 	c := plugins.CertStreamPlugin{}
 	if ok := c.Initialize(); !ok {
 		logrus.Info("certstream plugin not activated")
+	} else {
+		c.Run(&wg, dgraphClient)
 	}
-	c.Run(&wg, dgraphClient)
 
 	// pastebin
 	p := plugins.PastebinPlugin{}
 	if ok := p.Initialize(); !ok {
 		logrus.Info("pastebin plugin not activated")
+	} else {
+		p.Run(&wg)
 	}
-	p.Run(&wg)
 
 	// shodan
 	s := plugins.ShodanPlugin{}
 	if ok := s.Initialize(); !ok {
 		logrus.Info("shodan plugin not activated")
+	} else {
+		p.Run(&wg)
 	}
-	p.Run(&wg)
 
 	go func() {
 		<-stopChan

models/cerstream.go

@@ -61,7 +61,7 @@ type CertStreamData struct {
 // and the content is stored in Data.
 type CertStreamStruct struct {
 	MessageType string         `json:"message_data"`
-	Data        CertStreamData `json:"data"`
+	CSData      CertStreamData `json:"csdata"`
 }
 
 // ExtractCertFromStream builds the structures before saving them. It uses the
@@ -117,7 +117,7 @@ func ExtractCertFromStream(input jsonq.JsonQuery) (*CertStreamStruct, error) {
 
 	res := CertStreamStruct{
 		MessageType: messageType,
-		Data:        csd,
+		CSData:      csd,
 	}
 
 	return &res, nil

models/main.go

@@ -25,12 +25,12 @@ Structure of this file:
 // Styx terminology
 // (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx)
 type Node struct {
-	UID      string `json:"uid,omiempty"`
+	ID       string   `json:"id,omiempty"`
-	ID       string `json:"id,omiempty"`
+	Type     string   `json:"type,omiempty"`
-	Type     string `json:"type,omiempty"`
+	NData    string   `json:"ndata,omiempty"`
-	Data     string `json:"data,omiempty"`
+	Created  string   `json:"created,omiempty"`
-	Created  string `json:"created,omiempty"`
+	Modified string   `json:"modified,omiempty"`
-	Modified string `json:"modified,omiempty"`
+	DType    []string `json:"dgraph.type,omiempty"`
 }
 
 // BuildNode builds a node to send to MQ instance.
@@ -39,10 +39,9 @@ func BuildNode(flag string, dataType string, data string) *Node {
 	rfc3339time := t.Format(time.RFC3339)
 	uuid := uuid.New().String()
 	return &Node{
-		UID:      "_:" + flag + "--" + uuid,
 		ID:       flag + "--" + uuid,
 		Type:     dataType,
-		Data:     data,
+		NData:    data,
 		Created:  rfc3339time,
 		Modified: rfc3339time,
 	}
@@ -79,25 +78,24 @@ func SaveNode(filename string, node *Node) {
 
 // Edge defines a relation between two nodes.
 type Edge struct {
-	UID       string `json:"uid,omiempty"`
+	ID        string                 `json:"id,omiempty"`
-	ID        string `json:"id,omiempty"`
+	NodeOne   map[string]interface{} `json:"nodeOne,omiempty"`
-	NodeOneID string `json:"nodeOneID,omiempty"`
+	NodeTwo   map[string]interface{} `json:"nodeTwo,omiempty"`
-	NodeTwoID string `json:"nodeTwoID,omiempty"`
+	Timestamp string                 `json:"timestamp,omiempty"`
-	Timestamp string `json:"timestamp,omiempty"`
+	Source    string                 `json:"source,omiempty"`
-	Source    string `json:"source,omiempty"`
+	DType     []string               `json:"dgraph.type,omiempty"`
 }
 
 // BuildEdge build a send from two nodes with a given source type.
-func BuildEdge(source string, nodeOneUUID string, nodeTwoUUID string) *Edge {
+func BuildEdge(source string, nodeOne, nodeTwo map[string]interface{}) *Edge {
 	t := time.Now()
 	rfc3339time := t.Format(time.RFC3339)
 	uuid := uuid.New().String()
 	return &Edge{
-		UID:       "_:" + "edge--" + uuid,
 		ID:        "edge--" + uuid,
 		Source:    source,
-		NodeOneID: nodeOneUUID,
+		NodeOne:   nodeOne,
-		NodeTwoID: nodeTwoUUID,
+		NodeTwo:   nodeTwo,
 		Timestamp: rfc3339time,
 	}
 }
@@ -134,7 +132,6 @@ func SaveEdge(edge *Edge) {
 // CertStreamRaw is a wrapper around the stream function to unmarshall the
 // data receive in a Go structure.
 type CertStreamRaw struct {
-	UID      string           `json:"uid,omiempty"`
 	ID       string           `json:"id,omiempty"`
 	Type     string           `json:"type,omiempty"`
 	Data     CertStreamStruct `json:"data,omiempty"`
@@ -144,17 +141,16 @@ type CertStreamRaw struct {
 
 // CertNode represents our custom struct of data extraction from CertStream.
 type CertNode struct {
-	UID              string     `json:"uid,omiempty"`
+	ID               string        `json:"id,omiempty"`
-	ID               string     `json:"id,omiempty"`
+	Fingerprint      string        `json:"fingerprint,omiempty"`
-	Fingerprint      string     `json:"fingerprint,omiempty"`
+	NotBefore        string        `json:"notBefore,omiempty"`
-	NotBefore        string     `json:"notBefore,omiempty"`
+	NotAfter         string        `json:"notAfter,omiempty"`
-	NotAfter         string     `json:"notAfter,omiempty"`
+	CN               string        `json:"cn,omiempty"`
-	CN               string     `json:"cn,omiempty"`
+	SourceName       string        `json:"sourceName,omiempty"`
-	SourceName       string     `json:"sourceName,omiempty"`
+	SerialNumber     string        `json:"serialNumber,omiempty"`
-	SerialNumber     string     `json:"serialNumber,omiempty"`
+	BasicConstraints string        `json:"basicConstraints,omiempty"`
-	BasicConstraints string     `json:"basicConstraints,omiempty"`
+	Raw              CertStreamRaw `json:"raw,omiempty"`
-	RawUUID          string     `json:"rawUUID,omiempty"`
+	Chain            []CertNode    `json:"chainedTo,omiempty"`
-	Chain            []CertNode `json:"chainedTo,omiempty"`
 }
 
 // WrapCertStreamData is a wrapper around CertStreamStruct.
@@ -162,7 +158,6 @@ func WrapCertStreamData(data CertStreamStruct) *CertStreamRaw {
 	t := time.Now()
 	rfc3339time := t.Format(time.RFC3339)
 	return &CertStreamRaw{
-		UID:      "_:certstream--" + uuid.New().String(),
 		ID:       "certstream--" + uuid.New().String(),
 		Type:     "certstream_raw",
 		Data:     data,
@@ -174,29 +169,26 @@ func WrapCertStreamData(data CertStreamStruct) *CertStreamRaw {
 // BuildCertNode builds a custom node based on CertStream.
 func BuildCertNode(rawNode *CertStreamRaw) *CertNode {
 	main := &CertNode{
-		UID:              "_:certstream--" + uuid.New().String(),
 		ID:               "certstream--" + uuid.New().String(),
-		Fingerprint:      rawNode.Data.Data.LeafCert.Fingerprint,
+		Fingerprint:      rawNode.Data.CSData.LeafCert.Fingerprint,
-		NotBefore:        time.Unix(int64(rawNode.Data.Data.LeafCert.NotBefore), 0).Format(time.RFC3339),
+		NotBefore:        time.Unix(int64(rawNode.Data.CSData.LeafCert.NotBefore), 0).Format(time.RFC3339),
-		NotAfter:         time.Unix(int64(rawNode.Data.Data.LeafCert.NotAfter), 0).Format(time.RFC3339),
+		NotAfter:         time.Unix(int64(rawNode.Data.CSData.LeafCert.NotAfter), 0).Format(time.RFC3339),
-		CN:               rawNode.Data.Data.LeafCert.Subject.CN,
+		CN:               rawNode.Data.CSData.LeafCert.Subject.CN,
-		SourceName:       rawNode.Data.Data.Source.Name,
+		SourceName:       rawNode.Data.CSData.Source.Name,
-		BasicConstraints: rawNode.Data.Data.LeafCert.Extensions.BasicConstrains,
+		BasicConstraints: rawNode.Data.CSData.LeafCert.Extensions.BasicConstrains,
-		RawUUID:          rawNode.ID,
+		Raw:              *rawNode,
 	}
 
 	var res []CertNode
-	if len(rawNode.Data.Data.Chain) > 0 {
+	if len(rawNode.Data.CSData.Chain) > 0 {
 		chain := CertNode{
-			UID:              "_:certstream--" + uuid.New().String(),
 			ID:               "certstream--" + uuid.New().String(),
-			Fingerprint:      rawNode.Data.Data.LeafCert.Fingerprint,
+			Fingerprint:      rawNode.Data.CSData.LeafCert.Fingerprint,
-			NotBefore:        time.Unix(int64(rawNode.Data.Data.LeafCert.NotBefore), 0).Format(time.RFC3339),
+			NotBefore:        time.Unix(int64(rawNode.Data.CSData.LeafCert.NotBefore), 0).Format(time.RFC3339),
-			NotAfter:         time.Unix(int64(rawNode.Data.Data.LeafCert.NotAfter), 0).Format(time.RFC3339),
+			NotAfter:         time.Unix(int64(rawNode.Data.CSData.LeafCert.NotAfter), 0).Format(time.RFC3339),
-			CN:               rawNode.Data.Data.LeafCert.Subject.CN,
+			CN:               rawNode.Data.CSData.LeafCert.Subject.CN,
-			SourceName:       rawNode.Data.Data.Source.Name,
+			SourceName:       rawNode.Data.CSData.Source.Name,
-			BasicConstraints: rawNode.Data.Data.LeafCert.Extensions.BasicConstrains,
+			BasicConstraints: rawNode.Data.CSData.LeafCert.Extensions.BasicConstrains,
-			RawUUID:          rawNode.ID,
 		}
 		res = append(res, chain)
 	}

plugins/certstream.go

@@ -3,9 +3,11 @@ package plugins
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"sync"
 
 	"github.com/CaliDog/certstream-go"
+	"github.com/christalib/structs"
 	"github.com/dgraph-io/dgo/v2"
 	"github.com/dgraph-io/dgo/v2/protos/api"
 	"github.com/jmoiron/jsonq"
@@ -64,25 +66,26 @@ func (c *CertStreamPlugin) doRun(graphClient *dgo.Dgraph) {
 		case jq := <-c.Stream:
 			if data, err := models.ExtractCertFromStream(jq); err == nil {
 
-				allDomains := data.Data.LeafCert.AllDomains
+				allDomains := data.CSData.LeafCert.AllDomains
 				for _, domain := range allDomains {
 					if filters.RunDomainFilters(domain) {
 						rawNode := models.WrapCertStreamData(*data)
-						models.SaveCertStreamRaw("raw_certstream.json", rawNode)
+						// models.SaveCertStreamRaw("raw_certstream.json", rawNode)
 
 						certNode := models.BuildCertNode(rawNode)
 						models.SaveCertNode("cert_nodes.json", certNode)
 						mainNode := models.BuildNode("node", "certstream", certNode.ID)
 						models.SaveNode("nodes.json", mainNode)
-						edge := models.BuildEdge("certstream", rawNode.ID, mainNode.ID)
+						rawEdge := models.BuildEdge("certstream", structs.Map(rawNode), structs.Map(mainNode))
-						models.SaveEdge(edge)
+						models.SaveEdge(rawEdge)
-						edge = models.BuildEdge("certstream", mainNode.ID, certNode.ID)
+						edge := models.BuildEdge("certstream", structs.Map(mainNode), structs.Map(certNode))
 						models.SaveEdge(edge)
 						// saveSingleValues(conn, "certstream", "domain", certNode.ID, domain)
 
 						mu := &api.Mutation{
 							CommitNow: true,
 						}
+
 						marshaled, err := json.Marshal(mainNode)
 						if err != nil {
 							logrus.Fatal(err)
@@ -92,21 +95,51 @@ func (c *CertStreamPlugin) doRun(graphClient *dgo.Dgraph) {
 						if err != nil {
 							logrus.Fatal(err)
 						}
-						// variables := map[string]string{"$id": assigned.Uids[mainNode.ID]}
+
-						// q := `query Node($id: string){
+						variables := map[string]string{"$id": mainNode.ID}
-						// node(func: uid($id)) {
+						q := `query Node($id: string){
-						// uid
+						 node(func: eq(id, $id)) {
-						// ID
+						 uid
-						// type
+						 id
-						// data
+						 type
-						// dreated
+						 ndata
-						// modified
+						 created
-						// }
+						 modified
-						// }`
+						 }
-						// _, err := graphClient.NewTxn().QueryWithVars(context.Background(), q, variables)
+						 }`
-						// if err != nil {
+						node, err := graphClient.NewTxn().QueryWithVars(context.Background(), q, variables)
-						// logrus.Fatal(err)
+						if err != nil {
-						// }
+							logrus.Fatal(err)
+						}
+
+						marshaled, err = json.Marshal(certNode)
+						if err != nil {
+							logrus.Fatal(err)
+						}
+						mu.SetJson = marshaled
+						_, err = graphClient.NewTxn().Mutate(context.Background(), mu)
+						if err != nil {
+							logrus.Fatal(err)
+						}
+						query := `
+query Node($mainNodeID: string, $subNodeID: string) {
+node as var(func: eq(id, $mainNodeID))
+						}
+						`
+						mu = &api.Mutation{
+							SetNquads: []byte(`uid(node) <CertNode> "$subNodeID" .`),
+						}
+						req := &api.Request{
+							Query:     query,
+							Mutations: []*api.Mutation{mu},
+							CommitNow: true,
+							Vars:      map[string]string{"$mainNodeID": node.Uids[mainNode.ID], "$subNodeID": certNode.ID},
+						}
+						res, err := graphClient.NewTxn().Do(context.Background(), req)
+						if err != nil {
+							logrus.Error(err)
+						}
+						fmt.Println(res)
 
 					}
 				}

plugins/shodan.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"sync"
 
+	"github.com/christalib/structs"
 	"github.com/ns3777k/go-shodan/v4/shodan"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
@@ -85,7 +86,7 @@ func (s *ShodanPlugin) doRun() {
 						models.SaveShodanNode("raw_shodan.json", shodanNode)
 						node := models.BuildNode("shodan", "shodan_stream", shodanNode.ID)
 						models.SaveNode("nodes.json", node)
-						edge := models.BuildEdge("shodan", shodanNode.ID, node.ID)
+						edge := models.BuildEdge("shodan", structs.Map(shodanNode), structs.Map(node))
 						models.SaveEdge(edge)
 					}
 				} else {