From f61fe566a56f021f3a551bb0f228b9384b0479b4 Mon Sep 17 00:00:00 2001 From: Christopher Talib Date: Wed, 4 Mar 2020 15:16:59 +0100 Subject: [PATCH] Basic connection to Dgraph DB The first work and input to the graph db is set up in this work. It's for the moment very basic and doesn't cover relations and only works for certstream data. --- go.mod | 1 + go.sum | 6 ++ graph/main.go | 137 +++++++++++++++++++++--------------------- main.go | 4 +- models/main.go | 30 +++++---- plugins/certstream.go | 41 ++++++++++++- 6 files changed, 134 insertions(+), 85 deletions(-) diff --git a/go.mod b/go.mod index a57f271..ea68c0a 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/DataDog/zstd v1.4.4 // indirect github.com/cncf/udpa/go v0.0.0-20200124205748-db4b343e48c1 // indirect github.com/dgraph-io/dgo v1.0.0 + github.com/dgraph-io/dgo/v2 v2.2.0 github.com/envoyproxy/go-control-plane v0.9.4 // indirect github.com/frankban/quicktest v1.7.2 // indirect github.com/golang/mock v1.4.1 // indirect diff --git a/go.sum b/go.sum index b04447a..8aaab3c 100644 --- a/go.sum +++ b/go.sum @@ -56,6 +56,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/dgraph-io/dgo v1.0.0 h1:DRuI66G+j0XWDOXly4v5PSk2dGkbIopAZIirRjq7lzI= github.com/dgraph-io/dgo v1.0.0/go.mod h1:6K5zUB6Lsml4SEStX+fPzGhJtCLX9XxbkHJLsGOXS1E= +github.com/dgraph-io/dgo/v2 v2.2.0 h1:qYbm6mEF3wuKiRpgNOldk6PmPbBJFwj6vL7I7dTSdyc= +github.com/dgraph-io/dgo/v2 v2.2.0/go.mod h1:LJCkLxm5fUMcU+yb8gHFjHt7ChgNuz3YnQQ6MQkmscI= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= @@ -84,6 +86,7 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -146,6 +149,7 @@ github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= +github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -337,6 +341,7 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -358,6 +363,7 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20181030221726-6c7e314b6563/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= diff --git a/graph/main.go b/graph/main.go index 1116cf2..6e42b11 100644 --- a/graph/main.go +++ b/graph/main.go @@ -3,118 +3,119 @@ package graph import ( "context" - "github.com/dgraph-io/dgo" - "github.com/dgraph-io/dgo/protos/api" + "github.com/dgraph-io/dgo/v2" + "github.com/dgraph-io/dgo/v2/protos/api" "google.golang.org/grpc" ) -func ConnectToDgraph() error { +func ConnectToDgraph() (*dgo.Dgraph, error) { conn, err := grpc.Dial("localhost:9080", grpc.WithInsecure()) if err != nil { - return err + return nil, err } - defer conn.Close() dgraphClient := dgo.NewDgraphClient(api.NewDgraphClient(conn)) err = setupDgraphSchema(dgraphClient) if err != nil { - return err + return nil, err } - return nil + return dgraphClient, nil } func setupDgraphSchema(c *dgo.Dgraph) error { err := c.Alter(context.Background(), &api.Operation{ DropAll: true, + }) + + err = c.Alter(context.Background(), &api.Operation{ Schema: ` -ID: string @index(term) . -Type: string @index(term) . -Data: string . -NodeOneID: [uid] . -NodeTwoID: [uid] . -Source: string @index(term) . -Timestamp: string . -Created: dateTime . -Modified: dateTime . +id: string @index(term) . +type: string @index(term) . +data: string . +nodeOneID: uid . +nodeTwoID: uid . +source: string @index(term) . +timestamp: string . +created: dateTime . +modified: dateTime . type Node { -ID -Type -Data -Created -Modified +id +type +data +created +modified } type Edge { -ID -NodeOneID -NodeTwoID -Timestamp -Source +id +nodeOneID +nodeTwoID +timestamp +source } -Fingerprint: string . -NotBefore: string . -NotAfter: string . -CN: string . -SourceName: string . -SerialNumber: string . -BasicConstraints: string . -Chain: [uid] . -Data: [uid] . +fingerprint: string . +notBefore: string . +notAfter: string . +cn: string . +sourceName: string . +serialNumber: string . +basicConstraints: string . +chain: uid . type CertNode { -ID -Fingerprint -NotBefore -NotAfter -CN -SourceName -SerialNumber -BasicConstraints -Chain +id +fingerprint +notBefore +notAfter +cn +sourceName +serialNumber +basicConstraints +chain } type CertRaw { -ID -Type -Created -Modified -Data +id +type +created +modified +data } type PasteNode { -ID -Type -Created -Modified -Data +id +type +created +modified +data } -Meta: [uid] . -Full: string . +meta: uid . +full: string . type FullPaste { -Meta -Full +meta +full } type ShodanNode { -ID -Type -Data -Created -Modified +id +type +data +created +modified } type BalboaNode { -ID -Type -Data -Created -Modified +id +type +data +created +modified } `}) if err != nil { diff --git a/main.go b/main.go index 487e6aa..278dacf 100644 --- a/main.go +++ b/main.go @@ -36,7 +36,7 @@ func main() { // panic(err) // } - err = graph.ConnectToDgraph() + dgraphClient, err := graph.ConnectToDgraph() if err != nil { logrus.WithField("err", err).Error("error initialising the graph database") } @@ -52,7 +52,7 @@ func main() { if ok := c.Initialize(); !ok { logrus.Info("certstream plugin not activated") } - c.Run(&wg) + c.Run(&wg, dgraphClient) // pastebin p := plugins.PastebinPlugin{} diff --git a/models/main.go b/models/main.go index 32494b0..90ca7cd 100644 --- a/models/main.go +++ b/models/main.go @@ -25,7 +25,7 @@ Structure of this file: // Styx terminology // (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx) type Node struct { - Uid string `json:"uid,omiempty"` + UID string `json:"uid,omiempty"` ID string `json:"id,omiempty"` Type string `json:"type,omiempty"` Data string `json:"data,omiempty"` @@ -37,9 +37,10 @@ type Node struct { func BuildNode(flag string, dataType string, data string) *Node { t := time.Now() rfc3339time := t.Format(time.RFC3339) + uuid := uuid.New().String() return &Node{ - Uid: "_:" + flag + "--" + uuid.New().String(), - ID: flag + "--" + uuid.New().String(), + UID: "_:" + flag + "--" + uuid, + ID: flag + "--" + uuid, Type: dataType, Data: data, Created: rfc3339time, @@ -78,7 +79,7 @@ func SaveNode(filename string, node *Node) { // Edge defines a relation between two nodes. type Edge struct { - Uid string `json:"uid,omiempty"` + UID string `json:"uid,omiempty"` ID string `json:"id,omiempty"` NodeOneID string `json:"nodeOneID,omiempty"` NodeTwoID string `json:"nodeTwoID,omiempty"` @@ -90,9 +91,10 @@ type Edge struct { func BuildEdge(source string, nodeOneUUID string, nodeTwoUUID string) *Edge { t := time.Now() rfc3339time := t.Format(time.RFC3339) + uuid := uuid.New().String() return &Edge{ - Uid: "_:" + "edge--" + uuid.New().String(), - ID: "edge--" + uuid.New().String(), + UID: "_:" + "edge--" + uuid, + ID: "edge--" + uuid, Source: source, NodeOneID: nodeOneUUID, NodeTwoID: nodeTwoUUID, @@ -132,16 +134,17 @@ func SaveEdge(edge *Edge) { // CertStreamRaw is a wrapper around the stream function to unmarshall the // data receive in a Go structure. type CertStreamRaw struct { - ID string `json:"id"` - Type string `json:"type"` - Data CertStreamStruct `json:"data"` - Created string `json:"created"` - Modified string `json:"modified"` + UID string `json:"uid,omiempty"` + ID string `json:"id,omiempty"` + Type string `json:"type,omiempty"` + Data CertStreamStruct `json:"data,omiempty"` + Created string `json:"created,omiempty"` + Modified string `json:"modified,omiempty"` } // CertNode represents our custom struct of data extraction from CertStream. type CertNode struct { - Uid string `json:"uid,omiempty"` + UID string `json:"uid,omiempty"` ID string `json:"id,omiempty"` Fingerprint string `json:"fingerprint,omiempty"` NotBefore string `json:"notBefore,omiempty"` @@ -159,6 +162,7 @@ func WrapCertStreamData(data CertStreamStruct) *CertStreamRaw { t := time.Now() rfc3339time := t.Format(time.RFC3339) return &CertStreamRaw{ + UID: "_:certstream--" + uuid.New().String(), ID: "certstream--" + uuid.New().String(), Type: "certstream_raw", Data: data, @@ -170,6 +174,7 @@ func WrapCertStreamData(data CertStreamStruct) *CertStreamRaw { // BuildCertNode builds a custom node based on CertStream. func BuildCertNode(rawNode *CertStreamRaw) *CertNode { main := &CertNode{ + UID: "_:certstream--" + uuid.New().String(), ID: "certstream--" + uuid.New().String(), Fingerprint: rawNode.Data.Data.LeafCert.Fingerprint, NotBefore: time.Unix(int64(rawNode.Data.Data.LeafCert.NotBefore), 0).Format(time.RFC3339), @@ -183,6 +188,7 @@ func BuildCertNode(rawNode *CertStreamRaw) *CertNode { var res []CertNode if len(rawNode.Data.Data.Chain) > 0 { chain := CertNode{ + UID: "_:certstream--" + uuid.New().String(), ID: "certstream--" + uuid.New().String(), Fingerprint: rawNode.Data.Data.LeafCert.Fingerprint, NotBefore: time.Unix(int64(rawNode.Data.Data.LeafCert.NotBefore), 0).Format(time.RFC3339), diff --git a/plugins/certstream.go b/plugins/certstream.go index fe9cff6..99dca5d 100644 --- a/plugins/certstream.go +++ b/plugins/certstream.go @@ -1,9 +1,14 @@ package plugins import ( + "context" + "encoding/json" + "fmt" "sync" "github.com/CaliDog/certstream-go" + "github.com/dgraph-io/dgo/v2" + "github.com/dgraph-io/dgo/v2/protos/api" "github.com/jmoiron/jsonq" "github.com/sirupsen/logrus" "github.com/spf13/viper" @@ -34,11 +39,11 @@ func (c *CertStreamPlugin) Initialize() bool { } // Run runs the Certstream plugin. -func (c *CertStreamPlugin) Run(wg *sync.WaitGroup) { +func (c *CertStreamPlugin) Run(wg *sync.WaitGroup, dgraphClient *dgo.Dgraph) { if !c.Running { c.StopChan = make(chan bool) wg.Add(1) - go c.doRun() + go c.doRun(dgraphClient) c.Running = true } } @@ -54,7 +59,7 @@ func (c *CertStreamPlugin) Stop(wg *sync.WaitGroup) { } } -func (c *CertStreamPlugin) doRun() { +func (c *CertStreamPlugin) doRun(graphClient *dgo.Dgraph) { for { select { case jq := <-c.Stream: @@ -75,6 +80,36 @@ func (c *CertStreamPlugin) doRun() { edge = models.BuildEdge("certstream", mainNode.ID, certNode.ID) models.SaveEdge(edge) // saveSingleValues(conn, "certstream", "domain", certNode.ID, domain) + + mu := &api.Mutation{ + CommitNow: true, + } + marshaled, err := json.Marshal(mainNode) + if err != nil { + logrus.Fatal(err) + } + mu.SetJson = marshaled + assigned, err := graphClient.NewTxn().Mutate(context.Background(), mu) + if err != nil { + logrus.Fatal(err) + } + variables := map[string]string{"$id": assigned.Uids[mainNode.ID]} + q := `query Node($id: string){ +node(func: uid($id)) { +uid +ID +Type +Data +Created +Modified +} +}` + resp, err := graphClient.NewTxn().QueryWithVars(context.Background(), q, variables) + if err != nil { + logrus.Fatal(err) + } + fmt.Println(resp) + } }