diff --git a/go.mod b/go.mod index ea68c0a..91292a1 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( dmitri.shuralyov.com/gpu/mtl v0.0.0-20191203043605-d42048ed14fd // indirect github.com/CaliDog/certstream-go v0.0.0-20180219203951-6016c5462366 github.com/DataDog/zstd v1.4.4 // indirect + github.com/christalib/structs v1.1.0 github.com/cncf/udpa/go v0.0.0-20200124205748-db4b343e48c1 // indirect github.com/dgraph-io/dgo v1.0.0 github.com/dgraph-io/dgo/v2 v2.2.0 diff --git a/go.sum b/go.sum index 8aaab3c..96d4a58 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/christalib/structs v1.1.0 h1:vWXpO00DCoGj5jYo+HV2pvNmoDsKHAAHPdkK2aTZjXE= +github.com/christalib/structs v1.1.0/go.mod h1:sKgAVbcnUvqeaSB3E5Y8eRgaTyo1DFz7lG2EKiSy7SU= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -112,6 +114,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= diff --git a/graph/main.go b/graph/main.go index 6e42b11..a2a3018 100644 --- a/graph/main.go +++ b/graph/main.go @@ -26,35 +26,38 @@ func ConnectToDgraph() (*dgo.Dgraph, error) { func setupDgraphSchema(c *dgo.Dgraph) error { err := c.Alter(context.Background(), &api.Operation{ - DropAll: true, + DropOp: api.Operation_ALL, }) + if err != nil { + return err + } err = c.Alter(context.Background(), &api.Operation{ Schema: ` id: string @index(term) . type: string @index(term) . -data: string . -nodeOneID: uid . -nodeTwoID: uid . -source: string @index(term) . +ndata: string . +nodeOne: string @index(term) . +nodeTwo: string @index(term) . +sourceName: string @index(term) . timestamp: string . -created: dateTime . -modified: dateTime . +created: string . +modified: string . type Node { id type -data +ndata created modified } type Edge { id -nodeOneID -nodeTwoID +nodeOne +nodeTwo timestamp -source +sourceName } fingerprint: string . @@ -65,6 +68,7 @@ sourceName: string . serialNumber: string . basicConstraints: string . chain: uid . +csdata: uid . type CertNode { id @@ -83,7 +87,7 @@ id type created modified -data +csdata } type PasteNode { @@ -91,7 +95,7 @@ id type created modified -data +ndata } meta: uid . @@ -105,7 +109,7 @@ full type ShodanNode { id type -data +ndata created modified } @@ -113,7 +117,7 @@ modified type BalboaNode { id type -data +ndata created modified } diff --git a/main.go b/main.go index 278dacf..370559f 100644 --- a/main.go +++ b/main.go @@ -51,22 +51,25 @@ func main() { c := plugins.CertStreamPlugin{} if ok := c.Initialize(); !ok { logrus.Info("certstream plugin not activated") + } else { + c.Run(&wg, dgraphClient) } - c.Run(&wg, dgraphClient) // pastebin p := plugins.PastebinPlugin{} if ok := p.Initialize(); !ok { logrus.Info("pastebin plugin not activated") + } else { + p.Run(&wg) } - p.Run(&wg) // shodan s := plugins.ShodanPlugin{} if ok := s.Initialize(); !ok { logrus.Info("shodan plugin not activated") + } else { + p.Run(&wg) } - p.Run(&wg) go func() { <-stopChan diff --git a/models/cerstream.go b/models/cerstream.go index 265bb2f..b332913 100644 --- a/models/cerstream.go +++ b/models/cerstream.go @@ -61,7 +61,7 @@ type CertStreamData struct { // and the content is stored in Data. type CertStreamStruct struct { MessageType string `json:"message_data"` - Data CertStreamData `json:"data"` + CSData CertStreamData `json:"csdata"` } // ExtractCertFromStream builds the structures before saving them. It uses the @@ -117,7 +117,7 @@ func ExtractCertFromStream(input jsonq.JsonQuery) (*CertStreamStruct, error) { res := CertStreamStruct{ MessageType: messageType, - Data: csd, + CSData: csd, } return &res, nil diff --git a/models/main.go b/models/main.go index 90ca7cd..9d0ad36 100644 --- a/models/main.go +++ b/models/main.go @@ -25,12 +25,12 @@ Structure of this file: // Styx terminology // (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx) type Node struct { - UID string `json:"uid,omiempty"` - ID string `json:"id,omiempty"` - Type string `json:"type,omiempty"` - Data string `json:"data,omiempty"` - Created string `json:"created,omiempty"` - Modified string `json:"modified,omiempty"` + ID string `json:"id,omiempty"` + Type string `json:"type,omiempty"` + NData string `json:"ndata,omiempty"` + Created string `json:"created,omiempty"` + Modified string `json:"modified,omiempty"` + DType []string `json:"dgraph.type,omiempty"` } // BuildNode builds a node to send to MQ instance. @@ -39,10 +39,9 @@ func BuildNode(flag string, dataType string, data string) *Node { rfc3339time := t.Format(time.RFC3339) uuid := uuid.New().String() return &Node{ - UID: "_:" + flag + "--" + uuid, ID: flag + "--" + uuid, Type: dataType, - Data: data, + NData: data, Created: rfc3339time, Modified: rfc3339time, } @@ -79,25 +78,24 @@ func SaveNode(filename string, node *Node) { // Edge defines a relation between two nodes. type Edge struct { - UID string `json:"uid,omiempty"` - ID string `json:"id,omiempty"` - NodeOneID string `json:"nodeOneID,omiempty"` - NodeTwoID string `json:"nodeTwoID,omiempty"` - Timestamp string `json:"timestamp,omiempty"` - Source string `json:"source,omiempty"` + ID string `json:"id,omiempty"` + NodeOne map[string]interface{} `json:"nodeOne,omiempty"` + NodeTwo map[string]interface{} `json:"nodeTwo,omiempty"` + Timestamp string `json:"timestamp,omiempty"` + Source string `json:"source,omiempty"` + DType []string `json:"dgraph.type,omiempty"` } // BuildEdge build a send from two nodes with a given source type. -func BuildEdge(source string, nodeOneUUID string, nodeTwoUUID string) *Edge { +func BuildEdge(source string, nodeOne, nodeTwo map[string]interface{}) *Edge { t := time.Now() rfc3339time := t.Format(time.RFC3339) uuid := uuid.New().String() return &Edge{ - UID: "_:" + "edge--" + uuid, ID: "edge--" + uuid, Source: source, - NodeOneID: nodeOneUUID, - NodeTwoID: nodeTwoUUID, + NodeOne: nodeOne, + NodeTwo: nodeTwo, Timestamp: rfc3339time, } } @@ -134,7 +132,6 @@ func SaveEdge(edge *Edge) { // CertStreamRaw is a wrapper around the stream function to unmarshall the // data receive in a Go structure. type CertStreamRaw struct { - UID string `json:"uid,omiempty"` ID string `json:"id,omiempty"` Type string `json:"type,omiempty"` Data CertStreamStruct `json:"data,omiempty"` @@ -144,17 +141,16 @@ type CertStreamRaw struct { // CertNode represents our custom struct of data extraction from CertStream. type CertNode struct { - UID string `json:"uid,omiempty"` - ID string `json:"id,omiempty"` - Fingerprint string `json:"fingerprint,omiempty"` - NotBefore string `json:"notBefore,omiempty"` - NotAfter string `json:"notAfter,omiempty"` - CN string `json:"cn,omiempty"` - SourceName string `json:"sourceName,omiempty"` - SerialNumber string `json:"serialNumber,omiempty"` - BasicConstraints string `json:"basicConstraints,omiempty"` - RawUUID string `json:"rawUUID,omiempty"` - Chain []CertNode `json:"chainedTo,omiempty"` + ID string `json:"id,omiempty"` + Fingerprint string `json:"fingerprint,omiempty"` + NotBefore string `json:"notBefore,omiempty"` + NotAfter string `json:"notAfter,omiempty"` + CN string `json:"cn,omiempty"` + SourceName string `json:"sourceName,omiempty"` + SerialNumber string `json:"serialNumber,omiempty"` + BasicConstraints string `json:"basicConstraints,omiempty"` + Raw CertStreamRaw `json:"raw,omiempty"` + Chain []CertNode `json:"chainedTo,omiempty"` } // WrapCertStreamData is a wrapper around CertStreamStruct. @@ -162,7 +158,6 @@ func WrapCertStreamData(data CertStreamStruct) *CertStreamRaw { t := time.Now() rfc3339time := t.Format(time.RFC3339) return &CertStreamRaw{ - UID: "_:certstream--" + uuid.New().String(), ID: "certstream--" + uuid.New().String(), Type: "certstream_raw", Data: data, @@ -174,29 +169,26 @@ func WrapCertStreamData(data CertStreamStruct) *CertStreamRaw { // BuildCertNode builds a custom node based on CertStream. func BuildCertNode(rawNode *CertStreamRaw) *CertNode { main := &CertNode{ - UID: "_:certstream--" + uuid.New().String(), ID: "certstream--" + uuid.New().String(), - Fingerprint: rawNode.Data.Data.LeafCert.Fingerprint, - NotBefore: time.Unix(int64(rawNode.Data.Data.LeafCert.NotBefore), 0).Format(time.RFC3339), - NotAfter: time.Unix(int64(rawNode.Data.Data.LeafCert.NotAfter), 0).Format(time.RFC3339), - CN: rawNode.Data.Data.LeafCert.Subject.CN, - SourceName: rawNode.Data.Data.Source.Name, - BasicConstraints: rawNode.Data.Data.LeafCert.Extensions.BasicConstrains, - RawUUID: rawNode.ID, + Fingerprint: rawNode.Data.CSData.LeafCert.Fingerprint, + NotBefore: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotBefore), 0).Format(time.RFC3339), + NotAfter: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotAfter), 0).Format(time.RFC3339), + CN: rawNode.Data.CSData.LeafCert.Subject.CN, + SourceName: rawNode.Data.CSData.Source.Name, + BasicConstraints: rawNode.Data.CSData.LeafCert.Extensions.BasicConstrains, + Raw: *rawNode, } var res []CertNode - if len(rawNode.Data.Data.Chain) > 0 { + if len(rawNode.Data.CSData.Chain) > 0 { chain := CertNode{ - UID: "_:certstream--" + uuid.New().String(), ID: "certstream--" + uuid.New().String(), - Fingerprint: rawNode.Data.Data.LeafCert.Fingerprint, - NotBefore: time.Unix(int64(rawNode.Data.Data.LeafCert.NotBefore), 0).Format(time.RFC3339), - NotAfter: time.Unix(int64(rawNode.Data.Data.LeafCert.NotAfter), 0).Format(time.RFC3339), - CN: rawNode.Data.Data.LeafCert.Subject.CN, - SourceName: rawNode.Data.Data.Source.Name, - BasicConstraints: rawNode.Data.Data.LeafCert.Extensions.BasicConstrains, - RawUUID: rawNode.ID, + Fingerprint: rawNode.Data.CSData.LeafCert.Fingerprint, + NotBefore: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotBefore), 0).Format(time.RFC3339), + NotAfter: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotAfter), 0).Format(time.RFC3339), + CN: rawNode.Data.CSData.LeafCert.Subject.CN, + SourceName: rawNode.Data.CSData.Source.Name, + BasicConstraints: rawNode.Data.CSData.LeafCert.Extensions.BasicConstrains, } res = append(res, chain) } diff --git a/plugins/certstream.go b/plugins/certstream.go index 786a093..ab97c8a 100644 --- a/plugins/certstream.go +++ b/plugins/certstream.go @@ -3,9 +3,11 @@ package plugins import ( "context" "encoding/json" + "fmt" "sync" "github.com/CaliDog/certstream-go" + "github.com/christalib/structs" "github.com/dgraph-io/dgo/v2" "github.com/dgraph-io/dgo/v2/protos/api" "github.com/jmoiron/jsonq" @@ -64,25 +66,26 @@ func (c *CertStreamPlugin) doRun(graphClient *dgo.Dgraph) { case jq := <-c.Stream: if data, err := models.ExtractCertFromStream(jq); err == nil { - allDomains := data.Data.LeafCert.AllDomains + allDomains := data.CSData.LeafCert.AllDomains for _, domain := range allDomains { if filters.RunDomainFilters(domain) { rawNode := models.WrapCertStreamData(*data) - models.SaveCertStreamRaw("raw_certstream.json", rawNode) + // models.SaveCertStreamRaw("raw_certstream.json", rawNode) certNode := models.BuildCertNode(rawNode) models.SaveCertNode("cert_nodes.json", certNode) mainNode := models.BuildNode("node", "certstream", certNode.ID) models.SaveNode("nodes.json", mainNode) - edge := models.BuildEdge("certstream", rawNode.ID, mainNode.ID) - models.SaveEdge(edge) - edge = models.BuildEdge("certstream", mainNode.ID, certNode.ID) + rawEdge := models.BuildEdge("certstream", structs.Map(rawNode), structs.Map(mainNode)) + models.SaveEdge(rawEdge) + edge := models.BuildEdge("certstream", structs.Map(mainNode), structs.Map(certNode)) models.SaveEdge(edge) // saveSingleValues(conn, "certstream", "domain", certNode.ID, domain) mu := &api.Mutation{ CommitNow: true, } + marshaled, err := json.Marshal(mainNode) if err != nil { logrus.Fatal(err) @@ -92,21 +95,51 @@ func (c *CertStreamPlugin) doRun(graphClient *dgo.Dgraph) { if err != nil { logrus.Fatal(err) } - // variables := map[string]string{"$id": assigned.Uids[mainNode.ID]} - // q := `query Node($id: string){ - // node(func: uid($id)) { - // uid - // ID - // type - // data - // dreated - // modified - // } - // }` - // _, err := graphClient.NewTxn().QueryWithVars(context.Background(), q, variables) - // if err != nil { - // logrus.Fatal(err) - // } + + variables := map[string]string{"$id": mainNode.ID} + q := `query Node($id: string){ + node(func: eq(id, $id)) { + uid + id + type + ndata + created + modified + } + }` + node, err := graphClient.NewTxn().QueryWithVars(context.Background(), q, variables) + if err != nil { + logrus.Fatal(err) + } + + marshaled, err = json.Marshal(certNode) + if err != nil { + logrus.Fatal(err) + } + mu.SetJson = marshaled + _, err = graphClient.NewTxn().Mutate(context.Background(), mu) + if err != nil { + logrus.Fatal(err) + } + query := ` +query Node($mainNodeID: string, $subNodeID: string) { +node as var(func: eq(id, $mainNodeID)) + } + ` + mu = &api.Mutation{ + SetNquads: []byte(`uid(node) "$subNodeID" .`), + } + req := &api.Request{ + Query: query, + Mutations: []*api.Mutation{mu}, + CommitNow: true, + Vars: map[string]string{"$mainNodeID": node.Uids[mainNode.ID], "$subNodeID": certNode.ID}, + } + res, err := graphClient.NewTxn().Do(context.Background(), req) + if err != nil { + logrus.Error(err) + } + fmt.Println(res) } } diff --git a/plugins/shodan.go b/plugins/shodan.go index fb02eef..de4e5cf 100644 --- a/plugins/shodan.go +++ b/plugins/shodan.go @@ -4,6 +4,7 @@ import ( "fmt" "sync" + "github.com/christalib/structs" "github.com/ns3777k/go-shodan/v4/shodan" "github.com/sirupsen/logrus" "github.com/spf13/viper" @@ -85,7 +86,7 @@ func (s *ShodanPlugin) doRun() { models.SaveShodanNode("raw_shodan.json", shodanNode) node := models.BuildNode("shodan", "shodan_stream", shodanNode.ID) models.SaveNode("nodes.json", node) - edge := models.BuildEdge("shodan", shodanNode.ID, node.ID) + edge := models.BuildEdge("shodan", structs.Map(shodanNode), structs.Map(node)) models.SaveEdge(edge) } } else {