diff --git a/DEMO.md b/DEMO.md index 054d477..c67ed40 100644 --- a/DEMO.md +++ b/DEMO.md @@ -2,14 +2,17 @@ ```graphql { - Node(func: eq(type, "matcher")){ + Node(func: eq(nodeType, "matcher")){ id target - type + nodeType full nodes { uid full + meta { + full_url + } } } } @@ -19,7 +22,7 @@ { Node(func: has(nodes) ) { uid - type + nodeType target timestamp nodes { @@ -33,6 +36,23 @@ } ``` +```graphql +{ + Node(func: eq(nodeType, "certstream")){ + id + nodeType + certNode { + cn + sourceName + fingerprint + notBefore + notAfter + } + } + } + +``` + ## Notes * There is TOO MUCH junk data diff --git a/README.md b/README.md index d9eebbb..557720b 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ docker-compose up -d # or the other docker command # build the loader helper binary go build gitlab.dcso.lolcat/LABS/styx/cmd/iocloader # update the IOC list while the programm is already running -./iocloader +./iocloader ``` *Note*: if you have issues with the docker compose, make sure it runs on the @@ -79,7 +79,7 @@ query { uid ndata modified - type + nodeType id } } @@ -94,7 +94,7 @@ query { uid created modified - type + nodeType ndata certNode { uid @@ -126,7 +126,7 @@ query { uid created modified - type + nodeType ndata pasteNode { id @@ -161,7 +161,7 @@ query { uid created modified - type + nodeType full } } @@ -194,7 +194,7 @@ Node --[Edge]-- Node ```go type Node struct { ID string `json:"id"` - Type string `json:"type"` + NodeType string `json:"nodeType"` Data string `json:"data"` // For plain Node, the data is the ID of another typed node or a unique value like a domain or a host name. Created string `json:"created"` Modified string `json:"modified"` @@ -221,7 +221,7 @@ Node -- CertNode -- CertStreamRaw // data receive in a Go structure. type CertStreamRaw struct { ID string `json:"id"` - Type string `json:"type"` + NodeType string `json:"nodeType"` Data CertStreamStruct `json:"data"` Created string `json:"created"` Modified string `json:"modified"` @@ -251,7 +251,7 @@ Node -- PasteNode -- FullPaste // PasteNode is a node from PasteBin. type PasteNode struct { ID string `json:"id"` - Type string `json:"type"` + NodeType string `json:"nodeType"` Data FullPaste `json:"data"` Created string `json:"create"` Modified string `json:"modified"` @@ -259,9 +259,9 @@ type PasteNode struct { // FullPaste wrapes meta and information from Pastebin. type FullPaste struct { - Meta PasteMeta `json:"meta"` - Full string `json:"full"` - Type string `json:"type"` + Meta PasteMeta `json:"meta"` + Full string `json:"full"` + NodeType string `json:"nodeType"` } ``` @@ -271,11 +271,11 @@ Node -- ShodanNode -- Node(s) (hostnames and domains) ```go type ShodanNode struct { - ID string `json:"id"` - Type string `json:"type"` - Data *shodan.HostData `json:"data"` - Created string `json:"created"` - Modified string `json:"modified"` + ID string `json:"id"` + NodeType string `json:"nodeType"` + Data *shodan.HostData `json:"data"` + Created string `json:"created"` + Modified string `json:"modified"` } ``` @@ -288,10 +288,10 @@ Node -- ShodanNode -- Node (domain) -- BalboaNode ```go type BalboaNode struct { - ID string `json:"id"` - Type string `json:"type"` - Data []balboa.Entries `json:"data"` - Created string `json:"created"` - Modified string `json:"modified"` + ID string `json:"id"` + NodeType string `json:"nodeType"` + Data []balboa.Entries `json:"data"` + Created string `json:"created"` + Modified string `json:"modified"` } ``` diff --git a/cmd/iocloader/main.go b/cmd/iocloader/main.go index 395028d..c13b338 100644 --- a/cmd/iocloader/main.go +++ b/cmd/iocloader/main.go @@ -55,7 +55,7 @@ func loadTargets(graphClient *dgo.Dgraph) error { Timestamp: rfc3339time, Target: scanner.Text(), Nodes: []models.Node{}, - Type: "matcher", + NodeType: "matcher", } ctx := context.Background() query := `query eq($a: string){ diff --git a/connectors_test.go b/connectors_test.go index 0b03dba..9c592f0 100644 --- a/connectors_test.go +++ b/connectors_test.go @@ -1,4 +1,4 @@ -package main +package styx import ( "testing" diff --git a/graph/main.go b/graph/main.go index ebeaab0..539e2cf 100644 --- a/graph/main.go +++ b/graph/main.go @@ -37,7 +37,7 @@ func setupDgraphSchema(c *dgo.Dgraph) error { err := c.Alter(context.Background(), &api.Operation{ Schema: ` id: string @index(exact, term) . -type: string @index(term) . +nodeType: string @index(term) . ndata: string . nodeOne: uid . nodeTwo: uid . @@ -53,7 +53,7 @@ match: uid . type Node { id: string -type: string +nodeType: string ndata: string created: string modified: string @@ -107,7 +107,7 @@ chain: CertNode type CertRaw { id: string -type: string +nodeType: string created: string modified: string } @@ -116,7 +116,7 @@ hostData: uid . type ShodanNode { id: string -type: string +nodeType: string created: string modified: string hostData: uid @@ -187,12 +187,12 @@ user: string type FullPaste { meta: PasteMeta full: string -type: string +nodeType: string } type PasteNode { id: string -type: string +nodeType: string created: string modified: string fullPaste: FullPaste diff --git a/matcher/main.go b/matcher/main.go index 33dbeed..ea8be10 100644 --- a/matcher/main.go +++ b/matcher/main.go @@ -87,7 +87,7 @@ func loadTargets(graphClient *dgo.Dgraph) ([]string, error) { Timestamp: rfc3339time, Target: scanner.Text(), Nodes: []models.Node{}, - Type: "matcher", + NodeType: "matcher", } ctx := context.Background() query := `query eq($a: string){ @@ -175,7 +175,7 @@ func runPasteMatcher(target string, graphClient *dgo.Dgraph) { q := `query allofterms($a: string) { Node(func: allofterms(full, $a)) { uid - type + nodeType full } @@ -198,7 +198,7 @@ Node(func: allofterms(full, $a)) { Timestamp: rfc3339time, Target: target, Nodes: []models.Node{}, - Type: "matcher", + NodeType: "matcher", } if len(n.Result) != 0 { time.Sleep(time.Duration(2) * time.Second) @@ -221,7 +221,7 @@ Node(func: allofterms(full, $a)) { query := fmt.Sprintf(`query { match as var(func: eq(target, "%s")) }`, target) - pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, Type: "matcher", Timestamp: rfc3339time}) + pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, NodeType: "matcher", Timestamp: rfc3339time}) if err != nil { logrus.Fatal(err) } @@ -279,7 +279,7 @@ Node(func: allofterms(cn, $a)){ Timestamp: rfc3339time, Target: target, Nodes: []models.Node{}, - Type: "matcher", + NodeType: "matcher", } if len(n.Result) != 0 { time.Sleep(time.Duration(2) * time.Second) @@ -300,7 +300,7 @@ Node(func: allofterms(cn, $a)){ query := fmt.Sprintf(`query { match as var(func: eq(target, "%s")) }`, target) - pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, Type: "matcher"}) + pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, NodeType: "matcher"}) if err != nil { logrus.Fatal(err) } @@ -369,7 +369,7 @@ func runShodanMatcher(target string, graphClient *dgo.Dgraph) { Timestamp: rfc3339time, Target: target, Nodes: []models.Node{}, - Type: "matcher", + NodeType: "matcher", } if len(n.Result) != 0 { time.Sleep(time.Duration(2) * time.Second) @@ -392,7 +392,7 @@ func runShodanMatcher(target string, graphClient *dgo.Dgraph) { query := fmt.Sprintf(`query { match as var(func: eq(target, "%s")) }`, target) - pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, Type: "matcher", Timestamp: rfc3339time}) + pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, NodeType: "matcher", Timestamp: rfc3339time}) if err != nil { logrus.Fatal(err) } diff --git a/models/main.go b/models/main.go index 49aabe1..ec0e6b8 100644 --- a/models/main.go +++ b/models/main.go @@ -27,7 +27,7 @@ Structure of this file: type Node struct { UID string `json:"uid,omitempty"` ID string `json:"id,omitempty"` - Type string `json:"type,omitempty"` + NodeType string `json:"nodeType,omitempty"` NData string `json:"ndata,omitempty"` Created string `json:"created,omitempty"` Modified string `json:"modified,omitempty"` @@ -45,7 +45,7 @@ func BuildNode(flag string, dataType string, data string) *Node { uuid := uuid.New().String() return &Node{ ID: flag + "--" + uuid, - Type: dataType, + NodeType: dataType, NData: data, Created: rfc3339time, Modified: rfc3339time, @@ -141,14 +141,14 @@ type Match struct { Nodes []Node `json:"nodes,omitempty"` Target string `json:"target,omitempty"` Timestamp string `json:"timestamp,omitempty"` - Type string `json:"type,omitempty"` + NodeType string `json:"nodeType,omitempty"` } // CertStreamRaw is a wrapper around the stream function to unmarshall the // data receive in a Go structure. type CertStreamRaw struct { ID string `json:"id,omitempty"` - Type string `json:"type,omitempty"` + NodeType string `json:"nodeType,omitempty"` Data CertStreamStruct `json:"data,omitempty"` Created string `json:"created,omitempty"` Modified string `json:"modified,omitempty"` @@ -174,7 +174,7 @@ func WrapCertStreamData(data CertStreamStruct) *CertStreamRaw { rfc3339time := t.Format(time.RFC3339) return &CertStreamRaw{ ID: "certstream--" + uuid.New().String(), - Type: "certstream_raw", + NodeType: "certstream_raw", Data: data, Created: rfc3339time, Modified: rfc3339time, @@ -273,7 +273,7 @@ func SaveCertNode(filename string, node *CertNode) { // PasteNode is a node from PasteBin. type PasteNode struct { ID string `json:"id,omitempty"` - Type string `json:"type,omitempty"` + NodeType string `json:"nodeType,omitempty"` FullPaste FullPaste `json:"fullPaste,omitempty"` Created string `json:"create,omitempty"` Modified string `json:"modified,omitempty"` @@ -281,9 +281,9 @@ type PasteNode struct { // FullPaste wrapes meta and information from Pastebin. type FullPaste struct { - Meta PasteMeta `json:"meta,omitempty"` - Full string `json:"full,omitempty"` - Type string `json:"type,omitempty"` + Meta PasteMeta `json:"meta,omitempty"` + Full string `json:"full,omitempty"` + NodeType string `json:"nodeType,omitempty"` } // BuildPasteNode builds a node from a FullPaste data. @@ -292,7 +292,7 @@ func BuildPasteNode(data *FullPaste) *PasteNode { rfc3339time := t.Format(time.RFC3339) return &PasteNode{ ID: "pastebin--" + uuid.New().String(), - Type: "pastebin", + NodeType: "pastebin", FullPaste: *data, Created: rfc3339time, Modified: rfc3339time, @@ -332,7 +332,7 @@ func SavePaste(filename string, data *PasteNode) { // ShodanNode is node around the shodan.HostData struct. type ShodanNode struct { ID string `json:"id,omitempty"` - Type string `json:"type,omitempty"` + NodeType string `json:"nodeType,omitempty"` HostData ShodanHostData `json:"hostData,omitempty"` Created string `json:"created,omitempty"` Modified string `json:"modified,omitempty"` @@ -371,8 +371,8 @@ func BuildShodanNode(data *shodan.HostData) *ShodanNode { t := time.Now() rfc3339time := t.Format(time.RFC3339) return &ShodanNode{ - ID: "shodan--" + uuid.New().String(), - Type: "shodan_stream", + ID: "shodan--" + uuid.New().String(), + NodeType: "shodan_stream", HostData: ShodanHostData{ Product: data.Product, Hostnames: data.Hostnames, @@ -433,7 +433,7 @@ func SaveShodanNode(filename string, data *ShodanNode) { // BalboaNode represents a return from Balboa. type BalboaNode struct { ID string `json:"id"` - Type string `json:"type"` + NodeType string `json:"nodeType"` Data []balboa.Entries `json:"data"` Created string `json:"created"` Modified string `json:"modified"` @@ -445,7 +445,7 @@ func BuildBalboaNode(data []balboa.Entries) *BalboaNode { rfc3339time := t.Format(time.RFC3339) return &BalboaNode{ ID: "balboa--" + uuid.New().String(), - Type: "balboa", + NodeType: "balboa", Data: data, Created: rfc3339time, Modified: rfc3339time, diff --git a/plugins/certstream.go b/plugins/certstream.go index 173af31..529e778 100644 --- a/plugins/certstream.go +++ b/plugins/certstream.go @@ -83,7 +83,7 @@ func (c *CertStreamPlugin) doRun(graphClient *dgo.Dgraph) { // edge between Node and CertNode e := models.Node{ ID: mainNode.ID, - Type: mainNode.Type, + NodeType: mainNode.NodeType, NData: mainNode.NData, Created: mainNode.Created, Modified: mainNode.Modified, diff --git a/plugins/pastebin.go b/plugins/pastebin.go index b0120ac..f8b61ca 100644 --- a/plugins/pastebin.go +++ b/plugins/pastebin.go @@ -64,9 +64,9 @@ func (p *PastebinPlugin) doRun(graphClient *dgo.Dgraph) { logrus.Error("cannot fetch paste", err) } fp := models.FullPaste{ - Meta: p, - Full: paste, - Type: "fullPaste", + Meta: p, + Full: paste, + NodeType: "fullPaste", } pasteNode := models.BuildPasteNode(&fp) mainNode := models.BuildNode("node", "pastebin", pasteNode.ID) @@ -77,7 +77,7 @@ func (p *PastebinPlugin) doRun(graphClient *dgo.Dgraph) { e := models.Node{ ID: mainNode.ID, - Type: mainNode.Type, + NodeType: mainNode.NodeType, NData: mainNode.NData, Created: mainNode.Created, Modified: mainNode.Modified, diff --git a/plugins/shodan.go b/plugins/shodan.go index 1579951..ac0cf55 100644 --- a/plugins/shodan.go +++ b/plugins/shodan.go @@ -103,7 +103,7 @@ func (s *ShodanPlugin) doRun(graphClient *dgo.Dgraph) { // models.SaveEdge(edge) e := models.Node{ ID: mainNode.ID, - Type: mainNode.Type, + NodeType: mainNode.NodeType, NData: mainNode.NData, Created: mainNode.Created, Modified: mainNode.Modified,