chris/styx
chris/styx
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Changing Type to NodeType to avoid issues

Browse source
This commit is contained in:
Christopher Talib 2020-08-28 13:34:08 +02:00
parent 84e4937f85
commit 1528c28d7e
10 changed files with 81 additions and 61 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

26
DEMO.md
View file

@ -2,14 +2,17 @@
```graphql
{
Node(func: eq(type, "matcher")){
Node(func: eq(nodeType, "matcher")){
id
target
type
nodeType
full
nodes {
uid
full
meta {
full_url
}
}
}
}
@ -19,7 +22,7 @@
{
Node(func: has(nodes) ) {
uid
type
nodeType
target
timestamp
nodes {
@ -33,6 +36,23 @@
}
```
```graphql
{
Node(func: eq(nodeType, "certstream")){
id
nodeType
certNode {
cn
sourceName
fingerprint
notBefore
notAfter
}
}
}
```
## Notes
* There is TOO MUCH junk data

42
README.md
View file

@ -43,7 +43,7 @@ docker-compose up -d # or the other docker command
# build the loader helper binary
go build gitlab.dcso.lolcat/LABS/styx/cmd/iocloader
# update the IOC list while the programm is already running
./iocloader
./iocloader
```
*Note*: if you have issues with the docker compose, make sure it runs on the
@ -79,7 +79,7 @@ query {
uid
ndata
modified
type
nodeType
id
}
}
@ -94,7 +94,7 @@ query {
uid
created
modified
type
nodeType
ndata
certNode {
uid
@ -126,7 +126,7 @@ query {
uid
created
modified
type
nodeType
ndata
pasteNode {
id
@ -161,7 +161,7 @@ query {
uid
created
modified
type
nodeType
full
}
}
@ -194,7 +194,7 @@ Node --[Edge]-- Node
```go
type Node struct {
ID string `json:"id"`
Type string `json:"type"`
NodeType string `json:"nodeType"`
Data string `json:"data"` // For plain Node, the data is the ID of another typed node or a unique value like a domain or a host name.
Created string `json:"created"`
Modified string `json:"modified"`
@ -221,7 +221,7 @@ Node -- CertNode -- CertStreamRaw
// data receive in a Go structure.
type CertStreamRaw struct {
ID string `json:"id"`
Type string `json:"type"`
NodeType string `json:"nodeType"`
Data CertStreamStruct `json:"data"`
Created string `json:"created"`
Modified string `json:"modified"`
@ -251,7 +251,7 @@ Node -- PasteNode -- FullPaste
// PasteNode is a node from PasteBin.
type PasteNode struct {
ID string `json:"id"`
Type string `json:"type"`
NodeType string `json:"nodeType"`
Data FullPaste `json:"data"`
Created string `json:"create"`
Modified string `json:"modified"`
@ -259,9 +259,9 @@ type PasteNode struct {
// FullPaste wrapes meta and information from Pastebin.
type FullPaste struct {
Meta PasteMeta `json:"meta"`
Full string `json:"full"`
Type string `json:"type"`
Meta PasteMeta `json:"meta"`
Full string `json:"full"`
NodeType string `json:"nodeType"`
}
```
@ -271,11 +271,11 @@ Node -- ShodanNode -- Node(s) (hostnames and domains)
```go
type ShodanNode struct {
ID string `json:"id"`
Type string `json:"type"`
Data *shodan.HostData `json:"data"`
Created string `json:"created"`
Modified string `json:"modified"`
ID string `json:"id"`
NodeType string `json:"nodeType"`
Data *shodan.HostData `json:"data"`
Created string `json:"created"`
Modified string `json:"modified"`
}
```
@ -288,10 +288,10 @@ Node -- ShodanNode -- Node (domain) -- BalboaNode
```go
type BalboaNode struct {
ID string `json:"id"`
Type string `json:"type"`
Data []balboa.Entries `json:"data"`
Created string `json:"created"`
Modified string `json:"modified"`
ID string `json:"id"`
NodeType string `json:"nodeType"`
Data []balboa.Entries `json:"data"`
Created string `json:"created"`
Modified string `json:"modified"`
}
```

2
cmd/iocloader/main.go
View file

@ -55,7 +55,7 @@ func loadTargets(graphClient *dgo.Dgraph) error {
Timestamp: rfc3339time,
Target: scanner.Text(),
Nodes: []models.Node{},
Type: "matcher",
NodeType: "matcher",
}
ctx := context.Background()
query := `query eq($a: string){

2
connectors_test.go
View file

@ -1,4 +1,4 @@
package main
package styx
import (
"testing"

12
graph/main.go
View file

@ -37,7 +37,7 @@ func setupDgraphSchema(c *dgo.Dgraph) error {
err := c.Alter(context.Background(), &api.Operation{
Schema: `
id: string @index(exact, term) .
type: string @index(term) .
nodeType: string @index(term) .
ndata: string .
nodeOne: uid .
nodeTwo: uid .
@ -53,7 +53,7 @@ match: uid .
type Node {
id: string
type: string
nodeType: string
ndata: string
created: string
modified: string
@ -107,7 +107,7 @@ chain: CertNode
type CertRaw {
id: string
type: string
nodeType: string
created: string
modified: string
}
@ -116,7 +116,7 @@ hostData: uid .
type ShodanNode {
id: string
type: string
nodeType: string
created: string
modified: string
hostData: uid
@ -187,12 +187,12 @@ user: string
type FullPaste {
meta: PasteMeta
full: string
type: string
nodeType: string
}
type PasteNode {
id: string
type: string
nodeType: string
created: string
modified: string
fullPaste: FullPaste

16
matcher/main.go
View file

@ -87,7 +87,7 @@ func loadTargets(graphClient *dgo.Dgraph) ([]string, error) {
Timestamp: rfc3339time,
Target: scanner.Text(),
Nodes: []models.Node{},
Type: "matcher",
NodeType: "matcher",
}
ctx := context.Background()
query := `query eq($a: string){
@ -175,7 +175,7 @@ func runPasteMatcher(target string, graphClient *dgo.Dgraph) {
q := `query allofterms($a: string) {
Node(func: allofterms(full, $a)) {
uid
type
nodeType
full
}
@ -198,7 +198,7 @@ Node(func: allofterms(full, $a)) {
Timestamp: rfc3339time,
Target: target,
Nodes: []models.Node{},
Type: "matcher",
NodeType: "matcher",
}
if len(n.Result) != 0 {
time.Sleep(time.Duration(2) * time.Second)
@ -221,7 +221,7 @@ Node(func: allofterms(full, $a)) {
query := fmt.Sprintf(`query { match as var(func: eq(target, "%s")) }`, target)
pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, Type: "matcher", Timestamp: rfc3339time})
pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, NodeType: "matcher", Timestamp: rfc3339time})
if err != nil {
logrus.Fatal(err)
}
@ -279,7 +279,7 @@ Node(func: allofterms(cn, $a)){
Timestamp: rfc3339time,
Target: target,
Nodes: []models.Node{},
Type: "matcher",
NodeType: "matcher",
}
if len(n.Result) != 0 {
time.Sleep(time.Duration(2) * time.Second)
@ -300,7 +300,7 @@ Node(func: allofterms(cn, $a)){
query := fmt.Sprintf(`query { match as var(func: eq(target, "%s")) }`, target)
pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, Type: "matcher"})
pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, NodeType: "matcher"})
if err != nil {
logrus.Fatal(err)
}
@ -369,7 +369,7 @@ func runShodanMatcher(target string, graphClient *dgo.Dgraph) {
Timestamp: rfc3339time,
Target: target,
Nodes: []models.Node{},
Type: "matcher",
NodeType: "matcher",
}
if len(n.Result) != 0 {
time.Sleep(time.Duration(2) * time.Second)
@ -392,7 +392,7 @@ func runShodanMatcher(target string, graphClient *dgo.Dgraph) {
query := fmt.Sprintf(`query { match as var(func: eq(target, "%s")) }`, target)
pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, Type: "matcher", Timestamp: rfc3339time})
pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, NodeType: "matcher", Timestamp: rfc3339time})
if err != nil {
logrus.Fatal(err)
}

30
models/main.go
View file

@ -27,7 +27,7 @@ Structure of this file:
type Node struct {
UID string `json:"uid,omitempty"`
ID string `json:"id,omitempty"`
Type string `json:"type,omitempty"`
NodeType string `json:"nodeType,omitempty"`
NData string `json:"ndata,omitempty"`
Created string `json:"created,omitempty"`
Modified string `json:"modified,omitempty"`
@ -45,7 +45,7 @@ func BuildNode(flag string, dataType string, data string) *Node {
uuid := uuid.New().String()
return &Node{
ID: flag + "--" + uuid,
Type: dataType,
NodeType: dataType,
NData: data,
Created: rfc3339time,
Modified: rfc3339time,
@ -141,14 +141,14 @@ type Match struct {
Nodes []Node `json:"nodes,omitempty"`
Target string `json:"target,omitempty"`
Timestamp string `json:"timestamp,omitempty"`
Type string `json:"type,omitempty"`
NodeType string `json:"nodeType,omitempty"`
}
// CertStreamRaw is a wrapper around the stream function to unmarshall the
// data receive in a Go structure.
type CertStreamRaw struct {
ID string `json:"id,omitempty"`
Type string `json:"type,omitempty"`
NodeType string `json:"nodeType,omitempty"`
Data CertStreamStruct `json:"data,omitempty"`
Created string `json:"created,omitempty"`
Modified string `json:"modified,omitempty"`
@ -174,7 +174,7 @@ func WrapCertStreamData(data CertStreamStruct) *CertStreamRaw {
rfc3339time := t.Format(time.RFC3339)
return &CertStreamRaw{
ID: "certstream--" + uuid.New().String(),
Type: "certstream_raw",
NodeType: "certstream_raw",
Data: data,
Created: rfc3339time,
Modified: rfc3339time,
@ -273,7 +273,7 @@ func SaveCertNode(filename string, node *CertNode) {
// PasteNode is a node from PasteBin.
type PasteNode struct {
ID string `json:"id,omitempty"`
Type string `json:"type,omitempty"`
NodeType string `json:"nodeType,omitempty"`
FullPaste FullPaste `json:"fullPaste,omitempty"`
Created string `json:"create,omitempty"`
Modified string `json:"modified,omitempty"`
@ -281,9 +281,9 @@ type PasteNode struct {
// FullPaste wrapes meta and information from Pastebin.
type FullPaste struct {
Meta PasteMeta `json:"meta,omitempty"`
Full string `json:"full,omitempty"`
Type string `json:"type,omitempty"`
Meta PasteMeta `json:"meta,omitempty"`
Full string `json:"full,omitempty"`
NodeType string `json:"nodeType,omitempty"`
}
// BuildPasteNode builds a node from a FullPaste data.
@ -292,7 +292,7 @@ func BuildPasteNode(data *FullPaste) *PasteNode {
rfc3339time := t.Format(time.RFC3339)
return &PasteNode{
ID: "pastebin--" + uuid.New().String(),
Type: "pastebin",
NodeType: "pastebin",
FullPaste: *data,
Created: rfc3339time,
Modified: rfc3339time,
@ -332,7 +332,7 @@ func SavePaste(filename string, data *PasteNode) {
// ShodanNode is node around the shodan.HostData struct.
type ShodanNode struct {
ID string `json:"id,omitempty"`
Type string `json:"type,omitempty"`
NodeType string `json:"nodeType,omitempty"`
HostData ShodanHostData `json:"hostData,omitempty"`
Created string `json:"created,omitempty"`
Modified string `json:"modified,omitempty"`
@ -371,8 +371,8 @@ func BuildShodanNode(data *shodan.HostData) *ShodanNode {
t := time.Now()
rfc3339time := t.Format(time.RFC3339)
return &ShodanNode{
ID: "shodan--" + uuid.New().String(),
Type: "shodan_stream",
ID: "shodan--" + uuid.New().String(),
NodeType: "shodan_stream",
HostData: ShodanHostData{
Product: data.Product,
Hostnames: data.Hostnames,
@ -433,7 +433,7 @@ func SaveShodanNode(filename string, data *ShodanNode) {
// BalboaNode represents a return from Balboa.
type BalboaNode struct {
ID string `json:"id"`
Type string `json:"type"`
NodeType string `json:"nodeType"`
Data []balboa.Entries `json:"data"`
Created string `json:"created"`
Modified string `json:"modified"`
@ -445,7 +445,7 @@ func BuildBalboaNode(data []balboa.Entries) *BalboaNode {
rfc3339time := t.Format(time.RFC3339)
return &BalboaNode{
ID: "balboa--" + uuid.New().String(),
Type: "balboa",
NodeType: "balboa",
Data: data,
Created: rfc3339time,
Modified: rfc3339time,

2
plugins/certstream.go
View file

@ -83,7 +83,7 @@ func (c *CertStreamPlugin) doRun(graphClient *dgo.Dgraph) {
// edge between Node and CertNode
e := models.Node{
ID: mainNode.ID,
Type: mainNode.Type,
NodeType: mainNode.NodeType,
NData: mainNode.NData,
Created: mainNode.Created,
Modified: mainNode.Modified,

8
plugins/pastebin.go
View file

@ -64,9 +64,9 @@ func (p *PastebinPlugin) doRun(graphClient *dgo.Dgraph) {
logrus.Error("cannot fetch paste", err)
}
fp := models.FullPaste{
Meta: p,
Full: paste,
Type: "fullPaste",
Meta: p,
Full: paste,
NodeType: "fullPaste",
}
pasteNode := models.BuildPasteNode(&fp)
mainNode := models.BuildNode("node", "pastebin", pasteNode.ID)
@ -77,7 +77,7 @@ func (p *PastebinPlugin) doRun(graphClient *dgo.Dgraph) {
e := models.Node{
ID: mainNode.ID,
Type: mainNode.Type,
NodeType: mainNode.NodeType,
NData: mainNode.NData,
Created: mainNode.Created,
Modified: mainNode.Modified,

2
plugins/shodan.go
View file

@ -103,7 +103,7 @@ func (s *ShodanPlugin) doRun(graphClient *dgo.Dgraph) {
// models.SaveEdge(edge)
e := models.Node{
ID: mainNode.ID,
Type: mainNode.Type,
NodeType: mainNode.NodeType,
NData: mainNode.NData,
Created: mainNode.Created,
Modified: mainNode.Modified,