Simple linked model on certstream + better install instructions

README.md

@@ -1,5 +1,11 @@
 # Styx
 
+## IMPORTANT
+
+For development purposes, each time you restart Styx, the database and the
+schema is dropped. Currently, this is hardcoded and used to make development
+easier. Just so you know.
+
 ## Prerequisites
 
 Styx uses a couple of other services to run:
@@ -7,6 +13,7 @@ Styx uses a couple of other services to run:
 * Kafka for messaging (not implemented yet in the docker, but currently not
   necessary)
 * Dgraph for graph representation of results
+* Docker-compose to launch everything
 
 For that purposes, there is a `docker-compose.yml` file that you can spin up
 with the following command when in the directory:
@@ -28,10 +35,13 @@ docker run --rm -it -p 8080:8080 -p 9080:9080 -p 8000:8000 -v ~/dgraph:/dgraph d
 go get -u gitlab.dcso.lolcat/LABS/styx
 cd $GOPATH/src/gitlab.dcso.lolcat/LABS/styx
 go build
-docker-compose up -d # or the other command if you're connected with OpenVPN
+docker-compose up -d # or the other command
 ./styx
 ```
 
+*Note*: if you have issues with the docker compose, make sure it runs on the
+same subnet. Check [this](https://serverfault.com/questions/916941/configuring-docker-to-not-use-the-172-17-0-0-range) for inspiration.
+
 ### Example configuration:
 ```
 certstream:
@@ -56,6 +66,7 @@ kafka:
   partition: 0
 
 balboa:
+  # the url you tunneled to Balboa
   url: http://127.0.0.1:8030
   activated: true
 
@@ -63,7 +74,6 @@ elasticsearch:
   activated: true
   url: http://localhost:9200
   index: "pastebin"
-
 ```
 
 ## Dgraph Interface
@@ -84,6 +94,38 @@ query {
 
 ```
 
+Or filter node by type, this example works for certstream nodes:
+
+```graphql
+query {
+  Node(func: eq(type, "certstream")) {
+    uid
+    created
+    modified
+    type
+    ndata
+    cert_node {
+      uid
+      fingerprint
+      cn
+      raw {
+        uid
+        id
+      }
+      chain {
+        uid
+        id
+      }
+      sourceName
+      serialNumber
+      basicConstrains
+      notBefore
+      notAfter
+    }
+  }
+}
+```
+
 ## Datastructure
 
 ### Meta

graph/main.go

@@ -37,27 +37,30 @@ func setupDgraphSchema(c *dgo.Dgraph) error {
 id: string @index(term) .
 type: string @index(term) .
 ndata: string .
-nodeOne: string @index(term) .
+nodeOne: uid .
-nodeTwo: string @index(term) .
+nodeTwo: uid .
+subNode: uid .
 sourceName: string @index(term) .
 timestamp: string .
 created: string .
 modified: string .
+cert_node: uid .
 
 type Node {
-id
+id: string
-type
+type: string
-ndata
+ndata: string
-created
+created: string
-modified
+modified: string
+cert_node: CertNode
 }
 
 type Edge {
-id
+id: string
-nodeOne
+nodeOne: uid
-nodeTwo
+nodeTwo: uid
-timestamp
+timestamp: string
-sourceName
+sourceName: string
 }
 
 fingerprint: string .
@@ -67,59 +70,61 @@ cn: string .
 sourceName: string .
 serialNumber: string .
 basicConstraints: string .
-chain: uid .
+chain: [uid].
 csdata: uid .
+raw: uid .
 
 type CertNode {
-id
+id: string
-fingerprint
+fingerprint: string
-notBefore
+notBefore: string
-notAfter
+notAfter: string
-cn
+cn: string
-sourceName
+sourceName: string
-serialNumber
+serialNumber: string
-basicConstraints
+basicConstraints: string
-chain
+raw: CertRaw
+chain: CertNode
 }
 
 type CertRaw {
-id
+id: string
-type
+type: string
-created
+created: string
-modified
+modified: string
-csdata
+csdata: string
 }
 
 type PasteNode {
-id
+id: string
-type
+type: string
-created
+created: string
-modified
+modified: string
-ndata
+ndata: uid
 }
 
 meta: uid .
 full: string .
 
 type FullPaste {
-meta
+meta: PasteNode
-full
+full: string
 }
 
 type ShodanNode {
-id
+id: string
-type
+type: string
-ndata
+ndata: string
-created
+created: string
-modified
+modified: string
 }
 
 type BalboaNode {
-id
+id: string
-type
+type: string
-ndata
+ndata: string
-created
+created: string
-modified
+modified: string
 }
 `})
 	if err != nil {

models/main.go

@@ -31,6 +31,7 @@ type Node struct {
 	Created  string   `json:"created,omiempty"`
 	Modified string   `json:"modified,omiempty"`
 	DType    []string `json:"dgraph.type,omiempty"`
+	CertNode CertNode `json:"cert_node,omiempty"`
 }
 
 // BuildNode builds a node to send to MQ instance.

plugins/certstream.go

@@ -3,7 +3,6 @@ package plugins
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"sync"
 
 	"github.com/CaliDog/certstream-go"
@@ -82,65 +81,32 @@ func (c *CertStreamPlugin) doRun(graphClient *dgo.Dgraph) {
 						models.SaveEdge(edge)
 						// saveSingleValues(conn, "certstream", "domain", certNode.ID, domain)
 
+						// edge between Node and CertNode
+						e := models.Node{
+							ID:       mainNode.ID,
+							Type:     mainNode.Type,
+							NData:    mainNode.NData,
+							Created:  mainNode.Created,
+							Modified: mainNode.Modified,
+							CertNode: *certNode,
+						}
+
+						ctx := context.Background()
 						mu := &api.Mutation{
 							CommitNow: true,
 						}
 
-						marshaled, err := json.Marshal(mainNode)
+						pb, err := json.Marshal(e)
-						if err != nil {
-							logrus.Fatal(err)
-						}
-						mu.SetJson = marshaled
-						_, err = graphClient.NewTxn().Mutate(context.Background(), mu)
 						if err != nil {
 							logrus.Fatal(err)
 						}
 
-						variables := map[string]string{"$id": mainNode.ID}
+						mu.SetJson = pb
-						q := `query Node($id: string){
-						 node(func: eq(id, $id)) {
-						 uid
-						 id
-						 type
-						 ndata
-						 created
-						 modified
-						 }
-						 }`
-						node, err := graphClient.NewTxn().QueryWithVars(context.Background(), q, variables)
-						if err != nil {
-							logrus.Fatal(err)
-						}
 
-						marshaled, err = json.Marshal(certNode)
+						_, err = graphClient.NewTxn().Mutate(ctx, mu)
 						if err != nil {
 							logrus.Fatal(err)
 						}
-						mu.SetJson = marshaled
-						_, err = graphClient.NewTxn().Mutate(context.Background(), mu)
-						if err != nil {
-							logrus.Fatal(err)
-						}
-						query := `
-query Node($mainNodeID: string, $subNodeID: string) {
-node as var(func: eq(id, $mainNodeID))
-						}
-						`
-						mu = &api.Mutation{
-							SetNquads: []byte(`uid(node) <CertNode> "$subNodeID" .`),
-						}
-						req := &api.Request{
-							Query:     query,
-							Mutations: []*api.Mutation{mu},
-							CommitNow: true,
-							Vars:      map[string]string{"$mainNodeID": node.Uids[mainNode.ID], "$subNodeID": certNode.ID},
-						}
-						res, err := graphClient.NewTxn().Do(context.Background(), req)
-						if err != nil {
-							logrus.Error(err)
-						}
-						fmt.Println(res)
-
 					}
 				}