diff --git a/README.md b/README.md index 281a253..abcb228 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ # Styx +## IMPORTANT + +For development purposes, each time you restart Styx, the database and the +schema is dropped. Currently, this is hardcoded and used to make development +easier. Just so you know. + ## Prerequisites Styx uses a couple of other services to run: @@ -7,6 +13,7 @@ Styx uses a couple of other services to run: * Kafka for messaging (not implemented yet in the docker, but currently not necessary) * Dgraph for graph representation of results +* Docker-compose to launch everything For that purposes, there is a `docker-compose.yml` file that you can spin up with the following command when in the directory: @@ -28,10 +35,13 @@ docker run --rm -it -p 8080:8080 -p 9080:9080 -p 8000:8000 -v ~/dgraph:/dgraph d go get -u gitlab.dcso.lolcat/LABS/styx cd $GOPATH/src/gitlab.dcso.lolcat/LABS/styx go build -docker-compose up -d # or the other command if you're connected with OpenVPN +docker-compose up -d # or the other command ./styx ``` +*Note*: if you have issues with the docker compose, make sure it runs on the +same subnet. Check [this](https://serverfault.com/questions/916941/configuring-docker-to-not-use-the-172-17-0-0-range) for inspiration. + ### Example configuration: ``` certstream: @@ -56,6 +66,7 @@ kafka: partition: 0 balboa: + # the url you tunneled to Balboa url: http://127.0.0.1:8030 activated: true @@ -63,7 +74,6 @@ elasticsearch: activated: true url: http://localhost:9200 index: "pastebin" - ``` ## Dgraph Interface @@ -84,6 +94,38 @@ query { ``` +Or filter node by type, this example works for certstream nodes: + +```graphql +query { + Node(func: eq(type, "certstream")) { + uid + created + modified + type + ndata + cert_node { + uid + fingerprint + cn + raw { + uid + id + } + chain { + uid + id + } + sourceName + serialNumber + basicConstrains + notBefore + notAfter + } + } +} +``` + ## Datastructure ### Meta diff --git a/graph/main.go b/graph/main.go index a2a3018..8f7ca82 100644 --- a/graph/main.go +++ b/graph/main.go @@ -37,27 +37,30 @@ func setupDgraphSchema(c *dgo.Dgraph) error { id: string @index(term) . type: string @index(term) . ndata: string . -nodeOne: string @index(term) . -nodeTwo: string @index(term) . +nodeOne: uid . +nodeTwo: uid . +subNode: uid . sourceName: string @index(term) . timestamp: string . created: string . modified: string . +cert_node: uid . type Node { -id -type -ndata -created -modified +id: string +type: string +ndata: string +created: string +modified: string +cert_node: CertNode } type Edge { -id -nodeOne -nodeTwo -timestamp -sourceName +id: string +nodeOne: uid +nodeTwo: uid +timestamp: string +sourceName: string } fingerprint: string . @@ -67,59 +70,61 @@ cn: string . sourceName: string . serialNumber: string . basicConstraints: string . -chain: uid . +chain: [uid]. csdata: uid . +raw: uid . type CertNode { -id -fingerprint -notBefore -notAfter -cn -sourceName -serialNumber -basicConstraints -chain +id: string +fingerprint: string +notBefore: string +notAfter: string +cn: string +sourceName: string +serialNumber: string +basicConstraints: string +raw: CertRaw +chain: CertNode } type CertRaw { -id -type -created -modified -csdata +id: string +type: string +created: string +modified: string +csdata: string } type PasteNode { -id -type -created -modified -ndata +id: string +type: string +created: string +modified: string +ndata: uid } meta: uid . full: string . type FullPaste { -meta -full +meta: PasteNode +full: string } type ShodanNode { -id -type -ndata -created -modified +id: string +type: string +ndata: string +created: string +modified: string } type BalboaNode { -id -type -ndata -created -modified +id: string +type: string +ndata: string +created: string +modified: string } `}) if err != nil { diff --git a/models/main.go b/models/main.go index 9d0ad36..650de83 100644 --- a/models/main.go +++ b/models/main.go @@ -31,6 +31,7 @@ type Node struct { Created string `json:"created,omiempty"` Modified string `json:"modified,omiempty"` DType []string `json:"dgraph.type,omiempty"` + CertNode CertNode `json:"cert_node,omiempty"` } // BuildNode builds a node to send to MQ instance. diff --git a/plugins/certstream.go b/plugins/certstream.go index ab97c8a..0d0bd41 100644 --- a/plugins/certstream.go +++ b/plugins/certstream.go @@ -3,7 +3,6 @@ package plugins import ( "context" "encoding/json" - "fmt" "sync" "github.com/CaliDog/certstream-go" @@ -82,65 +81,32 @@ func (c *CertStreamPlugin) doRun(graphClient *dgo.Dgraph) { models.SaveEdge(edge) // saveSingleValues(conn, "certstream", "domain", certNode.ID, domain) + // edge between Node and CertNode + e := models.Node{ + ID: mainNode.ID, + Type: mainNode.Type, + NData: mainNode.NData, + Created: mainNode.Created, + Modified: mainNode.Modified, + CertNode: *certNode, + } + + ctx := context.Background() mu := &api.Mutation{ CommitNow: true, } - marshaled, err := json.Marshal(mainNode) - if err != nil { - logrus.Fatal(err) - } - mu.SetJson = marshaled - _, err = graphClient.NewTxn().Mutate(context.Background(), mu) + pb, err := json.Marshal(e) if err != nil { logrus.Fatal(err) } - variables := map[string]string{"$id": mainNode.ID} - q := `query Node($id: string){ - node(func: eq(id, $id)) { - uid - id - type - ndata - created - modified - } - }` - node, err := graphClient.NewTxn().QueryWithVars(context.Background(), q, variables) - if err != nil { - logrus.Fatal(err) - } + mu.SetJson = pb - marshaled, err = json.Marshal(certNode) + _, err = graphClient.NewTxn().Mutate(ctx, mu) if err != nil { logrus.Fatal(err) } - mu.SetJson = marshaled - _, err = graphClient.NewTxn().Mutate(context.Background(), mu) - if err != nil { - logrus.Fatal(err) - } - query := ` -query Node($mainNodeID: string, $subNodeID: string) { -node as var(func: eq(id, $mainNodeID)) - } - ` - mu = &api.Mutation{ - SetNquads: []byte(`uid(node) "$subNodeID" .`), - } - req := &api.Request{ - Query: query, - Mutations: []*api.Mutation{mu}, - CommitNow: true, - Vars: map[string]string{"$mainNodeID": node.Uids[mainNode.ID], "$subNodeID": certNode.ID}, - } - res, err := graphClient.NewTxn().Do(context.Background(), req) - if err != nil { - logrus.Error(err) - } - fmt.Println(res) - } }