Simple linked model on certstream + better install instructions
This commit is contained in:
Christopher Talib
parent
5dca0a0472
commit
cbdca52ab2
46
README.md
46
README.md
|
|
@ -1,5 +1,11 @@
|
|||
# Styx
|
||||
|
||||
## IMPORTANT
|
||||
|
||||
For development purposes, each time you restart Styx, the database and the
|
||||
schema is dropped. Currently, this is hardcoded and used to make development
|
||||
easier. Just so you know.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
Styx uses a couple of other services to run:
|
||||
|
|
@ -7,6 +13,7 @@ Styx uses a couple of other services to run:
|
|||
* Kafka for messaging (not implemented yet in the docker, but currently not
|
||||
necessary)
|
||||
* Dgraph for graph representation of results
|
||||
* Docker-compose to launch everything
|
||||
|
||||
For that purposes, there is a `docker-compose.yml` file that you can spin up
|
||||
with the following command when in the directory:
|
||||
|
|
@ -28,10 +35,13 @@ docker run --rm -it -p 8080:8080 -p 9080:9080 -p 8000:8000 -v ~/dgraph:/dgraph d
|
|||
go get -u gitlab.dcso.lolcat/LABS/styx
|
||||
cd $GOPATH/src/gitlab.dcso.lolcat/LABS/styx
|
||||
go build
|
||||
docker-compose up -d # or the other command if you're connected with OpenVPN
|
||||
docker-compose up -d # or the other command
|
||||
./styx
|
||||
```
|
||||
|
||||
*Note*: if you have issues with the docker compose, make sure it runs on the
|
||||
same subnet. Check [this](https://serverfault.com/questions/916941/configuring-docker-to-not-use-the-172-17-0-0-range) for inspiration.
|
||||
|
||||
### Example configuration:
|
||||
```
|
||||
certstream:
|
||||
|
|
@ -56,6 +66,7 @@ kafka:
|
|||
partition: 0
|
||||
|
||||
balboa:
|
||||
# the url you tunneled to Balboa
|
||||
url: http://127.0.0.1:8030
|
||||
activated: true
|
||||
|
||||
|
|
@ -63,7 +74,6 @@ elasticsearch:
|
|||
activated: true
|
||||
url: http://localhost:9200
|
||||
index: "pastebin"
|
||||
|
||||
```
|
||||
|
||||
## Dgraph Interface
|
||||
|
|
@ -84,6 +94,38 @@ query {
|
|||
|
||||
```
|
||||
|
||||
Or filter node by type, this example works for certstream nodes:
|
||||
|
||||
```graphql
|
||||
query {
|
||||
Node(func: eq(type, "certstream")) {
|
||||
uid
|
||||
created
|
||||
modified
|
||||
type
|
||||
ndata
|
||||
cert_node {
|
||||
uid
|
||||
fingerprint
|
||||
cn
|
||||
raw {
|
||||
uid
|
||||
id
|
||||
}
|
||||
chain {
|
||||
uid
|
||||
id
|
||||
}
|
||||
sourceName
|
||||
serialNumber
|
||||
basicConstrains
|
||||
notBefore
|
||||
notAfter
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Datastructure
|
||||
|
||||
### Meta
|
||||
|
|
|
|||
|
|
@ -37,27 +37,30 @@ func setupDgraphSchema(c *dgo.Dgraph) error {
|
|||
id: string @index(term) .
|
||||
type: string @index(term) .
|
||||
ndata: string .
|
||||
nodeOne: string @index(term) .
|
||||
nodeTwo: string @index(term) .
|
||||
nodeOne: uid .
|
||||
nodeTwo: uid .
|
||||
subNode: uid .
|
||||
sourceName: string @index(term) .
|
||||
timestamp: string .
|
||||
created: string .
|
||||
modified: string .
|
||||
cert_node: uid .
|
||||
|
||||
type Node {
|
||||
id
|
||||
type
|
||||
ndata
|
||||
created
|
||||
modified
|
||||
id: string
|
||||
type: string
|
||||
ndata: string
|
||||
created: string
|
||||
modified: string
|
||||
cert_node: CertNode
|
||||
}
|
||||
|
||||
type Edge {
|
||||
id
|
||||
nodeOne
|
||||
nodeTwo
|
||||
timestamp
|
||||
sourceName
|
||||
id: string
|
||||
nodeOne: uid
|
||||
nodeTwo: uid
|
||||
timestamp: string
|
||||
sourceName: string
|
||||
}
|
||||
|
||||
fingerprint: string .
|
||||
|
|
@ -67,59 +70,61 @@ cn: string .
|
|||
sourceName: string .
|
||||
serialNumber: string .
|
||||
basicConstraints: string .
|
||||
chain: uid .
|
||||
chain: [uid].
|
||||
csdata: uid .
|
||||
raw: uid .
|
||||
|
||||
type CertNode {
|
||||
id
|
||||
fingerprint
|
||||
notBefore
|
||||
notAfter
|
||||
cn
|
||||
sourceName
|
||||
serialNumber
|
||||
basicConstraints
|
||||
chain
|
||||
id: string
|
||||
fingerprint: string
|
||||
notBefore: string
|
||||
notAfter: string
|
||||
cn: string
|
||||
sourceName: string
|
||||
serialNumber: string
|
||||
basicConstraints: string
|
||||
raw: CertRaw
|
||||
chain: CertNode
|
||||
}
|
||||
|
||||
type CertRaw {
|
||||
id
|
||||
type
|
||||
created
|
||||
modified
|
||||
csdata
|
||||
id: string
|
||||
type: string
|
||||
created: string
|
||||
modified: string
|
||||
csdata: string
|
||||
}
|
||||
|
||||
type PasteNode {
|
||||
id
|
||||
type
|
||||
created
|
||||
modified
|
||||
ndata
|
||||
id: string
|
||||
type: string
|
||||
created: string
|
||||
modified: string
|
||||
ndata: uid
|
||||
}
|
||||
|
||||
meta: uid .
|
||||
full: string .
|
||||
|
||||
type FullPaste {
|
||||
meta
|
||||
full
|
||||
meta: PasteNode
|
||||
full: string
|
||||
}
|
||||
|
||||
type ShodanNode {
|
||||
id
|
||||
type
|
||||
ndata
|
||||
created
|
||||
modified
|
||||
id: string
|
||||
type: string
|
||||
ndata: string
|
||||
created: string
|
||||
modified: string
|
||||
}
|
||||
|
||||
type BalboaNode {
|
||||
id
|
||||
type
|
||||
ndata
|
||||
created
|
||||
modified
|
||||
id: string
|
||||
type: string
|
||||
ndata: string
|
||||
created: string
|
||||
modified: string
|
||||
}
|
||||
`})
|
||||
if err != nil {
|
||||
|
|
|
|||
|
|
@ -31,6 +31,7 @@ type Node struct {
|
|||
Created string `json:"created,omiempty"`
|
||||
Modified string `json:"modified,omiempty"`
|
||||
DType []string `json:"dgraph.type,omiempty"`
|
||||
CertNode CertNode `json:"cert_node,omiempty"`
|
||||
}
|
||||
|
||||
// BuildNode builds a node to send to MQ instance.
|
||||
|
|
|
|||
|
|
@ -3,7 +3,6 @@ package plugins
|
|||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/CaliDog/certstream-go"
|
||||
|
|
@ -82,65 +81,32 @@ func (c *CertStreamPlugin) doRun(graphClient *dgo.Dgraph) {
|
|||
models.SaveEdge(edge)
|
||||
// saveSingleValues(conn, "certstream", "domain", certNode.ID, domain)
|
||||
|
||||
// edge between Node and CertNode
|
||||
e := models.Node{
|
||||
ID: mainNode.ID,
|
||||
Type: mainNode.Type,
|
||||
NData: mainNode.NData,
|
||||
Created: mainNode.Created,
|
||||
Modified: mainNode.Modified,
|
||||
CertNode: *certNode,
|
||||
}
|
||||
|
||||
ctx := context.Background()
|
||||
mu := &api.Mutation{
|
||||
CommitNow: true,
|
||||
}
|
||||
|
||||
marshaled, err := json.Marshal(mainNode)
|
||||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
mu.SetJson = marshaled
|
||||
_, err = graphClient.NewTxn().Mutate(context.Background(), mu)
|
||||
pb, err := json.Marshal(e)
|
||||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
|
||||
variables := map[string]string{"$id": mainNode.ID}
|
||||
q := `query Node($id: string){
|
||||
node(func: eq(id, $id)) {
|
||||
uid
|
||||
id
|
||||
type
|
||||
ndata
|
||||
created
|
||||
modified
|
||||
}
|
||||
}`
|
||||
node, err := graphClient.NewTxn().QueryWithVars(context.Background(), q, variables)
|
||||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
mu.SetJson = pb
|
||||
|
||||
marshaled, err = json.Marshal(certNode)
|
||||
_, err = graphClient.NewTxn().Mutate(ctx, mu)
|
||||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
mu.SetJson = marshaled
|
||||
_, err = graphClient.NewTxn().Mutate(context.Background(), mu)
|
||||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
query := `
|
||||
query Node($mainNodeID: string, $subNodeID: string) {
|
||||
node as var(func: eq(id, $mainNodeID))
|
||||
}
|
||||
`
|
||||
mu = &api.Mutation{
|
||||
SetNquads: []byte(`uid(node) <CertNode> "$subNodeID" .`),
|
||||
}
|
||||
req := &api.Request{
|
||||
Query: query,
|
||||
Mutations: []*api.Mutation{mu},
|
||||
CommitNow: true,
|
||||
Vars: map[string]string{"$mainNodeID": node.Uids[mainNode.ID], "$subNodeID": certNode.ID},
|
||||
}
|
||||
res, err := graphClient.NewTxn().Do(context.Background(), req)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
fmt.Println(res)
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue