saving
This commit is contained in:
Christopher Talib
parent
fb270a1b66
commit
d0c8deae99
1
go.mod
1
go.mod
|
|
@ -7,6 +7,7 @@ require (
|
|||
dmitri.shuralyov.com/gpu/mtl v0.0.0-20191203043605-d42048ed14fd // indirect
|
||||
github.com/CaliDog/certstream-go v0.0.0-20180219203951-6016c5462366
|
||||
github.com/DataDog/zstd v1.4.4 // indirect
|
||||
github.com/christalib/structs v1.1.0
|
||||
github.com/cncf/udpa/go v0.0.0-20200124205748-db4b343e48c1 // indirect
|
||||
github.com/dgraph-io/dgo v1.0.0
|
||||
github.com/dgraph-io/dgo/v2 v2.2.0
|
||||
|
|
|
|||
3
go.sum
3
go.sum
|
|
@ -38,6 +38,8 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
|
|||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/christalib/structs v1.1.0 h1:vWXpO00DCoGj5jYo+HV2pvNmoDsKHAAHPdkK2aTZjXE=
|
||||
github.com/christalib/structs v1.1.0/go.mod h1:sKgAVbcnUvqeaSB3E5Y8eRgaTyo1DFz7lG2EKiSy7SU=
|
||||
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
|
||||
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
|
||||
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
|
||||
|
|
@ -112,6 +114,7 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a
|
|||
github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
|
|
|
|||
|
|
@ -26,35 +26,38 @@ func ConnectToDgraph() (*dgo.Dgraph, error) {
|
|||
|
||||
func setupDgraphSchema(c *dgo.Dgraph) error {
|
||||
err := c.Alter(context.Background(), &api.Operation{
|
||||
DropAll: true,
|
||||
DropOp: api.Operation_ALL,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
err = c.Alter(context.Background(), &api.Operation{
|
||||
Schema: `
|
||||
id: string @index(term) .
|
||||
type: string @index(term) .
|
||||
data: string .
|
||||
nodeOneID: uid .
|
||||
nodeTwoID: uid .
|
||||
source: string @index(term) .
|
||||
ndata: string .
|
||||
nodeOne: string @index(term) .
|
||||
nodeTwo: string @index(term) .
|
||||
sourceName: string @index(term) .
|
||||
timestamp: string .
|
||||
created: dateTime .
|
||||
modified: dateTime .
|
||||
created: string .
|
||||
modified: string .
|
||||
|
||||
type Node {
|
||||
id
|
||||
type
|
||||
data
|
||||
ndata
|
||||
created
|
||||
modified
|
||||
}
|
||||
|
||||
type Edge {
|
||||
id
|
||||
nodeOneID
|
||||
nodeTwoID
|
||||
nodeOne
|
||||
nodeTwo
|
||||
timestamp
|
||||
source
|
||||
sourceName
|
||||
}
|
||||
|
||||
fingerprint: string .
|
||||
|
|
@ -65,6 +68,7 @@ sourceName: string .
|
|||
serialNumber: string .
|
||||
basicConstraints: string .
|
||||
chain: uid .
|
||||
csdata: uid .
|
||||
|
||||
type CertNode {
|
||||
id
|
||||
|
|
@ -83,7 +87,7 @@ id
|
|||
type
|
||||
created
|
||||
modified
|
||||
data
|
||||
csdata
|
||||
}
|
||||
|
||||
type PasteNode {
|
||||
|
|
@ -91,7 +95,7 @@ id
|
|||
type
|
||||
created
|
||||
modified
|
||||
data
|
||||
ndata
|
||||
}
|
||||
|
||||
meta: uid .
|
||||
|
|
@ -105,7 +109,7 @@ full
|
|||
type ShodanNode {
|
||||
id
|
||||
type
|
||||
data
|
||||
ndata
|
||||
created
|
||||
modified
|
||||
}
|
||||
|
|
@ -113,7 +117,7 @@ modified
|
|||
type BalboaNode {
|
||||
id
|
||||
type
|
||||
data
|
||||
ndata
|
||||
created
|
||||
modified
|
||||
}
|
||||
|
|
|
|||
9
main.go
9
main.go
|
|
@ -51,22 +51,25 @@ func main() {
|
|||
c := plugins.CertStreamPlugin{}
|
||||
if ok := c.Initialize(); !ok {
|
||||
logrus.Info("certstream plugin not activated")
|
||||
} else {
|
||||
c.Run(&wg, dgraphClient)
|
||||
}
|
||||
c.Run(&wg, dgraphClient)
|
||||
|
||||
// pastebin
|
||||
p := plugins.PastebinPlugin{}
|
||||
if ok := p.Initialize(); !ok {
|
||||
logrus.Info("pastebin plugin not activated")
|
||||
} else {
|
||||
p.Run(&wg)
|
||||
}
|
||||
p.Run(&wg)
|
||||
|
||||
// shodan
|
||||
s := plugins.ShodanPlugin{}
|
||||
if ok := s.Initialize(); !ok {
|
||||
logrus.Info("shodan plugin not activated")
|
||||
} else {
|
||||
p.Run(&wg)
|
||||
}
|
||||
p.Run(&wg)
|
||||
|
||||
go func() {
|
||||
<-stopChan
|
||||
|
|
|
|||
|
|
@ -61,7 +61,7 @@ type CertStreamData struct {
|
|||
// and the content is stored in Data.
|
||||
type CertStreamStruct struct {
|
||||
MessageType string `json:"message_data"`
|
||||
Data CertStreamData `json:"data"`
|
||||
CSData CertStreamData `json:"csdata"`
|
||||
}
|
||||
|
||||
// ExtractCertFromStream builds the structures before saving them. It uses the
|
||||
|
|
@ -117,7 +117,7 @@ func ExtractCertFromStream(input jsonq.JsonQuery) (*CertStreamStruct, error) {
|
|||
|
||||
res := CertStreamStruct{
|
||||
MessageType: messageType,
|
||||
Data: csd,
|
||||
CSData: csd,
|
||||
}
|
||||
|
||||
return &res, nil
|
||||
|
|
|
|||
|
|
@ -25,12 +25,12 @@ Structure of this file:
|
|||
// Styx terminology
|
||||
// (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx)
|
||||
type Node struct {
|
||||
UID string `json:"uid,omiempty"`
|
||||
ID string `json:"id,omiempty"`
|
||||
Type string `json:"type,omiempty"`
|
||||
Data string `json:"data,omiempty"`
|
||||
Created string `json:"created,omiempty"`
|
||||
Modified string `json:"modified,omiempty"`
|
||||
ID string `json:"id,omiempty"`
|
||||
Type string `json:"type,omiempty"`
|
||||
NData string `json:"ndata,omiempty"`
|
||||
Created string `json:"created,omiempty"`
|
||||
Modified string `json:"modified,omiempty"`
|
||||
DType []string `json:"dgraph.type,omiempty"`
|
||||
}
|
||||
|
||||
// BuildNode builds a node to send to MQ instance.
|
||||
|
|
@ -39,10 +39,9 @@ func BuildNode(flag string, dataType string, data string) *Node {
|
|||
rfc3339time := t.Format(time.RFC3339)
|
||||
uuid := uuid.New().String()
|
||||
return &Node{
|
||||
UID: "_:" + flag + "--" + uuid,
|
||||
ID: flag + "--" + uuid,
|
||||
Type: dataType,
|
||||
Data: data,
|
||||
NData: data,
|
||||
Created: rfc3339time,
|
||||
Modified: rfc3339time,
|
||||
}
|
||||
|
|
@ -79,25 +78,24 @@ func SaveNode(filename string, node *Node) {
|
|||
|
||||
// Edge defines a relation between two nodes.
|
||||
type Edge struct {
|
||||
UID string `json:"uid,omiempty"`
|
||||
ID string `json:"id,omiempty"`
|
||||
NodeOneID string `json:"nodeOneID,omiempty"`
|
||||
NodeTwoID string `json:"nodeTwoID,omiempty"`
|
||||
Timestamp string `json:"timestamp,omiempty"`
|
||||
Source string `json:"source,omiempty"`
|
||||
ID string `json:"id,omiempty"`
|
||||
NodeOne map[string]interface{} `json:"nodeOne,omiempty"`
|
||||
NodeTwo map[string]interface{} `json:"nodeTwo,omiempty"`
|
||||
Timestamp string `json:"timestamp,omiempty"`
|
||||
Source string `json:"source,omiempty"`
|
||||
DType []string `json:"dgraph.type,omiempty"`
|
||||
}
|
||||
|
||||
// BuildEdge build a send from two nodes with a given source type.
|
||||
func BuildEdge(source string, nodeOneUUID string, nodeTwoUUID string) *Edge {
|
||||
func BuildEdge(source string, nodeOne, nodeTwo map[string]interface{}) *Edge {
|
||||
t := time.Now()
|
||||
rfc3339time := t.Format(time.RFC3339)
|
||||
uuid := uuid.New().String()
|
||||
return &Edge{
|
||||
UID: "_:" + "edge--" + uuid,
|
||||
ID: "edge--" + uuid,
|
||||
Source: source,
|
||||
NodeOneID: nodeOneUUID,
|
||||
NodeTwoID: nodeTwoUUID,
|
||||
NodeOne: nodeOne,
|
||||
NodeTwo: nodeTwo,
|
||||
Timestamp: rfc3339time,
|
||||
}
|
||||
}
|
||||
|
|
@ -134,7 +132,6 @@ func SaveEdge(edge *Edge) {
|
|||
// CertStreamRaw is a wrapper around the stream function to unmarshall the
|
||||
// data receive in a Go structure.
|
||||
type CertStreamRaw struct {
|
||||
UID string `json:"uid,omiempty"`
|
||||
ID string `json:"id,omiempty"`
|
||||
Type string `json:"type,omiempty"`
|
||||
Data CertStreamStruct `json:"data,omiempty"`
|
||||
|
|
@ -144,17 +141,16 @@ type CertStreamRaw struct {
|
|||
|
||||
// CertNode represents our custom struct of data extraction from CertStream.
|
||||
type CertNode struct {
|
||||
UID string `json:"uid,omiempty"`
|
||||
ID string `json:"id,omiempty"`
|
||||
Fingerprint string `json:"fingerprint,omiempty"`
|
||||
NotBefore string `json:"notBefore,omiempty"`
|
||||
NotAfter string `json:"notAfter,omiempty"`
|
||||
CN string `json:"cn,omiempty"`
|
||||
SourceName string `json:"sourceName,omiempty"`
|
||||
SerialNumber string `json:"serialNumber,omiempty"`
|
||||
BasicConstraints string `json:"basicConstraints,omiempty"`
|
||||
RawUUID string `json:"rawUUID,omiempty"`
|
||||
Chain []CertNode `json:"chainedTo,omiempty"`
|
||||
ID string `json:"id,omiempty"`
|
||||
Fingerprint string `json:"fingerprint,omiempty"`
|
||||
NotBefore string `json:"notBefore,omiempty"`
|
||||
NotAfter string `json:"notAfter,omiempty"`
|
||||
CN string `json:"cn,omiempty"`
|
||||
SourceName string `json:"sourceName,omiempty"`
|
||||
SerialNumber string `json:"serialNumber,omiempty"`
|
||||
BasicConstraints string `json:"basicConstraints,omiempty"`
|
||||
Raw CertStreamRaw `json:"raw,omiempty"`
|
||||
Chain []CertNode `json:"chainedTo,omiempty"`
|
||||
}
|
||||
|
||||
// WrapCertStreamData is a wrapper around CertStreamStruct.
|
||||
|
|
@ -162,7 +158,6 @@ func WrapCertStreamData(data CertStreamStruct) *CertStreamRaw {
|
|||
t := time.Now()
|
||||
rfc3339time := t.Format(time.RFC3339)
|
||||
return &CertStreamRaw{
|
||||
UID: "_:certstream--" + uuid.New().String(),
|
||||
ID: "certstream--" + uuid.New().String(),
|
||||
Type: "certstream_raw",
|
||||
Data: data,
|
||||
|
|
@ -174,29 +169,26 @@ func WrapCertStreamData(data CertStreamStruct) *CertStreamRaw {
|
|||
// BuildCertNode builds a custom node based on CertStream.
|
||||
func BuildCertNode(rawNode *CertStreamRaw) *CertNode {
|
||||
main := &CertNode{
|
||||
UID: "_:certstream--" + uuid.New().String(),
|
||||
ID: "certstream--" + uuid.New().String(),
|
||||
Fingerprint: rawNode.Data.Data.LeafCert.Fingerprint,
|
||||
NotBefore: time.Unix(int64(rawNode.Data.Data.LeafCert.NotBefore), 0).Format(time.RFC3339),
|
||||
NotAfter: time.Unix(int64(rawNode.Data.Data.LeafCert.NotAfter), 0).Format(time.RFC3339),
|
||||
CN: rawNode.Data.Data.LeafCert.Subject.CN,
|
||||
SourceName: rawNode.Data.Data.Source.Name,
|
||||
BasicConstraints: rawNode.Data.Data.LeafCert.Extensions.BasicConstrains,
|
||||
RawUUID: rawNode.ID,
|
||||
Fingerprint: rawNode.Data.CSData.LeafCert.Fingerprint,
|
||||
NotBefore: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotBefore), 0).Format(time.RFC3339),
|
||||
NotAfter: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotAfter), 0).Format(time.RFC3339),
|
||||
CN: rawNode.Data.CSData.LeafCert.Subject.CN,
|
||||
SourceName: rawNode.Data.CSData.Source.Name,
|
||||
BasicConstraints: rawNode.Data.CSData.LeafCert.Extensions.BasicConstrains,
|
||||
Raw: *rawNode,
|
||||
}
|
||||
|
||||
var res []CertNode
|
||||
if len(rawNode.Data.Data.Chain) > 0 {
|
||||
if len(rawNode.Data.CSData.Chain) > 0 {
|
||||
chain := CertNode{
|
||||
UID: "_:certstream--" + uuid.New().String(),
|
||||
ID: "certstream--" + uuid.New().String(),
|
||||
Fingerprint: rawNode.Data.Data.LeafCert.Fingerprint,
|
||||
NotBefore: time.Unix(int64(rawNode.Data.Data.LeafCert.NotBefore), 0).Format(time.RFC3339),
|
||||
NotAfter: time.Unix(int64(rawNode.Data.Data.LeafCert.NotAfter), 0).Format(time.RFC3339),
|
||||
CN: rawNode.Data.Data.LeafCert.Subject.CN,
|
||||
SourceName: rawNode.Data.Data.Source.Name,
|
||||
BasicConstraints: rawNode.Data.Data.LeafCert.Extensions.BasicConstrains,
|
||||
RawUUID: rawNode.ID,
|
||||
Fingerprint: rawNode.Data.CSData.LeafCert.Fingerprint,
|
||||
NotBefore: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotBefore), 0).Format(time.RFC3339),
|
||||
NotAfter: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotAfter), 0).Format(time.RFC3339),
|
||||
CN: rawNode.Data.CSData.LeafCert.Subject.CN,
|
||||
SourceName: rawNode.Data.CSData.Source.Name,
|
||||
BasicConstraints: rawNode.Data.CSData.LeafCert.Extensions.BasicConstrains,
|
||||
}
|
||||
res = append(res, chain)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -3,9 +3,11 @@ package plugins
|
|||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/CaliDog/certstream-go"
|
||||
"github.com/christalib/structs"
|
||||
"github.com/dgraph-io/dgo/v2"
|
||||
"github.com/dgraph-io/dgo/v2/protos/api"
|
||||
"github.com/jmoiron/jsonq"
|
||||
|
|
@ -64,25 +66,26 @@ func (c *CertStreamPlugin) doRun(graphClient *dgo.Dgraph) {
|
|||
case jq := <-c.Stream:
|
||||
if data, err := models.ExtractCertFromStream(jq); err == nil {
|
||||
|
||||
allDomains := data.Data.LeafCert.AllDomains
|
||||
allDomains := data.CSData.LeafCert.AllDomains
|
||||
for _, domain := range allDomains {
|
||||
if filters.RunDomainFilters(domain) {
|
||||
rawNode := models.WrapCertStreamData(*data)
|
||||
models.SaveCertStreamRaw("raw_certstream.json", rawNode)
|
||||
// models.SaveCertStreamRaw("raw_certstream.json", rawNode)
|
||||
|
||||
certNode := models.BuildCertNode(rawNode)
|
||||
models.SaveCertNode("cert_nodes.json", certNode)
|
||||
mainNode := models.BuildNode("node", "certstream", certNode.ID)
|
||||
models.SaveNode("nodes.json", mainNode)
|
||||
edge := models.BuildEdge("certstream", rawNode.ID, mainNode.ID)
|
||||
models.SaveEdge(edge)
|
||||
edge = models.BuildEdge("certstream", mainNode.ID, certNode.ID)
|
||||
rawEdge := models.BuildEdge("certstream", structs.Map(rawNode), structs.Map(mainNode))
|
||||
models.SaveEdge(rawEdge)
|
||||
edge := models.BuildEdge("certstream", structs.Map(mainNode), structs.Map(certNode))
|
||||
models.SaveEdge(edge)
|
||||
// saveSingleValues(conn, "certstream", "domain", certNode.ID, domain)
|
||||
|
||||
mu := &api.Mutation{
|
||||
CommitNow: true,
|
||||
}
|
||||
|
||||
marshaled, err := json.Marshal(mainNode)
|
||||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
|
|
@ -92,21 +95,51 @@ func (c *CertStreamPlugin) doRun(graphClient *dgo.Dgraph) {
|
|||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
// variables := map[string]string{"$id": assigned.Uids[mainNode.ID]}
|
||||
// q := `query Node($id: string){
|
||||
// node(func: uid($id)) {
|
||||
// uid
|
||||
// ID
|
||||
// type
|
||||
// data
|
||||
// dreated
|
||||
// modified
|
||||
// }
|
||||
// }`
|
||||
// _, err := graphClient.NewTxn().QueryWithVars(context.Background(), q, variables)
|
||||
// if err != nil {
|
||||
// logrus.Fatal(err)
|
||||
// }
|
||||
|
||||
variables := map[string]string{"$id": mainNode.ID}
|
||||
q := `query Node($id: string){
|
||||
node(func: eq(id, $id)) {
|
||||
uid
|
||||
id
|
||||
type
|
||||
ndata
|
||||
created
|
||||
modified
|
||||
}
|
||||
}`
|
||||
node, err := graphClient.NewTxn().QueryWithVars(context.Background(), q, variables)
|
||||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
|
||||
marshaled, err = json.Marshal(certNode)
|
||||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
mu.SetJson = marshaled
|
||||
_, err = graphClient.NewTxn().Mutate(context.Background(), mu)
|
||||
if err != nil {
|
||||
logrus.Fatal(err)
|
||||
}
|
||||
query := `
|
||||
query Node($mainNodeID: string, $subNodeID: string) {
|
||||
node as var(func: eq(id, $mainNodeID))
|
||||
}
|
||||
`
|
||||
mu = &api.Mutation{
|
||||
SetNquads: []byte(`uid(node) <CertNode> "$subNodeID" .`),
|
||||
}
|
||||
req := &api.Request{
|
||||
Query: query,
|
||||
Mutations: []*api.Mutation{mu},
|
||||
CommitNow: true,
|
||||
Vars: map[string]string{"$mainNodeID": node.Uids[mainNode.ID], "$subNodeID": certNode.ID},
|
||||
}
|
||||
res, err := graphClient.NewTxn().Do(context.Background(), req)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
fmt.Println(res)
|
||||
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ import (
|
|||
"fmt"
|
||||
"sync"
|
||||
|
||||
"github.com/christalib/structs"
|
||||
"github.com/ns3777k/go-shodan/v4/shodan"
|
||||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/viper"
|
||||
|
|
@ -85,7 +86,7 @@ func (s *ShodanPlugin) doRun() {
|
|||
models.SaveShodanNode("raw_shodan.json", shodanNode)
|
||||
node := models.BuildNode("shodan", "shodan_stream", shodanNode.ID)
|
||||
models.SaveNode("nodes.json", node)
|
||||
edge := models.BuildEdge("shodan", shodanNode.ID, node.ID)
|
||||
edge := models.BuildEdge("shodan", structs.Map(shodanNode), structs.Map(node))
|
||||
models.SaveEdge(edge)
|
||||
}
|
||||
} else {
|
||||
|
|
|
|||
Loading…
Reference in a new issue