chris/styx
chris/styx
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Parallel matching on multiple targets work

Browse source
This commit is contained in:
Christopher Talib 2020-06-08 10:49:19 +02:00
parent b26cc60d39
commit 929222aff4
5 changed files with 104 additions and 99 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
main.go
View file

@ -37,10 +37,12 @@ func main() {
// panic(err) // panic(err)
// } // }
logrus.Info("Initializing Dgraph...")
dgraphClient, err := graph.ConnectToDgraph() dgraphClient, err := graph.ConnectToDgraph()
if err != nil { if err != nil {
logrus.WithField("err", err).Error("error initialising the graph database") logrus.WithField("err", err).Error("error initialising the graph database")
} }
logrus.Info("done")
// stop channel // stop channel
stopChan := make(chan os.Signal) stopChan := make(chan os.Signal)

BIN
matcher/.main.go.swp Normal file
View file

Binary file not shown.

19
matcher/main.go
View file

@ -56,7 +56,7 @@ func (m *Matcher) Stop(wg *sync.WaitGroup) {
// Result is the result from the matching query. Probably going to change. // Result is the result from the matching query. Probably going to change.
type Result struct { type Result struct {
Result []models.Node `json:"Node,omiempty"` Result []models.Node `json:"Node,omitempty"`
} }
func loadTargets(graphClient *dgo.Dgraph) error { func loadTargets(graphClient *dgo.Dgraph) error {
@ -132,10 +132,12 @@ func (m *Matcher) Run(wg *sync.WaitGroup, graphClient *dgo.Dgraph) {
if !m.Running { if !m.Running {
m.StoppedChan = make(chan bool) m.StoppedChan = make(chan bool)
wg.Add(1) wg.Add(1)
targets := []string{"code", "password", "login", "covid", "coronavirus", "java"} targets := []string{"code", "password", "login", "covid", "coronavirus", "java", "php", "function"}
for _, target := range targets { for _, target := range targets {
go runMatcher(target, graphClient) go runMatcher(target, graphClient)
} }
// TODO: probably not the best design here // TODO: probably not the best design here
wg.Add(len(targets)) wg.Add(len(targets))
m.Running = true m.Running = true
@ -156,7 +158,6 @@ Node(func: allofterms(full, $a)) {
ctx := context.Background() ctx := context.Background()
txn := graphClient.NewTxn() txn := graphClient.NewTxn()
defer txn.Discard(ctx)
res, err := txn.QueryWithVars(ctx, q, map[string]string{"$a": target}) res, err := txn.QueryWithVars(ctx, q, map[string]string{"$a": target})
if err != nil { if err != nil {
logrus.Warn(err) logrus.Warn(err)
@ -175,7 +176,8 @@ Node(func: allofterms(full, $a)) {
Type: "matcher", Type: "matcher",
} }
if len(n.Result) != 0 { if len(n.Result) != 0 {
time.Sleep(3) time.Sleep(time.Duration(2) * time.Second)
logrus.Info("Found match for ", target)
// TODO: review time and id to be updated on new resulsts // TODO: review time and id to be updated on new resulsts
for _, res := range n.Result { for _, res := range n.Result {
@ -200,8 +202,7 @@ Node(func: allofterms(full, $a)) {
} }
mu := &api.Mutation{ mu := &api.Mutation{
SetJson: pb, SetJson: pb,
CommitNow: true,
} }
req := &api.Request{ req := &api.Request{
@ -213,10 +214,12 @@ Node(func: allofterms(full, $a)) {
txn := graphClient.NewTxn() txn := graphClient.NewTxn()
_, err = txn.Do(ctx, req) _, err = txn.Do(ctx, req)
if err != nil { if err != nil {
logrus.Fatal(err) logrus.Warn(err)
} }
time.Sleep(2) // txn.Discard(ctx)
time.Sleep(time.Duration(2) * time.Second)
} }
} }
} }

142
models/main.go
View file

@ -25,16 +25,16 @@ Structure of this file:
// Styx terminology // Styx terminology
// (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx) // (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx)
type Node struct { type Node struct {
UID string `json:"uid,omiempty"` UID string `json:"uid,omitempty"`
ID string `json:"id,omiempty"` ID string `json:"id,omitempty"`
Type string `json:"type,omiempty"` Type string `json:"type,omitempty"`
NData string `json:"ndata,omiempty"` NData string `json:"ndata,omitempty"`
Created string `json:"created,omiempty"` Created string `json:"created,omitempty"`
Modified string `json:"modified,omiempty"` Modified string `json:"modified,omitempty"`
DType []string `json:"dgraph.type,omiempty"` DType []string `json:"dgraph.type,omitempty"`
CertNode CertNode `json:"certNode,omiempty"` CertNode CertNode `json:"certNode,omitempty"`
ShodanNode ShodanNode `json:"shodanNode,omiempty"` ShodanNode ShodanNode `json:"shodanNode,omitempty"`
PasteNode PasteNode `json:"pasteNode,omiempty"` PasteNode PasteNode `json:"pasteNode,omitempty"`
} }
// BuildNode builds a node to send to MQ instance. // BuildNode builds a node to send to MQ instance.
@ -82,12 +82,12 @@ func SaveNode(filename string, node *Node) {
// Edge defines a relation between two nodes. // Edge defines a relation between two nodes.
type Edge struct { type Edge struct {
ID string `json:"id,omiempty"` ID string `json:"id,omitempty"`
NodeOne map[string]interface{} `json:"nodeOne,omiempty"` NodeOne map[string]interface{} `json:"nodeOne,omitempty"`
NodeTwo map[string]interface{} `json:"nodeTwo,omiempty"` NodeTwo map[string]interface{} `json:"nodeTwo,omitempty"`
Timestamp string `json:"timestamp,omiempty"` Timestamp string `json:"timestamp,omitempty"`
Source string `json:"source,omiempty"` Source string `json:"source,omitempty"`
DType []string `json:"dgraph.type,omiempty"` DType []string `json:"dgraph.type,omitempty"`
} }
// BuildEdge build a send from two nodes with a given source type. // BuildEdge build a send from two nodes with a given source type.
@ -135,36 +135,36 @@ func SaveEdge(edge *Edge) {
// Match represents clustered results based on a target. // Match represents clustered results based on a target.
type Match struct { type Match struct {
ID string `json:"id,omiempty"` ID string `json:"id,omitempty"`
UID string `json:"uid,omiempty"` UID string `json:"uid,omitempty"`
Nodes []Node `json:"nodes,omiempty"` Nodes []Node `json:"nodes,omitempty"`
Target string `json:"target,omiempty"` Target string `json:"target,omitempty"`
Timestamp string `json:"timestamp,omiempty"` Timestamp string `json:"timestamp,omitempty"`
Type string `json:"type,omiempty"` Type string `json:"type,omitempty"`
} }
// CertStreamRaw is a wrapper around the stream function to unmarshall the // CertStreamRaw is a wrapper around the stream function to unmarshall the
// data receive in a Go structure. // data receive in a Go structure.
type CertStreamRaw struct { type CertStreamRaw struct {
ID string `json:"id,omiempty"` ID string `json:"id,omitempty"`
Type string `json:"type,omiempty"` Type string `json:"type,omitempty"`
Data CertStreamStruct `json:"data,omiempty"` Data CertStreamStruct `json:"data,omitempty"`
Created string `json:"created,omiempty"` Created string `json:"created,omitempty"`
Modified string `json:"modified,omiempty"` Modified string `json:"modified,omitempty"`
} }
// CertNode represents our custom struct of data extraction from CertStream. // CertNode represents our custom struct of data extraction from CertStream.
type CertNode struct { type CertNode struct {
ID string `json:"id,omiempty"` ID string `json:"id,omitempty"`
Fingerprint string `json:"fingerprint,omiempty"` Fingerprint string `json:"fingerprint,omitempty"`
NotBefore string `json:"notBefore,omiempty"` NotBefore string `json:"notBefore,omitempty"`
NotAfter string `json:"notAfter,omiempty"` NotAfter string `json:"notAfter,omitempty"`
CN string `json:"cn,omiempty"` CN string `json:"cn,omitempty"`
SourceName string `json:"sourceName,omiempty"` SourceName string `json:"sourceName,omitempty"`
SerialNumber string `json:"serialNumber,omiempty"` SerialNumber string `json:"serialNumber,omitempty"`
BasicConstraints string `json:"basicConstraints,omiempty"` BasicConstraints string `json:"basicConstraints,omitempty"`
Raw CertStreamRaw `json:"raw,omiempty"` Raw CertStreamRaw `json:"raw,omitempty"`
Chain []CertNode `json:"chain,omiempty"` Chain []CertNode `json:"chain,omitempty"`
} }
// WrapCertStreamData is a wrapper around CertStreamStruct. // WrapCertStreamData is a wrapper around CertStreamStruct.
@ -271,18 +271,18 @@ func SaveCertNode(filename string, node *CertNode) {
// PasteNode is a node from PasteBin. // PasteNode is a node from PasteBin.
type PasteNode struct { type PasteNode struct {
ID string `json:"id,omiempty"` ID string `json:"id,omitempty"`
Type string `json:"type,omiempty"` Type string `json:"type,omitempty"`
FullPaste FullPaste `json:"fullPaste,omiempty"` FullPaste FullPaste `json:"fullPaste,omitempty"`
Created string `json:"create,omiempty"` Created string `json:"create,omitempty"`
Modified string `json:"modified,omiempty"` Modified string `json:"modified,omitempty"`
} }
// FullPaste wrapes meta and information from Pastebin. // FullPaste wrapes meta and information from Pastebin.
type FullPaste struct { type FullPaste struct {
Meta PasteMeta `json:"meta,omiempty"` Meta PasteMeta `json:"meta,omitempty"`
Full string `json:"full,omiempty"` Full string `json:"full,omitempty"`
Type string `json:"type,omiempty"` Type string `json:"type,omitempty"`
} }
// BuildPasteNode builds a node from a FullPaste data. // BuildPasteNode builds a node from a FullPaste data.
@ -330,39 +330,39 @@ func SavePaste(filename string, data *PasteNode) {
// ShodanNode is node around the shodan.HostData struct. // ShodanNode is node around the shodan.HostData struct.
type ShodanNode struct { type ShodanNode struct {
ID string `json:"id,omiempty"` ID string `json:"id,omitempty"`
Type string `json:"type,omiempty"` Type string `json:"type,omitempty"`
HostData ShodanHostData `json:"hostData,omiempty"` HostData ShodanHostData `json:"hostData,omitempty"`
Created string `json:"created,omiempty"` Created string `json:"created,omitempty"`
Modified string `json:"modified,omiempty"` Modified string `json:"modified,omitempty"`
} }
// ShodanHostData is a copy of the structure in the go shodan library. It's a // ShodanHostData is a copy of the structure in the go shodan library. It's a
// workaround to have more control on the data send. // workaround to have more control on the data send.
type ShodanHostData struct { type ShodanHostData struct {
Product string `json:"product,omiempty"` Product string `json:"product,omitempty"`
Hostnames []string `json:"hostnames,omiempty"` Hostnames []string `json:"hostnames,omitempty"`
Version string `json:"version,omiempty"` Version string `json:"version,omitempty"`
Title string `json:"title,omiempty"` Title string `json:"title,omitempty"`
// SSL *HostSSL `json:"ssl"` // SSL *HostSSL `json:"ssl"`
IP string `json:"ip_str,omiempty"` IP string `json:"ip_str,omitempty"`
OS string `json:"os,omiempty"` OS string `json:"os,omitempty"`
Organization string `json:"org,omiempty"` Organization string `json:"org,omitempty"`
ISP string `json:"isp,omiempty"` ISP string `json:"isp,omitempty"`
CPE []string `json:"cpe,omiempty"` CPE []string `json:"cpe,omitempty"`
// Data string `json:"data,omiempty"` // Data string `json:"data,omitempty"`
ASN string `json:"asn,omiempty"` ASN string `json:"asn,omitempty"`
Port int `json:"port,omiempty"` Port int `json:"port,omitempty"`
HTML string `json:"html,omiempty"` HTML string `json:"html,omitempty"`
Banner string `json:"banner,omiempty"` Banner string `json:"banner,omitempty"`
Link string `json:"link,omiempty"` Link string `json:"link,omitempty"`
Transport string `json:"transport,omiempty"` Transport string `json:"transport,omitempty"`
Domains []string `json:"domains,omiempty"` Domains []string `json:"domains,omitempty"`
Timestamp string `json:"timestamp,omiempty"` Timestamp string `json:"timestamp,omitempty"`
DeviceType string `json:"devicetype,omiempty"` DeviceType string `json:"devicetype,omitempty"`
// Location *HostLocation `json:"location"` // Location *HostLocation `json:"location"`
ShodanData map[string]interface{} `json:"_shodan,omiempty"` ShodanData map[string]interface{} `json:"_shodan,omitempty"`
Opts map[string]interface{} `json:"opts,omiempty"` Opts map[string]interface{} `json:"opts,omitempty"`
} }
// BuildShodanNode builds a wrapper node around shodan.HostData. // BuildShodanNode builds a wrapper node around shodan.HostData.

40
models/pastebin.go
View file

@ -13,31 +13,31 @@ import (
// PasteMeta is a set of descriptive information on a paste. // PasteMeta is a set of descriptive information on a paste.
// Camel case in the marshaling because it's the Pastebin API structure. // Camel case in the marshaling because it's the Pastebin API structure.
type PasteMeta struct { type PasteMeta struct {
ScrapeURL string `json:"scrape_url,omiempty"` ScrapeURL string `json:"scrape_url,omitempty"`
FullURL string `json:"full_url,omiempty"` FullURL string `json:"full_url,omitempty"`
Date string `json:"date,omiempty"` Date string `json:"date,omitempty"`
Key string `json:"key,omiempty"` Key string `json:"key,omitempty"`
Size string `json:"size,omiempty"` Size string `json:"size,omitempty"`
Expire string `json:"expire,omiempty"` Expire string `json:"expire,omitempty"`
Title string `json:"title,omiempty"` Title string `json:"title,omitempty"`
Syntax string `json:"syntax,omiempty"` Syntax string `json:"syntax,omitempty"`
User string `json:"user,omiempty"` User string `json:"user,omitempty"`
} }
// PasteFull extends PasteMeta by the actual content. // PasteFull extends PasteMeta by the actual content.
// Not used in our code. // Not used in our code.
type PasteFull struct { type PasteFull struct {
ScrapeURL string `json:"scrapeUrl,omiempty"` ScrapeURL string `json:"scrapeUrl,omitempty"`
FullURL string `json:"fullUrl,omiempty"` FullURL string `json:"fullUrl,omitempty"`
Date string `json:"date,omiempty"` Date string `json:"date,omitempty"`
Key string `json:"key,omiempty"` Key string `json:"key,omitempty"`
Size string `json:"size,omiempty"` Size string `json:"size,omitempty"`
Expire string `json:"expire,omiempty"` Expire string `json:"expire,omitempty"`
Title string `json:"title,omiempty"` Title string `json:"title,omitempty"`
Syntax string `json:"syntax,omiempty"` Syntax string `json:"syntax,omitempty"`
User string `json:"user,omiempty"` User string `json:"user,omitempty"`
Data string `json:"data,omiempty"` Data string `json:"data,omitempty"`
RFC3339 string `json:"time,omiempty"` RFC3339 string `json:"time,omitempty"`
} }
// Meta Information: https://pastebin.com/api_scraping.php // Meta Information: https://pastebin.com/api_scraping.php