452 lines
9.6 KiB
Go
452 lines
9.6 KiB
Go
package matcher
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"runtime"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/dgraph-io/dgo/v2"
|
|
"github.com/dgraph-io/dgo/v2/protos/api"
|
|
"github.com/google/uuid"
|
|
"github.com/sirupsen/logrus"
|
|
"github.com/spf13/viper"
|
|
"gitlab.dcso.lolcat/LABS/styx/models"
|
|
)
|
|
|
|
var (
|
|
_, b, _, _ = runtime.Caller(0)
|
|
basepath = filepath.Dir(b)
|
|
)
|
|
|
|
// Matcher is the structure for the matching logic within Styx.
|
|
type Matcher struct {
|
|
Running bool
|
|
StopChan chan bool
|
|
StoppedChan chan bool
|
|
}
|
|
|
|
// Initialize initialises the matcher based on the given configuration.
|
|
func (m *Matcher) Initialize() bool {
|
|
if !viper.GetBool("matcher.activated") {
|
|
return false
|
|
}
|
|
|
|
logrus.Info("matching is activated")
|
|
return true
|
|
}
|
|
|
|
// Stop gracefully stops the matching logic.
|
|
func (m *Matcher) Stop(wg *sync.WaitGroup) {
|
|
if m.Running {
|
|
m.StopChan = make(chan bool)
|
|
close(m.StopChan)
|
|
<-m.StopChan
|
|
wg.Done()
|
|
m.Running = false
|
|
}
|
|
}
|
|
|
|
// Result is the result from the matching query. Probably going to change.
|
|
type Result struct {
|
|
Result []models.Node `json:"Node,omitempty"`
|
|
}
|
|
|
|
func loadTargets(graphClient *dgo.Dgraph) ([]string, error) {
|
|
path := basepath + "/data/"
|
|
res := []string{}
|
|
|
|
sliceDomain, err := ioutil.ReadDir(path)
|
|
if err != nil {
|
|
logrus.Warn("matcher#ReadDir#domains", err)
|
|
return nil, err
|
|
}
|
|
|
|
for _, file := range sliceDomain {
|
|
logrus.Info("loading: ", file.Name(), " please wait...")
|
|
f, err := os.OpenFile(path+file.Name(), 0, 0644)
|
|
if err != nil {
|
|
logrus.Warn("matcher#OpenFile#", err)
|
|
return nil, err
|
|
}
|
|
scanner := bufio.NewScanner(f)
|
|
|
|
for scanner.Scan() {
|
|
uuid := uuid.New().String()
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
matcher := models.Match{
|
|
ID: uuid,
|
|
Timestamp: rfc3339time,
|
|
Target: scanner.Text(),
|
|
Nodes: []models.Node{},
|
|
NodeType: "matcher",
|
|
}
|
|
ctx := context.Background()
|
|
query := `query eq($a: string){
|
|
Node(func: eq(target, $a)){
|
|
uid
|
|
}
|
|
}`
|
|
|
|
txn := graphClient.NewTxn()
|
|
ret, err := txn.QueryWithVars(ctx, query, map[string]string{"$a": scanner.Text()})
|
|
if err != nil {
|
|
logrus.Warn(err)
|
|
}
|
|
|
|
n := Result{}
|
|
json.Unmarshal([]byte(ret.Json), &n)
|
|
|
|
// Check if the target already exists, if so, skipping not inserting
|
|
// the data
|
|
if len(n.Result) == 0 {
|
|
logrus.Info("new matcher, charging...")
|
|
mu := &api.Mutation{
|
|
CommitNow: true,
|
|
}
|
|
|
|
pb, err := json.Marshal(matcher)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
return nil, err
|
|
}
|
|
|
|
mu.SetJson = pb
|
|
|
|
txn = graphClient.NewTxn()
|
|
defer txn.Discard(ctx)
|
|
|
|
_, err = txn.Mutate(ctx, mu)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
return nil, err
|
|
}
|
|
|
|
}
|
|
|
|
res = append(res, scanner.Text())
|
|
if err := scanner.Err(); err != nil {
|
|
logrus.Error(err)
|
|
return nil, err
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
return res, nil
|
|
}
|
|
|
|
// Run runs the routine trying to find matches in the ingested Pastebin data.
|
|
func (m *Matcher) Run(wg *sync.WaitGroup, graphClient *dgo.Dgraph) {
|
|
logrus.Info("loading matcher targets, this might take some time...")
|
|
targets, err := loadTargets(graphClient)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
logrus.Info("finished loading matcher targets")
|
|
fmt.Println(targets)
|
|
|
|
if !m.Running {
|
|
m.StoppedChan = make(chan bool)
|
|
wg.Add(1)
|
|
logrus.Info("Running matchers")
|
|
for _, target := range targets {
|
|
go runPasteMatcher(target, graphClient)
|
|
go runCertstreamMatcher(target, graphClient)
|
|
go runShodanMatcher(target, graphClient)
|
|
}
|
|
|
|
// TODO: probably not the best design here
|
|
wg.Add(len(targets))
|
|
m.Running = true
|
|
}
|
|
}
|
|
|
|
func runPasteMatcher(target string, graphClient *dgo.Dgraph) {
|
|
for {
|
|
q := `query allofterms($a: string) {
|
|
Node(func: allofterms(full, $a)) {
|
|
uid
|
|
nodeType
|
|
full
|
|
}
|
|
|
|
}`
|
|
|
|
ctx := context.Background()
|
|
txn := graphClient.NewTxn()
|
|
res, err := txn.QueryWithVars(ctx, q, map[string]string{"$a": target})
|
|
if err != nil {
|
|
logrus.Warn(err)
|
|
}
|
|
|
|
n := Result{}
|
|
json.Unmarshal([]byte(res.Json), &n)
|
|
uuid := uuid.New().String()
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
matcher := models.Match{
|
|
ID: uuid,
|
|
Timestamp: rfc3339time,
|
|
Target: target,
|
|
Nodes: []models.Node{},
|
|
NodeType: "matcher",
|
|
}
|
|
if len(n.Result) != 0 {
|
|
time.Sleep(time.Duration(2) * time.Second)
|
|
logrus.Info("Found paste match for ", target)
|
|
// TODO: review time and id to be updated on new resulsts
|
|
for _, res := range n.Result {
|
|
if len(matcher.Nodes) == 0 {
|
|
matcher.Nodes = append(matcher.Nodes, res)
|
|
continue
|
|
} else {
|
|
for _, node := range matcher.Nodes {
|
|
if res.UID != node.UID {
|
|
matcher.Nodes = append(matcher.Nodes, res)
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
query := fmt.Sprintf(`query { match as var(func: eq(target, "%s")) }`, target)
|
|
|
|
pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, NodeType: "matcher", Timestamp: rfc3339time})
|
|
if err != nil {
|
|
logrus.Fatal(err)
|
|
}
|
|
|
|
mu := &api.Mutation{
|
|
SetJson: pb,
|
|
}
|
|
|
|
req := &api.Request{
|
|
Query: query,
|
|
Mutations: []*api.Mutation{mu},
|
|
CommitNow: true,
|
|
}
|
|
|
|
txn := graphClient.NewTxn()
|
|
_, err = txn.Do(ctx, req)
|
|
if err != nil {
|
|
logrus.Warn(err)
|
|
}
|
|
// txn.Discard(ctx)
|
|
time.Sleep(time.Duration(2) * time.Second)
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
func runCertstreamMatcher(target string, graphClient *dgo.Dgraph) {
|
|
for {
|
|
q := `query allofterms($a: string){
|
|
Node(func: allofterms(cn, $a)){
|
|
fingerprint
|
|
notBefore
|
|
notAfter
|
|
cn
|
|
sourceName
|
|
basicConstraints
|
|
raw
|
|
}
|
|
}`
|
|
|
|
ctx := context.Background()
|
|
txn := graphClient.NewTxn()
|
|
res, err := txn.QueryWithVars(ctx, q, map[string]string{"$a": target})
|
|
if err != nil {
|
|
logrus.Warn(err)
|
|
}
|
|
|
|
n := Result{}
|
|
json.Unmarshal([]byte(res.Json), &n)
|
|
uuid := uuid.New().String()
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
matcher := models.Match{
|
|
ID: uuid,
|
|
Timestamp: rfc3339time,
|
|
Target: target,
|
|
Nodes: []models.Node{},
|
|
NodeType: "matcher",
|
|
}
|
|
if len(n.Result) != 0 {
|
|
time.Sleep(time.Duration(2) * time.Second)
|
|
logrus.Info("Found certstream match for ", target)
|
|
|
|
for _, res := range n.Result {
|
|
if len(matcher.Nodes) == 0 {
|
|
matcher.Nodes = append(matcher.Nodes, res)
|
|
continue
|
|
} else {
|
|
for _, nodes := range matcher.Nodes {
|
|
if res.UID != nodes.UID {
|
|
matcher.Nodes = append(matcher.Nodes, res)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
query := fmt.Sprintf(`query { match as var(func: eq(target, "%s")) }`, target)
|
|
|
|
pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, NodeType: "matcher"})
|
|
if err != nil {
|
|
logrus.Fatal(err)
|
|
}
|
|
|
|
mu := &api.Mutation{
|
|
SetJson: pb,
|
|
}
|
|
|
|
req := &api.Request{
|
|
Query: query,
|
|
Mutations: []*api.Mutation{mu},
|
|
CommitNow: true,
|
|
}
|
|
|
|
txn := graphClient.NewTxn()
|
|
_, err = txn.Do(ctx, req)
|
|
if err != nil {
|
|
logrus.Warn(err)
|
|
}
|
|
|
|
time.Sleep(time.Duration(2) * time.Second)
|
|
}
|
|
}
|
|
}
|
|
|
|
func runShodanMatcher(target string, graphClient *dgo.Dgraph) {
|
|
for {
|
|
q := `query allofterms($a: string){
|
|
Node(func: eq(hostnames, $a)){
|
|
uid
|
|
type
|
|
hostnames
|
|
product
|
|
version
|
|
title
|
|
ip
|
|
os
|
|
organization
|
|
isp
|
|
cpe
|
|
asn
|
|
port
|
|
html
|
|
banner
|
|
link
|
|
transport
|
|
domains
|
|
timestamp
|
|
}
|
|
}
|
|
`
|
|
ctx := context.Background()
|
|
txn := graphClient.NewTxn()
|
|
res, err := txn.QueryWithVars(ctx, q, map[string]string{"$a": target})
|
|
if err != nil {
|
|
logrus.Warn(err)
|
|
}
|
|
|
|
n := Result{}
|
|
json.Unmarshal([]byte(res.Json), &n)
|
|
uuid := uuid.New().String()
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
matcher := models.Match{
|
|
ID: uuid,
|
|
Timestamp: rfc3339time,
|
|
Target: target,
|
|
Nodes: []models.Node{},
|
|
NodeType: "matcher",
|
|
}
|
|
if len(n.Result) != 0 {
|
|
time.Sleep(time.Duration(2) * time.Second)
|
|
logrus.Info("Found shodan match for ", target)
|
|
// TODO: review time and id to be updated on new resulsts
|
|
|
|
for _, res := range n.Result {
|
|
if len(matcher.Nodes) == 0 {
|
|
matcher.Nodes = append(matcher.Nodes, res)
|
|
continue
|
|
} else {
|
|
for _, node := range matcher.Nodes {
|
|
if res.UID != node.UID {
|
|
matcher.Nodes = append(matcher.Nodes, res)
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
query := fmt.Sprintf(`query { match as var(func: eq(target, "%s")) }`, target)
|
|
|
|
pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, NodeType: "matcher", Timestamp: rfc3339time})
|
|
if err != nil {
|
|
logrus.Fatal(err)
|
|
}
|
|
|
|
mu := &api.Mutation{
|
|
SetJson: pb,
|
|
}
|
|
|
|
req := &api.Request{
|
|
Query: query,
|
|
Mutations: []*api.Mutation{mu},
|
|
CommitNow: true,
|
|
}
|
|
|
|
txn := graphClient.NewTxn()
|
|
_, err = txn.Do(ctx, req)
|
|
if err != nil {
|
|
logrus.Warn(err)
|
|
}
|
|
// txn.Discard(ctx)
|
|
time.Sleep(time.Duration(2) * time.Second)
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
// RunDomainMatch looks for a target within the identified IOCs in /matcher/data.
|
|
// the function could be refactored with RunDomainFilters
|
|
func RunDomainMatch(domain string) bool {
|
|
path := basepath + "/data/domains.txt"
|
|
|
|
sliceDomain, err := ioutil.ReadDir(path)
|
|
if err != nil {
|
|
logrus.Warn("matcher#ReadDir#domains", err)
|
|
}
|
|
|
|
for _, file := range sliceDomain {
|
|
f, err := os.OpenFile(path+file.Name(), 1, 0644)
|
|
if err != nil {
|
|
logrus.Warn("matcher#OpenFile#", err)
|
|
}
|
|
scanner := bufio.NewScanner(f)
|
|
|
|
for scanner.Scan() {
|
|
r, err := regexp.Compile(scanner.Text())
|
|
if err != nil {
|
|
logrus.Warn("matcher#Compile#", err)
|
|
}
|
|
if r.MatchString(domain) {
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
return true
|
|
}
|