84e4937f85
This new work implements the server and the loader in two different binaries allowing the code while updating the IOC list. It updates also the documentation to reflect the new changes.
451 lines
9.6 KiB
Go
451 lines
9.6 KiB
Go
package matcher
|
|
|
|
import (
|
|
"bufio"
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"os"
|
|
"path/filepath"
|
|
"regexp"
|
|
"runtime"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/dgraph-io/dgo/v2"
|
|
"github.com/dgraph-io/dgo/v2/protos/api"
|
|
"github.com/google/uuid"
|
|
"github.com/sirupsen/logrus"
|
|
"github.com/spf13/viper"
|
|
"gitlab.dcso.lolcat/LABS/styx/models"
|
|
)
|
|
|
|
var (
|
|
_, b, _, _ = runtime.Caller(0)
|
|
basepath = filepath.Dir(b)
|
|
)
|
|
|
|
// Matcher is the structure for the matching logic within Styx.
|
|
type Matcher struct {
|
|
Running bool
|
|
StopChan chan bool
|
|
StoppedChan chan bool
|
|
}
|
|
|
|
// Initialize initialises the matcher based on the given configuration.
|
|
func (m *Matcher) Initialize() bool {
|
|
if !viper.GetBool("matcher.activated") {
|
|
return false
|
|
}
|
|
|
|
logrus.Info("matching is activated")
|
|
return true
|
|
}
|
|
|
|
// Stop gracefully stops the matching logic.
|
|
func (m *Matcher) Stop(wg *sync.WaitGroup) {
|
|
if m.Running {
|
|
m.StopChan = make(chan bool)
|
|
close(m.StopChan)
|
|
<-m.StopChan
|
|
wg.Done()
|
|
m.Running = false
|
|
}
|
|
}
|
|
|
|
// Result is the result from the matching query. Probably going to change.
|
|
type Result struct {
|
|
Result []models.Node `json:"Node,omitempty"`
|
|
}
|
|
|
|
func loadTargets(graphClient *dgo.Dgraph) ([]string, error) {
|
|
path := basepath + "/data/"
|
|
res := []string{}
|
|
|
|
sliceDomain, err := ioutil.ReadDir(path)
|
|
if err != nil {
|
|
logrus.Warn("matcher#ReadDir#domains", err)
|
|
return nil, err
|
|
}
|
|
|
|
for _, file := range sliceDomain {
|
|
logrus.Info("loading: ", file.Name(), " please wait...")
|
|
f, err := os.OpenFile(path+file.Name(), 0, 0644)
|
|
if err != nil {
|
|
logrus.Warn("matcher#OpenFile#", err)
|
|
return nil, err
|
|
}
|
|
scanner := bufio.NewScanner(f)
|
|
|
|
for scanner.Scan() {
|
|
uuid := uuid.New().String()
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
matcher := models.Match{
|
|
ID: uuid,
|
|
Timestamp: rfc3339time,
|
|
Target: scanner.Text(),
|
|
Nodes: []models.Node{},
|
|
Type: "matcher",
|
|
}
|
|
ctx := context.Background()
|
|
query := `query eq($a: string){
|
|
Node(func: eq(target, $a)){
|
|
uid
|
|
}
|
|
}`
|
|
|
|
txn := graphClient.NewTxn()
|
|
ret, err := txn.QueryWithVars(ctx, query, map[string]string{"$a": scanner.Text()})
|
|
if err != nil {
|
|
logrus.Warn(err)
|
|
}
|
|
|
|
n := Result{}
|
|
json.Unmarshal([]byte(ret.Json), &n)
|
|
|
|
// Check if the target already exists, if so, skipping not inserting
|
|
// the data
|
|
if len(n.Result) == 0 {
|
|
logrus.Info("new matcher, charging...")
|
|
mu := &api.Mutation{
|
|
CommitNow: true,
|
|
}
|
|
|
|
pb, err := json.Marshal(matcher)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
return nil, err
|
|
}
|
|
|
|
mu.SetJson = pb
|
|
|
|
txn = graphClient.NewTxn()
|
|
defer txn.Discard(ctx)
|
|
|
|
_, err = txn.Mutate(ctx, mu)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
return nil, err
|
|
}
|
|
|
|
}
|
|
|
|
res = append(res, scanner.Text())
|
|
if err := scanner.Err(); err != nil {
|
|
logrus.Error(err)
|
|
return nil, err
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
return res, nil
|
|
}
|
|
|
|
// Run runs the routine trying to find matches in the ingested Pastebin data.
|
|
func (m *Matcher) Run(wg *sync.WaitGroup, graphClient *dgo.Dgraph) {
|
|
logrus.Info("loading matcher targets, this might take some time...")
|
|
targets, err := loadTargets(graphClient)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
logrus.Info("finished loading matcher targets")
|
|
fmt.Println(targets)
|
|
|
|
if !m.Running {
|
|
m.StoppedChan = make(chan bool)
|
|
wg.Add(1)
|
|
logrus.Info("Running matchers")
|
|
for _, target := range targets {
|
|
go runPasteMatcher(target, graphClient)
|
|
go runCertstreamMatcher(target, graphClient)
|
|
go runShodanMatcher(target, graphClient)
|
|
}
|
|
|
|
// TODO: probably not the best design here
|
|
wg.Add(len(targets))
|
|
m.Running = true
|
|
}
|
|
}
|
|
|
|
func runPasteMatcher(target string, graphClient *dgo.Dgraph) {
|
|
for {
|
|
q := `query allofterms($a: string) {
|
|
Node(func: allofterms(full, $a)) {
|
|
uid
|
|
type
|
|
full
|
|
}
|
|
|
|
}`
|
|
|
|
ctx := context.Background()
|
|
txn := graphClient.NewTxn()
|
|
res, err := txn.QueryWithVars(ctx, q, map[string]string{"$a": target})
|
|
if err != nil {
|
|
logrus.Warn(err)
|
|
}
|
|
|
|
n := Result{}
|
|
json.Unmarshal([]byte(res.Json), &n)
|
|
uuid := uuid.New().String()
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
matcher := models.Match{
|
|
ID: uuid,
|
|
Timestamp: rfc3339time,
|
|
Target: target,
|
|
Nodes: []models.Node{},
|
|
Type: "matcher",
|
|
}
|
|
if len(n.Result) != 0 {
|
|
time.Sleep(time.Duration(2) * time.Second)
|
|
logrus.Info("Found paste match for ", target)
|
|
// TODO: review time and id to be updated on new resulsts
|
|
|
|
for _, res := range n.Result {
|
|
if len(matcher.Nodes) == 0 {
|
|
matcher.Nodes = append(matcher.Nodes, res)
|
|
continue
|
|
}
|
|
|
|
for _, node := range matcher.Nodes {
|
|
if res.UID != node.UID {
|
|
matcher.Nodes = append(matcher.Nodes, res)
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
query := fmt.Sprintf(`query { match as var(func: eq(target, "%s")) }`, target)
|
|
|
|
pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, Type: "matcher", Timestamp: rfc3339time})
|
|
if err != nil {
|
|
logrus.Fatal(err)
|
|
}
|
|
|
|
mu := &api.Mutation{
|
|
SetJson: pb,
|
|
}
|
|
|
|
req := &api.Request{
|
|
Query: query,
|
|
Mutations: []*api.Mutation{mu},
|
|
CommitNow: true,
|
|
}
|
|
|
|
txn := graphClient.NewTxn()
|
|
_, err = txn.Do(ctx, req)
|
|
if err != nil {
|
|
logrus.Warn(err)
|
|
}
|
|
// txn.Discard(ctx)
|
|
time.Sleep(time.Duration(2) * time.Second)
|
|
}
|
|
|
|
}
|
|
}
|
|
|
|
func runCertstreamMatcher(target string, graphClient *dgo.Dgraph) {
|
|
for {
|
|
q := `query allofterms($a: string){
|
|
Node(func: allofterms(cn, $a)){
|
|
fingerprint
|
|
notBefore
|
|
notAfter
|
|
cn
|
|
sourceName
|
|
basicConstraints
|
|
raw
|
|
}
|
|
}`
|
|
|
|
ctx := context.Background()
|
|
txn := graphClient.NewTxn()
|
|
res, err := txn.QueryWithVars(ctx, q, map[string]string{"$a": target})
|
|
if err != nil {
|
|
logrus.Warn(err)
|
|
}
|
|
|
|
n := Result{}
|
|
json.Unmarshal([]byte(res.Json), &n)
|
|
uuid := uuid.New().String()
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
matcher := models.Match{
|
|
ID: uuid,
|
|
Timestamp: rfc3339time,
|
|
Target: target,
|
|
Nodes: []models.Node{},
|
|
Type: "matcher",
|
|
}
|
|
if len(n.Result) != 0 {
|
|
time.Sleep(time.Duration(2) * time.Second)
|
|
logrus.Info("Found certstream match for ", target)
|
|
|
|
for _, res := range n.Result {
|
|
if len(matcher.Nodes) == 0 {
|
|
matcher.Nodes = append(matcher.Nodes, res)
|
|
continue
|
|
}
|
|
|
|
for _, nodes := range matcher.Nodes {
|
|
if res.UID != nodes.UID {
|
|
matcher.Nodes = append(matcher.Nodes, res)
|
|
}
|
|
}
|
|
}
|
|
|
|
query := fmt.Sprintf(`query { match as var(func: eq(target, "%s")) }`, target)
|
|
|
|
pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, Type: "matcher"})
|
|
if err != nil {
|
|
logrus.Fatal(err)
|
|
}
|
|
|
|
mu := &api.Mutation{
|
|
SetJson: pb,
|
|
}
|
|
|
|
req := &api.Request{
|
|
Query: query,
|
|
Mutations: []*api.Mutation{mu},
|
|
CommitNow: true,
|
|
}
|
|
|
|
txn := graphClient.NewTxn()
|
|
_, err = txn.Do(ctx, req)
|
|
if err != nil {
|
|
logrus.Warn(err)
|
|
}
|
|
|
|
time.Sleep(time.Duration(2) * time.Second)
|
|
}
|
|
}
|
|
}
|
|
|
|
func runShodanMatcher(target string, graphClient *dgo.Dgraph) {
|
|
for {
|
|
q := `query allofterms($a: string){
|
|
Node(func: eq(hostnames, $a)){
|
|
uid
|
|
type
|
|
hostnames
|
|
product
|
|
version
|
|
title
|
|
ip
|
|
os
|
|
organization
|
|
isp
|
|
cpe
|
|
asn
|
|
port
|
|
html
|
|
banner
|
|
link
|
|
transport
|
|
domains
|
|
timestamp
|
|
}
|
|
}
|
|
`
|
|
ctx := context.Background()
|
|
txn := graphClient.NewTxn()
|
|
res, err := txn.QueryWithVars(ctx, q, map[string]string{"$a": target})
|
|
if err != nil {
|
|
logrus.Warn(err)
|
|
}
|
|
|
|
n := Result{}
|
|
json.Unmarshal([]byte(res.Json), &n)
|
|
uuid := uuid.New().String()
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
matcher := models.Match{
|
|
ID: uuid,
|
|
Timestamp: rfc3339time,
|
|
Target: target,
|
|
Nodes: []models.Node{},
|
|
Type: "matcher",
|
|
}
|
|
if len(n.Result) != 0 {
|
|
time.Sleep(time.Duration(2) * time.Second)
|
|
logrus.Info("Found shodan match for ", target)
|
|
// TODO: review time and id to be updated on new resulsts
|
|
|
|
for _, res := range n.Result {
|
|
if len(matcher.Nodes) == 0 {
|
|
matcher.Nodes = append(matcher.Nodes, res)
|
|
continue
|
|
}
|
|
|
|
for _, node := range matcher.Nodes {
|
|
if res.UID != node.UID {
|
|
matcher.Nodes = append(matcher.Nodes, res)
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
query := fmt.Sprintf(`query { match as var(func: eq(target, "%s")) }`, target)
|
|
|
|
pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, Type: "matcher", Timestamp: rfc3339time})
|
|
if err != nil {
|
|
logrus.Fatal(err)
|
|
}
|
|
|
|
mu := &api.Mutation{
|
|
SetJson: pb,
|
|
}
|
|
|
|
req := &api.Request{
|
|
Query: query,
|
|
Mutations: []*api.Mutation{mu},
|
|
CommitNow: true,
|
|
}
|
|
|
|
txn := graphClient.NewTxn()
|
|
_, err = txn.Do(ctx, req)
|
|
if err != nil {
|
|
logrus.Warn(err)
|
|
}
|
|
// txn.Discard(ctx)
|
|
time.Sleep(time.Duration(2) * time.Second)
|
|
|
|
}
|
|
}
|
|
}
|
|
|
|
// RunDomainMatch looks for a target within the identified IOCs in /matcher/data.
|
|
// the function could be refactored with RunDomainFilters
|
|
func RunDomainMatch(domain string) bool {
|
|
path := basepath + "/data/domains.txt"
|
|
|
|
sliceDomain, err := ioutil.ReadDir(path)
|
|
if err != nil {
|
|
logrus.Warn("matcher#ReadDir#domains", err)
|
|
}
|
|
|
|
for _, file := range sliceDomain {
|
|
f, err := os.OpenFile(path+file.Name(), 1, 0644)
|
|
if err != nil {
|
|
logrus.Warn("matcher#OpenFile#", err)
|
|
}
|
|
scanner := bufio.NewScanner(f)
|
|
|
|
for scanner.Scan() {
|
|
r, err := regexp.Compile(scanner.Text())
|
|
if err != nil {
|
|
logrus.Warn("matcher#Compile#", err)
|
|
}
|
|
if r.MatchString(domain) {
|
|
return false
|
|
}
|
|
}
|
|
}
|
|
return true
|
|
}
|