diff --git a/matcher/.main.go.swp b/matcher/.main.go.swp deleted file mode 100644 index d0ca3aa..0000000 Binary files a/matcher/.main.go.swp and /dev/null differ diff --git a/matcher/main.go b/matcher/main.go index ea6bc8b..2d66646 100644 --- a/matcher/main.go +++ b/matcher/main.go @@ -59,13 +59,14 @@ type Result struct { Result []models.Node `json:"Node,omitempty"` } -func loadTargets(graphClient *dgo.Dgraph) error { +func loadTargets(graphClient *dgo.Dgraph) ([]string, error) { path := basepath + "/data/" + res := []string{} sliceDomain, err := ioutil.ReadDir(path) if err != nil { logrus.Warn("matcher#ReadDir#domains", err) - return err + return nil, err } for _, file := range sliceDomain { @@ -73,7 +74,7 @@ func loadTargets(graphClient *dgo.Dgraph) error { f, err := os.OpenFile(path+file.Name(), 0, 0644) if err != nil { logrus.Warn("matcher#OpenFile#", err) - return err + return nil, err } scanner := bufio.NewScanner(f) @@ -96,7 +97,7 @@ func loadTargets(graphClient *dgo.Dgraph) error { pb, err := json.Marshal(matcher) if err != nil { logrus.Error(err) - return err + return nil, err } mu.SetJson = pb @@ -107,24 +108,26 @@ func loadTargets(graphClient *dgo.Dgraph) error { _, err = txn.Mutate(ctx, mu) if err != nil { logrus.Error(err) - return err + return nil, err } + res = append(res, scanner.Text()) } if err := scanner.Err(); err != nil { logrus.Error(err) - return err + return nil, err } } - return nil + return res, nil } -// Run runs the routine trying to find matches in the ingested data. +// Run runs the routine trying to find matches in the ingested Pastebin data. func (m *Matcher) Run(wg *sync.WaitGroup, graphClient *dgo.Dgraph) { - logrus.Info("loading matcher targets, this might take some time...") - if err := loadTargets(graphClient); err != nil { + // logrus.Info("loading matcher targets, this might take some time...") + targets, err := loadTargets(graphClient) + if err != nil { logrus.Error(err) } logrus.Info("finished loading matcher targets") @@ -132,10 +135,9 @@ func (m *Matcher) Run(wg *sync.WaitGroup, graphClient *dgo.Dgraph) { if !m.Running { m.StoppedChan = make(chan bool) wg.Add(1) - targets := []string{"code", "password", "login", "covid", "coronavirus", "java", "php", "function"} - for _, target := range targets { - go runMatcher(target, graphClient) + go runPasteMatcher(target, graphClient) + go runCertstreamMatcher(target, graphClient) } // TODO: probably not the best design here @@ -144,7 +146,7 @@ func (m *Matcher) Run(wg *sync.WaitGroup, graphClient *dgo.Dgraph) { } } -func runMatcher(target string, graphClient *dgo.Dgraph) { +func runPasteMatcher(target string, graphClient *dgo.Dgraph) { logrus.Info("Running matcher for ", target) for { q := `query allofterms($a: string) { @@ -223,6 +225,85 @@ Node(func: allofterms(full, $a)) { } } +func runCertstreamMatcher(target string, graphClient *dgo.Dgraph) { + logrus.Info("Running Certstream matcher") + for { + q := `query allofterms($a: string){ +Node(func: allofterms(cn, $a)){ + fingerprint + notBefore + notAfter + cn + sourceName + basicConstraints + raw +} +}` + + ctx := context.Background() + txn := graphClient.NewTxn() + res, err := txn.QueryWithVars(ctx, q, map[string]string{"$a": target}) + if err != nil { + logrus.Warn(err) + } + + n := Result{} + json.Unmarshal([]byte(res.Json), &n) + uuid := uuid.New().String() + t := time.Now() + rfc3339time := t.Format(time.RFC3339) + matcher := models.Match{ + ID: uuid, + Timestamp: rfc3339time, + Target: target, + Nodes: []models.Node{}, + Type: "matcher", + } + if len(n.Result) != 0 { + time.Sleep(time.Duration(2) * time.Second) + logrus.Info("Found certstream match for ", target) + + for _, res := range n.Result { + if len(matcher.Nodes) == 0 { + matcher.Nodes = append(matcher.Nodes, res) + continue + } + + for _, nodes := range matcher.Nodes { + if res.UID != nodes.UID { + matcher.Nodes = append(matcher.Nodes, res) + } + } + } + + query := fmt.Sprintf(`query { match as var(func: eq(target, "%s")) }`, target) + + pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, Type: "matcher"}) + if err != nil { + logrus.Fatal(err) + } + + mu := &api.Mutation{ + SetJson: pb, + } + + req := &api.Request{ + Query: query, + Mutations: []*api.Mutation{mu}, + CommitNow: true, + } + + txn := graphClient.NewTxn() + _, err = txn.Do(ctx, req) + if err != nil { + logrus.Warn(err) + } + + time.Sleep(time.Duration(2) * time.Second) + } + } +} + // RunDomainMatch looks for a target within the identified IOCs in /matcher/data. // the function could be refactored with RunDomainFilters func RunDomainMatch(domain string) bool {