diff --git a/DEMO.md b/DEMO.md new file mode 100644 index 0000000..edbfc28 --- /dev/null +++ b/DEMO.md @@ -0,0 +1,23 @@ +# Demo notes + +```graphql +{ + Node(func: eq(type, "matcher")){ + id + target + type + full + nodes { + uid + full + } + } +} +``` + +## Notes + +* Currently only runs matchers on Pastebin data +* Upsert is not optimal +* What do we do with the data so it can be exploitable by analysts +* Sould we store matched data in an SQL-like db? diff --git a/matcher/main.go b/matcher/main.go index cfd464b..4219ae3 100644 --- a/matcher/main.go +++ b/matcher/main.go @@ -69,6 +69,7 @@ func loadTargets(graphClient *dgo.Dgraph) error { } for _, file := range sliceDomain { + logrus.Info("loading: ", file.Name(), " please wait...") f, err := os.OpenFile(path+file.Name(), 0, 0644) if err != nil { logrus.Warn("matcher#OpenFile#", err) @@ -100,12 +101,14 @@ func loadTargets(graphClient *dgo.Dgraph) error { mu.SetJson = pb - _, err = graphClient.NewTxn().Mutate(ctx, mu) + txn := graphClient.NewTxn() + defer txn.Discard(ctx) + + _, err = txn.Mutate(ctx, mu) if err != nil { logrus.Error(err) return err } - logrus.Info("adding", scanner.Text()) } if err := scanner.Err(); err != nil { @@ -120,93 +123,101 @@ func loadTargets(graphClient *dgo.Dgraph) error { // Run runs the routine trying to find matches in the ingested data. func (m *Matcher) Run(wg *sync.WaitGroup, graphClient *dgo.Dgraph) { - logrus.Info("loading matcher targets") + logrus.Info("loading matcher targets, this might take some time...") if err := loadTargets(graphClient); err != nil { logrus.Error(err) } logrus.Info("finished loading matcher targets") - // Created nodes based on the IOCs - // Upsert those nodes if the values are found + if !m.Running { m.StoppedChan = make(chan bool) wg.Add(1) - for { - q := `query allofterms($a: string) { - Node(func: allofterms(full, $a)) { - uid - type - full - } + targets := []string{"code", "password", "login", "covid", "coronavirus", "java"} + for _, target := range targets { + go runMatcher(target, graphClient) + } + // TODO: probably not the best design here + wg.Add(len(targets)) + m.Running = true + } +} + +func runMatcher(target string, graphClient *dgo.Dgraph) { + logrus.Info("Running matcher for ", target) + for { + q := `query allofterms($a: string) { +Node(func: allofterms(full, $a)) { + uid + type + full + } }` - ctx := context.Background() - txn := graphClient.NewTxn() - defer txn.Discard(ctx) - res, err := txn.QueryWithVars(ctx, q, map[string]string{"$a": "code"}) - if err != nil { - logrus.Warn(err) - } + ctx := context.Background() + txn := graphClient.NewTxn() + defer txn.Discard(ctx) + res, err := txn.QueryWithVars(ctx, q, map[string]string{"$a": target}) + if err != nil { + logrus.Warn(err) + } - n := Result{} - json.Unmarshal([]byte(res.Json), &n) - uuid := uuid.New().String() - t := time.Now() - rfc3339time := t.Format(time.RFC3339) - matcher := models.Match{ - ID: uuid, - Timestamp: rfc3339time, - Target: "java", - Nodes: []models.Node{}, - Type: "matcher", - } - if len(n.Result) != 0 { - fmt.Println("first query res", res) - // TODO: review time and id to be updated on new resulsts + n := Result{} + json.Unmarshal([]byte(res.Json), &n) + uuid := uuid.New().String() + t := time.Now() + rfc3339time := t.Format(time.RFC3339) + matcher := models.Match{ + ID: uuid, + Timestamp: rfc3339time, + Target: target, + Nodes: []models.Node{}, + Type: "matcher", + } + if len(n.Result) != 0 { + time.Sleep(3) + // TODO: review time and id to be updated on new resulsts - for _, res := range n.Result { - if len(matcher.Nodes) == 0 { + for _, res := range n.Result { + if len(matcher.Nodes) == 0 { + matcher.Nodes = append(matcher.Nodes, res) + continue + } + + for _, node := range matcher.Nodes { + if res.UID != node.UID { matcher.Nodes = append(matcher.Nodes, res) - continue + } - - for _, node := range matcher.Nodes { - if res.UID != node.UID { - matcher.Nodes = append(matcher.Nodes, res) - - } - } - } - - query := `query { match as var(func: eq(target, "code")) } ` - - fmt.Println("nodes", matcher.Nodes) - pb, err := json.Marshal(models.Match{UID: "uid(match)", Target: "code", Nodes: matcher.Nodes, Type: "matcher"}) - if err != nil { - logrus.Fatal(err) - } - - mu := &api.Mutation{ - SetJson: pb, - } - - req := &api.Request{ - Query: query, - Mutations: []*api.Mutation{mu}, - CommitNow: true, - } - - ret, err := graphClient.NewTxn().Do(ctx, req) - fmt.Println(ret) - if err != nil { - logrus.Fatal(err) } } - m.Running = true + query := fmt.Sprintf(`query { match as var(func: eq(target, "%s")) }`, target) + + pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, Type: "matcher"}) + if err != nil { + logrus.Fatal(err) + } + + mu := &api.Mutation{ + SetJson: pb, + CommitNow: true, + } + + req := &api.Request{ + Query: query, + Mutations: []*api.Mutation{mu}, + CommitNow: true, + } + + txn := graphClient.NewTxn() + _, err = txn.Do(ctx, req) + if err != nil { + logrus.Fatal(err) + } + time.Sleep(2) } } - } // RunDomainMatch looks for a target within the identified IOCs in /matcher/data.