multiple search works (kinda)

DEMO.md

@@ -0,0 +1,23 @@
+# Demo notes
+
+```graphql
+{
+  Node(func: eq(type, "matcher")){
+    id
+    target
+    type
+    full
+    nodes {
+      uid
+      full
+    }
+  }
+}
+```
+
+## Notes
+
+* Currently only runs matchers on Pastebin data
+* Upsert is not optimal
+* What do we do with the data so it can be exploitable by analysts
+* Sould we store matched data in an SQL-like db?

matcher/main.go

@@ -69,6 +69,7 @@ func loadTargets(graphClient *dgo.Dgraph) error {
 	}
 
 	for _, file := range sliceDomain {
+		logrus.Info("loading: ", file.Name(), " please wait...")
 		f, err := os.OpenFile(path+file.Name(), 0, 0644)
 		if err != nil {
 			logrus.Warn("matcher#OpenFile#", err)
@@ -100,12 +101,14 @@ func loadTargets(graphClient *dgo.Dgraph) error {
 
 			mu.SetJson = pb
 
-			_, err = graphClient.NewTxn().Mutate(ctx, mu)
+			txn := graphClient.NewTxn()
+			defer txn.Discard(ctx)
+
+			_, err = txn.Mutate(ctx, mu)
 			if err != nil {
 				logrus.Error(err)
 				return err
 			}
-			logrus.Info("adding", scanner.Text())
 
 		}
 		if err := scanner.Err(); err != nil {
@@ -120,93 +123,101 @@ func loadTargets(graphClient *dgo.Dgraph) error {
 
 // Run runs the routine trying to find matches in the ingested data.
 func (m *Matcher) Run(wg *sync.WaitGroup, graphClient *dgo.Dgraph) {
-	logrus.Info("loading matcher targets")
+	logrus.Info("loading matcher targets, this might take some time...")
 	if err := loadTargets(graphClient); err != nil {
 		logrus.Error(err)
 	}
 	logrus.Info("finished loading matcher targets")
-	// Created nodes based on the IOCs
+
-	// Upsert those nodes if the values are found
 	if !m.Running {
 		m.StoppedChan = make(chan bool)
 		wg.Add(1)
-		for {
+		targets := []string{"code", "password", "login", "covid", "coronavirus", "java"}
-			q := `query allofterms($a: string) {
+		for _, target := range targets {
-		 Node(func: allofterms(full, $a)) {
+			go runMatcher(target, graphClient)
-		 uid
+		}
-		 type
+		// TODO: probably not the best design here
-		 full
+		wg.Add(len(targets))
-		 }
+		m.Running = true
+	}
+}
+
+func runMatcher(target string, graphClient *dgo.Dgraph) {
+	logrus.Info("Running matcher for ", target)
+	for {
+		q := `query allofterms($a: string) {
+Node(func: allofterms(full, $a)) {
+			 uid
+			 type
+			 full
+			 }
 
 		 }`
 
-			ctx := context.Background()
+		ctx := context.Background()
-			txn := graphClient.NewTxn()
+		txn := graphClient.NewTxn()
-			defer txn.Discard(ctx)
+		defer txn.Discard(ctx)
-			res, err := txn.QueryWithVars(ctx, q, map[string]string{"$a": "code"})
+		res, err := txn.QueryWithVars(ctx, q, map[string]string{"$a": target})
-			if err != nil {
+		if err != nil {
-				logrus.Warn(err)
+			logrus.Warn(err)
-			}
+		}
 
-			n := Result{}
+		n := Result{}
-			json.Unmarshal([]byte(res.Json), &n)
+		json.Unmarshal([]byte(res.Json), &n)
-			uuid := uuid.New().String()
+		uuid := uuid.New().String()
-			t := time.Now()
+		t := time.Now()
-			rfc3339time := t.Format(time.RFC3339)
+		rfc3339time := t.Format(time.RFC3339)
-			matcher := models.Match{
+		matcher := models.Match{
-				ID:        uuid,
+			ID:        uuid,
-				Timestamp: rfc3339time,
+			Timestamp: rfc3339time,
-				Target:    "java",
+			Target:    target,
-				Nodes:     []models.Node{},
+			Nodes:     []models.Node{},
-				Type:      "matcher",
+			Type:      "matcher",
-			}
+		}
-			if len(n.Result) != 0 {
+		if len(n.Result) != 0 {
-				fmt.Println("first query res", res)
+			time.Sleep(3)
-				// TODO: review time and id to be updated on new resulsts
+			// TODO: review time and id to be updated on new resulsts
 
-				for _, res := range n.Result {
+			for _, res := range n.Result {
-					if len(matcher.Nodes) == 0 {
+				if len(matcher.Nodes) == 0 {
+					matcher.Nodes = append(matcher.Nodes, res)
+					continue
+				}
+
+				for _, node := range matcher.Nodes {
+					if res.UID != node.UID {
 						matcher.Nodes = append(matcher.Nodes, res)
-						continue
+
 					}
-
-					for _, node := range matcher.Nodes {
-						if res.UID != node.UID {
-							matcher.Nodes = append(matcher.Nodes, res)
-
-						}
-					}
-				}
-
-				query := `query { match as var(func: eq(target, "code")) } `
-
-				fmt.Println("nodes", matcher.Nodes)
-				pb, err := json.Marshal(models.Match{UID: "uid(match)", Target: "code", Nodes: matcher.Nodes, Type: "matcher"})
-				if err != nil {
-					logrus.Fatal(err)
-				}
-
-				mu := &api.Mutation{
-					SetJson: pb,
-				}
-
-				req := &api.Request{
-					Query:     query,
-					Mutations: []*api.Mutation{mu},
-					CommitNow: true,
-				}
-
-				ret, err := graphClient.NewTxn().Do(ctx, req)
-				fmt.Println(ret)
-				if err != nil {
-					logrus.Fatal(err)
 				}
 			}
 
-			m.Running = true
+			query := fmt.Sprintf(`query { match as var(func: eq(target, "%s")) }`, target)
+
+			pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, Type: "matcher"})
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			mu := &api.Mutation{
+				SetJson:   pb,
+				CommitNow: true,
+			}
+
+			req := &api.Request{
+				Query:     query,
+				Mutations: []*api.Mutation{mu},
+				CommitNow: true,
+			}
+
+			txn := graphClient.NewTxn()
+			_, err = txn.Do(ctx, req)
+			if err != nil {
+				logrus.Fatal(err)
+			}
+			time.Sleep(2)
 		}
 	}
-
 }
 
 // RunDomainMatch looks for a target within the identified IOCs in /matcher/data.