Adding certstream matching on CN

matcher/main.go

@@ -59,13 +59,14 @@ type Result struct {
 	Result []models.Node `json:"Node,omitempty"`
 }
 
-func loadTargets(graphClient *dgo.Dgraph) error {
+func loadTargets(graphClient *dgo.Dgraph) ([]string, error) {
 	path := basepath + "/data/"
+	res := []string{}
 
 	sliceDomain, err := ioutil.ReadDir(path)
 	if err != nil {
 		logrus.Warn("matcher#ReadDir#domains", err)
-		return err
+		return nil, err
 	}
 
 	for _, file := range sliceDomain {
@@ -73,7 +74,7 @@ func loadTargets(graphClient *dgo.Dgraph) error {
 		f, err := os.OpenFile(path+file.Name(), 0, 0644)
 		if err != nil {
 			logrus.Warn("matcher#OpenFile#", err)
-			return err
+			return nil, err
 		}
 		scanner := bufio.NewScanner(f)
 
@@ -96,7 +97,7 @@ func loadTargets(graphClient *dgo.Dgraph) error {
 			pb, err := json.Marshal(matcher)
 			if err != nil {
 				logrus.Error(err)
-				return err
+				return nil, err
 			}
 
 			mu.SetJson = pb
@@ -107,24 +108,26 @@ func loadTargets(graphClient *dgo.Dgraph) error {
 			_, err = txn.Mutate(ctx, mu)
 			if err != nil {
 				logrus.Error(err)
-				return err
+				return nil, err
 			}
+			res = append(res, scanner.Text())
 
 		}
 		if err := scanner.Err(); err != nil {
 			logrus.Error(err)
-			return err
+			return nil, err
 		}
 
 	}
 
-	return nil
+	return res, nil
 }
 
-// Run runs the routine trying to find matches in the ingested data.
+// Run runs the routine trying to find matches in the ingested Pastebin data.
 func (m *Matcher) Run(wg *sync.WaitGroup, graphClient *dgo.Dgraph) {
-	logrus.Info("loading matcher targets, this might take some time...")
-	if err := loadTargets(graphClient); err != nil {
+	// logrus.Info("loading matcher targets, this might take some time...")
+	targets, err := loadTargets(graphClient)
+	if err != nil {
 		logrus.Error(err)
 	}
 	logrus.Info("finished loading matcher targets")
@@ -132,10 +135,9 @@ func (m *Matcher) Run(wg *sync.WaitGroup, graphClient *dgo.Dgraph) {
 	if !m.Running {
 		m.StoppedChan = make(chan bool)
 		wg.Add(1)
-		targets := []string{"code", "password", "login", "covid", "coronavirus", "java", "php", "function"}
-
 		for _, target := range targets {
-			go runMatcher(target, graphClient)
+			go runPasteMatcher(target, graphClient)
+			go runCertstreamMatcher(target, graphClient)
 		}
 
 		// TODO: probably not the best design here
@@ -144,7 +146,7 @@ func (m *Matcher) Run(wg *sync.WaitGroup, graphClient *dgo.Dgraph) {
 	}
 }
 
-func runMatcher(target string, graphClient *dgo.Dgraph) {
+func runPasteMatcher(target string, graphClient *dgo.Dgraph) {
 	logrus.Info("Running matcher for ", target)
 	for {
 		q := `query allofterms($a: string) {
@@ -223,6 +225,85 @@ Node(func: allofterms(full, $a)) {
 	}
 }
 
+func runCertstreamMatcher(target string, graphClient *dgo.Dgraph) {
+	logrus.Info("Running Certstream matcher")
+	for {
+		q := `query allofterms($a: string){
+Node(func: allofterms(cn, $a)){
+		fingerprint      
+		notBefore        
+		notAfter         
+		cn               
+		sourceName       
+		basicConstraints 
+		raw              
+}
+}`
+
+		ctx := context.Background()
+		txn := graphClient.NewTxn()
+		res, err := txn.QueryWithVars(ctx, q, map[string]string{"$a": target})
+		if err != nil {
+			logrus.Warn(err)
+		}
+
+		n := Result{}
+		json.Unmarshal([]byte(res.Json), &n)
+		uuid := uuid.New().String()
+		t := time.Now()
+		rfc3339time := t.Format(time.RFC3339)
+		matcher := models.Match{
+			ID:        uuid,
+			Timestamp: rfc3339time,
+			Target:    target,
+			Nodes:     []models.Node{},
+			Type:      "matcher",
+		}
+		if len(n.Result) != 0 {
+			time.Sleep(time.Duration(2) * time.Second)
+			logrus.Info("Found certstream match for ", target)
+
+			for _, res := range n.Result {
+				if len(matcher.Nodes) == 0 {
+					matcher.Nodes = append(matcher.Nodes, res)
+					continue
+				}
+
+				for _, nodes := range matcher.Nodes {
+					if res.UID != nodes.UID {
+						matcher.Nodes = append(matcher.Nodes, res)
+					}
+				}
+			}
+
+			query := fmt.Sprintf(`query { match as var(func: eq(target, "%s")) }`, target)
+
+			pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, Type: "matcher"})
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			mu := &api.Mutation{
+				SetJson: pb,
+			}
+
+			req := &api.Request{
+				Query:     query,
+				Mutations: []*api.Mutation{mu},
+				CommitNow: true,
+			}
+
+			txn := graphClient.NewTxn()
+			_, err = txn.Do(ctx, req)
+			if err != nil {
+				logrus.Warn(err)
+			}
+
+			time.Sleep(time.Duration(2) * time.Second)
+		}
+	}
+}
+
 // RunDomainMatch looks for a target within the identified IOCs in /matcher/data.
 // the function could be refactored with RunDomainFilters
 func RunDomainMatch(domain string) bool {