From 44bc03b51dd514827d61c0643e28769513d9721c Mon Sep 17 00:00:00 2001
From: Christopher Talib <christopher.talib@dcso.de>
Date: Mon, 8 Jun 2020 12:58:07 +0200
Subject: [PATCH] Adding certstream matching on CN

---
 matcher/.main.go.swp | Bin 16384 -> 0 bytes
 matcher/main.go      | 109 +++++++++++++++++++++++++++++++++++++------
 2 files changed, 95 insertions(+), 14 deletions(-)
 delete mode 100644 matcher/.main.go.swp

diff --git a/matcher/.main.go.swp b/matcher/.main.go.swp
deleted file mode 100644
index d0ca3aab219f9bbbaf4414bf37210010b4665e6f..0000000000000000000000000000000000000000
GIT binary patch
literal 0
HcmV?d00001

literal 16384
zcmeI2eXJZ+6~G4+WWmw~qLGm3?Y7q4@;ciOBJiLf?W3fGzDnOKB-BFZ?auC-GCMP!
znSHO%QpNaz8Z;&*h7ZNWKu8n}m}m%66h9IZl|NKI3}O_65<&S${6m7^@7#N5-|p*c
zi$RTvv+1w1d+*G>_ndp~x%ZrTQ){o>x=o$yu5<V-bDTr3Jax^DC*FJ9%{v{ZIh_WX
zlpo`>>GJWlXOC}MyY8HGj(X=ubc2mf+{n^qqZ8zzH|4gPS?q>!*!1$o<ri<7Xk_`^
z{zli!o6~;EHHh3!eAH&k%2ETR29BkHUfy1F?h<Fi`gLpN))m#|>h#lC<mIQ-K&gRJ
z1EmH^4U`%vHBf4x)Ih0${~HbD$??udk@k2i@g4Sh+=A;{?C-1X_09$L_Gfu1HBf4x
z)Ih0$QUj$1N)416C^b-OpwvLAfl>pd2L1;%;I$k_V*Pt00l@qJBL07IspGr=e}t#t
z7w|AV0QbOM@FlnjJ_d8J4-&Wn-UmzIg;O2pQTQg@4c~w};giq-A6l>yPK8&OIL<Th
z2>c9w3Ll5vumM)Wa!~Nf+a2dG@ErUZ4#5}T2Jm1TTnOvo-EbzH0q=sJpW-<8!)M{s
za3k!5bKx9V0<WITIN)J;2oAw5a1ERVXTXaU$9Wol3y0uAxD!4F8C(V~oB{thiTmIo
zxC6clH^4MZz&LD!I=mAcc<F79^E^BUPr&1FKim#qfrGFYCgBqJ09*v;!|CwaiHs4R
zhv(o~_!B${zlJB^i!cr+!DA;l&JW;w@Ok(gTn|^nI6V7So`)a7?QkpH0wL^#O>hxx
zgbUz!c$DPPBXAdd0<MP)_CgDqup2%E7lMMvvBw|7eV}bq)=s5z5GrBZNqd=lm6t{}
zKTYeD1#QJQwS1$Bf>0fxB=xfqT{gN=<@+P|ik_d&T@~chS9)nyYv%h$)V>kb^^#p#
zo(558x49gsuJo#-D)-WkpC6Q_d05d#C-su)&0*k2xjXL9PVSFtQcFWp+l-^!-=Dji
zyykVCH10($YN=KS_sE5-{=eM|L;WS6OMLyM7qp!5xaHT{y{I{=yfBR0xu1433c_$i
zt@P@3EzdsbX3sv6V(_k&-l#Go*SEFfRPMGK+8VXvT6IS+iVzT4_21Mexu-f}MXWp0
zu}XW<Hd9?QqT5DC2IW?(Tb<Y))0&Xp9E1|Nx^B@0s!n?%*lfqWxYeqKeq`Ir>Y}Yi
zLrrcU+dit2G@kON!Z{Vixyq+~HRWfyYWZ2vi4=eM<M$`K6CXafp;1Y@BGR>mNTR6J
zi#om)F_WP(k9tZosom9|>?W#57uCeeve`Io$#;|&MDnE>&jh-p8K-gN%>-DTl<)Os
zJpIsgA{S9$9>h^qbYTTwC!^}R6M38@erxlz7paX*z;%ADDc`5!IOGnjcKN*WWRyb_
z*AAi}oAz5OjJ?+23|djDG$DQ3hb~FeIJFDHt_Nh5XD5y5wE-KhpTA(i4E2134$<&*
zkg0CanP%3IMrCmqH3wbabzO#|ZK>8(tF1BB6_!z(?Won+T-0R62Kl9|j$~*87n&7D
z!c#KqnKlg`$b*P((m7E=9xD{nvgVwrAkueHg$bdomX~|3w$0Sfdub#d0nHxlTa+!v
zU!c3ilvj8Yw3ju#DDqPm-PR6MnQ2o`f%qHUnQe8^W{Y+Qyi^6N<q?cSeAKpH?&X+>
zIHzt;JCI7tZ~LjP9Sayu+LCIquA@38sqgmO2|vFqi^cYmDUtupjWwC-d(^$O>Tbi6
zPJ3a^Iu<itF~~|~bKLC)`FK2wE5##>nDNNkTjeEz5tJn(j{2xBv_IrY9OkIfpQ`Pz
zs#~py_smSyuH9XJgqbU42M(HgWBmGtVm2pn)wQ~<#(1;b>n5WrZ8tY;*l?bhv#HxU
zX1i3XhojbowsR51cBXAzOES=a)aUM&IOYDDULZTRY?ju|)ahzjHmFGNBNgdDO64jW
zOs$Uhkc~-Ki-RjhKY>lwwEwMCY?(Yb)oTZ_JSp|qY^^O_>FF4@O}o<d_F3q1#rDLH
zwgjQS0*{74%A<$xsqOy!$C+C^QA@mh`m8q3y5nBgN3Uuv|K5N0+0w{jZe>)qpP>h;
zDhYir^VKXK!nl&cr^|gJp7H2e7&QH{SpF<jdvy_SNV-LZ+u?hyu^?U1(hrIcHzFP;
z;Xw0!Q3G*5p=;aF(u$>v)LQKfTYo=GRU6`a8dc*4kKlNRy&dDwJGN}4JN&E{=4?&_
zMy&S8N*k?;L65{NDZiWKbJabL93Yf^g8jeO%+ZH<PSXfpT6R?qLDFP9jl24{HePX}
z`W)e^4;I^SC$_Ftwx^D7{HG{)etX)e+FqF1&3!D6m}%-SSYviM+2Z)w4CBnN%@@fn
z^S_R;2hR6#V38d}_RvMP5Lp1oE46Y)Y5H<rn&a3KCr0+Zte8a59NH1gA!W!k`*))`
zkL`R0N8DmZGdrv`F{=RCwr7HbZQ)Ws-^5m4E0&Qv@F?Rud^_A41$p3w!H3bFJ_Q&l
z=8M_R%pI9OAO<_Y6n559_;Q)oPB7y~a_DRaonGq6>B9&(c;M)tG@ObQ%M6MC94Ic1
zZBi^p5@S`{>^X7Jtco&gSEj0$`uZ>_BkJo<Yg^57&}NBF@VDFy3@eb~>t&)a0$J1Y
z!(~y{rl@5XWN+dBYpG@9*A8<thhf~<+PX92rK+ZzsUH2W#<lmaIu*HXx;N!E<8C8M
z+Uqtn^n5L>;Xs#Ue$+6Qu%NaR$DPn`h(BIX)6(atH9_3qJlvRB-{3Tx$614&?HG4f
zv8SpCC-=#bNUm01{nhXI`xE&l>S+{ul7cvs|LN2CBl*$p=5p~PPO&WChPVnTu)&CY
z)tf@q3A#y~=Ay%d0L9~Itmn&59f|+1CGPtJv7*HP#ryr&C3c6Oz}Mkskl21FTnwvV
z8N75VF(mv79)O#`1BvHXz;W<9V)c9BYw%^b4N{nd6;OpU;T^CHmV(6c55jq{3YNhs
za5B71Z2o)r4Lk`ygnQsKkV6a+Y=`ym9yl3JfWH#6e;*!%@524?9r!jhp$@NCh(F;m
zxEpSRF6@N>E`ig)fyar_e+l1$``}iHU>B^1HLw(3L%)BAzroA!42aI3hVoNtpwvLA
zfl>pd2L1yY;23Yuujbq<$9iEn8BiSVCG1NH?a39|_|rs9I=xstV~@_mAtW(>G8{oJ
z5<*teeqfdA!;Whos}<p`3C<;OcymGTqGK$A)_+TwyRIMR+Wh0OW7S0m_1_IoZ4m6{
z`#C-82!$}fgw%T0YvzW2R#^fA6Q#&GzesixeDuN=X<K5<Hc?NU5}nAYUvhV*wJkwN
zCOgwnX;G$oASW4Lm+8>KvpL!#V300?7CH8y9nj#`?VELiuVIg?I>2^q;<+$m9V|?Q
zzMq&x%b1ObYU}H2wMkA+_?(GbVpEcWmQ>rqWSU+OMYdw_&B9_fJe0H>i%r5V_4nxx
zj`qG_e$_3OAgk#7AF&+itX;cT4<*XFJ)6$b?=ehTN(b>eDMjCsi_p2X9X{n*u5;dY
ziQ6<FY4<6U!|L_#F1j4N-5Ti@k{My~f@Lu^95Ekgl5l#i5A2jAT9qKmqzUUJU!=rN
zsJnG+L=_o2lhz(lHltVEqJMIi&HxsXc9nNCqD`#>k%nN%ok&QS<?Wh%O>)5WqP$EI
zLE1Mg{yzIgg3SGUvnp>_4(=)59GE7JDVYCJp!C9LXf#Br9*KS4@(dM^NWw)MfzF=s
z!$o}kRtlBq?>bLd<hCM8W$Y%mHPjR<PIqjFAt7B5=~pZN`dIMj@*-xa{kqz?QLUvH
znzP=E;m!{xBH^WTM`j$X8uQ{N^=ATqR<A1Mk%cE|NkSes&`U^unG8k54K&sZvpkdb
ztv#8yQCsBYn4K8u7<(}DZWfQ^MBnIIwsI9it&5_KNLp)0Oc`7IlB~y2vdw0U3v;B}
zTBHau$;gh7Ctf5?Y;`UAT$m+(qe<di9@f6C$PphsMXXcuJEN{m%J14eHOIRIvBAs8
U3y_sv6^V-V%S<!32Qcn`0+5_bng9R*

diff --git a/matcher/main.go b/matcher/main.go
index ea6bc8b..2d66646 100644
--- a/matcher/main.go
+++ b/matcher/main.go
@@ -59,13 +59,14 @@ type Result struct {
 	Result []models.Node `json:"Node,omitempty"`
 }
 
-func loadTargets(graphClient *dgo.Dgraph) error {
+func loadTargets(graphClient *dgo.Dgraph) ([]string, error) {
 	path := basepath + "/data/"
+	res := []string{}
 
 	sliceDomain, err := ioutil.ReadDir(path)
 	if err != nil {
 		logrus.Warn("matcher#ReadDir#domains", err)
-		return err
+		return nil, err
 	}
 
 	for _, file := range sliceDomain {
@@ -73,7 +74,7 @@ func loadTargets(graphClient *dgo.Dgraph) error {
 		f, err := os.OpenFile(path+file.Name(), 0, 0644)
 		if err != nil {
 			logrus.Warn("matcher#OpenFile#", err)
-			return err
+			return nil, err
 		}
 		scanner := bufio.NewScanner(f)
 
@@ -96,7 +97,7 @@ func loadTargets(graphClient *dgo.Dgraph) error {
 			pb, err := json.Marshal(matcher)
 			if err != nil {
 				logrus.Error(err)
-				return err
+				return nil, err
 			}
 
 			mu.SetJson = pb
@@ -107,24 +108,26 @@ func loadTargets(graphClient *dgo.Dgraph) error {
 			_, err = txn.Mutate(ctx, mu)
 			if err != nil {
 				logrus.Error(err)
-				return err
+				return nil, err
 			}
+			res = append(res, scanner.Text())
 
 		}
 		if err := scanner.Err(); err != nil {
 			logrus.Error(err)
-			return err
+			return nil, err
 		}
 
 	}
 
-	return nil
+	return res, nil
 }
 
-// Run runs the routine trying to find matches in the ingested data.
+// Run runs the routine trying to find matches in the ingested Pastebin data.
 func (m *Matcher) Run(wg *sync.WaitGroup, graphClient *dgo.Dgraph) {
-	logrus.Info("loading matcher targets, this might take some time...")
-	if err := loadTargets(graphClient); err != nil {
+	// logrus.Info("loading matcher targets, this might take some time...")
+	targets, err := loadTargets(graphClient)
+	if err != nil {
 		logrus.Error(err)
 	}
 	logrus.Info("finished loading matcher targets")
@@ -132,10 +135,9 @@ func (m *Matcher) Run(wg *sync.WaitGroup, graphClient *dgo.Dgraph) {
 	if !m.Running {
 		m.StoppedChan = make(chan bool)
 		wg.Add(1)
-		targets := []string{"code", "password", "login", "covid", "coronavirus", "java", "php", "function"}
-
 		for _, target := range targets {
-			go runMatcher(target, graphClient)
+			go runPasteMatcher(target, graphClient)
+			go runCertstreamMatcher(target, graphClient)
 		}
 
 		// TODO: probably not the best design here
@@ -144,7 +146,7 @@ func (m *Matcher) Run(wg *sync.WaitGroup, graphClient *dgo.Dgraph) {
 	}
 }
 
-func runMatcher(target string, graphClient *dgo.Dgraph) {
+func runPasteMatcher(target string, graphClient *dgo.Dgraph) {
 	logrus.Info("Running matcher for ", target)
 	for {
 		q := `query allofterms($a: string) {
@@ -223,6 +225,85 @@ Node(func: allofterms(full, $a)) {
 	}
 }
 
+func runCertstreamMatcher(target string, graphClient *dgo.Dgraph) {
+	logrus.Info("Running Certstream matcher")
+	for {
+		q := `query allofterms($a: string){
+Node(func: allofterms(cn, $a)){
+		fingerprint      
+		notBefore        
+		notAfter         
+		cn               
+		sourceName       
+		basicConstraints 
+		raw              
+}
+}`
+
+		ctx := context.Background()
+		txn := graphClient.NewTxn()
+		res, err := txn.QueryWithVars(ctx, q, map[string]string{"$a": target})
+		if err != nil {
+			logrus.Warn(err)
+		}
+
+		n := Result{}
+		json.Unmarshal([]byte(res.Json), &n)
+		uuid := uuid.New().String()
+		t := time.Now()
+		rfc3339time := t.Format(time.RFC3339)
+		matcher := models.Match{
+			ID:        uuid,
+			Timestamp: rfc3339time,
+			Target:    target,
+			Nodes:     []models.Node{},
+			Type:      "matcher",
+		}
+		if len(n.Result) != 0 {
+			time.Sleep(time.Duration(2) * time.Second)
+			logrus.Info("Found certstream match for ", target)
+
+			for _, res := range n.Result {
+				if len(matcher.Nodes) == 0 {
+					matcher.Nodes = append(matcher.Nodes, res)
+					continue
+				}
+
+				for _, nodes := range matcher.Nodes {
+					if res.UID != nodes.UID {
+						matcher.Nodes = append(matcher.Nodes, res)
+					}
+				}
+			}
+
+			query := fmt.Sprintf(`query { match as var(func: eq(target, "%s")) }`, target)
+
+			pb, err := json.Marshal(models.Match{UID: "uid(match)", ID: matcher.ID, Target: target, Nodes: matcher.Nodes, Type: "matcher"})
+			if err != nil {
+				logrus.Fatal(err)
+			}
+
+			mu := &api.Mutation{
+				SetJson: pb,
+			}
+
+			req := &api.Request{
+				Query:     query,
+				Mutations: []*api.Mutation{mu},
+				CommitNow: true,
+			}
+
+			txn := graphClient.NewTxn()
+			_, err = txn.Do(ctx, req)
+			if err != nil {
+				logrus.Warn(err)
+			}
+
+			time.Sleep(time.Duration(2) * time.Second)
+		}
+	}
+}
+
 // RunDomainMatch looks for a target within the identified IOCs in /matcher/data.
 // the function could be refactored with RunDomainFilters
 func RunDomainMatch(domain string) bool {