Setting up linking and and creating nodes already from the input source

main.go

@@ -5,7 +5,6 @@ import (
 
 	"github.com/CaliDog/certstream-go"
 	"github.com/sirupsen/logrus"
-	"gitlab.dcso.lolcat/LABS/styx/broker"
 	"gitlab.dcso.lolcat/LABS/styx/models"
 )
 
@@ -13,18 +12,30 @@ func main() {
 	// The false flag specifies that we want heartbeat messages.
 	stream, errStream := certstream.CertStreamEventStream(false)
 	fmt.Println("Starting to get data from CertStream...")
-	Conn, err := broker.SetUpKafkaConnecter()
-	if err != nil {
-		panic(err)
-	}
-	go broker.ReadEventFromKafka()
+	// Conn, err := broker.SetUpKafkaConnecter()
+	// if err != nil {
+	// panic(err)
+	// }
+	// go broker.ReadEventFromKafka()
 	for {
 		select {
 		case jq := <-stream:
 			if data, err := models.ExtractCertFromStream(jq); err == nil {
 
-				node := models.BuildNode("certstream", *data)
-				broker.SendEventToKafka(Conn, *node)
+				fingerprintNode := models.BuildNode("certstream", data.Data.LeafCert.Fingerprint)
+				models.SaveNode(fingerprintNode)
+				fmt.Println(fingerprintNode)
+				allDomains := data.Data.LeafCert.AllDomains
+				var edge *models.Edge
+				for _, domain := range allDomains {
+					domainNode := models.BuildNode("certstream", domain)
+					models.SaveNode(domainNode)
+					edge = models.BuildEdge("certstream", fingerprintNode.ID, domainNode.ID)
+					fmt.Println(edge)
+					models.SaveEdge(edge)
+
+				}
+				// broker.SendEventToKafka(Conn, *fingerprintNode)
 			}
 
 		case err := <-errStream:

models/main.go

@@ -11,7 +11,7 @@ import (
 )
 
 // BuildNode builds a node to send to MQ instance.
-func BuildNode(flag string, data CertStreamStruct) *Node {
+func BuildNode(flag string, data string) *Node {
 	t := time.Now()
 	rfc3339time := t.Format(time.RFC3339)
 	return &Node{
@@ -25,7 +25,7 @@ func BuildNode(flag string, data CertStreamStruct) *Node {
 }
 
 // BuildEdge build a send from two nodes with a given source type.
-func BuildEdge(source string, nodeOneUUID uuid.UUID, nodeTwoUUID uuid.UUID) *Edge {
+func BuildEdge(source string, nodeOneUUID string, nodeTwoUUID string) *Edge {
 	t := time.Now()
 	rfc3339time := t.Format(time.RFC3339)
 	return &Edge{

models/types.go

@@ -1,25 +1,21 @@
 package models
 
-import (
-	"github.com/google/uuid"
-)
-
 // Node defines the data we gather through the parsing. It should follow the
 // Styx terminology
 // (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx)
 type Node struct {
-	ID       string           `json:"id"`
-	Type     string           `json:"type"`
-	Data     CertStreamStruct `json:"data"`
-	Created  string           `json:"created"`
-	Modified string           `json:"modified"`
+	ID       string `json:"id"`
+	Type     string `json:"type"`
+	Data     string `json:"data"`
+	Created  string `json:"created"`
+	Modified string `json:"modified"`
 }
 
 // Edge defines a relation between two nodes.
 type Edge struct {
-	ID        string    `json:"id"`
-	NodeOneID uuid.UUID `json:"nodeOneID"`
-	NodeTwoID uuid.UUID `json:"nodeTwoID"`
-	Timestamp string    `json:"timestamp"`
-	Source    string    `json:"source"`
+	ID        string `json:"id"`
+	NodeOneID string `json:"nodeOneID"`
+	NodeTwoID string `json:"nodeTwoID"`
+	Timestamp string `json:"timestamp"`
+	Source    string `json:"source"`
 }

parser/main.go

@@ -1,12 +1,9 @@
 package parser
 
 import (
-	"encoding/json"
-	"io/ioutil"
 	"os"
 
 	"github.com/sirupsen/logrus"
-	"gitlab.dcso.lolcat/LABS/styx/models"
 )
 
 // read node recieved on kafka
@@ -22,24 +19,24 @@ const (
 	EdgesFilename = "edges.json"
 )
 
-func ParseEvent(domains []string) {
-	nodeFile, err := ioutil.ReadFile(NodesFilename)
-	if err != nil {
-		logrus.Error(err)
-	}
-	nodeDatas := []models.Node{}
+// func ParseEvent(domains []string) {
+// nodeFile, err := ioutil.ReadFile(NodesFilename)
+// if err != nil {
+// logrus.Error(err)
+// }
+// nodeDatas := []models.Node{}
 
-	if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil {
-		logrus.Error(err)
-	}
+// if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil {
+// logrus.Error(err)
+// }
 
-	for _, node := range nodeDatas {
-		SaveDomains(node.Data.Data.LeafCert.AllDomains)
-	}
+// for _, node := range nodeDatas {
+// SaveDomains(node.Data.Data.LeafCert.AllDomains)
+// }
 
-	// saveDomains()
-	// go findDomainEdges()
-}
+// // saveDomains()
+// // go findDomainEdges()
+// }
 
 func SaveDomains(domains []string) {
 	f, err := os.OpenFile("domains.txt", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)