Some temporal refactoring, saving domains and nodes

broker/main.go

@@ -3,12 +3,12 @@ package broker
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"time"
 
 	"github.com/segmentio/kafka-go"
 	"github.com/sirupsen/logrus"
 	"gitlab.dcso.lolcat/LABS/styx/models"
+	"gitlab.dcso.lolcat/LABS/styx/utils"
 )
 
 func SetUpKafkaConnecter() (*kafka.Conn, error) {
@@ -44,6 +44,7 @@ func ProduceEvent(conn *kafka.Conn, message string) error {
 
 func SendEventToKafka(conn *kafka.Conn, node models.Node) error {
 	conn.SetWriteDeadline(time.Now().Add(10 * time.Second))
+	utils.SaveNode(&node)
 	packaged, _ := json.Marshal(node)
 	_, err := conn.WriteMessages(kafka.Message{Value: packaged})
 	if err != nil {
@@ -72,7 +73,8 @@ func ReadEventFromKafka() {
 
 		var node models.Node
 		json.Unmarshal(m.Value, &node)
-		fmt.Println(node.Data.Data.LeafCert.AllDomains)
+
+		utils.SaveDomains(node.Data.Data.LeafCert.AllDomains)
 
 	}
 

main.go

@@ -2,7 +2,6 @@ package main
 
 import (
 	"fmt"
-	"os"
 
 	"github.com/CaliDog/certstream-go"
 	"github.com/sirupsen/logrus"
@@ -11,22 +10,7 @@ import (
 	"gitlab.dcso.lolcat/LABS/styx/utils"
 )
 
-const (
-	NodesFilename = "nodes.json"
-	EdgesFilename = "edges.json"
-)
-
 func main() {
-	// checking that data files exist
-	err := fileExists(NodesFilename)
-	if err != nil {
-		logrus.Error(err)
-	}
-
-	err = fileExists(EdgesFilename)
-	if err != nil {
-		logrus.Error(err)
-	}
 	// The false flag specifies that we want heartbeat messages.
 	stream, errStream := certstream.CertStreamEventStream(false)
 	fmt.Println("Starting to get data from CertStream...")
@@ -43,6 +27,7 @@ func main() {
 				if err != nil {
 					panic(err)
 				}
+				utils.SaveNode(node)
 
 				err = broker.SendEventToKafka(Conn, *node)
 				if err != nil {
@@ -57,15 +42,3 @@ func main() {
 }
 
 // Helpers
-
-func fileExists(filename string) error {
-	_, err := os.Stat(filename)
-	if os.IsNotExist(err) {
-		_, err := os.Create(filename)
-		if err != nil {
-			return err
-		}
-	}
-
-	return nil
-}

utils/main.go

@@ -1,11 +1,15 @@
 package utils
 
 import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
 	"strconv"
 
 	"github.com/jmoiron/jsonq"
 	"github.com/sirupsen/logrus"
 	"gitlab.dcso.lolcat/LABS/styx/models"
+	"gitlab.dcso.lolcat/LABS/styx/parser"
 )
 
 // ExtractCertFromStream builds the structures before saving them. It uses the
@@ -187,3 +191,45 @@ func extractLeafCertStruct(input jsonq.JsonQuery) (models.LeafCertStruct, error)
 	}, nil
 
 }
+
+func SaveNode(node *models.Node) {
+	err := FileExists(parser.NodesFilename)
+	if err != nil {
+		logrus.Error(err)
+	}
+	nodeFile, err := ioutil.ReadFile(parser.NodesFilename)
+	if err != nil {
+		logrus.Error(err)
+	}
+	nodeDatas := []models.Node{}
+
+	if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil {
+		logrus.Error(err)
+	}
+
+	nodeDatas = append(nodeDatas, *node)
+
+	nodeBytes, err := json.Marshal(nodeDatas)
+	if err != nil {
+		logrus.Error(err)
+	}
+
+	err = ioutil.WriteFile(parser.NodesFilename, nodeBytes, 0644)
+	if err != nil {
+		logrus.Error(err)
+	}
+}
+
+func SaveDomains(domains []string) {
+	f, err := os.OpenFile("domains.txt", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		logrus.Error(err)
+	}
+	defer f.Close()
+
+	for _, d := range domains {
+		if _, err := f.WriteString(d + ","); err != nil {
+			logrus.Error(err)
+		}
+	}
+}