diff --git a/main.go b/main.go index f130d6e..4d23471 100644 --- a/main.go +++ b/main.go @@ -38,7 +38,7 @@ func main() { if data, err := models.ExtractCertFromStream(jq); err == nil { rawNode := models.WrapCertStreamData(*data) - models.SaveRaw("raw_certstream.json", rawNode) + models.SaveCertStreamWrapper("raw_certstream.json", rawNode) certNode := models.BuildCertNode(rawNode) models.SaveCertNode("cert_nodes.json", certNode) diff --git a/models/cerstream.go b/models/cerstream.go index f3bcb72..265bb2f 100644 --- a/models/cerstream.go +++ b/models/cerstream.go @@ -7,16 +7,6 @@ import ( "github.com/sirupsen/logrus" ) -// CertStreamWrapper is a wrapper around the stream function to unmarshall the -// data receive in a Go structure. -type CertStreamWrapper struct { - ID string `json:"id"` - Type string `json:"type"` - Data CertStreamStruct `json:"data"` - Created string `json:"created"` - Modified string `json:"modified"` -} - // LeafCertExtensions extends the LeafCert object. type LeafCertExtensions struct { KeyUsage string `json:"keyUsage"` diff --git a/models/main.go b/models/main.go index e0f8d43..a518007 100644 --- a/models/main.go +++ b/models/main.go @@ -11,6 +11,26 @@ import ( "gitlab.dcso.lolcat/LABS/styx/utils" ) +/** + +Structure of this file: +* type +* build node functions +* save node functions + +**/ + +// Node defines the data we gather through the parsing. It should follow the +// Styx terminology +// (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx) +type Node struct { + ID string `json:"id"` + Type string `json:"type"` + Data string `json:"data"` + Created string `json:"created"` + Modified string `json:"modified"` +} + // BuildNode builds a node to send to MQ instance. func BuildNode(flag string, dataType string, data string) *Node { t := time.Now() @@ -24,6 +44,123 @@ func BuildNode(flag string, dataType string, data string) *Node { } } +// SaveNode saves a node to a file. +func SaveNode(filename string, node *Node) { + err := utils.FileExists(filename) + if err != nil { + logrus.Error(err) + } + nodeFile, err := ioutil.ReadFile(filename) + if err != nil { + logrus.Error(err) + } + nodeDatas := []Node{} + + if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil { + logrus.Error(err) + } + + nodeDatas = append(nodeDatas, *node) + + nodeBytes, err := json.Marshal(nodeDatas) + if err != nil { + logrus.Error(err) + } + + err = ioutil.WriteFile(filename, nodeBytes, 0644) + if err != nil { + logrus.Error(err) + } +} + +// Edge defines a relation between two nodes. +type Edge struct { + ID string `json:"id"` + NodeOneID string `json:"nodeOneID"` + NodeTwoID string `json:"nodeTwoID"` + Timestamp string `json:"timestamp"` + Source string `json:"source"` +} + +// BuildEdge build a send from two nodes with a given source type. +func BuildEdge(source string, nodeOneUUID string, nodeTwoUUID string) *Edge { + t := time.Now() + rfc3339time := t.Format(time.RFC3339) + return &Edge{ + ID: uuid.New().String(), + Source: source, + NodeOneID: nodeOneUUID, + NodeTwoID: nodeTwoUUID, + Timestamp: rfc3339time, + } +} + +// SaveEdge saves an edge to a file. +func SaveEdge(edge *Edge) { + err := utils.FileExists("edges.json") + if err != nil { + logrus.Error(err) + } + + edgeFile, err := ioutil.ReadFile("edges.json") + if err != nil { + logrus.Error(err) + } + edgeDatas := []Edge{} + + if err := json.Unmarshal(edgeFile, &edgeDatas); err != nil { + logrus.Error(err) + } + + edgeDatas = append(edgeDatas, *edge) + edgeBytes, err := json.Marshal(edgeDatas) + if err != nil { + logrus.Error(err) + } + + err = ioutil.WriteFile("edges.json", edgeBytes, 0644) + if err != nil { + logrus.Error(err) + } +} + +// CertStreamWrapper is a wrapper around the stream function to unmarshall the +// data receive in a Go structure. +type CertStreamWrapper struct { + ID string `json:"id"` + Type string `json:"type"` + Data CertStreamStruct `json:"data"` + Created string `json:"created"` + Modified string `json:"modified"` +} + +// CertNode represents our custom struct of data extraction from CertStream. +type CertNode struct { + ID string `json:"id"` + Fingerprint string `json:"fingerprint"` + NotBefore string `json:"notBefore"` + NotAfter string `json:"notAfter"` + CN string `json:"cn"` + SourceName string `json:"sourceName"` + SerialNumber string `json:"serialNumber"` + BasicConstraints string `json:"basicConstraints"` + RawUUID string `json:"rawUUID"` + Chain []CertNode `json:"chainedTo"` +} + +// WrapCertStreamData is a wrapper around CertStreamStruct. +func WrapCertStreamData(data CertStreamStruct) *CertStreamWrapper { + t := time.Now() + rfc3339time := t.Format(time.RFC3339) + return &CertStreamWrapper{ + ID: "certstream--" + uuid.New().String(), + Type: "certstream_raw", + Data: data, + Created: rfc3339time, + Modified: rfc3339time, + } +} + // BuildCertNode builds a custom node based on CertStream. func BuildCertNode(rawNode *CertStreamWrapper) *CertNode { main := &CertNode{ @@ -56,78 +193,8 @@ func BuildCertNode(rawNode *CertStreamWrapper) *CertNode { return main } -// CertNode represents our custom struct of data extraction from CertStream. -type CertNode struct { - ID string `json:"id"` - Fingerprint string `json:"fingerprint"` - NotBefore string `json:"notBefore"` - NotAfter string `json:"notAfter"` - CN string `json:"cn"` - SourceName string `json:"sourceName"` - SerialNumber string `json:"serialNumber"` - BasicConstraints string `json:"basicConstraints"` - RawUUID string `json:"rawUUID"` - Chain []CertNode `json:"chainedTo"` -} - -// WrapCertStreamData is a wrapper around CertStreamStruct. -func WrapCertStreamData(data CertStreamStruct) *CertStreamWrapper { - t := time.Now() - rfc3339time := t.Format(time.RFC3339) - return &CertStreamWrapper{ - ID: "certstream--" + uuid.New().String(), - Type: "certstream_raw", - Data: data, - Created: rfc3339time, - Modified: rfc3339time, - } -} - -// BuildEdge build a send from two nodes with a given source type. -func BuildEdge(source string, nodeOneUUID string, nodeTwoUUID string) *Edge { - t := time.Now() - rfc3339time := t.Format(time.RFC3339) - return &Edge{ - ID: uuid.New().String(), - Source: source, - NodeOneID: nodeOneUUID, - NodeTwoID: nodeTwoUUID, - Timestamp: rfc3339time, - } -} - -// SavePaste saves a object received from PasteBin. -func SavePaste(filename string, data *PasteWrapper) { - err := utils.FileExists(filename) - if err != nil { - logrus.Error(err) - } - - pasteFile, err := ioutil.ReadFile(filename) - if err != nil { - logrus.Error(err) - } - rawPaste := []PasteWrapper{} - - if err := json.Unmarshal(pasteFile, &rawPaste); err != nil { - logrus.Error(err) - } - - rawPaste = append(rawPaste, *data) - - rawBytes, err := json.Marshal(rawPaste) - if err != nil { - logrus.Error(err) - } - - err = ioutil.WriteFile(filename, rawBytes, 0644) - if err != nil { - logrus.Error(err) - } -} - -// SaveRaw save the raw CertStream data. -func SaveRaw(filename string, data *CertStreamWrapper) { +// SaveCertStreamWrapper save the raw CertStream data. +func SaveCertStreamWrapper(filename string, data *CertStreamWrapper) { err := utils.FileExists(filename) if err != nil { logrus.Error(err) @@ -184,59 +251,59 @@ func SaveCertNode(filename string, node *CertNode) { } } -// SaveNode saves a node to a file. -func SaveNode(filename string, node *Node) { +// PasteNode is a node from PasteBin. +type PasteNode struct { + ID string `json:"id"` + Type string `json:"type"` + Data FullPaste `json:"data"` + Created string `json:"create"` + Modified string `json:"modified"` +} + +// FullPaste wrapes meta and information from Pastebin. +type FullPaste struct { + Meta PasteMeta `json:"meta"` + Full string `json:"full"` +} + +// BuildPasteNode builds a node from a FullPaste data. +func BuildPasteNode(data *FullPaste) *PasteNode { + t := time.Now() + rfc3339time := t.Format(time.RFC3339) + return &PasteNode{ + ID: "pastebin--" + uuid.New().String(), + Type: "pastebin", + Data: *data, + Created: rfc3339time, + Modified: rfc3339time, + } +} + +// SavePaste saves a object received from PasteBin. +func SavePaste(filename string, data *PasteNode) { err := utils.FileExists(filename) if err != nil { logrus.Error(err) } - nodeFile, err := ioutil.ReadFile(filename) + + pasteFile, err := ioutil.ReadFile(filename) if err != nil { logrus.Error(err) } - nodeDatas := []Node{} + rawPaste := []PasteNode{} - if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil { + if err := json.Unmarshal(pasteFile, &rawPaste); err != nil { logrus.Error(err) } - nodeDatas = append(nodeDatas, *node) + rawPaste = append(rawPaste, *data) - nodeBytes, err := json.Marshal(nodeDatas) + rawBytes, err := json.Marshal(rawPaste) if err != nil { logrus.Error(err) } - err = ioutil.WriteFile(filename, nodeBytes, 0644) - if err != nil { - logrus.Error(err) - } -} - -// SaveEdge saves an edge to a file. -func SaveEdge(edge *Edge) { - err := utils.FileExists("edges.json") - if err != nil { - logrus.Error(err) - } - - edgeFile, err := ioutil.ReadFile("edges.json") - if err != nil { - logrus.Error(err) - } - edgeDatas := []Edge{} - - if err := json.Unmarshal(edgeFile, &edgeDatas); err != nil { - logrus.Error(err) - } - - edgeDatas = append(edgeDatas, *edge) - edgeBytes, err := json.Marshal(edgeDatas) - if err != nil { - logrus.Error(err) - } - - err = ioutil.WriteFile("edges.json", edgeBytes, 0644) + err = ioutil.WriteFile(filename, rawBytes, 0644) if err != nil { logrus.Error(err) } diff --git a/models/pastebin.go b/models/pastebin.go index 678057f..2e98c55 100644 --- a/models/pastebin.go +++ b/models/pastebin.go @@ -6,27 +6,10 @@ import ( "io/ioutil" "log" "net/http" - "time" - "github.com/google/uuid" "github.com/sirupsen/logrus" ) -// PasteWrapper is a node from PasteBin. -type PasteWrapper struct { - ID string `json:"id"` - Type string `json:"type"` - Data FullPaste `json:"data"` - Created string `json:"create"` - Modified string `json:"modified"` -} - -// FullPaste wrape meta and information from Pastebin. -type FullPaste struct { - Meta PasteMeta `json:"meta"` - Full string `json:"full"` -} - // PasteMeta is a set of descriptive information on a paste. type PasteMeta struct { ScrapeURL string `json:"scrape_url"` @@ -117,16 +100,3 @@ func FetchPaste(paste PasteMeta) (string, error) { return string(body), nil } - -// BuildPasteNode builds a node from a FullPaste data. -func BuildPasteNode(data *FullPaste) *PasteWrapper { - t := time.Now() - rfc3339time := t.Format(time.RFC3339) - return &PasteWrapper{ - ID: "pastebin--" + uuid.New().String(), - Type: "pastebin", - Data: *data, - Created: rfc3339time, - Modified: rfc3339time, - } -} diff --git a/models/types.go b/models/types.go index de14099..2640e7f 100644 --- a/models/types.go +++ b/models/types.go @@ -1,21 +1 @@ package models - -// Node defines the data we gather through the parsing. It should follow the -// Styx terminology -// (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx) -type Node struct { - ID string `json:"id"` - Type string `json:"type"` - Data string `json:"data"` - Created string `json:"created"` - Modified string `json:"modified"` -} - -// Edge defines a relation between two nodes. -type Edge struct { - ID string `json:"id"` - NodeOneID string `json:"nodeOneID"` - NodeTwoID string `json:"nodeTwoID"` - Timestamp string `json:"timestamp"` - Source string `json:"source"` -}