Cleaning up and and reorganizing the models package
This commit is contained in:
Christopher Talib
parent
b5caae0974
commit
cc87dab922
2
main.go
2
main.go
|
|
@ -38,7 +38,7 @@ func main() {
|
|||
if data, err := models.ExtractCertFromStream(jq); err == nil {
|
||||
|
||||
rawNode := models.WrapCertStreamData(*data)
|
||||
models.SaveRaw("raw_certstream.json", rawNode)
|
||||
models.SaveCertStreamWrapper("raw_certstream.json", rawNode)
|
||||
|
||||
certNode := models.BuildCertNode(rawNode)
|
||||
models.SaveCertNode("cert_nodes.json", certNode)
|
||||
|
|
|
|||
|
|
@ -7,16 +7,6 @@ import (
|
|||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// CertStreamWrapper is a wrapper around the stream function to unmarshall the
|
||||
// data receive in a Go structure.
|
||||
type CertStreamWrapper struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Data CertStreamStruct `json:"data"`
|
||||
Created string `json:"created"`
|
||||
Modified string `json:"modified"`
|
||||
}
|
||||
|
||||
// LeafCertExtensions extends the LeafCert object.
|
||||
type LeafCertExtensions struct {
|
||||
KeyUsage string `json:"keyUsage"`
|
||||
|
|
|
|||
285
models/main.go
285
models/main.go
|
|
@ -11,6 +11,26 @@ import (
|
|||
"gitlab.dcso.lolcat/LABS/styx/utils"
|
||||
)
|
||||
|
||||
/**
|
||||
|
||||
Structure of this file:
|
||||
* type
|
||||
* build node functions
|
||||
* save node functions
|
||||
|
||||
**/
|
||||
|
||||
// Node defines the data we gather through the parsing. It should follow the
|
||||
// Styx terminology
|
||||
// (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx)
|
||||
type Node struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Data string `json:"data"`
|
||||
Created string `json:"created"`
|
||||
Modified string `json:"modified"`
|
||||
}
|
||||
|
||||
// BuildNode builds a node to send to MQ instance.
|
||||
func BuildNode(flag string, dataType string, data string) *Node {
|
||||
t := time.Now()
|
||||
|
|
@ -24,6 +44,123 @@ func BuildNode(flag string, dataType string, data string) *Node {
|
|||
}
|
||||
}
|
||||
|
||||
// SaveNode saves a node to a file.
|
||||
func SaveNode(filename string, node *Node) {
|
||||
err := utils.FileExists(filename)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
nodeFile, err := ioutil.ReadFile(filename)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
nodeDatas := []Node{}
|
||||
|
||||
if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
nodeDatas = append(nodeDatas, *node)
|
||||
|
||||
nodeBytes, err := json.Marshal(nodeDatas)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile(filename, nodeBytes, 0644)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
// Edge defines a relation between two nodes.
|
||||
type Edge struct {
|
||||
ID string `json:"id"`
|
||||
NodeOneID string `json:"nodeOneID"`
|
||||
NodeTwoID string `json:"nodeTwoID"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
Source string `json:"source"`
|
||||
}
|
||||
|
||||
// BuildEdge build a send from two nodes with a given source type.
|
||||
func BuildEdge(source string, nodeOneUUID string, nodeTwoUUID string) *Edge {
|
||||
t := time.Now()
|
||||
rfc3339time := t.Format(time.RFC3339)
|
||||
return &Edge{
|
||||
ID: uuid.New().String(),
|
||||
Source: source,
|
||||
NodeOneID: nodeOneUUID,
|
||||
NodeTwoID: nodeTwoUUID,
|
||||
Timestamp: rfc3339time,
|
||||
}
|
||||
}
|
||||
|
||||
// SaveEdge saves an edge to a file.
|
||||
func SaveEdge(edge *Edge) {
|
||||
err := utils.FileExists("edges.json")
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
edgeFile, err := ioutil.ReadFile("edges.json")
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
edgeDatas := []Edge{}
|
||||
|
||||
if err := json.Unmarshal(edgeFile, &edgeDatas); err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
edgeDatas = append(edgeDatas, *edge)
|
||||
edgeBytes, err := json.Marshal(edgeDatas)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile("edges.json", edgeBytes, 0644)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
// CertStreamWrapper is a wrapper around the stream function to unmarshall the
|
||||
// data receive in a Go structure.
|
||||
type CertStreamWrapper struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Data CertStreamStruct `json:"data"`
|
||||
Created string `json:"created"`
|
||||
Modified string `json:"modified"`
|
||||
}
|
||||
|
||||
// CertNode represents our custom struct of data extraction from CertStream.
|
||||
type CertNode struct {
|
||||
ID string `json:"id"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
NotBefore string `json:"notBefore"`
|
||||
NotAfter string `json:"notAfter"`
|
||||
CN string `json:"cn"`
|
||||
SourceName string `json:"sourceName"`
|
||||
SerialNumber string `json:"serialNumber"`
|
||||
BasicConstraints string `json:"basicConstraints"`
|
||||
RawUUID string `json:"rawUUID"`
|
||||
Chain []CertNode `json:"chainedTo"`
|
||||
}
|
||||
|
||||
// WrapCertStreamData is a wrapper around CertStreamStruct.
|
||||
func WrapCertStreamData(data CertStreamStruct) *CertStreamWrapper {
|
||||
t := time.Now()
|
||||
rfc3339time := t.Format(time.RFC3339)
|
||||
return &CertStreamWrapper{
|
||||
ID: "certstream--" + uuid.New().String(),
|
||||
Type: "certstream_raw",
|
||||
Data: data,
|
||||
Created: rfc3339time,
|
||||
Modified: rfc3339time,
|
||||
}
|
||||
}
|
||||
|
||||
// BuildCertNode builds a custom node based on CertStream.
|
||||
func BuildCertNode(rawNode *CertStreamWrapper) *CertNode {
|
||||
main := &CertNode{
|
||||
|
|
@ -56,78 +193,8 @@ func BuildCertNode(rawNode *CertStreamWrapper) *CertNode {
|
|||
return main
|
||||
}
|
||||
|
||||
// CertNode represents our custom struct of data extraction from CertStream.
|
||||
type CertNode struct {
|
||||
ID string `json:"id"`
|
||||
Fingerprint string `json:"fingerprint"`
|
||||
NotBefore string `json:"notBefore"`
|
||||
NotAfter string `json:"notAfter"`
|
||||
CN string `json:"cn"`
|
||||
SourceName string `json:"sourceName"`
|
||||
SerialNumber string `json:"serialNumber"`
|
||||
BasicConstraints string `json:"basicConstraints"`
|
||||
RawUUID string `json:"rawUUID"`
|
||||
Chain []CertNode `json:"chainedTo"`
|
||||
}
|
||||
|
||||
// WrapCertStreamData is a wrapper around CertStreamStruct.
|
||||
func WrapCertStreamData(data CertStreamStruct) *CertStreamWrapper {
|
||||
t := time.Now()
|
||||
rfc3339time := t.Format(time.RFC3339)
|
||||
return &CertStreamWrapper{
|
||||
ID: "certstream--" + uuid.New().String(),
|
||||
Type: "certstream_raw",
|
||||
Data: data,
|
||||
Created: rfc3339time,
|
||||
Modified: rfc3339time,
|
||||
}
|
||||
}
|
||||
|
||||
// BuildEdge build a send from two nodes with a given source type.
|
||||
func BuildEdge(source string, nodeOneUUID string, nodeTwoUUID string) *Edge {
|
||||
t := time.Now()
|
||||
rfc3339time := t.Format(time.RFC3339)
|
||||
return &Edge{
|
||||
ID: uuid.New().String(),
|
||||
Source: source,
|
||||
NodeOneID: nodeOneUUID,
|
||||
NodeTwoID: nodeTwoUUID,
|
||||
Timestamp: rfc3339time,
|
||||
}
|
||||
}
|
||||
|
||||
// SavePaste saves a object received from PasteBin.
|
||||
func SavePaste(filename string, data *PasteWrapper) {
|
||||
err := utils.FileExists(filename)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
pasteFile, err := ioutil.ReadFile(filename)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
rawPaste := []PasteWrapper{}
|
||||
|
||||
if err := json.Unmarshal(pasteFile, &rawPaste); err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
rawPaste = append(rawPaste, *data)
|
||||
|
||||
rawBytes, err := json.Marshal(rawPaste)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
// SaveRaw save the raw CertStream data.
|
||||
func SaveRaw(filename string, data *CertStreamWrapper) {
|
||||
// SaveCertStreamWrapper save the raw CertStream data.
|
||||
func SaveCertStreamWrapper(filename string, data *CertStreamWrapper) {
|
||||
err := utils.FileExists(filename)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
|
|
@ -184,59 +251,59 @@ func SaveCertNode(filename string, node *CertNode) {
|
|||
}
|
||||
}
|
||||
|
||||
// SaveNode saves a node to a file.
|
||||
func SaveNode(filename string, node *Node) {
|
||||
// PasteNode is a node from PasteBin.
|
||||
type PasteNode struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Data FullPaste `json:"data"`
|
||||
Created string `json:"create"`
|
||||
Modified string `json:"modified"`
|
||||
}
|
||||
|
||||
// FullPaste wrapes meta and information from Pastebin.
|
||||
type FullPaste struct {
|
||||
Meta PasteMeta `json:"meta"`
|
||||
Full string `json:"full"`
|
||||
}
|
||||
|
||||
// BuildPasteNode builds a node from a FullPaste data.
|
||||
func BuildPasteNode(data *FullPaste) *PasteNode {
|
||||
t := time.Now()
|
||||
rfc3339time := t.Format(time.RFC3339)
|
||||
return &PasteNode{
|
||||
ID: "pastebin--" + uuid.New().String(),
|
||||
Type: "pastebin",
|
||||
Data: *data,
|
||||
Created: rfc3339time,
|
||||
Modified: rfc3339time,
|
||||
}
|
||||
}
|
||||
|
||||
// SavePaste saves a object received from PasteBin.
|
||||
func SavePaste(filename string, data *PasteNode) {
|
||||
err := utils.FileExists(filename)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
nodeFile, err := ioutil.ReadFile(filename)
|
||||
|
||||
pasteFile, err := ioutil.ReadFile(filename)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
nodeDatas := []Node{}
|
||||
rawPaste := []PasteNode{}
|
||||
|
||||
if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil {
|
||||
if err := json.Unmarshal(pasteFile, &rawPaste); err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
nodeDatas = append(nodeDatas, *node)
|
||||
rawPaste = append(rawPaste, *data)
|
||||
|
||||
nodeBytes, err := json.Marshal(nodeDatas)
|
||||
rawBytes, err := json.Marshal(rawPaste)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile(filename, nodeBytes, 0644)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
// SaveEdge saves an edge to a file.
|
||||
func SaveEdge(edge *Edge) {
|
||||
err := utils.FileExists("edges.json")
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
edgeFile, err := ioutil.ReadFile("edges.json")
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
edgeDatas := []Edge{}
|
||||
|
||||
if err := json.Unmarshal(edgeFile, &edgeDatas); err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
edgeDatas = append(edgeDatas, *edge)
|
||||
edgeBytes, err := json.Marshal(edgeDatas)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile("edges.json", edgeBytes, 0644)
|
||||
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -6,27 +6,10 @@ import (
|
|||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// PasteWrapper is a node from PasteBin.
|
||||
type PasteWrapper struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Data FullPaste `json:"data"`
|
||||
Created string `json:"create"`
|
||||
Modified string `json:"modified"`
|
||||
}
|
||||
|
||||
// FullPaste wrape meta and information from Pastebin.
|
||||
type FullPaste struct {
|
||||
Meta PasteMeta `json:"meta"`
|
||||
Full string `json:"full"`
|
||||
}
|
||||
|
||||
// PasteMeta is a set of descriptive information on a paste.
|
||||
type PasteMeta struct {
|
||||
ScrapeURL string `json:"scrape_url"`
|
||||
|
|
@ -117,16 +100,3 @@ func FetchPaste(paste PasteMeta) (string, error) {
|
|||
|
||||
return string(body), nil
|
||||
}
|
||||
|
||||
// BuildPasteNode builds a node from a FullPaste data.
|
||||
func BuildPasteNode(data *FullPaste) *PasteWrapper {
|
||||
t := time.Now()
|
||||
rfc3339time := t.Format(time.RFC3339)
|
||||
return &PasteWrapper{
|
||||
ID: "pastebin--" + uuid.New().String(),
|
||||
Type: "pastebin",
|
||||
Data: *data,
|
||||
Created: rfc3339time,
|
||||
Modified: rfc3339time,
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,21 +1 @@
|
|||
package models
|
||||
|
||||
// Node defines the data we gather through the parsing. It should follow the
|
||||
// Styx terminology
|
||||
// (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx)
|
||||
type Node struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Data string `json:"data"`
|
||||
Created string `json:"created"`
|
||||
Modified string `json:"modified"`
|
||||
}
|
||||
|
||||
// Edge defines a relation between two nodes.
|
||||
type Edge struct {
|
||||
ID string `json:"id"`
|
||||
NodeOneID string `json:"nodeOneID"`
|
||||
NodeTwoID string `json:"nodeTwoID"`
|
||||
Timestamp string `json:"timestamp"`
|
||||
Source string `json:"source"`
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue