chris/styx
chris/styx
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Refactoring from utils to models

Browse source 
This work refactors saving and extracting function from the utils
package to the models package as it is a main component of the tool.
`utils` will take care of not related to models functions (such as
finding the files for example).

Also creating unique files for each type of source we are parsing.
This commit is contained in:
Christopher Talib 2020-01-28 16:00:56 +01:00
parent 0e1e003ab2
commit 7785372e3a
8 changed files with 186 additions and 210 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
broker/main.go
View file

@ -9,7 +9,6 @@ import (
"github.com/segmentio/kafka-go"
"github.com/sirupsen/logrus"
"gitlab.dcso.lolcat/LABS/styx/models"
"gitlab.dcso.lolcat/LABS/styx/utils"
)
func SetUpKafkaConnecter() (*kafka.Conn, error) {
@ -55,7 +54,7 @@ func ReadEventFromKafka() {
json.Unmarshal(m.Value, &node)
if len(node.ID) != 0 {
fmt.Println(node)
utils.SaveNode(&node)
models.SaveNode(&node)
}
}
}

8
main.go
View file

@ -7,7 +7,6 @@ import (
"github.com/sirupsen/logrus"
"gitlab.dcso.lolcat/LABS/styx/broker"
"gitlab.dcso.lolcat/LABS/styx/models"
"gitlab.dcso.lolcat/LABS/styx/utils"
)
func main() {
@ -22,12 +21,9 @@ func main() {
for {
select {
case jq := <-stream:
if data, err := utils.ExtractCertFromStream(jq); err == nil {
if data, err := models.ExtractCertFromStream(jq); err == nil {
node, err := models.BuildNode("certstream", *data)
if err != nil {
panic(err)
}
node := models.BuildNode("certstream", *data)
broker.SendEventToKafka(Conn, *node)
}

58
models/cerstream.go Normal file
View file

@ -0,0 +1,58 @@
package models
// LeafCertExtensions extends the LeafCert object.
type LeafCertExtensions struct {
KeyUsage string `json:"keyUsage"`
ExtendedKeyUsage string `json:"extendedKeyUsage"`
BasicConstrains string `json:"basicConstrains"`
SubjectKeyIdentifier string `json:"subjectKeyIdentifier"`
AuthorityInfoAccess string `json:"authorityInfoAccess"`
SubjectAltName string `json:"subjectAltName"`
CertificatePolicies string `json:"certificatePolicies"`
}
// LeafCertSubject is the subject of the LeafCert object.
type LeafCertSubject struct {
Aggregated string `json:"aggregated"`
C string `json:"C"`
ST string `json:"ST"`
L string `json:"L"`
O string `json:"O"`
OU string `json:"OU"`
CN string `json:"CN"`
}
// LeafCertStruct represents the LeafCert object.
type LeafCertStruct struct {
Subject LeafCertSubject `json:"subject"`
Extensions LeafCertExtensions `json:"extensions"`
NotBefore int `json:"not_before"`
NotAfter int `json:"not_after"`
SerialNumber string `json:"serial_number"`
Fingerprint string `json:"fingerprint"`
AsDer string `json:"as_der"`
AllDomains []string `json:"all_domains"`
}
// Source is the object ofr the URL and its name.
type Source struct {
URL string `json:"url"`
Name string `json:"name"`
}
// CertStreamData is the data contained in a CertStream payload.
type CertStreamData struct {
UpdateType string `json:"update_type"`
LeafCert LeafCertStruct `json:"leaf_cert"`
Chain []LeafCertStruct `json:"chain"`
CertIndex int `json:"cert_index"`
Seen int `json:"seen"`
Source Source `json:"source"`
}
// CertStreamStruct reprensts a payload received from CertStream. It has a type
// and the content is stored in Data.
type CertStreamStruct struct {
MessageType string `json:"message_data"`
Data CertStreamData `json:"data"`
}

35
utils/extracts.go → models/extracts.go
View file

@ -1,4 +1,4 @@
package utils
package models
import (
"encoding/json"
@ -10,13 +10,12 @@ import (
"github.com/jmoiron/jsonq"
"github.com/sirupsen/logrus"
"gitlab.dcso.lolcat/LABS/styx/models"
)
// ExtractCertFromStream builds the structures before saving them. It uses the
// power of jsonq to parse quickly the json stream.
// The base structure is coming from : https://github.com/CaliDog/certstream-go#example-data-structure
func ExtractCertFromStream(input jsonq.JsonQuery) (*models.CertStreamStruct, error) {
func ExtractCertFromStream(input jsonq.JsonQuery) (*CertStreamStruct, error) {
// LeafCertStruct
leafCertStruct, err := extractLeafCertStruct(input)
if err != nil {
@ -30,7 +29,7 @@ func ExtractCertFromStream(input jsonq.JsonQuery) (*models.CertStreamStruct, err
logrus.Error(err)
}
source := models.Source{
source := Source{
URL: url,
Name: name,
}
@ -40,7 +39,7 @@ func ExtractCertFromStream(input jsonq.JsonQuery) (*models.CertStreamStruct, err
certIndex, err := input.Int("data", "cert_index")
seen, err := input.Int("data", "seen")
chain, err := input.ArrayOfObjects("data", "chain")
chainSlice := []models.LeafCertStruct{}
chainSlice := []LeafCertStruct{}
for i := 0; i < len(chain); i++ {
c, err := extractLeafCertChainStruct(input, strconv.Itoa(i))
if err != nil {
@ -49,7 +48,7 @@ func ExtractCertFromStream(input jsonq.JsonQuery) (*models.CertStreamStruct, err
chainSlice = append(chainSlice, c)
}
csd := models.CertStreamData{
csd := CertStreamData{
UpdateType: updateType,
LeafCert: leafCertStruct,
Chain: chainSlice,
@ -64,7 +63,7 @@ func ExtractCertFromStream(input jsonq.JsonQuery) (*models.CertStreamStruct, err
logrus.Error(err)
}
res := models.CertStreamStruct{
res := CertStreamStruct{
MessageType: messageType,
Data: csd,
}
@ -73,7 +72,7 @@ func ExtractCertFromStream(input jsonq.JsonQuery) (*models.CertStreamStruct, err
}
func extractLeafCertChainStruct(input jsonq.JsonQuery, index string) (models.LeafCertStruct, error) {
func extractLeafCertChainStruct(input jsonq.JsonQuery, index string) (LeafCertStruct, error) {
// LeafCertStruct > Subject
aggregated, _ := input.String("data", "chain", index, "subject", "aggregated")
c, _ := input.String("data", "chain", index, "subject", "C")
@ -83,7 +82,7 @@ func extractLeafCertChainStruct(input jsonq.JsonQuery, index string) (models.Lea
ou, _ := input.String("data", "chain", index, "subject", "OU")
cn, _ := input.String("data", "chain", index, "subject", "CN")
subject := models.LeafCertSubject{
subject := LeafCertSubject{
Aggregated: aggregated,
C: c,
ST: st,
@ -102,7 +101,7 @@ func extractLeafCertChainStruct(input jsonq.JsonQuery, index string) (models.Lea
subjectAltName, _ := input.String("data", "chain", index, "extensions", "subjectAltName")
certificatePolicies, _ := input.String("data", "chain", index, "extensions", "certificatePolicies")
extensions := models.LeafCertExtensions{
extensions := LeafCertExtensions{
KeyUsage: keyUsage,
ExtendedKeyUsage: extendedKeyUsage,
BasicConstrains: basicConstrains,
@ -119,7 +118,7 @@ func extractLeafCertChainStruct(input jsonq.JsonQuery, index string) (models.Lea
asDer, _ := input.String("data", "chain", "as_der")
allDomains, _ := input.ArrayOfStrings("data", "chain", "all_domains")
return models.LeafCertStruct{
return LeafCertStruct{
Subject: subject,
Extensions: extensions,
NotBefore: notBefore,
@ -131,7 +130,7 @@ func extractLeafCertChainStruct(input jsonq.JsonQuery, index string) (models.Lea
}, nil
}
func extractLeafCertStruct(input jsonq.JsonQuery) (models.LeafCertStruct, error) {
func extractLeafCertStruct(input jsonq.JsonQuery) (LeafCertStruct, error) {
// LeafCertStruct > Subject
aggregated, err := input.String("data", "leaf_cert", "subject", "aggregated")
c, err := input.String("data", "leaf_cert", "subject", "C")
@ -144,7 +143,7 @@ func extractLeafCertStruct(input jsonq.JsonQuery) (models.LeafCertStruct, error)
logrus.Error(err)
}
subject := models.LeafCertSubject{
subject := LeafCertSubject{
Aggregated: aggregated,
C: c,
ST: st,
@ -163,7 +162,7 @@ func extractLeafCertStruct(input jsonq.JsonQuery) (models.LeafCertStruct, error)
subjectAltName, _ := input.String("data", "leaf_cert", "extensions", "subjectAltName")
certificatePolicies, _ := input.String("data", "leaf_cert", "extensions", "certificatePolicies")
extensions := models.LeafCertExtensions{
extensions := LeafCertExtensions{
KeyUsage: keyUsage,
ExtendedKeyUsage: extendedKeyUsage,
BasicConstrains: basicConstrains,
@ -180,7 +179,7 @@ func extractLeafCertStruct(input jsonq.JsonQuery) (models.LeafCertStruct, error)
asDer, _ := input.String("data", "leaf_cert", "as_der")
allDomains, _ := input.ArrayOfStrings("data", "leaf_cert", "all_domains")
return models.LeafCertStruct{
return LeafCertStruct{
Subject: subject,
Extensions: extensions,
NotBefore: notBefore,
@ -197,7 +196,7 @@ func extractLeafCertStruct(input jsonq.JsonQuery) (models.LeafCertStruct, error)
// Content: http://pastebin.com/api_scrape_item.php
// QueryPastes returns metadata for the last 100 public pastes.
func QueryPastes() ([]models.PasteMeta, error) {
func QueryPastes() ([]PasteMeta, error) {
server := "pastebin.com"
req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/api_scraping.php?limit=100", server), nil)
@ -221,7 +220,7 @@ func QueryPastes() ([]models.PasteMeta, error) {
return nil, err
}
var pastes []models.PasteMeta
var pastes []PasteMeta
if err := json.Unmarshal(body, &pastes); err != nil {
logrus.Error("Could not decode response due to %v, body %s", err, string(body))
return nil, err
@ -231,7 +230,7 @@ func QueryPastes() ([]models.PasteMeta, error) {
}
// FetchPaste fetches paste contents via the web API.
func FetchPaste(paste models.PasteMeta) (string, error) {
func FetchPaste(paste PasteMeta) (string, error) {
url := paste.ScrapeURL
req, err := http.NewRequest("GET", url, nil)
if err != nil {

139
models/main.go
View file

@ -1,75 +1,17 @@
package models
import (
"encoding/json"
"io/ioutil"
"time"
"github.com/google/uuid"
"github.com/sirupsen/logrus"
"gitlab.dcso.lolcat/LABS/styx/utils"
)
// SaveData is the main function used to save data. You need to pass a specific
// flag to it and the data recieved. It just saves data, it doesn't filter or
// look for it in the stream.
// func SaveData(flag string, data CertStreamStruct) {
// nodeFile, err := ioutil.ReadFile(NodesFilename)
// if err != nil {
// logrus.Error(err)
// }
// edgeFile, err := ioutil.ReadFile(EdgesFilename)
// if err != nil {
// logrus.Error(err)
// }
// nodeDatas := []Node{}
// edgeDatas := []Edge{}
// if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil {
// logrus.Error(err)
// }
// if err := json.Unmarshal(edgeFile, &edgeDatas); err != nil {
// logrus.Error(err)
// }
// // res := CertStreamStruct{}
// // json.Unmarshal(data, &res)
// node, err := BuildNode(flag, data)
// if err != nil {
// logrus.Error(err)
// }
// // edge := &Edge{
// // ID: uuid.New(),
// // NodeOneID: node.ID,
// // Timestamp: time.Now(),
// // }
// nodeDatas = append(nodeDatas, *node)
// // edgeDatas = append(edgeDatas, *edge)
// nodeBytes, err := json.Marshal(nodeDatas)
// if err != nil {
// logrus.Error(err)
// }
// // edgeBytes, err := json.Marshal(edgeDatas)
// // if err != nil {
// // logrus.Error(err)
// // }
// err = ioutil.WriteFile(nodesFilename, nodeBytes, 0644)
// if err != nil {
// logrus.Error(err)
// }
// // err = ioutil.WriteFile(edgesFilename, edgeBytes, 0644)
// // if err != nil {
// // logrus.Error(err)
// // }
// }
// BuildNode builds a node to send to MQ instance.
func BuildNode(flag string, data CertStreamStruct) (*Node, error) {
func BuildNode(flag string, data CertStreamStruct) *Node {
t := time.Now()
rfc3339time := t.Format(time.RFC3339)
return &Node{
@ -78,6 +20,75 @@ func BuildNode(flag string, data CertStreamStruct) (*Node, error) {
Data: data,
Created: rfc3339time,
Modified: rfc3339time,
}, nil
}
}
// BuildEdge build a send from two nodes with a given source type.
func BuildEdge(source string, nodeOneUUID uuid.UUID, nodeTwoUUID uuid.UUID) *Edge {
t := time.Now()
rfc3339time := t.Format(time.RFC3339)
return &Edge{
ID: uuid.New().String(),
Source: source,
NodeOneID: nodeOneUUID,
NodeTwoID: nodeTwoUUID,
Timestamp: rfc3339time,
}
}
func SaveNode(node *Node) {
err := utils.FileExists("nodes.json")
if err != nil {
logrus.Error(err)
}
nodeFile, err := ioutil.ReadFile("nodes.json")
if err != nil {
logrus.Error(err)
}
nodeDatas := []Node{}
if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil {
logrus.Error(err)
}
nodeDatas = append(nodeDatas, *node)
nodeBytes, err := json.Marshal(nodeDatas)
if err != nil {
logrus.Error(err)
}
err = ioutil.WriteFile("nodes.json", nodeBytes, 0644)
if err != nil {
logrus.Error(err)
}
}
func SaveEdge(edge *Edge) {
err := utils.FileExists("edges.json")
if err != nil {
logrus.Error(err)
}
edgeFile, err := ioutil.ReadFile("edges.json")
if err != nil {
logrus.Error(err)
}
edgeDatas := []Edge{}
if err := json.Unmarshal(edgeFile, &edgeDatas); err != nil {
logrus.Error(err)
}
edgeDatas = append(edgeDatas, *edge)
edgeBytes, err := json.Marshal(edgeDatas)
if err != nil {
logrus.Error(err)
}
err = ioutil.WriteFile("edges.json", edgeBytes, 0644)
if err != nil {
logrus.Error(err)
}
}

29
models/pastebin.go Normal file
View file

@ -0,0 +1,29 @@
package models
// PasteMeta is a set of descriptive information on a paste.
type PasteMeta struct {
ScrapeURL string `json:"scrape_url"`
FullURL string `json:"full_url"`
Date string `json:"date"`
Key string `json:"key"`
Size string `json:"size"`
Expire string `json:"expire"`
Title string `json:"title"`
Syntax string `json:"syntax"`
User string `json:"user"`
}
// PasteFull extends PasteMeta by the actual content.
type PasteFull struct {
ScrapeURL string `json:"scrape_url"`
FullURL string `json:"full_url"`
Date string `json:"date"`
Key string `json:"key"`
Size string `json:"size"`
Expire string `json:"expire"`
Title string `json:"title"`
Syntax string `json:"syntax"`
User string `json:"user"`
Data string `json:"data"`
RFC3339 string `json:"time"`
}

92
models/types.go
View file

@ -18,92 +18,8 @@ type Node struct {
// Edge defines a relation between two nodes.
type Edge struct {
ID string `json:"id"`
NodeOneID uuid.UUID `json:"NodeOneID"`
// NodeTwoID uuid.UUID `json:NodeTwoID` to implement
Timestamp string `json:"Timestamp"`
}
// LeafCertExtensions extends the LeafCert object.
type LeafCertExtensions struct {
KeyUsage string `json:"keyUsage"`
ExtendedKeyUsage string `json:"extendedKeyUsage"`
BasicConstrains string `json:"basicConstrains"`
SubjectKeyIdentifier string `json:"subjectKeyIdentifier"`
AuthorityInfoAccess string `json:"authorityInfoAccess"`
SubjectAltName string `json:"subjectAltName"`
CertificatePolicies string `json:"certificatePolicies"`
}
// LeafCertSubject is the subject of the LeafCert object.
type LeafCertSubject struct {
Aggregated string `json:"aggregated"`
C string `json:"C"`
ST string `json:"ST"`
L string `json:"L"`
O string `json:"O"`
OU string `json:"OU"`
CN string `json:"CN"`
}
// LeafCertStruct represents the LeafCert object.
type LeafCertStruct struct {
Subject LeafCertSubject `json:"subject"`
Extensions LeafCertExtensions `json:"extensions"`
NotBefore int `json:"not_before"`
NotAfter int `json:"not_after"`
SerialNumber string `json:"serial_number"`
Fingerprint string `json:"fingerprint"`
AsDer string `json:"as_der"`
AllDomains []string `json:"all_domains"`
}
// Source is the object ofr the URL and its name.
type Source struct {
URL string `json:"url"`
Name string `json:"name"`
}
// CertStreamData is the data contained in a CertStream payload.
type CertStreamData struct {
UpdateType string `json:"update_type"`
LeafCert LeafCertStruct `json:"leaf_cert"`
Chain []LeafCertStruct `json:"chain"`
CertIndex int `json:"cert_index"`
Seen int `json:"seen"`
Source Source `json:"source"`
}
// CertStreamStruct reprensts a payload received from CertStream. It has a type
// and the content is stored in Data.
type CertStreamStruct struct {
MessageType string `json:"message_data"`
Data CertStreamData `json:"data"`
}
// PasteMeta is a set of descriptive information on a paste.
type PasteMeta struct {
ScrapeURL string `json:"scrape_url"`
FullURL string `json:"full_url"`
Date string `json:"date"`
Key string `json:"key"`
Size string `json:"size"`
Expire string `json:"expire"`
Title string `json:"title"`
Syntax string `json:"syntax"`
User string `json:"user"`
}
// PasteFull extends PasteMeta by the actual content.
type PasteFull struct {
ScrapeURL string `json:"scrape_url"`
FullURL string `json:"full_url"`
Date string `json:"date"`
Key string `json:"key"`
Size string `json:"size"`
Expire string `json:"expire"`
Title string `json:"title"`
Syntax string `json:"syntax"`
User string `json:"user"`
Data string `json:"data"`
RFC3339 string `json:"time"`
NodeOneID uuid.UUID `json:"nodeOneID"`
NodeTwoID uuid.UUID `json:"nodeTwoID"`
Timestamp string `json:"timestamp"`
Source string `json:"source"`
}

32
utils/saves.go
View file

@ -1,43 +1,11 @@
package utils
import (
"encoding/json"
"io/ioutil"
"os"
"github.com/sirupsen/logrus"
"gitlab.dcso.lolcat/LABS/styx/models"
"gitlab.dcso.lolcat/LABS/styx/parser"
)
func SaveNode(node *models.Node) {
err := FileExists(parser.NodesFilename)
if err != nil {
logrus.Error(err)
}
nodeFile, err := ioutil.ReadFile(parser.NodesFilename)
if err != nil {
logrus.Error(err)
}
nodeDatas := []models.Node{}
if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil {
logrus.Error(err)
}
nodeDatas = append(nodeDatas, *node)
nodeBytes, err := json.Marshal(nodeDatas)
if err != nil {
logrus.Error(err)
}
err = ioutil.WriteFile(parser.NodesFilename, nodeBytes, 0644)
if err != nil {
logrus.Error(err)
}
}
func SaveDomains(domains []string) {
f, err := os.OpenFile("domains.txt", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
if err != nil {