package models import ( "encoding/json" "io/ioutil" "time" "github.com/google/uuid" "github.com/ns3777k/go-shodan/v4/shodan" "github.com/sirupsen/logrus" "gitlab.dcso.lolcat/LABS/styx/balboa" "gitlab.dcso.lolcat/LABS/styx/utils" ) /** Structure of this file: * type * build node functions * save node functions **/ // Node defines the data we gather through the parsing. It should follow the // Styx terminology // (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx) type Node struct { UID string `json:"uid,omiempty"` ID string `json:"id,omiempty"` Type string `json:"type,omiempty"` NData string `json:"ndata,omiempty"` Created string `json:"created,omiempty"` Modified string `json:"modified,omiempty"` DType []string `json:"dgraph.type,omiempty"` CertNode CertNode `json:"certNode,omiempty"` ShodanNode ShodanNode `json:"shodanNode,omiempty"` PasteNode PasteNode `json:"pasteNode,omiempty"` } // BuildNode builds a node to send to MQ instance. func BuildNode(flag string, dataType string, data string) *Node { t := time.Now() rfc3339time := t.Format(time.RFC3339) uuid := uuid.New().String() return &Node{ ID: flag + "--" + uuid, Type: dataType, NData: data, Created: rfc3339time, Modified: rfc3339time, } } // SaveNode saves a node to a file. func SaveNode(filename string, node *Node) { err := utils.FileExists(filename) if err != nil { logrus.Error(err) } nodeFile, err := ioutil.ReadFile(filename) if err != nil { logrus.Error(err) } nodeDatas := []Node{} if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil { logrus.Error(err) } nodeDatas = append(nodeDatas, *node) nodeBytes, err := json.Marshal(nodeDatas) if err != nil { logrus.Error(err) } err = ioutil.WriteFile(filename, nodeBytes, 0644) if err != nil { logrus.Error(err) } } // Edge defines a relation between two nodes. type Edge struct { ID string `json:"id,omiempty"` NodeOne map[string]interface{} `json:"nodeOne,omiempty"` NodeTwo map[string]interface{} `json:"nodeTwo,omiempty"` Timestamp string `json:"timestamp,omiempty"` Source string `json:"source,omiempty"` DType []string `json:"dgraph.type,omiempty"` } // BuildEdge build a send from two nodes with a given source type. func BuildEdge(source string, nodeOne, nodeTwo map[string]interface{}) *Edge { t := time.Now() rfc3339time := t.Format(time.RFC3339) uuid := uuid.New().String() return &Edge{ ID: "edge--" + uuid, Source: source, NodeOne: nodeOne, NodeTwo: nodeTwo, Timestamp: rfc3339time, } } // SaveEdge saves an edge to a file. func SaveEdge(edge *Edge) { err := utils.FileExists("edges.json") if err != nil { logrus.Error(err) } edgeFile, err := ioutil.ReadFile("edges.json") if err != nil { logrus.Error(err) } edgeDatas := []Edge{} if err := json.Unmarshal(edgeFile, &edgeDatas); err != nil { logrus.Error(err) } edgeDatas = append(edgeDatas, *edge) edgeBytes, err := json.Marshal(edgeDatas) if err != nil { logrus.Error(err) } err = ioutil.WriteFile("edges.json", edgeBytes, 0644) if err != nil { logrus.Error(err) } } // Match represents clustered results based on a target. type Match struct { ID string `json:"id,omiempty"` UID string `json:"uid,omiempty"` Nodes []Node `json:"nodes,omiempty"` Target string `json:"target,omiempty"` Timestamp string `json:"timestamp,omiempty"` Type string `json:"type,omiempty"` } // CertStreamRaw is a wrapper around the stream function to unmarshall the // data receive in a Go structure. type CertStreamRaw struct { ID string `json:"id,omiempty"` Type string `json:"type,omiempty"` Data CertStreamStruct `json:"data,omiempty"` Created string `json:"created,omiempty"` Modified string `json:"modified,omiempty"` } // CertNode represents our custom struct of data extraction from CertStream. type CertNode struct { ID string `json:"id,omiempty"` Fingerprint string `json:"fingerprint,omiempty"` NotBefore string `json:"notBefore,omiempty"` NotAfter string `json:"notAfter,omiempty"` CN string `json:"cn,omiempty"` SourceName string `json:"sourceName,omiempty"` SerialNumber string `json:"serialNumber,omiempty"` BasicConstraints string `json:"basicConstraints,omiempty"` Raw CertStreamRaw `json:"raw,omiempty"` Chain []CertNode `json:"chain,omiempty"` } // WrapCertStreamData is a wrapper around CertStreamStruct. func WrapCertStreamData(data CertStreamStruct) *CertStreamRaw { t := time.Now() rfc3339time := t.Format(time.RFC3339) return &CertStreamRaw{ ID: "certstream--" + uuid.New().String(), Type: "certstream_raw", Data: data, Created: rfc3339time, Modified: rfc3339time, } } // BuildCertNode builds a custom node based on CertStream. func BuildCertNode(rawNode *CertStreamRaw) *CertNode { main := &CertNode{ ID: "certstream--" + uuid.New().String(), Fingerprint: rawNode.Data.CSData.LeafCert.Fingerprint, NotBefore: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotBefore), 0).Format(time.RFC3339), NotAfter: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotAfter), 0).Format(time.RFC3339), CN: rawNode.Data.CSData.LeafCert.Subject.CN, SourceName: rawNode.Data.CSData.Source.Name, BasicConstraints: rawNode.Data.CSData.LeafCert.Extensions.BasicConstrains, Raw: *rawNode, } var res []CertNode if len(rawNode.Data.CSData.Chain) > 0 { chain := CertNode{ ID: "certstream--" + uuid.New().String(), Fingerprint: rawNode.Data.CSData.LeafCert.Fingerprint, NotBefore: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotBefore), 0).Format(time.RFC3339), NotAfter: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotAfter), 0).Format(time.RFC3339), CN: rawNode.Data.CSData.LeafCert.Subject.CN, SourceName: rawNode.Data.CSData.Source.Name, BasicConstraints: rawNode.Data.CSData.LeafCert.Extensions.BasicConstrains, } res = append(res, chain) } main.Chain = res return main } // SaveCertStreamRaw save the raw CertStream data. func SaveCertStreamRaw(filename string, data *CertStreamRaw) { err := utils.FileExists(filename) if err != nil { logrus.Error(err) } nodeFile, err := ioutil.ReadFile(filename) if err != nil { logrus.Error(err) } rawDatas := []CertStreamRaw{} if err := json.Unmarshal(nodeFile, &rawDatas); err != nil { logrus.Error(err) } rawDatas = append(rawDatas, *data) rawBytes, err := json.Marshal(rawDatas) if err != nil { logrus.Error(err) } err = ioutil.WriteFile(filename, rawBytes, 0644) if err != nil { logrus.Error(err) } } // SaveCertNode saves a CertNode to a json file. func SaveCertNode(filename string, node *CertNode) { err := utils.FileExists(filename) if err != nil { logrus.Error(err) } nodeFile, err := ioutil.ReadFile(filename) if err != nil { logrus.Error(err) } nodeDatas := []CertNode{} if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil { logrus.Error(err) } nodeDatas = append(nodeDatas, *node) nodeBytes, err := json.Marshal(nodeDatas) if err != nil { logrus.Error(err) } err = ioutil.WriteFile(filename, nodeBytes, 0644) if err != nil { logrus.Error(err) } } // PasteNode is a node from PasteBin. type PasteNode struct { ID string `json:"id,omiempty"` Type string `json:"type,omiempty"` FullPaste FullPaste `json:"fullPaste,omiempty"` Created string `json:"create,omiempty"` Modified string `json:"modified,omiempty"` } // FullPaste wrapes meta and information from Pastebin. type FullPaste struct { Meta PasteMeta `json:"meta,omiempty"` Full string `json:"full,omiempty"` Type string `json:"type,omiempty"` } // BuildPasteNode builds a node from a FullPaste data. func BuildPasteNode(data *FullPaste) *PasteNode { t := time.Now() rfc3339time := t.Format(time.RFC3339) return &PasteNode{ ID: "pastebin--" + uuid.New().String(), Type: "pastebin", FullPaste: *data, Created: rfc3339time, Modified: rfc3339time, } } // SavePaste saves a object received from PasteBin. func SavePaste(filename string, data *PasteNode) { err := utils.FileExists(filename) if err != nil { logrus.Error(err) } pasteFile, err := ioutil.ReadFile(filename) if err != nil { logrus.Error(err) } rawPaste := []PasteNode{} if err := json.Unmarshal(pasteFile, &rawPaste); err != nil { logrus.Error(err) } rawPaste = append(rawPaste, *data) rawBytes, err := json.Marshal(rawPaste) if err != nil { logrus.Error(err) } err = ioutil.WriteFile(filename, rawBytes, 0644) if err != nil { logrus.Error(err) } } // ShodanNode is node around the shodan.HostData struct. type ShodanNode struct { ID string `json:"id,omiempty"` Type string `json:"type,omiempty"` HostData ShodanHostData `json:"hostData,omiempty"` Created string `json:"created,omiempty"` Modified string `json:"modified,omiempty"` } // ShodanHostData is a copy of the structure in the go shodan library. It's a // workaround to have more control on the data send. type ShodanHostData struct { Product string `json:"product,omiempty"` Hostnames []string `json:"hostnames,omiempty"` Version string `json:"version,omiempty"` Title string `json:"title,omiempty"` // SSL *HostSSL `json:"ssl"` IP string `json:"ip_str,omiempty"` OS string `json:"os,omiempty"` Organization string `json:"org,omiempty"` ISP string `json:"isp,omiempty"` CPE []string `json:"cpe,omiempty"` // Data string `json:"data,omiempty"` ASN string `json:"asn,omiempty"` Port int `json:"port,omiempty"` HTML string `json:"html,omiempty"` Banner string `json:"banner,omiempty"` Link string `json:"link,omiempty"` Transport string `json:"transport,omiempty"` Domains []string `json:"domains,omiempty"` Timestamp string `json:"timestamp,omiempty"` DeviceType string `json:"devicetype,omiempty"` // Location *HostLocation `json:"location"` ShodanData map[string]interface{} `json:"_shodan,omiempty"` Opts map[string]interface{} `json:"opts,omiempty"` } // BuildShodanNode builds a wrapper node around shodan.HostData. func BuildShodanNode(data *shodan.HostData) *ShodanNode { t := time.Now() rfc3339time := t.Format(time.RFC3339) return &ShodanNode{ ID: "shodan--" + uuid.New().String(), Type: "shodan_stream", HostData: ShodanHostData{ Product: data.Product, Hostnames: data.Hostnames, Version: data.Version.String(), Title: data.Title, IP: data.IP.String(), OS: data.OS, Organization: data.Organization, ISP: data.ISP, CPE: data.CPE, // Data: data.Data, ASN: data.ASN, Port: data.Port, HTML: data.HTML, Banner: data.Banner, Link: data.Link, Transport: data.Transport, Domains: data.Domains, Timestamp: data.Timestamp, DeviceType: data.DeviceType, ShodanData: data.ShodanData, Opts: data.Opts, }, Created: rfc3339time, Modified: rfc3339time, } } // SaveShodanNode saves the raw nodes from Shodan. func SaveShodanNode(filename string, data *ShodanNode) { err := utils.FileExists(filename) if err != nil { logrus.Error(err) } nodeFile, err := ioutil.ReadFile(filename) if err != nil { logrus.Error(err) } rawDatas := []ShodanNode{} if err := json.Unmarshal(nodeFile, &rawDatas); err != nil { logrus.Error(err) } rawDatas = append(rawDatas, *data) rawBytes, err := json.Marshal(rawDatas) if err != nil { logrus.Error(err) } err = ioutil.WriteFile(filename, rawBytes, 0644) if err != nil { logrus.Error(err) } } // BalboaNode represents a return from Balboa. type BalboaNode struct { ID string `json:"id"` Type string `json:"type"` Data []balboa.Entries `json:"data"` Created string `json:"created"` Modified string `json:"modified"` } // BuildBalboaNode builds a node coming from Balboa resolution. func BuildBalboaNode(data []balboa.Entries) *BalboaNode { t := time.Now() rfc3339time := t.Format(time.RFC3339) return &BalboaNode{ ID: "balboa--" + uuid.New().String(), Type: "balboa", Data: data, Created: rfc3339time, Modified: rfc3339time, } } // SaveBalboaNode saves a Balboa node. func SaveBalboaNode(filename string, data *BalboaNode) { err := utils.FileExists(filename) if err != nil { logrus.Error(err) } nodeFile, err := ioutil.ReadFile(filename) if err != nil { logrus.Error(err) } rawDatas := []BalboaNode{} if err := json.Unmarshal(nodeFile, &rawDatas); err != nil { logrus.Error(err) } rawDatas = append(rawDatas, *data) rawBytes, err := json.Marshal(rawDatas) if err != nil { logrus.Error(err) } err = ioutil.WriteFile(filename, rawBytes, 0644) if err != nil { logrus.Error(err) } }