483 lines
13 KiB
Go
483 lines
13 KiB
Go
package models
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io/ioutil"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/ns3777k/go-shodan/v4/shodan"
|
|
"github.com/sirupsen/logrus"
|
|
"gitlab.dcso.lolcat/LABS/styx/balboa"
|
|
"gitlab.dcso.lolcat/LABS/styx/utils"
|
|
)
|
|
|
|
/**
|
|
|
|
Structure of this file:
|
|
* type
|
|
* build node functions
|
|
* save node functions
|
|
|
|
**/
|
|
|
|
// Node defines the data we gather through the parsing. It should follow the
|
|
// Styx terminology
|
|
// (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx)
|
|
type Node struct {
|
|
UID string `json:"uid,omitempty"`
|
|
ID string `json:"id,omitempty"`
|
|
NodeType string `json:"nodeType,omitempty"`
|
|
NData string `json:"ndata,omitempty"`
|
|
Created string `json:"created,omitempty"`
|
|
Modified string `json:"modified,omitempty"`
|
|
DType []string `json:"dgraph.type,omitempty"`
|
|
CertNode CertNode `json:"certNode,omitempty"`
|
|
ShodanNode ShodanNode `json:"shodanNode,omitempty"`
|
|
PasteNode PasteNode `json:"pasteNode,omitempty"`
|
|
Match Match `json:"match,omitempty"`
|
|
}
|
|
|
|
// BuildNode builds a node to send to MQ instance.
|
|
func BuildNode(flag string, dataType string, data string) *Node {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
uuid := uuid.New().String()
|
|
return &Node{
|
|
ID: flag + "--" + uuid,
|
|
NodeType: dataType,
|
|
NData: data,
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// SaveNode saves a node to a file.
|
|
func SaveNode(filename string, node *Node) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeDatas := []Node{}
|
|
|
|
if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
nodeDatas = append(nodeDatas, *node)
|
|
|
|
nodeBytes, err := json.Marshal(nodeDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, nodeBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// Edge defines a relation between two nodes.
|
|
type Edge struct {
|
|
ID string `json:"id,omitempty"`
|
|
NodeOne map[string]interface{} `json:"nodeOne,omitempty"`
|
|
NodeTwo map[string]interface{} `json:"nodeTwo,omitempty"`
|
|
Timestamp string `json:"timestamp,omitempty"`
|
|
Source string `json:"source,omitempty"`
|
|
DType []string `json:"dgraph.type,omitempty"`
|
|
}
|
|
|
|
// BuildEdge build a send from two nodes with a given source type.
|
|
func BuildEdge(source string, nodeOne, nodeTwo map[string]interface{}) *Edge {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
uuid := uuid.New().String()
|
|
return &Edge{
|
|
ID: "edge--" + uuid,
|
|
Source: source,
|
|
NodeOne: nodeOne,
|
|
NodeTwo: nodeTwo,
|
|
Timestamp: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// SaveEdge saves an edge to a file.
|
|
func SaveEdge(edge *Edge) {
|
|
err := utils.FileExists("edges.json")
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
edgeFile, err := ioutil.ReadFile("edges.json")
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
edgeDatas := []Edge{}
|
|
|
|
if err := json.Unmarshal(edgeFile, &edgeDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
edgeDatas = append(edgeDatas, *edge)
|
|
edgeBytes, err := json.Marshal(edgeDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile("edges.json", edgeBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// Match represents clustered results based on a target.
|
|
type Match struct {
|
|
ID string `json:"id,omitempty"`
|
|
UID string `json:"uid,omitempty"`
|
|
Nodes []Node `json:"nodes,omitempty"`
|
|
Target string `json:"target,omitempty"`
|
|
Timestamp string `json:"timestamp,omitempty"`
|
|
NodeType string `json:"nodeType,omitempty"`
|
|
}
|
|
|
|
// CertStreamRaw is a wrapper around the stream function to unmarshall the
|
|
// data receive in a Go structure.
|
|
type CertStreamRaw struct {
|
|
ID string `json:"id,omitempty"`
|
|
NodeType string `json:"nodeType,omitempty"`
|
|
Data CertStreamStruct `json:"data,omitempty"`
|
|
Created string `json:"created,omitempty"`
|
|
Modified string `json:"modified,omitempty"`
|
|
}
|
|
|
|
// CertNode represents our custom struct of data extraction from CertStream.
|
|
type CertNode struct {
|
|
ID string `json:"id,omitempty"`
|
|
Fingerprint string `json:"fingerprint,omitempty"`
|
|
NotBefore string `json:"notBefore,omitempty"`
|
|
NotAfter string `json:"notAfter,omitempty"`
|
|
CN string `json:"cn,omitempty"`
|
|
SourceName string `json:"sourceName,omitempty"`
|
|
SerialNumber string `json:"serialNumber,omitempty"`
|
|
BasicConstraints string `json:"basicConstraints,omitempty"`
|
|
Raw CertStreamRaw `json:"raw,omitempty"`
|
|
Chain []CertNode `json:"chain,omitempty"`
|
|
}
|
|
|
|
// WrapCertStreamData is a wrapper around CertStreamStruct.
|
|
func WrapCertStreamData(data CertStreamStruct) *CertStreamRaw {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
return &CertStreamRaw{
|
|
ID: "certstream--" + uuid.New().String(),
|
|
NodeType: "certstream_raw",
|
|
Data: data,
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// BuildCertNode builds a custom node based on CertStream.
|
|
func BuildCertNode(rawNode *CertStreamRaw) *CertNode {
|
|
main := &CertNode{
|
|
ID: "certstream--" + uuid.New().String(),
|
|
Fingerprint: rawNode.Data.CSData.LeafCert.Fingerprint,
|
|
NotBefore: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotBefore), 0).Format(time.RFC3339),
|
|
NotAfter: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotAfter), 0).Format(time.RFC3339),
|
|
CN: rawNode.Data.CSData.LeafCert.Subject.CN,
|
|
SourceName: rawNode.Data.CSData.Source.Name,
|
|
BasicConstraints: rawNode.Data.CSData.LeafCert.Extensions.BasicConstrains,
|
|
Raw: *rawNode,
|
|
}
|
|
|
|
var res []CertNode
|
|
if len(rawNode.Data.CSData.Chain) > 0 {
|
|
chain := CertNode{
|
|
ID: "certstream--" + uuid.New().String(),
|
|
Fingerprint: rawNode.Data.CSData.LeafCert.Fingerprint,
|
|
NotBefore: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotBefore), 0).Format(time.RFC3339),
|
|
NotAfter: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotAfter), 0).Format(time.RFC3339),
|
|
CN: rawNode.Data.CSData.LeafCert.Subject.CN,
|
|
SourceName: rawNode.Data.CSData.Source.Name,
|
|
BasicConstraints: rawNode.Data.CSData.LeafCert.Extensions.BasicConstrains,
|
|
}
|
|
res = append(res, chain)
|
|
}
|
|
main.Chain = res
|
|
|
|
return main
|
|
}
|
|
|
|
// SaveCertStreamRaw save the raw CertStream data.
|
|
func SaveCertStreamRaw(filename string, data *CertStreamRaw) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
rawDatas := []CertStreamRaw{}
|
|
|
|
if err := json.Unmarshal(nodeFile, &rawDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
rawDatas = append(rawDatas, *data)
|
|
|
|
rawBytes, err := json.Marshal(rawDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// SaveCertNode saves a CertNode to a json file.
|
|
func SaveCertNode(filename string, node *CertNode) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeDatas := []CertNode{}
|
|
|
|
if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
nodeDatas = append(nodeDatas, *node)
|
|
|
|
nodeBytes, err := json.Marshal(nodeDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, nodeBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// PasteNode is a node from PasteBin.
|
|
type PasteNode struct {
|
|
ID string `json:"id,omitempty"`
|
|
NodeType string `json:"nodeType,omitempty"`
|
|
FullPaste FullPaste `json:"fullPaste,omitempty"`
|
|
Created string `json:"create,omitempty"`
|
|
Modified string `json:"modified,omitempty"`
|
|
}
|
|
|
|
// FullPaste wrapes meta and information from Pastebin.
|
|
type FullPaste struct {
|
|
Meta PasteMeta `json:"meta,omitempty"`
|
|
Full string `json:"full,omitempty"`
|
|
NodeType string `json:"nodeType,omitempty"`
|
|
}
|
|
|
|
// BuildPasteNode builds a node from a FullPaste data.
|
|
func BuildPasteNode(data *FullPaste) *PasteNode {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
return &PasteNode{
|
|
ID: "pastebin--" + uuid.New().String(),
|
|
NodeType: "pastebin",
|
|
FullPaste: *data,
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// SavePaste saves a object received from PasteBin.
|
|
func SavePaste(filename string, data *PasteNode) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
pasteFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
rawPaste := []PasteNode{}
|
|
|
|
if err := json.Unmarshal(pasteFile, &rawPaste); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
rawPaste = append(rawPaste, *data)
|
|
|
|
rawBytes, err := json.Marshal(rawPaste)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// ShodanNode is node around the shodan.HostData struct.
|
|
type ShodanNode struct {
|
|
ID string `json:"id,omitempty"`
|
|
NodeType string `json:"nodeType,omitempty"`
|
|
HostData ShodanHostData `json:"hostData,omitempty"`
|
|
Created string `json:"created,omitempty"`
|
|
Modified string `json:"modified,omitempty"`
|
|
}
|
|
|
|
// ShodanHostData is a copy of the structure in the go shodan library. It's a
|
|
// workaround to have more control on the data send.
|
|
type ShodanHostData struct {
|
|
Product string `json:"product,omitempty"`
|
|
Hostnames []string `json:"hostnames,omitempty"`
|
|
Version string `json:"version,omitempty"`
|
|
Title string `json:"title,omitempty"`
|
|
// SSL *HostSSL `json:"ssl"`
|
|
IP string `json:"ip_str,omitempty"`
|
|
OS string `json:"os,omitempty"`
|
|
Organization string `json:"org,omitempty"`
|
|
ISP string `json:"isp,omitempty"`
|
|
CPE []string `json:"cpe,omitempty"`
|
|
// Data string `json:"data,omitempty"`
|
|
ASN string `json:"asn,omitempty"`
|
|
Port int `json:"port,omitempty"`
|
|
HTML string `json:"html,omitempty"`
|
|
Banner string `json:"banner,omitempty"`
|
|
Link string `json:"link,omitempty"`
|
|
Transport string `json:"transport,omitempty"`
|
|
Domains []string `json:"domains,omitempty"`
|
|
Timestamp string `json:"timestamp,omitempty"`
|
|
DeviceType string `json:"devicetype,omitempty"`
|
|
// Location *HostLocation `json:"location"`
|
|
ShodanData map[string]interface{} `json:"_shodan,omitempty"`
|
|
Opts map[string]interface{} `json:"opts,omitempty"`
|
|
}
|
|
|
|
// BuildShodanNode builds a wrapper node around shodan.HostData.
|
|
func BuildShodanNode(data *shodan.HostData) *ShodanNode {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
return &ShodanNode{
|
|
ID: "shodan--" + uuid.New().String(),
|
|
NodeType: "shodan_stream",
|
|
HostData: ShodanHostData{
|
|
Product: data.Product,
|
|
Hostnames: data.Hostnames,
|
|
Version: data.Version.String(),
|
|
Title: data.Title,
|
|
IP: data.IP.String(),
|
|
OS: data.OS,
|
|
Organization: data.Organization,
|
|
ISP: data.ISP,
|
|
CPE: data.CPE,
|
|
// Data: data.Data,
|
|
ASN: data.ASN,
|
|
Port: data.Port,
|
|
HTML: data.HTML,
|
|
Banner: data.Banner,
|
|
Link: data.Link,
|
|
Transport: data.Transport,
|
|
Domains: data.Domains,
|
|
Timestamp: data.Timestamp,
|
|
DeviceType: data.DeviceType,
|
|
ShodanData: data.ShodanData,
|
|
Opts: data.Opts,
|
|
},
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// SaveShodanNode saves the raw nodes from Shodan.
|
|
func SaveShodanNode(filename string, data *ShodanNode) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
rawDatas := []ShodanNode{}
|
|
|
|
if err := json.Unmarshal(nodeFile, &rawDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
rawDatas = append(rawDatas, *data)
|
|
|
|
rawBytes, err := json.Marshal(rawDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// BalboaNode represents a return from Balboa.
|
|
type BalboaNode struct {
|
|
ID string `json:"id"`
|
|
NodeType string `json:"nodeType"`
|
|
Data []balboa.Entries `json:"data"`
|
|
Created string `json:"created"`
|
|
Modified string `json:"modified"`
|
|
}
|
|
|
|
// BuildBalboaNode builds a node coming from Balboa resolution.
|
|
func BuildBalboaNode(data []balboa.Entries) *BalboaNode {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
return &BalboaNode{
|
|
ID: "balboa--" + uuid.New().String(),
|
|
NodeType: "balboa",
|
|
Data: data,
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// SaveBalboaNode saves a Balboa node.
|
|
func SaveBalboaNode(filename string, data *BalboaNode) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
rawDatas := []BalboaNode{}
|
|
|
|
if err := json.Unmarshal(nodeFile, &rawDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
rawDatas = append(rawDatas, *data)
|
|
|
|
rawBytes, err := json.Marshal(rawDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|