9fa5d13bf6
Some of keywords are indexed and open for full text search, please refer to the README for more details. CertStream, Pastebin and Shodan are running as services and can be searched. Next steps: building the matcher and creating edges.
471 lines
13 KiB
Go
471 lines
13 KiB
Go
package models
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io/ioutil"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/ns3777k/go-shodan/v4/shodan"
|
|
"github.com/sirupsen/logrus"
|
|
"gitlab.dcso.lolcat/LABS/styx/balboa"
|
|
"gitlab.dcso.lolcat/LABS/styx/utils"
|
|
)
|
|
|
|
/**
|
|
|
|
Structure of this file:
|
|
* type
|
|
* build node functions
|
|
* save node functions
|
|
|
|
**/
|
|
|
|
// Node defines the data we gather through the parsing. It should follow the
|
|
// Styx terminology
|
|
// (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx)
|
|
type Node struct {
|
|
ID string `json:"id,omiempty"`
|
|
Type string `json:"type,omiempty"`
|
|
NData string `json:"ndata,omiempty"`
|
|
Created string `json:"created,omiempty"`
|
|
Modified string `json:"modified,omiempty"`
|
|
DType []string `json:"dgraph.type,omiempty"`
|
|
CertNode CertNode `json:"certNode,omiempty"`
|
|
ShodanNode ShodanNode `json:"shodanNode,omiempty"`
|
|
PasteNode PasteNode `json:"pasteNode,omiempty"`
|
|
}
|
|
|
|
// BuildNode builds a node to send to MQ instance.
|
|
func BuildNode(flag string, dataType string, data string) *Node {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
uuid := uuid.New().String()
|
|
return &Node{
|
|
ID: flag + "--" + uuid,
|
|
Type: dataType,
|
|
NData: data,
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// SaveNode saves a node to a file.
|
|
func SaveNode(filename string, node *Node) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeDatas := []Node{}
|
|
|
|
if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
nodeDatas = append(nodeDatas, *node)
|
|
|
|
nodeBytes, err := json.Marshal(nodeDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, nodeBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// Edge defines a relation between two nodes.
|
|
type Edge struct {
|
|
ID string `json:"id,omiempty"`
|
|
NodeOne map[string]interface{} `json:"nodeOne,omiempty"`
|
|
NodeTwo map[string]interface{} `json:"nodeTwo,omiempty"`
|
|
Timestamp string `json:"timestamp,omiempty"`
|
|
Source string `json:"source,omiempty"`
|
|
DType []string `json:"dgraph.type,omiempty"`
|
|
}
|
|
|
|
// BuildEdge build a send from two nodes with a given source type.
|
|
func BuildEdge(source string, nodeOne, nodeTwo map[string]interface{}) *Edge {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
uuid := uuid.New().String()
|
|
return &Edge{
|
|
ID: "edge--" + uuid,
|
|
Source: source,
|
|
NodeOne: nodeOne,
|
|
NodeTwo: nodeTwo,
|
|
Timestamp: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// SaveEdge saves an edge to a file.
|
|
func SaveEdge(edge *Edge) {
|
|
err := utils.FileExists("edges.json")
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
edgeFile, err := ioutil.ReadFile("edges.json")
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
edgeDatas := []Edge{}
|
|
|
|
if err := json.Unmarshal(edgeFile, &edgeDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
edgeDatas = append(edgeDatas, *edge)
|
|
edgeBytes, err := json.Marshal(edgeDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile("edges.json", edgeBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// CertStreamRaw is a wrapper around the stream function to unmarshall the
|
|
// data receive in a Go structure.
|
|
type CertStreamRaw struct {
|
|
ID string `json:"id,omiempty"`
|
|
Type string `json:"type,omiempty"`
|
|
Data CertStreamStruct `json:"data,omiempty"`
|
|
Created string `json:"created,omiempty"`
|
|
Modified string `json:"modified,omiempty"`
|
|
}
|
|
|
|
// CertNode represents our custom struct of data extraction from CertStream.
|
|
type CertNode struct {
|
|
ID string `json:"id,omiempty"`
|
|
Fingerprint string `json:"fingerprint,omiempty"`
|
|
NotBefore string `json:"notBefore,omiempty"`
|
|
NotAfter string `json:"notAfter,omiempty"`
|
|
CN string `json:"cn,omiempty"`
|
|
SourceName string `json:"sourceName,omiempty"`
|
|
SerialNumber string `json:"serialNumber,omiempty"`
|
|
BasicConstraints string `json:"basicConstraints,omiempty"`
|
|
Raw CertStreamRaw `json:"raw,omiempty"`
|
|
Chain []CertNode `json:"chain,omiempty"`
|
|
}
|
|
|
|
// WrapCertStreamData is a wrapper around CertStreamStruct.
|
|
func WrapCertStreamData(data CertStreamStruct) *CertStreamRaw {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
return &CertStreamRaw{
|
|
ID: "certstream--" + uuid.New().String(),
|
|
Type: "certstream_raw",
|
|
Data: data,
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// BuildCertNode builds a custom node based on CertStream.
|
|
func BuildCertNode(rawNode *CertStreamRaw) *CertNode {
|
|
main := &CertNode{
|
|
ID: "certstream--" + uuid.New().String(),
|
|
Fingerprint: rawNode.Data.CSData.LeafCert.Fingerprint,
|
|
NotBefore: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotBefore), 0).Format(time.RFC3339),
|
|
NotAfter: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotAfter), 0).Format(time.RFC3339),
|
|
CN: rawNode.Data.CSData.LeafCert.Subject.CN,
|
|
SourceName: rawNode.Data.CSData.Source.Name,
|
|
BasicConstraints: rawNode.Data.CSData.LeafCert.Extensions.BasicConstrains,
|
|
Raw: *rawNode,
|
|
}
|
|
|
|
var res []CertNode
|
|
if len(rawNode.Data.CSData.Chain) > 0 {
|
|
chain := CertNode{
|
|
ID: "certstream--" + uuid.New().String(),
|
|
Fingerprint: rawNode.Data.CSData.LeafCert.Fingerprint,
|
|
NotBefore: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotBefore), 0).Format(time.RFC3339),
|
|
NotAfter: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotAfter), 0).Format(time.RFC3339),
|
|
CN: rawNode.Data.CSData.LeafCert.Subject.CN,
|
|
SourceName: rawNode.Data.CSData.Source.Name,
|
|
BasicConstraints: rawNode.Data.CSData.LeafCert.Extensions.BasicConstrains,
|
|
}
|
|
res = append(res, chain)
|
|
}
|
|
main.Chain = res
|
|
|
|
return main
|
|
}
|
|
|
|
// SaveCertStreamRaw save the raw CertStream data.
|
|
func SaveCertStreamRaw(filename string, data *CertStreamRaw) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
rawDatas := []CertStreamRaw{}
|
|
|
|
if err := json.Unmarshal(nodeFile, &rawDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
rawDatas = append(rawDatas, *data)
|
|
|
|
rawBytes, err := json.Marshal(rawDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// SaveCertNode saves a CertNode to a json file.
|
|
func SaveCertNode(filename string, node *CertNode) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeDatas := []CertNode{}
|
|
|
|
if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
nodeDatas = append(nodeDatas, *node)
|
|
|
|
nodeBytes, err := json.Marshal(nodeDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, nodeBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// PasteNode is a node from PasteBin.
|
|
type PasteNode struct {
|
|
ID string `json:"id,omiempty"`
|
|
Type string `json:"type,omiempty"`
|
|
FullPaste FullPaste `json:"fullPaste,omiempty"`
|
|
Created string `json:"create,omiempty"`
|
|
Modified string `json:"modified,omiempty"`
|
|
}
|
|
|
|
// FullPaste wrapes meta and information from Pastebin.
|
|
type FullPaste struct {
|
|
Meta PasteMeta `json:"meta,omiempty"`
|
|
Full string `json:"full,omiempty"`
|
|
Type string `json:"type,omiempty"`
|
|
}
|
|
|
|
// BuildPasteNode builds a node from a FullPaste data.
|
|
func BuildPasteNode(data *FullPaste) *PasteNode {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
return &PasteNode{
|
|
ID: "pastebin--" + uuid.New().String(),
|
|
Type: "pastebin",
|
|
FullPaste: *data,
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// SavePaste saves a object received from PasteBin.
|
|
func SavePaste(filename string, data *PasteNode) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
pasteFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
rawPaste := []PasteNode{}
|
|
|
|
if err := json.Unmarshal(pasteFile, &rawPaste); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
rawPaste = append(rawPaste, *data)
|
|
|
|
rawBytes, err := json.Marshal(rawPaste)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// ShodanNode is node around the shodan.HostData struct.
|
|
type ShodanNode struct {
|
|
ID string `json:"id,omiempty"`
|
|
Type string `json:"type,omiempty"`
|
|
HostData ShodanHostData `json:"hostData,omiempty"`
|
|
Created string `json:"created,omiempty"`
|
|
Modified string `json:"modified,omiempty"`
|
|
}
|
|
|
|
// ShodanHostData is a copy of the structure in the go shodan library. It's a
|
|
// workaround to have more control on the data send.
|
|
type ShodanHostData struct {
|
|
Product string `json:"product,omiempty"`
|
|
Hostnames []string `json:"hostnames,omiempty"`
|
|
Version string `json:"version,omiempty"`
|
|
Title string `json:"title,omiempty"`
|
|
// SSL *HostSSL `json:"ssl"`
|
|
IP string `json:"ip_str,omiempty"`
|
|
OS string `json:"os,omiempty"`
|
|
Organization string `json:"org,omiempty"`
|
|
ISP string `json:"isp,omiempty"`
|
|
CPE []string `json:"cpe,omiempty"`
|
|
// Data string `json:"data,omiempty"`
|
|
ASN string `json:"asn,omiempty"`
|
|
Port int `json:"port,omiempty"`
|
|
HTML string `json:"html,omiempty"`
|
|
Banner string `json:"banner,omiempty"`
|
|
Link string `json:"link,omiempty"`
|
|
Transport string `json:"transport,omiempty"`
|
|
Domains []string `json:"domains,omiempty"`
|
|
Timestamp string `json:"timestamp,omiempty"`
|
|
DeviceType string `json:"devicetype,omiempty"`
|
|
// Location *HostLocation `json:"location"`
|
|
ShodanData map[string]interface{} `json:"_shodan,omiempty"`
|
|
Opts map[string]interface{} `json:"opts,omiempty"`
|
|
}
|
|
|
|
// BuildShodanNode builds a wrapper node around shodan.HostData.
|
|
func BuildShodanNode(data *shodan.HostData) *ShodanNode {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
return &ShodanNode{
|
|
ID: "shodan--" + uuid.New().String(),
|
|
Type: "shodan_stream",
|
|
HostData: ShodanHostData{
|
|
Product: data.Product,
|
|
Hostnames: data.Hostnames,
|
|
Version: data.Version.String(),
|
|
Title: data.Title,
|
|
IP: data.IP.String(),
|
|
OS: data.OS,
|
|
Organization: data.Organization,
|
|
ISP: data.ISP,
|
|
CPE: data.CPE,
|
|
// Data: data.Data,
|
|
ASN: data.ASN,
|
|
Port: data.Port,
|
|
HTML: data.HTML,
|
|
Banner: data.Banner,
|
|
Link: data.Link,
|
|
Transport: data.Transport,
|
|
Domains: data.Domains,
|
|
Timestamp: data.Timestamp,
|
|
DeviceType: data.DeviceType,
|
|
ShodanData: data.ShodanData,
|
|
Opts: data.Opts,
|
|
},
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// SaveShodanNode saves the raw nodes from Shodan.
|
|
func SaveShodanNode(filename string, data *ShodanNode) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
rawDatas := []ShodanNode{}
|
|
|
|
if err := json.Unmarshal(nodeFile, &rawDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
rawDatas = append(rawDatas, *data)
|
|
|
|
rawBytes, err := json.Marshal(rawDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// BalboaNode represents a return from Balboa.
|
|
type BalboaNode struct {
|
|
ID string `json:"id"`
|
|
Type string `json:"type"`
|
|
Data []balboa.Entries `json:"data"`
|
|
Created string `json:"created"`
|
|
Modified string `json:"modified"`
|
|
}
|
|
|
|
// BuildBalboaNode builds a node coming from Balboa resolution.
|
|
func BuildBalboaNode(data []balboa.Entries) *BalboaNode {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
return &BalboaNode{
|
|
ID: "balboa--" + uuid.New().String(),
|
|
Type: "balboa",
|
|
Data: data,
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// SaveBalboaNode saves a Balboa node.
|
|
func SaveBalboaNode(filename string, data *BalboaNode) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
rawDatas := []BalboaNode{}
|
|
|
|
if err := json.Unmarshal(nodeFile, &rawDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
rawDatas = append(rawDatas, *data)
|
|
|
|
rawBytes, err := json.Marshal(rawDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|