3961e79062
This work starts to build the matcher logic into styx. For the moment, the goal is to define IOCs and load them when the Matcher plugin is activated. To implement: Then, the matcher will run periodic queries to different types of nodes and index them to its one Matcher Dgraph Node. So be targetting a specific IOCs, the user will be able to list the observation that have been made to it.
482 lines
13 KiB
Go
482 lines
13 KiB
Go
package models
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io/ioutil"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/ns3777k/go-shodan/v4/shodan"
|
|
"github.com/sirupsen/logrus"
|
|
"gitlab.dcso.lolcat/LABS/styx/balboa"
|
|
"gitlab.dcso.lolcat/LABS/styx/utils"
|
|
)
|
|
|
|
/**
|
|
|
|
Structure of this file:
|
|
* type
|
|
* build node functions
|
|
* save node functions
|
|
|
|
**/
|
|
|
|
// Node defines the data we gather through the parsing. It should follow the
|
|
// Styx terminology
|
|
// (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx)
|
|
type Node struct {
|
|
UID string `json:"uid,omiempty"`
|
|
ID string `json:"id,omiempty"`
|
|
Type string `json:"type,omiempty"`
|
|
NData string `json:"ndata,omiempty"`
|
|
Created string `json:"created,omiempty"`
|
|
Modified string `json:"modified,omiempty"`
|
|
DType []string `json:"dgraph.type,omiempty"`
|
|
CertNode CertNode `json:"certNode,omiempty"`
|
|
ShodanNode ShodanNode `json:"shodanNode,omiempty"`
|
|
PasteNode PasteNode `json:"pasteNode,omiempty"`
|
|
}
|
|
|
|
// BuildNode builds a node to send to MQ instance.
|
|
func BuildNode(flag string, dataType string, data string) *Node {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
uuid := uuid.New().String()
|
|
return &Node{
|
|
ID: flag + "--" + uuid,
|
|
Type: dataType,
|
|
NData: data,
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// SaveNode saves a node to a file.
|
|
func SaveNode(filename string, node *Node) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeDatas := []Node{}
|
|
|
|
if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
nodeDatas = append(nodeDatas, *node)
|
|
|
|
nodeBytes, err := json.Marshal(nodeDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, nodeBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// Edge defines a relation between two nodes.
|
|
type Edge struct {
|
|
ID string `json:"id,omiempty"`
|
|
NodeOne map[string]interface{} `json:"nodeOne,omiempty"`
|
|
NodeTwo map[string]interface{} `json:"nodeTwo,omiempty"`
|
|
Timestamp string `json:"timestamp,omiempty"`
|
|
Source string `json:"source,omiempty"`
|
|
DType []string `json:"dgraph.type,omiempty"`
|
|
}
|
|
|
|
// BuildEdge build a send from two nodes with a given source type.
|
|
func BuildEdge(source string, nodeOne, nodeTwo map[string]interface{}) *Edge {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
uuid := uuid.New().String()
|
|
return &Edge{
|
|
ID: "edge--" + uuid,
|
|
Source: source,
|
|
NodeOne: nodeOne,
|
|
NodeTwo: nodeTwo,
|
|
Timestamp: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// SaveEdge saves an edge to a file.
|
|
func SaveEdge(edge *Edge) {
|
|
err := utils.FileExists("edges.json")
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
edgeFile, err := ioutil.ReadFile("edges.json")
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
edgeDatas := []Edge{}
|
|
|
|
if err := json.Unmarshal(edgeFile, &edgeDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
edgeDatas = append(edgeDatas, *edge)
|
|
edgeBytes, err := json.Marshal(edgeDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile("edges.json", edgeBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// Match represents clustered results based on a target.
|
|
type Match struct {
|
|
ID string `json:"id,omiempty"`
|
|
UID string `json:"uid,omiempty"`
|
|
Nodes []Node `json:"nodes,omiempty"`
|
|
Target string `json:"target,omiempty"`
|
|
Timestamp string `json:"timestamp,omiempty"`
|
|
Type string `json:"type,omiempty"`
|
|
}
|
|
|
|
// CertStreamRaw is a wrapper around the stream function to unmarshall the
|
|
// data receive in a Go structure.
|
|
type CertStreamRaw struct {
|
|
ID string `json:"id,omiempty"`
|
|
Type string `json:"type,omiempty"`
|
|
Data CertStreamStruct `json:"data,omiempty"`
|
|
Created string `json:"created,omiempty"`
|
|
Modified string `json:"modified,omiempty"`
|
|
}
|
|
|
|
// CertNode represents our custom struct of data extraction from CertStream.
|
|
type CertNode struct {
|
|
ID string `json:"id,omiempty"`
|
|
Fingerprint string `json:"fingerprint,omiempty"`
|
|
NotBefore string `json:"notBefore,omiempty"`
|
|
NotAfter string `json:"notAfter,omiempty"`
|
|
CN string `json:"cn,omiempty"`
|
|
SourceName string `json:"sourceName,omiempty"`
|
|
SerialNumber string `json:"serialNumber,omiempty"`
|
|
BasicConstraints string `json:"basicConstraints,omiempty"`
|
|
Raw CertStreamRaw `json:"raw,omiempty"`
|
|
Chain []CertNode `json:"chain,omiempty"`
|
|
}
|
|
|
|
// WrapCertStreamData is a wrapper around CertStreamStruct.
|
|
func WrapCertStreamData(data CertStreamStruct) *CertStreamRaw {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
return &CertStreamRaw{
|
|
ID: "certstream--" + uuid.New().String(),
|
|
Type: "certstream_raw",
|
|
Data: data,
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// BuildCertNode builds a custom node based on CertStream.
|
|
func BuildCertNode(rawNode *CertStreamRaw) *CertNode {
|
|
main := &CertNode{
|
|
ID: "certstream--" + uuid.New().String(),
|
|
Fingerprint: rawNode.Data.CSData.LeafCert.Fingerprint,
|
|
NotBefore: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotBefore), 0).Format(time.RFC3339),
|
|
NotAfter: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotAfter), 0).Format(time.RFC3339),
|
|
CN: rawNode.Data.CSData.LeafCert.Subject.CN,
|
|
SourceName: rawNode.Data.CSData.Source.Name,
|
|
BasicConstraints: rawNode.Data.CSData.LeafCert.Extensions.BasicConstrains,
|
|
Raw: *rawNode,
|
|
}
|
|
|
|
var res []CertNode
|
|
if len(rawNode.Data.CSData.Chain) > 0 {
|
|
chain := CertNode{
|
|
ID: "certstream--" + uuid.New().String(),
|
|
Fingerprint: rawNode.Data.CSData.LeafCert.Fingerprint,
|
|
NotBefore: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotBefore), 0).Format(time.RFC3339),
|
|
NotAfter: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotAfter), 0).Format(time.RFC3339),
|
|
CN: rawNode.Data.CSData.LeafCert.Subject.CN,
|
|
SourceName: rawNode.Data.CSData.Source.Name,
|
|
BasicConstraints: rawNode.Data.CSData.LeafCert.Extensions.BasicConstrains,
|
|
}
|
|
res = append(res, chain)
|
|
}
|
|
main.Chain = res
|
|
|
|
return main
|
|
}
|
|
|
|
// SaveCertStreamRaw save the raw CertStream data.
|
|
func SaveCertStreamRaw(filename string, data *CertStreamRaw) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
rawDatas := []CertStreamRaw{}
|
|
|
|
if err := json.Unmarshal(nodeFile, &rawDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
rawDatas = append(rawDatas, *data)
|
|
|
|
rawBytes, err := json.Marshal(rawDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// SaveCertNode saves a CertNode to a json file.
|
|
func SaveCertNode(filename string, node *CertNode) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeDatas := []CertNode{}
|
|
|
|
if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
nodeDatas = append(nodeDatas, *node)
|
|
|
|
nodeBytes, err := json.Marshal(nodeDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, nodeBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// PasteNode is a node from PasteBin.
|
|
type PasteNode struct {
|
|
ID string `json:"id,omiempty"`
|
|
Type string `json:"type,omiempty"`
|
|
FullPaste FullPaste `json:"fullPaste,omiempty"`
|
|
Created string `json:"create,omiempty"`
|
|
Modified string `json:"modified,omiempty"`
|
|
}
|
|
|
|
// FullPaste wrapes meta and information from Pastebin.
|
|
type FullPaste struct {
|
|
Meta PasteMeta `json:"meta,omiempty"`
|
|
Full string `json:"full,omiempty"`
|
|
Type string `json:"type,omiempty"`
|
|
}
|
|
|
|
// BuildPasteNode builds a node from a FullPaste data.
|
|
func BuildPasteNode(data *FullPaste) *PasteNode {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
return &PasteNode{
|
|
ID: "pastebin--" + uuid.New().String(),
|
|
Type: "pastebin",
|
|
FullPaste: *data,
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// SavePaste saves a object received from PasteBin.
|
|
func SavePaste(filename string, data *PasteNode) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
pasteFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
rawPaste := []PasteNode{}
|
|
|
|
if err := json.Unmarshal(pasteFile, &rawPaste); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
rawPaste = append(rawPaste, *data)
|
|
|
|
rawBytes, err := json.Marshal(rawPaste)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// ShodanNode is node around the shodan.HostData struct.
|
|
type ShodanNode struct {
|
|
ID string `json:"id,omiempty"`
|
|
Type string `json:"type,omiempty"`
|
|
HostData ShodanHostData `json:"hostData,omiempty"`
|
|
Created string `json:"created,omiempty"`
|
|
Modified string `json:"modified,omiempty"`
|
|
}
|
|
|
|
// ShodanHostData is a copy of the structure in the go shodan library. It's a
|
|
// workaround to have more control on the data send.
|
|
type ShodanHostData struct {
|
|
Product string `json:"product,omiempty"`
|
|
Hostnames []string `json:"hostnames,omiempty"`
|
|
Version string `json:"version,omiempty"`
|
|
Title string `json:"title,omiempty"`
|
|
// SSL *HostSSL `json:"ssl"`
|
|
IP string `json:"ip_str,omiempty"`
|
|
OS string `json:"os,omiempty"`
|
|
Organization string `json:"org,omiempty"`
|
|
ISP string `json:"isp,omiempty"`
|
|
CPE []string `json:"cpe,omiempty"`
|
|
// Data string `json:"data,omiempty"`
|
|
ASN string `json:"asn,omiempty"`
|
|
Port int `json:"port,omiempty"`
|
|
HTML string `json:"html,omiempty"`
|
|
Banner string `json:"banner,omiempty"`
|
|
Link string `json:"link,omiempty"`
|
|
Transport string `json:"transport,omiempty"`
|
|
Domains []string `json:"domains,omiempty"`
|
|
Timestamp string `json:"timestamp,omiempty"`
|
|
DeviceType string `json:"devicetype,omiempty"`
|
|
// Location *HostLocation `json:"location"`
|
|
ShodanData map[string]interface{} `json:"_shodan,omiempty"`
|
|
Opts map[string]interface{} `json:"opts,omiempty"`
|
|
}
|
|
|
|
// BuildShodanNode builds a wrapper node around shodan.HostData.
|
|
func BuildShodanNode(data *shodan.HostData) *ShodanNode {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
return &ShodanNode{
|
|
ID: "shodan--" + uuid.New().String(),
|
|
Type: "shodan_stream",
|
|
HostData: ShodanHostData{
|
|
Product: data.Product,
|
|
Hostnames: data.Hostnames,
|
|
Version: data.Version.String(),
|
|
Title: data.Title,
|
|
IP: data.IP.String(),
|
|
OS: data.OS,
|
|
Organization: data.Organization,
|
|
ISP: data.ISP,
|
|
CPE: data.CPE,
|
|
// Data: data.Data,
|
|
ASN: data.ASN,
|
|
Port: data.Port,
|
|
HTML: data.HTML,
|
|
Banner: data.Banner,
|
|
Link: data.Link,
|
|
Transport: data.Transport,
|
|
Domains: data.Domains,
|
|
Timestamp: data.Timestamp,
|
|
DeviceType: data.DeviceType,
|
|
ShodanData: data.ShodanData,
|
|
Opts: data.Opts,
|
|
},
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// SaveShodanNode saves the raw nodes from Shodan.
|
|
func SaveShodanNode(filename string, data *ShodanNode) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
rawDatas := []ShodanNode{}
|
|
|
|
if err := json.Unmarshal(nodeFile, &rawDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
rawDatas = append(rawDatas, *data)
|
|
|
|
rawBytes, err := json.Marshal(rawDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// BalboaNode represents a return from Balboa.
|
|
type BalboaNode struct {
|
|
ID string `json:"id"`
|
|
Type string `json:"type"`
|
|
Data []balboa.Entries `json:"data"`
|
|
Created string `json:"created"`
|
|
Modified string `json:"modified"`
|
|
}
|
|
|
|
// BuildBalboaNode builds a node coming from Balboa resolution.
|
|
func BuildBalboaNode(data []balboa.Entries) *BalboaNode {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
return &BalboaNode{
|
|
ID: "balboa--" + uuid.New().String(),
|
|
Type: "balboa",
|
|
Data: data,
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// SaveBalboaNode saves a Balboa node.
|
|
func SaveBalboaNode(filename string, data *BalboaNode) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
rawDatas := []BalboaNode{}
|
|
|
|
if err := json.Unmarshal(nodeFile, &rawDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
rawDatas = append(rawDatas, *data)
|
|
|
|
rawBytes, err := json.Marshal(rawDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|