b1ca4b3c5f
Implementing first version for shodan node, missing yet some models, but the overal approach works and can be queried in Ratel.
469 lines
13 KiB
Go
469 lines
13 KiB
Go
package models
|
|
|
|
import (
|
|
"encoding/json"
|
|
"io/ioutil"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/ns3777k/go-shodan/v4/shodan"
|
|
"github.com/sirupsen/logrus"
|
|
"gitlab.dcso.lolcat/LABS/styx/balboa"
|
|
"gitlab.dcso.lolcat/LABS/styx/utils"
|
|
)
|
|
|
|
/**
|
|
|
|
Structure of this file:
|
|
* type
|
|
* build node functions
|
|
* save node functions
|
|
|
|
**/
|
|
|
|
// Node defines the data we gather through the parsing. It should follow the
|
|
// Styx terminology
|
|
// (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx)
|
|
type Node struct {
|
|
ID string `json:"id,omiempty"`
|
|
Type string `json:"type,omiempty"`
|
|
NData string `json:"ndata,omiempty"`
|
|
Created string `json:"created,omiempty"`
|
|
Modified string `json:"modified,omiempty"`
|
|
DType []string `json:"dgraph.type,omiempty"`
|
|
CertNode CertNode `json:"certNode,omiempty"`
|
|
ShodanNode ShodanNode `json:"shodanNode,omiempty"`
|
|
}
|
|
|
|
// BuildNode builds a node to send to MQ instance.
|
|
func BuildNode(flag string, dataType string, data string) *Node {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
uuid := uuid.New().String()
|
|
return &Node{
|
|
ID: flag + "--" + uuid,
|
|
Type: dataType,
|
|
NData: data,
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// SaveNode saves a node to a file.
|
|
func SaveNode(filename string, node *Node) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeDatas := []Node{}
|
|
|
|
if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
nodeDatas = append(nodeDatas, *node)
|
|
|
|
nodeBytes, err := json.Marshal(nodeDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, nodeBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// Edge defines a relation between two nodes.
|
|
type Edge struct {
|
|
ID string `json:"id,omiempty"`
|
|
NodeOne map[string]interface{} `json:"nodeOne,omiempty"`
|
|
NodeTwo map[string]interface{} `json:"nodeTwo,omiempty"`
|
|
Timestamp string `json:"timestamp,omiempty"`
|
|
Source string `json:"source,omiempty"`
|
|
DType []string `json:"dgraph.type,omiempty"`
|
|
}
|
|
|
|
// BuildEdge build a send from two nodes with a given source type.
|
|
func BuildEdge(source string, nodeOne, nodeTwo map[string]interface{}) *Edge {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
uuid := uuid.New().String()
|
|
return &Edge{
|
|
ID: "edge--" + uuid,
|
|
Source: source,
|
|
NodeOne: nodeOne,
|
|
NodeTwo: nodeTwo,
|
|
Timestamp: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// SaveEdge saves an edge to a file.
|
|
func SaveEdge(edge *Edge) {
|
|
err := utils.FileExists("edges.json")
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
edgeFile, err := ioutil.ReadFile("edges.json")
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
edgeDatas := []Edge{}
|
|
|
|
if err := json.Unmarshal(edgeFile, &edgeDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
edgeDatas = append(edgeDatas, *edge)
|
|
edgeBytes, err := json.Marshal(edgeDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile("edges.json", edgeBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// CertStreamRaw is a wrapper around the stream function to unmarshall the
|
|
// data receive in a Go structure.
|
|
type CertStreamRaw struct {
|
|
ID string `json:"id,omiempty"`
|
|
Type string `json:"type,omiempty"`
|
|
Data CertStreamStruct `json:"data,omiempty"`
|
|
Created string `json:"created,omiempty"`
|
|
Modified string `json:"modified,omiempty"`
|
|
}
|
|
|
|
// CertNode represents our custom struct of data extraction from CertStream.
|
|
type CertNode struct {
|
|
ID string `json:"id,omiempty"`
|
|
Fingerprint string `json:"fingerprint,omiempty"`
|
|
NotBefore string `json:"notBefore,omiempty"`
|
|
NotAfter string `json:"notAfter,omiempty"`
|
|
CN string `json:"cn,omiempty"`
|
|
SourceName string `json:"sourceName,omiempty"`
|
|
SerialNumber string `json:"serialNumber,omiempty"`
|
|
BasicConstraints string `json:"basicConstraints,omiempty"`
|
|
Raw CertStreamRaw `json:"raw,omiempty"`
|
|
Chain []CertNode `json:"chain,omiempty"`
|
|
}
|
|
|
|
// WrapCertStreamData is a wrapper around CertStreamStruct.
|
|
func WrapCertStreamData(data CertStreamStruct) *CertStreamRaw {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
return &CertStreamRaw{
|
|
ID: "certstream--" + uuid.New().String(),
|
|
Type: "certstream_raw",
|
|
Data: data,
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// BuildCertNode builds a custom node based on CertStream.
|
|
func BuildCertNode(rawNode *CertStreamRaw) *CertNode {
|
|
main := &CertNode{
|
|
ID: "certstream--" + uuid.New().String(),
|
|
Fingerprint: rawNode.Data.CSData.LeafCert.Fingerprint,
|
|
NotBefore: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotBefore), 0).Format(time.RFC3339),
|
|
NotAfter: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotAfter), 0).Format(time.RFC3339),
|
|
CN: rawNode.Data.CSData.LeafCert.Subject.CN,
|
|
SourceName: rawNode.Data.CSData.Source.Name,
|
|
BasicConstraints: rawNode.Data.CSData.LeafCert.Extensions.BasicConstrains,
|
|
Raw: *rawNode,
|
|
}
|
|
|
|
var res []CertNode
|
|
if len(rawNode.Data.CSData.Chain) > 0 {
|
|
chain := CertNode{
|
|
ID: "certstream--" + uuid.New().String(),
|
|
Fingerprint: rawNode.Data.CSData.LeafCert.Fingerprint,
|
|
NotBefore: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotBefore), 0).Format(time.RFC3339),
|
|
NotAfter: time.Unix(int64(rawNode.Data.CSData.LeafCert.NotAfter), 0).Format(time.RFC3339),
|
|
CN: rawNode.Data.CSData.LeafCert.Subject.CN,
|
|
SourceName: rawNode.Data.CSData.Source.Name,
|
|
BasicConstraints: rawNode.Data.CSData.LeafCert.Extensions.BasicConstrains,
|
|
}
|
|
res = append(res, chain)
|
|
}
|
|
main.Chain = res
|
|
|
|
return main
|
|
}
|
|
|
|
// SaveCertStreamRaw save the raw CertStream data.
|
|
func SaveCertStreamRaw(filename string, data *CertStreamRaw) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
rawDatas := []CertStreamRaw{}
|
|
|
|
if err := json.Unmarshal(nodeFile, &rawDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
rawDatas = append(rawDatas, *data)
|
|
|
|
rawBytes, err := json.Marshal(rawDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// SaveCertNode saves a CertNode to a json file.
|
|
func SaveCertNode(filename string, node *CertNode) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeDatas := []CertNode{}
|
|
|
|
if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
nodeDatas = append(nodeDatas, *node)
|
|
|
|
nodeBytes, err := json.Marshal(nodeDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, nodeBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// PasteNode is a node from PasteBin.
|
|
type PasteNode struct {
|
|
ID string `json:"id"`
|
|
Type string `json:"type"`
|
|
Data FullPaste `json:"data"`
|
|
Created string `json:"create"`
|
|
Modified string `json:"modified"`
|
|
}
|
|
|
|
// FullPaste wrapes meta and information from Pastebin.
|
|
type FullPaste struct {
|
|
Meta PasteMeta `json:"meta"`
|
|
Full string `json:"full"`
|
|
}
|
|
|
|
// BuildPasteNode builds a node from a FullPaste data.
|
|
func BuildPasteNode(data *FullPaste) *PasteNode {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
return &PasteNode{
|
|
ID: "pastebin--" + uuid.New().String(),
|
|
Type: "pastebin",
|
|
Data: *data,
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// SavePaste saves a object received from PasteBin.
|
|
func SavePaste(filename string, data *PasteNode) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
pasteFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
rawPaste := []PasteNode{}
|
|
|
|
if err := json.Unmarshal(pasteFile, &rawPaste); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
rawPaste = append(rawPaste, *data)
|
|
|
|
rawBytes, err := json.Marshal(rawPaste)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// ShodanNode is node around the shodan.HostData struct.
|
|
type ShodanNode struct {
|
|
ID string `json:"id,omiempty"`
|
|
Type string `json:"type,omiempty"`
|
|
HostData ShodanHostData `json:"hostData,omiempty"`
|
|
Created string `json:"created,omiempty"`
|
|
Modified string `json:"modified,omiempty"`
|
|
}
|
|
|
|
// ShodanHostData is a copy of the structure in the go shodan library. It's a
|
|
// workaround to have more control on the data send.
|
|
type ShodanHostData struct {
|
|
Product string `json:"product,omiempty"`
|
|
Hostnames []string `json:"hostnames,omiempty"`
|
|
Version string `json:"version,omiempty"`
|
|
Title string `json:"title,omiempty"`
|
|
// SSL *HostSSL `json:"ssl"`
|
|
IP string `json:"ip_str,omiempty"`
|
|
OS string `json:"os,omiempty"`
|
|
Organization string `json:"org,omiempty"`
|
|
ISP string `json:"isp,omiempty"`
|
|
CPE []string `json:"cpe,omiempty"`
|
|
// Data string `json:"data,omiempty"`
|
|
ASN string `json:"asn,omiempty"`
|
|
Port int `json:"port,omiempty"`
|
|
HTML string `json:"html,omiempty"`
|
|
Banner string `json:"banner,omiempty"`
|
|
Link string `json:"link,omiempty"`
|
|
Transport string `json:"transport,omiempty"`
|
|
Domains []string `json:"domains,omiempty"`
|
|
Timestamp string `json:"timestamp,omiempty"`
|
|
DeviceType string `json:"devicetype,omiempty"`
|
|
// Location *HostLocation `json:"location"`
|
|
ShodanData map[string]interface{} `json:"_shodan,omiempty"`
|
|
Opts map[string]interface{} `json:"opts,omiempty"`
|
|
}
|
|
|
|
// BuildShodanNode builds a wrapper node around shodan.HostData.
|
|
func BuildShodanNode(data *shodan.HostData) *ShodanNode {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
return &ShodanNode{
|
|
ID: "shodan--" + uuid.New().String(),
|
|
Type: "shodan_stream",
|
|
HostData: ShodanHostData{
|
|
Product: data.Product,
|
|
Hostnames: data.Hostnames,
|
|
Version: data.Version.String(),
|
|
Title: data.Title,
|
|
IP: data.IP.String(),
|
|
OS: data.OS,
|
|
Organization: data.Organization,
|
|
ISP: data.ISP,
|
|
CPE: data.CPE,
|
|
// Data: data.Data,
|
|
ASN: data.ASN,
|
|
Port: data.Port,
|
|
HTML: data.HTML,
|
|
Banner: data.Banner,
|
|
Link: data.Link,
|
|
Transport: data.Transport,
|
|
Domains: data.Domains,
|
|
Timestamp: data.Timestamp,
|
|
DeviceType: data.DeviceType,
|
|
ShodanData: data.ShodanData,
|
|
Opts: data.Opts,
|
|
},
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// SaveShodanNode saves the raw nodes from Shodan.
|
|
func SaveShodanNode(filename string, data *ShodanNode) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
rawDatas := []ShodanNode{}
|
|
|
|
if err := json.Unmarshal(nodeFile, &rawDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
rawDatas = append(rawDatas, *data)
|
|
|
|
rawBytes, err := json.Marshal(rawDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
|
|
// BalboaNode represents a return from Balboa.
|
|
type BalboaNode struct {
|
|
ID string `json:"id"`
|
|
Type string `json:"type"`
|
|
Data []balboa.Entries `json:"data"`
|
|
Created string `json:"created"`
|
|
Modified string `json:"modified"`
|
|
}
|
|
|
|
// BuildBalboaNode builds a node coming from Balboa resolution.
|
|
func BuildBalboaNode(data []balboa.Entries) *BalboaNode {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
return &BalboaNode{
|
|
ID: "balboa--" + uuid.New().String(),
|
|
Type: "balboa",
|
|
Data: data,
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|
|
|
|
// SaveBalboaNode saves a Balboa node.
|
|
func SaveBalboaNode(filename string, data *BalboaNode) {
|
|
err := utils.FileExists(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
rawDatas := []BalboaNode{}
|
|
|
|
if err := json.Unmarshal(nodeFile, &rawDatas); err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
rawDatas = append(rawDatas, *data)
|
|
|
|
rawBytes, err := json.Marshal(rawDatas)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
|
|
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|