2020-01-14 14:49:14 +01:00
|
|
|
package models
|
|
|
|
|
|
|
|
import (
|
2020-01-28 16:00:56 +01:00
|
|
|
"encoding/json"
|
|
|
|
"io/ioutil"
|
2020-01-22 15:01:07 +01:00
|
|
|
"time"
|
|
|
|
|
2020-01-14 14:49:14 +01:00
|
|
|
"github.com/google/uuid"
|
2020-02-06 17:01:37 +01:00
|
|
|
"github.com/ns3777k/go-shodan/v4/shodan"
|
2020-01-28 16:00:56 +01:00
|
|
|
"github.com/sirupsen/logrus"
|
|
|
|
"gitlab.dcso.lolcat/LABS/styx/utils"
|
2020-01-14 14:49:14 +01:00
|
|
|
)
|
|
|
|
|
2020-01-21 16:50:50 +01:00
|
|
|
// BuildNode builds a node to send to MQ instance.
|
2020-01-29 10:03:52 +01:00
|
|
|
func BuildNode(flag string, dataType string, data string) *Node {
|
2020-01-26 17:26:03 +01:00
|
|
|
t := time.Now()
|
|
|
|
rfc3339time := t.Format(time.RFC3339)
|
2020-01-16 15:56:57 +01:00
|
|
|
return &Node{
|
2020-01-22 15:01:07 +01:00
|
|
|
ID: flag + "--" + uuid.New().String(),
|
2020-01-29 10:03:52 +01:00
|
|
|
Type: dataType,
|
2020-01-22 15:01:07 +01:00
|
|
|
Data: data,
|
2020-01-26 17:26:03 +01:00
|
|
|
Created: rfc3339time,
|
|
|
|
Modified: rfc3339time,
|
2020-01-28 16:00:56 +01:00
|
|
|
}
|
2020-01-29 12:47:01 +01:00
|
|
|
}
|
2020-01-28 16:00:56 +01:00
|
|
|
|
2020-02-04 10:22:48 +01:00
|
|
|
// BuildCertNode builds a custom node based on CertStream.
|
|
|
|
func BuildCertNode(rawNode *CertStreamWrapper) *CertNode {
|
2020-02-05 15:57:30 +01:00
|
|
|
main := &CertNode{
|
2020-02-04 10:58:56 +01:00
|
|
|
ID: "certstream--" + uuid.New().String(),
|
2020-02-04 10:22:48 +01:00
|
|
|
Fingerprint: rawNode.Data.Data.LeafCert.Fingerprint,
|
|
|
|
NotBefore: time.Unix(int64(rawNode.Data.Data.LeafCert.NotBefore), 0).Format(time.RFC3339),
|
|
|
|
NotAfter: time.Unix(int64(rawNode.Data.Data.LeafCert.NotAfter), 0).Format(time.RFC3339),
|
|
|
|
CN: rawNode.Data.Data.LeafCert.Subject.CN,
|
|
|
|
SourceName: rawNode.Data.Data.Source.Name,
|
|
|
|
BasicConstraints: rawNode.Data.Data.LeafCert.Extensions.BasicConstrains,
|
|
|
|
RawUUID: rawNode.ID,
|
|
|
|
}
|
2020-02-05 15:57:30 +01:00
|
|
|
|
|
|
|
var res []CertNode
|
|
|
|
if len(rawNode.Data.Data.Chain) > 0 {
|
|
|
|
chain := CertNode{
|
|
|
|
ID: "certstream--" + uuid.New().String(),
|
|
|
|
Fingerprint: rawNode.Data.Data.LeafCert.Fingerprint,
|
|
|
|
NotBefore: time.Unix(int64(rawNode.Data.Data.LeafCert.NotBefore), 0).Format(time.RFC3339),
|
|
|
|
NotAfter: time.Unix(int64(rawNode.Data.Data.LeafCert.NotAfter), 0).Format(time.RFC3339),
|
|
|
|
CN: rawNode.Data.Data.LeafCert.Subject.CN,
|
|
|
|
SourceName: rawNode.Data.Data.Source.Name,
|
|
|
|
BasicConstraints: rawNode.Data.Data.LeafCert.Extensions.BasicConstrains,
|
|
|
|
RawUUID: rawNode.ID,
|
|
|
|
}
|
|
|
|
res = append(res, chain)
|
|
|
|
}
|
|
|
|
main.Chain = res
|
|
|
|
|
|
|
|
return main
|
2020-02-04 10:22:48 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// CertNode represents our custom struct of data extraction from CertStream.
|
|
|
|
type CertNode struct {
|
2020-02-05 15:57:30 +01:00
|
|
|
ID string `json:"id"`
|
|
|
|
Fingerprint string `json:"fingerprint"`
|
|
|
|
NotBefore string `json:"notBefore"`
|
|
|
|
NotAfter string `json:"notAfter"`
|
|
|
|
CN string `json:"cn"`
|
|
|
|
SourceName string `json:"sourceName"`
|
|
|
|
SerialNumber string `json:"serialNumber"`
|
|
|
|
BasicConstraints string `json:"basicConstraints"`
|
|
|
|
RawUUID string `json:"rawUUID"`
|
|
|
|
Chain []CertNode `json:"chainedTo"`
|
2020-02-04 10:22:48 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// WrapCertStreamData is a wrapper around CertStreamStruct.
|
2020-01-29 12:47:01 +01:00
|
|
|
func WrapCertStreamData(data CertStreamStruct) *CertStreamWrapper {
|
|
|
|
t := time.Now()
|
|
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
|
|
return &CertStreamWrapper{
|
|
|
|
ID: "certstream--" + uuid.New().String(),
|
|
|
|
Type: "certstream_raw",
|
|
|
|
Data: data,
|
|
|
|
Created: rfc3339time,
|
|
|
|
Modified: rfc3339time,
|
|
|
|
}
|
2020-01-28 16:00:56 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// BuildEdge build a send from two nodes with a given source type.
|
2020-01-28 23:52:24 +01:00
|
|
|
func BuildEdge(source string, nodeOneUUID string, nodeTwoUUID string) *Edge {
|
2020-01-28 16:00:56 +01:00
|
|
|
t := time.Now()
|
|
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
|
|
return &Edge{
|
|
|
|
ID: uuid.New().String(),
|
|
|
|
Source: source,
|
|
|
|
NodeOneID: nodeOneUUID,
|
|
|
|
NodeTwoID: nodeTwoUUID,
|
|
|
|
Timestamp: rfc3339time,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-02-04 10:22:48 +01:00
|
|
|
// SavePaste saves a object received from PasteBin.
|
2020-01-30 17:43:10 +01:00
|
|
|
func SavePaste(filename string, data *PasteWrapper) {
|
|
|
|
err := utils.FileExists(filename)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
pasteFile, err := ioutil.ReadFile(filename)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
rawPaste := []PasteWrapper{}
|
|
|
|
|
|
|
|
if err := json.Unmarshal(pasteFile, &rawPaste); err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
rawPaste = append(rawPaste, *data)
|
|
|
|
|
|
|
|
rawBytes, err := json.Marshal(rawPaste)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-02-04 10:22:48 +01:00
|
|
|
// SaveRaw save the raw CertStream data.
|
2020-01-29 12:47:01 +01:00
|
|
|
func SaveRaw(filename string, data *CertStreamWrapper) {
|
|
|
|
err := utils.FileExists(filename)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
rawDatas := []CertStreamWrapper{}
|
|
|
|
|
|
|
|
if err := json.Unmarshal(nodeFile, &rawDatas); err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
rawDatas = append(rawDatas, *data)
|
|
|
|
|
|
|
|
rawBytes, err := json.Marshal(rawDatas)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-02-04 10:22:48 +01:00
|
|
|
// SaveCertNode saves a CertNode to a json file.
|
|
|
|
func SaveCertNode(filename string, node *CertNode) {
|
|
|
|
err := utils.FileExists(filename)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
nodeDatas := []CertNode{}
|
|
|
|
|
|
|
|
if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
nodeDatas = append(nodeDatas, *node)
|
|
|
|
|
|
|
|
nodeBytes, err := json.Marshal(nodeDatas)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = ioutil.WriteFile(filename, nodeBytes, 0644)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-01-29 10:03:52 +01:00
|
|
|
// SaveNode saves a node to a file.
|
2020-01-29 12:47:01 +01:00
|
|
|
func SaveNode(filename string, node *Node) {
|
|
|
|
err := utils.FileExists(filename)
|
2020-01-28 16:00:56 +01:00
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
2020-01-29 12:47:01 +01:00
|
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
2020-01-28 16:00:56 +01:00
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
nodeDatas := []Node{}
|
|
|
|
|
|
|
|
if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
nodeDatas = append(nodeDatas, *node)
|
|
|
|
|
|
|
|
nodeBytes, err := json.Marshal(nodeDatas)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
|
2020-01-29 12:47:01 +01:00
|
|
|
err = ioutil.WriteFile(filename, nodeBytes, 0644)
|
2020-01-28 16:00:56 +01:00
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
}
|
2020-01-16 15:56:57 +01:00
|
|
|
|
2020-01-29 10:03:52 +01:00
|
|
|
// SaveEdge saves an edge to a file.
|
2020-01-28 16:00:56 +01:00
|
|
|
func SaveEdge(edge *Edge) {
|
|
|
|
err := utils.FileExists("edges.json")
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
edgeFile, err := ioutil.ReadFile("edges.json")
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
edgeDatas := []Edge{}
|
|
|
|
|
|
|
|
if err := json.Unmarshal(edgeFile, &edgeDatas); err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
edgeDatas = append(edgeDatas, *edge)
|
|
|
|
edgeBytes, err := json.Marshal(edgeDatas)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = ioutil.WriteFile("edges.json", edgeBytes, 0644)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
2020-01-16 15:56:57 +01:00
|
|
|
}
|
2020-02-06 17:01:37 +01:00
|
|
|
|
|
|
|
// ShodanNode is node around the shodan.HostData struct.
|
|
|
|
type ShodanNode struct {
|
|
|
|
ID string `json:"id"`
|
|
|
|
Type string `json:"type"`
|
|
|
|
Data *shodan.HostData `json:"data"`
|
|
|
|
Created string `json:"created"`
|
|
|
|
Modified string `json:"modified"`
|
|
|
|
}
|
|
|
|
|
|
|
|
// BuildShodanNode builds a wrapper node around shodan.HostData.
|
|
|
|
func BuildShodanNode(data *shodan.HostData) *ShodanNode {
|
|
|
|
t := time.Now()
|
|
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
|
|
return &ShodanNode{
|
|
|
|
ID: "shodan--" + uuid.New().String(),
|
|
|
|
Type: "shodan_stream",
|
|
|
|
Data: data,
|
|
|
|
Created: rfc3339time,
|
|
|
|
Modified: rfc3339time,
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
// SaveShodanNode saves the raw nodes from Shodan.
|
|
|
|
func SaveShodanNode(filename string, data *ShodanNode) {
|
|
|
|
err := utils.FileExists(filename)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
nodeFile, err := ioutil.ReadFile(filename)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
rawDatas := []ShodanNode{}
|
|
|
|
|
|
|
|
if err := json.Unmarshal(nodeFile, &rawDatas); err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
rawDatas = append(rawDatas, *data)
|
|
|
|
|
|
|
|
rawBytes, err := json.Marshal(rawDatas)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
}
|