saving
This commit is contained in:
parent
cd43194873
commit
b72e82071d
1
go.sum
1
go.sum
|
@ -226,6 +226,7 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
|
||||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||||
|
github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
|
||||||
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
|
||||||
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
|
github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
|
||||||
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
|
github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
|
||||||
|
|
|
@ -1,8 +1,7 @@
|
||||||
package main
|
package graph
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"fmt"
|
|
||||||
|
|
||||||
"github.com/dgraph-io/dgo"
|
"github.com/dgraph-io/dgo"
|
||||||
"github.com/dgraph-io/dgo/protos/api"
|
"github.com/dgraph-io/dgo/protos/api"
|
||||||
|
@ -18,7 +17,6 @@ func ConnectToDgraph() error {
|
||||||
|
|
||||||
dgraphClient := dgo.NewDgraphClient(api.NewDgraphClient(conn))
|
dgraphClient := dgo.NewDgraphClient(api.NewDgraphClient(conn))
|
||||||
|
|
||||||
fmt.Println("pouet")
|
|
||||||
err = setupDgraphSchema(dgraphClient)
|
err = setupDgraphSchema(dgraphClient)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return err
|
return err
|
104
main.go
104
main.go
|
@ -1,19 +1,12 @@
|
||||||
package main
|
package main
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"fmt"
|
|
||||||
"os"
|
"os"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
|
||||||
|
|
||||||
"github.com/ns3777k/go-shodan/v4/shodan"
|
|
||||||
"github.com/segmentio/kafka-go"
|
|
||||||
"github.com/sirupsen/logrus"
|
"github.com/sirupsen/logrus"
|
||||||
"github.com/spf13/viper"
|
"github.com/spf13/viper"
|
||||||
"gitlab.dcso.lolcat/LABS/styx/broker"
|
"gitlab.dcso.lolcat/LABS/styx/graph"
|
||||||
"gitlab.dcso.lolcat/LABS/styx/elasticsearch"
|
|
||||||
"gitlab.dcso.lolcat/LABS/styx/filters"
|
|
||||||
"gitlab.dcso.lolcat/LABS/styx/models"
|
|
||||||
"gitlab.dcso.lolcat/LABS/styx/plugins"
|
"gitlab.dcso.lolcat/LABS/styx/plugins"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
@ -35,7 +28,7 @@ func main() {
|
||||||
}
|
}
|
||||||
|
|
||||||
os.Setenv("SHODAN_KEY", viper.GetString("shodan.key"))
|
os.Setenv("SHODAN_KEY", viper.GetString("shodan.key"))
|
||||||
fmt.Println("Starting to get data from the Internet...")
|
logrus.Info("Starting to get data from the Internet...")
|
||||||
|
|
||||||
// The false flag specifies that we want heartbeat messages.
|
// The false flag specifies that we want heartbeat messages.
|
||||||
// conn, err := broker.SetUpKafkaConnecter()
|
// conn, err := broker.SetUpKafkaConnecter()
|
||||||
|
@ -43,7 +36,7 @@ func main() {
|
||||||
// panic(err)
|
// panic(err)
|
||||||
// }
|
// }
|
||||||
|
|
||||||
err = ConnectToDgraph()
|
err = graph.ConnectToDgraph()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
logrus.WithField("err", err).Error("error initialising the graph database")
|
logrus.WithField("err", err).Error("error initialising the graph database")
|
||||||
}
|
}
|
||||||
|
@ -57,14 +50,14 @@ func main() {
|
||||||
// certstream
|
// certstream
|
||||||
c := plugins.CertStreamPlugin{}
|
c := plugins.CertStreamPlugin{}
|
||||||
if ok := c.Initialize(); !ok {
|
if ok := c.Initialize(); !ok {
|
||||||
logrus.Info("certstream module not activated")
|
logrus.Info("certstream plugin not activated")
|
||||||
}
|
}
|
||||||
c.Run(&wg)
|
c.Run(&wg)
|
||||||
|
|
||||||
// pastebin
|
// pastebin
|
||||||
p := plugins.PastebinPlugin{}
|
p := plugins.PastebinPlugin{}
|
||||||
if ok := p.Initialize(); !ok {
|
if ok := p.Initialize(); !ok {
|
||||||
logrus.Info("pastebin module not activated")
|
logrus.Info("pastebin plugin not activated")
|
||||||
}
|
}
|
||||||
p.Run(&wg)
|
p.Run(&wg)
|
||||||
|
|
||||||
|
@ -77,96 +70,13 @@ func main() {
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
<-stopChan
|
<-stopChan
|
||||||
logrus.Println("Shutting down...")
|
logrus.Info("Shutting down...")
|
||||||
c.Stop(&wg)
|
c.Stop(&wg)
|
||||||
p.Stop(&wg)
|
p.Stop(&wg)
|
||||||
s.Stop(&wg)
|
s.Stop(&wg)
|
||||||
}()
|
}()
|
||||||
|
|
||||||
wg.Wait()
|
wg.Wait()
|
||||||
fmt.Println("done")
|
logrus.Info("done")
|
||||||
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// routines
|
|
||||||
|
|
||||||
func pastebinRoutine(stopChan chan os.Signal, wg *sync.WaitGroup) {
|
|
||||||
fmt.Println("pastebin is activated")
|
|
||||||
|
|
||||||
elastic := viper.GetBool("elasticsearch.activated")
|
|
||||||
var e *elasticsearch.ElasticStorageModule
|
|
||||||
if elastic {
|
|
||||||
fmt.Println("elasticsearch is activated")
|
|
||||||
e = &elasticsearch.ElasticStorageModule{
|
|
||||||
ElasticURL: viper.GetString("elasticsearch.url"),
|
|
||||||
DailyIndexes: true,
|
|
||||||
UseIndex: "pastebin",
|
|
||||||
LastChk: time.Now(),
|
|
||||||
}
|
|
||||||
err := e.Initialize()
|
|
||||||
if err != nil {
|
|
||||||
panic(err)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
func shodanRoutine(client *shodan.Client, shodanChan chan *shodan.HostData, conn *kafka.Conn, stopChan chan os.Signal, wg *sync.WaitGroup) {
|
|
||||||
fmt.Println("shodan is activated")
|
|
||||||
for {
|
|
||||||
select {
|
|
||||||
default:
|
|
||||||
banner, ok := <-shodanChan
|
|
||||||
if !ok {
|
|
||||||
logrus.Error("channel is closed")
|
|
||||||
break
|
|
||||||
}
|
|
||||||
|
|
||||||
shodanNode := models.BuildShodanNode(banner)
|
|
||||||
// first filter poc
|
|
||||||
if shodanNode.Data.HTML != "" {
|
|
||||||
if !filters.RunIPFilters(shodanNode.Data.IP) {
|
|
||||||
hostnames := shodanNode.Data.Hostnames
|
|
||||||
var hostNotInFilters, domainNotInFilters bool
|
|
||||||
if len(hostnames) != 0 {
|
|
||||||
for _, hostname := range hostnames {
|
|
||||||
hostNotInFilters = filters.RunDomainFilters(hostname)
|
|
||||||
if hostNotInFilters {
|
|
||||||
saveSingleValues(conn, "shodan_stream", "hostname", shodanNode.ID, hostname)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
domains := shodanNode.Data.Domains
|
|
||||||
if len(domains) != 0 {
|
|
||||||
for _, domain := range domains {
|
|
||||||
domainNotInFilters = filters.RunDomainFilters(domain)
|
|
||||||
saveSingleValues(conn, "shodan_stream", "domain", shodanNode.ID, domain)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if domainNotInFilters && hostNotInFilters {
|
|
||||||
models.SaveShodanNode("raw_shodan.json", shodanNode)
|
|
||||||
node := models.BuildNode("shodan", "shodan_stream", shodanNode.ID)
|
|
||||||
models.SaveNode("nodes.json", node)
|
|
||||||
edge := models.BuildEdge("shodan", shodanNode.ID, node.ID)
|
|
||||||
models.SaveEdge(edge)
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
fmt.Println("is akamai", shodanNode.Data.IP)
|
|
||||||
}
|
|
||||||
}
|
|
||||||
case <-stopChan:
|
|
||||||
wg.Done()
|
|
||||||
return
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// helpers
|
|
||||||
func saveSingleValues(brokerConn *kafka.Conn, source string, datatype string, originNodeID string, values string) {
|
|
||||||
domainNode := models.BuildNode(source, datatype, values)
|
|
||||||
models.SaveNode("nodes.json", domainNode)
|
|
||||||
if domainNode.Type == "domain" || domainNode.Type == "hostname" {
|
|
||||||
broker.SendEventToKafka(brokerConn, *domainNode)
|
|
||||||
}
|
|
||||||
edge := models.BuildEdge(source, originNodeID, domainNode.ID)
|
|
||||||
models.SaveEdge(edge)
|
|
||||||
}
|
|
||||||
|
|
|
@ -25,11 +25,12 @@ Structure of this file:
|
||||||
// Styx terminology
|
// Styx terminology
|
||||||
// (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx)
|
// (https://docs.google.com/document/d/1dIrh1Lp3KAjEMm8o2VzAmuV0Peu-jt9aAh1IHrjAroM/pub#h.xzbicbtscatx)
|
||||||
type Node struct {
|
type Node struct {
|
||||||
ID string `json:"id"`
|
Uid string `json:"uid,omiempty"`
|
||||||
Type string `json:"type"`
|
ID string `json:"id,omiempty"`
|
||||||
Data string `json:"data"`
|
Type string `json:"type,omiempty"`
|
||||||
Created string `json:"created"`
|
Data string `json:"data,omiempty"`
|
||||||
Modified string `json:"modified"`
|
Created string `json:"created,omiempty"`
|
||||||
|
Modified string `json:"modified,omiempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// BuildNode builds a node to send to MQ instance.
|
// BuildNode builds a node to send to MQ instance.
|
||||||
|
@ -37,6 +38,7 @@ func BuildNode(flag string, dataType string, data string) *Node {
|
||||||
t := time.Now()
|
t := time.Now()
|
||||||
rfc3339time := t.Format(time.RFC3339)
|
rfc3339time := t.Format(time.RFC3339)
|
||||||
return &Node{
|
return &Node{
|
||||||
|
Uid: "_:" + flag + "--" + uuid.New().String(),
|
||||||
ID: flag + "--" + uuid.New().String(),
|
ID: flag + "--" + uuid.New().String(),
|
||||||
Type: dataType,
|
Type: dataType,
|
||||||
Data: data,
|
Data: data,
|
||||||
|
@ -76,11 +78,12 @@ func SaveNode(filename string, node *Node) {
|
||||||
|
|
||||||
// Edge defines a relation between two nodes.
|
// Edge defines a relation between two nodes.
|
||||||
type Edge struct {
|
type Edge struct {
|
||||||
ID string `json:"id"`
|
Uid string `json:"uid,omiempty"`
|
||||||
NodeOneID string `json:"nodeOneID"`
|
ID string `json:"id,omiempty"`
|
||||||
NodeTwoID string `json:"nodeTwoID"`
|
NodeOneID string `json:"nodeOneID,omiempty"`
|
||||||
Timestamp string `json:"timestamp"`
|
NodeTwoID string `json:"nodeTwoID,omiempty"`
|
||||||
Source string `json:"source"`
|
Timestamp string `json:"timestamp,omiempty"`
|
||||||
|
Source string `json:"source,omiempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// BuildEdge build a send from two nodes with a given source type.
|
// BuildEdge build a send from two nodes with a given source type.
|
||||||
|
@ -88,6 +91,7 @@ func BuildEdge(source string, nodeOneUUID string, nodeTwoUUID string) *Edge {
|
||||||
t := time.Now()
|
t := time.Now()
|
||||||
rfc3339time := t.Format(time.RFC3339)
|
rfc3339time := t.Format(time.RFC3339)
|
||||||
return &Edge{
|
return &Edge{
|
||||||
|
Uid: "_:" + "edge--" + uuid.New().String(),
|
||||||
ID: "edge--" + uuid.New().String(),
|
ID: "edge--" + uuid.New().String(),
|
||||||
Source: source,
|
Source: source,
|
||||||
NodeOneID: nodeOneUUID,
|
NodeOneID: nodeOneUUID,
|
||||||
|
@ -137,16 +141,17 @@ type CertStreamRaw struct {
|
||||||
|
|
||||||
// CertNode represents our custom struct of data extraction from CertStream.
|
// CertNode represents our custom struct of data extraction from CertStream.
|
||||||
type CertNode struct {
|
type CertNode struct {
|
||||||
ID string `json:"id"`
|
Uid string `json:"uid,omiempty"`
|
||||||
Fingerprint string `json:"fingerprint"`
|
ID string `json:"id,omiempty"`
|
||||||
NotBefore string `json:"notBefore"`
|
Fingerprint string `json:"fingerprint,omiempty"`
|
||||||
NotAfter string `json:"notAfter"`
|
NotBefore string `json:"notBefore,omiempty"`
|
||||||
CN string `json:"cn"`
|
NotAfter string `json:"notAfter,omiempty"`
|
||||||
SourceName string `json:"sourceName"`
|
CN string `json:"cn,omiempty"`
|
||||||
SerialNumber string `json:"serialNumber"`
|
SourceName string `json:"sourceName,omiempty"`
|
||||||
BasicConstraints string `json:"basicConstraints"`
|
SerialNumber string `json:"serialNumber,omiempty"`
|
||||||
RawUUID string `json:"rawUUID"`
|
BasicConstraints string `json:"basicConstraints,omiempty"`
|
||||||
Chain []CertNode `json:"chainedTo"`
|
RawUUID string `json:"rawUUID,omiempty"`
|
||||||
|
Chain []CertNode `json:"chainedTo,omiempty"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// WrapCertStreamData is a wrapper around CertStreamStruct.
|
// WrapCertStreamData is a wrapper around CertStreamStruct.
|
||||||
|
|
Loading…
Reference in a new issue