2020-01-14 09:43:12 +01:00
|
|
|
package main
|
|
|
|
|
2020-01-14 14:49:14 +01:00
|
|
|
import (
|
2020-02-06 17:01:37 +01:00
|
|
|
"context"
|
2020-01-15 16:46:13 +01:00
|
|
|
"fmt"
|
2020-01-30 17:43:10 +01:00
|
|
|
"os"
|
|
|
|
"sync"
|
|
|
|
"time"
|
2020-01-15 16:46:13 +01:00
|
|
|
|
2020-01-14 14:49:14 +01:00
|
|
|
"github.com/CaliDog/certstream-go"
|
2020-02-06 17:01:37 +01:00
|
|
|
"github.com/ns3777k/go-shodan/v4/shodan"
|
2020-02-07 17:39:33 +01:00
|
|
|
"github.com/segmentio/kafka-go"
|
2020-01-16 15:56:57 +01:00
|
|
|
"github.com/sirupsen/logrus"
|
2020-02-07 17:39:33 +01:00
|
|
|
"gitlab.dcso.lolcat/LABS/styx/broker"
|
2020-01-14 16:31:57 +01:00
|
|
|
"gitlab.dcso.lolcat/LABS/styx/models"
|
2020-01-14 14:49:14 +01:00
|
|
|
)
|
|
|
|
|
2020-02-07 14:52:29 +01:00
|
|
|
func init() {
|
|
|
|
// Setting up logging.
|
|
|
|
logrus.SetFormatter(&logrus.TextFormatter{
|
|
|
|
FullTimestamp: true,
|
|
|
|
})
|
|
|
|
logrus.SetReportCaller(true)
|
|
|
|
}
|
|
|
|
|
2020-01-14 09:43:12 +01:00
|
|
|
func main() {
|
2020-02-06 17:01:37 +01:00
|
|
|
fmt.Println("Starting to get data from the Internet...")
|
|
|
|
|
2020-01-14 14:49:14 +01:00
|
|
|
// The false flag specifies that we want heartbeat messages.
|
|
|
|
stream, errStream := certstream.CertStreamEventStream(false)
|
2020-02-07 17:39:33 +01:00
|
|
|
Conn, err := broker.SetUpKafkaConnecter()
|
|
|
|
if err != nil {
|
|
|
|
panic(err)
|
|
|
|
}
|
2020-01-30 17:43:10 +01:00
|
|
|
|
|
|
|
// stop channel
|
|
|
|
stopChan := make(chan os.Signal)
|
|
|
|
var wg sync.WaitGroup
|
2020-02-06 17:01:37 +01:00
|
|
|
wg.Add(3)
|
2020-01-30 17:43:10 +01:00
|
|
|
|
2020-02-07 17:39:33 +01:00
|
|
|
go broker.ReadEventFromKafka()
|
2020-02-04 10:35:18 +01:00
|
|
|
|
2020-01-29 11:05:05 +01:00
|
|
|
// certstream
|
2020-01-30 17:43:10 +01:00
|
|
|
go func() {
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
case jq := <-stream:
|
|
|
|
if data, err := models.ExtractCertFromStream(jq); err == nil {
|
|
|
|
|
|
|
|
rawNode := models.WrapCertStreamData(*data)
|
2020-02-07 15:50:21 +01:00
|
|
|
models.SaveCertStreamRaw("raw_certstream.json", rawNode)
|
2020-01-28 23:52:24 +01:00
|
|
|
|
2020-02-04 10:35:18 +01:00
|
|
|
certNode := models.BuildCertNode(rawNode)
|
2020-02-05 14:46:52 +01:00
|
|
|
models.SaveCertNode("cert_nodes.json", certNode)
|
2020-02-07 15:27:14 +01:00
|
|
|
mainNode := models.BuildNode("node", "certstream", certNode.ID)
|
2020-02-04 10:35:18 +01:00
|
|
|
models.SaveNode("nodes.json", mainNode)
|
2020-02-07 17:39:33 +01:00
|
|
|
edge := models.BuildEdge("certstream", rawNode.ID, mainNode.ID)
|
|
|
|
models.SaveEdge(edge)
|
|
|
|
edge = models.BuildEdge("certstream", mainNode.ID, certNode.ID)
|
|
|
|
models.SaveEdge(edge)
|
2020-01-30 17:43:10 +01:00
|
|
|
allDomains := data.Data.LeafCert.AllDomains
|
2020-02-07 17:39:33 +01:00
|
|
|
saveSingleValues(Conn, "certstream", "domain", certNode.ID, allDomains)
|
2020-01-28 23:52:24 +01:00
|
|
|
}
|
2020-01-14 16:31:57 +01:00
|
|
|
|
2020-01-30 17:43:10 +01:00
|
|
|
case err := <-errStream:
|
|
|
|
logrus.Error(err)
|
|
|
|
case <-stopChan:
|
|
|
|
wg.Done()
|
|
|
|
return
|
|
|
|
}
|
2020-01-14 14:49:14 +01:00
|
|
|
}
|
2020-02-04 10:35:18 +01:00
|
|
|
|
2020-01-30 17:43:10 +01:00
|
|
|
}()
|
2020-01-29 11:05:05 +01:00
|
|
|
|
2020-02-07 17:39:33 +01:00
|
|
|
// pastebin
|
2020-01-30 17:43:10 +01:00
|
|
|
go func() {
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
default:
|
|
|
|
pastes, err := models.QueryPastes()
|
|
|
|
if err != nil {
|
|
|
|
logrus.Panic(err)
|
|
|
|
}
|
|
|
|
for _, p := range pastes {
|
|
|
|
paste, err := models.FetchPaste(p)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error("cannot fetch paste", err)
|
|
|
|
}
|
|
|
|
fp := models.FullPaste{
|
|
|
|
Meta: p,
|
|
|
|
Full: paste,
|
|
|
|
}
|
|
|
|
res := models.BuildPasteNode(&fp)
|
|
|
|
models.SavePaste("paste_formatted.json", res)
|
|
|
|
time.Sleep(1 * time.Second)
|
|
|
|
|
|
|
|
}
|
|
|
|
time.Sleep(3 * time.Second)
|
|
|
|
case <-stopChan:
|
|
|
|
wg.Done()
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
2020-02-05 14:46:52 +01:00
|
|
|
|
|
|
|
// shodan
|
2020-02-06 17:01:37 +01:00
|
|
|
client := shodan.NewEnvClient(nil)
|
|
|
|
ch := make(chan *shodan.HostData)
|
2020-02-07 17:39:33 +01:00
|
|
|
err = client.GetBannersByPorts(context.Background(), []int{80, 443, 8443, 53}, ch)
|
2020-02-06 17:01:37 +01:00
|
|
|
if err != nil {
|
2020-02-07 15:27:14 +01:00
|
|
|
logrus.Panic(err)
|
2020-02-06 17:01:37 +01:00
|
|
|
}
|
2020-02-05 14:46:52 +01:00
|
|
|
|
2020-02-06 17:01:37 +01:00
|
|
|
go func() {
|
|
|
|
for {
|
|
|
|
select {
|
|
|
|
default:
|
|
|
|
banner, ok := <-ch
|
|
|
|
if !ok {
|
|
|
|
logrus.Error("channel is closed")
|
|
|
|
break
|
|
|
|
}
|
|
|
|
|
|
|
|
shodanNode := models.BuildShodanNode(banner)
|
2020-02-07 15:27:14 +01:00
|
|
|
hostnames := shodanNode.Data.Hostnames
|
|
|
|
if len(hostnames) != 0 {
|
2020-02-07 17:39:33 +01:00
|
|
|
saveSingleValues(Conn, "shodan_stream", "hostname", shodanNode.ID, hostnames)
|
2020-02-07 15:27:14 +01:00
|
|
|
}
|
|
|
|
domains := shodanNode.Data.Domains
|
|
|
|
if len(domains) != 0 {
|
2020-02-07 17:39:33 +01:00
|
|
|
saveSingleValues(Conn, "shodan_stream", "domain", shodanNode.ID, domains)
|
2020-02-07 15:27:14 +01:00
|
|
|
}
|
2020-02-06 17:01:37 +01:00
|
|
|
models.SaveShodanNode("shodan_raw.json", shodanNode)
|
|
|
|
node := models.BuildNode("shodan", "shodan_stream", shodanNode.ID)
|
|
|
|
models.SaveNode("nodes.json", node)
|
|
|
|
edge := models.BuildEdge("shodan", shodanNode.ID, node.ID)
|
|
|
|
models.SaveEdge(edge)
|
|
|
|
case <-stopChan:
|
|
|
|
wg.Done()
|
|
|
|
return
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}()
|
|
|
|
wg.Wait()
|
2020-02-05 14:46:52 +01:00
|
|
|
|
2020-01-14 09:43:12 +01:00
|
|
|
}
|
2020-02-07 15:27:14 +01:00
|
|
|
|
|
|
|
// helpers
|
2020-02-07 17:39:33 +01:00
|
|
|
func saveSingleValues(brokerConn *kafka.Conn, source string, datatype string, originNodeID string, values []string) {
|
2020-02-07 15:27:14 +01:00
|
|
|
for _, value := range values {
|
|
|
|
domainNode := models.BuildNode(source, datatype, value)
|
|
|
|
models.SaveNode("nodes.json", domainNode)
|
2020-02-07 17:39:33 +01:00
|
|
|
if domainNode.Type == "domain" || domainNode.Type == "hostname" {
|
|
|
|
broker.SendEventToKafka(brokerConn, *domainNode)
|
|
|
|
}
|
2020-02-07 15:27:14 +01:00
|
|
|
edge := models.BuildEdge(source, originNodeID, domainNode.ID)
|
|
|
|
models.SaveEdge(edge)
|
|
|
|
}
|
|
|
|
}
|