138 lines
3.5 KiB
Go
138 lines
3.5 KiB
Go
package plugins
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"sync"
|
|
|
|
"github.com/dgraph-io/dgo/v2"
|
|
"github.com/dgraph-io/dgo/v2/protos/api"
|
|
"github.com/ns3777k/go-shodan/v4/shodan"
|
|
"github.com/sirupsen/logrus"
|
|
"github.com/spf13/viper"
|
|
"gitlab.dcso.lolcat/LABS/styx/filters"
|
|
"gitlab.dcso.lolcat/LABS/styx/models"
|
|
)
|
|
|
|
// ShodanPlugin defines the general ShodanPlugin structure.
|
|
type ShodanPlugin struct {
|
|
Client *shodan.Client
|
|
ShodanChan chan *shodan.HostData
|
|
StopChan chan bool
|
|
StoppedChan chan bool
|
|
Running bool
|
|
}
|
|
|
|
// Initialize initialises the certstream configuration.
|
|
func (s *ShodanPlugin) Initialize() bool {
|
|
if !viper.GetBool("shodan.activated") {
|
|
return false
|
|
}
|
|
logrus.Info("shodan plugin is activated")
|
|
s.ShodanChan = make(chan *shodan.HostData)
|
|
return true
|
|
}
|
|
|
|
// Run runs the Shodan plugin.
|
|
func (s *ShodanPlugin) Run(wg *sync.WaitGroup, dgraphClient *dgo.Dgraph) {
|
|
if !s.Running {
|
|
s.StoppedChan = make(chan bool)
|
|
wg.Add(1)
|
|
go s.doRun(dgraphClient)
|
|
s.Running = true
|
|
}
|
|
}
|
|
|
|
// Stop stops the Shodan plugin.
|
|
func (s *ShodanPlugin) Stop(wg *sync.WaitGroup) {
|
|
if s.Running {
|
|
s.StoppedChan = make(chan bool)
|
|
close(s.StopChan)
|
|
<-s.StopChan
|
|
wg.Done()
|
|
s.Running = false
|
|
}
|
|
}
|
|
|
|
func (s *ShodanPlugin) doRun(graphClient *dgo.Dgraph) {
|
|
client := shodan.NewEnvClient(nil)
|
|
err := client.GetBannersByPorts(context.Background(), viper.GetIntSlice("shodan.ports"), s.ShodanChan)
|
|
if err != nil {
|
|
logrus.Panic(err)
|
|
}
|
|
|
|
for {
|
|
select {
|
|
default:
|
|
banner, ok := <-s.ShodanChan
|
|
if !ok {
|
|
logrus.Error("channel error: ", ok)
|
|
break
|
|
}
|
|
|
|
shodanNode := models.BuildShodanNode(banner)
|
|
// first filter poc
|
|
if shodanNode.HostData.HTML != "" {
|
|
if !filters.RunIPFilters(shodanNode.HostData.IP) {
|
|
hostnames := shodanNode.HostData.Hostnames
|
|
var hostNotInFilters, domainNotInFilters bool
|
|
if len(hostnames) != 0 {
|
|
for _, hostname := range hostnames {
|
|
hostNotInFilters = filters.RunDomainFilters(hostname)
|
|
if hostNotInFilters {
|
|
logrus.Info("host", hostname, " not in filters")
|
|
// keep track of new hostnames
|
|
// saveSingleValues(conn, "shodan_stream", "hostname", shodanNode.ID, hostname)
|
|
}
|
|
}
|
|
}
|
|
domains := shodanNode.HostData.Domains
|
|
if len(domains) != 0 {
|
|
for _, domain := range domains {
|
|
domainNotInFilters = filters.RunDomainFilters(domain)
|
|
logrus.Info("domain", domain, "not in filters")
|
|
// keep trakc of new domains
|
|
// saveSingleValues(conn, "shodan_stream", "domain", shodanNode.ID, domain)
|
|
}
|
|
}
|
|
if domainNotInFilters && hostNotInFilters {
|
|
// models.SaveShodanNode("raw_shodan.json", shodanNode)
|
|
mainNode := models.BuildNode("shodan", "shodan_stream", shodanNode.ID)
|
|
models.SaveNode("nodes.json", mainNode)
|
|
// edge := models.BuildEdge("shodan", structs.Map(shodanNode), structs.Map(mainNode))
|
|
// models.SaveEdge(edge)
|
|
e := models.Node{
|
|
ID: mainNode.ID,
|
|
NodeType: mainNode.NodeType,
|
|
NData: mainNode.NData,
|
|
Created: mainNode.Created,
|
|
Modified: mainNode.Modified,
|
|
ShodanNode: *shodanNode,
|
|
}
|
|
|
|
ctx := context.Background()
|
|
mu := &api.Mutation{
|
|
CommitNow: true,
|
|
}
|
|
|
|
pb, err := json.Marshal(e)
|
|
if err != nil {
|
|
logrus.Fatal(err)
|
|
}
|
|
|
|
mu.SetJson = pb
|
|
|
|
_, err = graphClient.NewTxn().Mutate(ctx, mu)
|
|
if err != nil {
|
|
logrus.Error(err)
|
|
}
|
|
}
|
|
} else {
|
|
logrus.Info(shodanNode.HostData.IP, "is akamain")
|
|
}
|
|
}
|
|
}
|
|
|
|
}
|
|
}
|