chris/styx
chris/styx
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Couple of quickfixes to allow run a dry installion (deactivating ES and balboa)

Browse source
This commit is contained in:
Christopher Talib 2020-02-19 15:26:47 +01:00
parent 56e0e52bb5
commit e7421931c2
2 changed files with 44 additions and 29 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
broker/main.go
View file

@ -64,9 +64,13 @@ func ReadEventFromKafka() {
if err != nil {
logrus.Error("error reading message:", err)
}
c, err := balboa.GetClient()
if err != nil {
logrus.Warn("cannot get balboa client:", err)
var c *balboa.Client
if viper.GetBool("balboa.activated") {
c, err = balboa.GetClient()
if err != nil {
logrus.Warn("cannot get balboa client:", err)
}
}
var node models.Node

63
main.go
View file

@ -66,7 +66,7 @@ func main() {
// shodan
client := shodan.NewEnvClient(nil)
ch := make(chan *shodan.HostData)
err = client.GetBannersByPorts(context.Background(), viper.GetIntSlice("shodan.ports"), ch)
err = client.GetBanners(context.Background(), ch)
if err != nil {
logrus.Panic(err)
}
@ -115,16 +115,21 @@ func certstreamRoutine(stream chan jsonq.JsonQuery, errStream chan error, conn *
func pastebinRoutine(stopChan chan os.Signal, wg *sync.WaitGroup) {
fmt.Println("pastebin is activated")
elastic := viper.GetBool("elasticsearch.activated")
var e *elasticsearch.ElasticStorageModule
e = &elasticsearch.ElasticStorageModule{
ElasticURL: viper.GetString("elasticsearch.url"),
DailyIndexes: true,
UseIndex: "pastebin",
LastChk: time.Now(),
}
err := e.Initialize()
if err != nil {
panic(err)
if elastic {
fmt.Println("elasticsearch is activated")
e = &elasticsearch.ElasticStorageModule{
ElasticURL: viper.GetString("elasticsearch.url"),
DailyIndexes: true,
UseIndex: "pastebin",
LastChk: time.Now(),
}
err := e.Initialize()
if err != nil {
panic(err)
}
}
for {
select {
@ -143,7 +148,9 @@ func pastebinRoutine(stopChan chan os.Signal, wg *sync.WaitGroup) {
Full: paste,
}
res := models.BuildPasteNode(&fp)
e.StorePaste(fp)
if elastic {
e.StorePaste(fp)
}
models.SavePaste("paste_formatted.json", res)
time.Sleep(1 * time.Second)
@ -169,22 +176,26 @@ func shodanRoutine(client *shodan.Client, shodanChan chan *shodan.HostData, conn
shodanNode := models.BuildShodanNode(banner)
// first filter poc
if !filters.RunIPFilters(shodanNode.Data.IP) {
hostnames := shodanNode.Data.Hostnames
if len(hostnames) != 0 {
saveSingleValues(conn, "shodan_stream", "hostname", shodanNode.ID, hostnames)
if shodanNode.Data.HTML != "" {
fmt.Println("##### not empty HTML", shodanNode.ID)
if !filters.RunIPFilters(shodanNode.Data.IP) {
fmt.Println("##### not in filters", shodanNode.ID)
hostnames := shodanNode.Data.Hostnames
if len(hostnames) != 0 {
saveSingleValues(conn, "shodan_stream", "hostname", shodanNode.ID, hostnames)
}
domains := shodanNode.Data.Domains
if len(domains) != 0 {
saveSingleValues(conn, "shodan_stream", "domain", shodanNode.ID, domains)
}
models.SaveShodanNode("raw_shodan.json", shodanNode)
node := models.BuildNode("shodan", "shodan_stream", shodanNode.ID)
models.SaveNode("nodes.json", node)
edge := models.BuildEdge("shodan", shodanNode.ID, node.ID)
models.SaveEdge(edge)
} else {
fmt.Println("is akamai", shodanNode.Data.IP)
}
domains := shodanNode.Data.Domains
if len(domains) != 0 {
saveSingleValues(conn, "shodan_stream", "domain", shodanNode.ID, domains)
}
models.SaveShodanNode("raw_shodan.json", shodanNode)
node := models.BuildNode("shodan", "shodan_stream", shodanNode.ID)
models.SaveNode("nodes.json", node)
edge := models.BuildEdge("shodan", shodanNode.ID, node.ID)
models.SaveEdge(edge)
} else {
fmt.Println("is akamai", shodanNode.Data.IP)
}
case <-stopChan:
wg.Done()