Couple of quickfixes to allow run a dry installion (deactivating ES and balboa)

broker/main.go

@@ -64,9 +64,13 @@ func ReadEventFromKafka() {
 		if err != nil {
 			logrus.Error("error reading message:", err)
 		}
-		c, err := balboa.GetClient()
+
-		if err != nil {
+		var c *balboa.Client
-			logrus.Warn("cannot get balboa client:", err)
+		if viper.GetBool("balboa.activated") {
+			c, err = balboa.GetClient()
+			if err != nil {
+				logrus.Warn("cannot get balboa client:", err)
+			}
 		}
 
 		var node models.Node

main.go

@@ -66,7 +66,7 @@ func main() {
 	// shodan
 	client := shodan.NewEnvClient(nil)
 	ch := make(chan *shodan.HostData)
-	err = client.GetBannersByPorts(context.Background(), viper.GetIntSlice("shodan.ports"), ch)
+	err = client.GetBanners(context.Background(), ch)
 	if err != nil {
 		logrus.Panic(err)
 	}
@@ -115,16 +115,21 @@ func certstreamRoutine(stream chan jsonq.JsonQuery, errStream chan error, conn *
 
 func pastebinRoutine(stopChan chan os.Signal, wg *sync.WaitGroup) {
 	fmt.Println("pastebin is activated")
+
+	elastic := viper.GetBool("elasticsearch.activated")
 	var e *elasticsearch.ElasticStorageModule
-	e = &elasticsearch.ElasticStorageModule{
+	if elastic {
-		ElasticURL:   viper.GetString("elasticsearch.url"),
+		fmt.Println("elasticsearch is activated")
-		DailyIndexes: true,
+		e = &elasticsearch.ElasticStorageModule{
-		UseIndex:     "pastebin",
+			ElasticURL:   viper.GetString("elasticsearch.url"),
-		LastChk:      time.Now(),
+			DailyIndexes: true,
-	}
+			UseIndex:     "pastebin",
-	err := e.Initialize()
+			LastChk:      time.Now(),
-	if err != nil {
+		}
-		panic(err)
+		err := e.Initialize()
+		if err != nil {
+			panic(err)
+		}
 	}
 	for {
 		select {
@@ -143,7 +148,9 @@ func pastebinRoutine(stopChan chan os.Signal, wg *sync.WaitGroup) {
 					Full: paste,
 				}
 				res := models.BuildPasteNode(&fp)
-				e.StorePaste(fp)
+				if elastic {
+					e.StorePaste(fp)
+				}
 				models.SavePaste("paste_formatted.json", res)
 				time.Sleep(1 * time.Second)
 
@@ -169,22 +176,26 @@ func shodanRoutine(client *shodan.Client, shodanChan chan *shodan.HostData, conn
 
 			shodanNode := models.BuildShodanNode(banner)
 			// first filter poc
-			if !filters.RunIPFilters(shodanNode.Data.IP) {
+			if shodanNode.Data.HTML != "" {
-				hostnames := shodanNode.Data.Hostnames
+				fmt.Println("##### not empty HTML", shodanNode.ID)
-				if len(hostnames) != 0 {
+				if !filters.RunIPFilters(shodanNode.Data.IP) {
-					saveSingleValues(conn, "shodan_stream", "hostname", shodanNode.ID, hostnames)
+					fmt.Println("##### not in filters", shodanNode.ID)
+					hostnames := shodanNode.Data.Hostnames
+					if len(hostnames) != 0 {
+						saveSingleValues(conn, "shodan_stream", "hostname", shodanNode.ID, hostnames)
+					}
+					domains := shodanNode.Data.Domains
+					if len(domains) != 0 {
+						saveSingleValues(conn, "shodan_stream", "domain", shodanNode.ID, domains)
+					}
+					models.SaveShodanNode("raw_shodan.json", shodanNode)
+					node := models.BuildNode("shodan", "shodan_stream", shodanNode.ID)
+					models.SaveNode("nodes.json", node)
+					edge := models.BuildEdge("shodan", shodanNode.ID, node.ID)
+					models.SaveEdge(edge)
+				} else {
+					fmt.Println("is akamai", shodanNode.Data.IP)
 				}
-				domains := shodanNode.Data.Domains
-				if len(domains) != 0 {
-					saveSingleValues(conn, "shodan_stream", "domain", shodanNode.ID, domains)
-				}
-				models.SaveShodanNode("raw_shodan.json", shodanNode)
-				node := models.BuildNode("shodan", "shodan_stream", shodanNode.ID)
-				models.SaveNode("nodes.json", node)
-				edge := models.BuildEdge("shodan", shodanNode.ID, node.ID)
-				models.SaveEdge(edge)
-			} else {
-				fmt.Println("is akamai", shodanNode.Data.IP)
 			}
 		case <-stopChan:
 			wg.Done()