Shodan connector and saving up and running

go.sum

@@ -46,6 +46,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
+github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
 github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
@@ -69,4 +70,5 @@ golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

main.go

@@ -1,20 +1,23 @@
 package main
 
 import (
+	"context"
 	"fmt"
 	"os"
 	"sync"
 	"time"
 
 	"github.com/CaliDog/certstream-go"
+	"github.com/ns3777k/go-shodan/v4/shodan"
 	"github.com/sirupsen/logrus"
 	"gitlab.dcso.lolcat/LABS/styx/models"
 )
 
 func main() {
+	fmt.Println("Starting to get data from the Internet...")
+
 	// The false flag specifies that we want heartbeat messages.
 	stream, errStream := certstream.CertStreamEventStream(false)
-	// fmt.Println("Starting to get data from CertStream...")
 	// Conn, err := broker.SetUpKafkaConnecter()
 	// if err != nil {
 	// panic(err)
@@ -23,7 +26,7 @@ func main() {
 	// stop channel
 	stopChan := make(chan os.Signal)
 	var wg sync.WaitGroup
-	wg.Add(2)
+	wg.Add(3)
 
 	// go broker.ReadEventFromKafka()
 
@@ -43,15 +46,12 @@ func main() {
 					models.SaveNode("nodes.json", mainNode)
 					models.BuildEdge("certstream", rawNode.ID, mainNode.ID)
 					models.BuildEdge("certstream", mainNode.ID, certNode.ID)
-					fmt.Println(certNode)
-					fmt.Println(certNode)
 					allDomains := data.Data.LeafCert.AllDomains
 					var edge *models.Edge
 					for _, domain := range allDomains {
 						domainNode := models.BuildNode("certstream", "domain", domain)
 						models.SaveNode("nodes.json", domainNode)
 						edge = models.BuildEdge("certstream", certNode.ID, domainNode.ID)
-						fmt.Println(edge)
 						models.SaveEdge(edge)
 					}
 					// broker.SendEventToKafka(Conn, *fingerprintNode)
@@ -78,7 +78,6 @@ func main() {
 					logrus.Panic(err)
 				}
 				for _, p := range pastes {
-					fmt.Println("fetching...", p.FullURL)
 					paste, err := models.FetchPaste(p)
 					if err != nil {
 						logrus.Error("cannot fetch paste", err)
@@ -99,24 +98,37 @@ func main() {
 			}
 		}
 	}()
-	wg.Wait()
 
 	// shodan
-	// client := shodan.NewEnvClient(nil)
-	// ch := make(chan *shodan.HostData)
-	// err := client.GetBannersByPorts(context.Background(), []int{80, 443}, ch)
-	// if err != nil {
-	// logrus.Error(err)
-	// }
+	client := shodan.NewEnvClient(nil)
+	ch := make(chan *shodan.HostData)
+	err := client.GetBannersByPorts(context.Background(), []int{80, 443}, ch)
+	if err != nil {
+		logrus.Error(err)
+	}
 
-	// for {
-	// banner, ok := <-ch
-	// fmt.Println("hello", ok)
-	// if !ok {
-	// logrus.Error("channel is closed")
-	// break
-	// }
-	// fmt.Println(banner)
-	// }
+	go func() {
+		for {
+			select {
+			default:
+				banner, ok := <-ch
+				if !ok {
+					logrus.Error("channel is closed")
+					break
+				}
+
+				shodanNode := models.BuildShodanNode(banner)
+				models.SaveShodanNode("shodan_raw.json", shodanNode)
+				node := models.BuildNode("shodan", "shodan_stream", shodanNode.ID)
+				models.SaveNode("nodes.json", node)
+				edge := models.BuildEdge("shodan", shodanNode.ID, node.ID)
+				models.SaveEdge(edge)
+			case <-stopChan:
+				wg.Done()
+				return
+			}
+		}
+	}()
+	wg.Wait()
 
 }

models/main.go

@@ -6,6 +6,7 @@ import (
 	"time"
 
 	"github.com/google/uuid"
+	"github.com/ns3777k/go-shodan/v4/shodan"
 	"github.com/sirupsen/logrus"
 	"gitlab.dcso.lolcat/LABS/styx/utils"
 )
@@ -240,3 +241,54 @@ func SaveEdge(edge *Edge) {
 		logrus.Error(err)
 	}
 }
+
+// ShodanNode is node around the shodan.HostData struct.
+type ShodanNode struct {
+	ID       string           `json:"id"`
+	Type     string           `json:"type"`
+	Data     *shodan.HostData `json:"data"`
+	Created  string           `json:"created"`
+	Modified string           `json:"modified"`
+}
+
+// BuildShodanNode builds a wrapper node around shodan.HostData.
+func BuildShodanNode(data *shodan.HostData) *ShodanNode {
+	t := time.Now()
+	rfc3339time := t.Format(time.RFC3339)
+	return &ShodanNode{
+		ID:       "shodan--" + uuid.New().String(),
+		Type:     "shodan_stream",
+		Data:     data,
+		Created:  rfc3339time,
+		Modified: rfc3339time,
+	}
+}
+
+// SaveShodanNode saves the raw nodes from Shodan.
+func SaveShodanNode(filename string, data *ShodanNode) {
+	err := utils.FileExists(filename)
+	if err != nil {
+		logrus.Error(err)
+	}
+	nodeFile, err := ioutil.ReadFile(filename)
+	if err != nil {
+		logrus.Error(err)
+	}
+	rawDatas := []ShodanNode{}
+
+	if err := json.Unmarshal(nodeFile, &rawDatas); err != nil {
+		logrus.Error(err)
+	}
+
+	rawDatas = append(rawDatas, *data)
+
+	rawBytes, err := json.Marshal(rawDatas)
+	if err != nil {
+		logrus.Error(err)
+	}
+
+	err = ioutil.WriteFile(filename, rawBytes, 0644)
+	if err != nil {
+		logrus.Error(err)
+	}
+}

models/shodan.go

@@ -0,0 +1 @@
+package models