diff --git a/README.md b/README.md index 8423d46..b85cdf9 100644 --- a/README.md +++ b/README.md @@ -115,3 +115,5 @@ type ShodanNode struct { Modified string `json:"modified"` } ``` + +### diff --git a/broker/main.go b/broker/main.go index 2e7de75..d65ddec 100644 --- a/broker/main.go +++ b/broker/main.go @@ -3,14 +3,15 @@ package broker import ( "context" "encoding/json" - "fmt" "time" "github.com/segmentio/kafka-go" "github.com/sirupsen/logrus" + "gitlab.dcso.lolcat/LABS/styx/balboa" "gitlab.dcso.lolcat/LABS/styx/models" ) +// SetUpKafkaConnecter builds the connection to Kafka with a timeout. func SetUpKafkaConnecter() (*kafka.Conn, error) { topic := "styx" partition := 0 @@ -23,6 +24,7 @@ func SetUpKafkaConnecter() (*kafka.Conn, error) { return conn, nil } +// SendEventToKafka sends a node to the broker. func SendEventToKafka(conn *kafka.Conn, node models.Node) { conn.SetWriteDeadline(time.Now().Add(10 * time.Second)) packaged, _ := json.Marshal(node) @@ -33,6 +35,7 @@ func SendEventToKafka(conn *kafka.Conn, node models.Node) { } +// ReadEventFromKafka read the event sent to Kafka and acts upon it. func ReadEventFromKafka() { r := kafka.NewReader(kafka.ReaderConfig{ Brokers: []string{"localhost:9092"}, @@ -49,11 +52,27 @@ func ReadEventFromKafka() { if err != nil { logrus.Error("error reading message:", err) } + c, err := balboa.GetClient() + if err != nil { + logrus.Warn("cannot get balboa client:", err) + } var node models.Node json.Unmarshal(m.Value, &node) if len(node.ID) != 0 { - fmt.Println(node) + // TODO: refactor this context + ctx := context.Background() + entries, err := c.GetAllEntries(ctx, node.Data, "", "", int32(1)) + if err != nil { + logrus.Error("error from balboa", err) + } + if len(entries) != 0 { + balboaNode := models.BuildBalboaNode(entries) + models.SaveBalboaNode("bnodes.json", balboaNode) + edge := models.BuildEdge("balboa", node.ID, balboaNode.ID) + models.SaveEdge(edge) + } + } } } diff --git a/go.mod b/go.mod index f61ba10..e4aad64 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,9 @@ require ( github.com/pkg/errors v0.9.0 // indirect github.com/segmentio/kafka-go v0.3.4 github.com/sirupsen/logrus v1.4.2 + gitlab.dcso.lolcat/go/golistic v1.12.14 + gitlab.dcso.lolcat/go/vizor v1.20.0 golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d // indirect golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa // indirect golang.org/x/sys v0.0.0-20200116001909-b77594299b42 // indirect - golang.org/x/text v0.3.2 // indirect ) diff --git a/go.sum b/go.sum index 7a2a8b3..2c4b456 100644 --- a/go.sum +++ b/go.sum @@ -3,13 +3,22 @@ github.com/CaliDog/certstream-go v0.0.0-20180219203951-6016c5462366/go.mod h1:JB github.com/DataDog/zstd v1.4.0/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE= github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= +github.com/aws/aws-lambda-go v1.8.1/go.mod h1:zUsUQhAUjYzR8AuduJPCfhBuKWUaDbQiPOG+ouzmE1A= +github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= +github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= @@ -20,10 +29,13 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/influxdata/influxdb1-client v0.0.0-20190124185755-16c852ea613f/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e h1:ZZCvgaRDZg1gC9/1xrsgaJzQUCQgniKtw0xjWywWAOE= github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e/go.mod h1:+rHyWac2R9oAZwFe1wGY2HBzFJJy++RHBg1cU23NkD8= +github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= @@ -31,6 +43,9 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/ns3777k/go-shodan/v4 v4.2.0 h1:18R6axS4f+l37ic14BfjnmMo1dLgNTiPi6dtPXd9qwc= github.com/ns3777k/go-shodan/v4 v4.2.0/go.mod h1:7kSWq/PQ/JCH6U4k2YjXRmnJKfPaJZAhOSMgAXRB23U= +github.com/nyaruka/phonenumbers v1.0.42/go.mod h1:Hhae+eypC1YKMaQlBJUCGZDzBrIHHNWhJX1xG/8sOC8= +github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pierrec/lz4 v2.4.0+incompatible h1:06usnXXDNcPvCHDkmPpkidf4jTc52UKld7UPfqKatY4= github.com/pierrec/lz4 v2.4.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= @@ -38,37 +53,88 @@ github.com/pkg/errors v0.9.0 h1:J8lpUdobwIeCI7OiSxHqEwJUKvJwicL5+3v1oe2Yb4k= github.com/pkg/errors v0.9.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pquerna/otp v1.0.0/go.mod h1:Zad1CMQfSQZI5KLpahDiSUX4tMMREnXw98IvL1nhgMk= +github.com/pquerna/otp v1.2.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= +github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/segmentio/kafka-go v0.3.4 h1:Mv9AcnCgU14/cU6Vd0wuRdG1FBO0HzXQLnjBduDLy70= github.com/segmentio/kafka-go v0.3.4/go.mod h1:OT5KXBPbaJJTcvokhWR2KFmm0niEx3mnccTwjmLvSi4= +github.com/sirupsen/logrus v1.1.1/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/skip2/go-qrcode v0.0.0-20171229120447-cf5f9fa2f0d8/go.mod h1:PLPIyL7ikehBD1OAjmKKiOEhbvWyHGaNDjquXMcYABo= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk= github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I= github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= +gitlab.dcso.lolcat/go/a3 v0.0.0-20190913150211-730e44550e9c/go.mod h1:WBCQF8UiFlsXmkPrYWLMKkPyphINaxQonHh08c+wHss= +gitlab.dcso.lolcat/go/a3client v1.4.0/go.mod h1:27iPJq3QKh4T0z7PX6A1wOlcyx8lUC6ae8nSIQoYTrc= +gitlab.dcso.lolcat/go/darq v0.0.0-20191119105836-4d936d8242b9/go.mod h1:LtNUmbMixMvc+Gd9CJ2pwwqOsnfHziJP/HEotRyzbAo= +gitlab.dcso.lolcat/go/golistic v0.0.0-20180830082801-b3fabf6e7d2c/go.mod h1:teEDykXNCW8/eEBLlQpTD2w88OCt+i4vVNLMW71JWs0= +gitlab.dcso.lolcat/go/golistic v1.4.0/go.mod h1:teEDykXNCW8/eEBLlQpTD2w88OCt+i4vVNLMW71JWs0= +gitlab.dcso.lolcat/go/golistic v1.6.0/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/mOOzo17vntZo= +gitlab.dcso.lolcat/go/golistic v1.7.0/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/mOOzo17vntZo= +gitlab.dcso.lolcat/go/golistic v1.7.1/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/mOOzo17vntZo= +gitlab.dcso.lolcat/go/golistic v1.8.3/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/mOOzo17vntZo= +gitlab.dcso.lolcat/go/golistic v1.10.0/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/mOOzo17vntZo= +gitlab.dcso.lolcat/go/golistic v1.12.14 h1:ijD8jrDqrevUeFafu2nRVcZBqhVkKcm5KSwQINkkJIg= +gitlab.dcso.lolcat/go/golistic v1.12.14/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/mOOzo17vntZo= +gitlab.dcso.lolcat/go/graphql-go v0.0.0-20180911083115-e96b19adf728/go.mod h1:EJYmIZeY2QsnndF3KPj25O6onHYTEwIMXaV/Sw92SGs= +gitlab.dcso.lolcat/go/konfigo v0.0.0-20190102144114-f254146e1d0d/go.mod h1:F8qWHeze5OYs08RwABIj+8eKR7q8OGh+8Fz/+wZFmnI= +gitlab.dcso.lolcat/go/konfigo v1.1.0 h1:sRJS7qlAWJacjjEfIIUr3jiqkrd9HBeYFpO8wN7+ym4= +gitlab.dcso.lolcat/go/konfigo v1.1.0/go.mod h1:ikRxQON5JN4xak7A4eUcMht3t4bGSvDXo0xKAsOC9K8= +gitlab.dcso.lolcat/go/mysqlgr v0.9.4/go.mod h1:hJgbZKJEb5mANBFspVndJ7Jd4cht+ZwBNgCxSB+HJXM= +gitlab.dcso.lolcat/go/mysqlgr v0.9.5 h1:Ay9+fmIopDt3EsNx1B4MGqdfe1k92rPl+M4iN/bwQpw= +gitlab.dcso.lolcat/go/mysqlgr v0.9.5/go.mod h1:hJgbZKJEb5mANBFspVndJ7Jd4cht+ZwBNgCxSB+HJXM= +gitlab.dcso.lolcat/go/notis v0.9.1-0.20190709124205-8e21d63e026d/go.mod h1:TYB/49R5S+dKUPW8EEOc+00ffVtoBvjn1gyLhGBDtu8= +gitlab.dcso.lolcat/go/notis v1.0.0/go.mod h1:Bqpjr054oCU2hOYe+wi6DnbaAP1E33YZkSenKmvXB7A= +gitlab.dcso.lolcat/go/vizor v1.12.4/go.mod h1:hXgYayDE8biBQB0MNIgc7LN0GmWjjS372MZtLVxvHcE= +gitlab.dcso.lolcat/go/vizor v1.20.0 h1:+iPrTQFux18sGT0aQMQGF2ZjEG9fxS+fcne1CD+0LlQ= +gitlab.dcso.lolcat/go/vizor v1.20.0/go.mod h1:xYkWYIo4Vj1iCfiyJ/I2bzdN0vaRNm6DXbD2CHjnVX8= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191117063200-497ca9f6d64f/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d h1:2+ZP7EfsZV7Vvmx3TIqSlSzATMkTAKqM14YGFPoSKjI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20180921000356-2f5d2388922f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190226215855-775f8194d0f9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.6.4/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.1.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/main.go b/main.go index 4053fdb..1a51285 100644 --- a/main.go +++ b/main.go @@ -9,7 +9,9 @@ import ( "github.com/CaliDog/certstream-go" "github.com/ns3777k/go-shodan/v4/shodan" + "github.com/segmentio/kafka-go" "github.com/sirupsen/logrus" + "gitlab.dcso.lolcat/LABS/styx/broker" "gitlab.dcso.lolcat/LABS/styx/models" ) @@ -26,17 +28,17 @@ func main() { // The false flag specifies that we want heartbeat messages. stream, errStream := certstream.CertStreamEventStream(false) - // Conn, err := broker.SetUpKafkaConnecter() - // if err != nil { - // panic(err) - // } + Conn, err := broker.SetUpKafkaConnecter() + if err != nil { + panic(err) + } // stop channel stopChan := make(chan os.Signal) var wg sync.WaitGroup wg.Add(3) - // go broker.ReadEventFromKafka() + go broker.ReadEventFromKafka() // certstream go func() { @@ -52,11 +54,12 @@ func main() { models.SaveCertNode("cert_nodes.json", certNode) mainNode := models.BuildNode("node", "certstream", certNode.ID) models.SaveNode("nodes.json", mainNode) - models.BuildEdge("certstream", rawNode.ID, mainNode.ID) - models.BuildEdge("certstream", mainNode.ID, certNode.ID) + edge := models.BuildEdge("certstream", rawNode.ID, mainNode.ID) + models.SaveEdge(edge) + edge = models.BuildEdge("certstream", mainNode.ID, certNode.ID) + models.SaveEdge(edge) allDomains := data.Data.LeafCert.AllDomains - saveSingleValues("certstream", "domain", certNode.ID, allDomains) - // broker.SendEventToKafka(Conn, *fingerprintNode) + saveSingleValues(Conn, "certstream", "domain", certNode.ID, allDomains) } case err := <-errStream: @@ -69,7 +72,7 @@ func main() { }() - // // pastebin + // pastebin go func() { for { select { @@ -103,7 +106,7 @@ func main() { // shodan client := shodan.NewEnvClient(nil) ch := make(chan *shodan.HostData) - err := client.GetBannersByPorts(context.Background(), []int{80, 443, 8443, 53}, ch) + err = client.GetBannersByPorts(context.Background(), []int{80, 443, 8443, 53}, ch) if err != nil { logrus.Panic(err) } @@ -121,11 +124,11 @@ func main() { shodanNode := models.BuildShodanNode(banner) hostnames := shodanNode.Data.Hostnames if len(hostnames) != 0 { - saveSingleValues("shodan_stream", "hostname", shodanNode.ID, hostnames) + saveSingleValues(Conn, "shodan_stream", "hostname", shodanNode.ID, hostnames) } domains := shodanNode.Data.Domains if len(domains) != 0 { - saveSingleValues("shodan_stream", "domain", shodanNode.ID, domains) + saveSingleValues(Conn, "shodan_stream", "domain", shodanNode.ID, domains) } models.SaveShodanNode("shodan_raw.json", shodanNode) node := models.BuildNode("shodan", "shodan_stream", shodanNode.ID) @@ -143,11 +146,13 @@ func main() { } // helpers - -func saveSingleValues(source string, datatype string, originNodeID string, values []string) { +func saveSingleValues(brokerConn *kafka.Conn, source string, datatype string, originNodeID string, values []string) { for _, value := range values { domainNode := models.BuildNode(source, datatype, value) models.SaveNode("nodes.json", domainNode) + if domainNode.Type == "domain" || domainNode.Type == "hostname" { + broker.SendEventToKafka(brokerConn, *domainNode) + } edge := models.BuildEdge(source, originNodeID, domainNode.ID) models.SaveEdge(edge) } diff --git a/models/main.go b/models/main.go index 8aa8fbc..c6a9f5b 100644 --- a/models/main.go +++ b/models/main.go @@ -8,6 +8,7 @@ import ( "github.com/google/uuid" "github.com/ns3777k/go-shodan/v4/shodan" "github.com/sirupsen/logrus" + "gitlab.dcso.lolcat/LABS/styx/balboa" "gitlab.dcso.lolcat/LABS/styx/utils" ) @@ -359,3 +360,54 @@ func SaveShodanNode(filename string, data *ShodanNode) { logrus.Error(err) } } + +// BalboaNode represents a return from Balboa. +type BalboaNode struct { + ID string `json:"id"` + Type string `json:"type"` + Data []balboa.Entries `json:"data"` + Created string `json:"created"` + Modified string `json:"modified"` +} + +// BuildBalboaNode builds a node coming from Balboa resolution. +func BuildBalboaNode(data []balboa.Entries) *BalboaNode { + t := time.Now() + rfc3339time := t.Format(time.RFC3339) + return &BalboaNode{ + ID: "balboa--" + uuid.New().String(), + Type: "balboa", + Data: data, + Created: rfc3339time, + Modified: rfc3339time, + } +} + +// SaveBalboaNode saves a Balboa node. +func SaveBalboaNode(filename string, data *BalboaNode) { + err := utils.FileExists(filename) + if err != nil { + logrus.Error(err) + } + nodeFile, err := ioutil.ReadFile(filename) + if err != nil { + logrus.Error(err) + } + rawDatas := []BalboaNode{} + + if err := json.Unmarshal(nodeFile, &rawDatas); err != nil { + logrus.Error(err) + } + + rawDatas = append(rawDatas, *data) + + rawBytes, err := json.Marshal(rawDatas) + if err != nil { + logrus.Error(err) + } + + err = ioutil.WriteFile(filename, rawBytes, 0644) + if err != nil { + logrus.Error(err) + } +}