Adding balboa enrichment for domains and hostnames + documentation
This commit is contained in:
Christopher Talib
parent
787e2c3d02
commit
afd99c0a4d
|
|
@ -115,3 +115,5 @@ type ShodanNode struct {
|
|||
Modified string `json:"modified"`
|
||||
}
|
||||
```
|
||||
|
||||
###
|
||||
|
|
|
|||
|
|
@ -3,14 +3,15 @@ package broker
|
|||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/segmentio/kafka-go"
|
||||
"github.com/sirupsen/logrus"
|
||||
"gitlab.dcso.lolcat/LABS/styx/balboa"
|
||||
"gitlab.dcso.lolcat/LABS/styx/models"
|
||||
)
|
||||
|
||||
// SetUpKafkaConnecter builds the connection to Kafka with a timeout.
|
||||
func SetUpKafkaConnecter() (*kafka.Conn, error) {
|
||||
topic := "styx"
|
||||
partition := 0
|
||||
|
|
@ -23,6 +24,7 @@ func SetUpKafkaConnecter() (*kafka.Conn, error) {
|
|||
return conn, nil
|
||||
}
|
||||
|
||||
// SendEventToKafka sends a node to the broker.
|
||||
func SendEventToKafka(conn *kafka.Conn, node models.Node) {
|
||||
conn.SetWriteDeadline(time.Now().Add(10 * time.Second))
|
||||
packaged, _ := json.Marshal(node)
|
||||
|
|
@ -33,6 +35,7 @@ func SendEventToKafka(conn *kafka.Conn, node models.Node) {
|
|||
|
||||
}
|
||||
|
||||
// ReadEventFromKafka read the event sent to Kafka and acts upon it.
|
||||
func ReadEventFromKafka() {
|
||||
r := kafka.NewReader(kafka.ReaderConfig{
|
||||
Brokers: []string{"localhost:9092"},
|
||||
|
|
@ -49,11 +52,27 @@ func ReadEventFromKafka() {
|
|||
if err != nil {
|
||||
logrus.Error("error reading message:", err)
|
||||
}
|
||||
c, err := balboa.GetClient()
|
||||
if err != nil {
|
||||
logrus.Warn("cannot get balboa client:", err)
|
||||
}
|
||||
|
||||
var node models.Node
|
||||
json.Unmarshal(m.Value, &node)
|
||||
if len(node.ID) != 0 {
|
||||
fmt.Println(node)
|
||||
// TODO: refactor this context
|
||||
ctx := context.Background()
|
||||
entries, err := c.GetAllEntries(ctx, node.Data, "", "", int32(1))
|
||||
if err != nil {
|
||||
logrus.Error("error from balboa", err)
|
||||
}
|
||||
if len(entries) != 0 {
|
||||
balboaNode := models.BuildBalboaNode(entries)
|
||||
models.SaveBalboaNode("bnodes.json", balboaNode)
|
||||
edge := models.BuildEdge("balboa", node.ID, balboaNode.ID)
|
||||
models.SaveEdge(edge)
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
3
go.mod
3
go.mod
|
|
@ -14,8 +14,9 @@ require (
|
|||
github.com/pkg/errors v0.9.0 // indirect
|
||||
github.com/segmentio/kafka-go v0.3.4
|
||||
github.com/sirupsen/logrus v1.4.2
|
||||
gitlab.dcso.lolcat/go/golistic v1.12.14
|
||||
gitlab.dcso.lolcat/go/vizor v1.20.0
|
||||
golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d // indirect
|
||||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa // indirect
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42 // indirect
|
||||
golang.org/x/text v0.3.2 // indirect
|
||||
)
|
||||
|
|
|
|||
66
go.sum
66
go.sum
|
|
@ -3,13 +3,22 @@ github.com/CaliDog/certstream-go v0.0.0-20180219203951-6016c5462366/go.mod h1:JB
|
|||
github.com/DataDog/zstd v1.4.0/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
|
||||
github.com/DataDog/zstd v1.4.4 h1:+IawcoXhCBylN7ccwdwf8LOH2jKq7NavGpEPanrlTzE=
|
||||
github.com/DataDog/zstd v1.4.4/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
|
||||
github.com/aws/aws-lambda-go v1.8.1/go.mod h1:zUsUQhAUjYzR8AuduJPCfhBuKWUaDbQiPOG+ouzmE1A=
|
||||
github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
|
||||
github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk=
|
||||
github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
|
||||
github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA=
|
||||
github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w=
|
||||
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
|
||||
github.com/golang/snappy v0.0.1 h1:Qgr9rKW7uDUkrbSmQeiDsGa8SjGyCOGtuasMWwvp2P4=
|
||||
github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
|
||||
|
|
@ -20,10 +29,13 @@ github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
|
|||
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
|
||||
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/influxdata/influxdb1-client v0.0.0-20190124185755-16c852ea613f/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
|
||||
github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e h1:ZZCvgaRDZg1gC9/1xrsgaJzQUCQgniKtw0xjWywWAOE=
|
||||
github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e/go.mod h1:+rHyWac2R9oAZwFe1wGY2HBzFJJy++RHBg1cU23NkD8=
|
||||
github.com/konsorten/go-windows-terminal-sequences v0.0.0-20180402223658-b729f2633dfe/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
|
||||
github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
|
||||
github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
|
||||
|
|
@ -31,6 +43,9 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
|||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/ns3777k/go-shodan/v4 v4.2.0 h1:18R6axS4f+l37ic14BfjnmMo1dLgNTiPi6dtPXd9qwc=
|
||||
github.com/ns3777k/go-shodan/v4 v4.2.0/go.mod h1:7kSWq/PQ/JCH6U4k2YjXRmnJKfPaJZAhOSMgAXRB23U=
|
||||
github.com/nyaruka/phonenumbers v1.0.42/go.mod h1:Hhae+eypC1YKMaQlBJUCGZDzBrIHHNWhJX1xG/8sOC8=
|
||||
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pierrec/lz4 v2.4.0+incompatible h1:06usnXXDNcPvCHDkmPpkidf4jTc52UKld7UPfqKatY4=
|
||||
github.com/pierrec/lz4 v2.4.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
|
|
@ -38,37 +53,88 @@ github.com/pkg/errors v0.9.0 h1:J8lpUdobwIeCI7OiSxHqEwJUKvJwicL5+3v1oe2Yb4k=
|
|||
github.com/pkg/errors v0.9.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pquerna/otp v1.0.0/go.mod h1:Zad1CMQfSQZI5KLpahDiSUX4tMMREnXw98IvL1nhgMk=
|
||||
github.com/pquerna/otp v1.2.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/segmentio/kafka-go v0.3.4 h1:Mv9AcnCgU14/cU6Vd0wuRdG1FBO0HzXQLnjBduDLy70=
|
||||
github.com/segmentio/kafka-go v0.3.4/go.mod h1:OT5KXBPbaJJTcvokhWR2KFmm0niEx3mnccTwjmLvSi4=
|
||||
github.com/sirupsen/logrus v1.1.1/go.mod h1:zrgwTnHtNr00buQ1vSptGe8m1f/BbgsPukg8qsT7A+A=
|
||||
github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
|
||||
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/skip2/go-qrcode v0.0.0-20171229120447-cf5f9fa2f0d8/go.mod h1:PLPIyL7ikehBD1OAjmKKiOEhbvWyHGaNDjquXMcYABo=
|
||||
github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
|
||||
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
|
||||
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
|
||||
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c h1:u40Z8hqBAAQyv+vATcGgV0YCnDjqSL7/q/JyPhhJSPk=
|
||||
github.com/xdg/scram v0.0.0-20180814205039-7eeb5667e42c/go.mod h1:lB8K/P019DLNhemzwFU4jHLhdvlE6uDZjXFejJXr49I=
|
||||
github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=
|
||||
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
|
||||
gitlab.dcso.lolcat/go/a3 v0.0.0-20190913150211-730e44550e9c/go.mod h1:WBCQF8UiFlsXmkPrYWLMKkPyphINaxQonHh08c+wHss=
|
||||
gitlab.dcso.lolcat/go/a3client v1.4.0/go.mod h1:27iPJq3QKh4T0z7PX6A1wOlcyx8lUC6ae8nSIQoYTrc=
|
||||
gitlab.dcso.lolcat/go/darq v0.0.0-20191119105836-4d936d8242b9/go.mod h1:LtNUmbMixMvc+Gd9CJ2pwwqOsnfHziJP/HEotRyzbAo=
|
||||
gitlab.dcso.lolcat/go/golistic v0.0.0-20180830082801-b3fabf6e7d2c/go.mod h1:teEDykXNCW8/eEBLlQpTD2w88OCt+i4vVNLMW71JWs0=
|
||||
gitlab.dcso.lolcat/go/golistic v1.4.0/go.mod h1:teEDykXNCW8/eEBLlQpTD2w88OCt+i4vVNLMW71JWs0=
|
||||
gitlab.dcso.lolcat/go/golistic v1.6.0/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/mOOzo17vntZo=
|
||||
gitlab.dcso.lolcat/go/golistic v1.7.0/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/mOOzo17vntZo=
|
||||
gitlab.dcso.lolcat/go/golistic v1.7.1/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/mOOzo17vntZo=
|
||||
gitlab.dcso.lolcat/go/golistic v1.8.3/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/mOOzo17vntZo=
|
||||
gitlab.dcso.lolcat/go/golistic v1.10.0/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/mOOzo17vntZo=
|
||||
gitlab.dcso.lolcat/go/golistic v1.12.14 h1:ijD8jrDqrevUeFafu2nRVcZBqhVkKcm5KSwQINkkJIg=
|
||||
gitlab.dcso.lolcat/go/golistic v1.12.14/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/mOOzo17vntZo=
|
||||
gitlab.dcso.lolcat/go/graphql-go v0.0.0-20180911083115-e96b19adf728/go.mod h1:EJYmIZeY2QsnndF3KPj25O6onHYTEwIMXaV/Sw92SGs=
|
||||
gitlab.dcso.lolcat/go/konfigo v0.0.0-20190102144114-f254146e1d0d/go.mod h1:F8qWHeze5OYs08RwABIj+8eKR7q8OGh+8Fz/+wZFmnI=
|
||||
gitlab.dcso.lolcat/go/konfigo v1.1.0 h1:sRJS7qlAWJacjjEfIIUr3jiqkrd9HBeYFpO8wN7+ym4=
|
||||
gitlab.dcso.lolcat/go/konfigo v1.1.0/go.mod h1:ikRxQON5JN4xak7A4eUcMht3t4bGSvDXo0xKAsOC9K8=
|
||||
gitlab.dcso.lolcat/go/mysqlgr v0.9.4/go.mod h1:hJgbZKJEb5mANBFspVndJ7Jd4cht+ZwBNgCxSB+HJXM=
|
||||
gitlab.dcso.lolcat/go/mysqlgr v0.9.5 h1:Ay9+fmIopDt3EsNx1B4MGqdfe1k92rPl+M4iN/bwQpw=
|
||||
gitlab.dcso.lolcat/go/mysqlgr v0.9.5/go.mod h1:hJgbZKJEb5mANBFspVndJ7Jd4cht+ZwBNgCxSB+HJXM=
|
||||
gitlab.dcso.lolcat/go/notis v0.9.1-0.20190709124205-8e21d63e026d/go.mod h1:TYB/49R5S+dKUPW8EEOc+00ffVtoBvjn1gyLhGBDtu8=
|
||||
gitlab.dcso.lolcat/go/notis v1.0.0/go.mod h1:Bqpjr054oCU2hOYe+wi6DnbaAP1E33YZkSenKmvXB7A=
|
||||
gitlab.dcso.lolcat/go/vizor v1.12.4/go.mod h1:hXgYayDE8biBQB0MNIgc7LN0GmWjjS372MZtLVxvHcE=
|
||||
gitlab.dcso.lolcat/go/vizor v1.20.0 h1:+iPrTQFux18sGT0aQMQGF2ZjEG9fxS+fcne1CD+0LlQ=
|
||||
gitlab.dcso.lolcat/go/vizor v1.20.0/go.mod h1:xYkWYIo4Vj1iCfiyJ/I2bzdN0vaRNm6DXbD2CHjnVX8=
|
||||
golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191117063200-497ca9f6d64f/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d h1:2+ZP7EfsZV7Vvmx3TIqSlSzATMkTAKqM14YGFPoSKjI=
|
||||
golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180921000356-2f5d2388922f/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
|
||||
golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
|
||||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa h1:F+8P+gmewFQYRk6JoLQLwjBCTu3mcIURZfNkVweuRKA=
|
||||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190226215855-775f8194d0f9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894 h1:Cz4ceDQGXuKRnVBDTS23GTn/pU5OE2C0WrNTOYK1Uuc=
|
||||
golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.4/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.1.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
|
|
|
|||
35
main.go
35
main.go
|
|
@ -9,7 +9,9 @@ import (
|
|||
|
||||
"github.com/CaliDog/certstream-go"
|
||||
"github.com/ns3777k/go-shodan/v4/shodan"
|
||||
"github.com/segmentio/kafka-go"
|
||||
"github.com/sirupsen/logrus"
|
||||
"gitlab.dcso.lolcat/LABS/styx/broker"
|
||||
"gitlab.dcso.lolcat/LABS/styx/models"
|
||||
)
|
||||
|
||||
|
|
@ -26,17 +28,17 @@ func main() {
|
|||
|
||||
// The false flag specifies that we want heartbeat messages.
|
||||
stream, errStream := certstream.CertStreamEventStream(false)
|
||||
// Conn, err := broker.SetUpKafkaConnecter()
|
||||
// if err != nil {
|
||||
// panic(err)
|
||||
// }
|
||||
Conn, err := broker.SetUpKafkaConnecter()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// stop channel
|
||||
stopChan := make(chan os.Signal)
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(3)
|
||||
|
||||
// go broker.ReadEventFromKafka()
|
||||
go broker.ReadEventFromKafka()
|
||||
|
||||
// certstream
|
||||
go func() {
|
||||
|
|
@ -52,11 +54,12 @@ func main() {
|
|||
models.SaveCertNode("cert_nodes.json", certNode)
|
||||
mainNode := models.BuildNode("node", "certstream", certNode.ID)
|
||||
models.SaveNode("nodes.json", mainNode)
|
||||
models.BuildEdge("certstream", rawNode.ID, mainNode.ID)
|
||||
models.BuildEdge("certstream", mainNode.ID, certNode.ID)
|
||||
edge := models.BuildEdge("certstream", rawNode.ID, mainNode.ID)
|
||||
models.SaveEdge(edge)
|
||||
edge = models.BuildEdge("certstream", mainNode.ID, certNode.ID)
|
||||
models.SaveEdge(edge)
|
||||
allDomains := data.Data.LeafCert.AllDomains
|
||||
saveSingleValues("certstream", "domain", certNode.ID, allDomains)
|
||||
// broker.SendEventToKafka(Conn, *fingerprintNode)
|
||||
saveSingleValues(Conn, "certstream", "domain", certNode.ID, allDomains)
|
||||
}
|
||||
|
||||
case err := <-errStream:
|
||||
|
|
@ -69,7 +72,7 @@ func main() {
|
|||
|
||||
}()
|
||||
|
||||
// // pastebin
|
||||
// pastebin
|
||||
go func() {
|
||||
for {
|
||||
select {
|
||||
|
|
@ -103,7 +106,7 @@ func main() {
|
|||
// shodan
|
||||
client := shodan.NewEnvClient(nil)
|
||||
ch := make(chan *shodan.HostData)
|
||||
err := client.GetBannersByPorts(context.Background(), []int{80, 443, 8443, 53}, ch)
|
||||
err = client.GetBannersByPorts(context.Background(), []int{80, 443, 8443, 53}, ch)
|
||||
if err != nil {
|
||||
logrus.Panic(err)
|
||||
}
|
||||
|
|
@ -121,11 +124,11 @@ func main() {
|
|||
shodanNode := models.BuildShodanNode(banner)
|
||||
hostnames := shodanNode.Data.Hostnames
|
||||
if len(hostnames) != 0 {
|
||||
saveSingleValues("shodan_stream", "hostname", shodanNode.ID, hostnames)
|
||||
saveSingleValues(Conn, "shodan_stream", "hostname", shodanNode.ID, hostnames)
|
||||
}
|
||||
domains := shodanNode.Data.Domains
|
||||
if len(domains) != 0 {
|
||||
saveSingleValues("shodan_stream", "domain", shodanNode.ID, domains)
|
||||
saveSingleValues(Conn, "shodan_stream", "domain", shodanNode.ID, domains)
|
||||
}
|
||||
models.SaveShodanNode("shodan_raw.json", shodanNode)
|
||||
node := models.BuildNode("shodan", "shodan_stream", shodanNode.ID)
|
||||
|
|
@ -143,11 +146,13 @@ func main() {
|
|||
}
|
||||
|
||||
// helpers
|
||||
|
||||
func saveSingleValues(source string, datatype string, originNodeID string, values []string) {
|
||||
func saveSingleValues(brokerConn *kafka.Conn, source string, datatype string, originNodeID string, values []string) {
|
||||
for _, value := range values {
|
||||
domainNode := models.BuildNode(source, datatype, value)
|
||||
models.SaveNode("nodes.json", domainNode)
|
||||
if domainNode.Type == "domain" || domainNode.Type == "hostname" {
|
||||
broker.SendEventToKafka(brokerConn, *domainNode)
|
||||
}
|
||||
edge := models.BuildEdge(source, originNodeID, domainNode.ID)
|
||||
models.SaveEdge(edge)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import (
|
|||
"github.com/google/uuid"
|
||||
"github.com/ns3777k/go-shodan/v4/shodan"
|
||||
"github.com/sirupsen/logrus"
|
||||
"gitlab.dcso.lolcat/LABS/styx/balboa"
|
||||
"gitlab.dcso.lolcat/LABS/styx/utils"
|
||||
)
|
||||
|
||||
|
|
@ -359,3 +360,54 @@ func SaveShodanNode(filename string, data *ShodanNode) {
|
|||
logrus.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
// BalboaNode represents a return from Balboa.
|
||||
type BalboaNode struct {
|
||||
ID string `json:"id"`
|
||||
Type string `json:"type"`
|
||||
Data []balboa.Entries `json:"data"`
|
||||
Created string `json:"created"`
|
||||
Modified string `json:"modified"`
|
||||
}
|
||||
|
||||
// BuildBalboaNode builds a node coming from Balboa resolution.
|
||||
func BuildBalboaNode(data []balboa.Entries) *BalboaNode {
|
||||
t := time.Now()
|
||||
rfc3339time := t.Format(time.RFC3339)
|
||||
return &BalboaNode{
|
||||
ID: "balboa--" + uuid.New().String(),
|
||||
Type: "balboa",
|
||||
Data: data,
|
||||
Created: rfc3339time,
|
||||
Modified: rfc3339time,
|
||||
}
|
||||
}
|
||||
|
||||
// SaveBalboaNode saves a Balboa node.
|
||||
func SaveBalboaNode(filename string, data *BalboaNode) {
|
||||
err := utils.FileExists(filename)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
nodeFile, err := ioutil.ReadFile(filename)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
rawDatas := []BalboaNode{}
|
||||
|
||||
if err := json.Unmarshal(nodeFile, &rawDatas); err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
rawDatas = append(rawDatas, *data)
|
||||
|
||||
rawBytes, err := json.Marshal(rawDatas)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
err = ioutil.WriteFile(filename, rawBytes, 0644)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue