From 84e4937f8534d6d895c26b781e61c1a1b1415a10 Mon Sep 17 00:00:00 2001 From: Christopher Talib Date: Mon, 24 Aug 2020 16:25:49 +0200 Subject: [PATCH] Major version update This new work implements the server and the loader in two different binaries allowing the code while updating the IOC list. It updates also the documentation to reflect the new changes. --- DEMO.md | 6 ++ README.md | 32 +++++----- broker/main.go | 56 ++++++++++++++--- cmd/iocloader/main.go | 117 +++++++++++++++++++++++++++++++++++ main.go => cmd/styxd/main.go | 15 +++-- docker-compose.yml | 58 ++++++++--------- graph/main.go | 10 +-- main_test.go | 2 +- matcher/main.go | 3 +- 9 files changed, 228 insertions(+), 71 deletions(-) create mode 100644 cmd/iocloader/main.go rename main.go => cmd/styxd/main.go (86%) diff --git a/DEMO.md b/DEMO.md index b37bb4a..054d477 100644 --- a/DEMO.md +++ b/DEMO.md @@ -39,3 +39,9 @@ * Upsert is not optimal * What do we do with the data so it can be exploitable by analysts * Sould we store matched data in an SQL-like db? + + +## TDH + +* patterns: stream to IP address and not a domain (so the HTTP hostname won't be interesting) => look into any canonical name resolving to a NXDOMAIN (tls connection directly to an IP address) NO DNS => hostname is a DGA (should be a way to identify visually) +* any canonical name that is a IP address and not a domain name diff --git a/README.md b/README.md index 6da602c..d9eebbb 100644 --- a/README.md +++ b/README.md @@ -36,15 +36,23 @@ docker run --rm -it -p 8080:8080 -p 9080:9080 -p 8000:8000 -v ~/dgraph:/dgraph d ```sh go get -u gitlab.dcso.lolcat/LABS/styx cd $GOPATH/src/gitlab.dcso.lolcat/LABS/styx -go build +go build gitlab.dcso.lolcat/LABS/styx/cmd/styxd docker-compose up -d # or the other docker command -./styx +./styxd + +# build the loader helper binary +go build gitlab.dcso.lolcat/LABS/styx/cmd/iocloader +# update the IOC list while the programm is already running +./iocloader ``` *Note*: if you have issues with the docker compose, make sure it runs on the same subnet. Check [this](https://serverfault.com/questions/916941/configuring-docker-to-not-use-the-172-17-0-0-range) for inspiration. ### Example configuration: + +*Note*: For Pastebin, you will have to authorise your IP address when you login through the web interface. + ``` certstream: activated: true @@ -62,12 +70,12 @@ shodan: ## Dgraph Interface -You can connect to the Dgraph interface at this default address: localhost:8000. +You can connect to the Dgraph interface at this default address: http://localhost:8000. There you would be able to run GraphQL+ queries, here to query a node. ```graphql query { - Node(func: eq(id, "node--cde8decb-0a8b-4d19-bd77-c2decb6dab9c")) { + Node(func: eq(uid, 0x23)) { uid ndata modified @@ -106,17 +114,6 @@ query { notBefore notAfter } - shodanNode { - uid - hostData { - product - ip - version - hostnames - port - html - } - } } } ``` @@ -158,7 +155,7 @@ query { Dgraph also supports full text search, so you can query things like: -``` +```graphql query { Node(func: allofterms(full, "code")) { uid @@ -190,8 +187,7 @@ each time you query something. ### Meta -Edges are not implemented yet. They will prove an existing relation between two -nodes of different origin. +Edges provide an existing relation between two nodes of different origin. They are part of Dgraph features. Node --[Edge]-- Node diff --git a/broker/main.go b/broker/main.go index 606d48b..49b3f3f 100644 --- a/broker/main.go +++ b/broker/main.go @@ -12,13 +12,33 @@ import ( "gitlab.dcso.lolcat/LABS/styx/models" ) +var ( + Topic string + Partition int + Protocol string + Host string + Port string +) + +func init() { + viper.SetConfigName("config") + viper.SetConfigType("yaml") + viper.AddConfigPath(".") + + err := viper.ReadInConfig() + if err != nil { + panic(err) + } + + Topic = viper.GetString("kafka.topic") + Partition = viper.GetInt("kafka.partition") + Protocol = viper.GetString("kafka.protocol") + Host = viper.GetString("kafka.host") + Port = viper.GetString("kafka.port") +} + // SetUpKafkaConnecter builds the connection to Kafka with a timeout. func SetUpKafkaConnecter() (*kafka.Conn, error) { - Topic := viper.GetString("kafka.topic") - Partition := viper.GetInt("kafka.partition") - Protocol := viper.GetString("kafka.protocol") - Host := viper.GetString("kafka.host") - Port := viper.GetString("kafka.port") conn, err := kafka.DialLeader(context.Background(), Protocol, Host+":"+Port, Topic, Partition) if err != nil { return nil, err @@ -28,11 +48,31 @@ func SetUpKafkaConnecter() (*kafka.Conn, error) { return conn, nil } +// CreateTopic creates a topic in the Kafka Broker based on a target. Don't +// forget to close them as they are channels. +func CreateTopic(conn *kafka.Conn, target string) *kafka.Reader { + r := kafka.NewReader(kafka.ReaderConfig{ + Brokers: []string{Host + ":" + Port}, + Topic: target, + Partition: Partition, + MinBytes: 10e3, + MaxBytes: 10e6, + }) + r.SetOffset(42) + + return r +} + +// CloseReader closes the Kafka reader. +func CloseReader(r kafka.Reader) { + r.Close() +} + // SendEventToKafka sends a node to the broker. func SendEventToKafka(conn *kafka.Conn, node models.Node) { conn.SetWriteDeadline(time.Now().Add(10 * time.Second)) packaged, _ := json.Marshal(node) - _, err := conn.WriteMessages(kafka.Message{Value: packaged}) + _, err := conn.WriteMessages(kafka.Message{Value: packaged, Topic: node.Type}) if err != nil { logrus.Error(err) } @@ -45,10 +85,6 @@ func ReadEventFromKafka() { return } - Topic := viper.GetString("kafka.topic") - Partition := viper.GetInt("kafka.partition") - Host := viper.GetString("kafka.host") - Port := viper.GetString("kafka.port") r := kafka.NewReader(kafka.ReaderConfig{ Brokers: []string{Host + ":" + Port}, Topic: Topic, diff --git a/cmd/iocloader/main.go b/cmd/iocloader/main.go new file mode 100644 index 0000000..395028d --- /dev/null +++ b/cmd/iocloader/main.go @@ -0,0 +1,117 @@ +package main + +import ( + "bufio" + "context" + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + "runtime" + "time" + + "github.com/dgraph-io/dgo/v2" + "github.com/dgraph-io/dgo/v2/protos/api" + "github.com/google/uuid" + "github.com/sirupsen/logrus" + "gitlab.dcso.lolcat/LABS/styx/graph" + "gitlab.dcso.lolcat/LABS/styx/models" +) + +var ( + _, b, _, _ = runtime.Caller(0) + basepath = filepath.Dir(b) +) + +// Result is the result from the matching query. Probably going to change. +type Result struct { + Result []models.Node `json:"Node,omitempty"` +} + +func loadTargets(graphClient *dgo.Dgraph) error { + path := basepath + "/../../matcher/data/" + + sliceDomain, err := ioutil.ReadDir(path) + if err != nil { + logrus.Warn("matcher#ReadDir#domains", err) + return err + } + + for _, file := range sliceDomain { + logrus.Info("loading: ", file.Name(), " please wait...") + f, err := os.OpenFile(path+file.Name(), 0, 0644) + if err != nil { + logrus.Warn("matcher#OpenFile#", err) + return err + } + scanner := bufio.NewScanner(f) + + for scanner.Scan() { + uuid := uuid.New().String() + t := time.Now() + rfc3339time := t.Format(time.RFC3339) + matcher := models.Match{ + ID: uuid, + Timestamp: rfc3339time, + Target: scanner.Text(), + Nodes: []models.Node{}, + Type: "matcher", + } + ctx := context.Background() + query := `query eq($a: string){ +Node(func: eq(target, $a)){ +uid +} + }` + + txn := graphClient.NewTxn() + ret, err := txn.QueryWithVars(ctx, query, map[string]string{"$a": scanner.Text()}) + if err != nil { + logrus.Warn(err) + } + + n := Result{} + json.Unmarshal([]byte(ret.Json), &n) + + // Check if the target already exists, if so, skipping not inserting + // the data + if len(n.Result) == 0 { + logrus.Info("new matcher, charging...") + mu := &api.Mutation{ + CommitNow: true, + } + + pb, err := json.Marshal(matcher) + if err != nil { + logrus.Error(err) + return err + } + + mu.SetJson = pb + + txn = graphClient.NewTxn() + defer txn.Discard(ctx) + + _, err = txn.Mutate(ctx, mu) + if err != nil { + logrus.Error(err) + return err + } + } + } + } + + return nil +} + +func main() { + logrus.Info("Initializing Dgraph for the importer...") + dgraphClient, err := graph.ConnectToDgraph(false) + if err != nil || dgraphClient == nil { + logrus.WithField("err", err).Error("error initialising the graph database") + } + logrus.Info("Loading data...") + loadTargets(dgraphClient) + logrus.Info("Done!") + +} diff --git a/main.go b/cmd/styxd/main.go similarity index 86% rename from main.go rename to cmd/styxd/main.go index 99dcafc..6140066 100644 --- a/main.go +++ b/cmd/styxd/main.go @@ -6,7 +6,6 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/viper" - "gitlab.dcso.lolcat/LABS/styx/broker" "gitlab.dcso.lolcat/LABS/styx/graph" "gitlab.dcso.lolcat/LABS/styx/matcher" "gitlab.dcso.lolcat/LABS/styx/plugins" @@ -32,15 +31,15 @@ func main() { os.Setenv("SHODAN_KEY", viper.GetString("shodan.key")) logrus.Info("Starting to get data from the Internet...") - logrus.Info("Initializing Kafka...") - _, err = broker.SetUpKafkaConnecter() - if err != nil { - logrus.WithField("err", err).Error("error initialising kafka") - } - logrus.Info("done") + // logrus.Info("Initializing Kafka...") + // _, err = broker.SetUpKafkaConnecter() + // if err != nil { + // logrus.WithField("err", err).Error("error initialising kafka") + // } + // logrus.Info("done") logrus.Info("Initializing Dgraph...") - dgraphClient, err := graph.ConnectToDgraph() + dgraphClient, err := graph.ConnectToDgraph(true) if err != nil { logrus.WithField("err", err).Error("error initialising the graph database") } diff --git a/docker-compose.yml b/docker-compose.yml index d80cde8..3137f3d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -23,34 +23,34 @@ services: ports: - 8000:8000 command: dgraph-ratel - zoo1: - image: zookeeper:3.4.9 - hostname: zoo1 - ports: - - "2181:2181" - environment: - ZOO_MY_ID: 1 - ZOO_PORT: 2181 - ZOO_SERVERS: server.1=zoo1:2888:3888 - volumes: - - ./zk-single-kafka-single/zoo1/data:/data - - ./zk-single-kafka-single/zoo1/datalog:/datalog + # zoo1: + # image: zookeeper:3.4.9 + # hostname: zoo1 + # ports: + # - "2181:2181" + # environment: + # ZOO_MY_ID: 1 + # ZOO_PORT: 2181 + # ZOO_SERVERS: server.1=zoo1:2888:3888 + # volumes: + # - ./zk-single-kafka-single/zoo1/data:/data + # - ./zk-single-kafka-single/zoo1/datalog:/datalog # https://github.com/simplesteph/kafka-stack-docker-compose/blob/master/zk-single-kafka-single.yml - kafka1: - image: confluentinc/cp-kafka:5.5.0 - hostname: kafka1 - ports: - - "9092:9092" - environment: - KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092 - KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT - KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL - KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" - KAFKA_BROKER_ID: 1 - KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" - KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 - volumes: - - ./zk-single-kafka-single/kafka1/data:/var/lib/kafka/data - depends_on: - - zoo1 + # kafka1: + # image: confluentinc/cp-kafka:5.5.0 + # hostname: kafka1 + # ports: + # - "9092:9092" + # environment: + # KAFKA_ADVERTISED_LISTENERS: LISTENER_DOCKER_INTERNAL://kafka1:19092,LISTENER_DOCKER_EXTERNAL://${DOCKER_HOST_IP:-127.0.0.1}:9092 + # KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: LISTENER_DOCKER_INTERNAL:PLAINTEXT,LISTENER_DOCKER_EXTERNAL:PLAINTEXT + # KAFKA_INTER_BROKER_LISTENER_NAME: LISTENER_DOCKER_INTERNAL + # KAFKA_ZOOKEEPER_CONNECT: "zoo1:2181" + # KAFKA_BROKER_ID: 1 + # KAFKA_LOG4J_LOGGERS: "kafka.controller=INFO,kafka.producer.async.DefaultEventHandler=INFO,state.change.logger=INFO" + # KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 + # volumes: + # - ./zk-single-kafka-single/kafka1/data:/var/lib/kafka/data + # depends_on: + # - zoo1 diff --git a/graph/main.go b/graph/main.go index afcbe3f..ebeaab0 100644 --- a/graph/main.go +++ b/graph/main.go @@ -8,7 +8,7 @@ import ( "google.golang.org/grpc" ) -func ConnectToDgraph() (*dgo.Dgraph, error) { +func ConnectToDgraph(setupSchema bool) (*dgo.Dgraph, error) { conn, err := grpc.Dial("localhost:9080", grpc.WithInsecure()) if err != nil { return nil, err @@ -16,9 +16,11 @@ func ConnectToDgraph() (*dgo.Dgraph, error) { dgraphClient := dgo.NewDgraphClient(api.NewDgraphClient(conn)) - err = setupDgraphSchema(dgraphClient) - if err != nil { - return nil, err + if setupSchema { + err = setupDgraphSchema(dgraphClient) + if err != nil { + return nil, err + } } return dgraphClient, nil diff --git a/main_test.go b/main_test.go index 3b7d16c..5848bd7 100644 --- a/main_test.go +++ b/main_test.go @@ -1,4 +1,4 @@ -package main +package styx import ( "os" diff --git a/matcher/main.go b/matcher/main.go index ac2b6ea..33dbeed 100644 --- a/matcher/main.go +++ b/matcher/main.go @@ -137,6 +137,7 @@ uid logrus.Error(err) return nil, err } + } } @@ -151,6 +152,7 @@ func (m *Matcher) Run(wg *sync.WaitGroup, graphClient *dgo.Dgraph) { logrus.Error(err) } logrus.Info("finished loading matcher targets") + fmt.Println(targets) if !m.Running { m.StoppedChan = make(chan bool) @@ -350,7 +352,6 @@ func runShodanMatcher(target string, graphClient *dgo.Dgraph) { } } ` - ctx := context.Background() txn := graphClient.NewTxn() res, err := txn.QueryWithVars(ctx, q, map[string]string{"$a": target})