From 02a014262aa5927dfdf29aaa75401cb6571ca519 Mon Sep 17 00:00:00 2001 From: Christopher Talib Date: Fri, 14 Feb 2020 11:36:47 +0100 Subject: [PATCH] Adding Akamai filters for data recieved in Shodan + allow deactivating Balboa search --- broker/main.go | 32 +- filters/akamai.cidr | 1923 +++++++++++++++++++++++++++++++++++++++++ filters/akamai.go | 54 ++ filters/akamaiv6.cidr | 357 ++++++++ main.go | 186 ++-- parser/main.go | 52 -- utils/files.go | 1 + 7 files changed, 2460 insertions(+), 145 deletions(-) create mode 100644 filters/akamai.cidr create mode 100644 filters/akamai.go create mode 100644 filters/akamaiv6.cidr diff --git a/broker/main.go b/broker/main.go index 4e8bbf4..00a3a20 100644 --- a/broker/main.go +++ b/broker/main.go @@ -41,6 +41,10 @@ func SendEventToKafka(conn *kafka.Conn, node models.Node) { // ReadEventFromKafka read the event sent to Kafka and acts upon it. func ReadEventFromKafka() { + if !viper.GetBool("kafka.activated") { + return + } + Topic := viper.GetString("kafka.topic") Partition := viper.GetInt("kafka.partition") Host := viper.GetString("kafka.host") @@ -67,20 +71,22 @@ func ReadEventFromKafka() { var node models.Node json.Unmarshal(m.Value, &node) - if len(node.ID) != 0 { - // TODO: refactor this context - ctx := context.Background() - entries, err := c.GetAllEntries(ctx, node.Data, "", "", int32(1)) - if err != nil { - logrus.Error("error from balboa", err) - } - if len(entries) != 0 { - balboaNode := models.BuildBalboaNode(entries) - models.SaveBalboaNode("bnodes.json", balboaNode) - edge := models.BuildEdge("balboa", node.ID, balboaNode.ID) - models.SaveEdge(edge) - } + if viper.GetBool("balboa.activated") { + if len(node.ID) != 0 { + // TODO: refactor this context + ctx := context.Background() + entries, err := c.GetAllEntries(ctx, node.Data, "", "", int32(1)) + if err != nil { + logrus.Error("error from balboa", err) + } + if len(entries) != 0 { + balboaNode := models.BuildBalboaNode(entries) + models.SaveBalboaNode("bnodes.json", balboaNode) + edge := models.BuildEdge("balboa", node.ID, balboaNode.ID) + models.SaveEdge(edge) + } + } } } } diff --git a/filters/akamai.cidr b/filters/akamai.cidr new file mode 100644 index 0000000..987726b --- /dev/null +++ b/filters/akamai.cidr @@ -0,0 +1,1923 @@ +209.200.128.0/18 +23.192.0.0/11 +104.64.0.0/10 +81.23.243.0/24 +80.231.197.217 +80.231.197.224 +77.67.20.0/25 +77.67.27.0/24 +88.221.0.0/16 +80.239.216.0/24 +194.25.94.128/25 +62.156.209.0/24 +195.10.51.0/24 +217.89.105.128/25 +95.100.64.0/20 +81.52.207.128/25 +2.20.181.0/24 +96.6.0.0/15 +62.41.62.8 +62.41.62.27 +2.19.240.0/20 +188.111.53.0/24 +66.171.231.15 +66.171.231.62 +23.14.94.0/23 +84.53.175.0/24 +84.53.172.0/24 +95.100.48.0/20 +80.157.170.0/24 +80.67.84.0/22 +80.67.88.0/21 +80.239.178.128/25 +62.154.232.0/24 +23.62.98.0/23 +23.32.0.0/11 +23.64.0.0/14 +95.100.96.0/23 +82.199.80.141 +92.122.0.0/15 +92.123.68.0/22 +95.101.0.0/22 +88.221.92.0/23 +62.159.74.0/24 +194.25.95.0/24 +80.150.142.0/25 +80.150.192.64/26 +80.150.191.128/25 +195.145.147.0/24 +62.156.238.0/24 +195.12.225.128/25 +195.95.193.0/24 +80.157.150.0/24 +193.45.10.128/25 +84.53.138.0/23 +188.111.53.57 +188.111.53.58 +195.27.183.128/25 +2.16.216.0/22 +79.140.94.0/23 +80.154.79.0/24 +23.209.164.223 +195.50.128.0/18 +2.16.0.0/13 +195.10.18.0/26 +80.150.193.0/24 +213.155.152.128/25 +80.157.151.0/25 +194.221.65.0/25 +208.31.170.0/24 +46.33.64.0/21 +46.33.72.0/22 +46.33.76.0/23 +64.145.86.0/25 +104.64.144.0/20 +104.64.16.0/20 +104.64.208.0/20 +104.64.224.0/20 +104.64.32.0/20 +104.64.48.0/20 +104.64.64.0/20 +104.64.80.0/20 +104.64.96.0/20 +104.65.0.0/20 +104.65.16.0/20 +104.65.32.0/20 +104.65.48.0/20 +104.66.224.0/20 +104.66.240.0/20 +104.67.0.0/20 +104.67.16.0/20 +104.67.224.0/20 +104.67.240.0/20 +104.67.32.0/20 +104.67.48.0/20 +104.67.64.0/20 +104.67.80.0/20 +104.68.0.0/20 +104.68.112.0/20 +104.68.16.0/20 +104.68.160.0/20 +104.68.176.0/20 +104.68.192.0/20 +104.68.208.0/20 +104.68.96.0/20 +104.69.112.0/20 +104.69.128.0/20 +104.69.144.0/20 +104.69.160.0/20 +104.69.176.0/20 +104.69.192.0/20 +104.69.208.0/22 +104.69.212.0/22 +104.69.48.0/20 +104.69.64.0/20 +104.69.80.0/20 +104.69.96.0/20 +104.70.116.0/22 +104.70.120.0/23 +104.70.122.0/23 +104.70.124.0/22 +104.70.128.0/20 +104.70.144.0/20 +104.70.16.0/20 +104.70.160.0/20 +104.70.176.0/20 +104.70.230.0/23 +104.70.232.0/22 +104.70.236.0/22 +104.70.240.0/20 +104.70.32.0/20 +104.71.0.0/20 +104.71.128.0/23 +104.71.130.0/24 +104.71.132.0/23 +104.71.134.0/23 +104.71.136.0/22 +104.71.140.0/23 +104.71.142.0/23 +104.71.144.0/20 +104.71.16.0/20 +104.71.160.0/20 +104.71.176.0/20 +104.71.192.0/20 +104.71.208.0/22 +104.71.212.0/22 +104.71.216.0/23 +104.71.224.0/20 +104.71.240.0/20 +104.71.32.0/20 +104.71.48.0/23 +104.71.62.0/23 +118.214.160.0/24 +118.214.161.0/24 +118.214.240.0/20 +118.215.176.0/20 +118.215.64.0/21 +125.56.144.0/21 +125.56.200.0/24 +125.56.201.0/24 +125.56.218.0/24 +125.56.219.0/24 +125.56.220.0/23 +125.56.224.0/20 +172.224.120.0/23 +172.224.172.0/22 +172.224.240.0/20 +172.225.0.0/20 +172.225.168.0/23 +172.225.170.0/23 +172.225.220.0/22 +172.226.216.0/23 +172.226.218.0/23 +172.226.220.0/23 +172.226.32.0/20 +172.226.48.0/20 +172.226.64.0/19 +172.226.96.0/22 +172.227.112.0/20 +172.227.12.0/22 +172.227.128.0/20 +172.227.144.0/22 +172.227.148.0/22 +172.227.160.0/19 +172.227.80.0/20 +172.227.96.0/20 +172.228.128.0/20 +172.228.144.0/20 +172.228.4.0/23 +172.228.52.0/22 +172.228.64.0/20 +172.228.80.0/20 +172.229.104.0/22 +172.229.108.0/22 +172.229.144.0/20 +172.229.160.0/20 +172.229.176.0/20 +172.229.192.0/20 +172.229.54.0/23 +172.229.56.0/23 +172.230.100.0/23 +172.230.102.0/23 +172.230.104.0/23 +172.230.148.0/22 +172.230.152.0/22 +172.230.156.0/22 +172.230.160.0/19 +172.230.192.0/20 +172.230.208.0/20 +172.230.224.0/19 +172.231.112.0/20 +172.231.128.0/20 +172.231.144.0/23 +172.231.150.0/23 +172.231.200.0/22 +172.231.240.0/20 +172.231.44.0/22 +172.232.0.0/20 +172.232.196.0/23 +172.232.244.0/22 +172.233.0.0/20 +172.233.16.0/20 +173.222.0.0/20 +173.222.112.0/22 +173.222.124.0/22 +173.222.128.0/22 +173.222.133.0/24 +173.222.134.0/23 +173.222.136.0/22 +173.222.148.0/22 +173.222.156.0/22 +173.222.184.0/22 +173.222.200.0/22 +173.222.208.0/23 +173.222.210.0/23 +173.222.220.0/22 +173.222.248.0/23 +173.222.250.0/23 +173.222.64.0/20 +173.223.10.0/23 +173.223.128.0/23 +173.223.144.0/21 +173.223.160.0/22 +173.223.164.0/22 +173.223.176.0/20 +173.223.20.0/23 +173.223.204.0/23 +173.223.206.0/24 +173.223.207.0/24 +173.223.208.0/20 +173.223.227.0/24 +173.223.235.0/24 +173.223.238.0/23 +173.223.24.0/22 +173.223.240.0/20 +173.223.28.0/22 +173.223.32.0/22 +173.223.4.0/22 +173.223.53.0/24 +173.223.55.0/24 +184.24.128.0/20 +184.24.148.0/23 +184.24.192.0/20 +184.25.0.0/20 +184.25.100.0/24 +184.25.101.0/24 +184.25.102.0/24 +184.25.107.0/24 +184.25.120.0/23 +184.25.144.0/24 +184.25.145.0/24 +184.25.157.0/24 +184.25.158.0/23 +184.25.16.0/20 +184.25.184.0/22 +184.25.192.0/22 +184.25.216.0/22 +184.25.248.0/22 +184.25.255.0/24 +184.25.56.0/23 +184.25.63.0/24 +184.25.80.0/20 +184.26.100.0/22 +184.26.136.0/24 +184.26.138.0/24 +184.26.139.0/24 +184.26.142.0/23 +184.26.156.0/22 +184.26.156.0/24 +184.26.160.0/24 +184.26.161.0/24 +184.26.163.0/24 +184.26.164.0/24 +184.26.166.0/24 +184.26.167.0/24 +184.26.184.0/22 +184.26.224.0/20 +184.26.24.0/22 +184.26.28.0/23 +184.26.38.0/24 +184.26.39.0/24 +184.26.44.0/24 +184.26.45.0/24 +184.26.60.0/23 +184.26.62.0/23 +184.26.64.0/20 +184.26.90.0/24 +184.26.93.0/24 +184.27.118.0/24 +184.27.119.0/24 +184.27.120.0/24 +184.27.121.0/24 +184.27.132.0/23 +184.27.136.0/22 +184.27.142.0/23 +184.27.144.0/20 +184.27.160.0/20 +184.27.176.0/23 +184.27.178.0/23 +184.27.186.0/23 +184.27.190.0/23 +184.27.195.0/24 +184.27.200.0/24 +184.27.204.0/24 +184.27.205.0/24 +184.27.206.0/23 +184.27.210.0/23 +184.27.24.0/22 +184.27.32.0/22 +184.27.45.0/24 +184.27.46.0/24 +184.27.48.0/20 +184.28.10.0/23 +184.28.100.0/24 +184.28.101.0/24 +184.28.111.0/24 +184.28.115.0/24 +184.28.119.0/24 +184.28.12.0/22 +184.28.120.0/24 +184.28.121.0/24 +184.28.124.0/24 +184.28.125.0/24 +184.28.126.0/24 +184.28.127.0/24 +184.28.128.0/24 +184.28.129.0/24 +184.28.130.0/23 +184.28.132.0/22 +184.28.148.0/22 +184.28.152.0/23 +184.28.158.0/23 +184.28.17.0/24 +184.28.170.0/23 +184.28.180.0/23 +184.28.184.0/24 +184.28.185.0/24 +184.28.188.0/24 +184.28.189.0/24 +184.28.190.0/23 +184.28.204.0/24 +184.28.205.0/24 +184.28.214.0/24 +184.28.215.0/24 +184.28.219.0/24 +184.28.220.0/23 +184.28.222.0/24 +184.28.223.0/24 +184.28.224.0/23 +184.28.23.0/24 +184.28.240.0/22 +184.28.250.0/23 +184.28.252.0/22 +184.28.30.0/23 +184.28.34.0/23 +184.28.38.0/23 +184.28.40.0/23 +184.28.52.0/23 +184.28.54.0/23 +184.28.56.0/23 +184.28.58.0/23 +184.28.60.0/23 +184.28.76.0/23 +184.28.82.0/24 +184.28.83.0/24 +184.28.88.0/22 +184.28.92.0/23 +184.28.94.0/23 +184.28.96.0/23 +184.29.104.0/22 +184.29.14.0/24 +184.29.15.0/24 +184.29.160.0/20 +184.29.176.0/20 +184.29.21.0/24 +184.29.24.0/22 +184.29.34.0/24 +184.29.35.0/24 +184.29.38.0/24 +184.29.39.0/24 +184.29.42.0/24 +184.29.43.0/24 +184.29.44.0/22 +184.29.54.0/23 +184.29.64.0/22 +184.29.84.0/22 +184.30.112.0/20 +184.30.160.0/20 +184.30.192.0/20 +184.30.208.0/20 +184.30.240.0/20 +184.30.64.0/20 +184.30.80.0/20 +184.30.96.0/20 +184.31.128.0/20 +184.31.80.0/20 +184.50.160.0/20 +184.50.192.0/20 +184.50.228.0/23 +184.50.238.0/23 +184.50.240.0/20 +184.50.26.0/24 +184.50.27.0/24 +184.50.32.0/20 +184.50.48.0/20 +184.50.80.0/22 +184.50.85.0/24 +184.50.86.0/24 +184.50.87.0/24 +184.50.88.0/24 +184.50.89.0/24 +184.50.90.0/23 +184.50.92.0/23 +184.50.96.0/20 +184.51.0.0/23 +184.51.100.0/24 +184.51.101.0/24 +184.51.108.0/24 +184.51.110.0/24 +184.51.111.0/24 +184.51.112.0/22 +184.51.112.0/23 +184.51.114.0/23 +184.51.120.0/24 +184.51.125.0/24 +184.51.126.0/24 +184.51.128.0/23 +184.51.13.0/24 +184.51.130.0/23 +184.51.140.0/22 +184.51.144.0/23 +184.51.146.0/24 +184.51.147.0/24 +184.51.15.0/24 +184.51.150.0/23 +184.51.154.0/24 +184.51.156.0/24 +184.51.158.0/24 +184.51.16.0/22 +184.51.176.0/20 +184.51.196.0/23 +184.51.198.0/23 +184.51.20.0/22 +184.51.200.0/24 +184.51.202.0/24 +184.51.206.0/24 +184.51.208.0/20 +184.51.252.0/23 +184.51.254.0/23 +184.51.28.0/22 +184.51.34.0/23 +184.51.4.0/24 +184.51.56.0/23 +184.51.58.0/23 +184.51.60.0/23 +184.51.62.0/23 +184.51.64.0/22 +184.51.7.0/24 +184.51.76.0/22 +184.51.88.0/22 +184.51.92.0/22 +184.84.177.0/24 +184.84.210.0/24 +184.84.211.0/24 +184.84.220.0/24 +184.84.228.0/22 +184.84.234.0/23 +184.84.238.0/24 +184.84.239.0/24 +184.84.245.0/24 +184.84.248.0/23 +184.84.254.0/23 +184.84.80.0/20 +184.85.128.0/20 +184.85.223.0/24 +184.85.228.0/22 +184.85.244.0/22 +184.85.249.0/24 +184.85.250.0/23 +184.85.250.0/24 +184.85.251.0/24 +184.85.253.0/24 +184.85.32.0/20 +184.85.80.0/20 +184.86.0.0/20 +184.86.100.0/23 +184.86.104.0/22 +184.86.108.0/22 +184.86.176.0/20 +184.86.208.0/23 +184.86.220.0/24 +184.86.222.0/24 +184.86.224.0/22 +184.86.240.0/22 +184.86.250.0/24 +184.86.32.0/20 +184.86.48.0/20 +184.86.88.0/22 +184.86.96.0/22 +184.87.132.0/22 +184.87.140.0/23 +184.87.142.0/24 +184.87.143.0/24 +184.87.164.0/22 +184.87.174.0/23 +184.87.176.0/20 +184.87.194.0/23 +184.87.195.0/24 +184.87.206.0/23 +184.87.208.0/23 +184.87.216.0/22 +184.87.220.0/23 +184.87.222.0/23 +184.87.224.0/20 +193.108.152.0/24 +193.108.153.0/24 +193.108.88.0/24 +193.108.89.0/24 +193.108.94.0/23 +193.108.94.0/24 +195.245.125.0/24 +195.245.127.0/24 +195.95.193.0/24 +195.95.194.0/24 +195.95.195.0/24 +2.16.0.0/23 +2.16.10.0/23 +2.16.114.0/23 +2.16.116.0/23 +2.16.117.0/24 +2.16.127.0/24 +2.16.128.0/23 +2.16.130.0/24 +2.16.153.0/24 +2.16.156.0/23 +2.16.159.0/24 +2.16.160.0/23 +2.16.162.0/24 +2.16.163.0/24 +2.16.168.0/24 +2.16.180.0/24 +2.16.192.0/20 +2.16.216.0/22 +2.16.224.0/20 +2.16.28.0/23 +2.16.30.0/23 +2.16.32.0/23 +2.16.34.0/23 +2.16.36.0/23 +2.16.38.0/23 +2.16.46.0/23 +2.16.48.0/23 +2.16.52.0/24 +2.16.56.0/23 +2.16.58.0/23 +2.16.6.0/23 +2.16.60.0/24 +2.16.61.0/24 +2.16.62.0/24 +2.16.63.0/24 +2.16.64.0/23 +2.16.66.0/23 +2.16.86.0/23 +2.16.94.0/23 +2.17.160.0/20 +2.17.176.0/20 +2.17.192.0/22 +2.17.208.0/20 +2.17.224.0/20 +2.17.24.0/22 +2.17.28.0/22 +2.17.32.0/20 +2.17.48.0/20 +2.18.0.0/20 +2.18.112.0/20 +2.18.128.0/20 +2.18.144.0/20 +2.18.16.0/20 +2.18.160.0/20 +2.18.208.0/22 +2.18.212.0/22 +2.18.220.0/22 +2.18.224.0/22 +2.18.228.0/22 +2.18.232.0/22 +2.18.240.0/24 +2.18.244.0/22 +2.18.248.0/24 +2.18.250.0/23 +2.18.252.0/22 +2.18.32.0/20 +2.18.48.0/22 +2.18.64.0/20 +2.18.96.0/20 +2.19.112.0/20 +2.19.128.0/20 +2.19.144.0/20 +2.19.176.0/20 +2.19.240.0/20 +2.19.80.0/20 +2.20.112.0/20 +2.20.132.0/24 +2.20.133.0/24 +2.20.134.0/23 +2.20.136.0/23 +2.20.138.0/23 +2.20.142.0/23 +2.20.178.0/24 +2.20.180.0/24 +2.20.181.0/24 +2.20.182.0/24 +2.20.183.0/24 +2.20.186.0/23 +2.20.208.0/20 +2.20.24.0/22 +2.20.243.0/24 +2.20.244.0/23 +2.20.32.0/22 +2.20.36.0/22 +2.20.40.0/22 +2.20.44.0/24 +2.20.45.0/24 +2.20.60.0/22 +2.20.64.0/20 +2.20.80.0/20 +2.20.96.0/20 +2.21.112.0/20 +2.21.208.0/20 +2.21.240.0/24 +2.21.242.0/24 +2.21.243.0/24 +2.21.246.0/24 +2.21.64.0/20 +2.21.80.0/22 +2.21.84.0/23 +2.21.88.0/22 +2.21.92.0/23 +2.21.96.0/20 +2.22.160.0/20 +2.22.176.0/20 +2.22.192.0/20 +2.22.208.0/20 +2.22.234.0/24 +2.22.237.0/24 +2.22.254.0/23 +2.22.32.0/22 +2.22.4.0/22 +2.22.48.0/24 +2.22.49.0/24 +2.22.51.0/24 +2.22.52.0/24 +2.22.55.0/24 +2.22.56.0/22 +2.22.61.0/24 +23.0.128.0/20 +23.0.160.0/24 +23.0.161.0/24 +23.0.162.0/24 +23.0.163.0/24 +23.0.168.0/22 +23.0.172.0/23 +23.0.174.0/24 +23.0.176.0/20 +23.0.196.0/22 +23.0.212.0/22 +23.0.216.0/22 +23.0.220.0/22 +23.0.230.0/23 +23.0.234.0/23 +23.0.236.0/23 +23.0.250.0/23 +23.0.252.0/23 +23.0.32.0/20 +23.0.48.0/20 +23.0.80.0/20 +23.1.112.0/20 +23.1.20.0/22 +23.1.64.0/20 +23.10.112.0/20 +23.10.192.0/20 +23.10.240.0/24 +23.10.250.0/24 +23.10.251.0/24 +23.10.252.0/24 +23.10.253.0/24 +23.10.48.0/20 +23.10.64.0/20 +23.10.80.0/20 +23.10.96.0/20 +23.11.48.0/20 +23.12.192.0/20 +23.12.208.0/20 +23.12.240.0/20 +23.12.33.0/24 +23.12.41.0/24 +23.12.96.0/20 +23.13.144.0/20 +23.13.160.0/20 +23.13.176.0/22 +23.13.180.0/23 +23.13.182.0/23 +23.13.184.0/21 +23.13.208.0/22 +23.13.220.0/22 +23.13.224.0/20 +23.13.48.0/20 +23.13.64.0/20 +23.14.128.0/24 +23.14.130.0/23 +23.14.136.0/22 +23.14.144.0/20 +23.14.160.0/20 +23.14.176.0/20 +23.14.52.0/22 +23.14.56.0/21 +23.14.84.0/23 +23.14.88.0/23 +23.14.90.0/23 +23.14.90.0/24 +23.14.91.0/24 +23.14.92.0/22 +23.15.128.0/23 +23.15.132.0/23 +23.15.135.0/24 +23.15.15.0/24 +23.15.180.0/22 +23.15.244.0/24 +23.15.252.0/23 +23.15.254.0/23 +23.15.4.0/24 +23.15.48.0/20 +23.15.8.0/23 +23.192.116.0/23 +23.192.162.0/23 +23.192.165.0/24 +23.193.16.0/20 +23.193.162.0/23 +23.193.164.0/23 +23.193.172.0/23 +23.193.176.0/20 +23.193.192.0/20 +23.193.208.0/22 +23.193.212.0/22 +23.193.224.0/20 +23.193.240.0/20 +23.193.32.0/20 +23.194.100.0/22 +23.194.112.0/20 +23.194.128.0/20 +23.194.210.0/23 +23.194.214.0/23 +23.194.224.0/20 +23.194.240.0/20 +23.195.0.0/22 +23.195.112.0/20 +23.195.128.0/20 +23.195.16.0/20 +23.195.224.0/20 +23.195.240.0/20 +23.195.32.0/20 +23.195.48.0/20 +23.195.64.0/20 +23.195.8.0/22 +23.196.0.0/23 +23.196.112.0/20 +23.196.128.0/20 +23.196.144.0/20 +23.196.230.0/23 +23.196.232.0/22 +23.196.236.0/22 +23.196.240.0/20 +23.196.56.0/22 +23.196.6.0/23 +23.196.60.0/22 +23.196.8.0/23 +23.196.96.0/20 +23.197.0.0/20 +23.197.16.0/20 +23.197.244.0/22 +23.197.32.0/20 +23.197.48.0/23 +23.198.0.0/20 +23.198.100.0/22 +23.198.104.0/23 +23.198.106.0/23 +23.198.108.0/22 +23.198.112.0/20 +23.198.128.0/20 +23.198.144.0/20 +23.198.16.0/20 +23.198.160.0/20 +23.198.99.0/24 +23.199.192.0/20 +23.199.208.0/20 +23.199.224.0/22 +23.2.112.0/20 +23.2.18.0/23 +23.2.192.0/20 +23.2.208.0/20 +23.2.224.0/20 +23.2.24.0/23 +23.2.26.0/23 +23.2.30.0/23 +23.2.34.0/23 +23.2.52.0/22 +23.2.56.0/22 +23.2.64.0/22 +23.2.68.0/22 +23.2.76.0/22 +23.2.92.0/22 +23.200.84.0/24 +23.200.86.0/23 +23.200.96.0/20 +23.201.100.0/24 +23.201.101.0/24 +23.201.102.0/24 +23.201.103.0/24 +23.201.108.0/23 +23.201.110.0/23 +23.201.152.0/22 +23.201.192.0/20 +23.201.208.0/20 +23.201.98.0/24 +23.201.99.0/24 +23.202.112.0/22 +23.202.160.0/20 +23.202.230.0/23 +23.202.232.0/22 +23.202.240.0/20 +23.202.44.0/24 +23.202.46.0/23 +23.202.80.0/20 +23.202.96.0/20 +23.203.0.0/20 +23.203.104.0/22 +23.203.108.0/22 +23.203.144.0/20 +23.203.160.0/20 +23.203.176.0/20 +23.203.192.0/20 +23.203.208.0/20 +23.203.224.0/20 +23.203.240.0/22 +23.203.244.0/22 +23.203.54.0/23 +23.203.56.0/23 +23.203.58.0/23 +23.203.60.0/23 +23.204.0.0/20 +23.204.106.0/23 +23.204.108.0/24 +23.204.109.0/24 +23.204.110.0/23 +23.204.16.0/20 +23.205.0.0/19 +23.205.113.0/24 +23.205.114.0/24 +23.205.119.0/24 +23.205.168.0/22 +23.205.172.0/23 +23.205.192.0/20 +23.205.208.0/22 +23.205.212.0/22 +23.205.220.0/23 +23.205.222.0/23 +23.205.32.0/22 +23.205.36.0/22 +23.205.48.0/20 +23.206.112.0/22 +23.206.116.0/23 +23.206.16.0/20 +23.206.194.0/23 +23.206.196.0/23 +23.206.198.0/23 +23.206.200.0/23 +23.206.208.0/20 +23.206.224.0/20 +23.206.240.0/22 +23.206.244.0/22 +23.206.248.0/22 +23.206.252.0/22 +23.206.32.0/20 +23.206.4.0/22 +23.206.8.0/23 +23.206.80.0/20 +23.206.96.0/20 +23.207.0.0/20 +23.207.112.0/20 +23.207.16.0/20 +23.207.32.0/20 +23.207.48.0/20 +23.207.96.0/20 +23.208.18.0/23 +23.208.20.0/23 +23.208.216.0/22 +23.208.22.0/23 +23.208.220.0/23 +23.208.222.0/23 +23.209.0.0/20 +23.209.112.0/20 +23.209.128.0/20 +23.209.144.0/20 +23.209.16.0/20 +23.209.160.0/20 +23.209.176.0/22 +23.209.180.0/23 +23.209.182.0/24 +23.209.184.0/22 +23.209.192.0/20 +23.209.208.0/20 +23.209.32.0/20 +23.209.48.0/20 +23.209.64.0/22 +23.209.68.0/22 +23.209.72.0/23 +23.209.74.0/23 +23.209.76.0/22 +23.209.80.0/20 +23.209.96.0/20 +23.210.128.0/20 +23.210.144.0/20 +23.210.88.0/22 +23.210.92.0/23 +23.211.132.0/24 +23.211.133.0/24 +23.211.176.0/20 +23.211.228.0/22 +23.211.232.0/23 +23.211.234.0/23 +23.211.236.0/22 +23.211.240.0/20 +23.211.61.0/24 +23.212.0.0/20 +23.212.108.0/23 +23.212.16.0/20 +23.212.192.0/20 +23.212.32.0/20 +23.212.50.0/24 +23.212.53.0/24 +23.212.56.0/24 +23.212.57.0/24 +23.212.60.0/24 +23.212.62.0/24 +23.212.63.0/24 +23.214.76.0/22 +23.215.10.0/24 +23.215.11.0/24 +23.215.15.0/24 +23.215.215.0/24 +23.215.4.0/22 +23.215.60.0/23 +23.215.8.0/24 +23.215.9.0/24 +23.216.10.0/23 +23.216.112.0/20 +23.216.56.0/22 +23.216.6.0/23 +23.216.64.0/20 +23.216.8.0/23 +23.216.96.0/20 +23.217.136.0/24 +23.217.142.0/23 +23.217.188.0/22 +23.218.0.0/20 +23.218.112.0/20 +23.218.128.0/20 +23.218.144.0/22 +23.218.148.0/23 +23.218.156.0/23 +23.218.16.0/20 +23.218.32.0/22 +23.218.36.0/22 +23.218.40.0/23 +23.218.42.0/23 +23.218.44.0/22 +23.218.48.0/20 +23.218.64.0/20 +23.218.80.0/20 +23.218.96.0/20 +23.219.173.0/24 +23.220.100.0/23 +23.220.102.0/23 +23.220.104.0/22 +23.220.108.0/23 +23.220.110.0/23 +23.220.112.0/20 +23.220.128.0/20 +23.220.148.0/24 +23.220.150.0/23 +23.220.152.0/24 +23.220.153.0/24 +23.220.154.0/23 +23.220.156.0/23 +23.220.158.0/23 +23.220.196.0/22 +23.220.200.0/23 +23.220.202.0/24 +23.220.206.0/23 +23.220.208.0/20 +23.220.224.0/20 +23.220.240.0/20 +23.221.0.0/20 +23.221.112.0/20 +23.221.16.0/22 +23.221.20.0/23 +23.221.32.0/20 +23.221.48.0/20 +23.221.64.0/20 +23.221.96.0/20 +23.222.112.0/20 +23.222.128.0/20 +23.222.144.0/20 +23.222.160.0/20 +23.222.18.0/23 +23.222.20.0/23 +23.222.22.0/23 +23.222.24.0/22 +23.222.28.0/23 +23.222.30.0/23 +23.222.32.0/20 +23.222.48.0/20 +23.222.64.0/22 +23.222.68.0/22 +23.222.72.0/22 +23.222.80.0/20 +23.222.96.0/20 +23.223.112.0/20 +23.223.128.0/20 +23.223.16.0/20 +23.223.208.0/20 +23.223.224.0/20 +23.223.240.0/20 +23.223.32.0/20 +23.223.48.0/20 +23.223.64.0/20 +23.223.80.0/20 +23.223.96.0/20 +23.3.0.0/23 +23.3.100.0/23 +23.3.102.0/23 +23.3.104.0/24 +23.3.11.0/24 +23.3.13.0/24 +23.3.32.0/20 +23.3.6.0/23 +23.3.64.0/22 +23.3.72.0/24 +23.3.73.0/24 +23.3.74.0/24 +23.3.8.0/24 +23.3.80.0/22 +23.3.84.0/22 +23.3.88.0/23 +23.3.90.0/23 +23.3.92.0/22 +23.3.96.0/23 +23.3.98.0/24 +23.3.99.0/24 +23.32.112.0/20 +23.32.12.0/23 +23.32.128.0/20 +23.32.148.0/22 +23.32.192.0/20 +23.32.208.0/22 +23.32.212.0/22 +23.32.22.0/23 +23.32.240.0/24 +23.32.241.0/24 +23.32.242.0/23 +23.32.244.0/22 +23.32.248.0/24 +23.32.249.0/24 +23.32.250.0/23 +23.32.252.0/22 +23.32.31.0/24 +23.32.90.0/23 +23.32.92.0/22 +23.33.160.0/20 +23.33.248.0/22 +23.33.36.0/22 +23.33.52.0/22 +23.33.60.0/22 +23.34.176.0/20 +23.34.32.0/20 +23.34.48.0/22 +23.34.52.0/22 +23.34.56.0/23 +23.34.80.0/20 +23.34.96.0/20 +23.35.192.0/20 +23.36.0.0/20 +23.36.96.0/20 +23.37.0.0/20 +23.37.16.0/20 +23.37.224.0/20 +23.37.64.0/20 +23.37.80.0/20 +23.38.102.0/23 +23.38.104.0/22 +23.38.114.0/23 +23.38.116.0/23 +23.38.120.0/22 +23.38.124.0/22 +23.38.128.0/20 +23.38.144.0/20 +23.38.176.0/20 +23.38.224.0/20 +23.38.240.0/20 +23.39.0.0/20 +23.39.112.0/20 +23.39.16.0/20 +23.39.192.0/20 +23.39.240.0/20 +23.39.64.0/20 +23.4.0.0/20 +23.4.112.0/20 +23.4.16.0/20 +23.4.80.0/23 +23.4.82.0/23 +23.4.84.0/24 +23.4.88.0/22 +23.4.93.0/24 +23.4.94.0/23 +23.40.0.0/20 +23.40.32.0/20 +23.40.64.0/20 +23.40.80.0/20 +23.41.112.0/20 +23.41.172.0/22 +23.41.48.0/20 +23.41.80.0/20 +23.42.128.0/20 +23.42.144.0/20 +23.42.160.0/20 +23.42.176.0/20 +23.42.208.0/20 +23.42.32.0/20 +23.42.96.0/20 +23.43.164.0/23 +23.43.166.0/23 +23.43.172.0/23 +23.43.200.0/22 +23.43.48.0/20 +23.44.112.0/20 +23.44.128.0/20 +23.44.176.0/20 +23.44.96.0/20 +23.45.236.0/22 +23.45.32.0/20 +23.45.67.0/24 +23.45.77.0/24 +23.45.84.0/22 +23.45.96.0/20 +23.46.10.0/23 +23.46.128.0/20 +23.46.2.0/23 +23.46.26.0/23 +23.46.4.0/23 +23.46.6.0/23 +23.47.128.0/20 +23.47.208.0/20 +23.48.144.0/22 +23.48.160.0/20 +23.48.86.0/23 +23.49.12.0/22 +23.49.176.0/20 +23.49.32.0/20 +23.49.57.0/24 +23.49.60.0/23 +23.49.62.0/23 +23.5.144.0/20 +23.5.160.0/24 +23.5.161.0/24 +23.5.162.0/23 +23.5.164.0/24 +23.5.168.0/23 +23.5.170.0/23 +23.5.192.0/20 +23.5.96.0/20 +23.50.16.0/20 +23.50.48.0/20 +23.51.80.0/20 +23.52.32.0/22 +23.52.40.0/21 +23.54.160.0/20 +23.54.208.0/20 +23.54.32.0/20 +23.55.112.0/20 +23.56.112.0/20 +23.56.176.0/20 +23.56.192.0/20 +23.56.208.0/23 +23.56.210.0/23 +23.56.214.0/23 +23.56.216.0/22 +23.56.224.0/20 +23.56.64.0/20 +23.56.80.0/20 +23.57.0.0/20 +23.57.116.0/22 +23.57.124.0/22 +23.57.128.0/20 +23.57.144.0/20 +23.57.192.0/20 +23.57.208.0/20 +23.57.224.0/20 +23.57.240.0/20 +23.57.70.0/23 +23.57.78.0/23 +23.58.0.0/22 +23.58.12.0/22 +23.58.152.0/22 +23.58.16.0/20 +23.58.192.0/20 +23.58.32.0/20 +23.58.4.0/23 +23.58.48.0/20 +23.58.6.0/23 +23.58.64.0/20 +23.58.8.0/22 +23.58.80.0/23 +23.58.89.0/24 +23.58.90.0/23 +23.59.0.0/20 +23.59.112.0/20 +23.59.128.0/20 +23.59.144.0/20 +23.59.16.0/20 +23.59.160.0/20 +23.59.176.0/22 +23.59.180.0/23 +23.59.182.0/23 +23.59.184.0/22 +23.59.192.0/20 +23.59.240.0/20 +23.59.84.0/22 +23.59.88.0/23 +23.59.90.0/23 +23.59.92.0/22 +23.59.96.0/20 +23.6.16.0/20 +23.6.208.0/20 +23.6.64.0/20 +23.6.96.0/20 +23.60.0.0/20 +23.60.148.0/22 +23.60.16.0/20 +23.60.208.0/20 +23.60.32.0/20 +23.60.48.0/20 +23.60.64.0/22 +23.60.69.0/24 +23.60.70.0/23 +23.61.112.0/22 +23.61.116.0/23 +23.61.16.0/20 +23.61.194.0/23 +23.61.198.0/24 +23.61.203.0/24 +23.61.206.0/24 +23.61.245.0/24 +23.61.246.0/24 +23.61.247.0/24 +23.61.248.0/24 +23.61.249.0/24 +23.61.250.0/24 +23.61.251.0/24 +23.61.252.0/23 +23.61.252.0/24 +23.61.254.0/23 +23.61.32.0/20 +23.62.0.0/23 +23.62.10.0/23 +23.62.100.0/24 +23.62.102.0/23 +23.62.104.0/23 +23.62.106.0/23 +23.62.108.0/24 +23.62.109.0/24 +23.62.112.0/20 +23.62.12.0/23 +23.62.128.0/20 +23.62.14.0/24 +23.62.144.0/22 +23.62.148.0/23 +23.62.150.0/23 +23.62.152.0/22 +23.62.156.0/24 +23.62.157.0/24 +23.62.160.0/20 +23.62.176.0/20 +23.62.192.0/20 +23.62.2.0/23 +23.62.208.0/20 +23.62.224.0/24 +23.62.225.0/24 +23.62.229.0/24 +23.62.231.0/24 +23.62.237.0/24 +23.62.238.0/24 +23.62.239.0/24 +23.62.240.0/20 +23.62.53.0/24 +23.62.6.0/23 +23.62.60.0/24 +23.62.61.0/24 +23.62.62.0/23 +23.62.8.0/24 +23.62.9.0/24 +23.62.98.0/23 +23.63.0.0/20 +23.63.100.0/23 +23.63.100.0/24 +23.63.101.0/24 +23.63.102.0/23 +23.63.104.0/22 +23.63.108.0/22 +23.63.112.0/20 +23.63.128.0/20 +23.63.144.0/22 +23.63.148.0/23 +23.63.150.0/23 +23.63.152.0/22 +23.63.156.0/22 +23.63.16.0/22 +23.63.160.0/20 +23.63.176.0/20 +23.63.192.0/20 +23.63.20.0/23 +23.63.208.0/20 +23.63.216.0/22 +23.63.22.0/23 +23.63.224.0/23 +23.63.226.0/23 +23.63.230.0/23 +23.63.232.0/22 +23.63.236.0/22 +23.63.24.0/22 +23.63.28.0/22 +23.63.32.0/20 +23.63.48.0/20 +23.63.64.0/20 +23.63.80.0/20 +23.63.96.0/23 +23.63.98.0/23 +23.64.0.0/24 +23.64.1.0/24 +23.64.112.0/20 +23.64.130.0/23 +23.64.132.0/23 +23.64.134.0/23 +23.64.136.0/23 +23.64.15.0/24 +23.64.2.0/24 +23.64.3.0/24 +23.64.54.0/23 +23.64.56.0/22 +23.64.64.0/19 +23.64.96.0/20 +23.65.112.0/22 +23.65.121.0/24 +23.65.124.0/24 +23.65.126.0/23 +23.65.128.0/20 +23.65.144.0/20 +23.65.160.0/20 +23.65.176.0/22 +23.65.181.0/24 +23.65.184.0/22 +23.65.188.0/24 +23.65.192.0/20 +23.65.208.0/20 +23.65.22.0/24 +23.65.28.0/24 +23.65.29.0/24 +23.65.30.0/23 +23.65.32.0/20 +23.65.64.0/20 +23.65.96.0/20 +23.66.112.0/23 +23.66.116.0/22 +23.66.12.0/22 +23.66.122.0/23 +23.66.128.0/20 +23.66.144.0/20 +23.66.16.0/20 +23.66.192.0/20 +23.66.208.0/20 +23.66.224.0/22 +23.66.228.0/23 +23.66.230.0/23 +23.66.234.0/23 +23.66.236.0/22 +23.66.240.0/20 +23.66.32.0/20 +23.66.4.0/22 +23.66.48.0/20 +23.66.64.0/20 +23.66.8.0/22 +23.67.0.0/20 +23.67.112.0/20 +23.67.128.0/20 +23.67.144.0/22 +23.67.148.0/23 +23.67.152.0/23 +23.67.154.0/23 +23.67.156.0/22 +23.67.16.0/20 +23.67.160.0/20 +23.67.176.0/20 +23.67.208.0/20 +23.67.224.0/20 +23.67.240.0/23 +23.67.242.0/23 +23.67.244.0/23 +23.67.250.0/23 +23.67.252.0/23 +23.67.254.0/24 +23.67.255.0/24 +23.67.36.0/24 +23.67.38.0/24 +23.67.41.0/24 +23.67.42.0/23 +23.67.44.0/24 +23.67.45.0/24 +23.67.47.0/24 +23.67.53.0/24 +23.67.54.0/24 +23.67.55.0/24 +23.67.56.0/24 +23.67.57.0/24 +23.67.70.0/23 +23.7.112.0/20 +23.7.176.0/20 +23.7.32.0/20 +23.7.48.0/20 +23.72.0.0/20 +23.72.112.0/20 +23.72.128.0/22 +23.72.132.0/22 +23.72.136.0/23 +23.72.138.0/23 +23.72.140.0/22 +23.72.144.0/20 +23.72.16.0/20 +23.72.160.0/20 +23.72.176.0/22 +23.72.180.0/23 +23.72.182.0/23 +23.72.184.0/22 +23.72.188.0/22 +23.72.192.0/20 +23.72.208.0/20 +23.72.224.0/23 +23.72.226.0/23 +23.72.230.0/23 +23.72.232.0/23 +23.72.234.0/23 +23.72.238.0/23 +23.72.240.0/20 +23.72.32.0/22 +23.72.36.0/23 +23.72.38.0/23 +23.72.40.0/22 +23.72.44.0/22 +23.72.48.0/20 +23.72.64.0/20 +23.72.80.0/23 +23.72.82.0/23 +23.72.84.0/22 +23.72.88.0/22 +23.72.92.0/23 +23.72.94.0/23 +23.72.96.0/20 +23.73.0.0/20 +23.73.135.0/24 +23.73.142.0/23 +23.73.144.0/20 +23.73.16.0/22 +23.73.160.0/20 +23.73.176.0/22 +23.73.180.0/23 +23.73.24.0/22 +23.73.28.0/22 +23.73.64.0/19 +23.74.10.0/23 +23.74.112.0/20 +23.74.12.0/23 +23.74.2.0/23 +23.74.24.0/24 +23.74.28.0/24 +23.74.32.0/20 +23.74.4.0/23 +23.74.48.0/20 +23.74.6.0/23 +23.74.64.0/20 +23.74.8.0/23 +23.74.80.0/20 +23.74.96.0/20 +23.75.112.0/20 +23.75.128.0/20 +23.75.144.0/20 +23.75.160.0/20 +23.75.176.0/20 +23.75.192.0/20 +23.75.208.0/22 +23.75.212.0/22 +23.75.216.0/23 +23.75.224.0/20 +23.75.24.0/23 +23.75.240.0/20 +23.75.26.0/23 +23.75.28.0/23 +23.75.30.0/23 +23.75.72.0/22 +23.75.76.0/22 +23.76.148.0/23 +23.76.153.0/24 +23.76.154.0/23 +23.76.156.0/23 +23.76.196.0/22 +23.76.204.0/23 +23.76.206.0/23 +23.76.208.0/20 +23.76.224.0/20 +23.77.0.0/20 +23.77.112.0/20 +23.77.16.0/20 +23.77.160.0/20 +23.77.176.0/20 +23.77.192.0/23 +23.77.195.0/24 +23.77.196.0/23 +23.77.198.0/23 +23.77.206.0/24 +23.77.207.0/24 +23.77.208.0/22 +23.77.208.0/23 +23.77.210.0/24 +23.77.212.0/23 +23.77.214.0/23 +23.77.216.0/21 +23.77.224.0/22 +23.77.228.0/22 +23.77.232.0/22 +23.77.232.0/23 +23.77.234.0/24 +23.77.236.0/22 +23.77.236.0/23 +23.77.238.0/24 +23.77.240.0/22 +23.77.244.0/22 +23.77.248.0/22 +23.77.48.0/20 +23.77.64.0/20 +23.77.80.0/22 +23.77.92.0/22 +23.78.0.0/20 +23.78.128.0/22 +23.78.132.0/22 +23.78.136.0/22 +23.78.140.0/23 +23.78.142.0/24 +23.78.143.0/24 +23.78.144.0/20 +23.78.16.0/20 +23.78.160.0/20 +23.78.176.0/20 +23.78.192.0/20 +23.78.208.0/22 +23.78.212.0/23 +23.78.214.0/23 +23.78.216.0/22 +23.78.220.0/23 +23.78.222.0/23 +23.78.224.0/20 +23.78.240.0/20 +23.78.32.0/20 +23.78.48.0/20 +23.79.0.0/20 +23.79.112.0/20 +23.79.128.0/20 +23.79.144.0/20 +23.79.16.0/20 +23.79.192.0/19 +23.79.240.0/24 +23.79.243.0/24 +23.79.244.0/24 +23.79.246.0/24 +23.79.251.0/24 +23.79.252.0/24 +23.79.255.0/24 +23.79.32.0/22 +23.79.36.0/22 +23.79.40.0/22 +23.79.44.0/22 +23.79.48.0/20 +23.79.64.0/20 +23.79.80.0/20 +23.79.96.0/20 +23.8.96.0/20 +23.9.128.0/20 +23.9.160.0/20 +23.9.192.0/20 +23.9.208.0/20 +59.151.136.0/22 +60.254.142.0/24 +60.254.153.0/24 +69.192.160.0/20 +69.192.2.0/23 +69.192.208.0/20 +69.192.240.0/20 +69.192.4.0/23 +69.192.4.0/24 +69.192.5.0/24 +69.192.64.0/20 +69.192.8.0/21 +72.246.100.0/23 +72.246.103.0/24 +72.246.104.0/21 +72.246.117.0/24 +72.246.124.0/22 +72.246.160.0/22 +72.246.168.0/22 +72.246.196.0/24 +72.246.197.0/24 +72.246.204.0/22 +72.246.218.0/24 +72.246.219.0/24 +72.246.220.0/22 +72.246.247.0/24 +72.246.25.0/24 +72.246.28.0/22 +72.246.40.0/24 +72.246.48.0/23 +72.246.54.0/24 +72.246.56.0/23 +72.246.68.0/23 +72.246.76.0/22 +72.246.80.0/23 +72.246.84.0/23 +72.246.86.0/23 +72.246.94.0/24 +72.246.97.0/24 +72.247.102.0/23 +72.247.104.0/21 +72.247.145.0/24 +72.247.146.0/23 +72.247.16.0/22 +72.247.168.0/22 +72.247.172.0/22 +72.247.192.0/23 +72.247.196.0/22 +72.247.20.0/23 +72.247.208.0/23 +72.247.210.0/24 +72.247.211.0/24 +72.247.212.0/22 +72.247.234.0/23 +72.247.24.0/22 +72.247.246.0/24 +72.247.247.0/24 +72.247.248.0/21 +72.247.28.0/22 +72.247.37.0/24 +72.247.48.0/22 +72.247.60.0/23 +72.247.8.0/22 +80.67.64.0/24 +80.67.72.0/24 +80.67.74.0/24 +80.67.81.0/24 +80.67.84.0/24 +80.67.85.0/24 +80.67.86.0/24 +80.67.87.0/24 +84.53.132.0/23 +84.53.132.0/24 +84.53.134.0/23 +84.53.134.0/24 +84.53.135.0/24 +84.53.136.0/24 +84.53.137.0/24 +84.53.138.0/24 +84.53.139.0/24 +84.53.142.0/24 +84.53.143.0/24 +84.53.144.0/24 +84.53.145.0/24 +84.53.146.0/24 +84.53.148.0/22 +84.53.156.0/22 +84.53.164.0/22 +84.53.174.0/24 +84.53.175.0/24 +84.53.176.0/23 +84.53.176.0/24 +84.53.177.0/24 +84.53.178.0/23 +84.53.178.0/24 +84.53.179.0/24 +84.53.180.0/23 +84.53.182.0/23 +88.221.110.0/24 +88.221.12.0/23 +88.221.120.0/22 +88.221.128.0/23 +88.221.132.0/23 +88.221.134.0/23 +88.221.14.0/23 +88.221.16.0/22 +88.221.160.0/21 +88.221.168.0/21 +88.221.176.0/21 +88.221.200.0/21 +88.221.210.0/24 +88.221.211.0/24 +88.221.216.0/24 +88.221.222.0/24 +88.221.4.0/22 +88.221.40.0/22 +88.221.48.0/22 +88.221.60.0/22 +88.221.64.0/22 +88.221.64.0/23 +88.221.66.0/23 +88.221.76.0/22 +88.221.81.0/24 +88.221.83.0/24 +88.221.84.0/24 +88.221.87.0/24 +88.221.88.0/22 +88.221.88.0/24 +88.221.90.0/24 +88.221.92.0/22 +88.221.92.0/23 +92.122.100.0/22 +92.122.102.0/23 +92.122.104.0/22 +92.122.114.0/23 +92.122.122.0/23 +92.122.122.0/24 +92.122.123.0/24 +92.122.125.0/24 +92.122.126.0/23 +92.122.144.0/20 +92.122.160.0/20 +92.122.188.0/24 +92.122.189.0/24 +92.122.190.0/24 +92.122.206.0/24 +92.122.207.0/24 +92.122.208.0/22 +92.122.212.0/22 +92.122.215.0/24 +92.122.216.0/22 +92.122.216.0/23 +92.122.218.0/23 +92.122.222.0/23 +92.122.226.0/23 +92.122.253.0/24 +92.122.255.0/24 +92.122.28.0/22 +92.122.44.0/22 +92.122.49.0/24 +92.122.50.0/24 +92.122.51.0/24 +92.122.54.0/24 +92.122.80.0/22 +92.122.88.0/22 +92.122.92.0/22 +92.122.96.0/22 +92.123.148.0/22 +92.123.155.0/24 +92.123.176.0/22 +92.123.184.0/21 +92.123.196.0/22 +92.123.32.0/20 +92.123.68.0/22 +92.123.69.0/24 +92.123.72.0/24 +92.123.73.0/24 +92.123.77.0/24 +92.123.78.0/24 +95.100.0.0/20 +95.100.104.0/21 +95.100.112.0/20 +95.100.128.0/20 +95.100.144.0/20 +95.100.160.0/22 +95.100.164.0/22 +95.100.170.0/24 +95.100.171.0/24 +95.100.174.0/24 +95.100.175.0/24 +95.100.176.0/20 +95.100.192.0/20 +95.100.240.0/23 +95.100.242.0/24 +95.100.243.0/24 +95.100.244.0/22 +95.100.248.0/24 +95.100.249.0/24 +95.100.254.0/24 +95.100.255.0/24 +95.100.64.0/20 +95.100.96.0/23 +95.100.98.0/23 +95.101.0.0/22 +95.101.176.0/22 +95.101.240.0/20 +95.101.28.0/23 +95.101.30.0/23 +95.101.34.0/24 +95.101.36.0/24 +95.101.44.0/22 +95.101.52.0/23 +95.101.64.0/22 +95.101.8.0/22 +95.101.84.0/22 +95.101.88.0/22 +95.101.92.0/22 +96.16.10.0/23 +96.16.12.0/23 +96.16.128.0/20 +96.16.14.0/23 +96.16.164.0/22 +96.16.168.0/22 +96.16.172.0/22 +96.16.216.0/22 +96.16.220.0/22 +96.16.224.0/20 +96.16.48.0/20 +96.16.6.0/23 +96.16.8.0/23 +96.16.99.0/24 +96.17.10.0/24 +96.17.101.0/24 +96.17.102.0/24 +96.17.106.0/23 +96.17.12.0/24 +96.17.14.0/24 +96.17.151.0/24 +96.17.153.0/24 +96.17.154.0/24 +96.17.156.0/23 +96.17.158.0/23 +96.17.163.0/24 +96.17.170.0/23 +96.17.177.0/24 +96.17.178.0/24 +96.17.184.0/22 +96.17.188.0/22 +96.17.192.0/22 +96.17.192.0/23 +96.17.194.0/23 +96.17.196.0/24 +96.17.197.0/24 +96.17.198.0/24 +96.17.202.0/23 +96.17.204.0/23 +96.17.206.0/23 +96.17.64.0/22 +96.17.72.0/24 +96.17.74.0/23 +96.17.9.0/24 +96.6.0.0/22 +96.6.113.0/24 +96.6.115.0/24 +96.6.12.0/22 +96.6.120.0/23 +96.6.122.0/24 +96.6.123.0/24 +96.6.124.0/24 +96.6.126.0/23 +96.6.192.0/20 +96.6.208.0/20 +96.6.4.0/22 +96.6.42.0/24 +96.6.45.0/24 +96.6.46.0/24 +96.6.64.0/20 +96.6.8.0/22 +96.7.176.0/22 +96.7.182.0/23 +96.7.208.0/23 +96.7.210.0/23 +96.7.220.0/23 +96.7.222.0/24 +96.7.224.0/21 +96.7.236.0/22 +96.7.246.0/23 +96.7.248.0/24 +96.7.249.0/24 +96.7.251.0/24 +96.7.40.0/24 +96.7.48.0/24 +96.7.50.0/24 +96.7.51.0/24 +96.7.52.0/22 +96.7.54.0/23 +96.7.56.0/22 +96.7.58.0/23 +96.7.96.0/20 diff --git a/filters/akamai.go b/filters/akamai.go new file mode 100644 index 0000000..95eacb7 --- /dev/null +++ b/filters/akamai.go @@ -0,0 +1,54 @@ +package filters + +import ( + "bufio" + "net" + "os" + + "github.com/sirupsen/logrus" +) + +// IsAkamai checks for the presence of the given IP in the Akamain CIDR. +func IsAkamai(ip net.IP) bool { + var file *os.File + var err error + if ip.To4() != nil { + file, err = os.Open("filters/akamai.cidr") + if err != nil { + logrus.Fatal("filters#IsAkamai", err) + } + } else if ip.To16() != nil { + file, err = os.Open("filters/akamaiv6.cidr") + if err != nil { + logrus.Fatal("filters#IsAkamai", err) + } + } else { + logrus.Error("filters#IsAkamai#invalid ip format") + return false + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + _, ipNet, err := net.ParseCIDR(scanner.Text()) + if err != nil { + continue + } + if ipNet.Contains(ip) { + return true + } + + } + return false +} + +// Check the version of the IP address (IPv4 or IPv6). +func checkIPversion(ip string) (string, bool) { + if net.ParseIP(ip).To4() != nil { + return "IPv4", true + } else if net.ParseIP(ip).To16() != nil { + return "IPv6", true + } else { + return "", false + } +} diff --git a/filters/akamaiv6.cidr b/filters/akamaiv6.cidr new file mode 100644 index 0000000..19a3f7f --- /dev/null +++ b/filters/akamaiv6.cidr @@ -0,0 +1,357 @@ +# 2001:4878::/48 +# 2600:1400:7::/48 +# 2600:1400:8::/48 +# 2600:1400:9::/48 +# 2600:1400:a::/48 +# 2600:1400:b::/48 +# 2600:1400:c::/48 +# 2600:1401:1::/48 +# 2600:1401:2::/48 +# 2600:1401:3::/48 +# 2600:1402:11::/48 +# 2600:1402:13::/48 +# 2600:1402:14::/48 +# 2600:1402:6::/48 +# 2600:1402:7::/48 +# 2600:1402:a::/48 +# 2600:1402:d::/48 +# 2600:1402:e::/48 +# 2600:1402:f::/48 +# 2600:1403:10::/64 +# 2600:1403:11::/48 +# 2600:1403:12::/48 +# 2600:1403:4::/48 +# 2600:1403:6::/48 +# 2600:1403:8::/48 +# 2600:1403:9::/48 +# 2600:1403:b::/48 +# 2600:1403:c::/48 +# 2600:1403:d::/48 +# 2600:1403:e::/48 +# 2600:1404:12::/48 +# 2600:1404:13::/48 +# 2600:1404:14::/48 +# 2600:1404:15::/48 +# 2600:1404:16::/48 +# 2600:1404:17::/48 +# 2600:1404:18::/48 +# 2600:1404:1a::/48 +# 2600:1404:1c::/48 +# 2600:1404:1d::/48 +# 2600:1404:1e::/48 +# 2600:1404:1f::/48 +# 2600:1404:20::/48 +# 2600:1404:7::/48 +# 2600:1404:8::/48 +# 2600:1404:a::/48 +# 2600:1404:b::/48 +# 2600:1404:c::/48 +# 2600:1404:d::/48 +# 2600:1404:e::/48 +# 2600:1404:f::/48 +# 2600:1405:1::/48 +# 2600:1405:2::/48 +# 2600:1406:12::/48 +# 2600:1406:13::/48 +# 2600:1406:16::/48 +# 2600:1406:19::/48 +# 2600:1406:1a::/48 +# 2600:1406:1b::/48 +# 2600:1406:1c::/48 +# 2600:1406:1f::/48 +# 2600:1406:20::/48 +# 2600:1406:21::/48 +# 2600:1406:22::/48 +# 2600:1406:27::/48 +# 2600:1406:28::/48 +# 2600:1406:29::/48 +# 2600:1406:2a::/48 +# 2600:1406:2b::/48 +# 2600:1406:2c::/48 +# 2600:1406:2f::/48 +# 2600:1406:30::/48 +# 2600:1406:31::/48 +# 2600:1406:32::/48 +# 2600:1406:34::/48 +# 2600:1406:f::/48 +# 2600:1407:10::/48 +# 2600:1407:11::/48 +# 2600:1407:12::/48 +# 2600:1407:13::/48 +# 2600:1407:15::/48 +# 2600:1407:16::/48 +# 2600:1407:17::/48 +# 2600:1407:19::/48 +# 2600:1407:8::/48 +# 2600:1407:9::/48 +# 2600:1407:a::/48 +# 2600:1407:b::/48 +# 2600:1407:c::/48 +# 2600:1407:d::/48 +# 2600:1407:f::/48 +# 2600:1408:10::/48 +# 2600:1408:12::/48 +# 2600:1408:13::/48 +# 2600:1408:14::/48 +# 2600:1408:15::/48 +# 2600:1408:16::/48 +# 2600:1408:17::/48 +# 2600:1408:19::/48 +# 2600:1408:1a::/48 +# 2600:1408:1c::/48 +# 2600:1408:1d::/48 +# 2600:1408:9::/48 +# 2600:1408:b::/48 +# 2600:1408:c::/48 +# 2600:1408:d::/48 +# 2600:1408:e::/48 +# 2600:1409:5::/48 +# 2600:1409:a::/48 +# 2600:1409:b::/48 +# 2600:1409:c::/48 +# 2600:1409:d::/48 +# 2600:1409:e::/48 +# 2600:1409:f::/48 +# 2600:140a:2::/48 +# 2600:140a:3::/48 +# 2600:140a:4::/48 +# 2600:140a:5::/48 +# 2600:140a:6::/48 +# 2600:140a:7::/48 +# 2600:140a:8::/48 +# 2600:140a:9::/48 +# 2600:140b:10::/48 +# 2600:140b:11::/48 +# 2600:140b:12::/48 +# 2600:140b:13::/48 +# 2600:140b:15::/48 +# 2600:140b:2::/48 +# 2600:140b:a::/48 +# 2600:140b:b::/48 +# 2600:140b:c::/48 +# 2600:140b:d::/48 +# 2600:140b:e::/48 +# 2600:140b:f::/48 +# 2600:140f:9::/48 +# 2600:140f:a::/48 +# 2600:1410:1::/48 +# 2600:1410:2::/48 +# 2600:1410:3::/48 +# 2600:1415:1::/48 +# 2600:1415:5::/48 +# 2600:1415:7::/48 +# 2600:1415:8::/48 +# 2600:1415:9::/48 +# 2600:1415:b::/48 +# 2600:1415:c::/48 +# 2600:1415:d::/48 +# 2600:1416::/48 +# 2600:1417:11::/48 +# 2600:1417:13::/48 +# 2600:1417:14::/48 +# 2600:1417:15::/48 +# 2600:1417:17::/48 +# 2600:1417:18::/48 +# 2600:1417:19::/48 +# 2600:1417:1::/48 +# 2600:1417:1b::/48 +# 2600:1417:1c::/48 +# 2600:1417:22::/48 +# 2600:1417:25::/48 +# 2600:1417:26::/48 +# 2600:1417:27::/48 +# 2600:1417:28::/48 +# 2600:1417:29::/48 +# 2600:1417:2a::/48 +# 2600:1417:2c::/48 +# 2600:1417:2d::/48 +# 2600:1417:2e::/48 +# 2600:1417:2f::/48 +# 2600:1417:34::/48 +# 2600:1417:35::/48 +# 2600:1417:36::/48 +# 2600:1417:37::/48 +# 2600:1417:38::/48 +# 2600:1417:39::/48 +# 2600:1417:3::/48 +# 2600:1417:3a::/48 +# 2600:1417:3b::/48 +# 2600:1417:3f::/48 +# 2600:1417:9::/48 +# 2600:1417::/48 +# 2600:1417:a::/48 +# 2600:1417:b::/48 +# 2600:1417:c::/48 +# 2600:1417:d::/48 +# 2600:1417:e::/48 +# 2600:1418:1::/48 +# 2600:1418:2::/48 +# 2600:1419:1::/48 +# 2600:1419:2::/48 +# 2600:1419:2::/64 +# 2600:1419:3::/48 +# 2600:1419:4::/48 +# 2600:1419:5::/48 +# 2600:1419:7::/48 +# 2600:1419:8::/48 +# 2600:1419:9::/48 +# 2600:1419::/48 +# 2600:1419:a::/48 +# 2600:1419:b::/48 +# 2600:141a::/48 +# 2600:141b:10::/48 +# 2600:141b:1::/48 +# 2600:141b:2::/48 +# 2600:141b:3::/48 +# 2600:141b:4::/48 +# 2600:141b:5::/48 +# 2600:141b:6::/48 +# 2600:141c:1::/48 +# 2600:141c:2::/48 +# 2600:141c:3::/48 +# 2600:141c:4::/48 +# 2600:141c::/48 +# 2600:141d:1::/48 +# 2600:141d::/48 +# 2600:141e:1::/48 +# 2600:141e:2::/48 +# 2600:141e::/48 +# 2600:141f:1::/48 +# 2600:141f::/48 +# 2600:807:320:100::/56 +# 2600:807:320:200::/56 +# 2600:807:320:300::/56 +# 2600:807:320:400::/56 +# 2600:807:320::/56 +# 2a02:26f0:10::/48 +# 2a02:26f0:11::/48 +# 2a02:26f0:12::/48 +# 2a02:26f0:13::/48 +# 2a02:26f0:14::/48 +# 2a02:26f0:15::/48 +# 2a02:26f0:16::/48 +# 2a02:26f0:17::/48 +# 2a02:26f0:18::/48 +# 2a02:26f0:19::/48 +# 2a02:26f0:20::/48 +# 2a02:26f0:21::/48 +# 2a02:26f0:22::/48 +# 2a02:26f0:23::/48 +# 2a02:26f0:26::/48 +# 2a02:26f0:27::/48 +# 2a02:26f0:28::/48 +# 2a02:26f0:2::/48 +# 2a02:26f0:2a::/48 +# 2a02:26f0:2c::/48 +# 2a02:26f0:2d::/48 +# 2a02:26f0:2e::/48 +# 2a02:26f0:2f::/48 +# 2a02:26f0:30::/48 +# 2a02:26f0:32::/48 +# 2a02:26f0:38::/48 +# 2a02:26f0:3::/48 +# 2a02:26f0:3a::/48 +# 2a02:26f0:3b::/48 +# 2a02:26f0:3c::/48 +# 2a02:26f0:3d::/48 +# 2a02:26f0:40::/48 +# 2a02:26f0:41::/48 +# 2a02:26f0:43::/48 +# 2a02:26f0:44::/48 +# 2a02:26f0:45::/48 +# 2a02:26f0:46::/48 +# 2a02:26f0:47::/48 +# 2a02:26f0:48::/48 +# 2a02:26f0:49::/48 +# 2a02:26f0:4a::/48 +# 2a02:26f0:4b::/48 +# 2a02:26f0:4c::/48 +# 2a02:26f0:4d::/48 +# 2a02:26f0:4e::/48 +# 2a02:26f0:4f::/48 +# 2a02:26f0:50::/48 +# 2a02:26f0:51::/48 +# 2a02:26f0:52::/48 +# 2a02:26f0:53::/48 +# 2a02:26f0:54::/48 +# 2a02:26f0:55::/48 +# 2a02:26f0:58::/48 +# 2a02:26f0:59::/48 +# 2a02:26f0:5::/48 +# 2a02:26f0:5a::/48 +# 2a02:26f0:5b::/48 +# 2a02:26f0:5c::/48 +# 2a02:26f0:5d::/48 +# 2a02:26f0:5f::/48 +# 2a02:26f0:60::/48 +# 2a02:26f0:61::/48 +# 2a02:26f0:62::/48 +# 2a02:26f0:63::/48 +# 2a02:26f0:64::/48 +# 2a02:26f0:65::/48 +# 2a02:26f0:67::/48 +# 2a02:26f0:68::/48 +# 2a02:26f0:69::/48 +# 2a02:26f0:6::/48 +# 2a02:26f0:6a::/48 +# 2a02:26f0:6b::/48 +# 2a02:26f0:6f::/48 +# 2a02:26f0:70::/48 +# 2a02:26f0:71::/48 +# 2a02:26f0:72::/48 +# 2a02:26f0:75::/48 +# 2a02:26f0:76::/48 +# 2a02:26f0:78::/48 +# 2a02:26f0:79::/48 +# 2a02:26f0:7b::/48 +# 2a02:26f0:7d::/48 +# 2a02:26f0:80::/48 +# 2a02:26f0:82::/48 +# 2a02:26f0:8::/48 +# 2a02:26f0:8a::/48 +# 2a02:26f0:8b::/48 +# 2a02:26f0:8c::/48 +# 2a02:26f0:8d::/48 +# 2a02:26f0:8e::/48 +# 2a02:26f0:8f::/48 +# 2a02:26f0:94::/48 +# 2a02:26f0:96::/48 +# 2a02:26f0:97::/48 +# 2a02:26f0:98::/48 +# 2a02:26f0:9::/48 +# 2a02:26f0::/48 +# 2a02:26f0:a0::/48 +# 2a02:26f0:a1::/48 +# 2a02:26f0:a4::/48 +# 2a02:26f0:a6::/48 +# 2a02:26f0:ab::/48 +# 2a02:26f0:ac::/48 +# 2a02:26f0:ad::/48 +# 2a02:26f0:ae::/48 +# 2a02:26f0:af::/48 +# 2a02:26f0:b0::/48 +# 2a02:26f0:b1::/48 +# 2a02:26f0:b2::/48 +# 2a02:26f0:b3::/48 +# 2a02:26f0:b5::/48 +# 2a02:26f0:b6::/48 +# 2a02:26f0:b7::/48 +# 2a02:26f0:b8::/48 +# 2a02:26f0:b9::/48 +# 2a02:26f0:b::/48 +# 2a02:26f0:ba::/48 +# 2a02:26f0:bd::/48 +# 2a02:26f0:be::/48 +# 2a02:26f0:c0::/48 +# 2a02:26f0:c2::/48 +# 2a02:26f0:c3::/48 +# 2a02:26f0:c5::/48 +# 2a02:26f0:c7::/48 +# 2a02:26f0:c8::/48 +# 2a02:26f0:c9::/48 +# 2a02:26f0:c::/48 +# 2a02:26f0:ca::/48 +# 2a02:26f0:cb::/48 +# 2a02:26f0:cc::/48 +# 2a02:26f0:cd::/48 +# 2a02:26f0:f::/48 diff --git a/main.go b/main.go index 594a5cb..6594a34 100644 --- a/main.go +++ b/main.go @@ -8,11 +8,13 @@ import ( "time" "github.com/CaliDog/certstream-go" + "github.com/jmoiron/jsonq" "github.com/ns3777k/go-shodan/v4/shodan" "github.com/segmentio/kafka-go" "github.com/sirupsen/logrus" "github.com/spf13/viper" "gitlab.dcso.lolcat/LABS/styx/broker" + "gitlab.dcso.lolcat/LABS/styx/filters" "gitlab.dcso.lolcat/LABS/styx/models" ) @@ -37,8 +39,7 @@ func main() { fmt.Println("Starting to get data from the Internet...") // The false flag specifies that we want heartbeat messages. - stream, errStream := certstream.CertStreamEventStream(false) - Conn, err := broker.SetUpKafkaConnecter() + conn, err := broker.SetUpKafkaConnecter() if err != nil { panic(err) } @@ -51,67 +52,15 @@ func main() { go broker.ReadEventFromKafka() // certstream - go func() { - for { - select { - case jq := <-stream: - if data, err := models.ExtractCertFromStream(jq); err == nil { - - rawNode := models.WrapCertStreamData(*data) - models.SaveCertStreamRaw("raw_certstream.json", rawNode) - - certNode := models.BuildCertNode(rawNode) - models.SaveCertNode("cert_nodes.json", certNode) - mainNode := models.BuildNode("node", "certstream", certNode.ID) - models.SaveNode("nodes.json", mainNode) - edge := models.BuildEdge("certstream", rawNode.ID, mainNode.ID) - models.SaveEdge(edge) - edge = models.BuildEdge("certstream", mainNode.ID, certNode.ID) - models.SaveEdge(edge) - allDomains := data.Data.LeafCert.AllDomains - saveSingleValues(Conn, "certstream", "domain", certNode.ID, allDomains) - } - - case err := <-errStream: - logrus.Error(err) - case <-stopChan: - wg.Done() - return - } - } - - }() + stream, errStream := certstream.CertStreamEventStream(false) + if viper.GetBool("certstream.activated") { + go certstreamRoutine(stream, errStream, conn, stopChan, &wg) + } // pastebin - go func() { - for { - select { - default: - pastes, err := models.QueryPastes() - if err != nil { - logrus.Panic(err) - } - for _, p := range pastes { - paste, err := models.FetchPaste(p) - if err != nil { - logrus.Error("cannot fetch paste", err) - } - fp := models.FullPaste{ - Meta: p, - Full: paste, - } - res := models.BuildPasteNode(&fp) - models.SavePaste("paste_formatted.json", res) - time.Sleep(1 * time.Second) - - } - time.Sleep(3 * time.Second) - case <-stopChan: - wg.Done() - return - } - } - }() + if viper.GetBool("pastebin.activated") { + go pastebinRoutine(stopChan, &wg) + } // shodan client := shodan.NewEnvClient(nil) @@ -121,38 +70,115 @@ func main() { logrus.Panic(err) } - go func() { - for { - select { - default: - banner, ok := <-ch - if !ok { - logrus.Error("channel is closed") - break - } + if viper.GetBool("shodan.activated") { + go shodanRoutine(client, ch, conn, stopChan, &wg) + } - shodanNode := models.BuildShodanNode(banner) + wg.Wait() + +} + +// routines + +// CertstreamRoutine wraps the routine for grabbing Certstream data. +func certstreamRoutine(stream chan jsonq.JsonQuery, errStream chan error, conn *kafka.Conn, stopChan chan os.Signal, wg *sync.WaitGroup) { + fmt.Println("certstream is activated") + for { + select { + case jq := <-stream: + if data, err := models.ExtractCertFromStream(jq); err == nil { + + rawNode := models.WrapCertStreamData(*data) + models.SaveCertStreamRaw("raw_certstream.json", rawNode) + + certNode := models.BuildCertNode(rawNode) + models.SaveCertNode("cert_nodes.json", certNode) + mainNode := models.BuildNode("node", "certstream", certNode.ID) + models.SaveNode("nodes.json", mainNode) + edge := models.BuildEdge("certstream", rawNode.ID, mainNode.ID) + models.SaveEdge(edge) + edge = models.BuildEdge("certstream", mainNode.ID, certNode.ID) + models.SaveEdge(edge) + allDomains := data.Data.LeafCert.AllDomains + saveSingleValues(conn, "certstream", "domain", certNode.ID, allDomains) + } + + case err := <-errStream: + logrus.Error(err) + case <-stopChan: + wg.Done() + return + } + } +} + +func pastebinRoutine(stopChan chan os.Signal, wg *sync.WaitGroup) { + fmt.Println("pastebin is activated") + for { + select { + default: + pastes, err := models.QueryPastes() + if err != nil { + logrus.Panic(err) + } + for _, p := range pastes { + paste, err := models.FetchPaste(p) + if err != nil { + logrus.Error("cannot fetch paste", err) + } + fp := models.FullPaste{ + Meta: p, + Full: paste, + } + res := models.BuildPasteNode(&fp) + models.SavePaste("paste_formatted.json", res) + time.Sleep(1 * time.Second) + + } + time.Sleep(3 * time.Second) + case <-stopChan: + wg.Done() + return + } + } +} + +func shodanRoutine(client *shodan.Client, shodanChan chan *shodan.HostData, conn *kafka.Conn, stopChan chan os.Signal, wg *sync.WaitGroup) { + fmt.Println("shodan is activated") + for { + select { + default: + banner, ok := <-shodanChan + if !ok { + logrus.Error("channel is closed") + break + } + + shodanNode := models.BuildShodanNode(banner) + // first filter poc + if !filters.IsAkamai(shodanNode.Data.IP) { + fmt.Println("is not Akamai", shodanNode.Data.IP) hostnames := shodanNode.Data.Hostnames if len(hostnames) != 0 { - saveSingleValues(Conn, "shodan_stream", "hostname", shodanNode.ID, hostnames) + saveSingleValues(conn, "shodan_stream", "hostname", shodanNode.ID, hostnames) } domains := shodanNode.Data.Domains if len(domains) != 0 { - saveSingleValues(Conn, "shodan_stream", "domain", shodanNode.ID, domains) + saveSingleValues(conn, "shodan_stream", "domain", shodanNode.ID, domains) } - models.SaveShodanNode("shodan_raw.json", shodanNode) + models.SaveShodanNode("raw_shodan.json", shodanNode) node := models.BuildNode("shodan", "shodan_stream", shodanNode.ID) models.SaveNode("nodes.json", node) edge := models.BuildEdge("shodan", shodanNode.ID, node.ID) models.SaveEdge(edge) - case <-stopChan: - wg.Done() - return + } else { + fmt.Println("is akamai", shodanNode.Data.IP) } + case <-stopChan: + wg.Done() + return } - }() - wg.Wait() - + } } // helpers diff --git a/parser/main.go b/parser/main.go index 59dd919..0bfe2c2 100644 --- a/parser/main.go +++ b/parser/main.go @@ -1,53 +1 @@ package parser - -import ( - "os" - - "github.com/sirupsen/logrus" -) - -// read node recieved on kafka -// create a node in the node file -// save domains in another file with node ID - -// parallel routine -// look throught domain names and if find some that exists already, create the -// edge file - -const ( - NodesFilename = "nodes.json" - EdgesFilename = "edges.json" -) - -// func ParseEvent(domains []string) { -// nodeFile, err := ioutil.ReadFile(NodesFilename) -// if err != nil { -// logrus.Error(err) -// } -// nodeDatas := []models.Node{} - -// if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil { -// logrus.Error(err) -// } - -// for _, node := range nodeDatas { -// SaveDomains(node.Data.Data.LeafCert.AllDomains) -// } - -// // saveDomains() -// // go findDomainEdges() -// } - -func SaveDomains(domains []string) { - f, err := os.OpenFile("domains.txt", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) - if err != nil { - logrus.Error(err) - } - defer f.Close() - - for _, d := range domains { - if _, err := f.WriteString(d + ","); err != nil { - logrus.Error(err) - } - } -} diff --git a/utils/files.go b/utils/files.go index 9ea71fe..57f72a0 100644 --- a/utils/files.go +++ b/utils/files.go @@ -2,6 +2,7 @@ package utils import "os" +// FileExists looks for a file, if it doesn't exist, it creates it. func FileExists(filename string) error { _, err := os.Stat(filename) if os.IsNotExist(err) {