From 5b1bfbc195269feecb24b756076fc8428037b2d1 Mon Sep 17 00:00:00 2001 From: Christopher Talib Date: Fri, 14 Feb 2020 20:46:09 +0100 Subject: [PATCH] Filters for IP and shodan --- filters/akamai.go | 66 ++++++++++----------- filters/data/{ => ipv4}/akamai.cidr | 0 filters/data/{ => ipv4}/cloudflare.cidr | 0 filters/data/{ => ipv4}/dns_poisoning.cidr | 0 filters/data/{ => ipv4}/godady_hosting.cidr | 0 filters/data/{ => ipv4}/ip_autoblock.cidr | 0 filters/data/{ => ipv4}/parking.cidr | 0 filters/data/{ => ipv4}/reserved.cidr | 0 filters/data/{ => ipv4}/sinkholes.cidr | 0 filters/data/{ => ipv4}/various.cidr | 0 filters/data/{ => ipv6}/akamaiv6.cidr | 0 filters/data/{ => ipv6}/cloudflarev6.cidr | 0 filters/main.go | 54 +++++++++++++++++ main.go | 4 +- 14 files changed, 88 insertions(+), 36 deletions(-) rename filters/data/{ => ipv4}/akamai.cidr (100%) rename filters/data/{ => ipv4}/cloudflare.cidr (100%) rename filters/data/{ => ipv4}/dns_poisoning.cidr (100%) rename filters/data/{ => ipv4}/godady_hosting.cidr (100%) rename filters/data/{ => ipv4}/ip_autoblock.cidr (100%) rename filters/data/{ => ipv4}/parking.cidr (100%) rename filters/data/{ => ipv4}/reserved.cidr (100%) rename filters/data/{ => ipv4}/sinkholes.cidr (100%) rename filters/data/{ => ipv4}/various.cidr (100%) rename filters/data/{ => ipv6}/akamaiv6.cidr (100%) rename filters/data/{ => ipv6}/cloudflarev6.cidr (100%) create mode 100644 filters/main.go diff --git a/filters/akamai.go b/filters/akamai.go index f9ebf35..94ed953 100644 --- a/filters/akamai.go +++ b/filters/akamai.go @@ -2,53 +2,51 @@ package filters import ( "bufio" + "io/ioutil" "net" "os" + "path/filepath" + "runtime" "github.com/sirupsen/logrus" ) -// IsAkamai checks for the presence of the given IP in the Akamain CIDR. -func IsAkamai(ip net.IP) bool { - var file *os.File - var err error +var ( + _, b, _, _ = runtime.Caller(0) + basepath = filepath.Dir(b) +) + +// RunFilters runs the battery of filters for an IP. +func RunIPFilters(ip net.IP) bool { if ip.To4() != nil { - file, err = os.Open("filters/data/akamai.cidr") + path := basepath + "/data/ipv4/" + sliceIPv4, err := ioutil.ReadDir(path) if err != nil { - logrus.Fatal("filters#IsAkamai", err) + logrus.Warn("filters#ReadDir#ipv4", err) + } + + for _, name := range sliceIPv4 { + f, err := os.OpenFile(path+name.Name(), 1, 0644) + if err != nil { + logrus.Warn("filters#OpenFile#", err) + } + scanner := bufio.NewScanner(f) + for scanner.Scan() { + _, ipNet, err := net.ParseCIDR(scanner.Text()) + if err != nil { + continue + } + if ipNet.Contains(ip) { + return true + } + } } } else if ip.To16() != nil { - file, err = os.Open("filters/data/akamaiv6.cidr") - if err != nil { - logrus.Fatal("filters#IsAkamai", err) - } + // run ipv6 filter battery } else { - logrus.Error("filters#IsAkamai#invalid ip format") + logrus.Error("filters#invalid IP format") return false } - defer file.Close() - scanner := bufio.NewScanner(file) - for scanner.Scan() { - _, ipNet, err := net.ParseCIDR(scanner.Text()) - if err != nil { - continue - } - if ipNet.Contains(ip) { - return true - } - - } return false } - -// Check the version of the IP address (IPv4 or IPv6). -func checkIPversion(ip string) (string, bool) { - if net.ParseIP(ip).To4() != nil { - return "IPv4", true - } else if net.ParseIP(ip).To16() != nil { - return "IPv6", true - } else { - return "", false - } -} diff --git a/filters/data/akamai.cidr b/filters/data/ipv4/akamai.cidr similarity index 100% rename from filters/data/akamai.cidr rename to filters/data/ipv4/akamai.cidr diff --git a/filters/data/cloudflare.cidr b/filters/data/ipv4/cloudflare.cidr similarity index 100% rename from filters/data/cloudflare.cidr rename to filters/data/ipv4/cloudflare.cidr diff --git a/filters/data/dns_poisoning.cidr b/filters/data/ipv4/dns_poisoning.cidr similarity index 100% rename from filters/data/dns_poisoning.cidr rename to filters/data/ipv4/dns_poisoning.cidr diff --git a/filters/data/godady_hosting.cidr b/filters/data/ipv4/godady_hosting.cidr similarity index 100% rename from filters/data/godady_hosting.cidr rename to filters/data/ipv4/godady_hosting.cidr diff --git a/filters/data/ip_autoblock.cidr b/filters/data/ipv4/ip_autoblock.cidr similarity index 100% rename from filters/data/ip_autoblock.cidr rename to filters/data/ipv4/ip_autoblock.cidr diff --git a/filters/data/parking.cidr b/filters/data/ipv4/parking.cidr similarity index 100% rename from filters/data/parking.cidr rename to filters/data/ipv4/parking.cidr diff --git a/filters/data/reserved.cidr b/filters/data/ipv4/reserved.cidr similarity index 100% rename from filters/data/reserved.cidr rename to filters/data/ipv4/reserved.cidr diff --git a/filters/data/sinkholes.cidr b/filters/data/ipv4/sinkholes.cidr similarity index 100% rename from filters/data/sinkholes.cidr rename to filters/data/ipv4/sinkholes.cidr diff --git a/filters/data/various.cidr b/filters/data/ipv4/various.cidr similarity index 100% rename from filters/data/various.cidr rename to filters/data/ipv4/various.cidr diff --git a/filters/data/akamaiv6.cidr b/filters/data/ipv6/akamaiv6.cidr similarity index 100% rename from filters/data/akamaiv6.cidr rename to filters/data/ipv6/akamaiv6.cidr diff --git a/filters/data/cloudflarev6.cidr b/filters/data/ipv6/cloudflarev6.cidr similarity index 100% rename from filters/data/cloudflarev6.cidr rename to filters/data/ipv6/cloudflarev6.cidr diff --git a/filters/main.go b/filters/main.go new file mode 100644 index 0000000..f9ebf35 --- /dev/null +++ b/filters/main.go @@ -0,0 +1,54 @@ +package filters + +import ( + "bufio" + "net" + "os" + + "github.com/sirupsen/logrus" +) + +// IsAkamai checks for the presence of the given IP in the Akamain CIDR. +func IsAkamai(ip net.IP) bool { + var file *os.File + var err error + if ip.To4() != nil { + file, err = os.Open("filters/data/akamai.cidr") + if err != nil { + logrus.Fatal("filters#IsAkamai", err) + } + } else if ip.To16() != nil { + file, err = os.Open("filters/data/akamaiv6.cidr") + if err != nil { + logrus.Fatal("filters#IsAkamai", err) + } + } else { + logrus.Error("filters#IsAkamai#invalid ip format") + return false + } + defer file.Close() + + scanner := bufio.NewScanner(file) + for scanner.Scan() { + _, ipNet, err := net.ParseCIDR(scanner.Text()) + if err != nil { + continue + } + if ipNet.Contains(ip) { + return true + } + + } + return false +} + +// Check the version of the IP address (IPv4 or IPv6). +func checkIPversion(ip string) (string, bool) { + if net.ParseIP(ip).To4() != nil { + return "IPv4", true + } else if net.ParseIP(ip).To16() != nil { + return "IPv6", true + } else { + return "", false + } +} diff --git a/main.go b/main.go index 6594a34..7121097 100644 --- a/main.go +++ b/main.go @@ -156,8 +156,8 @@ func shodanRoutine(client *shodan.Client, shodanChan chan *shodan.HostData, conn shodanNode := models.BuildShodanNode(banner) // first filter poc - if !filters.IsAkamai(shodanNode.Data.IP) { - fmt.Println("is not Akamai", shodanNode.Data.IP) + if !filters.RunIPFilters(shodanNode.Data.IP) { + fmt.Println("#### not found in filters", shodanNode.Data.IP) hostnames := shodanNode.Data.Hostnames if len(hostnames) != 0 { saveSingleValues(conn, "shodan_stream", "hostname", shodanNode.ID, hostnames)