package main import ( "fmt" "os" "sync" "time" "github.com/CaliDog/certstream-go" "github.com/sirupsen/logrus" "gitlab.dcso.lolcat/LABS/styx/models" ) func main() { // The false flag specifies that we want heartbeat messages. stream, errStream := certstream.CertStreamEventStream(false) // fmt.Println("Starting to get data from CertStream...") // Conn, err := broker.SetUpKafkaConnecter() // if err != nil { // panic(err) // } // stop channel stopChan := make(chan os.Signal) var wg sync.WaitGroup wg.Add(2) // go broker.ReadEventFromKafka() // certstream go func() { for { select { case jq := <-stream: if data, err := models.ExtractCertFromStream(jq); err == nil { rawNode := models.WrapCertStreamData(*data) models.SaveRaw("raw_certstream.json", rawNode) certNode := models.BuildCertNode(rawNode) models.SaveCertNode("cert_nodes.json", certNode) mainNode := models.BuildNode("certstream", "certstream", certNode.ID) models.SaveNode("nodes.json", mainNode) models.BuildEdge("certstream", rawNode.ID, mainNode.ID) models.BuildEdge("certstream", mainNode.ID, certNode.ID) fmt.Println(certNode) fmt.Println(certNode) allDomains := data.Data.LeafCert.AllDomains var edge *models.Edge for _, domain := range allDomains { domainNode := models.BuildNode("certstream", "domain", domain) models.SaveNode("nodes.json", domainNode) edge = models.BuildEdge("certstream", certNode.ID, domainNode.ID) fmt.Println(edge) models.SaveEdge(edge) } // broker.SendEventToKafka(Conn, *fingerprintNode) } case err := <-errStream: logrus.Error(err) case <-stopChan: wg.Done() return } } }() // pastebin go func() { for { select { default: fmt.Println("Querying pastebin...") pastes, err := models.QueryPastes() if err != nil { logrus.Panic(err) } for _, p := range pastes { fmt.Println("fetching...", p.FullURL) paste, err := models.FetchPaste(p) if err != nil { logrus.Error("cannot fetch paste", err) } fp := models.FullPaste{ Meta: p, Full: paste, } res := models.BuildPasteNode(&fp) models.SavePaste("paste_formatted.json", res) time.Sleep(1 * time.Second) } time.Sleep(3 * time.Second) case <-stopChan: wg.Done() return } } }() wg.Wait() // shodan // client := shodan.NewEnvClient(nil) // ch := make(chan *shodan.HostData) // err := client.GetBannersByPorts(context.Background(), []int{80, 443}, ch) // if err != nil { // logrus.Error(err) // } // for { // banner, ok := <-ch // fmt.Println("hello", ok) // if !ok { // logrus.Error("channel is closed") // break // } // fmt.Println(banner) // } }