package utils import ( "encoding/json" "fmt" "io/ioutil" "log" "net/http" "strconv" "github.com/jmoiron/jsonq" "github.com/sirupsen/logrus" "gitlab.dcso.lolcat/LABS/styx/models" ) // ExtractCertFromStream builds the structures before saving them. It uses the // power of jsonq to parse quickly the json stream. // The base structure is coming from : https://github.com/CaliDog/certstream-go#example-data-structure func ExtractCertFromStream(input jsonq.JsonQuery) (*models.CertStreamStruct, error) { // LeafCertStruct leafCertStruct, err := extractLeafCertStruct(input) if err != nil { logrus.Error(err) } // CertStreamData > Source url, err := input.String("data", "source", "url") name, err := input.String("data", "source", "name") if err != nil { logrus.Error(err) } source := models.Source{ URL: url, Name: name, } // CertStreamData updateType, err := input.String("data", "update_type") certIndex, err := input.Int("data", "cert_index") seen, err := input.Int("data", "seen") chain, err := input.ArrayOfObjects("data", "chain") chainSlice := []models.LeafCertStruct{} for i := 0; i < len(chain); i++ { c, err := extractLeafCertChainStruct(input, strconv.Itoa(i)) if err != nil { logrus.Error("error extractLeafCertChainStruct: ", err) } chainSlice = append(chainSlice, c) } csd := models.CertStreamData{ UpdateType: updateType, LeafCert: leafCertStruct, Chain: chainSlice, CertIndex: certIndex, Seen: seen, Source: source, } // CertStreamStruct messageType, err := input.String("message_type") if err != nil { logrus.Error(err) } res := models.CertStreamStruct{ MessageType: messageType, Data: csd, } return &res, nil } func extractLeafCertChainStruct(input jsonq.JsonQuery, index string) (models.LeafCertStruct, error) { // LeafCertStruct > Subject aggregated, _ := input.String("data", "chain", index, "subject", "aggregated") c, _ := input.String("data", "chain", index, "subject", "C") st, _ := input.String("data", "chain", index, "subject", "ST") l, _ := input.String("data", "chain", index, "subject", "L") o, _ := input.String("data", "chain", index, "subject", "O") ou, _ := input.String("data", "chain", index, "subject", "OU") cn, _ := input.String("data", "chain", index, "subject", "CN") subject := models.LeafCertSubject{ Aggregated: aggregated, C: c, ST: st, L: l, O: o, OU: ou, CN: cn, } // LeafCertStruct > Extensions keyUsage, _ := input.String("data", "chain", index, "extensions", "keyUsage") extendedKeyUsage, _ := input.String("data", "chain", index, "extensions", "extendedKeyUsage") basicConstrains, _ := input.String("data", "chain", index, "extensions", "basicConstrains") subjectKeyIdentifier, _ := input.String("data", "chain", index, "extensions", "subjectKeyIdentifier") authorityInfoAccess, _ := input.String("data", "chain", index, "extensions", "authorityInfoAccess") subjectAltName, _ := input.String("data", "chain", index, "extensions", "subjectAltName") certificatePolicies, _ := input.String("data", "chain", index, "extensions", "certificatePolicies") extensions := models.LeafCertExtensions{ KeyUsage: keyUsage, ExtendedKeyUsage: extendedKeyUsage, BasicConstrains: basicConstrains, SubjectKeyIdentifier: subjectKeyIdentifier, AuthorityInfoAccess: authorityInfoAccess, SubjectAltName: subjectAltName, CertificatePolicies: certificatePolicies, } notBefore, _ := input.Int("data", "chain", "not_before") notAfter, _ := input.Int("data", "chain", "not_after") serialNumber, _ := input.String("data", "chain", "serialNumber") fingerprint, _ := input.String("data", "chain", "fingerprint") asDer, _ := input.String("data", "chain", "as_der") allDomains, _ := input.ArrayOfStrings("data", "chain", "all_domains") return models.LeafCertStruct{ Subject: subject, Extensions: extensions, NotBefore: notBefore, NotAfter: notAfter, SerialNumber: serialNumber, Fingerprint: fingerprint, AsDer: asDer, AllDomains: allDomains, }, nil } func extractLeafCertStruct(input jsonq.JsonQuery) (models.LeafCertStruct, error) { // LeafCertStruct > Subject aggregated, err := input.String("data", "leaf_cert", "subject", "aggregated") c, err := input.String("data", "leaf_cert", "subject", "C") st, err := input.String("data", "leaf_cert", "subject", "ST") l, err := input.String("data", "leaf_cert", "subject", "L") o, err := input.String("data", "leaf_cert", "subject", "O") ou, err := input.String("data", "leaf_cert", "subject", "OU") cn, err := input.String("data", "leaf_cert", "subject", "CN") if err != nil { logrus.Error(err) } subject := models.LeafCertSubject{ Aggregated: aggregated, C: c, ST: st, L: l, O: o, OU: ou, CN: cn, } // LeafCertStruct > Extensions keyUsage, _ := input.String("data", "leaf_cert", "extensions", "keyUsage") extendedKeyUsage, _ := input.String("data", "leaf_cert", "extensions", "extendedKeyUsage") basicConstrains, _ := input.String("data", "leaf_cert", "extensions", "basicConstrains") subjectKeyIdentifier, _ := input.String("data", "leaf_cert", "extensions", "subjectKeyIdentifier") authorityInfoAccess, _ := input.String("data", "leaf_cert", "extensions", "authorityInfoAccess") subjectAltName, _ := input.String("data", "leaf_cert", "extensions", "subjectAltName") certificatePolicies, _ := input.String("data", "leaf_cert", "extensions", "certificatePolicies") extensions := models.LeafCertExtensions{ KeyUsage: keyUsage, ExtendedKeyUsage: extendedKeyUsage, BasicConstrains: basicConstrains, SubjectKeyIdentifier: subjectKeyIdentifier, AuthorityInfoAccess: authorityInfoAccess, SubjectAltName: subjectAltName, CertificatePolicies: certificatePolicies, } notBefore, _ := input.Int("data", "leaf_cert", "not_before") notAfter, _ := input.Int("data", "leaf_cert", "not_after") serialNumber, _ := input.String("data", "leaf_cert", "serialNumber") fingerprint, _ := input.String("data", "leaf_cert", "fingerprint") asDer, _ := input.String("data", "leaf_cert", "as_der") allDomains, _ := input.ArrayOfStrings("data", "leaf_cert", "all_domains") return models.LeafCertStruct{ Subject: subject, Extensions: extensions, NotBefore: notBefore, NotAfter: notAfter, SerialNumber: serialNumber, Fingerprint: fingerprint, AsDer: asDer, AllDomains: allDomains, }, nil } // Meta Information: https://pastebin.com/api_scraping.php // Content: http://pastebin.com/api_scrape_item.php // QueryPastes returns metadata for the last 100 public pastes. func QueryPastes() ([]models.PasteMeta, error) { server := "pastebin.com" req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/api_scraping.php?limit=100", server), nil) if err != nil { logrus.Fatal("Could not build http request", err) return nil, err } req.Header.Set("Content-Type", "application/json") client := &http.Client{} resp, err := client.Do(req) if err != nil { logrus.Error("Could not do requeest due to %v", err) return nil, err } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { logrus.Error("Could not fetch response due to %v", err) return nil, err } var pastes []models.PasteMeta if err := json.Unmarshal(body, &pastes); err != nil { logrus.Error("Could not decode response due to %v, body %s", err, string(body)) return nil, err } return pastes, err } // FetchPaste fetches paste contents via the web API. func FetchPaste(paste models.PasteMeta) (string, error) { url := paste.ScrapeURL req, err := http.NewRequest("GET", url, nil) if err != nil { log.Printf("Could build request %v due to %v", req, err) return "", err } client := &http.Client{} resp, err := client.Do(req) if err != nil { log.Printf("Could not do request %v due to %v", req, err) return "", err } defer resp.Body.Close() body, err := ioutil.ReadAll(resp.Body) if err != nil { log.Printf("Could not read response body %v due to %v", resp.Body, err) return "", err } return string(body), nil }