package utils import ( "encoding/json" "io/ioutil" "os" "strconv" "github.com/jmoiron/jsonq" "github.com/sirupsen/logrus" "gitlab.dcso.lolcat/LABS/styx/models" "gitlab.dcso.lolcat/LABS/styx/parser" ) // ExtractCertFromStream builds the structures before saving them. It uses the // power of jsonq to parse quickly the json stream. // The base structure is coming from : https://github.com/CaliDog/certstream-go#example-data-structure func ExtractCertFromStream(input jsonq.JsonQuery) (*models.CertStreamStruct, error) { // LeafCertStruct leafCertStruct, err := extractLeafCertStruct(input) if err != nil { logrus.Error(err) } // CertStreamData > Source url, err := input.String("data", "source", "url") name, err := input.String("data", "source", "name") if err != nil { logrus.Error(err) } source := models.Source{ URL: url, Name: name, } // CertStreamData updateType, err := input.String("data", "update_type") certIndex, err := input.Int("data", "cert_index") seen, err := input.Int("data", "seen") chain, err := input.ArrayOfObjects("data", "chain") chainSlice := []models.LeafCertStruct{} for i := 0; i < len(chain); i++ { c, err := extractLeafCertChainStruct(input, strconv.Itoa(i)) if err != nil { logrus.Error("error extractLeafCertChainStruct: ", err) } chainSlice = append(chainSlice, c) } csd := models.CertStreamData{ UpdateType: updateType, LeafCert: leafCertStruct, Chain: chainSlice, CertIndex: certIndex, Seen: seen, Source: source, } // CertStreamStruct messageType, err := input.String("message_type") if err != nil { logrus.Error(err) } res := models.CertStreamStruct{ MessageType: messageType, Data: csd, } return &res, nil } func extractLeafCertChainStruct(input jsonq.JsonQuery, index string) (models.LeafCertStruct, error) { // LeafCertStruct > Subject aggregated, _ := input.String("data", "chain", index, "subject", "aggregated") c, _ := input.String("data", "chain", index, "subject", "C") st, _ := input.String("data", "chain", index, "subject", "ST") l, _ := input.String("data", "chain", index, "subject", "L") o, _ := input.String("data", "chain", index, "subject", "O") ou, _ := input.String("data", "chain", index, "subject", "OU") cn, _ := input.String("data", "chain", index, "subject", "CN") subject := models.LeafCertSubject{ Aggregated: aggregated, C: c, ST: st, L: l, O: o, OU: ou, CN: cn, } // LeafCertStruct > Extensions keyUsage, _ := input.String("data", "chain", index, "extensions", "keyUsage") extendedKeyUsage, _ := input.String("data", "chain", index, "extensions", "extendedKeyUsage") basicConstrains, _ := input.String("data", "chain", index, "extensions", "basicConstrains") subjectKeyIdentifier, _ := input.String("data", "chain", index, "extensions", "subjectKeyIdentifier") authorityInfoAccess, _ := input.String("data", "chain", index, "extensions", "authorityInfoAccess") subjectAltName, _ := input.String("data", "chain", index, "extensions", "subjectAltName") certificatePolicies, _ := input.String("data", "chain", index, "extensions", "certificatePolicies") extensions := models.LeafCertExtensions{ KeyUsage: keyUsage, ExtendedKeyUsage: extendedKeyUsage, BasicConstrains: basicConstrains, SubjectKeyIdentifier: subjectKeyIdentifier, AuthorityInfoAccess: authorityInfoAccess, SubjectAltName: subjectAltName, CertificatePolicies: certificatePolicies, } notBefore, _ := input.Int("data", "chain", "not_before") notAfter, _ := input.Int("data", "chain", "not_after") serialNumber, _ := input.String("data", "chain", "serialNumber") fingerprint, _ := input.String("data", "chain", "fingerprint") asDer, _ := input.String("data", "chain", "as_der") allDomains, _ := input.ArrayOfStrings("data", "chain", "all_domains") return models.LeafCertStruct{ Subject: subject, Extensions: extensions, NotBefore: notBefore, NotAfter: notAfter, SerialNumber: serialNumber, Fingerprint: fingerprint, AsDer: asDer, AllDomains: allDomains, }, nil } func extractLeafCertStruct(input jsonq.JsonQuery) (models.LeafCertStruct, error) { // LeafCertStruct > Subject aggregated, err := input.String("data", "leaf_cert", "subject", "aggregated") c, err := input.String("data", "leaf_cert", "subject", "C") st, err := input.String("data", "leaf_cert", "subject", "ST") l, err := input.String("data", "leaf_cert", "subject", "L") o, err := input.String("data", "leaf_cert", "subject", "O") ou, err := input.String("data", "leaf_cert", "subject", "OU") cn, err := input.String("data", "leaf_cert", "subject", "CN") if err != nil { logrus.Error(err) } subject := models.LeafCertSubject{ Aggregated: aggregated, C: c, ST: st, L: l, O: o, OU: ou, CN: cn, } // LeafCertStruct > Extensions keyUsage, _ := input.String("data", "leaf_cert", "extensions", "keyUsage") extendedKeyUsage, _ := input.String("data", "leaf_cert", "extensions", "extendedKeyUsage") basicConstrains, _ := input.String("data", "leaf_cert", "extensions", "basicConstrains") subjectKeyIdentifier, _ := input.String("data", "leaf_cert", "extensions", "subjectKeyIdentifier") authorityInfoAccess, _ := input.String("data", "leaf_cert", "extensions", "authorityInfoAccess") subjectAltName, _ := input.String("data", "leaf_cert", "extensions", "subjectAltName") certificatePolicies, _ := input.String("data", "leaf_cert", "extensions", "certificatePolicies") extensions := models.LeafCertExtensions{ KeyUsage: keyUsage, ExtendedKeyUsage: extendedKeyUsage, BasicConstrains: basicConstrains, SubjectKeyIdentifier: subjectKeyIdentifier, AuthorityInfoAccess: authorityInfoAccess, SubjectAltName: subjectAltName, CertificatePolicies: certificatePolicies, } notBefore, _ := input.Int("data", "leaf_cert", "not_before") notAfter, _ := input.Int("data", "leaf_cert", "not_after") serialNumber, _ := input.String("data", "leaf_cert", "serialNumber") fingerprint, _ := input.String("data", "leaf_cert", "fingerprint") asDer, _ := input.String("data", "leaf_cert", "as_der") allDomains, _ := input.ArrayOfStrings("data", "leaf_cert", "all_domains") return models.LeafCertStruct{ Subject: subject, Extensions: extensions, NotBefore: notBefore, NotAfter: notAfter, SerialNumber: serialNumber, Fingerprint: fingerprint, AsDer: asDer, AllDomains: allDomains, }, nil } func SaveNode(node *models.Node) { err := FileExists(parser.NodesFilename) if err != nil { logrus.Error(err) } nodeFile, err := ioutil.ReadFile(parser.NodesFilename) if err != nil { logrus.Error(err) } nodeDatas := []models.Node{} if err := json.Unmarshal(nodeFile, &nodeDatas); err != nil { logrus.Error(err) } nodeDatas = append(nodeDatas, *node) nodeBytes, err := json.Marshal(nodeDatas) if err != nil { logrus.Error(err) } err = ioutil.WriteFile(parser.NodesFilename, nodeBytes, 0644) if err != nil { logrus.Error(err) } } func SaveDomains(domains []string) { f, err := os.OpenFile("domains.txt", os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) if err != nil { logrus.Error(err) } defer f.Close() for _, d := range domains { if _, err := f.WriteString(d + ","); err != nil { logrus.Error(err) } } }