From 6eaae996682c247a040f9cf8ba6185d46a78ba5d Mon Sep 17 00:00:00 2001 From: Christopher Talib Date: Wed, 15 Jan 2020 14:36:53 +0100 Subject: [PATCH] Extracting structures from CertStream This work builds an extractor for the data in the CertStream in order to save it. It builds itself from the previous work, so extensions and flags can be added to the structures. The work in `utils` is basically a big extractor for the data taking advantage of the JSONq library. Currently, there is not refactoring and the "chains" are not saved because they need additionnal computation which will come in a later commit. --- go.mod | 4 +-- go.sum | 1 + models/types.go | 8 ++--- utils/main.go | 94 ++++++++++++++++++++++++++++++++++++++++++++++++- 4 files changed, 100 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index c88885b..7eb545e 100644 --- a/go.mod +++ b/go.mod @@ -5,9 +5,9 @@ go 1.12 require ( github.com/CaliDog/certstream-go v0.0.0-20180219203951-6016c5462366 github.com/google/uuid v1.1.1 - github.com/gorilla/websocket v1.4.1 + github.com/gorilla/websocket v1.4.1 // indirect github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 - github.com/pkg/errors v0.9.0 + github.com/pkg/errors v0.9.0 // indirect github.com/sirupsen/logrus v1.4.2 ) diff --git a/go.sum b/go.sum index 12c0c24..598e00b 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,7 @@ github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvK github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e h1:ZZCvgaRDZg1gC9/1xrsgaJzQUCQgniKtw0xjWywWAOE= github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e/go.mod h1:+rHyWac2R9oAZwFe1wGY2HBzFJJy++RHBg1cU23NkD8= +github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= diff --git a/models/types.go b/models/types.go index 6a13147..fc568b4 100644 --- a/models/types.go +++ b/models/types.go @@ -42,7 +42,7 @@ type LeafCertSubject struct { } type LeafCertStruct struct { - Subject *LeafCertSubject `json:"aggregated"` + Subject *LeafCertSubject `json:"subject"` Extensions *LeafCertExtensions `json:"extensions"` NotBefore string `json:"not_before"` NotAfter string `json:"not_after"` @@ -62,11 +62,11 @@ type CertStreamData struct { LeafCert *LeafCertStruct `json:"leaf_cert"` Chain []*LeafCertStruct `json:"chain"` CertIndex int `json:"cert_index"` - Seen time.Time `json:"seen"` + Seen int `json:"seen"` Source *Source `json:"source"` } type CertStreamStruct struct { - MessageType string `json:"message_data"` - Data CertStreamData `json:"data"` + MessageType string `json:"message_data"` + Data *CertStreamData `json:"data"` } diff --git a/utils/main.go b/utils/main.go index d9df92d..e871a8f 100644 --- a/utils/main.go +++ b/utils/main.go @@ -2,17 +2,109 @@ package utils import ( "github.com/jmoiron/jsonq" + "github.com/sirupsen/logrus" "gitlab.dcso.lolcat/LABS/styx/models" ) +// ExtractCertFromStream builds the structures before saving them. It uses the +// power of jsonq to parse quickly the json stream. +// The base structure is coming from : https://github.com/CaliDog/certstream-go#example-data-structure func ExtractCertFromStream(input jsonq.JsonQuery) (*models.CertStreamStruct, error) { + // LeafCertStruct + // LeafCertStruct > Subject + aggregated, err := input.String("data", "leaf_cert", "subject", "aggregated") + c, err := input.String("data", "leaf_cert", "subject", "C") + st, err := input.String("data", "leaf_cert", "subject", "ST") + l, err := input.String("data", "leaf_cert", "subject", "L") + o, err := input.String("data", "leaf_cert", "subject", "O") + ou, err := input.String("data", "leaf_cert", "subject", "OU") + cn, err := input.String("data", "leaf_cert", "subject", "CN") + if err != nil { + logrus.Error(err) + } + + subject := models.LeafCertSubject{ + Aggregated: aggregated, + C: c, + ST: st, + L: l, + O: o, + OU: ou, + CN: cn, + } + + // LeafCertStruct > Extensions + keyUsage, err := input.String("data", "leaf_cert", "extensions", "keyUsage") + extendedKeyUsage, err := input.String("data", "leaf_cert", "extensions", "extendedKeyUsage") + basicConstrains, err := input.String("data", "leaf_cert", "extensions", "basicConstrains") + subjectKeyIdentifier, err := input.String("data", "leaf_cert", "extensions", "subjectKeyIdentifier") + authorityInfoAccess, err := input.String("data", "leaf_cert", "extensions", "authorityInfoAccess") + subjectAltName, err := input.String("data", "leaf_cert", "extensions", "subjectAltName") + certificatePolicies, err := input.String("data", "leaf_cert", "extensions", "certificatePolicies") + + extensions := models.LeafCertExtensions{ + KeyUsage: keyUsage, + ExtendedKeyUsage: extendedKeyUsage, + BasicConstrains: basicConstrains, + SubjectKeyIdentifier: subjectKeyIdentifier, + AuthorityInfoAccess: authorityInfoAccess, + SubjectAltName: subjectAltName, + CertificatePolicies: certificatePolicies, + } + + notBefore, err := input.String("data", "leaf_cert", "not_before") + notAfter, err := input.String("data", "leaf_cert", "not_after") + serialNumber, err := input.String("data", "leaf_cert", "serialNumber") + fingerprint, err := input.String("data", "leaf_cert", "fingerprint") + asDer, err := input.String("data", "leaf_cert", "as_der") + allDomains, err := input.ArrayOfStrings("data", "leaf_cert", "all_domains") + + leafCertStruct := models.LeafCertStruct{ + Subject: &subject, + Extensions: &extensions, + NotBefore: notBefore, + NotAfter: notAfter, + SerialNumber: serialNumber, + Fingerprint: fingerprint, + AsDer: asDer, + AllDomains: allDomains, + } + + // CertStreamData > Source + url, err := input.String("data", "source", "url") + name, err := input.String("data", "source", "name") + if err != nil { + logrus.Error(err) + } + + source := models.Source{ + URL: url, + Name: name, + } + + // CertStreamData + updateType, err := input.String("data", "update_type") + certIndex, err := input.Int("data", "cert_index") + seen, err := input.Int("data", "seen") + + csd := models.CertStreamData{ + UpdateType: updateType, + LeafCert: &leafCertStruct, + // chain + CertIndex: certIndex, + Seen: seen, + Source: &source, + } + + // CertStreamStruct messageType, err := input.String("message_type") if err != nil { - return nil, err + logrus.Error(err) } res := models.CertStreamStruct{ MessageType: messageType, + Data: &csd, } return &res, nil