Extracting structures from CertStream
This work builds an extractor for the data in the CertStream in order to save it. It builds itself from the previous work, so extensions and flags can be added to the structures. The work in `utils` is basically a big extractor for the data taking advantage of the JSONq library. Currently, there is not refactoring and the "chains" are not saved because they need additionnal computation which will come in a later commit.
This commit is contained in:
Christopher Talib
parent
b7dce16c9e
commit
6eaae99668
4
go.mod
4
go.mod
|
|
@ -5,9 +5,9 @@ go 1.12
|
|||
require (
|
||||
github.com/CaliDog/certstream-go v0.0.0-20180219203951-6016c5462366
|
||||
github.com/google/uuid v1.1.1
|
||||
github.com/gorilla/websocket v1.4.1
|
||||
github.com/gorilla/websocket v1.4.1 // indirect
|
||||
github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e
|
||||
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
|
||||
github.com/pkg/errors v0.9.0
|
||||
github.com/pkg/errors v0.9.0 // indirect
|
||||
github.com/sirupsen/logrus v1.4.2
|
||||
)
|
||||
|
|
|
|||
1
go.sum
1
go.sum
|
|
@ -8,6 +8,7 @@ github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvK
|
|||
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e h1:ZZCvgaRDZg1gC9/1xrsgaJzQUCQgniKtw0xjWywWAOE=
|
||||
github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e/go.mod h1:+rHyWac2R9oAZwFe1wGY2HBzFJJy++RHBg1cU23NkD8=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
|
||||
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
|
||||
|
|
|
|||
|
|
@ -42,7 +42,7 @@ type LeafCertSubject struct {
|
|||
}
|
||||
|
||||
type LeafCertStruct struct {
|
||||
Subject *LeafCertSubject `json:"aggregated"`
|
||||
Subject *LeafCertSubject `json:"subject"`
|
||||
Extensions *LeafCertExtensions `json:"extensions"`
|
||||
NotBefore string `json:"not_before"`
|
||||
NotAfter string `json:"not_after"`
|
||||
|
|
@ -62,11 +62,11 @@ type CertStreamData struct {
|
|||
LeafCert *LeafCertStruct `json:"leaf_cert"`
|
||||
Chain []*LeafCertStruct `json:"chain"`
|
||||
CertIndex int `json:"cert_index"`
|
||||
Seen time.Time `json:"seen"`
|
||||
Seen int `json:"seen"`
|
||||
Source *Source `json:"source"`
|
||||
}
|
||||
|
||||
type CertStreamStruct struct {
|
||||
MessageType string `json:"message_data"`
|
||||
Data CertStreamData `json:"data"`
|
||||
MessageType string `json:"message_data"`
|
||||
Data *CertStreamData `json:"data"`
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,17 +2,109 @@ package utils
|
|||
|
||||
import (
|
||||
"github.com/jmoiron/jsonq"
|
||||
"github.com/sirupsen/logrus"
|
||||
"gitlab.dcso.lolcat/LABS/styx/models"
|
||||
)
|
||||
|
||||
// ExtractCertFromStream builds the structures before saving them. It uses the
|
||||
// power of jsonq to parse quickly the json stream.
|
||||
// The base structure is coming from : https://github.com/CaliDog/certstream-go#example-data-structure
|
||||
func ExtractCertFromStream(input jsonq.JsonQuery) (*models.CertStreamStruct, error) {
|
||||
// LeafCertStruct
|
||||
// LeafCertStruct > Subject
|
||||
aggregated, err := input.String("data", "leaf_cert", "subject", "aggregated")
|
||||
c, err := input.String("data", "leaf_cert", "subject", "C")
|
||||
st, err := input.String("data", "leaf_cert", "subject", "ST")
|
||||
l, err := input.String("data", "leaf_cert", "subject", "L")
|
||||
o, err := input.String("data", "leaf_cert", "subject", "O")
|
||||
ou, err := input.String("data", "leaf_cert", "subject", "OU")
|
||||
cn, err := input.String("data", "leaf_cert", "subject", "CN")
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
subject := models.LeafCertSubject{
|
||||
Aggregated: aggregated,
|
||||
C: c,
|
||||
ST: st,
|
||||
L: l,
|
||||
O: o,
|
||||
OU: ou,
|
||||
CN: cn,
|
||||
}
|
||||
|
||||
// LeafCertStruct > Extensions
|
||||
keyUsage, err := input.String("data", "leaf_cert", "extensions", "keyUsage")
|
||||
extendedKeyUsage, err := input.String("data", "leaf_cert", "extensions", "extendedKeyUsage")
|
||||
basicConstrains, err := input.String("data", "leaf_cert", "extensions", "basicConstrains")
|
||||
subjectKeyIdentifier, err := input.String("data", "leaf_cert", "extensions", "subjectKeyIdentifier")
|
||||
authorityInfoAccess, err := input.String("data", "leaf_cert", "extensions", "authorityInfoAccess")
|
||||
subjectAltName, err := input.String("data", "leaf_cert", "extensions", "subjectAltName")
|
||||
certificatePolicies, err := input.String("data", "leaf_cert", "extensions", "certificatePolicies")
|
||||
|
||||
extensions := models.LeafCertExtensions{
|
||||
KeyUsage: keyUsage,
|
||||
ExtendedKeyUsage: extendedKeyUsage,
|
||||
BasicConstrains: basicConstrains,
|
||||
SubjectKeyIdentifier: subjectKeyIdentifier,
|
||||
AuthorityInfoAccess: authorityInfoAccess,
|
||||
SubjectAltName: subjectAltName,
|
||||
CertificatePolicies: certificatePolicies,
|
||||
}
|
||||
|
||||
notBefore, err := input.String("data", "leaf_cert", "not_before")
|
||||
notAfter, err := input.String("data", "leaf_cert", "not_after")
|
||||
serialNumber, err := input.String("data", "leaf_cert", "serialNumber")
|
||||
fingerprint, err := input.String("data", "leaf_cert", "fingerprint")
|
||||
asDer, err := input.String("data", "leaf_cert", "as_der")
|
||||
allDomains, err := input.ArrayOfStrings("data", "leaf_cert", "all_domains")
|
||||
|
||||
leafCertStruct := models.LeafCertStruct{
|
||||
Subject: &subject,
|
||||
Extensions: &extensions,
|
||||
NotBefore: notBefore,
|
||||
NotAfter: notAfter,
|
||||
SerialNumber: serialNumber,
|
||||
Fingerprint: fingerprint,
|
||||
AsDer: asDer,
|
||||
AllDomains: allDomains,
|
||||
}
|
||||
|
||||
// CertStreamData > Source
|
||||
url, err := input.String("data", "source", "url")
|
||||
name, err := input.String("data", "source", "name")
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
source := models.Source{
|
||||
URL: url,
|
||||
Name: name,
|
||||
}
|
||||
|
||||
// CertStreamData
|
||||
updateType, err := input.String("data", "update_type")
|
||||
certIndex, err := input.Int("data", "cert_index")
|
||||
seen, err := input.Int("data", "seen")
|
||||
|
||||
csd := models.CertStreamData{
|
||||
UpdateType: updateType,
|
||||
LeafCert: &leafCertStruct,
|
||||
// chain
|
||||
CertIndex: certIndex,
|
||||
Seen: seen,
|
||||
Source: &source,
|
||||
}
|
||||
|
||||
// CertStreamStruct
|
||||
messageType, err := input.String("message_type")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
res := models.CertStreamStruct{
|
||||
MessageType: messageType,
|
||||
Data: &csd,
|
||||
}
|
||||
|
||||
return &res, nil
|
||||
|
|
|
|||
Loading…
Reference in a new issue