Extracting structures from CertStream
This work builds an extractor for the data in the CertStream in order to save it. It builds itself from the previous work, so extensions and flags can be added to the structures. The work in `utils` is basically a big extractor for the data taking advantage of the JSONq library. Currently, there is not refactoring and the "chains" are not saved because they need additionnal computation which will come in a later commit.
This commit is contained in:
parent
b7dce16c9e
commit
6eaae99668
4
go.mod
4
go.mod
|
@ -5,9 +5,9 @@ go 1.12
|
||||||
require (
|
require (
|
||||||
github.com/CaliDog/certstream-go v0.0.0-20180219203951-6016c5462366
|
github.com/CaliDog/certstream-go v0.0.0-20180219203951-6016c5462366
|
||||||
github.com/google/uuid v1.1.1
|
github.com/google/uuid v1.1.1
|
||||||
github.com/gorilla/websocket v1.4.1
|
github.com/gorilla/websocket v1.4.1 // indirect
|
||||||
github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e
|
github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e
|
||||||
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
|
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7
|
||||||
github.com/pkg/errors v0.9.0
|
github.com/pkg/errors v0.9.0 // indirect
|
||||||
github.com/sirupsen/logrus v1.4.2
|
github.com/sirupsen/logrus v1.4.2
|
||||||
)
|
)
|
||||||
|
|
1
go.sum
1
go.sum
|
@ -8,6 +8,7 @@ github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvK
|
||||||
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||||
github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e h1:ZZCvgaRDZg1gC9/1xrsgaJzQUCQgniKtw0xjWywWAOE=
|
github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e h1:ZZCvgaRDZg1gC9/1xrsgaJzQUCQgniKtw0xjWywWAOE=
|
||||||
github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e/go.mod h1:+rHyWac2R9oAZwFe1wGY2HBzFJJy++RHBg1cU23NkD8=
|
github.com/jmoiron/jsonq v0.0.0-20150511023944-e874b168d07e/go.mod h1:+rHyWac2R9oAZwFe1wGY2HBzFJJy++RHBg1cU23NkD8=
|
||||||
|
github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk=
|
||||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||||
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
|
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7 h1:lDH9UUVJtmYCjyT0CI4q8xvlXPxeZ0gYCVvWbmPlp88=
|
||||||
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
|
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
|
||||||
|
|
|
@ -42,7 +42,7 @@ type LeafCertSubject struct {
|
||||||
}
|
}
|
||||||
|
|
||||||
type LeafCertStruct struct {
|
type LeafCertStruct struct {
|
||||||
Subject *LeafCertSubject `json:"aggregated"`
|
Subject *LeafCertSubject `json:"subject"`
|
||||||
Extensions *LeafCertExtensions `json:"extensions"`
|
Extensions *LeafCertExtensions `json:"extensions"`
|
||||||
NotBefore string `json:"not_before"`
|
NotBefore string `json:"not_before"`
|
||||||
NotAfter string `json:"not_after"`
|
NotAfter string `json:"not_after"`
|
||||||
|
@ -62,11 +62,11 @@ type CertStreamData struct {
|
||||||
LeafCert *LeafCertStruct `json:"leaf_cert"`
|
LeafCert *LeafCertStruct `json:"leaf_cert"`
|
||||||
Chain []*LeafCertStruct `json:"chain"`
|
Chain []*LeafCertStruct `json:"chain"`
|
||||||
CertIndex int `json:"cert_index"`
|
CertIndex int `json:"cert_index"`
|
||||||
Seen time.Time `json:"seen"`
|
Seen int `json:"seen"`
|
||||||
Source *Source `json:"source"`
|
Source *Source `json:"source"`
|
||||||
}
|
}
|
||||||
|
|
||||||
type CertStreamStruct struct {
|
type CertStreamStruct struct {
|
||||||
MessageType string `json:"message_data"`
|
MessageType string `json:"message_data"`
|
||||||
Data CertStreamData `json:"data"`
|
Data *CertStreamData `json:"data"`
|
||||||
}
|
}
|
||||||
|
|
|
@ -2,17 +2,109 @@ package utils
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"github.com/jmoiron/jsonq"
|
"github.com/jmoiron/jsonq"
|
||||||
|
"github.com/sirupsen/logrus"
|
||||||
"gitlab.dcso.lolcat/LABS/styx/models"
|
"gitlab.dcso.lolcat/LABS/styx/models"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// ExtractCertFromStream builds the structures before saving them. It uses the
|
||||||
|
// power of jsonq to parse quickly the json stream.
|
||||||
|
// The base structure is coming from : https://github.com/CaliDog/certstream-go#example-data-structure
|
||||||
func ExtractCertFromStream(input jsonq.JsonQuery) (*models.CertStreamStruct, error) {
|
func ExtractCertFromStream(input jsonq.JsonQuery) (*models.CertStreamStruct, error) {
|
||||||
|
// LeafCertStruct
|
||||||
|
// LeafCertStruct > Subject
|
||||||
|
aggregated, err := input.String("data", "leaf_cert", "subject", "aggregated")
|
||||||
|
c, err := input.String("data", "leaf_cert", "subject", "C")
|
||||||
|
st, err := input.String("data", "leaf_cert", "subject", "ST")
|
||||||
|
l, err := input.String("data", "leaf_cert", "subject", "L")
|
||||||
|
o, err := input.String("data", "leaf_cert", "subject", "O")
|
||||||
|
ou, err := input.String("data", "leaf_cert", "subject", "OU")
|
||||||
|
cn, err := input.String("data", "leaf_cert", "subject", "CN")
|
||||||
|
if err != nil {
|
||||||
|
logrus.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
subject := models.LeafCertSubject{
|
||||||
|
Aggregated: aggregated,
|
||||||
|
C: c,
|
||||||
|
ST: st,
|
||||||
|
L: l,
|
||||||
|
O: o,
|
||||||
|
OU: ou,
|
||||||
|
CN: cn,
|
||||||
|
}
|
||||||
|
|
||||||
|
// LeafCertStruct > Extensions
|
||||||
|
keyUsage, err := input.String("data", "leaf_cert", "extensions", "keyUsage")
|
||||||
|
extendedKeyUsage, err := input.String("data", "leaf_cert", "extensions", "extendedKeyUsage")
|
||||||
|
basicConstrains, err := input.String("data", "leaf_cert", "extensions", "basicConstrains")
|
||||||
|
subjectKeyIdentifier, err := input.String("data", "leaf_cert", "extensions", "subjectKeyIdentifier")
|
||||||
|
authorityInfoAccess, err := input.String("data", "leaf_cert", "extensions", "authorityInfoAccess")
|
||||||
|
subjectAltName, err := input.String("data", "leaf_cert", "extensions", "subjectAltName")
|
||||||
|
certificatePolicies, err := input.String("data", "leaf_cert", "extensions", "certificatePolicies")
|
||||||
|
|
||||||
|
extensions := models.LeafCertExtensions{
|
||||||
|
KeyUsage: keyUsage,
|
||||||
|
ExtendedKeyUsage: extendedKeyUsage,
|
||||||
|
BasicConstrains: basicConstrains,
|
||||||
|
SubjectKeyIdentifier: subjectKeyIdentifier,
|
||||||
|
AuthorityInfoAccess: authorityInfoAccess,
|
||||||
|
SubjectAltName: subjectAltName,
|
||||||
|
CertificatePolicies: certificatePolicies,
|
||||||
|
}
|
||||||
|
|
||||||
|
notBefore, err := input.String("data", "leaf_cert", "not_before")
|
||||||
|
notAfter, err := input.String("data", "leaf_cert", "not_after")
|
||||||
|
serialNumber, err := input.String("data", "leaf_cert", "serialNumber")
|
||||||
|
fingerprint, err := input.String("data", "leaf_cert", "fingerprint")
|
||||||
|
asDer, err := input.String("data", "leaf_cert", "as_der")
|
||||||
|
allDomains, err := input.ArrayOfStrings("data", "leaf_cert", "all_domains")
|
||||||
|
|
||||||
|
leafCertStruct := models.LeafCertStruct{
|
||||||
|
Subject: &subject,
|
||||||
|
Extensions: &extensions,
|
||||||
|
NotBefore: notBefore,
|
||||||
|
NotAfter: notAfter,
|
||||||
|
SerialNumber: serialNumber,
|
||||||
|
Fingerprint: fingerprint,
|
||||||
|
AsDer: asDer,
|
||||||
|
AllDomains: allDomains,
|
||||||
|
}
|
||||||
|
|
||||||
|
// CertStreamData > Source
|
||||||
|
url, err := input.String("data", "source", "url")
|
||||||
|
name, err := input.String("data", "source", "name")
|
||||||
|
if err != nil {
|
||||||
|
logrus.Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
source := models.Source{
|
||||||
|
URL: url,
|
||||||
|
Name: name,
|
||||||
|
}
|
||||||
|
|
||||||
|
// CertStreamData
|
||||||
|
updateType, err := input.String("data", "update_type")
|
||||||
|
certIndex, err := input.Int("data", "cert_index")
|
||||||
|
seen, err := input.Int("data", "seen")
|
||||||
|
|
||||||
|
csd := models.CertStreamData{
|
||||||
|
UpdateType: updateType,
|
||||||
|
LeafCert: &leafCertStruct,
|
||||||
|
// chain
|
||||||
|
CertIndex: certIndex,
|
||||||
|
Seen: seen,
|
||||||
|
Source: &source,
|
||||||
|
}
|
||||||
|
|
||||||
|
// CertStreamStruct
|
||||||
messageType, err := input.String("message_type")
|
messageType, err := input.String("message_type")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
logrus.Error(err)
|
||||||
}
|
}
|
||||||
|
|
||||||
res := models.CertStreamStruct{
|
res := models.CertStreamStruct{
|
||||||
MessageType: messageType,
|
MessageType: messageType,
|
||||||
|
Data: &csd,
|
||||||
}
|
}
|
||||||
|
|
||||||
return &res, nil
|
return &res, nil
|
||||||
|
|
Loading…
Reference in a new issue