Removing extract.go file and refactoring the adhoc functions to files with name of the service

This commit is contained in:
Christopher Talib 2020-01-29 11:05:05 +01:00
parent 668686fbab
commit b9f0ac688c
4 changed files with 263 additions and 256 deletions

View file

@ -18,6 +18,7 @@ func main() {
panic(err)
}
go broker.ReadEventFromKafka()
// certstream
for {
select {
case jq := <-stream:
@ -45,4 +46,6 @@ func main() {
logrus.Error(err)
}
}
// pastebin
}

View file

@ -1,5 +1,12 @@
package models
import (
"strconv"
"github.com/jmoiron/jsonq"
"github.com/sirupsen/logrus"
)
// LeafCertExtensions extends the LeafCert object.
type LeafCertExtensions struct {
KeyUsage string `json:"keyUsage"`
@ -56,3 +63,183 @@ type CertStreamStruct struct {
MessageType string `json:"message_data"`
Data CertStreamData `json:"data"`
}
// ExtractCertFromStream builds the structures before saving them. It uses the
// power of jsonq to parse quickly the json stream.
// The base structure is coming from : https://github.com/CaliDog/certstream-go#example-data-structure
func ExtractCertFromStream(input jsonq.JsonQuery) (*CertStreamStruct, error) {
// LeafCertStruct
leafCertStruct, err := extractLeafCertStruct(input)
if err != nil {
logrus.Error(err)
}
// CertStreamData > Source
url, err := input.String("data", "source", "url")
name, err := input.String("data", "source", "name")
if err != nil {
logrus.Error(err)
}
source := Source{
URL: url,
Name: name,
}
// CertStreamData
updateType, err := input.String("data", "update_type")
certIndex, err := input.Int("data", "cert_index")
seen, err := input.Int("data", "seen")
chain, err := input.ArrayOfObjects("data", "chain")
chainSlice := []LeafCertStruct{}
for i := 0; i < len(chain); i++ {
c, err := extractLeafCertChainStruct(input, strconv.Itoa(i))
if err != nil {
logrus.Error("error extractLeafCertChainStruct: ", err)
}
chainSlice = append(chainSlice, c)
}
csd := CertStreamData{
UpdateType: updateType,
LeafCert: leafCertStruct,
Chain: chainSlice,
CertIndex: certIndex,
Seen: seen,
Source: source,
}
// CertStreamStruct
messageType, err := input.String("message_type")
if err != nil {
logrus.Error(err)
}
res := CertStreamStruct{
MessageType: messageType,
Data: csd,
}
return &res, nil
}
func extractLeafCertChainStruct(input jsonq.JsonQuery, index string) (LeafCertStruct, error) {
// LeafCertStruct > Subject
aggregated, _ := input.String("data", "chain", index, "subject", "aggregated")
c, _ := input.String("data", "chain", index, "subject", "C")
st, _ := input.String("data", "chain", index, "subject", "ST")
l, _ := input.String("data", "chain", index, "subject", "L")
o, _ := input.String("data", "chain", index, "subject", "O")
ou, _ := input.String("data", "chain", index, "subject", "OU")
cn, _ := input.String("data", "chain", index, "subject", "CN")
subject := LeafCertSubject{
Aggregated: aggregated,
C: c,
ST: st,
L: l,
O: o,
OU: ou,
CN: cn,
}
// LeafCertStruct > Extensions
keyUsage, _ := input.String("data", "chain", index, "extensions", "keyUsage")
extendedKeyUsage, _ := input.String("data", "chain", index, "extensions", "extendedKeyUsage")
basicConstrains, _ := input.String("data", "chain", index, "extensions", "basicConstrains")
subjectKeyIdentifier, _ := input.String("data", "chain", index, "extensions", "subjectKeyIdentifier")
authorityInfoAccess, _ := input.String("data", "chain", index, "extensions", "authorityInfoAccess")
subjectAltName, _ := input.String("data", "chain", index, "extensions", "subjectAltName")
certificatePolicies, _ := input.String("data", "chain", index, "extensions", "certificatePolicies")
extensions := LeafCertExtensions{
KeyUsage: keyUsage,
ExtendedKeyUsage: extendedKeyUsage,
BasicConstrains: basicConstrains,
SubjectKeyIdentifier: subjectKeyIdentifier,
AuthorityInfoAccess: authorityInfoAccess,
SubjectAltName: subjectAltName,
CertificatePolicies: certificatePolicies,
}
notBefore, _ := input.Int("data", "chain", "not_before")
notAfter, _ := input.Int("data", "chain", "not_after")
serialNumber, _ := input.String("data", "chain", "serialNumber")
fingerprint, _ := input.String("data", "chain", "fingerprint")
asDer, _ := input.String("data", "chain", "as_der")
allDomains, _ := input.ArrayOfStrings("data", "chain", "all_domains")
return LeafCertStruct{
Subject: subject,
Extensions: extensions,
NotBefore: notBefore,
NotAfter: notAfter,
SerialNumber: serialNumber,
Fingerprint: fingerprint,
AsDer: asDer,
AllDomains: allDomains,
}, nil
}
func extractLeafCertStruct(input jsonq.JsonQuery) (LeafCertStruct, error) {
// LeafCertStruct > Subject
aggregated, err := input.String("data", "leaf_cert", "subject", "aggregated")
c, err := input.String("data", "leaf_cert", "subject", "C")
st, err := input.String("data", "leaf_cert", "subject", "ST")
l, err := input.String("data", "leaf_cert", "subject", "L")
o, err := input.String("data", "leaf_cert", "subject", "O")
ou, err := input.String("data", "leaf_cert", "subject", "OU")
cn, err := input.String("data", "leaf_cert", "subject", "CN")
if err != nil {
logrus.Error(err)
}
subject := LeafCertSubject{
Aggregated: aggregated,
C: c,
ST: st,
L: l,
O: o,
OU: ou,
CN: cn,
}
// LeafCertStruct > Extensions
keyUsage, _ := input.String("data", "leaf_cert", "extensions", "keyUsage")
extendedKeyUsage, _ := input.String("data", "leaf_cert", "extensions", "extendedKeyUsage")
basicConstrains, _ := input.String("data", "leaf_cert", "extensions", "basicConstrains")
subjectKeyIdentifier, _ := input.String("data", "leaf_cert", "extensions", "subjectKeyIdentifier")
authorityInfoAccess, _ := input.String("data", "leaf_cert", "extensions", "authorityInfoAccess")
subjectAltName, _ := input.String("data", "leaf_cert", "extensions", "subjectAltName")
certificatePolicies, _ := input.String("data", "leaf_cert", "extensions", "certificatePolicies")
extensions := LeafCertExtensions{
KeyUsage: keyUsage,
ExtendedKeyUsage: extendedKeyUsage,
BasicConstrains: basicConstrains,
SubjectKeyIdentifier: subjectKeyIdentifier,
AuthorityInfoAccess: authorityInfoAccess,
SubjectAltName: subjectAltName,
CertificatePolicies: certificatePolicies,
}
notBefore, _ := input.Int("data", "leaf_cert", "not_before")
notAfter, _ := input.Int("data", "leaf_cert", "not_after")
serialNumber, _ := input.String("data", "leaf_cert", "serialNumber")
fingerprint, _ := input.String("data", "leaf_cert", "fingerprint")
asDer, _ := input.String("data", "leaf_cert", "as_der")
allDomains, _ := input.ArrayOfStrings("data", "leaf_cert", "all_domains")
return LeafCertStruct{
Subject: subject,
Extensions: extensions,
NotBefore: notBefore,
NotAfter: notAfter,
SerialNumber: serialNumber,
Fingerprint: fingerprint,
AsDer: asDer,
AllDomains: allDomains,
}, nil
}

View file

@ -1,256 +0,0 @@
package models
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"strconv"
"github.com/jmoiron/jsonq"
"github.com/sirupsen/logrus"
)
// ExtractCertFromStream builds the structures before saving them. It uses the
// power of jsonq to parse quickly the json stream.
// The base structure is coming from : https://github.com/CaliDog/certstream-go#example-data-structure
func ExtractCertFromStream(input jsonq.JsonQuery) (*CertStreamStruct, error) {
// LeafCertStruct
leafCertStruct, err := extractLeafCertStruct(input)
if err != nil {
logrus.Error(err)
}
// CertStreamData > Source
url, err := input.String("data", "source", "url")
name, err := input.String("data", "source", "name")
if err != nil {
logrus.Error(err)
}
source := Source{
URL: url,
Name: name,
}
// CertStreamData
updateType, err := input.String("data", "update_type")
certIndex, err := input.Int("data", "cert_index")
seen, err := input.Int("data", "seen")
chain, err := input.ArrayOfObjects("data", "chain")
chainSlice := []LeafCertStruct{}
for i := 0; i < len(chain); i++ {
c, err := extractLeafCertChainStruct(input, strconv.Itoa(i))
if err != nil {
logrus.Error("error extractLeafCertChainStruct: ", err)
}
chainSlice = append(chainSlice, c)
}
csd := CertStreamData{
UpdateType: updateType,
LeafCert: leafCertStruct,
Chain: chainSlice,
CertIndex: certIndex,
Seen: seen,
Source: source,
}
// CertStreamStruct
messageType, err := input.String("message_type")
if err != nil {
logrus.Error(err)
}
res := CertStreamStruct{
MessageType: messageType,
Data: csd,
}
return &res, nil
}
func extractLeafCertChainStruct(input jsonq.JsonQuery, index string) (LeafCertStruct, error) {
// LeafCertStruct > Subject
aggregated, _ := input.String("data", "chain", index, "subject", "aggregated")
c, _ := input.String("data", "chain", index, "subject", "C")
st, _ := input.String("data", "chain", index, "subject", "ST")
l, _ := input.String("data", "chain", index, "subject", "L")
o, _ := input.String("data", "chain", index, "subject", "O")
ou, _ := input.String("data", "chain", index, "subject", "OU")
cn, _ := input.String("data", "chain", index, "subject", "CN")
subject := LeafCertSubject{
Aggregated: aggregated,
C: c,
ST: st,
L: l,
O: o,
OU: ou,
CN: cn,
}
// LeafCertStruct > Extensions
keyUsage, _ := input.String("data", "chain", index, "extensions", "keyUsage")
extendedKeyUsage, _ := input.String("data", "chain", index, "extensions", "extendedKeyUsage")
basicConstrains, _ := input.String("data", "chain", index, "extensions", "basicConstrains")
subjectKeyIdentifier, _ := input.String("data", "chain", index, "extensions", "subjectKeyIdentifier")
authorityInfoAccess, _ := input.String("data", "chain", index, "extensions", "authorityInfoAccess")
subjectAltName, _ := input.String("data", "chain", index, "extensions", "subjectAltName")
certificatePolicies, _ := input.String("data", "chain", index, "extensions", "certificatePolicies")
extensions := LeafCertExtensions{
KeyUsage: keyUsage,
ExtendedKeyUsage: extendedKeyUsage,
BasicConstrains: basicConstrains,
SubjectKeyIdentifier: subjectKeyIdentifier,
AuthorityInfoAccess: authorityInfoAccess,
SubjectAltName: subjectAltName,
CertificatePolicies: certificatePolicies,
}
notBefore, _ := input.Int("data", "chain", "not_before")
notAfter, _ := input.Int("data", "chain", "not_after")
serialNumber, _ := input.String("data", "chain", "serialNumber")
fingerprint, _ := input.String("data", "chain", "fingerprint")
asDer, _ := input.String("data", "chain", "as_der")
allDomains, _ := input.ArrayOfStrings("data", "chain", "all_domains")
return LeafCertStruct{
Subject: subject,
Extensions: extensions,
NotBefore: notBefore,
NotAfter: notAfter,
SerialNumber: serialNumber,
Fingerprint: fingerprint,
AsDer: asDer,
AllDomains: allDomains,
}, nil
}
func extractLeafCertStruct(input jsonq.JsonQuery) (LeafCertStruct, error) {
// LeafCertStruct > Subject
aggregated, err := input.String("data", "leaf_cert", "subject", "aggregated")
c, err := input.String("data", "leaf_cert", "subject", "C")
st, err := input.String("data", "leaf_cert", "subject", "ST")
l, err := input.String("data", "leaf_cert", "subject", "L")
o, err := input.String("data", "leaf_cert", "subject", "O")
ou, err := input.String("data", "leaf_cert", "subject", "OU")
cn, err := input.String("data", "leaf_cert", "subject", "CN")
if err != nil {
logrus.Error(err)
}
subject := LeafCertSubject{
Aggregated: aggregated,
C: c,
ST: st,
L: l,
O: o,
OU: ou,
CN: cn,
}
// LeafCertStruct > Extensions
keyUsage, _ := input.String("data", "leaf_cert", "extensions", "keyUsage")
extendedKeyUsage, _ := input.String("data", "leaf_cert", "extensions", "extendedKeyUsage")
basicConstrains, _ := input.String("data", "leaf_cert", "extensions", "basicConstrains")
subjectKeyIdentifier, _ := input.String("data", "leaf_cert", "extensions", "subjectKeyIdentifier")
authorityInfoAccess, _ := input.String("data", "leaf_cert", "extensions", "authorityInfoAccess")
subjectAltName, _ := input.String("data", "leaf_cert", "extensions", "subjectAltName")
certificatePolicies, _ := input.String("data", "leaf_cert", "extensions", "certificatePolicies")
extensions := LeafCertExtensions{
KeyUsage: keyUsage,
ExtendedKeyUsage: extendedKeyUsage,
BasicConstrains: basicConstrains,
SubjectKeyIdentifier: subjectKeyIdentifier,
AuthorityInfoAccess: authorityInfoAccess,
SubjectAltName: subjectAltName,
CertificatePolicies: certificatePolicies,
}
notBefore, _ := input.Int("data", "leaf_cert", "not_before")
notAfter, _ := input.Int("data", "leaf_cert", "not_after")
serialNumber, _ := input.String("data", "leaf_cert", "serialNumber")
fingerprint, _ := input.String("data", "leaf_cert", "fingerprint")
asDer, _ := input.String("data", "leaf_cert", "as_der")
allDomains, _ := input.ArrayOfStrings("data", "leaf_cert", "all_domains")
return LeafCertStruct{
Subject: subject,
Extensions: extensions,
NotBefore: notBefore,
NotAfter: notAfter,
SerialNumber: serialNumber,
Fingerprint: fingerprint,
AsDer: asDer,
AllDomains: allDomains,
}, nil
}
// Meta Information: https://pastebin.com/api_scraping.php
// Content: http://pastebin.com/api_scrape_item.php
// QueryPastes returns metadata for the last 100 public pastes.
func QueryPastes() ([]PasteMeta, error) {
server := "pastebin.com"
req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/api_scraping.php?limit=100", server), nil)
if err != nil {
logrus.Fatal("Could not build http request", err)
return nil, err
}
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
logrus.Error("Could not do requeest due to %v", err)
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
logrus.Error("Could not fetch response due to %v", err)
return nil, err
}
var pastes []PasteMeta
if err := json.Unmarshal(body, &pastes); err != nil {
logrus.Error("Could not decode response due to %v, body %s", err, string(body))
return nil, err
}
return pastes, err
}
// FetchPaste fetches paste contents via the web API.
func FetchPaste(paste PasteMeta) (string, error) {
url := paste.ScrapeURL
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Printf("Could build request %v due to %v", req, err)
return "", err
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
log.Printf("Could not do request %v due to %v", req, err)
return "", err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("Could not read response body %v due to %v", resp.Body, err)
return "", err
}
return string(body), nil
}

View file

@ -1,5 +1,15 @@
package models
import (
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"github.com/sirupsen/logrus"
)
// PasteMeta is a set of descriptive information on a paste.
type PasteMeta struct {
ScrapeURL string `json:"scrape_url"`
@ -27,3 +37,66 @@ type PasteFull struct {
Data string `json:"data"`
RFC3339 string `json:"time"`
}
// Meta Information: https://pastebin.com/api_scraping.php
// Content: http://pastebin.com/api_scrape_item.php
// QueryPastes returns metadata for the last 100 public pastes.
func QueryPastes() ([]PasteMeta, error) {
server := "pastebin.com"
req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/api_scraping.php?limit=100", server), nil)
if err != nil {
logrus.Fatal("Could not build http request", err)
return nil, err
}
req.Header.Set("Content-Type", "application/json")
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
logrus.Error("Could not do requeest due to %v", err)
return nil, err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
logrus.Error("Could not fetch response due to %v", err)
return nil, err
}
var pastes []PasteMeta
if err := json.Unmarshal(body, &pastes); err != nil {
logrus.Error("Could not decode response due to %v, body %s", err, string(body))
return nil, err
}
return pastes, err
}
// FetchPaste fetches paste contents via the web API.
func FetchPaste(paste PasteMeta) (string, error) {
url := paste.ScrapeURL
req, err := http.NewRequest("GET", url, nil)
if err != nil {
log.Printf("Could build request %v due to %v", req, err)
return "", err
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
log.Printf("Could not do request %v due to %v", req, err)
return "", err
}
defer resp.Body.Close()
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
log.Printf("Could not read response body %v due to %v", resp.Body, err)
return "", err
}
return string(body), nil
}