Removing extract.go file and refactoring the adhoc functions to files with name of the service
This commit is contained in:
parent
668686fbab
commit
b9f0ac688c
3
main.go
3
main.go
|
@ -18,6 +18,7 @@ func main() {
|
|||
panic(err)
|
||||
}
|
||||
go broker.ReadEventFromKafka()
|
||||
// certstream
|
||||
for {
|
||||
select {
|
||||
case jq := <-stream:
|
||||
|
@ -45,4 +46,6 @@ func main() {
|
|||
logrus.Error(err)
|
||||
}
|
||||
}
|
||||
|
||||
// pastebin
|
||||
}
|
||||
|
|
|
@ -1,5 +1,12 @@
|
|||
package models
|
||||
|
||||
import (
|
||||
"strconv"
|
||||
|
||||
"github.com/jmoiron/jsonq"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// LeafCertExtensions extends the LeafCert object.
|
||||
type LeafCertExtensions struct {
|
||||
KeyUsage string `json:"keyUsage"`
|
||||
|
@ -56,3 +63,183 @@ type CertStreamStruct struct {
|
|||
MessageType string `json:"message_data"`
|
||||
Data CertStreamData `json:"data"`
|
||||
}
|
||||
|
||||
// ExtractCertFromStream builds the structures before saving them. It uses the
|
||||
// power of jsonq to parse quickly the json stream.
|
||||
// The base structure is coming from : https://github.com/CaliDog/certstream-go#example-data-structure
|
||||
func ExtractCertFromStream(input jsonq.JsonQuery) (*CertStreamStruct, error) {
|
||||
// LeafCertStruct
|
||||
leafCertStruct, err := extractLeafCertStruct(input)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
// CertStreamData > Source
|
||||
url, err := input.String("data", "source", "url")
|
||||
name, err := input.String("data", "source", "name")
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
source := Source{
|
||||
URL: url,
|
||||
Name: name,
|
||||
}
|
||||
|
||||
// CertStreamData
|
||||
updateType, err := input.String("data", "update_type")
|
||||
certIndex, err := input.Int("data", "cert_index")
|
||||
seen, err := input.Int("data", "seen")
|
||||
chain, err := input.ArrayOfObjects("data", "chain")
|
||||
chainSlice := []LeafCertStruct{}
|
||||
for i := 0; i < len(chain); i++ {
|
||||
c, err := extractLeafCertChainStruct(input, strconv.Itoa(i))
|
||||
if err != nil {
|
||||
logrus.Error("error extractLeafCertChainStruct: ", err)
|
||||
}
|
||||
chainSlice = append(chainSlice, c)
|
||||
}
|
||||
|
||||
csd := CertStreamData{
|
||||
UpdateType: updateType,
|
||||
LeafCert: leafCertStruct,
|
||||
Chain: chainSlice,
|
||||
CertIndex: certIndex,
|
||||
Seen: seen,
|
||||
Source: source,
|
||||
}
|
||||
|
||||
// CertStreamStruct
|
||||
messageType, err := input.String("message_type")
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
res := CertStreamStruct{
|
||||
MessageType: messageType,
|
||||
Data: csd,
|
||||
}
|
||||
|
||||
return &res, nil
|
||||
|
||||
}
|
||||
|
||||
func extractLeafCertChainStruct(input jsonq.JsonQuery, index string) (LeafCertStruct, error) {
|
||||
// LeafCertStruct > Subject
|
||||
aggregated, _ := input.String("data", "chain", index, "subject", "aggregated")
|
||||
c, _ := input.String("data", "chain", index, "subject", "C")
|
||||
st, _ := input.String("data", "chain", index, "subject", "ST")
|
||||
l, _ := input.String("data", "chain", index, "subject", "L")
|
||||
o, _ := input.String("data", "chain", index, "subject", "O")
|
||||
ou, _ := input.String("data", "chain", index, "subject", "OU")
|
||||
cn, _ := input.String("data", "chain", index, "subject", "CN")
|
||||
|
||||
subject := LeafCertSubject{
|
||||
Aggregated: aggregated,
|
||||
C: c,
|
||||
ST: st,
|
||||
L: l,
|
||||
O: o,
|
||||
OU: ou,
|
||||
CN: cn,
|
||||
}
|
||||
|
||||
// LeafCertStruct > Extensions
|
||||
keyUsage, _ := input.String("data", "chain", index, "extensions", "keyUsage")
|
||||
extendedKeyUsage, _ := input.String("data", "chain", index, "extensions", "extendedKeyUsage")
|
||||
basicConstrains, _ := input.String("data", "chain", index, "extensions", "basicConstrains")
|
||||
subjectKeyIdentifier, _ := input.String("data", "chain", index, "extensions", "subjectKeyIdentifier")
|
||||
authorityInfoAccess, _ := input.String("data", "chain", index, "extensions", "authorityInfoAccess")
|
||||
subjectAltName, _ := input.String("data", "chain", index, "extensions", "subjectAltName")
|
||||
certificatePolicies, _ := input.String("data", "chain", index, "extensions", "certificatePolicies")
|
||||
|
||||
extensions := LeafCertExtensions{
|
||||
KeyUsage: keyUsage,
|
||||
ExtendedKeyUsage: extendedKeyUsage,
|
||||
BasicConstrains: basicConstrains,
|
||||
SubjectKeyIdentifier: subjectKeyIdentifier,
|
||||
AuthorityInfoAccess: authorityInfoAccess,
|
||||
SubjectAltName: subjectAltName,
|
||||
CertificatePolicies: certificatePolicies,
|
||||
}
|
||||
|
||||
notBefore, _ := input.Int("data", "chain", "not_before")
|
||||
notAfter, _ := input.Int("data", "chain", "not_after")
|
||||
serialNumber, _ := input.String("data", "chain", "serialNumber")
|
||||
fingerprint, _ := input.String("data", "chain", "fingerprint")
|
||||
asDer, _ := input.String("data", "chain", "as_der")
|
||||
allDomains, _ := input.ArrayOfStrings("data", "chain", "all_domains")
|
||||
|
||||
return LeafCertStruct{
|
||||
Subject: subject,
|
||||
Extensions: extensions,
|
||||
NotBefore: notBefore,
|
||||
NotAfter: notAfter,
|
||||
SerialNumber: serialNumber,
|
||||
Fingerprint: fingerprint,
|
||||
AsDer: asDer,
|
||||
AllDomains: allDomains,
|
||||
}, nil
|
||||
|
||||
}
|
||||
func extractLeafCertStruct(input jsonq.JsonQuery) (LeafCertStruct, error) {
|
||||
// LeafCertStruct > Subject
|
||||
aggregated, err := input.String("data", "leaf_cert", "subject", "aggregated")
|
||||
c, err := input.String("data", "leaf_cert", "subject", "C")
|
||||
st, err := input.String("data", "leaf_cert", "subject", "ST")
|
||||
l, err := input.String("data", "leaf_cert", "subject", "L")
|
||||
o, err := input.String("data", "leaf_cert", "subject", "O")
|
||||
ou, err := input.String("data", "leaf_cert", "subject", "OU")
|
||||
cn, err := input.String("data", "leaf_cert", "subject", "CN")
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
subject := LeafCertSubject{
|
||||
Aggregated: aggregated,
|
||||
C: c,
|
||||
ST: st,
|
||||
L: l,
|
||||
O: o,
|
||||
OU: ou,
|
||||
CN: cn,
|
||||
}
|
||||
|
||||
// LeafCertStruct > Extensions
|
||||
keyUsage, _ := input.String("data", "leaf_cert", "extensions", "keyUsage")
|
||||
extendedKeyUsage, _ := input.String("data", "leaf_cert", "extensions", "extendedKeyUsage")
|
||||
basicConstrains, _ := input.String("data", "leaf_cert", "extensions", "basicConstrains")
|
||||
subjectKeyIdentifier, _ := input.String("data", "leaf_cert", "extensions", "subjectKeyIdentifier")
|
||||
authorityInfoAccess, _ := input.String("data", "leaf_cert", "extensions", "authorityInfoAccess")
|
||||
subjectAltName, _ := input.String("data", "leaf_cert", "extensions", "subjectAltName")
|
||||
certificatePolicies, _ := input.String("data", "leaf_cert", "extensions", "certificatePolicies")
|
||||
|
||||
extensions := LeafCertExtensions{
|
||||
KeyUsage: keyUsage,
|
||||
ExtendedKeyUsage: extendedKeyUsage,
|
||||
BasicConstrains: basicConstrains,
|
||||
SubjectKeyIdentifier: subjectKeyIdentifier,
|
||||
AuthorityInfoAccess: authorityInfoAccess,
|
||||
SubjectAltName: subjectAltName,
|
||||
CertificatePolicies: certificatePolicies,
|
||||
}
|
||||
|
||||
notBefore, _ := input.Int("data", "leaf_cert", "not_before")
|
||||
notAfter, _ := input.Int("data", "leaf_cert", "not_after")
|
||||
serialNumber, _ := input.String("data", "leaf_cert", "serialNumber")
|
||||
fingerprint, _ := input.String("data", "leaf_cert", "fingerprint")
|
||||
asDer, _ := input.String("data", "leaf_cert", "as_der")
|
||||
allDomains, _ := input.ArrayOfStrings("data", "leaf_cert", "all_domains")
|
||||
|
||||
return LeafCertStruct{
|
||||
Subject: subject,
|
||||
Extensions: extensions,
|
||||
NotBefore: notBefore,
|
||||
NotAfter: notAfter,
|
||||
SerialNumber: serialNumber,
|
||||
Fingerprint: fingerprint,
|
||||
AsDer: asDer,
|
||||
AllDomains: allDomains,
|
||||
}, nil
|
||||
|
||||
}
|
||||
|
|
|
@ -1,256 +0,0 @@
|
|||
package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/jmoiron/jsonq"
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// ExtractCertFromStream builds the structures before saving them. It uses the
|
||||
// power of jsonq to parse quickly the json stream.
|
||||
// The base structure is coming from : https://github.com/CaliDog/certstream-go#example-data-structure
|
||||
func ExtractCertFromStream(input jsonq.JsonQuery) (*CertStreamStruct, error) {
|
||||
// LeafCertStruct
|
||||
leafCertStruct, err := extractLeafCertStruct(input)
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
// CertStreamData > Source
|
||||
url, err := input.String("data", "source", "url")
|
||||
name, err := input.String("data", "source", "name")
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
source := Source{
|
||||
URL: url,
|
||||
Name: name,
|
||||
}
|
||||
|
||||
// CertStreamData
|
||||
updateType, err := input.String("data", "update_type")
|
||||
certIndex, err := input.Int("data", "cert_index")
|
||||
seen, err := input.Int("data", "seen")
|
||||
chain, err := input.ArrayOfObjects("data", "chain")
|
||||
chainSlice := []LeafCertStruct{}
|
||||
for i := 0; i < len(chain); i++ {
|
||||
c, err := extractLeafCertChainStruct(input, strconv.Itoa(i))
|
||||
if err != nil {
|
||||
logrus.Error("error extractLeafCertChainStruct: ", err)
|
||||
}
|
||||
chainSlice = append(chainSlice, c)
|
||||
}
|
||||
|
||||
csd := CertStreamData{
|
||||
UpdateType: updateType,
|
||||
LeafCert: leafCertStruct,
|
||||
Chain: chainSlice,
|
||||
CertIndex: certIndex,
|
||||
Seen: seen,
|
||||
Source: source,
|
||||
}
|
||||
|
||||
// CertStreamStruct
|
||||
messageType, err := input.String("message_type")
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
res := CertStreamStruct{
|
||||
MessageType: messageType,
|
||||
Data: csd,
|
||||
}
|
||||
|
||||
return &res, nil
|
||||
|
||||
}
|
||||
|
||||
func extractLeafCertChainStruct(input jsonq.JsonQuery, index string) (LeafCertStruct, error) {
|
||||
// LeafCertStruct > Subject
|
||||
aggregated, _ := input.String("data", "chain", index, "subject", "aggregated")
|
||||
c, _ := input.String("data", "chain", index, "subject", "C")
|
||||
st, _ := input.String("data", "chain", index, "subject", "ST")
|
||||
l, _ := input.String("data", "chain", index, "subject", "L")
|
||||
o, _ := input.String("data", "chain", index, "subject", "O")
|
||||
ou, _ := input.String("data", "chain", index, "subject", "OU")
|
||||
cn, _ := input.String("data", "chain", index, "subject", "CN")
|
||||
|
||||
subject := LeafCertSubject{
|
||||
Aggregated: aggregated,
|
||||
C: c,
|
||||
ST: st,
|
||||
L: l,
|
||||
O: o,
|
||||
OU: ou,
|
||||
CN: cn,
|
||||
}
|
||||
|
||||
// LeafCertStruct > Extensions
|
||||
keyUsage, _ := input.String("data", "chain", index, "extensions", "keyUsage")
|
||||
extendedKeyUsage, _ := input.String("data", "chain", index, "extensions", "extendedKeyUsage")
|
||||
basicConstrains, _ := input.String("data", "chain", index, "extensions", "basicConstrains")
|
||||
subjectKeyIdentifier, _ := input.String("data", "chain", index, "extensions", "subjectKeyIdentifier")
|
||||
authorityInfoAccess, _ := input.String("data", "chain", index, "extensions", "authorityInfoAccess")
|
||||
subjectAltName, _ := input.String("data", "chain", index, "extensions", "subjectAltName")
|
||||
certificatePolicies, _ := input.String("data", "chain", index, "extensions", "certificatePolicies")
|
||||
|
||||
extensions := LeafCertExtensions{
|
||||
KeyUsage: keyUsage,
|
||||
ExtendedKeyUsage: extendedKeyUsage,
|
||||
BasicConstrains: basicConstrains,
|
||||
SubjectKeyIdentifier: subjectKeyIdentifier,
|
||||
AuthorityInfoAccess: authorityInfoAccess,
|
||||
SubjectAltName: subjectAltName,
|
||||
CertificatePolicies: certificatePolicies,
|
||||
}
|
||||
|
||||
notBefore, _ := input.Int("data", "chain", "not_before")
|
||||
notAfter, _ := input.Int("data", "chain", "not_after")
|
||||
serialNumber, _ := input.String("data", "chain", "serialNumber")
|
||||
fingerprint, _ := input.String("data", "chain", "fingerprint")
|
||||
asDer, _ := input.String("data", "chain", "as_der")
|
||||
allDomains, _ := input.ArrayOfStrings("data", "chain", "all_domains")
|
||||
|
||||
return LeafCertStruct{
|
||||
Subject: subject,
|
||||
Extensions: extensions,
|
||||
NotBefore: notBefore,
|
||||
NotAfter: notAfter,
|
||||
SerialNumber: serialNumber,
|
||||
Fingerprint: fingerprint,
|
||||
AsDer: asDer,
|
||||
AllDomains: allDomains,
|
||||
}, nil
|
||||
|
||||
}
|
||||
func extractLeafCertStruct(input jsonq.JsonQuery) (LeafCertStruct, error) {
|
||||
// LeafCertStruct > Subject
|
||||
aggregated, err := input.String("data", "leaf_cert", "subject", "aggregated")
|
||||
c, err := input.String("data", "leaf_cert", "subject", "C")
|
||||
st, err := input.String("data", "leaf_cert", "subject", "ST")
|
||||
l, err := input.String("data", "leaf_cert", "subject", "L")
|
||||
o, err := input.String("data", "leaf_cert", "subject", "O")
|
||||
ou, err := input.String("data", "leaf_cert", "subject", "OU")
|
||||
cn, err := input.String("data", "leaf_cert", "subject", "CN")
|
||||
if err != nil {
|
||||
logrus.Error(err)
|
||||
}
|
||||
|
||||
subject := LeafCertSubject{
|
||||
Aggregated: aggregated,
|
||||
C: c,
|
||||
ST: st,
|
||||
L: l,
|
||||
O: o,
|
||||
OU: ou,
|
||||
CN: cn,
|
||||
}
|
||||
|
||||
// LeafCertStruct > Extensions
|
||||
keyUsage, _ := input.String("data", "leaf_cert", "extensions", "keyUsage")
|
||||
extendedKeyUsage, _ := input.String("data", "leaf_cert", "extensions", "extendedKeyUsage")
|
||||
basicConstrains, _ := input.String("data", "leaf_cert", "extensions", "basicConstrains")
|
||||
subjectKeyIdentifier, _ := input.String("data", "leaf_cert", "extensions", "subjectKeyIdentifier")
|
||||
authorityInfoAccess, _ := input.String("data", "leaf_cert", "extensions", "authorityInfoAccess")
|
||||
subjectAltName, _ := input.String("data", "leaf_cert", "extensions", "subjectAltName")
|
||||
certificatePolicies, _ := input.String("data", "leaf_cert", "extensions", "certificatePolicies")
|
||||
|
||||
extensions := LeafCertExtensions{
|
||||
KeyUsage: keyUsage,
|
||||
ExtendedKeyUsage: extendedKeyUsage,
|
||||
BasicConstrains: basicConstrains,
|
||||
SubjectKeyIdentifier: subjectKeyIdentifier,
|
||||
AuthorityInfoAccess: authorityInfoAccess,
|
||||
SubjectAltName: subjectAltName,
|
||||
CertificatePolicies: certificatePolicies,
|
||||
}
|
||||
|
||||
notBefore, _ := input.Int("data", "leaf_cert", "not_before")
|
||||
notAfter, _ := input.Int("data", "leaf_cert", "not_after")
|
||||
serialNumber, _ := input.String("data", "leaf_cert", "serialNumber")
|
||||
fingerprint, _ := input.String("data", "leaf_cert", "fingerprint")
|
||||
asDer, _ := input.String("data", "leaf_cert", "as_der")
|
||||
allDomains, _ := input.ArrayOfStrings("data", "leaf_cert", "all_domains")
|
||||
|
||||
return LeafCertStruct{
|
||||
Subject: subject,
|
||||
Extensions: extensions,
|
||||
NotBefore: notBefore,
|
||||
NotAfter: notAfter,
|
||||
SerialNumber: serialNumber,
|
||||
Fingerprint: fingerprint,
|
||||
AsDer: asDer,
|
||||
AllDomains: allDomains,
|
||||
}, nil
|
||||
|
||||
}
|
||||
|
||||
// Meta Information: https://pastebin.com/api_scraping.php
|
||||
// Content: http://pastebin.com/api_scrape_item.php
|
||||
|
||||
// QueryPastes returns metadata for the last 100 public pastes.
|
||||
func QueryPastes() ([]PasteMeta, error) {
|
||||
server := "pastebin.com"
|
||||
req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/api_scraping.php?limit=100", server), nil)
|
||||
|
||||
if err != nil {
|
||||
logrus.Fatal("Could not build http request", err)
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
logrus.Error("Could not do requeest due to %v", err)
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
logrus.Error("Could not fetch response due to %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var pastes []PasteMeta
|
||||
if err := json.Unmarshal(body, &pastes); err != nil {
|
||||
logrus.Error("Could not decode response due to %v, body %s", err, string(body))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return pastes, err
|
||||
}
|
||||
|
||||
// FetchPaste fetches paste contents via the web API.
|
||||
func FetchPaste(paste PasteMeta) (string, error) {
|
||||
url := paste.ScrapeURL
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
log.Printf("Could build request %v due to %v", req, err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
log.Printf("Could not do request %v due to %v", req, err)
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
log.Printf("Could not read response body %v due to %v", resp.Body, err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
return string(body), nil
|
||||
}
|
|
@ -1,5 +1,15 @@
|
|||
package models
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"net/http"
|
||||
|
||||
"github.com/sirupsen/logrus"
|
||||
)
|
||||
|
||||
// PasteMeta is a set of descriptive information on a paste.
|
||||
type PasteMeta struct {
|
||||
ScrapeURL string `json:"scrape_url"`
|
||||
|
@ -27,3 +37,66 @@ type PasteFull struct {
|
|||
Data string `json:"data"`
|
||||
RFC3339 string `json:"time"`
|
||||
}
|
||||
|
||||
// Meta Information: https://pastebin.com/api_scraping.php
|
||||
// Content: http://pastebin.com/api_scrape_item.php
|
||||
|
||||
// QueryPastes returns metadata for the last 100 public pastes.
|
||||
func QueryPastes() ([]PasteMeta, error) {
|
||||
server := "pastebin.com"
|
||||
req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/api_scraping.php?limit=100", server), nil)
|
||||
|
||||
if err != nil {
|
||||
logrus.Fatal("Could not build http request", err)
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
logrus.Error("Could not do requeest due to %v", err)
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
logrus.Error("Could not fetch response due to %v", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var pastes []PasteMeta
|
||||
if err := json.Unmarshal(body, &pastes); err != nil {
|
||||
logrus.Error("Could not decode response due to %v, body %s", err, string(body))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return pastes, err
|
||||
}
|
||||
|
||||
// FetchPaste fetches paste contents via the web API.
|
||||
func FetchPaste(paste PasteMeta) (string, error) {
|
||||
url := paste.ScrapeURL
|
||||
req, err := http.NewRequest("GET", url, nil)
|
||||
if err != nil {
|
||||
log.Printf("Could build request %v due to %v", req, err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
client := &http.Client{}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
log.Printf("Could not do request %v due to %v", req, err)
|
||||
return "", err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
body, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
log.Printf("Could not read response body %v due to %v", resp.Body, err)
|
||||
return "", err
|
||||
}
|
||||
|
||||
return string(body), nil
|
||||
}
|
||||
|
|
Loading…
Reference in a new issue