2020-01-14 16:31:57 +01:00
|
|
|
package utils
|
|
|
|
|
|
|
|
import (
|
2020-01-26 17:27:20 +01:00
|
|
|
"encoding/json"
|
|
|
|
"fmt"
|
|
|
|
"io/ioutil"
|
|
|
|
"log"
|
|
|
|
"net/http"
|
2020-01-15 16:07:11 +01:00
|
|
|
"strconv"
|
|
|
|
|
2020-01-14 16:31:57 +01:00
|
|
|
"github.com/jmoiron/jsonq"
|
2020-01-15 14:36:53 +01:00
|
|
|
"github.com/sirupsen/logrus"
|
2020-01-14 16:31:57 +01:00
|
|
|
"gitlab.dcso.lolcat/LABS/styx/models"
|
|
|
|
)
|
|
|
|
|
2020-01-15 14:36:53 +01:00
|
|
|
// ExtractCertFromStream builds the structures before saving them. It uses the
|
|
|
|
// power of jsonq to parse quickly the json stream.
|
|
|
|
// The base structure is coming from : https://github.com/CaliDog/certstream-go#example-data-structure
|
2020-01-14 16:31:57 +01:00
|
|
|
func ExtractCertFromStream(input jsonq.JsonQuery) (*models.CertStreamStruct, error) {
|
2020-01-15 14:36:53 +01:00
|
|
|
// LeafCertStruct
|
2020-01-15 16:07:11 +01:00
|
|
|
leafCertStruct, err := extractLeafCertStruct(input)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
// CertStreamData > Source
|
|
|
|
url, err := input.String("data", "source", "url")
|
|
|
|
name, err := input.String("data", "source", "name")
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
source := models.Source{
|
|
|
|
URL: url,
|
|
|
|
Name: name,
|
|
|
|
}
|
|
|
|
|
|
|
|
// CertStreamData
|
|
|
|
updateType, err := input.String("data", "update_type")
|
|
|
|
certIndex, err := input.Int("data", "cert_index")
|
|
|
|
seen, err := input.Int("data", "seen")
|
|
|
|
chain, err := input.ArrayOfObjects("data", "chain")
|
2020-01-22 15:01:07 +01:00
|
|
|
chainSlice := []models.LeafCertStruct{}
|
2020-01-15 16:07:11 +01:00
|
|
|
for i := 0; i < len(chain); i++ {
|
|
|
|
c, err := extractLeafCertChainStruct(input, strconv.Itoa(i))
|
|
|
|
if err != nil {
|
2020-01-22 15:01:07 +01:00
|
|
|
logrus.Error("error extractLeafCertChainStruct: ", err)
|
2020-01-15 16:07:11 +01:00
|
|
|
}
|
2020-01-22 15:01:07 +01:00
|
|
|
chainSlice = append(chainSlice, c)
|
2020-01-15 16:07:11 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
csd := models.CertStreamData{
|
|
|
|
UpdateType: updateType,
|
2020-01-22 15:01:07 +01:00
|
|
|
LeafCert: leafCertStruct,
|
2020-01-15 16:07:11 +01:00
|
|
|
Chain: chainSlice,
|
|
|
|
CertIndex: certIndex,
|
|
|
|
Seen: seen,
|
2020-01-22 15:01:07 +01:00
|
|
|
Source: source,
|
2020-01-15 16:07:11 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// CertStreamStruct
|
|
|
|
messageType, err := input.String("message_type")
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
res := models.CertStreamStruct{
|
|
|
|
MessageType: messageType,
|
2020-01-22 15:01:07 +01:00
|
|
|
Data: csd,
|
2020-01-15 16:07:11 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
return &res, nil
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
func extractLeafCertChainStruct(input jsonq.JsonQuery, index string) (models.LeafCertStruct, error) {
|
|
|
|
// LeafCertStruct > Subject
|
2020-01-22 15:01:07 +01:00
|
|
|
aggregated, _ := input.String("data", "chain", index, "subject", "aggregated")
|
|
|
|
c, _ := input.String("data", "chain", index, "subject", "C")
|
|
|
|
st, _ := input.String("data", "chain", index, "subject", "ST")
|
|
|
|
l, _ := input.String("data", "chain", index, "subject", "L")
|
|
|
|
o, _ := input.String("data", "chain", index, "subject", "O")
|
|
|
|
ou, _ := input.String("data", "chain", index, "subject", "OU")
|
|
|
|
cn, _ := input.String("data", "chain", index, "subject", "CN")
|
2020-01-15 16:07:11 +01:00
|
|
|
|
|
|
|
subject := models.LeafCertSubject{
|
|
|
|
Aggregated: aggregated,
|
|
|
|
C: c,
|
|
|
|
ST: st,
|
|
|
|
L: l,
|
|
|
|
O: o,
|
|
|
|
OU: ou,
|
|
|
|
CN: cn,
|
|
|
|
}
|
|
|
|
|
|
|
|
// LeafCertStruct > Extensions
|
2020-01-22 15:01:07 +01:00
|
|
|
keyUsage, _ := input.String("data", "chain", index, "extensions", "keyUsage")
|
|
|
|
extendedKeyUsage, _ := input.String("data", "chain", index, "extensions", "extendedKeyUsage")
|
|
|
|
basicConstrains, _ := input.String("data", "chain", index, "extensions", "basicConstrains")
|
|
|
|
subjectKeyIdentifier, _ := input.String("data", "chain", index, "extensions", "subjectKeyIdentifier")
|
|
|
|
authorityInfoAccess, _ := input.String("data", "chain", index, "extensions", "authorityInfoAccess")
|
|
|
|
subjectAltName, _ := input.String("data", "chain", index, "extensions", "subjectAltName")
|
|
|
|
certificatePolicies, _ := input.String("data", "chain", index, "extensions", "certificatePolicies")
|
2020-01-15 16:07:11 +01:00
|
|
|
|
|
|
|
extensions := models.LeafCertExtensions{
|
|
|
|
KeyUsage: keyUsage,
|
|
|
|
ExtendedKeyUsage: extendedKeyUsage,
|
|
|
|
BasicConstrains: basicConstrains,
|
|
|
|
SubjectKeyIdentifier: subjectKeyIdentifier,
|
|
|
|
AuthorityInfoAccess: authorityInfoAccess,
|
|
|
|
SubjectAltName: subjectAltName,
|
|
|
|
CertificatePolicies: certificatePolicies,
|
|
|
|
}
|
|
|
|
|
2020-01-22 15:01:07 +01:00
|
|
|
notBefore, _ := input.Int("data", "chain", "not_before")
|
|
|
|
notAfter, _ := input.Int("data", "chain", "not_after")
|
|
|
|
serialNumber, _ := input.String("data", "chain", "serialNumber")
|
|
|
|
fingerprint, _ := input.String("data", "chain", "fingerprint")
|
|
|
|
asDer, _ := input.String("data", "chain", "as_der")
|
|
|
|
allDomains, _ := input.ArrayOfStrings("data", "chain", "all_domains")
|
2020-01-15 16:07:11 +01:00
|
|
|
|
|
|
|
return models.LeafCertStruct{
|
2020-01-22 15:01:07 +01:00
|
|
|
Subject: subject,
|
|
|
|
Extensions: extensions,
|
2020-01-15 16:07:11 +01:00
|
|
|
NotBefore: notBefore,
|
|
|
|
NotAfter: notAfter,
|
|
|
|
SerialNumber: serialNumber,
|
|
|
|
Fingerprint: fingerprint,
|
|
|
|
AsDer: asDer,
|
|
|
|
AllDomains: allDomains,
|
|
|
|
}, nil
|
|
|
|
|
|
|
|
}
|
|
|
|
func extractLeafCertStruct(input jsonq.JsonQuery) (models.LeafCertStruct, error) {
|
2020-01-15 14:36:53 +01:00
|
|
|
// LeafCertStruct > Subject
|
|
|
|
aggregated, err := input.String("data", "leaf_cert", "subject", "aggregated")
|
|
|
|
c, err := input.String("data", "leaf_cert", "subject", "C")
|
|
|
|
st, err := input.String("data", "leaf_cert", "subject", "ST")
|
|
|
|
l, err := input.String("data", "leaf_cert", "subject", "L")
|
|
|
|
o, err := input.String("data", "leaf_cert", "subject", "O")
|
|
|
|
ou, err := input.String("data", "leaf_cert", "subject", "OU")
|
|
|
|
cn, err := input.String("data", "leaf_cert", "subject", "CN")
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error(err)
|
|
|
|
}
|
|
|
|
|
|
|
|
subject := models.LeafCertSubject{
|
|
|
|
Aggregated: aggregated,
|
|
|
|
C: c,
|
|
|
|
ST: st,
|
|
|
|
L: l,
|
|
|
|
O: o,
|
|
|
|
OU: ou,
|
|
|
|
CN: cn,
|
|
|
|
}
|
|
|
|
|
|
|
|
// LeafCertStruct > Extensions
|
2020-01-22 15:01:07 +01:00
|
|
|
keyUsage, _ := input.String("data", "leaf_cert", "extensions", "keyUsage")
|
|
|
|
extendedKeyUsage, _ := input.String("data", "leaf_cert", "extensions", "extendedKeyUsage")
|
|
|
|
basicConstrains, _ := input.String("data", "leaf_cert", "extensions", "basicConstrains")
|
|
|
|
subjectKeyIdentifier, _ := input.String("data", "leaf_cert", "extensions", "subjectKeyIdentifier")
|
|
|
|
authorityInfoAccess, _ := input.String("data", "leaf_cert", "extensions", "authorityInfoAccess")
|
|
|
|
subjectAltName, _ := input.String("data", "leaf_cert", "extensions", "subjectAltName")
|
|
|
|
certificatePolicies, _ := input.String("data", "leaf_cert", "extensions", "certificatePolicies")
|
2020-01-15 14:36:53 +01:00
|
|
|
|
|
|
|
extensions := models.LeafCertExtensions{
|
|
|
|
KeyUsage: keyUsage,
|
|
|
|
ExtendedKeyUsage: extendedKeyUsage,
|
|
|
|
BasicConstrains: basicConstrains,
|
|
|
|
SubjectKeyIdentifier: subjectKeyIdentifier,
|
|
|
|
AuthorityInfoAccess: authorityInfoAccess,
|
|
|
|
SubjectAltName: subjectAltName,
|
|
|
|
CertificatePolicies: certificatePolicies,
|
|
|
|
}
|
|
|
|
|
2020-01-22 15:01:07 +01:00
|
|
|
notBefore, _ := input.Int("data", "leaf_cert", "not_before")
|
|
|
|
notAfter, _ := input.Int("data", "leaf_cert", "not_after")
|
|
|
|
serialNumber, _ := input.String("data", "leaf_cert", "serialNumber")
|
|
|
|
fingerprint, _ := input.String("data", "leaf_cert", "fingerprint")
|
|
|
|
asDer, _ := input.String("data", "leaf_cert", "as_der")
|
|
|
|
allDomains, _ := input.ArrayOfStrings("data", "leaf_cert", "all_domains")
|
2020-01-15 14:36:53 +01:00
|
|
|
|
2020-01-15 16:07:11 +01:00
|
|
|
return models.LeafCertStruct{
|
2020-01-22 15:01:07 +01:00
|
|
|
Subject: subject,
|
|
|
|
Extensions: extensions,
|
2020-01-15 14:36:53 +01:00
|
|
|
NotBefore: notBefore,
|
|
|
|
NotAfter: notAfter,
|
|
|
|
SerialNumber: serialNumber,
|
|
|
|
Fingerprint: fingerprint,
|
|
|
|
AsDer: asDer,
|
|
|
|
AllDomains: allDomains,
|
2020-01-15 16:07:11 +01:00
|
|
|
}, nil
|
2020-01-14 16:31:57 +01:00
|
|
|
|
|
|
|
}
|
2020-01-26 17:27:20 +01:00
|
|
|
|
|
|
|
// Meta Information: https://pastebin.com/api_scraping.php
|
|
|
|
// Content: http://pastebin.com/api_scrape_item.php
|
|
|
|
|
|
|
|
// QueryPastes returns metadata for the last 100 public pastes.
|
|
|
|
func QueryPastes() ([]models.PasteMeta, error) {
|
|
|
|
server := "pastebin.com"
|
|
|
|
req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/api_scraping.php?limit=100", server), nil)
|
|
|
|
|
|
|
|
if err != nil {
|
|
|
|
logrus.Fatal("Could not build http request", err)
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
req.Header.Set("Content-Type", "application/json")
|
|
|
|
|
|
|
|
client := &http.Client{}
|
|
|
|
resp, err := client.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error("Could not do requeest due to %v", err)
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
defer resp.Body.Close()
|
|
|
|
|
|
|
|
body, err := ioutil.ReadAll(resp.Body)
|
|
|
|
if err != nil {
|
|
|
|
logrus.Error("Could not fetch response due to %v", err)
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
var pastes []models.PasteMeta
|
|
|
|
if err := json.Unmarshal(body, &pastes); err != nil {
|
|
|
|
logrus.Error("Could not decode response due to %v, body %s", err, string(body))
|
|
|
|
return nil, err
|
|
|
|
}
|
|
|
|
|
|
|
|
return pastes, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// FetchPaste fetches paste contents via the web API.
|
|
|
|
func FetchPaste(paste models.PasteMeta) (string, error) {
|
|
|
|
url := paste.ScrapeURL
|
|
|
|
req, err := http.NewRequest("GET", url, nil)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("Could build request %v due to %v", req, err)
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
|
|
|
client := &http.Client{}
|
|
|
|
resp, err := client.Do(req)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("Could not do request %v due to %v", req, err)
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
defer resp.Body.Close()
|
|
|
|
|
|
|
|
body, err := ioutil.ReadAll(resp.Body)
|
|
|
|
if err != nil {
|
|
|
|
log.Printf("Could not read response body %v due to %v", resp.Body, err)
|
|
|
|
return "", err
|
|
|
|
}
|
|
|
|
|
|
|
|
return string(body), nil
|
|
|
|
}
|