styx/models/extracts.go

package models

import (
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"strconv"

	"github.com/jmoiron/jsonq"
	"github.com/sirupsen/logrus"
)

// ExtractCertFromStream builds the structures before saving them. It uses the
// power of jsonq to parse quickly the json stream.
// The base structure is coming from : https://github.com/CaliDog/certstream-go#example-data-structure
func ExtractCertFromStream(input jsonq.JsonQuery) (*CertStreamStruct, error) {
	// LeafCertStruct
	leafCertStruct, err := extractLeafCertStruct(input)
	if err != nil {
		logrus.Error(err)
	}

	// CertStreamData > Source
	url, err := input.String("data", "source", "url")
	name, err := input.String("data", "source", "name")
	if err != nil {
		logrus.Error(err)
	}

	source := Source{
		URL:  url,
		Name: name,
	}

	// CertStreamData
	updateType, err := input.String("data", "update_type")
	certIndex, err := input.Int("data", "cert_index")
	seen, err := input.Int("data", "seen")
	chain, err := input.ArrayOfObjects("data", "chain")
	chainSlice := []LeafCertStruct{}
	for i := 0; i < len(chain); i++ {
		c, err := extractLeafCertChainStruct(input, strconv.Itoa(i))
		if err != nil {
			logrus.Error("error extractLeafCertChainStruct: ", err)
		}
		chainSlice = append(chainSlice, c)
	}

	csd := CertStreamData{
		UpdateType: updateType,
		LeafCert:   leafCertStruct,
		Chain:      chainSlice,
		CertIndex:  certIndex,
		Seen:       seen,
		Source:     source,
	}

	// CertStreamStruct
	messageType, err := input.String("message_type")
	if err != nil {
		logrus.Error(err)
	}

	res := CertStreamStruct{
		MessageType: messageType,
		Data:        csd,
	}

	return &res, nil

}

func extractLeafCertChainStruct(input jsonq.JsonQuery, index string) (LeafCertStruct, error) {
	// LeafCertStruct > Subject
	aggregated, _ := input.String("data", "chain", index, "subject", "aggregated")
	c, _ := input.String("data", "chain", index, "subject", "C")
	st, _ := input.String("data", "chain", index, "subject", "ST")
	l, _ := input.String("data", "chain", index, "subject", "L")
	o, _ := input.String("data", "chain", index, "subject", "O")
	ou, _ := input.String("data", "chain", index, "subject", "OU")
	cn, _ := input.String("data", "chain", index, "subject", "CN")

	subject := LeafCertSubject{
		Aggregated: aggregated,
		C:          c,
		ST:         st,
		L:          l,
		O:          o,
		OU:         ou,
		CN:         cn,
	}

	// LeafCertStruct > Extensions
	keyUsage, _ := input.String("data", "chain", index, "extensions", "keyUsage")
	extendedKeyUsage, _ := input.String("data", "chain", index, "extensions", "extendedKeyUsage")
	basicConstrains, _ := input.String("data", "chain", index, "extensions", "basicConstrains")
	subjectKeyIdentifier, _ := input.String("data", "chain", index, "extensions", "subjectKeyIdentifier")
	authorityInfoAccess, _ := input.String("data", "chain", index, "extensions", "authorityInfoAccess")
	subjectAltName, _ := input.String("data", "chain", index, "extensions", "subjectAltName")
	certificatePolicies, _ := input.String("data", "chain", index, "extensions", "certificatePolicies")

	extensions := LeafCertExtensions{
		KeyUsage:             keyUsage,
		ExtendedKeyUsage:     extendedKeyUsage,
		BasicConstrains:      basicConstrains,
		SubjectKeyIdentifier: subjectKeyIdentifier,
		AuthorityInfoAccess:  authorityInfoAccess,
		SubjectAltName:       subjectAltName,
		CertificatePolicies:  certificatePolicies,
	}

	notBefore, _ := input.Int("data", "chain", "not_before")
	notAfter, _ := input.Int("data", "chain", "not_after")
	serialNumber, _ := input.String("data", "chain", "serialNumber")
	fingerprint, _ := input.String("data", "chain", "fingerprint")
	asDer, _ := input.String("data", "chain", "as_der")
	allDomains, _ := input.ArrayOfStrings("data", "chain", "all_domains")

	return LeafCertStruct{
		Subject:      subject,
		Extensions:   extensions,
		NotBefore:    notBefore,
		NotAfter:     notAfter,
		SerialNumber: serialNumber,
		Fingerprint:  fingerprint,
		AsDer:        asDer,
		AllDomains:   allDomains,
	}, nil

}
func extractLeafCertStruct(input jsonq.JsonQuery) (LeafCertStruct, error) {
	// LeafCertStruct > Subject
	aggregated, err := input.String("data", "leaf_cert", "subject", "aggregated")
	c, err := input.String("data", "leaf_cert", "subject", "C")
	st, err := input.String("data", "leaf_cert", "subject", "ST")
	l, err := input.String("data", "leaf_cert", "subject", "L")
	o, err := input.String("data", "leaf_cert", "subject", "O")
	ou, err := input.String("data", "leaf_cert", "subject", "OU")
	cn, err := input.String("data", "leaf_cert", "subject", "CN")
	if err != nil {
		logrus.Error(err)
	}

	subject := LeafCertSubject{
		Aggregated: aggregated,
		C:          c,
		ST:         st,
		L:          l,
		O:          o,
		OU:         ou,
		CN:         cn,
	}

	// LeafCertStruct > Extensions
	keyUsage, _ := input.String("data", "leaf_cert", "extensions", "keyUsage")
	extendedKeyUsage, _ := input.String("data", "leaf_cert", "extensions", "extendedKeyUsage")
	basicConstrains, _ := input.String("data", "leaf_cert", "extensions", "basicConstrains")
	subjectKeyIdentifier, _ := input.String("data", "leaf_cert", "extensions", "subjectKeyIdentifier")
	authorityInfoAccess, _ := input.String("data", "leaf_cert", "extensions", "authorityInfoAccess")
	subjectAltName, _ := input.String("data", "leaf_cert", "extensions", "subjectAltName")
	certificatePolicies, _ := input.String("data", "leaf_cert", "extensions", "certificatePolicies")

	extensions := LeafCertExtensions{
		KeyUsage:             keyUsage,
		ExtendedKeyUsage:     extendedKeyUsage,
		BasicConstrains:      basicConstrains,
		SubjectKeyIdentifier: subjectKeyIdentifier,
		AuthorityInfoAccess:  authorityInfoAccess,
		SubjectAltName:       subjectAltName,
		CertificatePolicies:  certificatePolicies,
	}

	notBefore, _ := input.Int("data", "leaf_cert", "not_before")
	notAfter, _ := input.Int("data", "leaf_cert", "not_after")
	serialNumber, _ := input.String("data", "leaf_cert", "serialNumber")
	fingerprint, _ := input.String("data", "leaf_cert", "fingerprint")
	asDer, _ := input.String("data", "leaf_cert", "as_der")
	allDomains, _ := input.ArrayOfStrings("data", "leaf_cert", "all_domains")

	return LeafCertStruct{
		Subject:      subject,
		Extensions:   extensions,
		NotBefore:    notBefore,
		NotAfter:     notAfter,
		SerialNumber: serialNumber,
		Fingerprint:  fingerprint,
		AsDer:        asDer,
		AllDomains:   allDomains,
	}, nil

}

// Meta Information: https://pastebin.com/api_scraping.php
// Content: http://pastebin.com/api_scrape_item.php

// QueryPastes returns metadata for the last 100 public pastes.
func QueryPastes() ([]PasteMeta, error) {
	server := "pastebin.com"
	req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/api_scraping.php?limit=100", server), nil)

	if err != nil {
		logrus.Fatal("Could not build http request", err)
		return nil, err
	}
	req.Header.Set("Content-Type", "application/json")

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		logrus.Error("Could not do requeest due to %v", err)
		return nil, err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		logrus.Error("Could not fetch response due to %v", err)
		return nil, err
	}

	var pastes []PasteMeta
	if err := json.Unmarshal(body, &pastes); err != nil {
		logrus.Error("Could not decode response due to %v, body %s", err, string(body))
		return nil, err
	}

	return pastes, err
}

// FetchPaste fetches paste contents via the web API.
func FetchPaste(paste PasteMeta) (string, error) {
	url := paste.ScrapeURL
	req, err := http.NewRequest("GET", url, nil)
	if err != nil {
		log.Printf("Could build request %v due to %v", req, err)
		return "", err
	}

	client := &http.Client{}
	resp, err := client.Do(req)
	if err != nil {
		log.Printf("Could not do request %v due to %v", req, err)
		return "", err
	}
	defer resp.Body.Close()

	body, err := ioutil.ReadAll(resp.Body)
	if err != nil {
		log.Printf("Could not read response body %v due to %v", resp.Body, err)
		return "", err
	}

	return string(body), nil
}