0cd0faa6fa
Parallel running of Certstream and Pastebin connectors in go routine with wait groups to exit them properly.
129 lines
3.1 KiB
Go
129 lines
3.1 KiB
Go
package models
|
|
|
|
import (
|
|
"encoding/json"
|
|
"fmt"
|
|
"io/ioutil"
|
|
"log"
|
|
"net/http"
|
|
"time"
|
|
|
|
"github.com/google/uuid"
|
|
"github.com/sirupsen/logrus"
|
|
)
|
|
|
|
type PasteWrapper struct {
|
|
ID string `json:"id"`
|
|
Type string `json:"type"`
|
|
Data FullPaste `json:"data"`
|
|
Created string `json:"create"`
|
|
Modified string `json:"modified"`
|
|
}
|
|
|
|
type FullPaste struct {
|
|
Meta PasteMeta `json:"meta"`
|
|
Full string `json:"full"`
|
|
}
|
|
|
|
// PasteMeta is a set of descriptive information on a paste.
|
|
type PasteMeta struct {
|
|
ScrapeURL string `json:"scrape_url"`
|
|
FullURL string `json:"full_url"`
|
|
Date string `json:"date"`
|
|
Key string `json:"key"`
|
|
Size string `json:"size"`
|
|
Expire string `json:"expire"`
|
|
Title string `json:"title"`
|
|
Syntax string `json:"syntax"`
|
|
User string `json:"user"`
|
|
}
|
|
|
|
// PasteFull extends PasteMeta by the actual content.
|
|
type PasteFull struct {
|
|
ScrapeURL string `json:"scrape_url"`
|
|
FullURL string `json:"full_url"`
|
|
Date string `json:"date"`
|
|
Key string `json:"key"`
|
|
Size string `json:"size"`
|
|
Expire string `json:"expire"`
|
|
Title string `json:"title"`
|
|
Syntax string `json:"syntax"`
|
|
User string `json:"user"`
|
|
Data string `json:"data"`
|
|
RFC3339 string `json:"time"`
|
|
}
|
|
|
|
// Meta Information: https://pastebin.com/api_scraping.php
|
|
// Content: http://pastebin.com/api_scrape_item.php
|
|
|
|
// QueryPastes returns metadata for the last 100 public pastes.
|
|
func QueryPastes() ([]PasteMeta, error) {
|
|
server := "scrape.pastebin.com"
|
|
req, err := http.NewRequest("GET", fmt.Sprintf("https://%s/api_scraping.php?limit=100", server), nil)
|
|
|
|
if err != nil {
|
|
logrus.Fatal("Could not build http request", err)
|
|
return nil, err
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
|
|
client := &http.Client{}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
logrus.Error("Could not do request due to", err)
|
|
return nil, err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, err := ioutil.ReadAll(resp.Body)
|
|
if err != nil {
|
|
logrus.Error("Could not fetch response due to", err)
|
|
return nil, err
|
|
}
|
|
|
|
var pastes []PasteMeta
|
|
if err := json.Unmarshal(body, &pastes); err != nil {
|
|
logrus.Error("Could not decode response due to ", err, " body", string(body))
|
|
return nil, err
|
|
}
|
|
|
|
return pastes, err
|
|
}
|
|
|
|
// FetchPaste fetches paste contents via the web API.
|
|
func FetchPaste(paste PasteMeta) (string, error) {
|
|
url := paste.ScrapeURL
|
|
req, err := http.NewRequest("GET", url, nil)
|
|
if err != nil {
|
|
logrus.Error("Could build request", req, " due to", err)
|
|
return "", err
|
|
}
|
|
|
|
client := &http.Client{}
|
|
resp, err := client.Do(req)
|
|
if err != nil {
|
|
log.Printf("Could not do request %v due to %v", req, err)
|
|
return "", err
|
|
}
|
|
defer resp.Body.Close()
|
|
|
|
body, err := ioutil.ReadAll(resp.Body)
|
|
if err != nil {
|
|
log.Printf("Could not read response body %v due to %v", resp.Body, err)
|
|
return "", err
|
|
}
|
|
|
|
return string(body), nil
|
|
}
|
|
|
|
func BuildPasteNode(data *FullPaste) *PasteWrapper {
|
|
t := time.Now()
|
|
rfc3339time := t.Format(time.RFC3339)
|
|
return &PasteWrapper{
|
|
ID: "pastebin--" + uuid.New().String(),
|
|
Type: "pastebin",
|
|
Data: *data,
|
|
Created: rfc3339time,
|
|
Modified: rfc3339time,
|
|
}
|
|
}
|