From 7163147a4fe87b95d41688af4080f46f2eacab6f Mon Sep 17 00:00:00 2001 From: Christopher Talib Date: Tue, 19 May 2020 10:10:42 +0200 Subject: [PATCH] Pastebin nodes simple Pastebin data is also sent to Dgraph and can be queried. --- README.md | 35 +++++++++++++++++++++++++++++++++++ graph/main.go | 41 +++++++++++++++++++++++++++++++++++++++++ main.go | 2 +- models/main.go | 25 +++++++++++++------------ models/pastebin.go | 42 ++++++++++++++++++++++-------------------- plugins/pastebin.go | 41 ++++++++++++++++++++++++++++++++++++----- 6 files changed, 148 insertions(+), 38 deletions(-) diff --git a/README.md b/README.md index 53ac2c5..b5a72f9 100644 --- a/README.md +++ b/README.md @@ -137,6 +137,41 @@ query { } ``` +Example query for pastebin data: + +```graphql +query { + Node(func: eq(type, "pastebin")) { + uid + created + modified + type + ndata + pasteNode { + id + type + created + modified + fullPaste { + full + meta { + full_url + size + expire + title + syntax + user + scrape_url + date + key + } + } + } + } +} + +``` + ## Datastructure ### Meta diff --git a/graph/main.go b/graph/main.go index fba0227..f48c86d 100644 --- a/graph/main.go +++ b/graph/main.go @@ -46,6 +46,7 @@ created: string . modified: string . certNode: uid . shodanNode: uid . +pasteNode: uid . type Node { id: string @@ -55,6 +56,7 @@ created: string modified: string certNode: CertNode shodanNode: ShodanNode +pasteNode: PasteNode } type Edge { @@ -143,6 +145,45 @@ domains: [string] timestamp: string } +fullPaste: uid . +meta: uid . + +full: string . +scrape_url: string . +full_url: string . +date: string . +key: string . +size: string . +expire: string . +title: string . +syntax: string . +user: string . + +type PasteMeta { +scrape_url: string +full_url: string +date: string +key: string +size: string +expire: string +title: string +syntax: string +user: string +} + +type FullPaste { +meta: PasteMeta +full: string +} + +type PasteNode { +id: string +type: string +created: string +modified: string +fullPaste: FullPaste +} + `}) if err != nil { return err diff --git a/main.go b/main.go index c1476ae..8e5c2c0 100644 --- a/main.go +++ b/main.go @@ -60,7 +60,7 @@ func main() { if ok := p.Initialize(); !ok { logrus.Info("pastebin plugin not activated") } else { - p.Run(&wg) + p.Run(&wg, dgraphClient) } // shodan diff --git a/models/main.go b/models/main.go index 0479e35..7fc80f3 100644 --- a/models/main.go +++ b/models/main.go @@ -33,6 +33,7 @@ type Node struct { DType []string `json:"dgraph.type,omiempty"` CertNode CertNode `json:"certNode,omiempty"` ShodanNode ShodanNode `json:"shodanNode,omiempty"` + PasteNode PasteNode `json:"pasteNode,omiempty"` } // BuildNode builds a node to send to MQ instance. @@ -259,17 +260,17 @@ func SaveCertNode(filename string, node *CertNode) { // PasteNode is a node from PasteBin. type PasteNode struct { - ID string `json:"id"` - Type string `json:"type"` - Data FullPaste `json:"data"` - Created string `json:"create"` - Modified string `json:"modified"` + ID string `json:"id,omiempty"` + Type string `json:"type,omiempty"` + FullPaste FullPaste `json:"fullPaste,omiempty"` + Created string `json:"create,omiempty"` + Modified string `json:"modified,omiempty"` } // FullPaste wrapes meta and information from Pastebin. type FullPaste struct { - Meta PasteMeta `json:"meta"` - Full string `json:"full"` + Meta PasteMeta `json:"meta,omiempty"` + Full string `json:"full,omiempty"` } // BuildPasteNode builds a node from a FullPaste data. @@ -277,11 +278,11 @@ func BuildPasteNode(data *FullPaste) *PasteNode { t := time.Now() rfc3339time := t.Format(time.RFC3339) return &PasteNode{ - ID: "pastebin--" + uuid.New().String(), - Type: "pastebin", - Data: *data, - Created: rfc3339time, - Modified: rfc3339time, + ID: "pastebin--" + uuid.New().String(), + Type: "pastebin", + FullPaste: *data, + Created: rfc3339time, + Modified: rfc3339time, } } diff --git a/models/pastebin.go b/models/pastebin.go index 2e98c55..d9a2560 100644 --- a/models/pastebin.go +++ b/models/pastebin.go @@ -11,31 +11,33 @@ import ( ) // PasteMeta is a set of descriptive information on a paste. +// Camel case in the marshaling because it's the Pastebin API structure. type PasteMeta struct { - ScrapeURL string `json:"scrape_url"` - FullURL string `json:"full_url"` - Date string `json:"date"` - Key string `json:"key"` - Size string `json:"size"` - Expire string `json:"expire"` - Title string `json:"title"` - Syntax string `json:"syntax"` - User string `json:"user"` + ScrapeURL string `json:"scrape_url,omiempty"` + FullURL string `json:"full_url,omiempty"` + Date string `json:"date,omiempty"` + Key string `json:"key,omiempty"` + Size string `json:"size,omiempty"` + Expire string `json:"expire,omiempty"` + Title string `json:"title,omiempty"` + Syntax string `json:"syntax,omiempty"` + User string `json:"user,omiempty"` } // PasteFull extends PasteMeta by the actual content. +// Not used in our code. type PasteFull struct { - ScrapeURL string `json:"scrape_url"` - FullURL string `json:"full_url"` - Date string `json:"date"` - Key string `json:"key"` - Size string `json:"size"` - Expire string `json:"expire"` - Title string `json:"title"` - Syntax string `json:"syntax"` - User string `json:"user"` - Data string `json:"data"` - RFC3339 string `json:"time"` + ScrapeURL string `json:"scrapeUrl,omiempty"` + FullURL string `json:"fullUrl,omiempty"` + Date string `json:"date,omiempty"` + Key string `json:"key,omiempty"` + Size string `json:"size,omiempty"` + Expire string `json:"expire,omiempty"` + Title string `json:"title,omiempty"` + Syntax string `json:"syntax,omiempty"` + User string `json:"user,omiempty"` + Data string `json:"data,omiempty"` + RFC3339 string `json:"time,omiempty"` } // Meta Information: https://pastebin.com/api_scraping.php diff --git a/plugins/pastebin.go b/plugins/pastebin.go index 18ccbf2..80754ac 100644 --- a/plugins/pastebin.go +++ b/plugins/pastebin.go @@ -1,9 +1,13 @@ package plugins import ( + "context" + "encoding/json" "sync" "time" + "github.com/dgraph-io/dgo/v2" + "github.com/dgraph-io/dgo/v2/protos/api" "github.com/sirupsen/logrus" "github.com/spf13/viper" "gitlab.dcso.lolcat/LABS/styx/models" @@ -26,11 +30,11 @@ func (p *PastebinPlugin) Initialize() bool { } // Run runs the Pastebin plugin. -func (p *PastebinPlugin) Run(wg *sync.WaitGroup) { +func (p *PastebinPlugin) Run(wg *sync.WaitGroup, dgraphClient *dgo.Dgraph) { if !p.Running { p.StoppedChan = make(chan bool) wg.Add(1) - go p.doRun() + go p.doRun(dgraphClient) p.Running = true } } @@ -46,7 +50,7 @@ func (p *PastebinPlugin) Stop(wg *sync.WaitGroup) { } } -func (p *PastebinPlugin) doRun() { +func (p *PastebinPlugin) doRun(graphClient *dgo.Dgraph) { for { select { default: @@ -63,11 +67,38 @@ func (p *PastebinPlugin) doRun() { Meta: p, Full: paste, } - res := models.BuildPasteNode(&fp) + pasteNode := models.BuildPasteNode(&fp) + mainNode := models.BuildNode("node", "pastebin", pasteNode.ID) // if elastic { // e.StorePaste(fp) // } - models.SavePaste("paste_formatted.json", res) + // models.SavePaste("paste_formatted.json", res) + + e := models.Node{ + ID: mainNode.ID, + Type: mainNode.Type, + NData: mainNode.NData, + Created: mainNode.Created, + Modified: mainNode.Modified, + PasteNode: *pasteNode, + } + + ctx := context.Background() + mu := &api.Mutation{ + CommitNow: true, + } + + pb, err := json.Marshal(e) + if err != nil { + logrus.Fatal(err) + } + + mu.SetJson = pb + + _, err = graphClient.NewTxn().Mutate(ctx, mu) + if err != nil { + logrus.Fatal(err) + } time.Sleep(1 * time.Second) }