Pastebin nodes simple

Pastebin data is also sent to Dgraph and can be queried.

README.md

@@ -137,6 +137,41 @@ query {
 }
 ```
 
+Example query for pastebin data:
+
+```graphql
+query {
+  Node(func: eq(type, "pastebin")) {
+    uid
+    created
+    modified
+    type
+    ndata
+    pasteNode {
+      id
+      type
+      created
+      modified
+      fullPaste {
+        full
+        meta {
+          full_url
+          size
+          expire
+          title
+          syntax
+          user
+          scrape_url
+          date
+          key
+        }
+      }
+    }
+  }
+}
+
+```
+
 ## Datastructure
 
 ### Meta

graph/main.go

@@ -46,6 +46,7 @@ created: string .
 modified: string .
 certNode: uid .
 shodanNode: uid .
+pasteNode: uid .
 
 type Node {
 id: string
@@ -55,6 +56,7 @@ created: string
 modified: string
 certNode: CertNode
 shodanNode: ShodanNode
+pasteNode: PasteNode
 }
 
 type Edge {
@@ -143,6 +145,45 @@ domains: [string]
 timestamp: string
 }
 
+fullPaste: uid .
+meta: uid .
+
+full: string .
+scrape_url: string .
+full_url: string .
+date: string .
+key: string .
+size: string .
+expire: string .
+title: string .
+syntax: string .
+user: string .
+
+type PasteMeta {
+scrape_url: string
+full_url: string
+date: string
+key: string
+size: string
+expire: string
+title: string
+syntax: string
+user: string
+}
+
+type FullPaste {
+meta: PasteMeta
+full: string
+}
+
+type PasteNode {
+id: string
+type: string
+created: string
+modified: string
+fullPaste: FullPaste
+}
+
 `})
 	if err != nil {
 		return err

main.go

@@ -60,7 +60,7 @@ func main() {
 	if ok := p.Initialize(); !ok {
 		logrus.Info("pastebin plugin not activated")
 	} else {
-		p.Run(&wg)
+		p.Run(&wg, dgraphClient)
 	}
 
 	// shodan

models/main.go

@@ -33,6 +33,7 @@ type Node struct {
 	DType      []string   `json:"dgraph.type,omiempty"`
 	CertNode   CertNode   `json:"certNode,omiempty"`
 	ShodanNode ShodanNode `json:"shodanNode,omiempty"`
+	PasteNode  PasteNode  `json:"pasteNode,omiempty"`
 }
 
 // BuildNode builds a node to send to MQ instance.
@@ -259,17 +260,17 @@ func SaveCertNode(filename string, node *CertNode) {
 
 // PasteNode is a node from PasteBin.
 type PasteNode struct {
-	ID       string    `json:"id"`
-	Type     string    `json:"type"`
-	Data     FullPaste `json:"data"`
-	Created  string    `json:"create"`
-	Modified string    `json:"modified"`
+	ID        string    `json:"id,omiempty"`
+	Type      string    `json:"type,omiempty"`
+	FullPaste FullPaste `json:"fullPaste,omiempty"`
+	Created   string    `json:"create,omiempty"`
+	Modified  string    `json:"modified,omiempty"`
 }
 
 // FullPaste wrapes meta and information from Pastebin.
 type FullPaste struct {
-	Meta PasteMeta `json:"meta"`
-	Full string    `json:"full"`
+	Meta PasteMeta `json:"meta,omiempty"`
+	Full string    `json:"full,omiempty"`
 }
 
 // BuildPasteNode builds a node from a FullPaste data.
@@ -277,11 +278,11 @@ func BuildPasteNode(data *FullPaste) *PasteNode {
 	t := time.Now()
 	rfc3339time := t.Format(time.RFC3339)
 	return &PasteNode{
-		ID:       "pastebin--" + uuid.New().String(),
-		Type:     "pastebin",
-		Data:     *data,
-		Created:  rfc3339time,
-		Modified: rfc3339time,
+		ID:        "pastebin--" + uuid.New().String(),
+		Type:      "pastebin",
+		FullPaste: *data,
+		Created:   rfc3339time,
+		Modified:  rfc3339time,
 	}
 }
 

models/pastebin.go

@@ -11,31 +11,33 @@ import (
 )
 
 // PasteMeta is a set of descriptive information on a paste.
+// Camel case in the marshaling because it's the Pastebin API structure.
 type PasteMeta struct {
-	ScrapeURL string `json:"scrape_url"`
-	FullURL   string `json:"full_url"`
-	Date      string `json:"date"`
-	Key       string `json:"key"`
-	Size      string `json:"size"`
-	Expire    string `json:"expire"`
-	Title     string `json:"title"`
-	Syntax    string `json:"syntax"`
-	User      string `json:"user"`
+	ScrapeURL string `json:"scrape_url,omiempty"`
+	FullURL   string `json:"full_url,omiempty"`
+	Date      string `json:"date,omiempty"`
+	Key       string `json:"key,omiempty"`
+	Size      string `json:"size,omiempty"`
+	Expire    string `json:"expire,omiempty"`
+	Title     string `json:"title,omiempty"`
+	Syntax    string `json:"syntax,omiempty"`
+	User      string `json:"user,omiempty"`
 }
 
 // PasteFull extends PasteMeta by the actual content.
+// Not used in our code.
 type PasteFull struct {
-	ScrapeURL string `json:"scrape_url"`
-	FullURL   string `json:"full_url"`
-	Date      string `json:"date"`
-	Key       string `json:"key"`
-	Size      string `json:"size"`
-	Expire    string `json:"expire"`
-	Title     string `json:"title"`
-	Syntax    string `json:"syntax"`
-	User      string `json:"user"`
-	Data      string `json:"data"`
-	RFC3339   string `json:"time"`
+	ScrapeURL string `json:"scrapeUrl,omiempty"`
+	FullURL   string `json:"fullUrl,omiempty"`
+	Date      string `json:"date,omiempty"`
+	Key       string `json:"key,omiempty"`
+	Size      string `json:"size,omiempty"`
+	Expire    string `json:"expire,omiempty"`
+	Title     string `json:"title,omiempty"`
+	Syntax    string `json:"syntax,omiempty"`
+	User      string `json:"user,omiempty"`
+	Data      string `json:"data,omiempty"`
+	RFC3339   string `json:"time,omiempty"`
 }
 
 // Meta Information: https://pastebin.com/api_scraping.php

plugins/pastebin.go

@@ -1,9 +1,13 @@
 package plugins
 
 import (
+	"context"
+	"encoding/json"
 	"sync"
 	"time"
 
+	"github.com/dgraph-io/dgo/v2"
+	"github.com/dgraph-io/dgo/v2/protos/api"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/viper"
 	"gitlab.dcso.lolcat/LABS/styx/models"
@@ -26,11 +30,11 @@ func (p *PastebinPlugin) Initialize() bool {
 }
 
 // Run runs the Pastebin plugin.
-func (p *PastebinPlugin) Run(wg *sync.WaitGroup) {
+func (p *PastebinPlugin) Run(wg *sync.WaitGroup, dgraphClient *dgo.Dgraph) {
 	if !p.Running {
 		p.StoppedChan = make(chan bool)
 		wg.Add(1)
-		go p.doRun()
+		go p.doRun(dgraphClient)
 		p.Running = true
 	}
 }
@@ -46,7 +50,7 @@ func (p *PastebinPlugin) Stop(wg *sync.WaitGroup) {
 	}
 }
 
-func (p *PastebinPlugin) doRun() {
+func (p *PastebinPlugin) doRun(graphClient *dgo.Dgraph) {
 	for {
 		select {
 		default:
@@ -63,11 +67,38 @@ func (p *PastebinPlugin) doRun() {
 					Meta: p,
 					Full: paste,
 				}
-				res := models.BuildPasteNode(&fp)
+				pasteNode := models.BuildPasteNode(&fp)
+				mainNode := models.BuildNode("node", "pastebin", pasteNode.ID)
 				// if elastic {
 				// e.StorePaste(fp)
 				// }
-				models.SavePaste("paste_formatted.json", res)
+				// models.SavePaste("paste_formatted.json", res)
+
+				e := models.Node{
+					ID:        mainNode.ID,
+					Type:      mainNode.Type,
+					NData:     mainNode.NData,
+					Created:   mainNode.Created,
+					Modified:  mainNode.Modified,
+					PasteNode: *pasteNode,
+				}
+
+				ctx := context.Background()
+				mu := &api.Mutation{
+					CommitNow: true,
+				}
+
+				pb, err := json.Marshal(e)
+				if err != nil {
+					logrus.Fatal(err)
+				}
+
+				mu.SetJson = pb
+
+				_, err = graphClient.NewTxn().Mutate(ctx, mu)
+				if err != nil {
+					logrus.Fatal(err)
+				}
 				time.Sleep(1 * time.Second)
 
 			}