chris/styx
chris/styx
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Pastebin nodes simple

Browse source 
Pastebin data is also sent to Dgraph and can be queried.
This commit is contained in:
Christopher Talib 2020-05-19 10:10:42 +02:00
parent b1ca4b3c5f
commit 7163147a4f
6 changed files with 148 additions and 38 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
README.md
View file

@ -137,6 +137,41 @@ query {
} }
``` ```
Example query for pastebin data:
```graphql
query {
Node(func: eq(type, "pastebin")) {
uid
created
modified
type
ndata
pasteNode {
id
type
created
modified
fullPaste {
full
meta {
full_url
size
expire
title
syntax
user
scrape_url
date
key
}
}
}
}
}
```
## Datastructure ## Datastructure
### Meta ### Meta

41
graph/main.go
View file

@ -46,6 +46,7 @@ created: string .
modified: string . modified: string .
certNode: uid . certNode: uid .
shodanNode: uid . shodanNode: uid .
pasteNode: uid .
type Node { type Node {
id: string id: string
@ -55,6 +56,7 @@ created: string
modified: string modified: string
certNode: CertNode certNode: CertNode
shodanNode: ShodanNode shodanNode: ShodanNode
pasteNode: PasteNode
} }
type Edge { type Edge {
@ -143,6 +145,45 @@ domains: [string]
timestamp: string timestamp: string
} }
fullPaste: uid .
meta: uid .
full: string .
scrape_url: string .
full_url: string .
date: string .
key: string .
size: string .
expire: string .
title: string .
syntax: string .
user: string .
type PasteMeta {
scrape_url: string
full_url: string
date: string
key: string
size: string
expire: string
title: string
syntax: string
user: string
}
type FullPaste {
meta: PasteMeta
full: string
}
type PasteNode {
id: string
type: string
created: string
modified: string
fullPaste: FullPaste
}
`}) `})
if err != nil { if err != nil {
return err return err

2
main.go
View file

@ -60,7 +60,7 @@ func main() {
if ok := p.Initialize(); !ok { if ok := p.Initialize(); !ok {
logrus.Info("pastebin plugin not activated") logrus.Info("pastebin plugin not activated")
} else { } else {
p.Run(&wg) p.Run(&wg, dgraphClient)
} }
// shodan // shodan

25
models/main.go
View file

@ -33,6 +33,7 @@ type Node struct {
DType []string `json:"dgraph.type,omiempty"` DType []string `json:"dgraph.type,omiempty"`
CertNode CertNode `json:"certNode,omiempty"` CertNode CertNode `json:"certNode,omiempty"`
ShodanNode ShodanNode `json:"shodanNode,omiempty"` ShodanNode ShodanNode `json:"shodanNode,omiempty"`
PasteNode PasteNode `json:"pasteNode,omiempty"`
} }
// BuildNode builds a node to send to MQ instance. // BuildNode builds a node to send to MQ instance.
@ -259,17 +260,17 @@ func SaveCertNode(filename string, node *CertNode) {
// PasteNode is a node from PasteBin. // PasteNode is a node from PasteBin.
type PasteNode struct { type PasteNode struct {
ID string `json:"id"` ID string `json:"id,omiempty"`
Type string `json:"type"` Type string `json:"type,omiempty"`
Data FullPaste `json:"data"` FullPaste FullPaste `json:"fullPaste,omiempty"`
Created string `json:"create"` Created string `json:"create,omiempty"`
Modified string `json:"modified"` Modified string `json:"modified,omiempty"`
} }
// FullPaste wrapes meta and information from Pastebin. // FullPaste wrapes meta and information from Pastebin.
type FullPaste struct { type FullPaste struct {
Meta PasteMeta `json:"meta"` Meta PasteMeta `json:"meta,omiempty"`
Full string `json:"full"` Full string `json:"full,omiempty"`
} }
// BuildPasteNode builds a node from a FullPaste data. // BuildPasteNode builds a node from a FullPaste data.
@ -277,11 +278,11 @@ func BuildPasteNode(data *FullPaste) *PasteNode {
t := time.Now() t := time.Now()
rfc3339time := t.Format(time.RFC3339) rfc3339time := t.Format(time.RFC3339)
return &PasteNode{ return &PasteNode{
ID: "pastebin--" + uuid.New().String(), ID: "pastebin--" + uuid.New().String(),
Type: "pastebin", Type: "pastebin",
Data: *data, FullPaste: *data,
Created: rfc3339time, Created: rfc3339time,
Modified: rfc3339time, Modified: rfc3339time,
} }
} }

42
models/pastebin.go
View file

@ -11,31 +11,33 @@ import (
) )
// PasteMeta is a set of descriptive information on a paste. // PasteMeta is a set of descriptive information on a paste.
// Camel case in the marshaling because it's the Pastebin API structure.
type PasteMeta struct { type PasteMeta struct {
ScrapeURL string `json:"scrape_url"` ScrapeURL string `json:"scrape_url,omiempty"`
FullURL string `json:"full_url"` FullURL string `json:"full_url,omiempty"`
Date string `json:"date"` Date string `json:"date,omiempty"`
Key string `json:"key"` Key string `json:"key,omiempty"`
Size string `json:"size"` Size string `json:"size,omiempty"`
Expire string `json:"expire"` Expire string `json:"expire,omiempty"`
Title string `json:"title"` Title string `json:"title,omiempty"`
Syntax string `json:"syntax"` Syntax string `json:"syntax,omiempty"`
User string `json:"user"` User string `json:"user,omiempty"`
} }
// PasteFull extends PasteMeta by the actual content. // PasteFull extends PasteMeta by the actual content.
// Not used in our code.
type PasteFull struct { type PasteFull struct {
ScrapeURL string `json:"scrape_url"` ScrapeURL string `json:"scrapeUrl,omiempty"`
FullURL string `json:"full_url"` FullURL string `json:"fullUrl,omiempty"`
Date string `json:"date"` Date string `json:"date,omiempty"`
Key string `json:"key"` Key string `json:"key,omiempty"`
Size string `json:"size"` Size string `json:"size,omiempty"`
Expire string `json:"expire"` Expire string `json:"expire,omiempty"`
Title string `json:"title"` Title string `json:"title,omiempty"`
Syntax string `json:"syntax"` Syntax string `json:"syntax,omiempty"`
User string `json:"user"` User string `json:"user,omiempty"`
Data string `json:"data"` Data string `json:"data,omiempty"`
RFC3339 string `json:"time"` RFC3339 string `json:"time,omiempty"`
} }
// Meta Information: https://pastebin.com/api_scraping.php // Meta Information: https://pastebin.com/api_scraping.php

41
plugins/pastebin.go
View file

@ -1,9 +1,13 @@
package plugins package plugins
import ( import (
"context"
"encoding/json"
"sync" "sync"
"time" "time"
"github.com/dgraph-io/dgo/v2"
"github.com/dgraph-io/dgo/v2/protos/api"
"github.com/sirupsen/logrus" "github.com/sirupsen/logrus"
"github.com/spf13/viper" "github.com/spf13/viper"
"gitlab.dcso.lolcat/LABS/styx/models" "gitlab.dcso.lolcat/LABS/styx/models"
@ -26,11 +30,11 @@ func (p *PastebinPlugin) Initialize() bool {
} }
// Run runs the Pastebin plugin. // Run runs the Pastebin plugin.
func (p *PastebinPlugin) Run(wg *sync.WaitGroup) { func (p *PastebinPlugin) Run(wg *sync.WaitGroup, dgraphClient *dgo.Dgraph) {
if !p.Running { if !p.Running {
p.StoppedChan = make(chan bool) p.StoppedChan = make(chan bool)
wg.Add(1) wg.Add(1)
go p.doRun() go p.doRun(dgraphClient)
p.Running = true p.Running = true
} }
} }
@ -46,7 +50,7 @@ func (p *PastebinPlugin) Stop(wg *sync.WaitGroup) {
} }
} }
func (p *PastebinPlugin) doRun() { func (p *PastebinPlugin) doRun(graphClient *dgo.Dgraph) {
for { for {
select { select {
default: default:
@ -63,11 +67,38 @@ func (p *PastebinPlugin) doRun() {
Meta: p, Meta: p,
Full: paste, Full: paste,
} }
res := models.BuildPasteNode(&fp) pasteNode := models.BuildPasteNode(&fp)
mainNode := models.BuildNode("node", "pastebin", pasteNode.ID)
// if elastic { // if elastic {
// e.StorePaste(fp) // e.StorePaste(fp)
// } // }
models.SavePaste("paste_formatted.json", res) // models.SavePaste("paste_formatted.json", res)
e := models.Node{
ID: mainNode.ID,
Type: mainNode.Type,
NData: mainNode.NData,
Created: mainNode.Created,
Modified: mainNode.Modified,
PasteNode: *pasteNode,
}
ctx := context.Background()
mu := &api.Mutation{
CommitNow: true,
}
pb, err := json.Marshal(e)
if err != nil {
logrus.Fatal(err)
}
mu.SetJson = pb
_, err = graphClient.NewTxn().Mutate(ctx, mu)
if err != nil {
logrus.Fatal(err)
}
time.Sleep(1 * time.Second) time.Sleep(1 * time.Second)
} }