Adding code and setting up elastic search
This commit is contained in:
Christopher Talib
parent
5b1bfbc195
commit
f4f82d9e1c
150
elasticsearch/main.go
Normal file
150
elasticsearch/main.go
Normal file
|
|
@ -0,0 +1,150 @@
|
|||
package elasticsearch
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"gitlab.dcso.lolcat/LABS/styx/models"
|
||||
elastic "gopkg.in/olivere/elastic.v5"
|
||||
)
|
||||
|
||||
const mapping = `{
|
||||
"settings":{
|
||||
"number_of_shards":1,
|
||||
"number_of_replicas":0
|
||||
},
|
||||
"mappings":{
|
||||
"_default_":{
|
||||
"_all":{
|
||||
"enabled":true
|
||||
}
|
||||
},
|
||||
"paste":{
|
||||
"properties":{
|
||||
"key":{
|
||||
"type":"keyword"
|
||||
},
|
||||
"data":{
|
||||
"type":"text",
|
||||
"store":true,
|
||||
"fielddata":true
|
||||
},
|
||||
"size":{
|
||||
"type":"long"
|
||||
},
|
||||
"syntax":{
|
||||
"type":"keyword"
|
||||
},
|
||||
"date":{
|
||||
"type":"date",
|
||||
"format":"epoch_second"
|
||||
},
|
||||
"expire":{
|
||||
"type":"date",
|
||||
"format":"epoch_second"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}`
|
||||
|
||||
// ElasticStorageModule is a StorageModule that stores pastes in an
|
||||
// Elasticsearch instance.
|
||||
type ElasticStorageModule struct {
|
||||
Index string
|
||||
ElasticURL string
|
||||
DailyIndexes bool
|
||||
UseIndex string
|
||||
LastChk time.Time
|
||||
Client *elastic.Client
|
||||
}
|
||||
|
||||
func (e *ElasticStorageModule) makeIndexIfNotExists() error {
|
||||
index := e.Index
|
||||
if e.DailyIndexes {
|
||||
index = time.Now().Format(fmt.Sprintf("%s-2006-1-2", e.Index))
|
||||
}
|
||||
|
||||
exists, err := e.Client.IndexExists(index).Do(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if !exists {
|
||||
log.Printf("Creating new index %s", index)
|
||||
e.Client.CreateIndex(index).Body(mapping).Do(context.Background())
|
||||
}
|
||||
|
||||
e.UseIndex = index
|
||||
return nil
|
||||
}
|
||||
|
||||
// Initialize prepares the storage modules for use.
|
||||
func (e *ElasticStorageModule) Initialize() error {
|
||||
log.Printf("Connecting to %v", e.ElasticURL)
|
||||
var err error
|
||||
e.Index = "pastebin"
|
||||
e.Client, err = elastic.NewSimpleClient(elastic.SetURL(e.ElasticURL))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
info, code, err := e.Client.Ping(e.ElasticURL).Do(context.Background())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
log.Printf("Elasticsearch returned with code %d and version %s", code,
|
||||
info.Version.Number)
|
||||
|
||||
err = e.makeIndexIfNotExists()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
e.LastChk = time.Now()
|
||||
log.Printf("Using index %s", e.UseIndex)
|
||||
return err
|
||||
}
|
||||
|
||||
// StorePaste stores a single paste in the storage backend.
|
||||
func (e *ElasticStorageModule) StorePaste(paste models.FullPaste) {
|
||||
// if e.DailyIndexes && time.Since(e.LastChk) > 12*time.Hour {
|
||||
// e.makeIndexIfNotExists()
|
||||
// e.LastChk = time.Now()
|
||||
// }
|
||||
_, err := e.Client.Index().
|
||||
Index(e.UseIndex).
|
||||
Type("paste").
|
||||
Id(paste.Meta.Key).
|
||||
BodyJson(paste).
|
||||
Do(context.Background())
|
||||
if err != nil {
|
||||
log.Printf("Could not store paste %s due to %v", paste.Meta.FullURL, err)
|
||||
}
|
||||
}
|
||||
|
||||
// Check returns true if the given paste is new in the current storage backend,
|
||||
// false otherwise.
|
||||
func (e *ElasticStorageModule) Check(paste models.PasteMeta) bool {
|
||||
q := elastic.NewMatchQuery("_id", paste.Key)
|
||||
|
||||
searchResult, err := e.Client.Search().
|
||||
Index(e.Index).
|
||||
Query(q).
|
||||
Pretty(true).
|
||||
Do(context.Background())
|
||||
if err != nil {
|
||||
log.Printf("Could not check paste due to %v", err)
|
||||
return true
|
||||
}
|
||||
|
||||
return (searchResult.TotalHits() == 0)
|
||||
}
|
||||
|
||||
// Destroy finishes all operations on the module.
|
||||
func (e *ElasticStorageModule) Destroy() error {
|
||||
_, err := e.Client.Flush().Index(e.Index).Do(context.Background())
|
||||
return err
|
||||
}
|
||||
5
go.mod
5
go.mod
|
|
@ -15,9 +15,10 @@ require (
|
|||
github.com/segmentio/kafka-go v0.3.4
|
||||
github.com/sirupsen/logrus v1.4.2
|
||||
github.com/spf13/viper v1.6.2
|
||||
gitlab.dcso.lolcat/go/golistic v1.12.14
|
||||
gitlab.dcso.lolcat/go/a3/v3 v3.0.0-dev2 // indirect
|
||||
gitlab.dcso.lolcat/go/golistic v1.12.18
|
||||
gitlab.dcso.lolcat/go/vizor v1.20.0
|
||||
golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d // indirect
|
||||
golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa // indirect
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42 // indirect
|
||||
gopkg.in/olivere/elastic.v5 v5.0.84
|
||||
)
|
||||
|
|
|
|||
31
go.sum
31
go.sum
|
|
@ -13,6 +13,7 @@ github.com/aws/aws-lambda-go v1.8.1/go.mod h1:zUsUQhAUjYzR8AuduJPCfhBuKWUaDbQiPO
|
|||
github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q=
|
||||
github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
|
||||
github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI=
|
||||
github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8=
|
||||
github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
|
||||
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
|
||||
|
|
@ -24,10 +25,13 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc
|
|||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
|
||||
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no=
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw=
|
||||
github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU=
|
||||
github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw=
|
||||
github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g=
|
||||
github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk=
|
||||
github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o=
|
||||
github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I=
|
||||
|
|
@ -61,6 +65,7 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR
|
|||
github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ=
|
||||
github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM=
|
||||
github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
|
||||
github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc=
|
||||
github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs=
|
||||
github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
|
||||
github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
|
||||
|
|
@ -86,6 +91,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE=
|
|||
github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
|
||||
github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4=
|
||||
github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
|
||||
github.com/mailru/easyjson v0.0.0-20180730094502-03f2033d19d5 h1:0x4qcEHDpruK6ML/m/YSlFUUu0UpRD3I2PHsNCuGnyA=
|
||||
github.com/mailru/easyjson v0.0.0-20180730094502-03f2033d19d5/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc=
|
||||
github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0=
|
||||
github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
|
|
@ -95,6 +102,7 @@ github.com/ns3777k/go-shodan/v4 v4.2.0/go.mod h1:7kSWq/PQ/JCH6U4k2YjXRmnJKfPaJZA
|
|||
github.com/nyaruka/phonenumbers v1.0.42/go.mod h1:Hhae+eypC1YKMaQlBJUCGZDzBrIHHNWhJX1xG/8sOC8=
|
||||
github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U=
|
||||
github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU=
|
||||
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
|
||||
github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc=
|
||||
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
|
||||
|
|
@ -102,11 +110,13 @@ github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi
|
|||
github.com/pierrec/lz4 v2.4.0+incompatible h1:06usnXXDNcPvCHDkmPpkidf4jTc52UKld7UPfqKatY4=
|
||||
github.com/pierrec/lz4 v2.4.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
|
||||
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pkg/errors v0.9.0 h1:J8lpUdobwIeCI7OiSxHqEwJUKvJwicL5+3v1oe2Yb4k=
|
||||
github.com/pkg/errors v0.9.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/pquerna/otp v1.0.0/go.mod h1:Zad1CMQfSQZI5KLpahDiSUX4tMMREnXw98IvL1nhgMk=
|
||||
github.com/pquerna/otp v1.2.0 h1:/A3+Jn+cagqayeR3iHs/L62m5ue7710D35zl1zJ1kok=
|
||||
github.com/pquerna/otp v1.2.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg=
|
||||
github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
|
||||
github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso=
|
||||
|
|
@ -118,6 +128,7 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R
|
|||
github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
|
||||
github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU=
|
||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
github.com/segmentio/kafka-go v0.3.4 h1:Mv9AcnCgU14/cU6Vd0wuRdG1FBO0HzXQLnjBduDLy70=
|
||||
github.com/segmentio/kafka-go v0.3.4/go.mod h1:OT5KXBPbaJJTcvokhWR2KFmm0niEx3mnccTwjmLvSi4=
|
||||
|
|
@ -127,7 +138,9 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB
|
|||
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/skip2/go-qrcode v0.0.0-20171229120447-cf5f9fa2f0d8/go.mod h1:PLPIyL7ikehBD1OAjmKKiOEhbvWyHGaNDjquXMcYABo=
|
||||
github.com/skip2/go-qrcode v0.0.0-20191027152451-9434209cb086/go.mod h1:PLPIyL7ikehBD1OAjmKKiOEhbvWyHGaNDjquXMcYABo=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM=
|
||||
github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM=
|
||||
github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
|
||||
|
|
@ -141,6 +154,7 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg=
|
|||
github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4=
|
||||
github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E=
|
||||
github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k=
|
||||
github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 h1:WhxRHzgeVGETMlmVfqhRn8RIeeNoPr2Czh33I4Zdccw=
|
||||
github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
|
|
@ -159,9 +173,15 @@ github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0=
|
|||
github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y=
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU=
|
||||
github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q=
|
||||
gitlab.dcso.lolcat/go/a3 v0.0.0-20190913150211-730e44550e9c h1:DLOJygGbGcWR0O9IkvFSmDR5EOXMrg41/SQFmmuB/LE=
|
||||
gitlab.dcso.lolcat/go/a3 v0.0.0-20190913150211-730e44550e9c/go.mod h1:WBCQF8UiFlsXmkPrYWLMKkPyphINaxQonHh08c+wHss=
|
||||
gitlab.dcso.lolcat/go/a3/v3 v3.0.0-dev2 h1:4FHwKUC8z4SlV9zIHMsMC7g/gEyuJG9yFTanBUx3/Mg=
|
||||
gitlab.dcso.lolcat/go/a3/v3 v3.0.0-dev2/go.mod h1:5I6xp9PUSoxMfAn/8yWm7RdUNF8uiDvOTdFikVfw6Zo=
|
||||
gitlab.dcso.lolcat/go/a3client v1.4.0/go.mod h1:27iPJq3QKh4T0z7PX6A1wOlcyx8lUC6ae8nSIQoYTrc=
|
||||
gitlab.dcso.lolcat/go/darq v0.0.0-20191119105836-4d936d8242b9 h1:hKZLpC9vOw5tmzuaBxERrz1XH+oyMVjJO0+Or6rozyQ=
|
||||
gitlab.dcso.lolcat/go/darq v0.0.0-20191119105836-4d936d8242b9/go.mod h1:LtNUmbMixMvc+Gd9CJ2pwwqOsnfHziJP/HEotRyzbAo=
|
||||
gitlab.dcso.lolcat/go/darq v0.0.0-20191218130934-9ed195a57624 h1:UsoQvYurF5qWe3Qz/Qp6xNlK2iz6keAfuyvsATCO9fE=
|
||||
gitlab.dcso.lolcat/go/darq v0.0.0-20191218130934-9ed195a57624/go.mod h1:LtNUmbMixMvc+Gd9CJ2pwwqOsnfHziJP/HEotRyzbAo=
|
||||
gitlab.dcso.lolcat/go/golistic v0.0.0-20180830082801-b3fabf6e7d2c/go.mod h1:teEDykXNCW8/eEBLlQpTD2w88OCt+i4vVNLMW71JWs0=
|
||||
gitlab.dcso.lolcat/go/golistic v1.4.0/go.mod h1:teEDykXNCW8/eEBLlQpTD2w88OCt+i4vVNLMW71JWs0=
|
||||
gitlab.dcso.lolcat/go/golistic v1.6.0/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/mOOzo17vntZo=
|
||||
|
|
@ -171,6 +191,9 @@ gitlab.dcso.lolcat/go/golistic v1.8.3/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/m
|
|||
gitlab.dcso.lolcat/go/golistic v1.10.0/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/mOOzo17vntZo=
|
||||
gitlab.dcso.lolcat/go/golistic v1.12.14 h1:ijD8jrDqrevUeFafu2nRVcZBqhVkKcm5KSwQINkkJIg=
|
||||
gitlab.dcso.lolcat/go/golistic v1.12.14/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/mOOzo17vntZo=
|
||||
gitlab.dcso.lolcat/go/golistic v1.12.18 h1:T80mDpyA+U2m/rTp2fOO+oYjA6YyCifuXzlACD6V+7M=
|
||||
gitlab.dcso.lolcat/go/golistic v1.12.18/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/mOOzo17vntZo=
|
||||
gitlab.dcso.lolcat/go/graphql-go v0.0.0-20180911083115-e96b19adf728 h1:DeguiLLOPA2qj0tzD62b60NoOh/4egFz9CT9fMQt+lg=
|
||||
gitlab.dcso.lolcat/go/graphql-go v0.0.0-20180911083115-e96b19adf728/go.mod h1:EJYmIZeY2QsnndF3KPj25O6onHYTEwIMXaV/Sw92SGs=
|
||||
gitlab.dcso.lolcat/go/konfigo v0.0.0-20190102144114-f254146e1d0d/go.mod h1:F8qWHeze5OYs08RwABIj+8eKR7q8OGh+8Fz/+wZFmnI=
|
||||
gitlab.dcso.lolcat/go/konfigo v1.1.0 h1:sRJS7qlAWJacjjEfIIUr3jiqkrd9HBeYFpO8wN7+ym4=
|
||||
|
|
@ -192,6 +215,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
|
|||
golang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
|
||||
golang.org/x/crypto v0.0.0-20191117063200-497ca9f6d64f/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d h1:2+ZP7EfsZV7Vvmx3TIqSlSzATMkTAKqM14YGFPoSKjI=
|
||||
golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
|
||||
|
|
@ -224,6 +248,8 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w
|
|||
golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg=
|
||||
golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU=
|
||||
golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
|
|
@ -244,8 +270,11 @@ google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij
|
|||
gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno=
|
||||
gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/olivere/elastic.v5 v5.0.84 h1:acF/tRSg5geZpE3rqLglkS79CQMIMzOpWZE7hRXIkjs=
|
||||
gopkg.in/olivere/elastic.v5 v5.0.84/go.mod h1:LXF6q9XNBxpMqrcgax95C6xyARXWbbCXUrtTxrNrxJI=
|
||||
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
|
||||
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
|
||||
gopkg.in/yaml.v2 v2.1.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
|
|
@ -254,4 +283,6 @@ gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw=
|
|||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
|
||||
gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c=
|
||||
gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
|
||||
|
|
|
|||
14
main.go
14
main.go
|
|
@ -14,6 +14,7 @@ import (
|
|||
"github.com/sirupsen/logrus"
|
||||
"github.com/spf13/viper"
|
||||
"gitlab.dcso.lolcat/LABS/styx/broker"
|
||||
"gitlab.dcso.lolcat/LABS/styx/elasticsearch"
|
||||
"gitlab.dcso.lolcat/LABS/styx/filters"
|
||||
"gitlab.dcso.lolcat/LABS/styx/models"
|
||||
)
|
||||
|
|
@ -114,6 +115,17 @@ func certstreamRoutine(stream chan jsonq.JsonQuery, errStream chan error, conn *
|
|||
|
||||
func pastebinRoutine(stopChan chan os.Signal, wg *sync.WaitGroup) {
|
||||
fmt.Println("pastebin is activated")
|
||||
var e *elasticsearch.ElasticStorageModule
|
||||
e = &elasticsearch.ElasticStorageModule{
|
||||
ElasticURL: viper.GetString("elasticsearch.url"),
|
||||
DailyIndexes: true,
|
||||
UseIndex: "pastebin",
|
||||
LastChk: time.Now(),
|
||||
}
|
||||
err := e.Initialize()
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
for {
|
||||
select {
|
||||
default:
|
||||
|
|
@ -131,6 +143,7 @@ func pastebinRoutine(stopChan chan os.Signal, wg *sync.WaitGroup) {
|
|||
Full: paste,
|
||||
}
|
||||
res := models.BuildPasteNode(&fp)
|
||||
e.StorePaste(fp)
|
||||
models.SavePaste("paste_formatted.json", res)
|
||||
time.Sleep(1 * time.Second)
|
||||
|
||||
|
|
@ -157,7 +170,6 @@ func shodanRoutine(client *shodan.Client, shodanChan chan *shodan.HostData, conn
|
|||
shodanNode := models.BuildShodanNode(banner)
|
||||
// first filter poc
|
||||
if !filters.RunIPFilters(shodanNode.Data.IP) {
|
||||
fmt.Println("#### not found in filters", shodanNode.Data.IP)
|
||||
hostnames := shodanNode.Data.Hostnames
|
||||
if len(hostnames) != 0 {
|
||||
saveSingleValues(conn, "shodan_stream", "hostname", shodanNode.ID, hostnames)
|
||||
|
|
|
|||
Loading…
Reference in a new issue