From f4f82d9e1c2a993e5f16c08ad5a286989ff480e4 Mon Sep 17 00:00:00 2001 From: Christopher Talib Date: Mon, 17 Feb 2020 12:08:49 +0100 Subject: [PATCH] Adding code and setting up elastic search --- elasticsearch/main.go | 150 ++++++++++++++++++++++++++++++++++++++++++ go.mod | 5 +- go.sum | 31 +++++++++ main.go | 14 +++- 4 files changed, 197 insertions(+), 3 deletions(-) create mode 100644 elasticsearch/main.go diff --git a/elasticsearch/main.go b/elasticsearch/main.go new file mode 100644 index 0000000..31da5ac --- /dev/null +++ b/elasticsearch/main.go @@ -0,0 +1,150 @@ +package elasticsearch + +import ( + "context" + "fmt" + "log" + "time" + + "gitlab.dcso.lolcat/LABS/styx/models" + elastic "gopkg.in/olivere/elastic.v5" +) + +const mapping = `{ + "settings":{ + "number_of_shards":1, + "number_of_replicas":0 + }, + "mappings":{ + "_default_":{ + "_all":{ + "enabled":true + } + }, + "paste":{ + "properties":{ + "key":{ + "type":"keyword" + }, + "data":{ + "type":"text", + "store":true, + "fielddata":true + }, + "size":{ + "type":"long" + }, + "syntax":{ + "type":"keyword" + }, + "date":{ + "type":"date", + "format":"epoch_second" + }, + "expire":{ + "type":"date", + "format":"epoch_second" + } + } + } + } +}` + +// ElasticStorageModule is a StorageModule that stores pastes in an +// Elasticsearch instance. +type ElasticStorageModule struct { + Index string + ElasticURL string + DailyIndexes bool + UseIndex string + LastChk time.Time + Client *elastic.Client +} + +func (e *ElasticStorageModule) makeIndexIfNotExists() error { + index := e.Index + if e.DailyIndexes { + index = time.Now().Format(fmt.Sprintf("%s-2006-1-2", e.Index)) + } + + exists, err := e.Client.IndexExists(index).Do(context.Background()) + if err != nil { + return err + } + + if !exists { + log.Printf("Creating new index %s", index) + e.Client.CreateIndex(index).Body(mapping).Do(context.Background()) + } + + e.UseIndex = index + return nil +} + +// Initialize prepares the storage modules for use. +func (e *ElasticStorageModule) Initialize() error { + log.Printf("Connecting to %v", e.ElasticURL) + var err error + e.Index = "pastebin" + e.Client, err = elastic.NewSimpleClient(elastic.SetURL(e.ElasticURL)) + if err != nil { + return err + } + + info, code, err := e.Client.Ping(e.ElasticURL).Do(context.Background()) + if err != nil { + return err + } + log.Printf("Elasticsearch returned with code %d and version %s", code, + info.Version.Number) + + err = e.makeIndexIfNotExists() + if err != nil { + return err + } + + e.LastChk = time.Now() + log.Printf("Using index %s", e.UseIndex) + return err +} + +// StorePaste stores a single paste in the storage backend. +func (e *ElasticStorageModule) StorePaste(paste models.FullPaste) { + // if e.DailyIndexes && time.Since(e.LastChk) > 12*time.Hour { + // e.makeIndexIfNotExists() + // e.LastChk = time.Now() + // } + _, err := e.Client.Index(). + Index(e.UseIndex). + Type("paste"). + Id(paste.Meta.Key). + BodyJson(paste). + Do(context.Background()) + if err != nil { + log.Printf("Could not store paste %s due to %v", paste.Meta.FullURL, err) + } +} + +// Check returns true if the given paste is new in the current storage backend, +// false otherwise. +func (e *ElasticStorageModule) Check(paste models.PasteMeta) bool { + q := elastic.NewMatchQuery("_id", paste.Key) + + searchResult, err := e.Client.Search(). + Index(e.Index). + Query(q). + Pretty(true). + Do(context.Background()) + if err != nil { + log.Printf("Could not check paste due to %v", err) + return true + } + + return (searchResult.TotalHits() == 0) +} + +// Destroy finishes all operations on the module. +func (e *ElasticStorageModule) Destroy() error { + _, err := e.Client.Flush().Index(e.Index).Do(context.Background()) + return err +} diff --git a/go.mod b/go.mod index 5bad594..11dbd48 100644 --- a/go.mod +++ b/go.mod @@ -15,9 +15,10 @@ require ( github.com/segmentio/kafka-go v0.3.4 github.com/sirupsen/logrus v1.4.2 github.com/spf13/viper v1.6.2 - gitlab.dcso.lolcat/go/golistic v1.12.14 + gitlab.dcso.lolcat/go/a3/v3 v3.0.0-dev2 // indirect + gitlab.dcso.lolcat/go/golistic v1.12.18 gitlab.dcso.lolcat/go/vizor v1.20.0 golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d // indirect golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa // indirect - golang.org/x/sys v0.0.0-20200116001909-b77594299b42 // indirect + gopkg.in/olivere/elastic.v5 v5.0.84 ) diff --git a/go.sum b/go.sum index cc6dec5..811ff28 100644 --- a/go.sum +++ b/go.sum @@ -13,6 +13,7 @@ github.com/aws/aws-lambda-go v1.8.1/go.mod h1:zUsUQhAUjYzR8AuduJPCfhBuKWUaDbQiPO github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= +github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc h1:biVzkmvwrH8WK8raXaxBx6fRVTlJILwEwQGL1I/ByEI= github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= @@ -24,10 +25,13 @@ github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfc github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21 h1:YEetp8/yCZMuEPMUDHG0CW/brkkEp8mzqk2+ODEitlw= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= +github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/frankban/quicktest v1.7.2 h1:2QxQoC1TS09S7fhCPsrvqYdvP1H5M1P1ih5ABm3BTYk= github.com/frankban/quicktest v1.7.2/go.mod h1:jaStnuzAqU1AJdCO0l53JDCJrVDKcS03DbaAcR7Ks/o= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= @@ -61,6 +65,7 @@ github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORR github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/graph-gophers/graphql-go v0.0.0-20191115155744-f33e81362277/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/grpc-ecosystem/go-grpc-middleware v1.0.0/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= @@ -86,6 +91,8 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.1 h1:ZC2Vc7/ZFkGmsVC9KvOjumD+G5lXy2RtTKyzRKO2BQ4= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mailru/easyjson v0.0.0-20180730094502-03f2033d19d5 h1:0x4qcEHDpruK6ML/m/YSlFUUu0UpRD3I2PHsNCuGnyA= +github.com/mailru/easyjson v0.0.0-20180730094502-03f2033d19d5/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -95,6 +102,7 @@ github.com/ns3777k/go-shodan/v4 v4.2.0/go.mod h1:7kSWq/PQ/JCH6U4k2YjXRmnJKfPaJZA github.com/nyaruka/phonenumbers v1.0.42/go.mod h1:Hhae+eypC1YKMaQlBJUCGZDzBrIHHNWhJX1xG/8sOC8= github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/pelletier/go-toml v1.2.0 h1:T5zMGML61Wp+FlcbWjRDT7yAxhJNAiPPLOFECq181zc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -102,11 +110,13 @@ github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi github.com/pierrec/lz4 v2.4.0+incompatible h1:06usnXXDNcPvCHDkmPpkidf4jTc52UKld7UPfqKatY4= github.com/pierrec/lz4 v2.4.0+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.0 h1:J8lpUdobwIeCI7OiSxHqEwJUKvJwicL5+3v1oe2Yb4k= github.com/pkg/errors v0.9.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pquerna/otp v1.0.0/go.mod h1:Zad1CMQfSQZI5KLpahDiSUX4tMMREnXw98IvL1nhgMk= +github.com/pquerna/otp v1.2.0 h1:/A3+Jn+cagqayeR3iHs/L62m5ue7710D35zl1zJ1kok= github.com/pquerna/otp v1.2.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= @@ -118,6 +128,7 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/segmentio/kafka-go v0.3.4 h1:Mv9AcnCgU14/cU6Vd0wuRdG1FBO0HzXQLnjBduDLy70= github.com/segmentio/kafka-go v0.3.4/go.mod h1:OT5KXBPbaJJTcvokhWR2KFmm0niEx3mnccTwjmLvSi4= @@ -127,7 +138,9 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/skip2/go-qrcode v0.0.0-20171229120447-cf5f9fa2f0d8/go.mod h1:PLPIyL7ikehBD1OAjmKKiOEhbvWyHGaNDjquXMcYABo= +github.com/skip2/go-qrcode v0.0.0-20191027152451-9434209cb086/go.mod h1:PLPIyL7ikehBD1OAjmKKiOEhbvWyHGaNDjquXMcYABo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= @@ -141,6 +154,7 @@ github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.6.2 h1:7aKfF+e8/k68gda3LOjo5RxiUqddoFxVq4BKBPrxk5E= github.com/spf13/viper v1.6.2/go.mod h1:t3iDnF5Jlj76alVNuyFBk5oUMCvsrkbvZK0WQdfDi5k= +github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 h1:WhxRHzgeVGETMlmVfqhRn8RIeeNoPr2Czh33I4Zdccw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -159,9 +173,15 @@ github.com/xdg/stringprep v1.0.0 h1:d9X0esnoa3dFsV0FG35rAT0RIhYFlPq7MiP+DW89La0= github.com/xdg/stringprep v1.0.0/go.mod h1:Jhud4/sHMO4oL310DaZAKk9ZaJ08SJfe+sJh0HrGL1Y= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +gitlab.dcso.lolcat/go/a3 v0.0.0-20190913150211-730e44550e9c h1:DLOJygGbGcWR0O9IkvFSmDR5EOXMrg41/SQFmmuB/LE= gitlab.dcso.lolcat/go/a3 v0.0.0-20190913150211-730e44550e9c/go.mod h1:WBCQF8UiFlsXmkPrYWLMKkPyphINaxQonHh08c+wHss= +gitlab.dcso.lolcat/go/a3/v3 v3.0.0-dev2 h1:4FHwKUC8z4SlV9zIHMsMC7g/gEyuJG9yFTanBUx3/Mg= +gitlab.dcso.lolcat/go/a3/v3 v3.0.0-dev2/go.mod h1:5I6xp9PUSoxMfAn/8yWm7RdUNF8uiDvOTdFikVfw6Zo= gitlab.dcso.lolcat/go/a3client v1.4.0/go.mod h1:27iPJq3QKh4T0z7PX6A1wOlcyx8lUC6ae8nSIQoYTrc= +gitlab.dcso.lolcat/go/darq v0.0.0-20191119105836-4d936d8242b9 h1:hKZLpC9vOw5tmzuaBxERrz1XH+oyMVjJO0+Or6rozyQ= gitlab.dcso.lolcat/go/darq v0.0.0-20191119105836-4d936d8242b9/go.mod h1:LtNUmbMixMvc+Gd9CJ2pwwqOsnfHziJP/HEotRyzbAo= +gitlab.dcso.lolcat/go/darq v0.0.0-20191218130934-9ed195a57624 h1:UsoQvYurF5qWe3Qz/Qp6xNlK2iz6keAfuyvsATCO9fE= +gitlab.dcso.lolcat/go/darq v0.0.0-20191218130934-9ed195a57624/go.mod h1:LtNUmbMixMvc+Gd9CJ2pwwqOsnfHziJP/HEotRyzbAo= gitlab.dcso.lolcat/go/golistic v0.0.0-20180830082801-b3fabf6e7d2c/go.mod h1:teEDykXNCW8/eEBLlQpTD2w88OCt+i4vVNLMW71JWs0= gitlab.dcso.lolcat/go/golistic v1.4.0/go.mod h1:teEDykXNCW8/eEBLlQpTD2w88OCt+i4vVNLMW71JWs0= gitlab.dcso.lolcat/go/golistic v1.6.0/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/mOOzo17vntZo= @@ -171,6 +191,9 @@ gitlab.dcso.lolcat/go/golistic v1.8.3/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/m gitlab.dcso.lolcat/go/golistic v1.10.0/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/mOOzo17vntZo= gitlab.dcso.lolcat/go/golistic v1.12.14 h1:ijD8jrDqrevUeFafu2nRVcZBqhVkKcm5KSwQINkkJIg= gitlab.dcso.lolcat/go/golistic v1.12.14/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/mOOzo17vntZo= +gitlab.dcso.lolcat/go/golistic v1.12.18 h1:T80mDpyA+U2m/rTp2fOO+oYjA6YyCifuXzlACD6V+7M= +gitlab.dcso.lolcat/go/golistic v1.12.18/go.mod h1:FYxfvcYN/7/YzVmfoyGIVqwR+WSj5F/mOOzo17vntZo= +gitlab.dcso.lolcat/go/graphql-go v0.0.0-20180911083115-e96b19adf728 h1:DeguiLLOPA2qj0tzD62b60NoOh/4egFz9CT9fMQt+lg= gitlab.dcso.lolcat/go/graphql-go v0.0.0-20180911083115-e96b19adf728/go.mod h1:EJYmIZeY2QsnndF3KPj25O6onHYTEwIMXaV/Sw92SGs= gitlab.dcso.lolcat/go/konfigo v0.0.0-20190102144114-f254146e1d0d/go.mod h1:F8qWHeze5OYs08RwABIj+8eKR7q8OGh+8Fz/+wZFmnI= gitlab.dcso.lolcat/go/konfigo v1.1.0 h1:sRJS7qlAWJacjjEfIIUr3jiqkrd9HBeYFpO8wN7+ym4= @@ -192,6 +215,7 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190506204251-e1dfcc566284/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191117063200-497ca9f6d64f/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d h1:2+ZP7EfsZV7Vvmx3TIqSlSzATMkTAKqM14YGFPoSKjI= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -224,6 +248,8 @@ golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42 h1:vEOn+mP2zCOVzKckCZy6YsCtDblrpj/w7B9nxGNELpg= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82 h1:ywK/j/KkyTHcdyYSZNXGjMwgmDSfjglYZ3vStQ/gSCU= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= @@ -244,8 +270,11 @@ google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/ini.v1 v1.51.0 h1:AQvPpx3LzTDM0AjnIRlVFwFFGC+npRopjZxLJj6gdno= gopkg.in/ini.v1 v1.51.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/olivere/elastic.v5 v5.0.84 h1:acF/tRSg5geZpE3rqLglkS79CQMIMzOpWZE7hRXIkjs= +gopkg.in/olivere/elastic.v5 v5.0.84/go.mod h1:LXF6q9XNBxpMqrcgax95C6xyARXWbbCXUrtTxrNrxJI= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.1.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -254,4 +283,6 @@ gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.5 h1:ymVxjfMaHvXD8RqPRmzHHsB3VvucivSkIAvJFDI5O3c= +gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/main.go b/main.go index 7121097..13a0269 100644 --- a/main.go +++ b/main.go @@ -14,6 +14,7 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/viper" "gitlab.dcso.lolcat/LABS/styx/broker" + "gitlab.dcso.lolcat/LABS/styx/elasticsearch" "gitlab.dcso.lolcat/LABS/styx/filters" "gitlab.dcso.lolcat/LABS/styx/models" ) @@ -114,6 +115,17 @@ func certstreamRoutine(stream chan jsonq.JsonQuery, errStream chan error, conn * func pastebinRoutine(stopChan chan os.Signal, wg *sync.WaitGroup) { fmt.Println("pastebin is activated") + var e *elasticsearch.ElasticStorageModule + e = &elasticsearch.ElasticStorageModule{ + ElasticURL: viper.GetString("elasticsearch.url"), + DailyIndexes: true, + UseIndex: "pastebin", + LastChk: time.Now(), + } + err := e.Initialize() + if err != nil { + panic(err) + } for { select { default: @@ -131,6 +143,7 @@ func pastebinRoutine(stopChan chan os.Signal, wg *sync.WaitGroup) { Full: paste, } res := models.BuildPasteNode(&fp) + e.StorePaste(fp) models.SavePaste("paste_formatted.json", res) time.Sleep(1 * time.Second) @@ -157,7 +170,6 @@ func shodanRoutine(client *shodan.Client, shodanChan chan *shodan.HostData, conn shodanNode := models.BuildShodanNode(banner) // first filter poc if !filters.RunIPFilters(shodanNode.Data.IP) { - fmt.Println("#### not found in filters", shodanNode.Data.IP) hostnames := shodanNode.Data.Hostnames if len(hostnames) != 0 { saveSingleValues(conn, "shodan_stream", "hostname", shodanNode.ID, hostnames)