From 40f9546b1a34083a18312a809af09d052d079b09 Mon Sep 17 00:00:00 2001 From: christalib Date: Sun, 26 Jul 2020 01:41:30 +0200 Subject: [PATCH] Adding results and building results from it --- crawlers/driver.go | 25 +----- crawlers/race.go | 46 ----------- crawlers/races.go | 43 ++++------ crawlers/results.go | 107 +++++++++++++++++++++++++ crawlers/results.json | 180 ++++++++++++++++++++++++++++++++++++++++++ crawlers/types.go | 114 ++++++++++++++++++++++++++ go.mod | 1 + go.sum | 3 + graph/main.go | 58 +++++++++++--- main.go | 5 +- 10 files changed, 473 insertions(+), 109 deletions(-) create mode 100644 crawlers/results.go create mode 100644 crawlers/results.json create mode 100644 crawlers/types.go diff --git a/crawlers/driver.go b/crawlers/driver.go index 6ad9565..562ed41 100644 --- a/crawlers/driver.go +++ b/crawlers/driver.go @@ -13,29 +13,6 @@ import ( "github.com/dgraph-io/dgo/v2/protos/api" ) -// Driver represents a structure of driver entry in the Eargast API. -type Driver struct { - FirstName string `json:"givenName"` - LastName string `json:"familyName"` - DateOrBirth Date `json:"dateOfBirth"` - Nationality string `json:"nationality"` -} - -// Drivers represents an array of driver in the Eargast API. -type Drivers struct { - Drivers []Driver `json:"Drivers"` -} - -// DriverTable reprensts a table of drivers in the Eargast API. -type DriverTable struct { - DriverTable Drivers `json:"DriverTable"` -} - -// MRDriverData is the wrapper for the datta in the Eargast API. -type MRDriverData struct { - MRData DriverTable `json:"MRData"` -} - // GetDriversFromErgast gets the driver data from the API. func GetDriversFromErgast(year string, client *dgo.Dgraph) { resp, err := http.Get("http://ergast.com/api/f1/" + year + "/drivers.json") @@ -66,7 +43,7 @@ func GetDriversFromErgast(year string, client *dgo.Dgraph) { FirstName: driver.FirstName, LastName: driver.LastName, Nationality: driver.Nationality, - DoB: (driver.DateOrBirth).Format(time.RFC3339), + DoB: (driver.DateOfBirth).Format(time.RFC3339), Type: "driver", } pb, err := json.Marshal(model) diff --git a/crawlers/race.go b/crawlers/race.go index ab04fa5..10ed9bd 100644 --- a/crawlers/race.go +++ b/crawlers/race.go @@ -15,52 +15,6 @@ import ( "github.com/sirupsen/logrus" ) -// Location represents the details of a circuit. -type Location struct { - Country string `json:"country"` - Latitude string `json:"lat"` - Locatlity string `json:"locality"` - Longitude string `json:"long"` -} - -// Circuit represents the details of a circuit. -type Circuit struct { - CircuitID string `json:"circuitID"` - URL string `json:"url"` - Name string `json:"circuitName"` - Location Location `json:"Location"` -} - -// Race represents the details of a race. -type Race struct { - Season string `json:"season"` - Round string `json:"round"` - RaceName string `json:"raceName"` - Circuit Circuit `json:"Circuit"` - Date Date `json:"date"` - Time string `json:"time"` - URL string `json:"url"` -} - -// Races is the list of races. -type Races struct { - Races []Race `json:"Races"` - Season string `json:"season"` -} - -// RaceTable is the list of races. -type RaceTable struct { - RaceTable Races `json:"RaceTable"` -} - -// MRData is the general wrapper from Eargast API. -type MRData struct { - MRData RaceTable `json:"MRData"` -} - -// Date is the date struct. -type Date struct{ time.Time } - // UnmarshalJSON is something. func (d *Date) UnmarshalJSON(data []byte) error { t, err := time.Parse("2006-01-02", strings.Trim(string(data), "\"")) diff --git a/crawlers/races.go b/crawlers/races.go index cca9eeb..66daa5c 100644 --- a/crawlers/races.go +++ b/crawlers/races.go @@ -1,34 +1,25 @@ package crawlers -import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" - - "github.com/sirupsen/logrus" -) - // GetResultsFromErgast retrieves the results of the races from Ergast API. -func GetResultsFromErgast(year string) { - resp, err := http.Get("http://ergast.com/api/f1/results" + year + ".json") - if err != nil { - panic(err.Error()) - } +// func GetResultsFromErgast(year string) { +// resp, err := http.Get("http://ergast.com/api/f1/results" + year + ".json") +// if err != nil { +// panic(err.Error()) +// } - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - panic(err.Error()) - } - defer resp.Body.Close() +// body, err := ioutil.ReadAll(resp.Body) +// if err != nil { +// panic(err.Error()) +// } +// defer resp.Body.Close() - var jsonData MRData - err = json.Unmarshal(body, &jsonData) - if err != nil { - logrus.Error(err) - } - fmt.Println(jsonData) -} +// var jsonData MRData +// err = json.Unmarshal(body, &jsonData) +// if err != nil { +// logrus.Error(err) +// } +// fmt.Println(jsonData) +// } /** MRDATA { diff --git a/crawlers/results.go b/crawlers/results.go new file mode 100644 index 0000000..eed3f9c --- /dev/null +++ b/crawlers/results.go @@ -0,0 +1,107 @@ +package crawlers + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + "github.com/dgraph-io/dgo/v2" + "github.com/dgraph-io/dgo/v2/protos/api" + "github.com/opentracing/opentracing-go/log" +) + +// GetResultsFromErgast gets the results from the API based on a year and +// driver. +func GetResultsFromErgast(year string, driver string, client *dgo.Dgraph) { + // resp, err := http.Get("http://ergast.com/api/f1/" + year + "/" + driver + ".json") + resp, err := http.Get("http://ergast.com/api/f1/2019/drivers/vettel/results.json") + if err != nil { + log.Error(err) + } + + if resp.StatusCode != 200 { + log.Message("Error not a 200 response") + return + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + panic(err.Error()) + } + defer resp.Body.Close() + + var jsonData MRData + err = json.Unmarshal(body, &jsonData) + if err != nil { + log.Error(err) + } + + races := jsonData.MRData.RaceTable.Races + for _, race := range races { + ctx := context.Background() + mu := &api.Mutation{ + CommitNow: true, + } + + var results []Result + + for _, result := range race.Results { + res := Result{ + Number: result.Number, + Position: result.Position, + Points: result.Points, + Driver: Driver{ + FirstName: result.Driver.FirstName, + LastName: result.Driver.LastName, + DateOfBirth: result.Driver.DateOfBirth, + Nationality: result.Driver.Nationality, + }, + // Constructor: result.Constructor{}, + Grid: result.Grid, + Laps: result.Laps, + Status: result.Status, + Time: result.Time, + // FastestLap: result.FastestLap{}, + // AverageSpeed: result.AverageSpeed{}, + } + + results = append(results, res) + + } + + model := Race{ + Season: race.Season, + Round: race.Round, + RaceName: race.RaceName, + Circuit: Circuit{ + CircuitID: race.Circuit.CircuitID, + URL: race.Circuit.URL, + Name: race.Circuit.Name, + Location: Location{ + Country: race.Circuit.Location.Country, + Latitude: race.Circuit.Location.Latitude, + Locality: race.Circuit.Location.Locality, + Longitude: race.Circuit.Location.Longitude, + }, + }, + Results: results, + Date: race.Date, + Time: race.Time, + } + + pb, err := json.Marshal(model) + if err != nil { + log.Error(err) + } + mu.SetJson = pb + data, err := client.NewTxn().Mutate(ctx, mu) + if err != nil { + log.Error(err) + } + fmt.Print(data) + + } + +} diff --git a/crawlers/results.json b/crawlers/results.json new file mode 100644 index 0000000..ce345e9 --- /dev/null +++ b/crawlers/results.json @@ -0,0 +1,180 @@ +{ + "MRData": { + "xmlns": "http://ergast.com/mrd/1.4", + "series": "f1", + "url": "http://ergast.com/api/f1/2020/drivers/vettel/results.json", + "limit": "30", + "offset": "0", + "total": "3", + "RaceTable": { + "season": "2020", + "driverId": "vettel", + "Races": [ + { + "season": "2020", + "round": "1", + "url": "https://en.wikipedia.org/wiki/2020_Austrian_Grand_Prix", + "raceName": "Austrian Grand Prix", + "Circuit": { + "circuitId": "red_bull_ring", + "url": "http://en.wikipedia.org/wiki/Red_Bull_Ring", + "circuitName": "Red Bull Ring", + "Location": { + "lat": "47.2197", + "long": "14.7647", + "locality": "Spielburg", + "country": "Austria" + } + }, + "date": "2020-07-05", + "time": "13:10:00Z", + "Results": [ + { + "number": "5", + "position": "10", + "positionText": "10", + "points": "1", + "Driver": { + "driverId": "vettel", + "permanentNumber": "5", + "code": "VET", + "url": "http://en.wikipedia.org/wiki/Sebastian_Vettel", + "givenName": "Sebastian", + "familyName": "Vettel", + "dateOfBirth": "1987-07-03", + "nationality": "German" + }, + "Constructor": { + "constructorId": "ferrari", + "url": "http://en.wikipedia.org/wiki/Scuderia_Ferrari", + "name": "Ferrari", + "nationality": "Italian" + }, + "grid": "11", + "laps": "71", + "status": "Finished", + "Time": { + "millis": "5480284", + "time": "+24.545" + }, + "FastestLap": { + "rank": "8", + "lap": "71", + "Time": { + "time": "1:08.623" + }, + "AverageSpeed": { + "units": "kph", + "speed": "226.524" + } + } + } + ] + }, + { + "season": "2020", + "round": "2", + "url": "https://en.wikipedia.org/wiki/2020_Styrian_Grand_Prix", + "raceName": "Styrian Grand Prix", + "Circuit": { + "circuitId": "red_bull_ring", + "url": "http://en.wikipedia.org/wiki/Red_Bull_Ring", + "circuitName": "Red Bull Ring", + "Location": { + "lat": "47.2197", + "long": "14.7647", + "locality": "Spielburg", + "country": "Austria" + } + }, + "date": "2020-07-12", + "time": "13:10:00Z", + "Results": [ + { + "number": "5", + "position": "20", + "positionText": "R", + "points": "0", + "Driver": { + "driverId": "vettel", + "permanentNumber": "5", + "code": "VET", + "url": "http://en.wikipedia.org/wiki/Sebastian_Vettel", + "givenName": "Sebastian", + "familyName": "Vettel", + "dateOfBirth": "1987-07-03", + "nationality": "German" + }, + "Constructor": { + "constructorId": "ferrari", + "url": "http://en.wikipedia.org/wiki/Scuderia_Ferrari", + "name": "Ferrari", + "nationality": "Italian" + }, + "grid": "10", + "laps": "1", + "status": "Collision damage" + } + ] + }, + { + "season": "2020", + "round": "3", + "url": "https://en.wikipedia.org/wiki/2020_Hungarian_Grand_Prix", + "raceName": "Hungarian Grand Prix", + "Circuit": { + "circuitId": "hungaroring", + "url": "http://en.wikipedia.org/wiki/Hungaroring", + "circuitName": "Hungaroring", + "Location": { + "lat": "47.5789", + "long": "19.2486", + "locality": "Budapest", + "country": "Hungary" + } + }, + "date": "2020-07-19", + "time": "13:10:00Z", + "Results": [ + { + "number": "5", + "position": "6", + "positionText": "6", + "points": "8", + "Driver": { + "driverId": "vettel", + "permanentNumber": "5", + "code": "VET", + "url": "http://en.wikipedia.org/wiki/Sebastian_Vettel", + "givenName": "Sebastian", + "familyName": "Vettel", + "dateOfBirth": "1987-07-03", + "nationality": "German" + }, + "Constructor": { + "constructorId": "ferrari", + "url": "http://en.wikipedia.org/wiki/Scuderia_Ferrari", + "name": "Ferrari", + "nationality": "Italian" + }, + "grid": "5", + "laps": "69", + "status": "+1 Lap", + "FastestLap": { + "rank": "14", + "lap": "49", + "Time": { + "time": "1:20.363" + }, + "AverageSpeed": { + "units": "kph", + "speed": "196.254" + } + } + } + ] + } + ] + } + } +} diff --git a/crawlers/types.go b/crawlers/types.go new file mode 100644 index 0000000..6d34def --- /dev/null +++ b/crawlers/types.go @@ -0,0 +1,114 @@ +package crawlers + +import "time" + +// Driver represents a structure of driver entry in the Eargast API. +type Driver struct { + FirstName string `json:"givenName,omitemtpy"` + LastName string `json:"familyName,omitemtpy"` + DateOfBirth Date `json:"dateOfBirth,omitemtpy"` + Nationality string `json:"nationality,omitemtpy"` +} + +// Drivers represents an array of driver in the Eargast API. +type Drivers struct { + Drivers []Driver `json:"Drivers,omitemtpy"` +} + +// DriverTable reprensts a table of drivers in the Eargast API. +type DriverTable struct { + DriverTable Drivers `json:"DriverTable,omitemtpy"` +} + +// MRData is the general wrapper from Eargast API. +type MRData struct { + MRData RaceTable `json:"MRData,omitempty"` +} + +// MRDriverData is the wrapper for the datta in the Eargast API. +type MRDriverData struct { + MRData DriverTable `json:"MRData,omitemtpy"` +} + +// Location represents the details of a circuit. +type Location struct { + Country string `json:"country,omitempty"` + Latitude string `json:"lat,omitempty"` + Locality string `json:"locality,omitempty"` + Longitude string `json:"long,omitempty"` +} + +// Circuit represents the details of a circuit. +type Circuit struct { + CircuitID string `json:"circuitID,omitempty"` + URL string `json:"url,omitempty"` + Name string `json:"circuitName,omitempty"` + Location Location `json:"Location,omitempty"` +} + +// Race represents the details of a race. +type Race struct { + Season string `json:"season,omitempty"` + Round string `json:"round,omitempty"` + RaceName string `json:"raceName,omitempty"` + Circuit Circuit `json:"Circuit,omitempty"` + Date Date `json:"date,omitempty"` + Time string `json:"time,omitempty"` + URL string `json:"url,omitempty"` + Results []Result `json:"Results,omitempty"` +} + +// Races is the list of races. +type Races struct { + Races []Race `json:"Races,omitempty"` + // Season string `json:"season,omitempty"` +} + +// RaceTable is the list of races. +type RaceTable struct { + RaceTable Races `json:"RaceTable,omitempty"` +} + +// Date is the date struct. +type Date struct{ time.Time } + +type Constructor struct { + ConstructorID string `json:"constructorID,omitempty"` + URL string `json:"url,omitempty"` + Name string `json:"name,omitempty"` + Nationality string `json:"nationality,omitempty"` +} + +type FastestLap struct { + Rank string `json:"rank,omitempty"` + Lap string `json:"lap,omitempty"` + Time Time `json:"time,omitempty"` +} + +type Time struct { + Millis string `json:"millis,omitempty"` + Time string `json:"time,omitempty"` +} + +type AverageSpeed struct { + Units string `json:"units,omitempty"` + Speed string `json:"speed,omitempty"` +} + +type Result struct { + Number string `json:"numbe,omitemptyr"` + Position string `json:"position,omitempty"` + Points string `json:"points,omitempty"` + Driver Driver `json:"driver,omitempty"` + Constructor Constructor `json:"Constructor,omitempty"` + Grid string `json:"grid,omitempty"` + Laps string `json:"laps,omitempty"` + Status string `json:"status,omitempty"` + Time Time `json:"time,omitempty"` + FastestLap FastestLap `json:"FastestLap,omitempty"` + AverageSpeed AverageSpeed `json:"AvergareSpeed,omitempty"` +} + +type Results struct { + Results []Result `json:"Result,omitempty"` +} diff --git a/go.mod b/go.mod index d9d5127..2dee2b4 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/dgraph-io/dgo/v2 v2.2.0 github.com/go-sql-driver/mysql v1.4.1 github.com/jinzhu/gorm v1.9.11 + github.com/opentracing/opentracing-go v1.2.0 github.com/sirupsen/logrus v1.2.0 github.com/urfave/cli v1.22.4 github.com/urfave/cli/v2 v2.2.0 // indirect diff --git a/go.sum b/go.sum index 5d9ff72..8725a20 100644 --- a/go.sum +++ b/go.sum @@ -81,6 +81,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= +github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs= +github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc= github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -110,6 +112,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/urfave/cli v1.22.4 h1:u7tSpNPPswAFymm8IehJhy4uJMlUuU/GmqSkvJ1InXA= diff --git a/graph/main.go b/graph/main.go index 606717f..216097b 100644 --- a/graph/main.go +++ b/graph/main.go @@ -22,12 +22,38 @@ func ConnectToDgraph() (*dgo.Dgraph, error) { func InitSchema(c *dgo.Dgraph) error { err := c.Alter(context.Background(), &api.Operation{ + DropAll: true, Schema: ` -firstName: string . -lastName: string @index(term) . -nationality: string @index(term) . +averageSpeed: uid . +circuit: uid . +constructor: uid . +constructorId: string . +date: string . dob: string . +driver: uid . +fastestLap: uid . +firstName: string . +grid: string . +lap: string . +laps: string . +lastName: string @index(term) . +location: string @index(term) . +millis: string . +name: string @index(term) . +nationality: string @index(term) . +number: string . +points: string . +position: string @index(term) . +raceName: string @index(term) . +rank: string . +round: string . +season: string @index(term) . +speed: string . +status: string . +time: string . type: string @index(term) . +units: string . +url: string . type Driver { firstName: string @@ -37,12 +63,24 @@ dob: string type: string } -season: string @index(term) . -round: string . -raceName: string @index(term) . -circuit: uid . -date: string . -url: string . +type Constructor { +constructorId: string +url: string +name: string +nationality: string +} + +type Result { +number: string +position: string +points: string +driver: Driver +constructor: Constructor +grid: string +laps: string +status: string +time: Time +} type Circuit { name: string @@ -61,8 +99,6 @@ url: string type: string } -name: string @index(term) . -location: string @index(term) . `}) if err != nil { diff --git a/main.go b/main.go index 2ce2021..7d7b64a 100644 --- a/main.go +++ b/main.go @@ -29,7 +29,8 @@ func main() { app.Description = "Run with the docker-compose file and the app will data from Ergast and display it in Dgraph." fmt.Println("Hello there! It is", year) - crawlers.GetSeasonFromErgast(year, client) - crawlers.GetDriversFromErgast(year, client) + crawlers.GetResultsFromErgast(year, "alonso", client) + // crawlers.GetSeasonFromErgast(year, client) + // crawlers.GetDriversFromErgast(year, client) // crawlers.GetResultsFromErgast(year) }