chris/paddockpass
1
0
Fork 0
Code Issues Pull requests Releases Wiki Activity

Adding results and building results from it

Browse source
This commit is contained in:
christalib 2020-07-26 01:41:30 +02:00
parent 56ad1acf3c
commit 40f9546b1a
10 changed files with 473 additions and 109 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

25
crawlers/driver.go
View file

@ -13,29 +13,6 @@ import (
"github.com/dgraph-io/dgo/v2/protos/api"
)
// Driver represents a structure of driver entry in the Eargast API.
type Driver struct {
FirstName string `json:"givenName"`
LastName string `json:"familyName"`
DateOrBirth Date `json:"dateOfBirth"`
Nationality string `json:"nationality"`
}
// Drivers represents an array of driver in the Eargast API.
type Drivers struct {
Drivers []Driver `json:"Drivers"`
}
// DriverTable reprensts a table of drivers in the Eargast API.
type DriverTable struct {
DriverTable Drivers `json:"DriverTable"`
}
// MRDriverData is the wrapper for the datta in the Eargast API.
type MRDriverData struct {
MRData DriverTable `json:"MRData"`
}
// GetDriversFromErgast gets the driver data from the API.
func GetDriversFromErgast(year string, client *dgo.Dgraph) {
resp, err := http.Get("http://ergast.com/api/f1/" + year + "/drivers.json")
@ -66,7 +43,7 @@ func GetDriversFromErgast(year string, client *dgo.Dgraph) {
FirstName: driver.FirstName,
LastName: driver.LastName,
Nationality: driver.Nationality,
DoB: (driver.DateOrBirth).Format(time.RFC3339),
DoB: (driver.DateOfBirth).Format(time.RFC3339),
Type: "driver",
}
pb, err := json.Marshal(model)

46
crawlers/race.go
View file

@ -15,52 +15,6 @@ import (
"github.com/sirupsen/logrus"
)
// Location represents the details of a circuit.
type Location struct {
Country string `json:"country"`
Latitude string `json:"lat"`
Locatlity string `json:"locality"`
Longitude string `json:"long"`
}
// Circuit represents the details of a circuit.
type Circuit struct {
CircuitID string `json:"circuitID"`
URL string `json:"url"`
Name string `json:"circuitName"`
Location Location `json:"Location"`
}
// Race represents the details of a race.
type Race struct {
Season string `json:"season"`
Round string `json:"round"`
RaceName string `json:"raceName"`
Circuit Circuit `json:"Circuit"`
Date Date `json:"date"`
Time string `json:"time"`
URL string `json:"url"`
}
// Races is the list of races.
type Races struct {
Races []Race `json:"Races"`
Season string `json:"season"`
}
// RaceTable is the list of races.
type RaceTable struct {
RaceTable Races `json:"RaceTable"`
}
// MRData is the general wrapper from Eargast API.
type MRData struct {
MRData RaceTable `json:"MRData"`
}
// Date is the date struct.
type Date struct{ time.Time }
// UnmarshalJSON is something.
func (d *Date) UnmarshalJSON(data []byte) error {
t, err := time.Parse("2006-01-02", strings.Trim(string(data), "\""))

43
crawlers/races.go
View file

@ -1,34 +1,25 @@
package crawlers
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"github.com/sirupsen/logrus"
)
// GetResultsFromErgast retrieves the results of the races from Ergast API.
func GetResultsFromErgast(year string) {
resp, err := http.Get("http://ergast.com/api/f1/results" + year + ".json")
if err != nil {
panic(err.Error())
}
// func GetResultsFromErgast(year string) {
// resp, err := http.Get("http://ergast.com/api/f1/results" + year + ".json")
// if err != nil {
// panic(err.Error())
// }
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err.Error())
}
defer resp.Body.Close()
// body, err := ioutil.ReadAll(resp.Body)
// if err != nil {
// panic(err.Error())
// }
// defer resp.Body.Close()
var jsonData MRData
err = json.Unmarshal(body, &jsonData)
if err != nil {
logrus.Error(err)
}
fmt.Println(jsonData)
}
// var jsonData MRData
// err = json.Unmarshal(body, &jsonData)
// if err != nil {
// logrus.Error(err)
// }
// fmt.Println(jsonData)
// }
/**
MRDATA {

107
crawlers/results.go Normal file
View file

@ -0,0 +1,107 @@
package crawlers
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"github.com/dgraph-io/dgo/v2"
"github.com/dgraph-io/dgo/v2/protos/api"
"github.com/opentracing/opentracing-go/log"
)
// GetResultsFromErgast gets the results from the API based on a year and
// driver.
func GetResultsFromErgast(year string, driver string, client *dgo.Dgraph) {
// resp, err := http.Get("http://ergast.com/api/f1/" + year + "/" + driver + ".json")
resp, err := http.Get("http://ergast.com/api/f1/2019/drivers/vettel/results.json")
if err != nil {
log.Error(err)
}
if resp.StatusCode != 200 {
log.Message("Error not a 200 response")
return
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
panic(err.Error())
}
defer resp.Body.Close()
var jsonData MRData
err = json.Unmarshal(body, &jsonData)
if err != nil {
log.Error(err)
}
races := jsonData.MRData.RaceTable.Races
for _, race := range races {
ctx := context.Background()
mu := &api.Mutation{
CommitNow: true,
}
var results []Result
for _, result := range race.Results {
res := Result{
Number: result.Number,
Position: result.Position,
Points: result.Points,
Driver: Driver{
FirstName: result.Driver.FirstName,
LastName: result.Driver.LastName,
DateOfBirth: result.Driver.DateOfBirth,
Nationality: result.Driver.Nationality,
},
// Constructor: result.Constructor{},
Grid: result.Grid,
Laps: result.Laps,
Status: result.Status,
Time: result.Time,
// FastestLap: result.FastestLap{},
// AverageSpeed: result.AverageSpeed{},
}
results = append(results, res)
}
model := Race{
Season: race.Season,
Round: race.Round,
RaceName: race.RaceName,
Circuit: Circuit{
CircuitID: race.Circuit.CircuitID,
URL: race.Circuit.URL,
Name: race.Circuit.Name,
Location: Location{
Country: race.Circuit.Location.Country,
Latitude: race.Circuit.Location.Latitude,
Locality: race.Circuit.Location.Locality,
Longitude: race.Circuit.Location.Longitude,
},
},
Results: results,
Date: race.Date,
Time: race.Time,
}
pb, err := json.Marshal(model)
if err != nil {
log.Error(err)
}
mu.SetJson = pb
data, err := client.NewTxn().Mutate(ctx, mu)
if err != nil {
log.Error(err)
}
fmt.Print(data)
}
}

180
crawlers/results.json Normal file
View file

@ -0,0 +1,180 @@
{
"MRData": {
"xmlns": "http://ergast.com/mrd/1.4",
"series": "f1",
"url": "http://ergast.com/api/f1/2020/drivers/vettel/results.json",
"limit": "30",
"offset": "0",
"total": "3",
"RaceTable": {
"season": "2020",
"driverId": "vettel",
"Races": [
{
"season": "2020",
"round": "1",
"url": "https://en.wikipedia.org/wiki/2020_Austrian_Grand_Prix",
"raceName": "Austrian Grand Prix",
"Circuit": {
"circuitId": "red_bull_ring",
"url": "http://en.wikipedia.org/wiki/Red_Bull_Ring",
"circuitName": "Red Bull Ring",
"Location": {
"lat": "47.2197",
"long": "14.7647",
"locality": "Spielburg",
"country": "Austria"
}
},
"date": "2020-07-05",
"time": "13:10:00Z",
"Results": [
{
"number": "5",
"position": "10",
"positionText": "10",
"points": "1",
"Driver": {
"driverId": "vettel",
"permanentNumber": "5",
"code": "VET",
"url": "http://en.wikipedia.org/wiki/Sebastian_Vettel",
"givenName": "Sebastian",
"familyName": "Vettel",
"dateOfBirth": "1987-07-03",
"nationality": "German"
},
"Constructor": {
"constructorId": "ferrari",
"url": "http://en.wikipedia.org/wiki/Scuderia_Ferrari",
"name": "Ferrari",
"nationality": "Italian"
},
"grid": "11",
"laps": "71",
"status": "Finished",
"Time": {
"millis": "5480284",
"time": "+24.545"
},
"FastestLap": {
"rank": "8",
"lap": "71",
"Time": {
"time": "1:08.623"
},
"AverageSpeed": {
"units": "kph",
"speed": "226.524"
}
}
}
]
},
{
"season": "2020",
"round": "2",
"url": "https://en.wikipedia.org/wiki/2020_Styrian_Grand_Prix",
"raceName": "Styrian Grand Prix",
"Circuit": {
"circuitId": "red_bull_ring",
"url": "http://en.wikipedia.org/wiki/Red_Bull_Ring",
"circuitName": "Red Bull Ring",
"Location": {
"lat": "47.2197",
"long": "14.7647",
"locality": "Spielburg",
"country": "Austria"
}
},
"date": "2020-07-12",
"time": "13:10:00Z",
"Results": [
{
"number": "5",
"position": "20",
"positionText": "R",
"points": "0",
"Driver": {
"driverId": "vettel",
"permanentNumber": "5",
"code": "VET",
"url": "http://en.wikipedia.org/wiki/Sebastian_Vettel",
"givenName": "Sebastian",
"familyName": "Vettel",
"dateOfBirth": "1987-07-03",
"nationality": "German"
},
"Constructor": {
"constructorId": "ferrari",
"url": "http://en.wikipedia.org/wiki/Scuderia_Ferrari",
"name": "Ferrari",
"nationality": "Italian"
},
"grid": "10",
"laps": "1",
"status": "Collision damage"
}
]
},
{
"season": "2020",
"round": "3",
"url": "https://en.wikipedia.org/wiki/2020_Hungarian_Grand_Prix",
"raceName": "Hungarian Grand Prix",
"Circuit": {
"circuitId": "hungaroring",
"url": "http://en.wikipedia.org/wiki/Hungaroring",
"circuitName": "Hungaroring",
"Location": {
"lat": "47.5789",
"long": "19.2486",
"locality": "Budapest",
"country": "Hungary"
}
},
"date": "2020-07-19",
"time": "13:10:00Z",
"Results": [
{
"number": "5",
"position": "6",
"positionText": "6",
"points": "8",
"Driver": {
"driverId": "vettel",
"permanentNumber": "5",
"code": "VET",
"url": "http://en.wikipedia.org/wiki/Sebastian_Vettel",
"givenName": "Sebastian",
"familyName": "Vettel",
"dateOfBirth": "1987-07-03",
"nationality": "German"
},
"Constructor": {
"constructorId": "ferrari",
"url": "http://en.wikipedia.org/wiki/Scuderia_Ferrari",
"name": "Ferrari",
"nationality": "Italian"
},
"grid": "5",
"laps": "69",
"status": "+1 Lap",
"FastestLap": {
"rank": "14",
"lap": "49",
"Time": {
"time": "1:20.363"
},
"AverageSpeed": {
"units": "kph",
"speed": "196.254"
}
}
}
]
}
]
}
}
}

114
crawlers/types.go Normal file
View file

@ -0,0 +1,114 @@
package crawlers
import "time"
// Driver represents a structure of driver entry in the Eargast API.
type Driver struct {
FirstName string `json:"givenName,omitemtpy"`
LastName string `json:"familyName,omitemtpy"`
DateOfBirth Date `json:"dateOfBirth,omitemtpy"`
Nationality string `json:"nationality,omitemtpy"`
}
// Drivers represents an array of driver in the Eargast API.
type Drivers struct {
Drivers []Driver `json:"Drivers,omitemtpy"`
}
// DriverTable reprensts a table of drivers in the Eargast API.
type DriverTable struct {
DriverTable Drivers `json:"DriverTable,omitemtpy"`
}
// MRData is the general wrapper from Eargast API.
type MRData struct {
MRData RaceTable `json:"MRData,omitempty"`
}
// MRDriverData is the wrapper for the datta in the Eargast API.
type MRDriverData struct {
MRData DriverTable `json:"MRData,omitemtpy"`
}
// Location represents the details of a circuit.
type Location struct {
Country string `json:"country,omitempty"`
Latitude string `json:"lat,omitempty"`
Locality string `json:"locality,omitempty"`
Longitude string `json:"long,omitempty"`
}
// Circuit represents the details of a circuit.
type Circuit struct {
CircuitID string `json:"circuitID,omitempty"`
URL string `json:"url,omitempty"`
Name string `json:"circuitName,omitempty"`
Location Location `json:"Location,omitempty"`
}
// Race represents the details of a race.
type Race struct {
Season string `json:"season,omitempty"`
Round string `json:"round,omitempty"`
RaceName string `json:"raceName,omitempty"`
Circuit Circuit `json:"Circuit,omitempty"`
Date Date `json:"date,omitempty"`
Time string `json:"time,omitempty"`
URL string `json:"url,omitempty"`
Results []Result `json:"Results,omitempty"`
}
// Races is the list of races.
type Races struct {
Races []Race `json:"Races,omitempty"`
// Season string `json:"season,omitempty"`
}
// RaceTable is the list of races.
type RaceTable struct {
RaceTable Races `json:"RaceTable,omitempty"`
}
// Date is the date struct.
type Date struct{ time.Time }
type Constructor struct {
ConstructorID string `json:"constructorID,omitempty"`
URL string `json:"url,omitempty"`
Name string `json:"name,omitempty"`
Nationality string `json:"nationality,omitempty"`
}
type FastestLap struct {
Rank string `json:"rank,omitempty"`
Lap string `json:"lap,omitempty"`
Time Time `json:"time,omitempty"`
}
type Time struct {
Millis string `json:"millis,omitempty"`
Time string `json:"time,omitempty"`
}
type AverageSpeed struct {
Units string `json:"units,omitempty"`
Speed string `json:"speed,omitempty"`
}
type Result struct {
Number string `json:"numbe,omitemptyr"`
Position string `json:"position,omitempty"`
Points string `json:"points,omitempty"`
Driver Driver `json:"driver,omitempty"`
Constructor Constructor `json:"Constructor,omitempty"`
Grid string `json:"grid,omitempty"`
Laps string `json:"laps,omitempty"`
Status string `json:"status,omitempty"`
Time Time `json:"time,omitempty"`
FastestLap FastestLap `json:"FastestLap,omitempty"`
AverageSpeed AverageSpeed `json:"AvergareSpeed,omitempty"`
}
type Results struct {
Results []Result `json:"Result,omitempty"`
}

1
go.mod
View file

@ -7,6 +7,7 @@ require (
github.com/dgraph-io/dgo/v2 v2.2.0
github.com/go-sql-driver/mysql v1.4.1
github.com/jinzhu/gorm v1.9.11
github.com/opentracing/opentracing-go v1.2.0
github.com/sirupsen/logrus v1.2.0
github.com/urfave/cli v1.22.4
github.com/urfave/cli/v2 v2.2.0 // indirect

3
go.sum
View file

@ -81,6 +81,8 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/opentracing/opentracing-go v1.2.0 h1:uEJPy/1a5RIPAJ0Ov+OIO8OxWu77jEv+1B0VhjKrZUs=
github.com/opentracing/opentracing-go v1.2.0/go.mod h1:GxEUsuufX4nBwe+T+Wl9TAgYrxe9dPLANfrWvHYVTgc=
github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@ -110,6 +112,7 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w=
github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/urfave/cli v1.22.4 h1:u7tSpNPPswAFymm8IehJhy4uJMlUuU/GmqSkvJ1InXA=

58
graph/main.go
View file

@ -22,12 +22,38 @@ func ConnectToDgraph() (*dgo.Dgraph, error) {
func InitSchema(c *dgo.Dgraph) error {
err := c.Alter(context.Background(), &api.Operation{
DropAll: true,
Schema: `
firstName: string .
lastName: string @index(term) .
nationality: string @index(term) .
averageSpeed: uid .
circuit: uid .
constructor: uid .
constructorId: string .
date: string .
dob: string .
driver: uid .
fastestLap: uid .
firstName: string .
grid: string .
lap: string .
laps: string .
lastName: string @index(term) .
location: string @index(term) .
millis: string .
name: string @index(term) .
nationality: string @index(term) .
number: string .
points: string .
position: string @index(term) .
raceName: string @index(term) .
rank: string .
round: string .
season: string @index(term) .
speed: string .
status: string .
time: string .
type: string @index(term) .
units: string .
url: string .
type Driver {
firstName: string
@ -37,12 +63,24 @@ dob: string
type: string
}
season: string @index(term) .
round: string .
raceName: string @index(term) .
circuit: uid .
date: string .
url: string .
type Constructor {
constructorId: string
url: string
name: string
nationality: string
}
type Result {
number: string
position: string
points: string
driver: Driver
constructor: Constructor
grid: string
laps: string
status: string
time: Time
}
type Circuit {
name: string
@ -61,8 +99,6 @@ url: string
type: string
}
name: string @index(term) .
location: string @index(term) .
`})
if err != nil {

5
main.go
View file

@ -29,7 +29,8 @@ func main() {
app.Description = "Run with the docker-compose file and the app will data from Ergast and display it in Dgraph."
fmt.Println("Hello there! It is", year)
crawlers.GetSeasonFromErgast(year, client)
crawlers.GetDriversFromErgast(year, client)
crawlers.GetResultsFromErgast(year, "alonso", client)
// crawlers.GetSeasonFromErgast(year, client)
// crawlers.GetDriversFromErgast(year, client)
// crawlers.GetResultsFromErgast(year)
}