adding drivers to the upload

This commit is contained in:
christalib 2020-07-25 00:52:24 +02:00
parent c33dd20c4b
commit e3ee93d8b7
5 changed files with 31 additions and 15 deletions

View file

@ -1,14 +1,16 @@
package crawlers package crawlers
import ( import (
"context"
"encoding/json" "encoding/json"
"fmt"
"io/ioutil" "io/ioutil"
"log"
"net/http" "net/http"
"time" "time"
"git.postblue.info/chris/paddockpass/db" "git.postblue.info/chris/paddockpass/db"
"github.com/jinzhu/gorm" "github.com/dgraph-io/dgo/v2"
"github.com/dgraph-io/dgo/v2/protos/api"
) )
// Driver represents a structure of driver entry in the Eargast API. // Driver represents a structure of driver entry in the Eargast API.
@ -35,7 +37,7 @@ type MRDriverData struct {
} }
// GetDriversFromErgast gets the driver data from the API. // GetDriversFromErgast gets the driver data from the API.
func GetDriversFromErgast(year string) { func GetDriversFromErgast(year string, client *dgo.Dgraph) {
resp, err := http.Get("http://ergast.com/api/f1/" + year + "/drivers.json") resp, err := http.Get("http://ergast.com/api/f1/" + year + "/drivers.json")
if err != nil { if err != nil {
panic(err.Error()) panic(err.Error())
@ -55,22 +57,30 @@ func GetDriversFromErgast(year string) {
drivers := jsonData.MRData.DriverTable.Drivers drivers := jsonData.MRData.DriverTable.Drivers
tx, err := gorm.Open("sqlite3", "test.db")
if err != nil {
fmt.Println(err)
}
for _, driver := range drivers { for _, driver := range drivers {
fmt.Println(driver) ctx := context.Background()
mu := &api.Mutation{
CommitNow: true,
}
model := db.Driver{ model := db.Driver{
FirstName: driver.FirstName, FirstName: driver.FirstName,
LastName: driver.LastName, LastName: driver.LastName,
Nationality: driver.Nationality, Nationality: driver.Nationality,
DoB: (driver.DateOrBirth).Format(time.RFC3339), DoB: (driver.DateOrBirth).Format(time.RFC3339),
Type: "driver",
} }
tx.Where(Driver{FirstName: driver.FirstName, LastName: driver.LastName}).Assign(model).FirstOrCreate(&model) pb, err := json.Marshal(model)
if err != nil {
log.Fatal(err)
}
mu.SetJson = pb
data, err := client.NewTxn().Mutate(ctx, mu)
if err != nil {
log.Fatal(err)
}
log.Print(data)
} }
defer tx.Close()
} }
// { // {

View file

@ -106,6 +106,7 @@ func GetSeasonFromErgast(year string, client *dgo.Dgraph) {
CircuitID: race.Circuit.CircuitID, CircuitID: race.Circuit.CircuitID,
Name: race.Circuit.Name, Name: race.Circuit.Name,
URL: race.Circuit.URL, URL: race.Circuit.URL,
Type: "circuit",
} }
race := db.Race{ race := db.Race{
@ -114,19 +115,17 @@ func GetSeasonFromErgast(year string, client *dgo.Dgraph) {
Round: race.Round, Round: race.Round,
Season: race.Season, Season: race.Season,
URL: race.URL, URL: race.URL,
Type: "race",
} }
pb, err := json.Marshal(race) pb, err := json.Marshal(race)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
mu.SetJson = pb mu.SetJson = pb
data, err := client.NewTxn().Mutate(ctx, mu) _, err = client.NewTxn().Mutate(ctx, mu)
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
log.Print(race.RaceName, race.Circuit.Name)
log.Print(data)
} }
} }

View file

@ -13,6 +13,7 @@ type Race struct {
Circuit Circuit `json:"circuit,omitempty"` Circuit Circuit `json:"circuit,omitempty"`
Date time.Time `json:"date,omitempty"` Date time.Time `json:"date,omitempty"`
URL string `json:"url,omitempty"` URL string `json:"url,omitempty"`
Type string `json:"type,omitempty"`
} }
// Circuit model describe a circuit. // Circuit model describe a circuit.
@ -22,6 +23,7 @@ type Circuit struct {
Location string `json:"location,omitempty"` Location string `json:"location,omitempty"`
CircuitID string `json:"circuitID,omitempty"` CircuitID string `json:"circuitID,omitempty"`
URL string `json:"url,omitempty"` URL string `json:"url,omitempty"`
Type string `json:"type,omitempty"`
} }
// Driver describes a driver. // Driver describes a driver.
@ -31,4 +33,5 @@ type Driver struct {
LastName string `json:"lastName,omitempty"` LastName string `json:"lastName,omitempty"`
Nationality string `json:"nationality,omitempty"` Nationality string `json:"nationality,omitempty"`
DoB string `json:"doB,omitempty"` DoB string `json:"doB,omitempty"`
Type string `json:"type,omitempty"`
} }

View file

@ -27,12 +27,14 @@ firstName: string .
lastName: string @index(term) . lastName: string @index(term) .
nationality: string @index(term) . nationality: string @index(term) .
dob: string . dob: string .
type: string @index(term) .
type Driver { type Driver {
firstName: string firstName: string
lastName: string lastName: string
nationality: string nationality: string
dob: string dob: string
type: string
} }
season: string @index(term) . season: string @index(term) .
@ -46,6 +48,7 @@ type Circuit {
name: string name: string
location: string location: string
url: string url: string
type: string
} }
type Race { type Race {
@ -55,6 +58,7 @@ raceName: string
circuit: Circuit circuit: Circuit
date: string date: string
url: string url: string
type: string
} }
name: string @index(term) . name: string @index(term) .

View file

@ -25,6 +25,6 @@ func main() {
fmt.Println("Hello there! It is", year) fmt.Println("Hello there! It is", year)
crawlers.GetSeasonFromErgast(year, client) crawlers.GetSeasonFromErgast(year, client)
// crawlers.GetDriversFromErgast(year) crawlers.GetDriversFromErgast(year, client)
// crawlers.GetResultsFromErgast(year) // crawlers.GetResultsFromErgast(year)
} }