adding drivers to the upload

This commit is contained in:
christalib 2020-07-25 00:52:24 +02:00
parent c33dd20c4b
commit e3ee93d8b7
5 changed files with 31 additions and 15 deletions

View file

@ -1,14 +1,16 @@
package crawlers
import (
"context"
"encoding/json"
"fmt"
"io/ioutil"
"log"
"net/http"
"time"
"git.postblue.info/chris/paddockpass/db"
"github.com/jinzhu/gorm"
"github.com/dgraph-io/dgo/v2"
"github.com/dgraph-io/dgo/v2/protos/api"
)
// Driver represents a structure of driver entry in the Eargast API.
@ -35,7 +37,7 @@ type MRDriverData struct {
}
// GetDriversFromErgast gets the driver data from the API.
func GetDriversFromErgast(year string) {
func GetDriversFromErgast(year string, client *dgo.Dgraph) {
resp, err := http.Get("http://ergast.com/api/f1/" + year + "/drivers.json")
if err != nil {
panic(err.Error())
@ -55,22 +57,30 @@ func GetDriversFromErgast(year string) {
drivers := jsonData.MRData.DriverTable.Drivers
tx, err := gorm.Open("sqlite3", "test.db")
if err != nil {
fmt.Println(err)
}
for _, driver := range drivers {
fmt.Println(driver)
ctx := context.Background()
mu := &api.Mutation{
CommitNow: true,
}
model := db.Driver{
FirstName: driver.FirstName,
LastName: driver.LastName,
Nationality: driver.Nationality,
DoB: (driver.DateOrBirth).Format(time.RFC3339),
Type: "driver",
}
tx.Where(Driver{FirstName: driver.FirstName, LastName: driver.LastName}).Assign(model).FirstOrCreate(&model)
pb, err := json.Marshal(model)
if err != nil {
log.Fatal(err)
}
mu.SetJson = pb
data, err := client.NewTxn().Mutate(ctx, mu)
if err != nil {
log.Fatal(err)
}
log.Print(data)
}
defer tx.Close()
}
// {

View file

@ -106,6 +106,7 @@ func GetSeasonFromErgast(year string, client *dgo.Dgraph) {
CircuitID: race.Circuit.CircuitID,
Name: race.Circuit.Name,
URL: race.Circuit.URL,
Type: "circuit",
}
race := db.Race{
@ -114,19 +115,17 @@ func GetSeasonFromErgast(year string, client *dgo.Dgraph) {
Round: race.Round,
Season: race.Season,
URL: race.URL,
Type: "race",
}
pb, err := json.Marshal(race)
if err != nil {
log.Fatal(err)
}
mu.SetJson = pb
data, err := client.NewTxn().Mutate(ctx, mu)
_, err = client.NewTxn().Mutate(ctx, mu)
if err != nil {
log.Fatal(err)
}
log.Print(race.RaceName, race.Circuit.Name)
log.Print(data)
}
}

View file

@ -13,6 +13,7 @@ type Race struct {
Circuit Circuit `json:"circuit,omitempty"`
Date time.Time `json:"date,omitempty"`
URL string `json:"url,omitempty"`
Type string `json:"type,omitempty"`
}
// Circuit model describe a circuit.
@ -22,6 +23,7 @@ type Circuit struct {
Location string `json:"location,omitempty"`
CircuitID string `json:"circuitID,omitempty"`
URL string `json:"url,omitempty"`
Type string `json:"type,omitempty"`
}
// Driver describes a driver.
@ -31,4 +33,5 @@ type Driver struct {
LastName string `json:"lastName,omitempty"`
Nationality string `json:"nationality,omitempty"`
DoB string `json:"doB,omitempty"`
Type string `json:"type,omitempty"`
}

View file

@ -27,12 +27,14 @@ firstName: string .
lastName: string @index(term) .
nationality: string @index(term) .
dob: string .
type: string @index(term) .
type Driver {
firstName: string
lastName: string
nationality: string
dob: string
type: string
}
season: string @index(term) .
@ -46,6 +48,7 @@ type Circuit {
name: string
location: string
url: string
type: string
}
type Race {
@ -55,6 +58,7 @@ raceName: string
circuit: Circuit
date: string
url: string
type: string
}
name: string @index(term) .

View file

@ -25,6 +25,6 @@ func main() {
fmt.Println("Hello there! It is", year)
crawlers.GetSeasonFromErgast(year, client)
// crawlers.GetDriversFromErgast(year)
crawlers.GetDriversFromErgast(year, client)
// crawlers.GetResultsFromErgast(year)
}