close #65 (#66)

* close #65

CHANGELOG.md

@@ -5,6 +5,12 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.1.0/),
 and this project adheres to [Calendar Versioning](https://calver.org/) (`YYYY.MM.MICRO-TAG`).
 
+## [v2024.11.2-beta] - 2024-11-13
+
+## Added
+
+- Added rate limiter to geocoding module (PR [#66](https://github.com/felix-schott/jamsessions/pull/66))
+
 ## [v2024.11.1-beta] - 2024-11-13
 
 ## Fixed

backend/go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/rs/cors v1.11.1
 	github.com/twpayne/go-geom v1.5.7
 	github.com/twpayne/pgx-geom v0.0.2
+	golang.org/x/time v0.8.0
 )
 
 require (

backend/go.sum

@@ -91,6 +91,8 @@ golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
 golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
 golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/time v0.8.0 h1:9i3RxcPv3PZnitoVGMPDKZSq1xW1gK1Xy3ArNOGZfEg=
+golang.org/x/time v0.8.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk=
 gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q=

backend/internal/geocoding/client.go

@@ -0,0 +1,49 @@
+package geocoding
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	"golang.org/x/time/rate"
+)
+
+// most code taken from https://gist.github.com/MelchiSalins/27c11566184116ec1629a0726e0f9af5
+
+type httpClientWithRateLimit struct {
+	client      *http.Client
+	Ratelimiter *rate.Limiter
+	UserAgent   string
+}
+
+// Do dispatches the HTTP request to the network
+func (c *httpClientWithRateLimit) Do(req *http.Request) (*http.Response, error) {
+	if c.UserAgent != "" {
+		req.Header.Set("User-Agent", c.UserAgent)
+	}
+	// Comment out the below 5 lines to turn off ratelimiting
+	ctx := context.Background()
+	err := c.Ratelimiter.Wait(ctx) // This is a blocking call. Honors the rate limit
+	if err != nil {
+		return nil, err
+	}
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	return resp, nil
+}
+
+// Returns http client with a ratelimiter
+func NewHttpClient(rl *rate.Limiter, userAgent string) *httpClientWithRateLimit {
+	var tr = &http.Transport{
+		IdleConnTimeout: 30 * time.Second,
+	}
+	var client = &http.Client{Transport: tr}
+	c := &httpClientWithRateLimit{
+		client:      client,
+		Ratelimiter: rl,
+		UserAgent:   userAgent,
+	}
+	return c
+}

backend/internal/geocoding/geocoding.go

@@ -9,8 +9,11 @@ import (
 	"time"
 
 	geom "github.com/twpayne/go-geom"
+	"golang.org/x/time/rate"
 )
 
+var client *httpClientWithRateLimit
+
 type geometry struct {
 	Type        string
 	Coordinates []float64
@@ -27,14 +30,6 @@ type nominatimResponse struct {
 	Features []feature
 }
 
-var tr = &http.Transport{
-	IdleConnTimeout: 30 * time.Second,
-}
-var client = &http.Client{Transport: tr}
-
-// nominatim
-// 1 request per second!!
-
 type NominatimDownError struct {
 	StatusCode int
 	Body       []byte
@@ -47,7 +42,11 @@ func (r NominatimDownError) Error() string {
 
 // Returns a NominatimDownError if the service is not healthy, otherwise nil
 func serviceIsHealthy() error {
-	resp, err := client.Get("https://nominatim.openstreetmap.org/status")
+	req, err := http.NewRequest("GET", "https://nominatim.openstreetmap.org/status", nil)
+	if err != nil {
+		return err
+	}
+	resp, err := client.Do(req)
 	var b []byte
 	if err != nil {
 		b, err = io.ReadAll(resp.Body)
@@ -67,13 +66,23 @@ func serviceIsHealthy() error {
 }
 
 func Geocode(street string, city string, postcode string) (*geom.Point, error) {
+	// instantiate client that respects nominatim rate limit (max 1 request per second)
+	if client == nil {
+		rl := rate.NewLimiter(rate.Every(time.Second*1), 1)
+		client = NewHttpClient(rl, "github.com/felix-schott/jamsessions")
+	}
+
 	err := serviceIsHealthy()
 	if err != nil {
 		return nil, err
 	}
 
 	reqUrl := fmt.Sprintf("https://nominatim.openstreetmap.org/search?street=%v&city=%v&country=UK&postcode=%v&format=geojson&limit=1", url.QueryEscape(street), url.QueryEscape(city), url.QueryEscape(postcode))
-	resp, err2 := client.Get(reqUrl)
+	req, err := http.NewRequest("GET", reqUrl, nil)
+	if err != nil {
+		return nil, err
+	}
+	resp, err2 := client.Do(req)
 	if err2 != nil {
 		return nil, fmt.Errorf("an unkown error occured when making request to %v: %v", reqUrl, err2)
 	}
@@ -92,5 +101,6 @@ func Geocode(street string, city string, postcode string) (*geom.Point, error) {
 	if len(result.Features) == 0 {
 		return nil, fmt.Errorf("no matches for %v, %v, %v (url %v)", street, city, postcode, reqUrl)
 	}
+
 	return geom.NewPoint(geom.XY).MustSetCoords(result.Features[0].Geometry.Coordinates).SetSRID(4326), nil
 }