From cc64358a96b4a9367a93e6d6a5e7446f6992fdd2 Mon Sep 17 00:00:00 2001 From: Ryan Mello Date: Fri, 24 Jul 2026 06:58:07 +0100 Subject: [PATCH 01/30] fix: stop country-only IP locations from tearing down connections SetConnectionLocation panicked when a client's IP resolved to a country- or region-only location (no city), because it inserted the location row's NULL city_location_id / region_location_id into network_client_location, where both are NOT NULL. The panic is the real damage, not just a failed lookup: it propagates out of the connection announce goroutine (connect/transport_announce.go), whose HandleError wrapper cancels the connection-level context -- tearing down the entire connect connection right after auth. And because the panic hits before the disconnect-cleanup defer is registered, the connection row is orphaned as connected=true forever. How often this fires depends on the geo database's city coverage, so it is rare with a full commercial dataset and constant with a sparse one. Fix: fall back to the coarsest available granularity so the columns are always non-null -- a country-only location stores its country id for city/region too, keeping the provider locatable at country level instead of crashing the connection. If even the country id is absent, return a clean error (the caller already has a graceful retry path) rather than panic. --- model/network_client_location_model.go | 30 +++++++++++ model/network_client_location_model_test.go | 59 +++++++++++++++++++++ 2 files changed, 89 insertions(+) diff --git a/model/network_client_location_model.go b/model/network_client_location_model.go index a2650fc4..cbab0e61 100644 --- a/model/network_client_location_model.go +++ b/model/network_client_location_model.go @@ -1242,6 +1242,36 @@ func SetConnectionLocation( } }) + // fix(beta): the free-tier ipinfo geo db resolves many IPs -- + // datacenter, mobile, and VPN egress especially -- to country or + // region granularity only, with no city. network_client_location + // requires city_location_id and region_location_id NOT NULL, so a + // country-only location's NULL city/region made this INSERT panic + // inside server.Tx. That panic propagated out of the connection + // announce goroutine (connect/transport_announce.go), whose + // HandleError wrapper then cancelled the whole connection context -- + // tearing down every country-only client's connect connection right + // after auth (the app itself included, whichever egress it resolved + // to), and, because the panic hit before the disconnect-cleanup + // defer was registered, orphaning the connection row as + // connected=true forever. Fall back to the coarsest available + // granularity so the columns are always non-null: a country-only + // location stores its country id for city/region too, which keeps + // the provider locatable at country level instead of crashing the + // connection. If even the country id is missing (location row absent + // or malformed), return a clean error so the caller's existing + // graceful retry path handles it -- never panic here. + if countryLocationId == nil { + returnErr = fmt.Errorf("Location %s has no country granularity.", locationId) + return + } + if cityLocationId == nil { + cityLocationId = countryLocationId + } + if regionLocationId == nil { + regionLocationId = countryLocationId + } + server.RaisePgResult(tx.Exec( ctx, ` diff --git a/model/network_client_location_model_test.go b/model/network_client_location_model_test.go index ab91c7ee..888a1ce2 100644 --- a/model/network_client_location_model_test.go +++ b/model/network_client_location_model_test.go @@ -1198,3 +1198,62 @@ func TestFindProviders2ReliabilityDeployGap(t *testing.T) { connect.AssertEqual(t, len(res.Providers), n) }) } + +// SetConnectionLocation must not panic when the resolved location has no city +// (country-only). network_client_location requires city_location_id and +// region_location_id NOT NULL, but a country-granularity location row has them +// NULL, so the insert panicked inside server.Tx. That panic propagated out of +// the connection announce goroutine, whose HandleError wrapper cancelled the +// connection context -- tearing down the client's connection right after auth, +// and (because the panic hit before the disconnect-cleanup defer was +// registered) orphaning the connection row as connected=true. This asserts a +// country-only location is stored, falling back to country granularity, with +// no panic and no error. +func TestSetConnectionLocationToleratesCountryOnlyLocation(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + // country-only location -- its row has NULL city_location_id and + // NULL region_location_id, exactly what crashed the insert before + country := &Location{ + LocationType: LocationTypeCountry, + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, country) + + networkId := server.NewId() + clientId := server.NewId() + Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := ConnectNetworkClient(ctx, clientId, "0.0.0.1:0", handlerId) + connect.AssertEqual(t, err, nil) + + // this call panicked before the fix; now it must succeed and store + // the connection at country granularity + err = SetConnectionLocation(ctx, connectionId, country.LocationId, &ConnectionLocationScores{}) + connect.AssertEqual(t, err, nil) + + var city, region, cty *server.Id + server.Db(ctx, func(conn server.PgConn) { + result, qerr := conn.Query( + ctx, + `SELECT city_location_id, region_location_id, country_location_id FROM network_client_location WHERE connection_id = $1`, + connectionId, + ) + server.WithPgResult(result, qerr, func() { + if result.Next() { + server.Raise(result.Scan(&city, ®ion, &cty)) + } + }) + }) + if city == nil || region == nil || cty == nil { + t.Fatal("expected all three location ids to be set (falling back to country), got a nil") + } + // city and region fall back to the country id for a country-only location + connect.AssertEqual(t, *city, country.CountryLocationId) + connect.AssertEqual(t, *region, country.CountryLocationId) + connect.AssertEqual(t, *cty, country.CountryLocationId) + }) +} From 0bf9387afa56a3941e19c02c92a2a31c269c648d Mon Sep 17 00:00:00 2001 From: ryanmello07 Date: Sat, 25 Jul 2026 04:15:25 +0100 Subject: [PATCH 02/30] feat(model): provider_egress_location storage --- db_migrations.go | 26 ++++ model/provider_egress_location_model.go | 154 +++++++++++++++++++ model/provider_egress_location_model_test.go | 129 ++++++++++++++++ 3 files changed, 309 insertions(+) create mode 100644 model/provider_egress_location_model.go create mode 100644 model/provider_egress_location_model_test.go diff --git a/db_migrations.go b/db_migrations.go index 063f091e..204afa4b 100644 --- a/db_migrations.go +++ b/db_migrations.go @@ -4385,4 +4385,30 @@ var migrations = []any{ CREATE INDEX IF NOT EXISTS network_client_top_level_contract_time ON network_client (contract_time) WHERE (active = true AND source_client_id IS NULL AND contract_time IS NOT NULL) `), + + // provider egress locations: locations learned by probing a provider's own + // egress (see docs/superpowers/specs/2026-07-24-provider-egress-geolocation-design.md). + // Keyed by client_id, one row per provider, upserted by the operator's + // prober. location_id is the canonical country (or city, when the probe was + // city-confident) location row. observed_at is when the probe ran, and is + // what freshness is judged against. + newSqlMigration(` + CREATE TABLE IF NOT EXISTS provider_egress_location ( + client_id uuid NOT NULL PRIMARY KEY, + location_id uuid NOT NULL, + country_code varchar(2) NOT NULL, + asn int NOT NULL DEFAULT 0, + org varchar(256) NOT NULL DEFAULT '', + hosting bool NOT NULL DEFAULT false, + proxy bool NOT NULL DEFAULT false, + mobile bool NOT NULL DEFAULT false, + city_confident bool NOT NULL DEFAULT false, + observed_at timestamp NOT NULL, + update_time timestamp NOT NULL + ) + `), + newSqlMigration(` + CREATE INDEX IF NOT EXISTS provider_egress_location_observed_at + ON provider_egress_location (observed_at) + `), } diff --git a/model/provider_egress_location_model.go b/model/provider_egress_location_model.go new file mode 100644 index 00000000..797d82fd --- /dev/null +++ b/model/provider_egress_location_model.go @@ -0,0 +1,154 @@ +package model + +import ( + "context" + "time" + + "github.com/urnetwork/server" +) + +// ProviderEgressLocationMaxAge bounds how long a probed egress location is +// trusted. Past this, the location is ignored and the caller falls back to the +// mmdb lookup on the observed control ip. +const ProviderEgressLocationMaxAge = 7 * 24 * time.Hour + +// ProviderEgressLocation is a provider location learned by probing the +// provider's own egress, rather than by looking up its control-connection ip. +type ProviderEgressLocation struct { + ClientId server.Id + LocationId server.Id + CountryCode string + ASN int + Org string + Hosting bool + Proxy bool + Mobile bool + CityConfident bool + ObservedAt time.Time + UpdateTime time.Time +} + +// SetProviderEgressLocation upserts the probed location for a provider. +func SetProviderEgressLocation(ctx context.Context, e *ProviderEgressLocation) { + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO provider_egress_location ( + client_id, + location_id, + country_code, + asn, + org, + hosting, + proxy, + mobile, + city_confident, + observed_at, + update_time + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + ON CONFLICT (client_id) DO UPDATE + SET + location_id = $2, + country_code = $3, + asn = $4, + org = $5, + hosting = $6, + proxy = $7, + mobile = $8, + city_confident = $9, + observed_at = $10, + update_time = $11 + `, + e.ClientId, + e.LocationId, + e.CountryCode, + e.ASN, + e.Org, + e.Hosting, + e.Proxy, + e.Mobile, + e.CityConfident, + e.ObservedAt.UTC(), + server.NowUtc(), + )) + }) +} + +// GetProviderEgressLocation returns the stored location for a provider, or nil. +func GetProviderEgressLocation(ctx context.Context, clientId server.Id) *ProviderEgressLocation { + var e *ProviderEgressLocation + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT + client_id, + location_id, + country_code, + asn, + org, + hosting, + proxy, + mobile, + city_confident, + observed_at, + update_time + FROM provider_egress_location + WHERE client_id = $1 + `, + clientId, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + e = &ProviderEgressLocation{} + server.Raise(result.Scan( + &e.ClientId, + &e.LocationId, + &e.CountryCode, + &e.ASN, + &e.Org, + &e.Hosting, + &e.Proxy, + &e.Mobile, + &e.CityConfident, + &e.ObservedAt, + &e.UpdateTime, + )) + } + }) + }) + return e +} + +// GetFreshProviderEgressLocation is GetProviderEgressLocation, filtered to +// entries probed within maxAge. The cutoff is computed in Go and bound as a +// parameter: observed_at is a naive timestamp holding utc, and comparing it +// against sql now() would cast through the session timezone. +func GetFreshProviderEgressLocation( + ctx context.Context, + clientId server.Id, + maxAge time.Duration, +) *ProviderEgressLocation { + e := GetProviderEgressLocation(ctx, clientId) + if e == nil { + return nil + } + if e.ObservedAt.Before(server.NowUtc().Add(-maxAge)) { + return nil + } + return e +} + +// RemoveExpiredProviderEgressLocations drops entries probed before +// minObservedAt. +func RemoveExpiredProviderEgressLocations(ctx context.Context, minObservedAt time.Time) { + server.MaintenanceTx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + `DELETE FROM provider_egress_location WHERE observed_at < $1`, + minObservedAt.UTC(), + )) + }) +} diff --git a/model/provider_egress_location_model_test.go b/model/provider_egress_location_model_test.go new file mode 100644 index 00000000..37d9c494 --- /dev/null +++ b/model/provider_egress_location_model_test.go @@ -0,0 +1,129 @@ +package model + +import ( + "context" + "testing" + "time" + + "github.com/go-playground/assert/v2" + + "github.com/urnetwork/server" +) + +func TestProviderEgressLocationUpsertAndGet(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + country := &Location{ + LocationType: LocationTypeCountry, + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, country) + + clientId := server.NewId() + now := server.NowUtc() + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: clientId, + LocationId: country.LocationId, + CountryCode: "us", + ASN: 401486, + Org: "RAVNIX LLC", + Hosting: true, + ObservedAt: now, + }) + + got := GetProviderEgressLocation(ctx, clientId) + if got == nil { + t.Fatal("expected a stored egress location") + } + assert.Equal(t, got.LocationId, country.LocationId) + assert.Equal(t, got.CountryCode, "us") + assert.Equal(t, got.ASN, 401486) + assert.Equal(t, got.Hosting, true) + assert.Equal(t, got.Proxy, false) + + // upsert replaces + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: clientId, + LocationId: country.LocationId, + CountryCode: "us", + ASN: 999, + Hosting: false, + Proxy: true, + ObservedAt: now, + }) + got = GetProviderEgressLocation(ctx, clientId) + assert.Equal(t, got.ASN, 999) + assert.Equal(t, got.Hosting, false) + assert.Equal(t, got.Proxy, true) + }) +} + +func TestProviderEgressLocationFreshness(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + country := &Location{ + LocationType: LocationTypeCountry, + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, country) + + fresh := server.NewId() + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: fresh, LocationId: country.LocationId, CountryCode: "us", + ObservedAt: server.NowUtc(), + }) + stale := server.NewId() + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: stale, LocationId: country.LocationId, CountryCode: "us", + ObservedAt: server.NowUtc().Add(-8 * 24 * time.Hour), + }) + + if GetFreshProviderEgressLocation(ctx, fresh, ProviderEgressLocationMaxAge) == nil { + t.Fatal("fresh entry must be returned") + } + if GetFreshProviderEgressLocation(ctx, stale, ProviderEgressLocationMaxAge) != nil { + t.Fatal("stale entry must not be returned") + } + // absent + if GetFreshProviderEgressLocation(ctx, server.NewId(), ProviderEgressLocationMaxAge) != nil { + t.Fatal("absent entry must return nil") + } + }) +} + +func TestRemoveExpiredProviderEgressLocations(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + country := &Location{ + LocationType: LocationTypeCountry, + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, country) + + keep := server.NewId() + drop := server.NewId() + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: keep, LocationId: country.LocationId, CountryCode: "us", + ObservedAt: server.NowUtc(), + }) + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: drop, LocationId: country.LocationId, CountryCode: "us", + ObservedAt: server.NowUtc().Add(-30 * 24 * time.Hour), + }) + + RemoveExpiredProviderEgressLocations(ctx, server.NowUtc().Add(-14*24*time.Hour)) + + if GetProviderEgressLocation(ctx, keep) == nil { + t.Fatal("recent entry must survive the sweep") + } + if GetProviderEgressLocation(ctx, drop) != nil { + t.Fatal("old entry must be swept") + } + }) +} From 88f9afd083b885c8d793f0b5a78b7207e328eb76 Mon Sep 17 00:00:00 2001 From: ryanmello07 Date: Sat, 25 Jul 2026 04:28:58 +0100 Subject: [PATCH 03/30] fix(model): lowercase country code in SetProviderEgressLocation The sub-project's binding constraint is that country codes are stored/ compared lowercased, and CreateLocation (network_client_location_model.go) already enforces this at the model layer via strings.ToLower before writing. SetProviderEgressLocation wrote e.CountryCode verbatim, so an uppercase code from an operator-run prober (the raw "US" the real geolocation APIs return) would silently persist uppercase and violate the invariant that countryCodeLocationIds and other lookups depend on. Normalize to lowercase before the INSERT/UPDATE, mirroring CreateLocation's idiom, and add a test asserting SetProviderEgressLocation("US") round-trips as "us" via GetProviderEgressLocation. --- model/provider_egress_location_model.go | 8 ++++- model/provider_egress_location_model_test.go | 32 ++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/model/provider_egress_location_model.go b/model/provider_egress_location_model.go index 797d82fd..980e42f7 100644 --- a/model/provider_egress_location_model.go +++ b/model/provider_egress_location_model.go @@ -2,6 +2,7 @@ package model import ( "context" + "strings" "time" "github.com/urnetwork/server" @@ -30,6 +31,11 @@ type ProviderEgressLocation struct { // SetProviderEgressLocation upserts the probed location for a provider. func SetProviderEgressLocation(ctx context.Context, e *ProviderEgressLocation) { + // country codes are stored/compared lowercased (see CreateLocation in + // network_client_location_model.go); the geolocation APIs that feed this + // return uppercase codes (e.g. "US"), so normalize before writing. + countryCode := strings.ToLower(e.CountryCode) + server.Tx(ctx, func(tx server.PgTx) { server.RaisePgResult(tx.Exec( ctx, @@ -63,7 +69,7 @@ func SetProviderEgressLocation(ctx context.Context, e *ProviderEgressLocation) { `, e.ClientId, e.LocationId, - e.CountryCode, + countryCode, e.ASN, e.Org, e.Hosting, diff --git a/model/provider_egress_location_model_test.go b/model/provider_egress_location_model_test.go index 37d9c494..bf229eb5 100644 --- a/model/provider_egress_location_model_test.go +++ b/model/provider_egress_location_model_test.go @@ -60,6 +60,38 @@ func TestProviderEgressLocationUpsertAndGet(t *testing.T) { }) } +func TestProviderEgressLocationCountryCodeLowercased(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + country := &Location{ + LocationType: LocationTypeCountry, + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, country) + + clientId := server.NewId() + // geolocation APIs return uppercase codes (e.g. "US"); the model must + // normalize to lowercase before storing, matching CreateLocation's + // established invariant that country codes are stored/compared lowercased. + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: clientId, + LocationId: country.LocationId, + CountryCode: "US", + ASN: 12345, + Org: "TEST ORG", + ObservedAt: server.NowUtc(), + }) + + got := GetProviderEgressLocation(ctx, clientId) + if got == nil { + t.Fatal("expected a stored egress location") + } + assert.Equal(t, got.CountryCode, "us") + }) +} + func TestProviderEgressLocationFreshness(t *testing.T) { server.DefaultTestEnv().Run(t, func(t testing.TB) { ctx := context.Background() From cae15a35a41d2f2fda1aca2b8a88ec951ed95752 Mon Sep 17 00:00:00 2001 From: ryanmello07 Date: Sat, 25 Jul 2026 04:37:46 +0100 Subject: [PATCH 04/30] feat(controller): resolve and store provider egress location submissions --- .../provider_egress_location_controller.go | 94 ++++++++++++ ...rovider_egress_location_controller_test.go | 139 ++++++++++++++++++ model/provider_egress_location_model.go | 55 +++++++ 3 files changed, 288 insertions(+) create mode 100644 controller/provider_egress_location_controller.go create mode 100644 controller/provider_egress_location_controller_test.go diff --git a/controller/provider_egress_location_controller.go b/controller/provider_egress_location_controller.go new file mode 100644 index 00000000..57a11d78 --- /dev/null +++ b/controller/provider_egress_location_controller.go @@ -0,0 +1,94 @@ +package controller + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" +) + +// MaxProviderEgressLocationSubmissionAge rejects a submission whose probe is +// already older than this when it arrives. It bounds replay of an old probe. +const MaxProviderEgressLocationSubmissionAge = 24 * time.Hour + +type SubmitProviderEgressLocationArgs struct { + ClientId server.Id `json:"client_id"` + CountryCode string `json:"country_code"` + Country string `json:"country"` + Region string `json:"region,omitempty"` + City string `json:"city,omitempty"` + ASN int `json:"asn,omitempty"` + Org string `json:"org,omitempty"` + Hosting bool `json:"hosting,omitempty"` + Proxy bool `json:"proxy,omitempty"` + Mobile bool `json:"mobile,omitempty"` + CountryConfident bool `json:"country_confident"` + CityConfident bool `json:"city_confident,omitempty"` + ObservedAt time.Time `json:"observed_at"` +} + +type SubmitProviderEgressLocationResult struct { + LocationId server.Id `json:"location_id"` +} + +// SubmitProviderEgressLocation records a probed egress location for a provider. +// Only country-confident submissions are accepted; city/region are stored only +// when the probe was also city-confident (free geolocation sources disagree on +// city often enough that an unconfirmed city is worse than none). +func SubmitProviderEgressLocation( + ctx context.Context, + args *SubmitProviderEgressLocationArgs, +) (*SubmitProviderEgressLocationResult, error) { + if !args.CountryConfident { + return nil, fmt.Errorf("Submission is not country-confident.") + } + countryCode := strings.ToLower(strings.TrimSpace(args.CountryCode)) + if len(countryCode) != 2 { + return nil, fmt.Errorf("Country code must be alpha-2.") + } + if args.ObservedAt.IsZero() { + return nil, fmt.Errorf("Missing observed_at.") + } + if args.ObservedAt.Before(server.NowUtc().Add(-MaxProviderEgressLocationSubmissionAge)) { + return nil, fmt.Errorf("Submission is too old.") + } + if networkId := model.GetNetworkClientNetwork(ctx, args.ClientId); networkId == nil { + return nil, fmt.Errorf("Unknown client.") + } + + // resolve to a canonical location row. city granularity only when the + // probe agreed on a city; otherwise country. + location := &model.Location{ + LocationType: model.LocationTypeCountry, + Country: args.Country, + CountryCode: countryCode, + } + if args.CityConfident && args.City != "" { + location = &model.Location{ + LocationType: model.LocationTypeCity, + City: args.City, + Region: args.Region, + Country: args.Country, + CountryCode: countryCode, + } + } + model.CreateLocation(ctx, location) + + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: args.ClientId, + LocationId: location.LocationId, + CountryCode: countryCode, + ASN: args.ASN, + Org: args.Org, + Hosting: args.Hosting, + Proxy: args.Proxy, + Mobile: args.Mobile, + CityConfident: args.CityConfident, + ObservedAt: args.ObservedAt, + }) + + return &SubmitProviderEgressLocationResult{LocationId: location.LocationId}, nil +} diff --git a/controller/provider_egress_location_controller_test.go b/controller/provider_egress_location_controller_test.go new file mode 100644 index 00000000..459e1183 --- /dev/null +++ b/controller/provider_egress_location_controller_test.go @@ -0,0 +1,139 @@ +package controller + +import ( + "context" + "testing" + "time" + + "github.com/go-playground/assert/v2" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" +) + +func TestSubmitProviderEgressLocationCountryOnly(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + res, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "US", + Country: "United States", + ASN: 401486, + Org: "RAVNIX LLC", + Hosting: true, + CountryConfident: true, + ObservedAt: server.NowUtc(), + }) + assert.Equal(t, err, nil) + if res.LocationId == (server.Id{}) { + t.Fatal("expected a resolved location id") + } + + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected the submission to be stored") + } + assert.Equal(t, stored.CountryCode, "us") + assert.Equal(t, stored.ASN, 401486) + assert.Equal(t, stored.Hosting, true) + assert.Equal(t, stored.CityConfident, false) + }) +} + +func TestSubmitProviderEgressLocationRejectsNotCountryConfident(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "us", + CountryConfident: false, + ObservedAt: server.NowUtc(), + }) + if err == nil { + t.Fatal("a submission that is not country-confident must be rejected") + } + if model.GetProviderEgressLocation(ctx, clientId) != nil { + t.Fatal("rejected submission must not be stored") + } + }) +} + +func TestSubmitProviderEgressLocationRejectsUnknownClient(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: server.NewId(), // never created + CountryCode: "us", + CountryConfident: true, + ObservedAt: server.NowUtc(), + }) + if err == nil { + t.Fatal("unknown client_id must be rejected") + } + }) +} + +func TestSubmitProviderEgressLocationCityConfidentStoresCity(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "us", + Country: "United States", + Region: "Colorado", + City: "Denver", + CountryConfident: true, + CityConfident: true, + ObservedAt: server.NowUtc(), + }) + assert.Equal(t, err, nil) + + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected the submission to be stored") + } + assert.Equal(t, stored.CityConfident, true) + + // the resolved location must be the city-granular row + loc := model.GetLocation(ctx, stored.LocationId) + if loc == nil { + t.Fatal("expected the resolved location row to exist") + } + assert.Equal(t, loc.LocationType, model.LocationTypeCity) + }) +} + +func TestSubmitProviderEgressLocationRejectsStaleObservedAt(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "us", + CountryConfident: true, + ObservedAt: server.NowUtc().Add(-30 * 24 * time.Hour), + }) + if err == nil { + t.Fatal("a submission observed long ago must be rejected") + } + }) +} diff --git a/model/provider_egress_location_model.go b/model/provider_egress_location_model.go index 980e42f7..2d78d751 100644 --- a/model/provider_egress_location_model.go +++ b/model/provider_egress_location_model.go @@ -147,6 +147,61 @@ func GetFreshProviderEgressLocation( return e } +// GetLocation returns the canonical location row, or nil. +// +// Note: the location table also has a location_name column, but it holds the +// name for whichever granularity that specific row represents (e.g. a city +// row's own name), not a single name field on the Location struct. Location +// instead splits City/Region/Country by joining sibling rows (see +// IndexSearchLocationsInTx in network_client_location_model.go). This helper +// only needs to resolve identity/type, so it selects the columns that map +// directly onto Location's fields and leaves City/Region/Country empty. +func GetLocation(ctx context.Context, locationId server.Id) *Location { + var loc *Location + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT location_id, location_type, city_location_id, region_location_id, country_location_id, country_code + FROM location + WHERE location_id = $1 + `, + locationId, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + loc = &Location{} + // city_location_id/region_location_id are only set once the + // row's hierarchy reaches that granularity (e.g. a country + // row has both NULL); server.Id.Scan errors on a nil source, + // so scan through nullable pointers as in + // IndexSearchLocationsInTx (network_client_location_model.go). + var cityLocationId *server.Id + var regionLocationId *server.Id + var countryLocationId *server.Id + server.Raise(result.Scan( + &loc.LocationId, + &loc.LocationType, + &cityLocationId, + ®ionLocationId, + &countryLocationId, + &loc.CountryCode, + )) + if cityLocationId != nil { + loc.CityLocationId = *cityLocationId + } + if regionLocationId != nil { + loc.RegionLocationId = *regionLocationId + } + if countryLocationId != nil { + loc.CountryLocationId = *countryLocationId + } + } + }) + }) + return loc +} + // RemoveExpiredProviderEgressLocations drops entries probed before // minObservedAt. func RemoveExpiredProviderEgressLocations(ctx context.Context, minObservedAt time.Time) { From 206bdd50d2eaa3568a2d955f311e6f18f1c0b3d1 Mon Sep 17 00:00:00 2001 From: ryanmello07 Date: Sat, 25 Jul 2026 04:55:34 +0100 Subject: [PATCH 05/30] test(controller): assert country granularity in provider egress location test TestSubmitProviderEgressLocationCountryOnly only checked the persisted CityConfident flag, which is copied straight from args independent of the location-resolution branch. It never asserted the resolved location's LocationType, so a broken granularity gate (resolving city unconditionally) would slip past all 5 existing tests. Add an assertion that the resolved location is LocationTypeCountry, mirroring the pattern already used by TestSubmitProviderEgressLocationCityConfidentStoresCity. Also assert CityLocationId/RegionLocationId are unset, since a country-granularity row's INSERT never populates those columns (verified against CreateLocation and GetLocation). --- .../provider_egress_location_controller_test.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/controller/provider_egress_location_controller_test.go b/controller/provider_egress_location_controller_test.go index 459e1183..9f8c70a4 100644 --- a/controller/provider_egress_location_controller_test.go +++ b/controller/provider_egress_location_controller_test.go @@ -42,6 +42,20 @@ func TestSubmitProviderEgressLocationCountryOnly(t *testing.T) { assert.Equal(t, stored.ASN, 401486) assert.Equal(t, stored.Hosting, true) assert.Equal(t, stored.CityConfident, false) + + // the resolved location must be the country-granular row, with no + // city/region association + loc := model.GetLocation(ctx, stored.LocationId) + if loc == nil { + t.Fatal("expected the resolved location row to exist") + } + assert.Equal(t, loc.LocationType, model.LocationTypeCountry) + if loc.CityLocationId != (server.Id{}) { + t.Fatal("a country-granularity row must not have a city association") + } + if loc.RegionLocationId != (server.Id{}) { + t.Fatal("a country-granularity row must not have a region association") + } }) } From feaed919dab23b9a7d1e32534c947a9edbcae7cb Mon Sep 17 00:00:00 2001 From: ryanmello07 Date: Sat, 25 Jul 2026 05:04:20 +0100 Subject: [PATCH 06/30] feat(api): operator-authenticated provider egress location ingest Adds POST /network/provider-egress-location, an operator-to-server endpoint (not a client route) that ingests probed provider egress locations. Authenticated by a shared secret from the vault (beta-vault/vault/provider_egress.yml, key ingest_secret) compared with hmac.Equal, not a network jwt. Fails closed: an absent vault resource or empty/missing key disables the endpoint (every request rejected) without panicking the api process at startup or per-request. --- api/api.go | 1 + .../provider_egress_location_handlers.go | 83 +++++++++++++++++++ .../provider_egress_location_handlers_test.go | 38 +++++++++ beta-vault/vault/provider_egress.yml.example | 6 ++ 4 files changed, 128 insertions(+) create mode 100644 api/handlers/provider_egress_location_handlers.go create mode 100644 api/handlers/provider_egress_location_handlers_test.go create mode 100644 beta-vault/vault/provider_egress.yml.example diff --git a/api/api.go b/api/api.go index fef528b8..c3dc801d 100644 --- a/api/api.go +++ b/api/api.go @@ -51,6 +51,7 @@ func Routes() []*router.Route { router.NewRoute("POST", "/auth/upgrade-guest-existing", handlers.UpgradeGuestExisting), router.NewRoute("POST", "/network/auth-client", handlers.AuthNetworkClient), router.NewRoute("POST", "/network/remove-client", handlers.RemoveNetworkClient), + router.NewRoute("POST", "/network/provider-egress-location", handlers.ProviderEgressLocationSubmit), router.NewRoute("GET", "/network/clients", handlers.NetworkClients), router.NewRoute("GET", "/network/peers", handlers.NetworkPeers), router.NewRoute("GET", "/network/provider-locations", handlers.NetworkGetProviderLocations), diff --git a/api/handlers/provider_egress_location_handlers.go b/api/handlers/provider_egress_location_handlers.go new file mode 100644 index 00000000..13557886 --- /dev/null +++ b/api/handlers/provider_egress_location_handlers.go @@ -0,0 +1,83 @@ +package handlers + +import ( + "crypto/hmac" + "encoding/json" + "io" + "net/http" + "sync" + + "github.com/urnetwork/glog" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/controller" +) + +// operatorSecretHeader carries the operator ingest secret. This endpoint is +// operator-to-server, not a client route: it is authenticated by a shared +// secret from the vault, not by a network jwt. +const operatorSecretHeader = "X-UR-Operator-Secret" + +// maxProviderEgressLocationBody bounds the request body. +const maxProviderEgressLocationBody = 16 * 1024 + +// operatorIngestSecret reads the operator ingest secret from the vault +// (beta-vault/vault/provider_egress.yml, key "ingest_secret"). It returns "" +// when the vault resource is absent, the key is absent, or the key is empty, +// which makes the endpoint fail closed (every request is rejected) rather +// than open. `SimpleResource`/`String` are the non-panicking lookups (unlike +// `RequireSimpleResource`/`RequireString`), so a missing vault resource +// disables the endpoint instead of panicking the api process at startup or +// per-request. +var operatorIngestSecret = sync.OnceValue(func() string { + res, err := server.Vault.SimpleResource("provider_egress.yml") + if err != nil { + glog.Infof("[pegl]no provider_egress.yml in the vault; ingest endpoint disabled\n") + return "" + } + values := res.String("ingest_secret") + if len(values) != 1 || values[0] == "" { + glog.Infof("[pegl]no ingest_secret in provider_egress.yml; ingest endpoint disabled\n") + return "" + } + return values[0] +}) + +// ProviderEgressLocationSubmit ingests a probed provider egress location from +// the operator's prober. See +// docs/superpowers/specs/2026-07-24-provider-egress-geolocation-design.md. +func ProviderEgressLocationSubmit(w http.ResponseWriter, r *http.Request) { + secret := operatorIngestSecret() + provided := r.Header.Get(operatorSecretHeader) + if secret == "" || provided == "" || !hmac.Equal([]byte(secret), []byte(provided)) { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + body, err := io.ReadAll(io.LimitReader(r.Body, maxProviderEgressLocationBody+1)) + if err != nil { + http.Error(w, "Bad request", http.StatusBadRequest) + return + } + if len(body) > maxProviderEgressLocationBody { + http.Error(w, "Request too large", http.StatusRequestEntityTooLarge) + return + } + + var args controller.SubmitProviderEgressLocationArgs + if err := json.Unmarshal(body, &args); err != nil { + http.Error(w, "Bad request", http.StatusBadRequest) + return + } + + result, err := controller.SubmitProviderEgressLocation(r.Context(), &args) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(result); err != nil { + glog.Infof("[pegl]could not write response. err = %s\n", err) + } +} diff --git a/api/handlers/provider_egress_location_handlers_test.go b/api/handlers/provider_egress_location_handlers_test.go new file mode 100644 index 00000000..a2afd35f --- /dev/null +++ b/api/handlers/provider_egress_location_handlers_test.go @@ -0,0 +1,38 @@ +package handlers + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "testing" +) + +func TestProviderEgressLocationSubmitRejectsMissingSecret(t *testing.T) { + body, _ := json.Marshal(map[string]any{ + "client_id": "019f8835-158d-6fd8-e9dd-fd0e4c6d6792", + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-egress-location", bytes.NewReader(body)) + w := httptest.NewRecorder() + + ProviderEgressLocationSubmit(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret header is absent", w.Code) + } +} + +func TestProviderEgressLocationSubmitRejectsWrongSecret(t *testing.T) { + body, _ := json.Marshal(map[string]any{ + "client_id": "019f8835-158d-6fd8-e9dd-fd0e4c6d6792", + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-egress-location", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, "definitely-not-the-secret") + w := httptest.NewRecorder() + + ProviderEgressLocationSubmit(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 on a wrong operator secret", w.Code) + } +} diff --git a/beta-vault/vault/provider_egress.yml.example b/beta-vault/vault/provider_egress.yml.example new file mode 100644 index 00000000..9402cafa --- /dev/null +++ b/beta-vault/vault/provider_egress.yml.example @@ -0,0 +1,6 @@ +# Operator ingest secret for POST /network/provider-egress-location. +# The operator's egress prober sends this value in the X-UR-Operator-Secret +# header. Generate with: openssl rand -base64 32 +# If this file or key is absent, the ingest endpoint rejects every request +# (fails closed, does not crash the api process). +ingest_secret: "replace-me" From 208b989437f643decf3293e65e84f868f009ab8b Mon Sep 17 00:00:00 2001 From: ryanmello07 Date: Sat, 25 Jul 2026 05:24:55 +0100 Subject: [PATCH 07/30] test(api): prove the provider-egress ingest auth gate can accept, not just reject Both committed tests for ProviderEgressLocationSubmit ran with the vault unconfigured, so hmac.Equal never executed - a handler that always returned 401 would have passed the same suite. Extract the memoized secret reader (sync.OnceValue) into a reassignable package var plus a plain readOperatorIngestSecret function, with no change to the read logic or the handler's auth gate, so tests can inject a known secret without racing the existing unconfigured-vault reject tests in the same process. Add a positive-path test (correct secret clears auth and reaches the controller, 400 "Unknown client." for an unregistered id) and a negative discrimination test (wrong secret against a configured vault still 401), plus a direct vault-read test. Document in the vault example that the secret is memoized at first request, so rotating it requires an api restart. --- .../provider_egress_location_handlers.go | 12 +- .../provider_egress_location_handlers_test.go | 112 ++++++++++++++++++ beta-vault/vault/provider_egress.yml.example | 3 + 3 files changed, 124 insertions(+), 3 deletions(-) diff --git a/api/handlers/provider_egress_location_handlers.go b/api/handlers/provider_egress_location_handlers.go index 13557886..3271b167 100644 --- a/api/handlers/provider_egress_location_handlers.go +++ b/api/handlers/provider_egress_location_handlers.go @@ -21,7 +21,13 @@ const operatorSecretHeader = "X-UR-Operator-Secret" // maxProviderEgressLocationBody bounds the request body. const maxProviderEgressLocationBody = 16 * 1024 -// operatorIngestSecret reads the operator ingest secret from the vault +// operatorIngestSecret memoizes readOperatorIngestSecret for the life of the +// process. It is a package-level var (not a plain sync.OnceValue call site) +// so tests can swap it for a stub and restore it with defer; production code +// never reassigns it. +var operatorIngestSecret func() string = sync.OnceValue(readOperatorIngestSecret) + +// readOperatorIngestSecret reads the operator ingest secret from the vault // (beta-vault/vault/provider_egress.yml, key "ingest_secret"). It returns "" // when the vault resource is absent, the key is absent, or the key is empty, // which makes the endpoint fail closed (every request is rejected) rather @@ -29,7 +35,7 @@ const maxProviderEgressLocationBody = 16 * 1024 // `RequireSimpleResource`/`RequireString`), so a missing vault resource // disables the endpoint instead of panicking the api process at startup or // per-request. -var operatorIngestSecret = sync.OnceValue(func() string { +func readOperatorIngestSecret() string { res, err := server.Vault.SimpleResource("provider_egress.yml") if err != nil { glog.Infof("[pegl]no provider_egress.yml in the vault; ingest endpoint disabled\n") @@ -41,7 +47,7 @@ var operatorIngestSecret = sync.OnceValue(func() string { return "" } return values[0] -}) +} // ProviderEgressLocationSubmit ingests a probed provider egress location from // the operator's prober. See diff --git a/api/handlers/provider_egress_location_handlers_test.go b/api/handlers/provider_egress_location_handlers_test.go index a2afd35f..4af00971 100644 --- a/api/handlers/provider_egress_location_handlers_test.go +++ b/api/handlers/provider_egress_location_handlers_test.go @@ -5,7 +5,11 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "strings" "testing" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/controller" ) func TestProviderEgressLocationSubmitRejectsMissingSecret(t *testing.T) { @@ -36,3 +40,111 @@ func TestProviderEgressLocationSubmitRejectsWrongSecret(t *testing.T) { t.Fatalf("status = %d, want 401 on a wrong operator secret", w.Code) } } + +// withStubOperatorIngestSecret swaps the package-level operatorIngestSecret +// memo for a stub that always returns secret, and returns a func to restore +// the original (real, still-memoized) reader. This lets a test exercise the +// "configured vault" path without the sync.OnceValue in the real reader ever +// touching the vault, and without the reject tests above (which rely on an +// unconfigured vault) observing any change. +func withStubOperatorIngestSecret(secret string) (restore func()) { + prev := operatorIngestSecret + operatorIngestSecret = func() string { return secret } + return func() { operatorIngestSecret = prev } +} + +// TestProviderEgressLocationSubmitAcceptsCorrectSecret proves the auth gate +// can ACCEPT a correct secret and hand off to the controller. Without this +// test, the two reject tests above (which both run with the vault +// unconfigured and take the secret=="" short-circuit) would pass unchanged +// even if the handler's entire body were replaced with an unconditional 401 - +// hmac.Equal would never be proven to run on a real match. +// +// Clearing auth hands the request to controller.SubmitProviderEgressLocation, +// which looks up the client in the database before it can return "Unknown +// client.", so this test needs a real (throwaway) test database - see +// server.DefaultTestEnv, the same harness +// controller/provider_egress_location_controller_test.go uses. t.Setenv makes +// it self-sufficient under a plain `go test`, matching the pattern in +// router/warp_handlers_status_test.go. +func TestProviderEgressLocationSubmitAcceptsCorrectSecret(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + // A syntactically valid, semantically unregistered submission: once + // auth clears, the controller looks up the client and (since it does + // not exist in the fresh test database) returns "Unknown client.", + // surfaced by the handler as 400. That 400 is proof the request + // reached the controller, i.e. proof auth passed. + args := controller.SubmitProviderEgressLocationArgs{ + ClientId: server.NewId(), + CountryCode: "US", + Country: "United States", + CountryConfident: true, + ObservedAt: server.NowUtc(), + } + body, err := json.Marshal(args) + if err != nil { + t.Fatalf("marshal args: %s", err) + } + + req := httptest.NewRequest(http.MethodPost, "/network/provider-egress-location", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderEgressLocationSubmit(w, req) + + if w.Code == http.StatusUnauthorized { + t.Fatalf("status = %d, want the correct secret to clear auth (not 401)", w.Code) + } + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 (Unknown client.) once auth clears for an unregistered client id; body = %s", w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), "Unknown client.") { + t.Fatalf("body = %q, want it to report the unknown client", w.Body.String()) + } + }) +} + +// TestProviderEgressLocationSubmitRejectsAlteredSecret proves that once the +// vault is configured, hmac.Equal is actually consulted rather than the +// endpoint accepting any request once secret != "". Same configured secret as +// the accept test above, but the request carries a one-character-altered +// value. +func TestProviderEgressLocationSubmitRejectsAlteredSecret(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + const wrongSecret = "correct-operator-secret-0123456780" // last char changed + defer withStubOperatorIngestSecret(secret)() + + body, _ := json.Marshal(map[string]any{ + "client_id": "019f8835-158d-6fd8-e9dd-fd0e4c6d6792", + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-egress-location", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, wrongSecret) + w := httptest.NewRecorder() + + ProviderEgressLocationSubmit(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret is configured but the request's secret is wrong", w.Code) + } +} + +// TestProviderEgressLocationSubmitReadsSecretFromVault proves the vault +// plumbing itself - not the test stub - returns the configured secret: +// readOperatorIngestSecret (the un-memoized reader) reads a +// PushSimpleResource-injected provider_egress.yml. +func TestProviderEgressLocationSubmitReadsSecretFromVault(t *testing.T) { + const secret = "vault-provisioned-secret-abcdef" + pop := server.Vault.PushSimpleResource( + "provider_egress.yml", + []byte(`ingest_secret: "`+secret+`"`), + ) + defer pop() + + if got := readOperatorIngestSecret(); got != secret { + t.Fatalf("readOperatorIngestSecret() = %q, want %q", got, secret) + } +} diff --git a/beta-vault/vault/provider_egress.yml.example b/beta-vault/vault/provider_egress.yml.example index 9402cafa..2024763f 100644 --- a/beta-vault/vault/provider_egress.yml.example +++ b/beta-vault/vault/provider_egress.yml.example @@ -3,4 +3,7 @@ # header. Generate with: openssl rand -base64 32 # If this file or key is absent, the ingest endpoint rejects every request # (fails closed, does not crash the api process). +# The secret is read once, on the first request, and memoized for the life of +# the process: after creating or rotating this file, restart the api for the +# new value to take effect. ingest_secret: "replace-me" From 010bf9170876ed32d5f9b12a9393f866eccc1a7e Mon Sep 17 00:00:00 2001 From: ryanmello07 Date: Sat, 25 Jul 2026 05:31:52 +0100 Subject: [PATCH 08/30] feat(controller): prefer probed provider egress location over mmdb --- controller/network_client_controller.go | 30 +++++++ controller/network_client_controller_test.go | 94 ++++++++++++++++++++ model/provider_egress_location_model.go | 18 ++++ 3 files changed, 142 insertions(+) create mode 100644 controller/network_client_controller_test.go diff --git a/controller/network_client_controller.go b/controller/network_client_controller.go index 5ecd7252..5bd46cf8 100644 --- a/controller/network_client_controller.go +++ b/controller/network_client_controller.go @@ -56,6 +56,36 @@ func SetConnectionLocation( connectionId server.Id, clientIp string, ) error { + // a provider probed through its own egress is located from that probe, not + // from a lookup on its control-connection ip: the egress is where user + // traffic actually exits, and the probed value is cross-checked across + // several sources. see + // docs/superpowers/specs/2026-07-24-provider-egress-geolocation-design.md + if clientId := model.GetNetworkClientForConnection(ctx, connectionId); clientId != nil { + if egress := model.GetFreshProviderEgressLocation( + ctx, + *clientId, + model.ProviderEgressLocationMaxAge, + ); egress != nil { + scores := &model.ConnectionLocationScores{} + if egress.Hosting { + scores.NetTypeHosting = 1 + } + if egress.Proxy { + scores.NetTypePrivacy = 1 + } + if egress.Mobile { + scores.NetTypeVirtual = 1 + } + err := model.SetConnectionLocation(ctx, connectionId, egress.LocationId, scores) + if err == nil { + return nil + } + // fall through to the mmdb path on a storage error + glog.Infof("[ncc][%s]could not set probed egress location. err = %s\n", connectionId, err) + } + } + location, connectionLocationScores, err := GetLocationForIp(ctx, clientIp) if err != nil { // server.Logger().Printf("Get ip for location error: %s", err) diff --git a/controller/network_client_controller_test.go b/controller/network_client_controller_test.go new file mode 100644 index 00000000..863f412c --- /dev/null +++ b/controller/network_client_controller_test.go @@ -0,0 +1,94 @@ +package controller + +import ( + "context" + "testing" + + "github.com/go-playground/assert/v2" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" +) + +// A provider with a fresh probed egress location must be located from that +// entry, not from the mmdb lookup on its control ip. +func TestSetConnectionLocationPrefersEgressLocation(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + // the probed egress location: japan + probed := &model.Location{ + LocationType: model.LocationTypeCountry, + Country: "Japan", + CountryCode: "jp", + } + model.CreateLocation(ctx, probed) + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, "8.8.8.8:0", handlerId) + assert.Equal(t, err, nil) + + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: clientId, + LocationId: probed.LocationId, + CountryCode: "jp", + ObservedAt: server.NowUtc(), + }) + + err = SetConnectionLocation(ctx, connectionId, "8.8.8.8") + assert.Equal(t, err, nil) + + var countryLocationId server.Id + server.Db(ctx, func(conn server.PgConn) { + result, qerr := conn.Query( + ctx, + `SELECT country_location_id FROM network_client_location WHERE connection_id = $1`, + connectionId, + ) + server.WithPgResult(result, qerr, func() { + if result.Next() { + server.Raise(result.Scan(&countryLocationId)) + } + }) + }) + assert.Equal(t, countryLocationId, probed.CountryLocationId) + }) +} + +// With no probed entry, the existing mmdb path still applies. +func TestSetConnectionLocationFallsBackToMmdb(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, "8.8.8.8:0", handlerId) + assert.Equal(t, err, nil) + + // no SetProviderEgressLocation call -> mmdb path + err = SetConnectionLocation(ctx, connectionId, "8.8.8.8") + assert.Equal(t, err, nil) + + var count int + server.Db(ctx, func(conn server.PgConn) { + result, qerr := conn.Query( + ctx, + `SELECT COUNT(*) FROM network_client_location WHERE connection_id = $1`, + connectionId, + ) + server.WithPgResult(result, qerr, func() { + if result.Next() { + server.Raise(result.Scan(&count)) + } + }) + }) + assert.Equal(t, count, 1) + }) +} diff --git a/model/provider_egress_location_model.go b/model/provider_egress_location_model.go index 2d78d751..c12950a6 100644 --- a/model/provider_egress_location_model.go +++ b/model/provider_egress_location_model.go @@ -128,6 +128,24 @@ func GetProviderEgressLocation(ctx context.Context, clientId server.Id) *Provide return e } +// GetNetworkClientForConnection returns the client id for a connection, or nil. +func GetNetworkClientForConnection(ctx context.Context, connectionId server.Id) *server.Id { + var clientId *server.Id + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + `SELECT client_id FROM network_client_connection WHERE connection_id = $1`, + connectionId, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + server.Raise(result.Scan(&clientId)) + } + }) + }) + return clientId +} + // GetFreshProviderEgressLocation is GetProviderEgressLocation, filtered to // entries probed within maxAge. The cutoff is computed in Go and bound as a // parameter: observed_at is a naive timestamp holding utc, and comparing it From 2a58b1efe073aa9a15830c45b324fc13d1787a4e Mon Sep 17 00:00:00 2001 From: ryanmello07 Date: Sat, 25 Jul 2026 05:56:34 +0100 Subject: [PATCH 09/30] fix(controller): collapse egress lookup to one query, add ARIN foreign parity, cover unprotected branches SetConnectionLocation ran on the connect-announce hot path (every client, every connection, inside a retry loop) with two separate DB round trips before ever reaching the mmdb fallback. Fix review findings against the probed-egress location change: - model.GetFreshProviderEgressLocationForConnection replaces the GetNetworkClientForConnection + GetFreshProviderEgressLocation pair with a single query joining network_client_connection to provider_egress_location. Freshness cutoff is still computed/compared in Go, not SQL now(). Cuts the probed-hit path from 3 DB round trips to 2, and the fallback path from 4 to 3. - The probed path now also runs the ARIN org-vs-country foreign check (against the probed country code), matching the mmdb path's GetLocationForIp, so net_type_foreign no longer gives probed providers an unearned ranking advantage over unprobed ones. Factored into a shared arinForeignScore helper; on any ARIN/ip-parse failure it just leaves NetTypeForeign at 0, never erroring or panicking the hot path. - Added tests for the stale-probed fallback, a probed write error (bad location_id) falling back to mmdb without panicking, and the Hosting/Proxy flag-to-score mapping. Verified the stale-fallback test has teeth by temporarily widening maxAge and confirming it fails. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01QtgqtCmKJRXdsQ5ktiqwkg --- controller/network_client_controller.go | 57 +++--- controller/network_client_controller_test.go | 169 ++++++++++++++++++ .../network_client_location_controller.go | 39 ++-- model/provider_egress_location_model.go | 68 +++++++ 4 files changed, 297 insertions(+), 36 deletions(-) diff --git a/controller/network_client_controller.go b/controller/network_client_controller.go index 5bd46cf8..d5b457cd 100644 --- a/controller/network_client_controller.go +++ b/controller/network_client_controller.go @@ -2,6 +2,7 @@ package controller import ( "context" + "net/netip" // "strings" "time" @@ -61,29 +62,41 @@ func SetConnectionLocation( // traffic actually exits, and the probed value is cross-checked across // several sources. see // docs/superpowers/specs/2026-07-24-provider-egress-geolocation-design.md - if clientId := model.GetNetworkClientForConnection(ctx, connectionId); clientId != nil { - if egress := model.GetFreshProviderEgressLocation( - ctx, - *clientId, - model.ProviderEgressLocationMaxAge, - ); egress != nil { - scores := &model.ConnectionLocationScores{} - if egress.Hosting { - scores.NetTypeHosting = 1 - } - if egress.Proxy { - scores.NetTypePrivacy = 1 - } - if egress.Mobile { - scores.NetTypeVirtual = 1 - } - err := model.SetConnectionLocation(ctx, connectionId, egress.LocationId, scores) - if err == nil { - return nil - } - // fall through to the mmdb path on a storage error - glog.Infof("[ncc][%s]could not set probed egress location. err = %s\n", connectionId, err) + // a single query joining network_client_connection to + // provider_egress_location: this runs for every connection (provider or + // not) on the connect-announce path and inside a retry loop, so it must + // not cost the two round trips (client lookup, then egress lookup) the + // naive version would. + if egress := model.GetFreshProviderEgressLocationForConnection( + ctx, + connectionId, + model.ProviderEgressLocationMaxAge, + ); egress != nil { + scores := &model.ConnectionLocationScores{} + if egress.Hosting { + scores.NetTypeHosting = 1 + } + if egress.Proxy { + scores.NetTypePrivacy = 1 + } + if egress.Mobile { + scores.NetTypeVirtual = 1 + } + // keep the ARIN org-vs-country foreign check on the probed path too, + // so a probed provider is ranked on equal terms with an equivalent + // unprobed one (net_type_foreign feeds the ranking columns). Compare + // against the probed country -- that's the country now being + // claimed. Any lookup failure just leaves NetTypeForeign at 0; it + // must never fail or panic this path. + if addr, err := netip.ParseAddr(clientIp); err == nil { + scores.NetTypeForeign = arinForeignScore(addr, egress.CountryCode) + } + err := model.SetConnectionLocation(ctx, connectionId, egress.LocationId, scores) + if err == nil { + return nil } + // fall through to the mmdb path on a storage error + glog.Infof("[ncc][%s]could not set probed egress location. err = %s\n", connectionId, err) } location, connectionLocationScores, err := GetLocationForIp(ctx, clientIp) diff --git a/controller/network_client_controller_test.go b/controller/network_client_controller_test.go index 863f412c..0f476477 100644 --- a/controller/network_client_controller_test.go +++ b/controller/network_client_controller_test.go @@ -3,6 +3,7 @@ package controller import ( "context" "testing" + "time" "github.com/go-playground/assert/v2" @@ -92,3 +93,171 @@ func TestSetConnectionLocationFallsBackToMmdb(t *testing.T) { assert.Equal(t, count, 1) }) } + +// A probed egress location observed longer ago than ProviderEgressLocationMaxAge +// must be ignored, falling back to the mmdb lookup exactly as if there were no +// probed entry at all. +func TestSetConnectionLocationStaleProbedFallsBackToMmdb(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + clientIp := "8.8.8.8" + + // the mmdb location this ip actually resolves to; created up front so + // its canonical (deduped) location id is known for the assertion. + mmdbLocation, _, err := GetLocationForIp(ctx, clientIp) + assert.Equal(t, err, nil) + model.CreateLocation(ctx, mmdbLocation) + + // a stale probed location, deliberately a different country than + // whatever the mmdb lookup returns, so a wrongly-preferred probed + // value would be caught by the assertion below. + probed := &model.Location{ + LocationType: model.LocationTypeCountry, + Country: "Japan", + CountryCode: "jp", + } + model.CreateLocation(ctx, probed) + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientIp+":0", handlerId) + assert.Equal(t, err, nil) + + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: clientId, + LocationId: probed.LocationId, + CountryCode: "jp", + ObservedAt: server.NowUtc().Add(-(model.ProviderEgressLocationMaxAge + time.Hour)), + }) + + err = SetConnectionLocation(ctx, connectionId, clientIp) + assert.Equal(t, err, nil) + + var countryLocationId server.Id + server.Db(ctx, func(conn server.PgConn) { + result, qerr := conn.Query( + ctx, + `SELECT country_location_id FROM network_client_location WHERE connection_id = $1`, + connectionId, + ) + server.WithPgResult(result, qerr, func() { + if result.Next() { + server.Raise(result.Scan(&countryLocationId)) + } + }) + }) + assert.Equal(t, countryLocationId, mmdbLocation.CountryLocationId) + if countryLocationId == probed.CountryLocationId { + t.Fatal("stale probed location must not be used") + } + }) +} + +// A probed egress location whose location_id points at no existing location +// row makes the storage write fail (SetConnectionLocation returns an error +// rather than panicking, see model.SetConnectionLocation). The connection +// must still end up located via the mmdb path, and SetConnectionLocation +// itself must not panic or error out on this. +func TestSetConnectionLocationProbedWriteErrorFallsBackToMmdb(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + clientIp := "8.8.8.8" + + mmdbLocation, _, err := GetLocationForIp(ctx, clientIp) + assert.Equal(t, err, nil) + model.CreateLocation(ctx, mmdbLocation) + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientIp+":0", handlerId) + assert.Equal(t, err, nil) + + // a fresh probed entry pointing at a location id that was never + // created via CreateLocation -- the storage write in + // model.SetConnectionLocation must fail cleanly on this, not panic. + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: clientId, + LocationId: server.NewId(), + CountryCode: "jp", + ObservedAt: server.NowUtc(), + }) + + err = SetConnectionLocation(ctx, connectionId, clientIp) + assert.Equal(t, err, nil) + + var countryLocationId server.Id + server.Db(ctx, func(conn server.PgConn) { + result, qerr := conn.Query( + ctx, + `SELECT country_location_id FROM network_client_location WHERE connection_id = $1`, + connectionId, + ) + server.WithPgResult(result, qerr, func() { + if result.Next() { + server.Raise(result.Scan(&countryLocationId)) + } + }) + }) + assert.Equal(t, countryLocationId, mmdbLocation.CountryLocationId) + }) +} + +// A fresh probed location's Hosting/Proxy flags must map onto the stored +// connection's net_type_hosting/net_type_privacy scores. +func TestSetConnectionLocationMapsProbedFlagsToScores(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + probed := &model.Location{ + LocationType: model.LocationTypeCountry, + Country: "Japan", + CountryCode: "jp", + } + model.CreateLocation(ctx, probed) + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, "8.8.8.8:0", handlerId) + assert.Equal(t, err, nil) + + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: clientId, + LocationId: probed.LocationId, + CountryCode: "jp", + Hosting: true, + Proxy: true, + ObservedAt: server.NowUtc(), + }) + + err = SetConnectionLocation(ctx, connectionId, "8.8.8.8") + assert.Equal(t, err, nil) + + var netTypeHosting int + var netTypePrivacy int + server.Db(ctx, func(conn server.PgConn) { + result, qerr := conn.Query( + ctx, + `SELECT net_type_hosting, net_type_privacy FROM network_client_location WHERE connection_id = $1`, + connectionId, + ) + server.WithPgResult(result, qerr, func() { + if result.Next() { + server.Raise(result.Scan(&netTypeHosting, &netTypePrivacy)) + } + }) + }) + assert.Equal(t, netTypeHosting, 1) + assert.Equal(t, netTypePrivacy, 1) + }) +} diff --git a/controller/network_client_location_controller.go b/controller/network_client_location_controller.go index 3376a9af..0535db77 100644 --- a/controller/network_client_location_controller.go +++ b/controller/network_client_location_controller.go @@ -55,21 +55,32 @@ func GetLocationForIp(ctx context.Context, clientIp string) (*model.Location, *m connectionLocationScores.NetTypeVirtual = 1 } + connectionLocationScores.NetTypeForeign = arinForeignScore(addr, ipInfo.CountryCode) + + return location, connectionLocationScores, nil +} + +// arinForeignScore cross-checks the ARIN org registration country for addr +// against countryCode, the country code being claimed for this connection +// (the mmdb-resolved country on the ordinary path, or the probed egress +// country on the provider-egress path). If the org's registered country +// differs, the use case is considered foreign (VPN/proxy-like), matching the +// heuristic previously inlined in GetLocationForIp. +// +// If the ARIN lookup fails, this returns 0 without error: it must never fail +// or panic a caller on the connect-announce hot path over a missing/failed +// foreign check. +func arinForeignScore(addr netip.Addr, countryCode string) int { arinInfo, err := server.GetArinInfo(addr) - if err == nil { - // if the org ownership does not match the ip country, - // we consider the use case of the ip to be virtual - foreign := false - for _, orgCountryCode := range arinInfo.OrgCountryCodes { - if orgCountryCode != ipInfo.CountryCode { - foreign = true - break - } - } - if foreign { - connectionLocationScores.NetTypeForeign = 1 + if err != nil { + return 0 + } + // if the org ownership does not match the claimed country, + // we consider the use case of the ip to be foreign + for _, orgCountryCode := range arinInfo.OrgCountryCodes { + if orgCountryCode != countryCode { + return 1 } } - - return location, connectionLocationScores, nil + return 0 } diff --git a/model/provider_egress_location_model.go b/model/provider_egress_location_model.go index c12950a6..f7d0c69f 100644 --- a/model/provider_egress_location_model.go +++ b/model/provider_egress_location_model.go @@ -165,6 +165,74 @@ func GetFreshProviderEgressLocation( return e } +// GetFreshProviderEgressLocationForConnection resolves the probed provider +// egress location for a connection in a single query, joining +// network_client_connection to provider_egress_location on client_id. This +// exists for the connect-announce hot path (SetConnectionLocation), which +// previously spent two round trips per connection -- GetNetworkClientForConnection +// then GetFreshProviderEgressLocation -- before ever reaching the mmdb +// fallback; collapsing to one query matters on a path that runs for every +// connection and inside a retry loop. +// +// As with GetFreshProviderEgressLocation, the maxAge cutoff is computed in Go +// and compared in Go: observed_at is a naive timestamp holding utc, and +// comparing it against sql now() would cast through the session timezone. +func GetFreshProviderEgressLocationForConnection( + ctx context.Context, + connectionId server.Id, + maxAge time.Duration, +) *ProviderEgressLocation { + var e *ProviderEgressLocation + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT + pel.client_id, + pel.location_id, + pel.country_code, + pel.asn, + pel.org, + pel.hosting, + pel.proxy, + pel.mobile, + pel.city_confident, + pel.observed_at, + pel.update_time + FROM network_client_connection ncc + INNER JOIN provider_egress_location pel ON pel.client_id = ncc.client_id + WHERE ncc.connection_id = $1 + `, + connectionId, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + e = &ProviderEgressLocation{} + server.Raise(result.Scan( + &e.ClientId, + &e.LocationId, + &e.CountryCode, + &e.ASN, + &e.Org, + &e.Hosting, + &e.Proxy, + &e.Mobile, + &e.CityConfident, + &e.ObservedAt, + &e.UpdateTime, + )) + } + }) + }) + if e == nil { + return nil + } + if e.ObservedAt.Before(server.NowUtc().Add(-maxAge)) { + return nil + } + return e +} + // GetLocation returns the canonical location row, or nil. // // Note: the location table also has a location_name column, but it holds the From 35ba895e8810af2858cef367884f402827e4a7bf Mon Sep 17 00:00:00 2001 From: ryanmello07 Date: Sat, 25 Jul 2026 05:58:30 +0100 Subject: [PATCH 10/30] feat(taskworker): sweep expired provider egress locations --- taskworker/taskworker.go | 5 ++ .../work/provider_egress_location_work.go | 49 +++++++++++++++++++ 2 files changed, 54 insertions(+) create mode 100644 taskworker/work/provider_egress_location_work.go diff --git a/taskworker/taskworker.go b/taskworker/taskworker.go index 3364dce0..9cbc1c05 100644 --- a/taskworker/taskworker.go +++ b/taskworker/taskworker.go @@ -52,6 +52,7 @@ func InitTasks(ctx context.Context) { work.ScheduleRemoveExpiredAuthAttempts(clientSession, tx) work.ScheduleRemoveExpiredWalletAuthChallenges(clientSession, tx) work.ScheduleRemoveExpiredWalletNonces(clientSession, tx) + work.ScheduleRemoveExpiredProviderEgressLocations(clientSession, tx) work.ScheduleRemoveOldAuditNetworkEvents(clientSession, tx) work.ScheduleRemoveOldAuditEvents(clientSession, tx) work.ScheduleRemoveOldClientReliabilityStats(clientSession, tx) @@ -236,6 +237,10 @@ func InitTaskWorkerWithSettings(ctx context.Context, settings *task.TaskWorkerSe work.RemoveExpiredWalletNoncesPost, "github.com/urnetwork/server/taskworker/work.RemoveExpiredWalletNonces", ), + task.NewTaskTargetWithPost( + work.RemoveExpiredProviderEgressLocations, + work.RemoveExpiredProviderEgressLocationsPost, + ), task.NewTaskTargetWithPost( work.RemoveOldAuditNetworkEvents, work.RemoveOldAuditNetworkEventsPost, diff --git a/taskworker/work/provider_egress_location_work.go b/taskworker/work/provider_egress_location_work.go new file mode 100644 index 00000000..94901c34 --- /dev/null +++ b/taskworker/work/provider_egress_location_work.go @@ -0,0 +1,49 @@ +package work + +import ( + "time" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" + "github.com/urnetwork/server/session" + "github.com/urnetwork/server/task" +) + +type RemoveExpiredProviderEgressLocationsArgs struct{} + +type RemoveExpiredProviderEgressLocationsResult struct{} + +func ScheduleRemoveExpiredProviderEgressLocations(clientSession *session.ClientSession, tx server.PgTx) { + task.ScheduleTaskInTx( + tx, + RemoveExpiredProviderEgressLocations, + &RemoveExpiredProviderEgressLocationsArgs{}, + clientSession, + task.RunOnce("remove_expired_provider_egress_locations"), + task.RunAt(server.NowUtc().Add(6*time.Hour)), + ) +} + +// RemoveExpiredProviderEgressLocations drops probed locations well past their +// trust window, so a provider that stops being probed eventually falls back to +// the mmdb path instead of being pinned to a stale location forever. The cutoff +// is deliberately looser than ProviderEgressLocationMaxAge: reads already +// ignore stale rows, so this is only reclaiming storage. +func RemoveExpiredProviderEgressLocations( + _ *RemoveExpiredProviderEgressLocationsArgs, + clientSession *session.ClientSession, +) (*RemoveExpiredProviderEgressLocationsResult, error) { + minObservedAt := server.NowUtc().Add(-4 * model.ProviderEgressLocationMaxAge) + model.RemoveExpiredProviderEgressLocations(clientSession.Ctx, minObservedAt) + return &RemoveExpiredProviderEgressLocationsResult{}, nil +} + +func RemoveExpiredProviderEgressLocationsPost( + _ *RemoveExpiredProviderEgressLocationsArgs, + _ *RemoveExpiredProviderEgressLocationsResult, + clientSession *session.ClientSession, + tx server.PgTx, +) error { + ScheduleRemoveExpiredProviderEgressLocations(clientSession, tx) + return nil +} From 211e5a18ba0e87ad233320a37e666bd6e2c44e11 Mon Sep 17 00:00:00 2001 From: ryanmello07 Date: Sat, 25 Jul 2026 06:49:45 +0100 Subject: [PATCH 11/30] fix(provider-egress): correct ARIN foreign parity, reject corrupting/oversized submissions, wire up ingest secret Pre-merge fixes from the final whole-branch review of provider-egress geolocation: - Fix the probed-path ARIN "foreign" check: it compared the control ip's ARIN org country against the probed egress country (two different ips), flagging a provider foreign precisely when probing changed the answer. Now matches the mmdb path exactly: ARIN org country of the control ip vs. the mmdb country of that same control ip, restoring probed/unprobed ranking parity. - Reject provider-egress submissions with an empty (post-trim) Country/City/Region instead of silently creating a permanently-blank canonical location row via CreateLocation's dedupe-on-name behavior. - Reject over-long Country/City/Region (>128) or Org (>256) instead of panicking inside CreateLocation on a Postgres "value too long" error. - Make SetProviderEgressLocation's upsert monotonic in observed_at so a replayed older submission cannot clobber a newer stored row. - Replace three production-comment references to the (upstream-absent) design-spec doc path with inline prose, and drop the hardcoded beta vault filesystem path from the ingest-secret handler comment. - Delete GetNetworkClientForConnection (model/provider_egress_location_model.go), a fully superseded, zero-caller helper. - Provision the ingest secret end-to-end: beta-setup.sh now generates beta-vault/vault/provider_egress.yml (guarded the same way as the other generated secrets), and BETA.md documents it plus the api-restart requirement, since the endpoint otherwise ships silently disabled. Adds tests for each behavioral fix; all pass, including TestSetConnectionLocationToleratesCountryOnlyLocation. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01QtgqtCmKJRXdsQ5ktiqwkg --- .../provider_egress_location_handlers.go | 10 +- controller/network_client_controller.go | 24 ++- controller/network_client_controller_test.go | 63 ++++++++ .../provider_egress_location_controller.go | 61 +++++++- ...rovider_egress_location_controller_test.go | 142 ++++++++++++++++++ db_migrations.go | 15 +- model/provider_egress_location_model.go | 32 ++-- model/provider_egress_location_model_test.go | 58 ++++++- 8 files changed, 358 insertions(+), 47 deletions(-) diff --git a/api/handlers/provider_egress_location_handlers.go b/api/handlers/provider_egress_location_handlers.go index 3271b167..b843c05b 100644 --- a/api/handlers/provider_egress_location_handlers.go +++ b/api/handlers/provider_egress_location_handlers.go @@ -28,7 +28,7 @@ const maxProviderEgressLocationBody = 16 * 1024 var operatorIngestSecret func() string = sync.OnceValue(readOperatorIngestSecret) // readOperatorIngestSecret reads the operator ingest secret from the vault -// (beta-vault/vault/provider_egress.yml, key "ingest_secret"). It returns "" +// resource "provider_egress.yml", key "ingest_secret". It returns "" // when the vault resource is absent, the key is absent, or the key is empty, // which makes the endpoint fail closed (every request is rejected) rather // than open. `SimpleResource`/`String` are the non-panicking lookups (unlike @@ -50,8 +50,12 @@ func readOperatorIngestSecret() string { } // ProviderEgressLocationSubmit ingests a probed provider egress location from -// the operator's prober. See -// docs/superpowers/specs/2026-07-24-provider-egress-geolocation-design.md. +// the operator's prober. The prober routes geolocation lookups through a +// provider's own egress -- rather than relying on a lookup against the +// provider's control-connection ip -- and submits the result here so the +// server can prefer it over the built-in mmdb lookup. The route is +// operator-to-server, authenticated by the shared secret above rather than a +// network jwt. func ProviderEgressLocationSubmit(w http.ResponseWriter, r *http.Request) { secret := operatorIngestSecret() provided := r.Header.Get(operatorSecretHeader) diff --git a/controller/network_client_controller.go b/controller/network_client_controller.go index d5b457cd..382aa913 100644 --- a/controller/network_client_controller.go +++ b/controller/network_client_controller.go @@ -59,9 +59,11 @@ func SetConnectionLocation( ) error { // a provider probed through its own egress is located from that probe, not // from a lookup on its control-connection ip: the egress is where user - // traffic actually exits, and the probed value is cross-checked across - // several sources. see - // docs/superpowers/specs/2026-07-24-provider-egress-geolocation-design.md + // traffic actually exits, and an operator-run prober learns it by routing + // geolocation lookups through the provider itself and cross-checking them + // across several sources, then submits the result here. When a fresh + // probed entry exists we prefer it over the built-in mmdb lookup on the + // control ip. GetFreshProviderEgressLocationForConnection is // a single query joining network_client_connection to // provider_egress_location: this runs for every connection (provider or // not) on the connect-announce path and inside a retry loop, so it must @@ -84,12 +86,18 @@ func SetConnectionLocation( } // keep the ARIN org-vs-country foreign check on the probed path too, // so a probed provider is ranked on equal terms with an equivalent - // unprobed one (net_type_foreign feeds the ranking columns). Compare - // against the probed country -- that's the country now being - // claimed. Any lookup failure just leaves NetTypeForeign at 0; it - // must never fail or panic this path. + // unprobed one (net_type_foreign feeds the ranking columns). Compute + // it exactly as the mmdb path does in GetLocationForIp: the ARIN org + // country of the control ip against the mmdb country of that SAME + // control ip -- not the probed country, which is a different + // question (whether probing changed the answer) and must not be + // silently folded into this ranking penalty. Any lookup failure + // just leaves NetTypeForeign at 0; it must never fail or panic this + // path. if addr, err := netip.ParseAddr(clientIp); err == nil { - scores.NetTypeForeign = arinForeignScore(addr, egress.CountryCode) + if ipInfo, err := server.GetIpInfo(addr); err == nil { + scores.NetTypeForeign = arinForeignScore(addr, ipInfo.CountryCode) + } } err := model.SetConnectionLocation(ctx, connectionId, egress.LocationId, scores) if err == nil { diff --git a/controller/network_client_controller_test.go b/controller/network_client_controller_test.go index 0f476477..ea6c593c 100644 --- a/controller/network_client_controller_test.go +++ b/controller/network_client_controller_test.go @@ -210,6 +210,69 @@ func TestSetConnectionLocationProbedWriteErrorFallsBackToMmdb(t *testing.T) { }) } +// net_type_foreign on the probed path must be computed the same way as the +// mmdb path: the ARIN org country of the control ip against the mmdb country +// of that SAME control ip, not against the probed country. 8.8.8.8 resolves +// (both via mmdb and ARIN org registration) to "us", so it is non-foreign by +// construction -- this is the same ip used by the parity tests above. The +// probed country here is deliberately "jp", a different country than the +// control ip's mmdb country: probing having changed the answer must not, by +// itself, flip net_type_foreign, or a probed provider is penalized a full +// ranking tier precisely for the reason this feature exists. A prior version +// of this code compared the ARIN org country against the probed country +// instead of the control ip's mmdb country, which made this exact scenario +// (egress country differs from control-ip country) foreign=1 instead of the +// correct foreign=0. +func TestSetConnectionLocationProbedNetTypeForeignMatchesMmdbParity(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + clientIp := "8.8.8.8" + + probed := &model.Location{ + LocationType: model.LocationTypeCountry, + Country: "Japan", + CountryCode: "jp", + } + model.CreateLocation(ctx, probed) + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientIp+":0", handlerId) + assert.Equal(t, err, nil) + + // probed country ("jp") deliberately differs from the control ip's + // mmdb country ("us" for 8.8.8.8). + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: clientId, + LocationId: probed.LocationId, + CountryCode: "jp", + ObservedAt: server.NowUtc(), + }) + + err = SetConnectionLocation(ctx, connectionId, clientIp) + assert.Equal(t, err, nil) + + var netTypeForeign int + server.Db(ctx, func(conn server.PgConn) { + result, qerr := conn.Query( + ctx, + `SELECT net_type_foreign FROM network_client_location WHERE connection_id = $1`, + connectionId, + ) + server.WithPgResult(result, qerr, func() { + if result.Next() { + server.Raise(result.Scan(&netTypeForeign)) + } + }) + }) + assert.Equal(t, netTypeForeign, 0) + }) +} + // A fresh probed location's Hosting/Proxy flags must map onto the stored // connection's net_type_hosting/net_type_privacy scores. func TestSetConnectionLocationMapsProbedFlagsToScores(t *testing.T) { diff --git a/controller/provider_egress_location_controller.go b/controller/provider_egress_location_controller.go index 57a11d78..169de94a 100644 --- a/controller/provider_egress_location_controller.go +++ b/controller/provider_egress_location_controller.go @@ -14,6 +14,17 @@ import ( // already older than this when it arrives. It bounds replay of an old probe. const MaxProviderEgressLocationSubmissionAge = 24 * time.Hour +// maxLocationNameLen bounds country/city/region as submitted: these flow into +// model.CreateLocation, whose location_name column is varchar(128). Rejecting +// an over-long value here with a clear error is preferable to letting +// CreateLocation panic on a Postgres "value too long for type character +// varying(128)" error. +const maxLocationNameLen = 128 + +// maxOrgLen mirrors maxLocationNameLen for org, which is stored in +// provider_egress_location.org, a varchar(256) column. +const maxOrgLen = 256 + type SubmitProviderEgressLocationArgs struct { ClientId server.Id `json:"client_id"` CountryCode string `json:"country_code"` @@ -59,19 +70,59 @@ func SubmitProviderEgressLocation( return nil, fmt.Errorf("Unknown client.") } + // country is always used to resolve/create a location row (at minimum + // the country-granular one), and model.CreateLocation dedupes country + // rows on (location_type, country_code): an empty name here would create + // a canonical row with location_name='' that every later lookup for this + // country reuses forever, even after a subsequent real mmdb lookup. Reject + // rather than silently falling back, so the prober learns it sent a bad + // payload instead of the server permanently corrupting shared data. + country := strings.TrimSpace(args.Country) + if country == "" { + return nil, fmt.Errorf("Missing country.") + } + if maxLocationNameLen < len(country) { + return nil, fmt.Errorf("Country is too long.") + } + if maxOrgLen < len(args.Org) { + return nil, fmt.Errorf("Org is too long.") + } + + // city/region are only used (and their rows only created) when the probe + // was city-confident; the same empty-name corruption applies to them, so + // require both are present and reject rather than silently dropping to + // country granularity on a bad payload. + var city, region string + if args.CityConfident { + city = strings.TrimSpace(args.City) + region = strings.TrimSpace(args.Region) + if city == "" { + return nil, fmt.Errorf("Missing city for a city-confident submission.") + } + if region == "" { + return nil, fmt.Errorf("Missing region for a city-confident submission.") + } + if maxLocationNameLen < len(city) { + return nil, fmt.Errorf("City is too long.") + } + if maxLocationNameLen < len(region) { + return nil, fmt.Errorf("Region is too long.") + } + } + // resolve to a canonical location row. city granularity only when the // probe agreed on a city; otherwise country. location := &model.Location{ LocationType: model.LocationTypeCountry, - Country: args.Country, + Country: country, CountryCode: countryCode, } - if args.CityConfident && args.City != "" { + if args.CityConfident { location = &model.Location{ LocationType: model.LocationTypeCity, - City: args.City, - Region: args.Region, - Country: args.Country, + City: city, + Region: region, + Country: country, CountryCode: countryCode, } } diff --git a/controller/provider_egress_location_controller_test.go b/controller/provider_egress_location_controller_test.go index 9f8c70a4..25a04c6d 100644 --- a/controller/provider_egress_location_controller_test.go +++ b/controller/provider_egress_location_controller_test.go @@ -2,6 +2,7 @@ package controller import ( "context" + "strings" "testing" "time" @@ -132,6 +133,147 @@ func TestSubmitProviderEgressLocationCityConfidentStoresCity(t *testing.T) { }) } +// An empty Country must be rejected, not silently stored: model.CreateLocation +// dedupes country rows on (location_type, country_code), so an empty name +// would create a canonical, permanently-blank row that every later lookup for +// that country reuses forever. +func TestSubmitProviderEgressLocationRejectsEmptyCountry(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "us", + Country: " ", // blank after trimming + CountryConfident: true, + ObservedAt: server.NowUtc(), + }) + if err == nil { + t.Fatal("an empty country must be rejected") + } + if model.GetProviderEgressLocation(ctx, clientId) != nil { + t.Fatal("rejected submission must not be stored") + } + }) +} + +// A city-confident submission with an empty City must be rejected rather than +// silently falling back to country granularity: the same empty-canonical-row +// corruption applies to city/region rows. +func TestSubmitProviderEgressLocationRejectsEmptyCityWhenCityConfident(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "us", + Country: "United States", + Region: "Colorado", + City: "", + CountryConfident: true, + CityConfident: true, + ObservedAt: server.NowUtc(), + }) + if err == nil { + t.Fatal("a city-confident submission with an empty city must be rejected") + } + if model.GetProviderEgressLocation(ctx, clientId) != nil { + t.Fatal("rejected submission must not be stored") + } + }) +} + +// A city-confident submission with an empty Region must likewise be rejected: +// the region row is created with the same dedupe-on-empty-name hazard. +func TestSubmitProviderEgressLocationRejectsEmptyRegionWhenCityConfident(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "us", + Country: "United States", + Region: "", + City: "Denver", + CountryConfident: true, + CityConfident: true, + ObservedAt: server.NowUtc(), + }) + if err == nil { + t.Fatal("a city-confident submission with an empty region must be rejected") + } + if model.GetProviderEgressLocation(ctx, clientId) != nil { + t.Fatal("rejected submission must not be stored") + } + }) +} + +// An over-long Country must be rejected with a clear error instead of +// panicking inside model.CreateLocation on a Postgres "value too long for +// type character varying(128)" error. +func TestSubmitProviderEgressLocationRejectsOverLongCountry(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "us", + Country: strings.Repeat("a", 129), + CountryConfident: true, + ObservedAt: server.NowUtc(), + }) + if err == nil { + t.Fatal("an over-long country must be rejected") + } + if model.GetProviderEgressLocation(ctx, clientId) != nil { + t.Fatal("rejected submission must not be stored") + } + }) +} + +// An over-long Org must likewise be rejected rather than panicking inside +// storage. +func TestSubmitProviderEgressLocationRejectsOverLongOrg(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "us", + Country: "United States", + Org: strings.Repeat("a", 257), + CountryConfident: true, + ObservedAt: server.NowUtc(), + }) + if err == nil { + t.Fatal("an over-long org must be rejected") + } + if model.GetProviderEgressLocation(ctx, clientId) != nil { + t.Fatal("rejected submission must not be stored") + } + }) +} + func TestSubmitProviderEgressLocationRejectsStaleObservedAt(t *testing.T) { server.DefaultTestEnv().Run(t, func(t testing.TB) { ctx := context.Background() diff --git a/db_migrations.go b/db_migrations.go index 204afa4b..9ae8a614 100644 --- a/db_migrations.go +++ b/db_migrations.go @@ -4386,12 +4386,15 @@ var migrations = []any{ ON network_client (contract_time) WHERE (active = true AND source_client_id IS NULL AND contract_time IS NOT NULL) `), - // provider egress locations: locations learned by probing a provider's own - // egress (see docs/superpowers/specs/2026-07-24-provider-egress-geolocation-design.md). - // Keyed by client_id, one row per provider, upserted by the operator's - // prober. location_id is the canonical country (or city, when the probe was - // city-confident) location row. observed_at is when the probe ran, and is - // what freshness is judged against. + // provider egress locations: locations learned by an operator-run prober + // that routes geolocation lookups through a provider's own egress rather + // than trusting a lookup on the provider's control-connection ip, since + // the egress is where user traffic actually exits and can differ from + // where the provider's control connection originates (e.g. behind a VPN + // or hosting network). Keyed by client_id, one row per provider, upserted + // by the operator's prober. location_id is the canonical country (or + // city, when the probe was city-confident) location row. observed_at is + // when the probe ran, and is what freshness is judged against. newSqlMigration(` CREATE TABLE IF NOT EXISTS provider_egress_location ( client_id uuid NOT NULL PRIMARY KEY, diff --git a/model/provider_egress_location_model.go b/model/provider_egress_location_model.go index f7d0c69f..240bb8b4 100644 --- a/model/provider_egress_location_model.go +++ b/model/provider_egress_location_model.go @@ -29,7 +29,10 @@ type ProviderEgressLocation struct { UpdateTime time.Time } -// SetProviderEgressLocation upserts the probed location for a provider. +// SetProviderEgressLocation upserts the probed location for a provider. The +// upsert is monotonic in observed_at: a replayed or out-of-order submission +// older than what is already stored is silently dropped rather than +// clobbering a newer probe result. func SetProviderEgressLocation(ctx context.Context, e *ProviderEgressLocation) { // country codes are stored/compared lowercased (see CreateLocation in // network_client_location_model.go); the geolocation APIs that feed this @@ -66,6 +69,7 @@ func SetProviderEgressLocation(ctx context.Context, e *ProviderEgressLocation) { city_confident = $9, observed_at = $10, update_time = $11 + WHERE provider_egress_location.observed_at < EXCLUDED.observed_at `, e.ClientId, e.LocationId, @@ -128,24 +132,6 @@ func GetProviderEgressLocation(ctx context.Context, clientId server.Id) *Provide return e } -// GetNetworkClientForConnection returns the client id for a connection, or nil. -func GetNetworkClientForConnection(ctx context.Context, connectionId server.Id) *server.Id { - var clientId *server.Id - server.Db(ctx, func(conn server.PgConn) { - result, err := conn.Query( - ctx, - `SELECT client_id FROM network_client_connection WHERE connection_id = $1`, - connectionId, - ) - server.WithPgResult(result, err, func() { - if result.Next() { - server.Raise(result.Scan(&clientId)) - } - }) - }) - return clientId -} - // GetFreshProviderEgressLocation is GetProviderEgressLocation, filtered to // entries probed within maxAge. The cutoff is computed in Go and bound as a // parameter: observed_at is a naive timestamp holding utc, and comparing it @@ -169,10 +155,10 @@ func GetFreshProviderEgressLocation( // egress location for a connection in a single query, joining // network_client_connection to provider_egress_location on client_id. This // exists for the connect-announce hot path (SetConnectionLocation), which -// previously spent two round trips per connection -- GetNetworkClientForConnection -// then GetFreshProviderEgressLocation -- before ever reaching the mmdb -// fallback; collapsing to one query matters on a path that runs for every -// connection and inside a retry loop. +// previously spent two round trips per connection -- resolving the client id +// for the connection, then fetching its fresh egress location -- before ever +// reaching the mmdb fallback; collapsing to one query matters on a path that +// runs for every connection and inside a retry loop. // // As with GetFreshProviderEgressLocation, the maxAge cutoff is computed in Go // and compared in Go: observed_at is a naive timestamp holding utc, and diff --git a/model/provider_egress_location_model_test.go b/model/provider_egress_location_model_test.go index bf229eb5..874f5391 100644 --- a/model/provider_egress_location_model_test.go +++ b/model/provider_egress_location_model_test.go @@ -43,7 +43,9 @@ func TestProviderEgressLocationUpsertAndGet(t *testing.T) { assert.Equal(t, got.Hosting, true) assert.Equal(t, got.Proxy, false) - // upsert replaces + // upsert replaces, given a strictly newer observed_at: the upsert is + // monotonic (see TestProviderEgressLocationUpsertIgnoresOlderReplay below), + // so a second submission at the same observed_at would not win. SetProviderEgressLocation(ctx, &ProviderEgressLocation{ ClientId: clientId, LocationId: country.LocationId, @@ -51,7 +53,7 @@ func TestProviderEgressLocationUpsertAndGet(t *testing.T) { ASN: 999, Hosting: false, Proxy: true, - ObservedAt: now, + ObservedAt: now.Add(time.Minute), }) got = GetProviderEgressLocation(ctx, clientId) assert.Equal(t, got.ASN, 999) @@ -60,6 +62,58 @@ func TestProviderEgressLocationUpsertAndGet(t *testing.T) { }) } +// The upsert is monotonic in observed_at: a replayed submission older than +// what is already stored must not clobber the newer row. +func TestProviderEgressLocationUpsertIgnoresOlderReplay(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + usCountry := &Location{ + LocationType: LocationTypeCountry, + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, usCountry) + + jpCountry := &Location{ + LocationType: LocationTypeCountry, + Country: "Japan", + CountryCode: "jp", + } + CreateLocation(ctx, jpCountry) + + clientId := server.NewId() + newer := server.NowUtc() + older := newer.Add(-1 * time.Hour) + + // the newer probe lands first + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: clientId, + LocationId: jpCountry.LocationId, + CountryCode: "jp", + ASN: 111, + ObservedAt: newer, + }) + + // a stale/replayed older probe arrives afterward and must not win + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: clientId, + LocationId: usCountry.LocationId, + CountryCode: "us", + ASN: 222, + ObservedAt: older, + }) + + got := GetProviderEgressLocation(ctx, clientId) + if got == nil { + t.Fatal("expected a stored egress location") + } + assert.Equal(t, got.CountryCode, "jp") + assert.Equal(t, got.ASN, 111) + assert.Equal(t, got.LocationId, jpCountry.LocationId) + }) +} + func TestProviderEgressLocationCountryCodeLowercased(t *testing.T) { server.DefaultTestEnv().Run(t, func(t testing.TB) { ctx := context.Background() From 1c7bebebf1fa670b6c4d211f1c2918cbe2cd9eef Mon Sep 17 00:00:00 2001 From: ryanmello07 Date: Sat, 25 Jul 2026 07:10:05 +0100 Subject: [PATCH 12/30] chore: drop beta-only vault example from the upstream change The example config lives in this fork's beta deployment tree, which does not exist upstream. Operators configure the ingest secret through their own vault; the handler documents the resource name and key. --- beta-vault/vault/provider_egress.yml.example | 9 --------- 1 file changed, 9 deletions(-) delete mode 100644 beta-vault/vault/provider_egress.yml.example diff --git a/beta-vault/vault/provider_egress.yml.example b/beta-vault/vault/provider_egress.yml.example deleted file mode 100644 index 2024763f..00000000 --- a/beta-vault/vault/provider_egress.yml.example +++ /dev/null @@ -1,9 +0,0 @@ -# Operator ingest secret for POST /network/provider-egress-location. -# The operator's egress prober sends this value in the X-UR-Operator-Secret -# header. Generate with: openssl rand -base64 32 -# If this file or key is absent, the ingest endpoint rejects every request -# (fails closed, does not crash the api process). -# The secret is read once, on the first request, and memoized for the life of -# the process: after creating or rotating this file, restart the api for the -# new value to take effect. -ingest_secret: "replace-me" From 39f29346c42bd2619c1881841f7ed9784f7cd2b8 Mon Sep 17 00:00:00 2001 From: ryanmello07 Date: Sat, 25 Jul 2026 07:25:36 +0100 Subject: [PATCH 13/30] test: port provider-egress tests off go-playground/assert to connect.AssertEqual The three provider-egress-location test files imported github.com/go-playground/assert/v2, which is present in this fork's go.mod but not in upstream's, breaking compilation on this upstream-based branch (no required module provides package github.com/go-playground/assert/v2). Upstream already has an equivalent helper, connect.AssertEqual (from github.com/urnetwork/connect, used throughout model/ and controller/ test files), so no new third-party dependency is needed. This is a mechanical 1:1 swap (assert.Equal(t, v1, v2) -> connect.AssertEqual(t, v1, v2), same argument order and semantics) with no change to test logic or assertions. None of the three files used assert.NotEqual. No go.mod/go.sum changes: go-playground/assert was never present in this branch's go.mod/go.sum to begin with. --- controller/network_client_controller_test.go | 45 +++++++++---------- ...rovider_egress_location_controller_test.go | 21 +++++---- model/provider_egress_location_model_test.go | 27 ++++++----- 3 files changed, 45 insertions(+), 48 deletions(-) diff --git a/controller/network_client_controller_test.go b/controller/network_client_controller_test.go index ea6c593c..17feb8c9 100644 --- a/controller/network_client_controller_test.go +++ b/controller/network_client_controller_test.go @@ -5,8 +5,7 @@ import ( "testing" "time" - "github.com/go-playground/assert/v2" - + "github.com/urnetwork/connect" "github.com/urnetwork/server" "github.com/urnetwork/server/model" ) @@ -31,7 +30,7 @@ func TestSetConnectionLocationPrefersEgressLocation(t *testing.T) { handlerId := model.CreateNetworkClientHandler(ctx) connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, "8.8.8.8:0", handlerId) - assert.Equal(t, err, nil) + connect.AssertEqual(t, err, nil) model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ ClientId: clientId, @@ -41,7 +40,7 @@ func TestSetConnectionLocationPrefersEgressLocation(t *testing.T) { }) err = SetConnectionLocation(ctx, connectionId, "8.8.8.8") - assert.Equal(t, err, nil) + connect.AssertEqual(t, err, nil) var countryLocationId server.Id server.Db(ctx, func(conn server.PgConn) { @@ -56,7 +55,7 @@ func TestSetConnectionLocationPrefersEgressLocation(t *testing.T) { } }) }) - assert.Equal(t, countryLocationId, probed.CountryLocationId) + connect.AssertEqual(t, countryLocationId, probed.CountryLocationId) }) } @@ -71,11 +70,11 @@ func TestSetConnectionLocationFallsBackToMmdb(t *testing.T) { handlerId := model.CreateNetworkClientHandler(ctx) connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, "8.8.8.8:0", handlerId) - assert.Equal(t, err, nil) + connect.AssertEqual(t, err, nil) // no SetProviderEgressLocation call -> mmdb path err = SetConnectionLocation(ctx, connectionId, "8.8.8.8") - assert.Equal(t, err, nil) + connect.AssertEqual(t, err, nil) var count int server.Db(ctx, func(conn server.PgConn) { @@ -90,7 +89,7 @@ func TestSetConnectionLocationFallsBackToMmdb(t *testing.T) { } }) }) - assert.Equal(t, count, 1) + connect.AssertEqual(t, count, 1) }) } @@ -106,7 +105,7 @@ func TestSetConnectionLocationStaleProbedFallsBackToMmdb(t *testing.T) { // the mmdb location this ip actually resolves to; created up front so // its canonical (deduped) location id is known for the assertion. mmdbLocation, _, err := GetLocationForIp(ctx, clientIp) - assert.Equal(t, err, nil) + connect.AssertEqual(t, err, nil) model.CreateLocation(ctx, mmdbLocation) // a stale probed location, deliberately a different country than @@ -125,7 +124,7 @@ func TestSetConnectionLocationStaleProbedFallsBackToMmdb(t *testing.T) { handlerId := model.CreateNetworkClientHandler(ctx) connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientIp+":0", handlerId) - assert.Equal(t, err, nil) + connect.AssertEqual(t, err, nil) model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ ClientId: clientId, @@ -135,7 +134,7 @@ func TestSetConnectionLocationStaleProbedFallsBackToMmdb(t *testing.T) { }) err = SetConnectionLocation(ctx, connectionId, clientIp) - assert.Equal(t, err, nil) + connect.AssertEqual(t, err, nil) var countryLocationId server.Id server.Db(ctx, func(conn server.PgConn) { @@ -150,7 +149,7 @@ func TestSetConnectionLocationStaleProbedFallsBackToMmdb(t *testing.T) { } }) }) - assert.Equal(t, countryLocationId, mmdbLocation.CountryLocationId) + connect.AssertEqual(t, countryLocationId, mmdbLocation.CountryLocationId) if countryLocationId == probed.CountryLocationId { t.Fatal("stale probed location must not be used") } @@ -169,7 +168,7 @@ func TestSetConnectionLocationProbedWriteErrorFallsBackToMmdb(t *testing.T) { clientIp := "8.8.8.8" mmdbLocation, _, err := GetLocationForIp(ctx, clientIp) - assert.Equal(t, err, nil) + connect.AssertEqual(t, err, nil) model.CreateLocation(ctx, mmdbLocation) networkId := server.NewId() @@ -178,7 +177,7 @@ func TestSetConnectionLocationProbedWriteErrorFallsBackToMmdb(t *testing.T) { handlerId := model.CreateNetworkClientHandler(ctx) connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientIp+":0", handlerId) - assert.Equal(t, err, nil) + connect.AssertEqual(t, err, nil) // a fresh probed entry pointing at a location id that was never // created via CreateLocation -- the storage write in @@ -191,7 +190,7 @@ func TestSetConnectionLocationProbedWriteErrorFallsBackToMmdb(t *testing.T) { }) err = SetConnectionLocation(ctx, connectionId, clientIp) - assert.Equal(t, err, nil) + connect.AssertEqual(t, err, nil) var countryLocationId server.Id server.Db(ctx, func(conn server.PgConn) { @@ -206,7 +205,7 @@ func TestSetConnectionLocationProbedWriteErrorFallsBackToMmdb(t *testing.T) { } }) }) - assert.Equal(t, countryLocationId, mmdbLocation.CountryLocationId) + connect.AssertEqual(t, countryLocationId, mmdbLocation.CountryLocationId) }) } @@ -242,7 +241,7 @@ func TestSetConnectionLocationProbedNetTypeForeignMatchesMmdbParity(t *testing.T handlerId := model.CreateNetworkClientHandler(ctx) connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientIp+":0", handlerId) - assert.Equal(t, err, nil) + connect.AssertEqual(t, err, nil) // probed country ("jp") deliberately differs from the control ip's // mmdb country ("us" for 8.8.8.8). @@ -254,7 +253,7 @@ func TestSetConnectionLocationProbedNetTypeForeignMatchesMmdbParity(t *testing.T }) err = SetConnectionLocation(ctx, connectionId, clientIp) - assert.Equal(t, err, nil) + connect.AssertEqual(t, err, nil) var netTypeForeign int server.Db(ctx, func(conn server.PgConn) { @@ -269,7 +268,7 @@ func TestSetConnectionLocationProbedNetTypeForeignMatchesMmdbParity(t *testing.T } }) }) - assert.Equal(t, netTypeForeign, 0) + connect.AssertEqual(t, netTypeForeign, 0) }) } @@ -292,7 +291,7 @@ func TestSetConnectionLocationMapsProbedFlagsToScores(t *testing.T) { handlerId := model.CreateNetworkClientHandler(ctx) connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, "8.8.8.8:0", handlerId) - assert.Equal(t, err, nil) + connect.AssertEqual(t, err, nil) model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ ClientId: clientId, @@ -304,7 +303,7 @@ func TestSetConnectionLocationMapsProbedFlagsToScores(t *testing.T) { }) err = SetConnectionLocation(ctx, connectionId, "8.8.8.8") - assert.Equal(t, err, nil) + connect.AssertEqual(t, err, nil) var netTypeHosting int var netTypePrivacy int @@ -320,7 +319,7 @@ func TestSetConnectionLocationMapsProbedFlagsToScores(t *testing.T) { } }) }) - assert.Equal(t, netTypeHosting, 1) - assert.Equal(t, netTypePrivacy, 1) + connect.AssertEqual(t, netTypeHosting, 1) + connect.AssertEqual(t, netTypePrivacy, 1) }) } diff --git a/controller/provider_egress_location_controller_test.go b/controller/provider_egress_location_controller_test.go index 25a04c6d..27b4163c 100644 --- a/controller/provider_egress_location_controller_test.go +++ b/controller/provider_egress_location_controller_test.go @@ -6,8 +6,7 @@ import ( "testing" "time" - "github.com/go-playground/assert/v2" - + "github.com/urnetwork/connect" "github.com/urnetwork/server" "github.com/urnetwork/server/model" ) @@ -30,7 +29,7 @@ func TestSubmitProviderEgressLocationCountryOnly(t *testing.T) { CountryConfident: true, ObservedAt: server.NowUtc(), }) - assert.Equal(t, err, nil) + connect.AssertEqual(t, err, nil) if res.LocationId == (server.Id{}) { t.Fatal("expected a resolved location id") } @@ -39,10 +38,10 @@ func TestSubmitProviderEgressLocationCountryOnly(t *testing.T) { if stored == nil { t.Fatal("expected the submission to be stored") } - assert.Equal(t, stored.CountryCode, "us") - assert.Equal(t, stored.ASN, 401486) - assert.Equal(t, stored.Hosting, true) - assert.Equal(t, stored.CityConfident, false) + connect.AssertEqual(t, stored.CountryCode, "us") + connect.AssertEqual(t, stored.ASN, 401486) + connect.AssertEqual(t, stored.Hosting, true) + connect.AssertEqual(t, stored.CityConfident, false) // the resolved location must be the country-granular row, with no // city/region association @@ -50,7 +49,7 @@ func TestSubmitProviderEgressLocationCountryOnly(t *testing.T) { if loc == nil { t.Fatal("expected the resolved location row to exist") } - assert.Equal(t, loc.LocationType, model.LocationTypeCountry) + connect.AssertEqual(t, loc.LocationType, model.LocationTypeCountry) if loc.CityLocationId != (server.Id{}) { t.Fatal("a country-granularity row must not have a city association") } @@ -116,20 +115,20 @@ func TestSubmitProviderEgressLocationCityConfidentStoresCity(t *testing.T) { CityConfident: true, ObservedAt: server.NowUtc(), }) - assert.Equal(t, err, nil) + connect.AssertEqual(t, err, nil) stored := model.GetProviderEgressLocation(ctx, clientId) if stored == nil { t.Fatal("expected the submission to be stored") } - assert.Equal(t, stored.CityConfident, true) + connect.AssertEqual(t, stored.CityConfident, true) // the resolved location must be the city-granular row loc := model.GetLocation(ctx, stored.LocationId) if loc == nil { t.Fatal("expected the resolved location row to exist") } - assert.Equal(t, loc.LocationType, model.LocationTypeCity) + connect.AssertEqual(t, loc.LocationType, model.LocationTypeCity) }) } diff --git a/model/provider_egress_location_model_test.go b/model/provider_egress_location_model_test.go index 874f5391..f92bb2a7 100644 --- a/model/provider_egress_location_model_test.go +++ b/model/provider_egress_location_model_test.go @@ -5,8 +5,7 @@ import ( "testing" "time" - "github.com/go-playground/assert/v2" - + "github.com/urnetwork/connect" "github.com/urnetwork/server" ) @@ -37,11 +36,11 @@ func TestProviderEgressLocationUpsertAndGet(t *testing.T) { if got == nil { t.Fatal("expected a stored egress location") } - assert.Equal(t, got.LocationId, country.LocationId) - assert.Equal(t, got.CountryCode, "us") - assert.Equal(t, got.ASN, 401486) - assert.Equal(t, got.Hosting, true) - assert.Equal(t, got.Proxy, false) + connect.AssertEqual(t, got.LocationId, country.LocationId) + connect.AssertEqual(t, got.CountryCode, "us") + connect.AssertEqual(t, got.ASN, 401486) + connect.AssertEqual(t, got.Hosting, true) + connect.AssertEqual(t, got.Proxy, false) // upsert replaces, given a strictly newer observed_at: the upsert is // monotonic (see TestProviderEgressLocationUpsertIgnoresOlderReplay below), @@ -56,9 +55,9 @@ func TestProviderEgressLocationUpsertAndGet(t *testing.T) { ObservedAt: now.Add(time.Minute), }) got = GetProviderEgressLocation(ctx, clientId) - assert.Equal(t, got.ASN, 999) - assert.Equal(t, got.Hosting, false) - assert.Equal(t, got.Proxy, true) + connect.AssertEqual(t, got.ASN, 999) + connect.AssertEqual(t, got.Hosting, false) + connect.AssertEqual(t, got.Proxy, true) }) } @@ -108,9 +107,9 @@ func TestProviderEgressLocationUpsertIgnoresOlderReplay(t *testing.T) { if got == nil { t.Fatal("expected a stored egress location") } - assert.Equal(t, got.CountryCode, "jp") - assert.Equal(t, got.ASN, 111) - assert.Equal(t, got.LocationId, jpCountry.LocationId) + connect.AssertEqual(t, got.CountryCode, "jp") + connect.AssertEqual(t, got.ASN, 111) + connect.AssertEqual(t, got.LocationId, jpCountry.LocationId) }) } @@ -142,7 +141,7 @@ func TestProviderEgressLocationCountryCodeLowercased(t *testing.T) { if got == nil { t.Fatal("expected a stored egress location") } - assert.Equal(t, got.CountryCode, "us") + connect.AssertEqual(t, got.CountryCode, "us") }) } From 6b5de2be82f548daabcd66033311d9ae5cfe43b6 Mon Sep 17 00:00:00 2001 From: ryanmello07 Date: Sat, 25 Jul 2026 08:34:27 +0100 Subject: [PATCH 14/30] fix(provider-egress): reject future-dated submissions, fix ASN column overflow, and other review findings Five review findings on the provider-egress-location PR, each empirically reproduced before being fixed: - A far-future observed_at had no upper bound: it would win the monotonic upsert forever, read as fresh forever, and outlive the taskworker sweep, permanently pinning a provider's location with no API-side recovery. Add MaxProviderEgressLocationSubmissionSkew (5m) alongside the existing MaxProviderEgressLocationSubmissionAge check, and reject observed_at further in the future than that. - provider_egress_location.asn was `int` (Postgres int4, max ~2.147e9); ASNs are 32-bit unsigned (max ~4.295e9), so a real ASN like 4200000000 panicked pgx's arg encoding after a ~78s retry-storm hang. The table is new in this same unmerged PR, so the fix edits the CREATE TABLE migration directly (asn int -> asn bigint) rather than adding a second migration. - Removed a "fix(beta)" fork marker comment from model/network_client_location_model.go, rewritten to be vendor-neutral while keeping the NULL-city/region panic explanation intact. - arinForeignScore's doc comment said its second argument was "the mmdb-resolved country ... or the probed egress country" -- a later commit made both callers pass the mmdb country for ranking parity, so the doc was updated to describe what the code actually does and why. - SetConnectionLocation mapped the probed Mobile flag onto NetTypeVirtual, but IpInfo has no Mobile concept and NetTypeVirtual is only ever set from the ipinfo schema's is_satellite field. This gave probed mobile providers a ranking penalty an identical unprobed provider never takes -- the opposite of the parity this feature promises. Mobile stays in the model/wire contract as metadata but no longer feeds net_type_virtual; Hosting/Proxy keep feeding their scores since they do have direct mmdb-path equivalents (ipInfo.Hosting/ipInfo.Privacy). Tests added: future/within-skew observed_at rejection and acceptance, large-ASN round-trip, and a Mobile-does-not-set-net_type_virtual assertion folded into the existing probed-flags-to-scores test. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01QtgqtCmKJRXdsQ5ktiqwkg --- controller/network_client_controller.go | 14 ++- controller/network_client_controller_test.go | 16 +++- .../network_client_location_controller.go | 13 ++- .../provider_egress_location_controller.go | 16 ++++ ...rovider_egress_location_controller_test.go | 95 +++++++++++++++++++ db_migrations.go | 2 +- model/network_client_location_model.go | 6 +- 7 files changed, 147 insertions(+), 15 deletions(-) diff --git a/controller/network_client_controller.go b/controller/network_client_controller.go index 382aa913..4a723b6e 100644 --- a/controller/network_client_controller.go +++ b/controller/network_client_controller.go @@ -81,9 +81,17 @@ func SetConnectionLocation( if egress.Proxy { scores.NetTypePrivacy = 1 } - if egress.Mobile { - scores.NetTypeVirtual = 1 - } + // egress.Mobile deliberately does NOT feed NetTypeVirtual: unlike + // Hosting/Proxy, Mobile has no mmdb-path equivalent (IpInfo has no + // Mobile concept; NetTypeVirtual is set from the ipinfo schema's + // is_satellite field only, see GetLocationForIp, and never from + // DB-IP). Deriving NetTypeVirtual from Mobile here would penalize a + // probed mobile provider's ranking with no equivalent penalty for an + // otherwise-identical unprobed one -- the opposite of the parity + // this feature is meant to preserve (see arinForeignScore's doc for + // the same parity reasoning applied to net_type_foreign). Mobile + // stays on the model/wire contract as metadata; it just does not + // feed the ranking score. // keep the ARIN org-vs-country foreign check on the probed path too, // so a probed provider is ranked on equal terms with an equivalent // unprobed one (net_type_foreign feeds the ranking columns). Compute diff --git a/controller/network_client_controller_test.go b/controller/network_client_controller_test.go index 17feb8c9..f6561fb1 100644 --- a/controller/network_client_controller_test.go +++ b/controller/network_client_controller_test.go @@ -273,7 +273,14 @@ func TestSetConnectionLocationProbedNetTypeForeignMatchesMmdbParity(t *testing.T } // A fresh probed location's Hosting/Proxy flags must map onto the stored -// connection's net_type_hosting/net_type_privacy scores. +// connection's net_type_hosting/net_type_privacy scores. Mobile must NOT map +// onto net_type_virtual: Hosting/Proxy have direct mmdb-path equivalents +// (ipInfo.Hosting/ipInfo.Privacy, see GetLocationForIp), but Mobile has none +// (IpInfo has no Mobile concept at all, and NetTypeVirtual is only ever set +// from the ipinfo schema's is_satellite field, never from DB-IP or from +// anything Mobile-shaped) -- deriving NetTypeVirtual from Mobile would give +// a probed mobile provider a ranking penalty an identical unprobed mobile +// provider never takes, breaking the parity this feature promises. func TestSetConnectionLocationMapsProbedFlagsToScores(t *testing.T) { server.DefaultTestEnv().Run(t, func(t testing.TB) { ctx := context.Background() @@ -299,6 +306,7 @@ func TestSetConnectionLocationMapsProbedFlagsToScores(t *testing.T) { CountryCode: "jp", Hosting: true, Proxy: true, + Mobile: true, ObservedAt: server.NowUtc(), }) @@ -307,19 +315,21 @@ func TestSetConnectionLocationMapsProbedFlagsToScores(t *testing.T) { var netTypeHosting int var netTypePrivacy int + var netTypeVirtual int server.Db(ctx, func(conn server.PgConn) { result, qerr := conn.Query( ctx, - `SELECT net_type_hosting, net_type_privacy FROM network_client_location WHERE connection_id = $1`, + `SELECT net_type_hosting, net_type_privacy, net_type_virtual FROM network_client_location WHERE connection_id = $1`, connectionId, ) server.WithPgResult(result, qerr, func() { if result.Next() { - server.Raise(result.Scan(&netTypeHosting, &netTypePrivacy)) + server.Raise(result.Scan(&netTypeHosting, &netTypePrivacy, &netTypeVirtual)) } }) }) connect.AssertEqual(t, netTypeHosting, 1) connect.AssertEqual(t, netTypePrivacy, 1) + connect.AssertEqual(t, netTypeVirtual, 0) }) } diff --git a/controller/network_client_location_controller.go b/controller/network_client_location_controller.go index 0535db77..a8796b8b 100644 --- a/controller/network_client_location_controller.go +++ b/controller/network_client_location_controller.go @@ -61,11 +61,14 @@ func GetLocationForIp(ctx context.Context, clientIp string) (*model.Location, *m } // arinForeignScore cross-checks the ARIN org registration country for addr -// against countryCode, the country code being claimed for this connection -// (the mmdb-resolved country on the ordinary path, or the probed egress -// country on the provider-egress path). If the org's registered country -// differs, the use case is considered foreign (VPN/proxy-like), matching the -// heuristic previously inlined in GetLocationForIp. +// against countryCode. Both the ordinary path (GetLocationForIp) and the +// provider-egress path (SetConnectionLocation, in network_client_controller.go) pass the +// mmdb-resolved country of addr here, not the probed egress country: this is +// deliberate parity, so a probed and an unprobed provider on the same +// control ip are scored on the same basis and probing does not, by itself, +// change this ranking signal. If the org's registered country differs, the +// use case is considered foreign (VPN/proxy-like), matching the heuristic +// previously inlined in GetLocationForIp. // // If the ARIN lookup fails, this returns 0 without error: it must never fail // or panic a caller on the connect-announce hot path over a missing/failed diff --git a/controller/provider_egress_location_controller.go b/controller/provider_egress_location_controller.go index 169de94a..2c28c494 100644 --- a/controller/provider_egress_location_controller.go +++ b/controller/provider_egress_location_controller.go @@ -14,6 +14,19 @@ import ( // already older than this when it arrives. It bounds replay of an old probe. const MaxProviderEgressLocationSubmissionAge = 24 * time.Hour +// MaxProviderEgressLocationSubmissionSkew rejects a submission whose +// observed_at is further in the future than this. The prober and server +// clocks should be roughly in sync, so a few minutes of allowance covers +// ordinary clock drift without opening the door to a far-future timestamp. +// Without this bound, a future observed_at would defeat every other +// safeguard at once: it always wins the monotonic upsert in +// model.SetProviderEgressLocation (so no later, legitimate probe can ever +// overwrite it), it reads as "fresh" forever against +// ProviderEgressLocationMaxAge, and it outlives the taskworker sweep in +// RemoveExpiredProviderEgressLocations -- permanently pinning a provider to +// whatever location was submitted, with no API-side recovery. +const MaxProviderEgressLocationSubmissionSkew = 5 * time.Minute + // maxLocationNameLen bounds country/city/region as submitted: these flow into // model.CreateLocation, whose location_name column is varchar(128). Rejecting // an over-long value here with a clear error is preferable to letting @@ -66,6 +79,9 @@ func SubmitProviderEgressLocation( if args.ObservedAt.Before(server.NowUtc().Add(-MaxProviderEgressLocationSubmissionAge)) { return nil, fmt.Errorf("Submission is too old.") } + if server.NowUtc().Add(MaxProviderEgressLocationSubmissionSkew).Before(args.ObservedAt) { + return nil, fmt.Errorf("Submission is too far in the future.") + } if networkId := model.GetNetworkClientNetwork(ctx, args.ClientId); networkId == nil { return nil, fmt.Errorf("Unknown client.") } diff --git a/controller/provider_egress_location_controller_test.go b/controller/provider_egress_location_controller_test.go index 27b4163c..d11c08c8 100644 --- a/controller/provider_egress_location_controller_test.go +++ b/controller/provider_egress_location_controller_test.go @@ -292,3 +292,98 @@ func TestSubmitProviderEgressLocationRejectsStaleObservedAt(t *testing.T) { } }) } + +// A far-future observed_at must be rejected, not just an old one: unchecked, +// it would defeat the monotonic upsert (it always "wins"), read as fresh +// forever, and outlive the taskworker sweep -- permanently pinning a +// provider's location with no API-side recovery. See +// MaxProviderEgressLocationSubmissionSkew. +func TestSubmitProviderEgressLocationRejectsFutureObservedAt(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "jp", + Country: "Japan", + CountryConfident: true, + ObservedAt: server.NowUtc().Add(10 * 365 * 24 * time.Hour), + }) + if err == nil { + t.Fatal("a far-future observed_at must be rejected") + } + if model.GetProviderEgressLocation(ctx, clientId) != nil { + t.Fatal("rejected submission must not be stored") + } + }) +} + +// A submission within the allowed clock-skew window must still be accepted: +// the future-timestamp rejection must not be so strict that ordinary clock +// drift between the prober and server breaks legitimate submissions. +func TestSubmitProviderEgressLocationAcceptsWithinSkewObservedAt(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + res, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "us", + Country: "United States", + CountryConfident: true, + ObservedAt: server.NowUtc().Add(1 * time.Minute), + }) + connect.AssertEqual(t, err, nil) + if res.LocationId == (server.Id{}) { + t.Fatal("expected a resolved location id") + } + + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected the submission to be stored") + } + }) +} + +// A large 32-bit ASN (e.g. from the private-use range, common on +// hosting/VPN infrastructure) must round-trip cleanly, not panic. The asn +// column used to be `int` (Postgres int4, max ~2.147e9); ASNs are 32-bit +// unsigned (max ~4.295e9), so a value above int4's range panicked deep in +// pgx's arg encoding. +func TestSubmitProviderEgressLocationAcceptsLargeAsn(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + const largeAsn = 4200000000 + + res, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "us", + Country: "United States", + ASN: largeAsn, + CountryConfident: true, + ObservedAt: server.NowUtc(), + }) + connect.AssertEqual(t, err, nil) + if res.LocationId == (server.Id{}) { + t.Fatal("expected a resolved location id") + } + + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected the submission to be stored") + } + connect.AssertEqual(t, stored.ASN, largeAsn) + }) +} diff --git a/db_migrations.go b/db_migrations.go index 9ae8a614..dfc27b62 100644 --- a/db_migrations.go +++ b/db_migrations.go @@ -4400,7 +4400,7 @@ var migrations = []any{ client_id uuid NOT NULL PRIMARY KEY, location_id uuid NOT NULL, country_code varchar(2) NOT NULL, - asn int NOT NULL DEFAULT 0, + asn bigint NOT NULL DEFAULT 0, org varchar(256) NOT NULL DEFAULT '', hosting bool NOT NULL DEFAULT false, proxy bool NOT NULL DEFAULT false, diff --git a/model/network_client_location_model.go b/model/network_client_location_model.go index cbab0e61..811cc80e 100644 --- a/model/network_client_location_model.go +++ b/model/network_client_location_model.go @@ -1242,9 +1242,9 @@ func SetConnectionLocation( } }) - // fix(beta): the free-tier ipinfo geo db resolves many IPs -- - // datacenter, mobile, and VPN egress especially -- to country or - // region granularity only, with no city. network_client_location + // geo databases vary in how deep their coverage goes: many IPs -- + // datacenter, mobile, and VPN egress especially -- only resolve to + // country or region granularity, with no city. network_client_location // requires city_location_id and region_location_id NOT NULL, so a // country-only location's NULL city/region made this INSERT panic // inside server.Tx. That panic propagated out of the connection From d176fc99650f619e19e1795bf41e1454bcc7866e Mon Sep 17 00:00:00 2001 From: Ryan Mello Date: Sun, 26 Jul 2026 09:09:23 +0100 Subject: [PATCH 15/30] feat(api): serve the provider egress probe schedule from the server The operator's prober decided what to probe from an in-memory ttl cache, so a restart re-probed the whole provider population and nothing durable recorded what was actually due. The freshness data already exists server-side in provider_egress_location.observed_at; this exposes it so the server owns the schedule. model.GetProviderEgressLocationDue sources candidates from the live provider population (network_client_location_reliability, connected + valid) and LEFT JOINs the egress row, rather than selecting from provider_egress_location. The dominant case is a provider that has never been probed and so has no egress row at all; selecting from that table would return exactly the providers that least need probing and none of the ones that most do. Only providers holding a Public provide key are returned. Probing tunnels through the provider itself, i.e. opens a contract from outside the provider's own network, which a provider without a Public key refuses -- offering one to the prober would burn a probe slot on a guaranteed failure. Same EXISTS filter UpdateClientLocations and UpdateClientScores already apply. The staleness cutoff is computed in Go and bound as a query argument. observed_at is a naive `timestamp` holding utc, so comparing it against sql now() would cast through the session timezone and silently skip a window. GET /network/provider-egress-due is operator-to-server, authenticated by the same X-UR-Operator-Secret shared secret as the ingest endpoint (hmac.Equal against the memoized, fail-closed operatorIngestSecret) rather than a network jwt. Its cutoff is half ProviderEgressLocationMaxAge: at the full max age every location would lapse to the mmdb fallback at the exact moment it became due and stay lapsed until the prober reached it. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01QtgqtCmKJRXdsQ5ktiqwkg --- api/api.go | 1 + .../provider_egress_location_handlers.go | 75 +++++++ .../provider_egress_location_handlers_test.go | 192 ++++++++++++++++++ model/provider_egress_location_model.go | 79 +++++++ model/provider_egress_location_model_test.go | 111 ++++++++++ 5 files changed, 458 insertions(+) diff --git a/api/api.go b/api/api.go index c3dc801d..e9a99f18 100644 --- a/api/api.go +++ b/api/api.go @@ -52,6 +52,7 @@ func Routes() []*router.Route { router.NewRoute("POST", "/network/auth-client", handlers.AuthNetworkClient), router.NewRoute("POST", "/network/remove-client", handlers.RemoveNetworkClient), router.NewRoute("POST", "/network/provider-egress-location", handlers.ProviderEgressLocationSubmit), + router.NewRoute("GET", "/network/provider-egress-due", handlers.ProviderEgressLocationDue), router.NewRoute("GET", "/network/clients", handlers.NetworkClients), router.NewRoute("GET", "/network/peers", handlers.NetworkPeers), router.NewRoute("GET", "/network/provider-locations", handlers.NetworkGetProviderLocations), diff --git a/api/handlers/provider_egress_location_handlers.go b/api/handlers/provider_egress_location_handlers.go index b843c05b..7f6feac8 100644 --- a/api/handlers/provider_egress_location_handlers.go +++ b/api/handlers/provider_egress_location_handlers.go @@ -5,12 +5,14 @@ import ( "encoding/json" "io" "net/http" + "strconv" "sync" "github.com/urnetwork/glog" "github.com/urnetwork/server" "github.com/urnetwork/server/controller" + "github.com/urnetwork/server/model" ) // operatorSecretHeader carries the operator ingest secret. This endpoint is @@ -91,3 +93,76 @@ func ProviderEgressLocationSubmit(w http.ResponseWriter, r *http.Request) { glog.Infof("[pegl]could not write response. err = %s\n", err) } } + +const ( + // defaultProviderEgressDueLimit is the batch size when the caller does not + // ask for one. + defaultProviderEgressDueLimit = 100 + // maxProviderEgressDueLimit bounds the batch size regardless of what the + // caller asks for, so one request cannot ask the database for the entire + // provider population. + maxProviderEgressDueLimit = 500 +) + +// providerEgressDueAge is how stale a stored probe must be before its provider +// is offered up for re-probing. It is deliberately shorter than +// model.ProviderEgressLocationMaxAge -- the age past which a stored location +// stops being trusted at all. If the two were equal, every location would lapse +// to the mmdb fallback at the exact moment it became due and stay lapsed until +// the prober worked its way around to it; at half the max age the prober has a +// full max-age/2 window to refresh a location before it expires. +const providerEgressDueAge = model.ProviderEgressLocationMaxAge / 2 + +// ProviderEgressLocationDueResult is the response body of +// ProviderEgressLocationDue. +type ProviderEgressLocationDueResult struct { + ClientIds []server.Id `json:"client_ids"` +} + +// ProviderEgressLocationDue tells the operator's prober which providers to +// probe next: those whose egress location has gone stale, and those that have +// never been probed at all, oldest first. +// +// This moves the probe schedule from the prober's memory into the database. +// The prober used to decide what to probe from an in-memory ttl cache, so a +// restart re-probed the whole population and nothing durable recorded what was +// actually due; observed_at already carries that information server-side, and +// this exposes it. +// +// Same auth as ProviderEgressLocationSubmit above: operator-to-server, the +// shared secret header rather than a network jwt, fail-closed when the vault +// resource is missing. +func ProviderEgressLocationDue(w http.ResponseWriter, r *http.Request) { + secret := operatorIngestSecret() + provided := r.Header.Get(operatorSecretHeader) + if secret == "" || provided == "" || !hmac.Equal([]byte(secret), []byte(provided)) { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + limit := defaultProviderEgressDueLimit + if raw := r.URL.Query().Get("limit"); raw != "" { + parsed, err := strconv.Atoi(raw) + if err != nil || parsed < 1 { + // not clamped up to 1: `limit=0` would come back as an empty list, + // which the prober cannot distinguish from "nothing is due" + http.Error(w, "Bad request", http.StatusBadRequest) + return + } + limit = min(parsed, maxProviderEgressDueLimit) + } + + // the cutoff is computed here and passed as an argument; observed_at is a + // naive timestamp holding utc, so comparing it to sql now() in the query + // would cast through the session timezone + minObservedAt := server.NowUtc().Add(-providerEgressDueAge) + + result := &ProviderEgressLocationDueResult{ + ClientIds: model.GetProviderEgressLocationDue(r.Context(), minObservedAt, limit), + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(result); err != nil { + glog.Infof("[pegl]could not write response. err = %s\n", err) + } +} diff --git a/api/handlers/provider_egress_location_handlers_test.go b/api/handlers/provider_egress_location_handlers_test.go index 4af00971..ab298a94 100644 --- a/api/handlers/provider_egress_location_handlers_test.go +++ b/api/handlers/provider_egress_location_handlers_test.go @@ -2,14 +2,18 @@ package handlers import ( "bytes" + "context" "encoding/json" "net/http" "net/http/httptest" + "slices" "strings" "testing" + "time" "github.com/urnetwork/server" "github.com/urnetwork/server/controller" + "github.com/urnetwork/server/model" ) func TestProviderEgressLocationSubmitRejectsMissingSecret(t *testing.T) { @@ -148,3 +152,191 @@ func TestProviderEgressLocationSubmitReadsSecretFromVault(t *testing.T) { t.Fatalf("readOperatorIngestSecret() = %q, want %q", got, secret) } } + +func TestProviderEgressLocationDueRejectsMissingSecret(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/network/provider-egress-due", nil) + w := httptest.NewRecorder() + + ProviderEgressLocationDue(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret header is absent", w.Code) + } +} + +func TestProviderEgressLocationDueRejectsWrongSecret(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/network/provider-egress-due", nil) + req.Header.Set(operatorSecretHeader, "definitely-not-the-secret") + w := httptest.NewRecorder() + + ProviderEgressLocationDue(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 on a wrong operator secret", w.Code) + } +} + +// TestProviderEgressLocationDueRejectsAlteredSecret is the reject case with +// the vault *configured*, so the request gets past the secret == "" fail-closed +// short-circuit and hmac.Equal is what does the rejecting. +func TestProviderEgressLocationDueRejectsAlteredSecret(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + const wrongSecret = "correct-operator-secret-0123456780" // last char changed + defer withStubOperatorIngestSecret(secret)() + + req := httptest.NewRequest(http.MethodGet, "/network/provider-egress-due", nil) + req.Header.Set(operatorSecretHeader, wrongSecret) + w := httptest.NewRecorder() + + ProviderEgressLocationDue(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret is configured but the request's secret is wrong", w.Code) + } +} + +// testing_connectDueProvider stands up a connected + valid provider holding a +// Public provide key and no probe result, i.e. a provider the due query must +// return. The caller runs model.UpdateClientLocationReliabilities afterward. +func testing_connectDueProvider( + t testing.TB, + ctx context.Context, + clientId server.Id, + locationId server.Id, + clientAddress string, +) { + model.Testing_CreateDevice(ctx, server.NewId(), server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientAddress, handlerId) + if err != nil { + t.Fatalf("connect client: %s", err) + } + if err := model.SetConnectionLocation(ctx, connectionId, locationId, &model.ConnectionLocationScores{}); err != nil { + t.Fatalf("set connection location: %s", err) + } + model.SetProvide(ctx, clientId, map[model.ProvideMode][]byte{ + model.ProvideModePublic: []byte("provide-secret"), + }) +} + +// TestProviderEgressLocationDueAcceptsCorrectSecret proves the auth gate can +// ACCEPT. This is the test that gives the three reject tests above their +// meaning: without it, a handler whose entire body was replaced with an +// unconditional `http.Error(w, "Unauthorized", 401)` would still pass all +// three, because a suite that only ever asserts rejections cannot tell a +// working auth check from a broken-shut one. +// +// It deliberately asserts more than "not 401": a real, never-probed provider +// is stood up in the test database and must come back in the response body, so +// the test also fails if the handler clears auth but never reaches the model +// query or writes the wrong json shape. +func TestProviderEgressLocationDueAcceptsCorrectSecret(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + + city := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + model.CreateLocation(ctx, city) + + due := server.NewId() + testing_connectDueProvider(t, ctx, due, city.LocationId, "0.0.0.1:0") + model.UpdateClientLocationReliabilities(ctx, server.NowUtc().Add(-time.Hour), server.NowUtc()) + + req := httptest.NewRequest(http.MethodGet, "/network/provider-egress-due?limit=10", nil) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderEgressLocationDue(w, req) + + if w.Code == http.StatusUnauthorized { + t.Fatalf("status = %d, want the correct secret to clear auth (not 401)", w.Code) + } + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body = %s", w.Code, w.Body.String()) + } + + var result ProviderEgressLocationDueResult + if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { + t.Fatalf("decode body %q: %s", w.Body.String(), err) + } + if !slices.Contains(result.ClientIds, due) { + t.Fatalf("client_ids = %v, want it to contain the never-probed provider %s", result.ClientIds, due) + } + // the wire name the prober reads + if !strings.Contains(w.Body.String(), `"client_ids"`) { + t.Fatalf("body = %s, want a client_ids field", w.Body.String()) + } + }) +} + +// The prober asks for a batch; the server must not hand back more than asked. +func TestProviderEgressLocationDueHonoursLimit(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + + city := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + model.CreateLocation(ctx, city) + + testing_connectDueProvider(t, ctx, server.NewId(), city.LocationId, "0.0.0.1:0") + testing_connectDueProvider(t, ctx, server.NewId(), city.LocationId, "0.0.0.2:0") + model.UpdateClientLocationReliabilities(ctx, server.NowUtc().Add(-time.Hour), server.NowUtc()) + + req := httptest.NewRequest(http.MethodGet, "/network/provider-egress-due?limit=1", nil) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderEgressLocationDue(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body = %s", w.Code, w.Body.String()) + } + var result ProviderEgressLocationDueResult + if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { + t.Fatalf("decode body %q: %s", w.Body.String(), err) + } + if len(result.ClientIds) != 1 { + t.Fatalf("len(client_ids) = %d, want 1 for limit=1; body = %s", len(result.ClientIds), w.Body.String()) + } + }) +} + +// A limit that is not a positive integer is a caller bug. Silently clamping it +// to 1 (or to the default) would answer a question the prober did not ask -- +// `limit=0` would come back as an empty list, indistinguishable from "nothing +// is due" -- so it is rejected instead. +func TestProviderEgressLocationDueRejectsBadLimit(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + for _, raw := range []string{"0", "-1", "abc", "1.5"} { + req := httptest.NewRequest(http.MethodGet, "/network/provider-egress-due?limit="+raw, nil) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderEgressLocationDue(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("limit=%q: status = %d, want 400", raw, w.Code) + } + } +} diff --git a/model/provider_egress_location_model.go b/model/provider_egress_location_model.go index 240bb8b4..7bf9ae00 100644 --- a/model/provider_egress_location_model.go +++ b/model/provider_egress_location_model.go @@ -274,6 +274,85 @@ func GetLocation(ctx context.Context, locationId server.Id) *Location { return loc } +// GetProviderEgressLocationDue returns the client ids of providers whose +// egress location is due for a probe: their newest probe is older than +// minObservedAt, or they have never been probed at all. Oldest first, so the +// longest-unprobed are handed out first, capped at limit. +// +// This is the durable replacement for the prober's in-memory ttl cache: the +// schedule lives in the database, so a prober restart resumes where it left +// off instead of re-probing everything. +// +// Two things about the shape of this query matter. +// +// First, candidates are sourced from the live provider population +// (network_client_location_reliability, connected + valid) and the egress row +// is LEFT JOINed on. The dominant case by far is a provider that has *never* +// been probed and therefore has no provider_egress_location row at all; +// selecting from provider_egress_location would return exactly the providers +// that least need probing and none of the ones that most do. +// +// Second, only providers holding a Public provide key are returned. Probing +// tunnels through the provider itself, which means opening a contract from +// outside the provider's own network -- something a provider without a Public +// key refuses. Offering one to the prober would burn a probe slot on a +// guaranteed failure. This is the same filter UpdateClientLocations and +// UpdateClientScores apply (network_client_location_model.go). +// +// The cutoff is computed by the caller in Go and bound as a parameter: +// observed_at is a naive `timestamp` holding utc, and comparing it against sql +// now() would cast through the session timezone and silently skip a window. +func GetProviderEgressLocationDue( + ctx context.Context, + minObservedAt time.Time, + limit int, +) []server.Id { + clientIds := []server.Id{} + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT + network_client_location_reliability.client_id + FROM network_client_location_reliability + + LEFT JOIN provider_egress_location ON + provider_egress_location.client_id = network_client_location_reliability.client_id + + WHERE + network_client_location_reliability.connected = true AND + network_client_location_reliability.valid = true AND + EXISTS ( + SELECT 1 FROM provide_key + WHERE + provide_key.client_id = network_client_location_reliability.client_id AND + provide_key.provide_mode = $1 + ) AND + ( + provider_egress_location.observed_at IS NULL OR + provider_egress_location.observed_at < $2 + ) + + -- never-probed sorts ahead of merely stale: a missing observed_at + -- is infinitely old + ORDER BY provider_egress_location.observed_at ASC NULLS FIRST + LIMIT $3 + `, + ProvideModePublic, + minObservedAt.UTC(), + limit, + ) + server.WithPgResult(result, err, func() { + for result.Next() { + var clientId server.Id + server.Raise(result.Scan(&clientId)) + clientIds = append(clientIds, clientId) + } + }) + }) + return clientIds +} + // RemoveExpiredProviderEgressLocations drops entries probed before // minObservedAt. func RemoveExpiredProviderEgressLocations(ctx context.Context, minObservedAt time.Time) { diff --git a/model/provider_egress_location_model_test.go b/model/provider_egress_location_model_test.go index f92bb2a7..c6f5f8cb 100644 --- a/model/provider_egress_location_model_test.go +++ b/model/provider_egress_location_model_test.go @@ -2,6 +2,7 @@ package model import ( "context" + "slices" "testing" "time" @@ -212,3 +213,113 @@ func TestRemoveExpiredProviderEgressLocations(t *testing.T) { } }) } + +// testing_connectProbeableProvider stands up the minimum a client needs to +// look like a live provider to the due-selection query: a device, a live +// connection with a resolved location, and a provide key of the given mode. +// The caller must run UpdateClientLocationReliabilities afterward -- that is +// what rolls the live connection tables up into the +// network_client_location_reliability row (connected + valid) the query reads. +func testing_connectProbeableProvider( + t testing.TB, + ctx context.Context, + clientId server.Id, + locationId server.Id, + clientAddress string, + provideMode ProvideMode, +) { + Testing_CreateDevice(ctx, server.NewId(), server.NewId(), clientId, "", "") + + handlerId := CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := ConnectNetworkClient(ctx, clientId, clientAddress, handlerId) + if err != nil { + t.Fatalf("connect client: %s", err) + } + + if err := SetConnectionLocation(ctx, connectionId, locationId, &ConnectionLocationScores{}); err != nil { + t.Fatalf("set connection location: %s", err) + } + + SetProvide(ctx, clientId, map[ProvideMode][]byte{ + provideMode: []byte("provide-secret"), + }) +} + +// The prober asks the server what to probe next. The answer must be sourced +// from the live provider population and not from provider_egress_location, +// because the dominant case -- a provider that has never been probed at all -- +// has no row there. +func TestGetProviderEgressLocationDue(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + + city := &Location{ + LocationType: LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, city) + + fresh := server.NewId() + stale := server.NewId() + never := server.NewId() + // a provider that cannot serve a stranger is unprobeable: the tunnel + // contract would be refused, so it must never be offered to the prober + nonPublic := server.NewId() + + testing_connectProbeableProvider(t, ctx, fresh, city.LocationId, "0.0.0.1:0", ProvideModePublic) + testing_connectProbeableProvider(t, ctx, stale, city.LocationId, "0.0.0.2:0", ProvideModePublic) + testing_connectProbeableProvider(t, ctx, never, city.LocationId, "0.0.0.3:0", ProvideModePublic) + testing_connectProbeableProvider(t, ctx, nonPublic, city.LocationId, "0.0.0.4:0", ProvideModeNetwork) + + UpdateClientLocationReliabilities(ctx, now.Add(-time.Hour), now) + + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: fresh, LocationId: city.LocationId, + CountryCode: "us", ObservedAt: now.Add(-1 * time.Hour), + }) + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: stale, LocationId: city.LocationId, + CountryCode: "us", ObservedAt: now.Add(-72 * time.Hour), + }) + // `never` and `nonPublic` deliberately get no row at all + + due := GetProviderEgressLocationDue(ctx, now.Add(-24*time.Hour), 100) + + // a provider probed an hour ago must not be re-probed; one probed three + // days ago must be; one never probed must be + if slices.Contains(due, fresh) { + t.Fatalf("due = %v, must not contain the provider probed an hour ago (%s)", due, fresh) + } + if !slices.Contains(due, stale) { + t.Fatalf("due = %v, must contain the provider probed three days ago (%s)", due, stale) + } + if !slices.Contains(due, never) { + t.Fatalf("due = %v, must contain the never-probed provider (%s)", due, never) + } + // unprobeable regardless of freshness + if slices.Contains(due, nonPublic) { + t.Fatalf("due = %v, must not contain the provider without a Public provide key (%s)", due, nonPublic) + } + + // oldest first, so the longest-unprobed are probed first: the + // never-probed provider sorts ahead of the three-days-stale one + neverIndex := slices.Index(due, never) + staleIndex := slices.Index(due, stale) + if staleIndex < neverIndex { + t.Fatalf("never-probed provider at %d must sort before the stale one at %d", neverIndex, staleIndex) + } + + // limit is honoured + limited := GetProviderEgressLocationDue(ctx, now.Add(-24*time.Hour), 1) + if len(limited) != 1 { + t.Fatalf("len(due) = %d for limit 1, want 1", len(limited)) + } + if limited[0] != never { + t.Fatalf("due[0] = %s for limit 1, want the never-probed provider %s", limited[0], never) + } + }) +} From 813d986fca643cb7925909d3a4e1f86a217e72e1 Mon Sep 17 00:00:00 2001 From: Ryan Mello Date: Sun, 26 Jul 2026 09:46:43 +0100 Subject: [PATCH 16/30] fix(model,api): defer providers that fail to probe, not only ones that succeed The only thing that moved a provider off the head of the due queue was a successful probe writing provider_egress_location. A provider that connects and holds a Public provide key but always fails to probe -- firewalled egress, dead upstream, anything other than the missing Public key the query already screens for -- never gets a row, so its observed_at stays NULL, so it sorts ahead of every stale-but-refreshable provider on every poll, forever. Five hundred such providers and a prober asking for limit=500 gets the same five hundred dead providers every time; no healthy provider's location is ever refreshed. It fails silently: the endpoint keeps returning a full, plausible-looking batch. The in-memory ttl cache this replaced was incidentally immune, because it marked a provider probed whether or not the probe worked; moving the schedule server-side dropped that protection. Record attempts, not just successes. provider_egress_probe_attempt is a small table keyed by client_id -- the attempt cannot live on provider_egress_location, because the case it exists to handle is precisely a provider with no row there. GetProviderEgressLocationDue now requires both no fresh success and no recent attempt, with a much shorter backoff for attempts (6h) than for success freshness (half ProviderEgressLocationMaxAge, 3.5d): a failing provider should be retried periodically, just not on every poll. Both cutoffs are computed in Go and bound as arguments -- attempt_at, like observed_at, is a naive `timestamp` holding utc, and comparing it to sql now() casts through the session timezone. The prober reports an attempt to POST /network/provider-egress-attempt, on the same operator-authenticated surface as the other two endpoints (X-UR-Operator-Secret, hmac.Equal, the memoized fail-closed operatorIngestSecret). The server timestamps the attempt rather than trusting the prober's clock. Attempt rows are swept by the existing expiry task. This pulls probe_attempt_at/probe_failure forward from the P2 data model in docs/superpowers/specs/2026-07-25-enforced-provider-geo-probing-design.md, because P1's schedule cannot function without them. Only those two columns, not the rest of the verdict model. Also adds a client_id secondary sort key, so batch composition under a limit is deterministic rather than plan-dependent -- the whole never-probed population otherwise ties on NULL observed_at. Tests: a provider never probed successfully but attempted seconds ago must not be due, must not take the single batch slot from a stale-but-refreshable provider, and must become due again once the backoff elapses; the same over http via the attempt endpoint. Plus the two coverage gaps review flagged: deleting either the connected or the valid predicate now fails a test, and the handler's staleness cutoff is exercised by a probed (not merely never-probed) provider. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01QtgqtCmKJRXdsQ5ktiqwkg --- api/api.go | 1 + .../provider_egress_location_handlers.go | 71 +++++- .../provider_egress_location_handlers_test.go | 210 ++++++++++++++++ .../provider_egress_location_controller.go | 52 ++++ db_migrations.go | 36 +++ model/provider_egress_location_model.go | 152 +++++++++++- model/provider_egress_location_model_test.go | 226 +++++++++++++++++- .../work/provider_egress_location_work.go | 8 +- 8 files changed, 738 insertions(+), 18 deletions(-) diff --git a/api/api.go b/api/api.go index e9a99f18..bb9d6c21 100644 --- a/api/api.go +++ b/api/api.go @@ -53,6 +53,7 @@ func Routes() []*router.Route { router.NewRoute("POST", "/network/remove-client", handlers.RemoveNetworkClient), router.NewRoute("POST", "/network/provider-egress-location", handlers.ProviderEgressLocationSubmit), router.NewRoute("GET", "/network/provider-egress-due", handlers.ProviderEgressLocationDue), + router.NewRoute("POST", "/network/provider-egress-attempt", handlers.ProviderEgressLocationAttempt), router.NewRoute("GET", "/network/clients", handlers.NetworkClients), router.NewRoute("GET", "/network/peers", handlers.NetworkPeers), router.NewRoute("GET", "/network/provider-locations", handlers.NetworkGetProviderLocations), diff --git a/api/handlers/provider_egress_location_handlers.go b/api/handlers/provider_egress_location_handlers.go index 7f6feac8..264da746 100644 --- a/api/handlers/provider_egress_location_handlers.go +++ b/api/handlers/provider_egress_location_handlers.go @@ -94,6 +94,59 @@ func ProviderEgressLocationSubmit(w http.ResponseWriter, r *http.Request) { } } +// ProviderEgressLocationAttempt records that the operator's prober tried to +// probe a provider, whether or not the try produced a location. +// +// The prober reports a *failure* here; a success is reported by +// ProviderEgressLocationSubmit above, whose provider_egress_location row +// already defers the provider for the full staleness window. Reporting a +// success here as well is harmless -- the attempt backoff is far shorter than +// that window -- but redundant. +// +// This exists because ProviderEgressLocationDue would otherwise be starved by +// providers that can never be probed successfully: they never get an egress +// row, so they sort to the head of the queue on every poll forever. See +// model.GetProviderEgressLocationDue. +// +// Same auth as the two endpoints around it: operator-to-server, the shared +// secret header rather than a network jwt, fail-closed when the vault resource +// is missing. +func ProviderEgressLocationAttempt(w http.ResponseWriter, r *http.Request) { + secret := operatorIngestSecret() + provided := r.Header.Get(operatorSecretHeader) + if secret == "" || provided == "" || !hmac.Equal([]byte(secret), []byte(provided)) { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + body, err := io.ReadAll(io.LimitReader(r.Body, maxProviderEgressLocationBody+1)) + if err != nil { + http.Error(w, "Bad request", http.StatusBadRequest) + return + } + if len(body) > maxProviderEgressLocationBody { + http.Error(w, "Request too large", http.StatusRequestEntityTooLarge) + return + } + + var args controller.RecordProviderEgressProbeAttemptArgs + if err := json.Unmarshal(body, &args); err != nil { + http.Error(w, "Bad request", http.StatusBadRequest) + return + } + + result, err := controller.RecordProviderEgressProbeAttempt(r.Context(), &args) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(result); err != nil { + glog.Infof("[pegl]could not write response. err = %s\n", err) + } +} + const ( // defaultProviderEgressDueLimit is the batch size when the caller does not // ask for one. @@ -129,6 +182,12 @@ type ProviderEgressLocationDueResult struct { // actually due; observed_at already carries that information server-side, and // this exposes it. // +// A provider is skipped if it has a fresh success *or* a recent attempt. The +// second cutoff, ProviderEgressProbeAttemptBackoff, is much shorter than the +// first: a provider that failed to probe should be retried within hours, but +// must not be handed back on every poll, which is what would starve the rest of +// the queue (see ProviderEgressLocationAttempt above). +// // Same auth as ProviderEgressLocationSubmit above: operator-to-server, the // shared secret header rather than a network jwt, fail-closed when the vault // resource is missing. @@ -152,13 +211,15 @@ func ProviderEgressLocationDue(w http.ResponseWriter, r *http.Request) { limit = min(parsed, maxProviderEgressDueLimit) } - // the cutoff is computed here and passed as an argument; observed_at is a - // naive timestamp holding utc, so comparing it to sql now() in the query - // would cast through the session timezone - minObservedAt := server.NowUtc().Add(-providerEgressDueAge) + // both cutoffs are computed here and passed as arguments; observed_at and + // attempt_at are naive timestamps holding utc, so comparing them to sql + // now() in the query would cast through the session timezone + now := server.NowUtc() + minObservedAt := now.Add(-providerEgressDueAge) + minAttemptAt := now.Add(-model.ProviderEgressProbeAttemptBackoff) result := &ProviderEgressLocationDueResult{ - ClientIds: model.GetProviderEgressLocationDue(r.Context(), minObservedAt, limit), + ClientIds: model.GetProviderEgressLocationDue(r.Context(), minObservedAt, minAttemptAt, limit), } w.Header().Set("Content-Type", "application/json") diff --git a/api/handlers/provider_egress_location_handlers_test.go b/api/handlers/provider_egress_location_handlers_test.go index ab298a94..0f094ca0 100644 --- a/api/handlers/provider_egress_location_handlers_test.go +++ b/api/handlers/provider_egress_location_handlers_test.go @@ -320,6 +320,216 @@ func TestProviderEgressLocationDueHonoursLimit(t *testing.T) { }) } +// Every other due test in this file stands up never-probed providers, which +// come back regardless of what cutoff the handler computes -- so nothing here +// actually exercised providerEgressDueAge. This one does: a provider probed +// just now must be held back, and one probed past the cutoff must come through. +// Defeating the cutoff (dropping it, computing it in the wrong direction, +// comparing against sql now() through the session timezone) fails this. +func TestProviderEgressLocationDueHonoursStalenessCutoff(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + + city := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + model.CreateLocation(ctx, city) + + fresh := server.NewId() + stale := server.NewId() + testing_connectDueProvider(t, ctx, fresh, city.LocationId, "0.0.0.1:0") + testing_connectDueProvider(t, ctx, stale, city.LocationId, "0.0.0.2:0") + model.UpdateClientLocationReliabilities(ctx, server.NowUtc().Add(-time.Hour), server.NowUtc()) + + now := server.NowUtc() + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: fresh, LocationId: city.LocationId, + CountryCode: "us", ObservedAt: now, + }) + // comfortably past providerEgressDueAge, which is half + // model.ProviderEgressLocationMaxAge + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: stale, LocationId: city.LocationId, + CountryCode: "us", ObservedAt: now.Add(-providerEgressDueAge - time.Hour), + }) + + req := httptest.NewRequest(http.MethodGet, "/network/provider-egress-due?limit=100", nil) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderEgressLocationDue(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body = %s", w.Code, w.Body.String()) + } + var result ProviderEgressLocationDueResult + if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { + t.Fatalf("decode body %q: %s", w.Body.String(), err) + } + if slices.Contains(result.ClientIds, fresh) { + t.Fatalf("client_ids = %v, must not contain the just-probed provider %s", result.ClientIds, fresh) + } + if !slices.Contains(result.ClientIds, stale) { + t.Fatalf("client_ids = %v, must contain the provider probed past the cutoff %s", result.ClientIds, stale) + } + }) +} + +func TestProviderEgressLocationAttemptRejectsMissingSecret(t *testing.T) { + body, _ := json.Marshal(map[string]any{ + "client_id": "019f8835-158d-6fd8-e9dd-fd0e4c6d6792", + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-egress-attempt", bytes.NewReader(body)) + w := httptest.NewRecorder() + + ProviderEgressLocationAttempt(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret header is absent", w.Code) + } +} + +// The reject case with the vault *configured*, so the request gets past the +// secret == "" fail-closed short-circuit and hmac.Equal is what rejects. +func TestProviderEgressLocationAttemptRejectsAlteredSecret(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + const wrongSecret = "correct-operator-secret-0123456780" // last char changed + defer withStubOperatorIngestSecret(secret)() + + body, _ := json.Marshal(map[string]any{ + "client_id": "019f8835-158d-6fd8-e9dd-fd0e4c6d6792", + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-egress-attempt", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, wrongSecret) + w := httptest.NewRecorder() + + ProviderEgressLocationAttempt(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret is configured but the request's secret is wrong", w.Code) + } +} + +// The whole point of the attempt endpoint, end to end over http: a provider +// that has never been probed successfully is due; the prober reports that it +// tried and failed; the provider stops being due. Without that, a provider +// whose probes always fail sits at the head of the queue on every poll forever +// (observed_at IS NULL sorts first) and starves every provider behind it. +func TestProviderEgressLocationAttemptDefersProvider(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + + city := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + model.CreateLocation(ctx, city) + + dead := server.NewId() + testing_connectDueProvider(t, ctx, dead, city.LocationId, "0.0.0.1:0") + model.UpdateClientLocationReliabilities(ctx, server.NowUtc().Add(-time.Hour), server.NowUtc()) + + if !slices.Contains(due(t, secret), dead) { + t.Fatalf("the never-probed provider %s must be due before any attempt is reported", dead) + } + + attemptBody, err := json.Marshal(controller.RecordProviderEgressProbeAttemptArgs{ + ClientId: dead, + ProbeFailure: "tunnel_failed", + }) + if err != nil { + t.Fatalf("marshal attempt: %s", err) + } + req := httptest.NewRequest(http.MethodPost, "/network/provider-egress-attempt", bytes.NewReader(attemptBody)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderEgressLocationAttempt(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body = %s", w.Code, w.Body.String()) + } + + attempt := model.GetProviderEgressProbeAttempt(ctx, dead) + if attempt == nil { + t.Fatal("expected the attempt to be recorded") + } + if attempt.ProbeFailure != "tunnel_failed" { + t.Fatalf("probe_failure = %q, want %q", attempt.ProbeFailure, "tunnel_failed") + } + + if slices.Contains(due(t, secret), dead) { + t.Fatalf("the provider %s must not be due again immediately after a failed attempt", dead) + } + }) +} + +// due drives the due endpoint over http and returns the batch. +func due(t testing.TB, secret string) []server.Id { + req := httptest.NewRequest(http.MethodGet, "/network/provider-egress-due?limit=100", nil) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderEgressLocationDue(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("due: status = %d, want 200; body = %s", w.Code, w.Body.String()) + } + var result ProviderEgressLocationDueResult + if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { + t.Fatalf("due: decode body %q: %s", w.Body.String(), err) + } + return result.ClientIds +} + +// An unknown client id must be rejected rather than writing an attempt row +// keyed to a client that does not exist, which nothing would ever read. +func TestProviderEgressLocationAttemptRejectsUnknownClient(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + body, err := json.Marshal(controller.RecordProviderEgressProbeAttemptArgs{ + ClientId: server.NewId(), + ProbeFailure: "tunnel_failed", + }) + if err != nil { + t.Fatalf("marshal attempt: %s", err) + } + req := httptest.NewRequest(http.MethodPost, "/network/provider-egress-attempt", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderEgressLocationAttempt(w, req) + + if w.Code == http.StatusUnauthorized { + t.Fatalf("status = %d, want the correct secret to clear auth (not 401)", w.Code) + } + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 for an unregistered client id; body = %s", w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), "Unknown client.") { + t.Fatalf("body = %q, want it to report the unknown client", w.Body.String()) + } + }) +} + // A limit that is not a positive integer is a caller bug. Silently clamping it // to 1 (or to the default) would answer a question the prober did not ask -- // `limit=0` would come back as an empty list, indistinguishable from "nothing diff --git a/controller/provider_egress_location_controller.go b/controller/provider_egress_location_controller.go index 2c28c494..4c688150 100644 --- a/controller/provider_egress_location_controller.go +++ b/controller/provider_egress_location_controller.go @@ -159,3 +159,55 @@ func SubmitProviderEgressLocation( return &SubmitProviderEgressLocationResult{LocationId: location.LocationId}, nil } + +// maxProbeFailureLen bounds the failure class as submitted: +// provider_egress_probe_attempt.probe_failure is a varchar(64), and rejecting +// an over-long value with a clear error beats letting the insert panic on a +// Postgres "value too long" error and spin in the retry loop. +const maxProbeFailureLen = 64 + +type RecordProviderEgressProbeAttemptArgs struct { + ClientId server.Id `json:"client_id"` + // ProbeFailure is "" when the attempt succeeded, otherwise a short failure + // class (`contract_failed`, `tunnel_failed`, `no_consensus`, ...). + ProbeFailure string `json:"probe_failure,omitempty"` +} + +type RecordProviderEgressProbeAttemptResult struct { + AttemptAt time.Time `json:"attempt_at"` +} + +// RecordProviderEgressProbeAttempt records that the prober tried this provider. +// A failed attempt defers the provider from the due queue for +// ProviderEgressProbeAttemptBackoff, exactly as a successful probe defers it +// for the (much longer) staleness window -- without this, a provider that +// always fails to probe never gets a provider_egress_location row and so stays +// permanently at the head of the queue, starving every other provider. See +// model.GetProviderEgressLocationDue. +// +// The attempt is timestamped by the server, not the prober: the prober is +// reporting something it just did, and a prober whose clock ran fast could +// otherwise defer a provider far past the backoff window. +func RecordProviderEgressProbeAttempt( + ctx context.Context, + args *RecordProviderEgressProbeAttemptArgs, +) (*RecordProviderEgressProbeAttemptResult, error) { + if maxProbeFailureLen < len(args.ProbeFailure) { + return nil, fmt.Errorf("Probe failure class is too long.") + } + // same check as SubmitProviderEgressLocation: without it a typo'd or stale + // client id writes a row keyed to a client that does not exist, which + // nothing ever reads and only the sweep ever removes. + if networkId := model.GetNetworkClientNetwork(ctx, args.ClientId); networkId == nil { + return nil, fmt.Errorf("Unknown client.") + } + + attemptAt := server.NowUtc() + model.SetProviderEgressProbeAttempt(ctx, &model.ProviderEgressProbeAttempt{ + ClientId: args.ClientId, + AttemptAt: attemptAt, + ProbeFailure: args.ProbeFailure, + }) + + return &RecordProviderEgressProbeAttemptResult{AttemptAt: attemptAt}, nil +} diff --git a/db_migrations.go b/db_migrations.go index dfc27b62..7bcb3344 100644 --- a/db_migrations.go +++ b/db_migrations.go @@ -4414,4 +4414,40 @@ var migrations = []any{ CREATE INDEX IF NOT EXISTS provider_egress_location_observed_at ON provider_egress_location (observed_at) `), + + // provider egress probe attempts: when the prober last *tried* a provider, + // successful or not, and how the try failed. + // + // This cannot live on provider_egress_location, because the case it exists + // to handle is precisely a provider that has no row there. A provider that + // connects, holds a Public provide key and fails every probe (firewalled + // egress, dead upstream) never gets an egress row, so its observed_at stays + // NULL, so it sorts to the head of the due queue forever. Enough of them and + // every batch the prober asks for is the same set of permanently-dead + // providers, and no healthy provider's location is ever refreshed -- while + // the endpoint keeps returning a full, plausible-looking batch. + // GetProviderEgressLocationDue defers on a recent attempt as well as a fresh + // success, which needs somewhere to record the attempt. + // + // Pulled forward from the P2 verdict model + // (docs/superpowers/specs/2026-07-25-enforced-provider-geo-probing-design.md, + // probe_attempt_at / probe_failure) because the P1 schedule cannot function + // without it. Deliberately only the two columns the schedule reads, not the + // rest of that model. + newSqlMigration(` + CREATE TABLE IF NOT EXISTS provider_egress_probe_attempt ( + client_id uuid NOT NULL PRIMARY KEY, + attempt_at timestamp NOT NULL, + probe_failure varchar(64) NOT NULL DEFAULT '', + update_time timestamp NOT NULL + ) + `), + + // serves the sweep in RemoveExpiredProviderEgressProbeAttempts. The due + // query reaches this table by primary key through the left join, so it + // needs no index of its own. + newSqlMigration(` + CREATE INDEX IF NOT EXISTS provider_egress_probe_attempt_attempt_at + ON provider_egress_probe_attempt (attempt_at) + `), } diff --git a/model/provider_egress_location_model.go b/model/provider_egress_location_model.go index 7bf9ae00..55d0aaf3 100644 --- a/model/provider_egress_location_model.go +++ b/model/provider_egress_location_model.go @@ -13,6 +13,16 @@ import ( // mmdb lookup on the observed control ip. const ProviderEgressLocationMaxAge = 7 * 24 * time.Hour +// ProviderEgressProbeAttemptBackoff is how long a probe *attempt* defers a +// provider from being offered up again, whether or not the attempt succeeded. +// +// It is much shorter than the staleness window a successful probe buys +// (providerEgressDueAge in api/handlers, half ProviderEgressLocationMaxAge): a +// provider that fails to probe should be retried periodically -- the fault may +// be transient -- just not on every single poll, which is what starves the rest +// of the queue. +const ProviderEgressProbeAttemptBackoff = 6 * time.Hour + // ProviderEgressLocation is a provider location learned by probing the // provider's own egress, rather than by looking up its control-connection ip. type ProviderEgressLocation struct { @@ -86,6 +96,87 @@ func SetProviderEgressLocation(ctx context.Context, e *ProviderEgressLocation) { }) } +// ProviderEgressProbeAttempt records that the prober tried a provider, whether +// or not the try produced a location. +// +// A provider that has never been probed successfully has no +// ProviderEgressLocation row at all, so an attempt cannot be recorded there -- +// see the provider_egress_probe_attempt migration for why that matters. +// ProbeFailure is "" for a successful attempt, otherwise a short failure class +// (`tunnel_failed`, `no_consensus`, ...). +type ProviderEgressProbeAttempt struct { + ClientId server.Id + AttemptAt time.Time + ProbeFailure string + UpdateTime time.Time +} + +// SetProviderEgressProbeAttempt upserts the last probe attempt for a provider. +// +// Like SetProviderEgressLocation the upsert is monotonic in its timestamp: a +// replayed or out-of-order report older than what is already stored is dropped +// rather than moving the provider's last-attempt time backwards, which would +// hand it back to the prober early. +func SetProviderEgressProbeAttempt(ctx context.Context, a *ProviderEgressProbeAttempt) { + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO provider_egress_probe_attempt ( + client_id, + attempt_at, + probe_failure, + update_time + ) + VALUES ($1, $2, $3, $4) + ON CONFLICT (client_id) DO UPDATE + SET + attempt_at = $2, + probe_failure = $3, + update_time = $4 + WHERE provider_egress_probe_attempt.attempt_at < EXCLUDED.attempt_at + `, + a.ClientId, + a.AttemptAt.UTC(), + a.ProbeFailure, + server.NowUtc(), + )) + }) +} + +// GetProviderEgressProbeAttempt returns the last recorded probe attempt for a +// provider, or nil. +func GetProviderEgressProbeAttempt(ctx context.Context, clientId server.Id) *ProviderEgressProbeAttempt { + var a *ProviderEgressProbeAttempt + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT + client_id, + attempt_at, + probe_failure, + update_time + FROM provider_egress_probe_attempt + WHERE client_id = $1 + `, + clientId, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + a = &ProviderEgressProbeAttempt{} + server.Raise(result.Scan( + &a.ClientId, + &a.AttemptAt, + &a.ProbeFailure, + &a.UpdateTime, + )) + } + }) + }) + return a +} + // GetProviderEgressLocation returns the stored location for a provider, or nil. func GetProviderEgressLocation(ctx context.Context, clientId server.Id) *ProviderEgressLocation { var e *ProviderEgressLocation @@ -275,15 +366,16 @@ func GetLocation(ctx context.Context, locationId server.Id) *Location { } // GetProviderEgressLocationDue returns the client ids of providers whose -// egress location is due for a probe: their newest probe is older than -// minObservedAt, or they have never been probed at all. Oldest first, so the +// egress location is due for a probe: no fresh success (newest probe older than +// minObservedAt, or never probed) *and* no recent attempt (last attempt older +// than minAttemptAt, or never attempted). Oldest first, so the // longest-unprobed are handed out first, capped at limit. // // This is the durable replacement for the prober's in-memory ttl cache: the // schedule lives in the database, so a prober restart resumes where it left // off instead of re-probing everything. // -// Two things about the shape of this query matter. +// Three things about the shape of this query matter. // // First, candidates are sourced from the live provider population // (network_client_location_reliability, connected + valid) and the egress row @@ -299,12 +391,26 @@ func GetLocation(ctx context.Context, locationId server.Id) *Location { // guaranteed failure. This is the same filter UpdateClientLocations and // UpdateClientScores apply (network_client_location_model.go). // -// The cutoff is computed by the caller in Go and bound as a parameter: -// observed_at is a naive `timestamp` holding utc, and comparing it against sql -// now() would cast through the session timezone and silently skip a window. +// Third, a recent *attempt* defers a provider the same way a recent success +// does. Without that, a provider that connects and holds a Public provide key +// but always fails to probe -- for any reason other than the missing Public key +// screened for above -- never gets an egress row, so its observed_at stays +// NULL, so it sorts ahead of every stale-but-refreshable provider on every +// single poll, forever. Enough such providers to fill a batch and no healthy +// provider is ever refreshed again, while this endpoint goes on returning a +// full, plausible-looking batch. The in-memory ttl cache this replaced was +// incidentally immune, because it marked a provider probed whether or not the +// probe worked; moving the schedule server-side dropped that protection, and +// provider_egress_probe_attempt is what restores it. +// +// Both cutoffs are computed by the caller in Go and bound as parameters: +// observed_at and attempt_at are naive `timestamp` columns holding utc, and +// comparing them against sql now() would cast through the session timezone and +// silently skip a window. func GetProviderEgressLocationDue( ctx context.Context, minObservedAt time.Time, + minAttemptAt time.Time, limit int, ) []server.Id { clientIds := []server.Id{} @@ -319,6 +425,9 @@ func GetProviderEgressLocationDue( LEFT JOIN provider_egress_location ON provider_egress_location.client_id = network_client_location_reliability.client_id + LEFT JOIN provider_egress_probe_attempt ON + provider_egress_probe_attempt.client_id = network_client_location_reliability.client_id + WHERE network_client_location_reliability.connected = true AND network_client_location_reliability.valid = true AND @@ -331,15 +440,26 @@ func GetProviderEgressLocationDue( ( provider_egress_location.observed_at IS NULL OR provider_egress_location.observed_at < $2 + ) AND + ( + provider_egress_probe_attempt.attempt_at IS NULL OR + provider_egress_probe_attempt.attempt_at < $3 ) -- never-probed sorts ahead of merely stale: a missing observed_at - -- is infinitely old - ORDER BY provider_egress_location.observed_at ASC NULLS FIRST - LIMIT $3 + -- is infinitely old. client_id breaks the tie so batch composition + -- is deterministic instead of plan-dependent -- otherwise the whole + -- never-probed population ties on NULL and which slice of it the + -- prober gets back under a limit is whatever order the executor + -- happened to produce. + ORDER BY + provider_egress_location.observed_at ASC NULLS FIRST, + network_client_location_reliability.client_id ASC + LIMIT $4 `, ProvideModePublic, minObservedAt.UTC(), + minAttemptAt.UTC(), limit, ) server.WithPgResult(result, err, func() { @@ -364,3 +484,17 @@ func RemoveExpiredProviderEgressLocations(ctx context.Context, minObservedAt tim )) }) } + +// RemoveExpiredProviderEgressProbeAttempts drops attempts older than +// minAttemptAt. An attempt only carries information for as long as it defers +// the provider (ProviderEgressProbeAttemptBackoff); past that the row is just +// storage held for a client id that may no longer exist. +func RemoveExpiredProviderEgressProbeAttempts(ctx context.Context, minAttemptAt time.Time) { + server.MaintenanceTx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + `DELETE FROM provider_egress_probe_attempt WHERE attempt_at < $1`, + minAttemptAt.UTC(), + )) + }) +} diff --git a/model/provider_egress_location_model_test.go b/model/provider_egress_location_model_test.go index c6f5f8cb..892f01b3 100644 --- a/model/provider_egress_location_model_test.go +++ b/model/provider_egress_location_model_test.go @@ -220,6 +220,7 @@ func TestRemoveExpiredProviderEgressLocations(t *testing.T) { // The caller must run UpdateClientLocationReliabilities afterward -- that is // what rolls the live connection tables up into the // network_client_location_reliability row (connected + valid) the query reads. +// It returns the connection id, so a caller can disconnect the provider again. func testing_connectProbeableProvider( t testing.TB, ctx context.Context, @@ -227,7 +228,7 @@ func testing_connectProbeableProvider( locationId server.Id, clientAddress string, provideMode ProvideMode, -) { +) server.Id { Testing_CreateDevice(ctx, server.NewId(), server.NewId(), clientId, "", "") handlerId := CreateNetworkClientHandler(ctx) @@ -243,6 +244,8 @@ func testing_connectProbeableProvider( SetProvide(ctx, clientId, map[ProvideMode][]byte{ provideMode: []byte("provide-secret"), }) + + return connectionId } // The prober asks the server what to probe next. The answer must be sourced @@ -287,7 +290,9 @@ func TestGetProviderEgressLocationDue(t *testing.T) { }) // `never` and `nonPublic` deliberately get no row at all - due := GetProviderEgressLocationDue(ctx, now.Add(-24*time.Hour), 100) + // no attempt rows exist in this test, so the attempt cutoff never + // excludes anything; freshness is the only variable + due := GetProviderEgressLocationDue(ctx, now.Add(-24*time.Hour), now, 100) // a provider probed an hour ago must not be re-probed; one probed three // days ago must be; one never probed must be @@ -314,7 +319,7 @@ func TestGetProviderEgressLocationDue(t *testing.T) { } // limit is honoured - limited := GetProviderEgressLocationDue(ctx, now.Add(-24*time.Hour), 1) + limited := GetProviderEgressLocationDue(ctx, now.Add(-24*time.Hour), now, 1) if len(limited) != 1 { t.Fatalf("len(due) = %d for limit 1, want 1", len(limited)) } @@ -323,3 +328,218 @@ func TestGetProviderEgressLocationDue(t *testing.T) { } }) } + +// A provider that connects, holds a Public provide key and fails every probe +// never gets a provider_egress_location row, so its observed_at stays NULL, so +// it sorts ahead of every stale-but-refreshable provider -- forever, on every +// poll. Enough of them to fill a batch and no healthy provider's location is +// ever refreshed again, silently: the endpoint keeps returning a full, +// plausible-looking batch of the same dead providers. +// +// A recent attempt must therefore defer a provider exactly as a fresh success +// does. Deleting the attempt predicate from GetProviderEgressLocationDue must +// fail this test. +func TestGetProviderEgressLocationDueDefersRecentlyAttempted(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + + city := &Location{ + LocationType: LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, city) + + // never probed successfully, and the prober just tried it and failed + dead := server.NewId() + // probed successfully three days ago, never attempted since: the + // provider that actually needs the next probe slot + healthyStale := server.NewId() + + testing_connectProbeableProvider(t, ctx, dead, city.LocationId, "0.0.0.1:0", ProvideModePublic) + testing_connectProbeableProvider(t, ctx, healthyStale, city.LocationId, "0.0.0.2:0", ProvideModePublic) + + UpdateClientLocationReliabilities(ctx, now.Add(-time.Hour), now) + + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: healthyStale, LocationId: city.LocationId, + CountryCode: "us", ObservedAt: now.Add(-72 * time.Hour), + }) + // `dead` deliberately gets no location row -- it has never succeeded -- + // only a failed attempt seconds ago + SetProviderEgressProbeAttempt(ctx, &ProviderEgressProbeAttempt{ + ClientId: dead, + AttemptAt: now.Add(-5 * time.Second), + ProbeFailure: "tunnel_failed", + }) + + minObservedAt := now.Add(-24 * time.Hour) + minAttemptAt := now.Add(-ProviderEgressProbeAttemptBackoff) + + due := GetProviderEgressLocationDue(ctx, minObservedAt, minAttemptAt, 100) + + if slices.Contains(due, dead) { + t.Fatalf("due = %v, must not contain the provider attempted seconds ago (%s)", due, dead) + } + if !slices.Contains(due, healthyStale) { + t.Fatalf("due = %v, must contain the stale-but-refreshable provider (%s)", due, healthyStale) + } + + // the starvation itself: with a batch big enough for exactly one + // provider, the slot must go to the one that can actually be refreshed, + // not to the never-probed one that just failed. Without the attempt + // predicate `dead` wins this on observed_at IS NULL every single poll. + limited := GetProviderEgressLocationDue(ctx, minObservedAt, minAttemptAt, 1) + if len(limited) != 1 { + t.Fatalf("len(due) = %d for limit 1, want 1", len(limited)) + } + if limited[0] != healthyStale { + t.Fatalf("due[0] = %s for limit 1, want the refreshable provider %s, not the just-failed one %s", limited[0], healthyStale, dead) + } + + // ... and the deferral is a backoff, not a ban: once the backoff has + // elapsed the same provider is offered again. The caller computes the + // cutoff as (wall clock - backoff), so a poll exactly one backoff period + // after the attempt computes `now`. + afterBackoff := GetProviderEgressLocationDue(ctx, minObservedAt, now, 100) + if !slices.Contains(afterBackoff, dead) { + t.Fatalf("due = %v, must contain the failed provider (%s) again once the attempt backoff has elapsed", afterBackoff, dead) + } + }) +} + +// Only live, routable providers are probeable. A provider that has gone offline +// (connected = false) or that looks messed up from a routing perspective +// (valid = false, a generated column: more than one address hash or location on +// its live connections) must not be handed to the prober. Deleting either +// predicate from GetProviderEgressLocationDue must fail this test. +func TestGetProviderEgressLocationDueRequiresConnectedAndValid(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + + city := &Location{ + LocationType: LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, city) + + good := server.NewId() + disconnected := server.NewId() + invalid := server.NewId() + + testing_connectProbeableProvider(t, ctx, good, city.LocationId, "0.0.0.1:0", ProvideModePublic) + disconnectedConnectionId := testing_connectProbeableProvider(t, ctx, disconnected, city.LocationId, "0.0.0.2:0", ProvideModePublic) + + // `invalid` holds two simultaneous connections from two different + // addresses, which makes client_address_hash_count = 2 and so the + // generated `valid` column false. The two addresses must be in + // different /29s: server.ClientIpHash buckets ipv4 to the /29 network, + // so e.g. 0.0.0.3 and 0.0.0.4 would hash the same and count as one. + testing_connectProbeableProvider(t, ctx, invalid, city.LocationId, "0.0.0.3:0", ProvideModePublic) + secondHandlerId := CreateNetworkClientHandler(ctx) + secondConnectionId, _, _, _, err := ConnectNetworkClient(ctx, invalid, "0.0.8.3:0", secondHandlerId) + if err != nil { + t.Fatalf("connect second address: %s", err) + } + if err := SetConnectionLocation(ctx, secondConnectionId, city.LocationId, &ConnectionLocationScores{}); err != nil { + t.Fatalf("set second connection location: %s", err) + } + + // first roll-up: everything above is connected + UpdateClientLocationReliabilities(ctx, now.Add(-time.Hour), now) + + // `disconnected` drops off, and a second roll-up flips its reliability + // row's connected to false (the row itself survives) + if err := DisconnectNetworkClient(ctx, disconnectedConnectionId); err != nil { + t.Fatalf("disconnect client: %s", err) + } + UpdateClientLocationReliabilities(ctx, now.Add(-time.Hour), server.NowUtc()) + + due := GetProviderEgressLocationDue(ctx, now.Add(-24*time.Hour), now, 100) + + if !slices.Contains(due, good) { + t.Fatalf("due = %v, must contain the connected, valid provider (%s)", due, good) + } + if slices.Contains(due, disconnected) { + t.Fatalf("due = %v, must not contain the disconnected provider (%s)", due, disconnected) + } + if slices.Contains(due, invalid) { + t.Fatalf("due = %v, must not contain the provider whose reliability row is not valid (%s)", due, invalid) + } + }) +} + +// The attempt upsert is monotonic in attempt_at, for the same reason the +// location upsert is: a replayed or out-of-order report must not move the last +// attempt backwards and hand the provider back to the prober early. +func TestProviderEgressProbeAttemptUpsertIgnoresOlderReplay(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + clientId := server.NewId() + newer := server.NowUtc() + older := newer.Add(-time.Hour) + + SetProviderEgressProbeAttempt(ctx, &ProviderEgressProbeAttempt{ + ClientId: clientId, AttemptAt: newer, ProbeFailure: "no_consensus", + }) + SetProviderEgressProbeAttempt(ctx, &ProviderEgressProbeAttempt{ + ClientId: clientId, AttemptAt: older, ProbeFailure: "tunnel_failed", + }) + + got := GetProviderEgressProbeAttempt(ctx, clientId) + if got == nil { + t.Fatal("expected a stored probe attempt") + } + connect.AssertEqual(t, got.ProbeFailure, "no_consensus") + // postgres `timestamp` keeps microseconds, Go keeps nanoseconds, so + // compare with a tolerance rather than for equality + if delta := got.AttemptAt.Sub(newer); delta < -time.Millisecond || time.Millisecond < delta { + t.Fatalf("attempt_at = %s, want the newer attempt %s", got.AttemptAt, newer) + } + + // a strictly newer report does win + newest := newer.Add(time.Minute) + SetProviderEgressProbeAttempt(ctx, &ProviderEgressProbeAttempt{ + ClientId: clientId, AttemptAt: newest, ProbeFailure: "", + }) + got = GetProviderEgressProbeAttempt(ctx, clientId) + connect.AssertEqual(t, got.ProbeFailure, "") + + // absent + if GetProviderEgressProbeAttempt(ctx, server.NewId()) != nil { + t.Fatal("absent attempt must return nil") + } + }) +} + +func TestRemoveExpiredProviderEgressProbeAttempts(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + keep := server.NewId() + drop := server.NewId() + SetProviderEgressProbeAttempt(ctx, &ProviderEgressProbeAttempt{ + ClientId: keep, AttemptAt: server.NowUtc(), + }) + SetProviderEgressProbeAttempt(ctx, &ProviderEgressProbeAttempt{ + ClientId: drop, AttemptAt: server.NowUtc().Add(-30 * 24 * time.Hour), + }) + + RemoveExpiredProviderEgressProbeAttempts(ctx, server.NowUtc().Add(-24*time.Hour)) + + if GetProviderEgressProbeAttempt(ctx, keep) == nil { + t.Fatal("recent attempt must survive the sweep") + } + if GetProviderEgressProbeAttempt(ctx, drop) != nil { + t.Fatal("old attempt must be swept") + } + }) +} diff --git a/taskworker/work/provider_egress_location_work.go b/taskworker/work/provider_egress_location_work.go index 94901c34..533d9609 100644 --- a/taskworker/work/provider_egress_location_work.go +++ b/taskworker/work/provider_egress_location_work.go @@ -33,8 +33,14 @@ func RemoveExpiredProviderEgressLocations( _ *RemoveExpiredProviderEgressLocationsArgs, clientSession *session.ClientSession, ) (*RemoveExpiredProviderEgressLocationsResult, error) { - minObservedAt := server.NowUtc().Add(-4 * model.ProviderEgressLocationMaxAge) + now := server.NowUtc() + minObservedAt := now.Add(-4 * model.ProviderEgressLocationMaxAge) model.RemoveExpiredProviderEgressLocations(clientSession.Ctx, minObservedAt) + // probe attempts stop meaning anything once they no longer defer the + // provider; same reasoning as above, a looser multiple of the window that + // actually matters. + minAttemptAt := now.Add(-4 * model.ProviderEgressProbeAttemptBackoff) + model.RemoveExpiredProviderEgressProbeAttempts(clientSession.Ctx, minAttemptAt) return &RemoveExpiredProviderEgressLocationsResult{}, nil } From e84775e9333255e4fc65dd4cc862f4c813992a9e Mon Sep 17 00:00:00 2001 From: Ryan Mello Date: Mon, 27 Jul 2026 03:22:51 +0100 Subject: [PATCH 17/30] fix(controller,model): the egress probe must match existing locations, never create them SubmitProviderEgressLocation validated a probe-supplied city/region only as trimmed, non-empty and <=128 chars, then handed it to model.CreateLocation. CreateLocation dedupes a city on its exact location_name, so an unrecognised spelling did not fail -- it silently inserted a new permanent row into the shared `location` table and added it to the search index. This is not hypothetical. The prober's consensus stores the winning source's original display string, and the three free geolocation APIs demonstrably disagree on spelling: we observed "Frankfurt am Main (Innenstadt I)" against "Frankfurt am Main" for the same host. "Frankfurt am Main", "Frankfurt Am Main" and "Frankfurt/Main" would each become their own row. Those rows are permanent, they survive a code revert, they feed the location search and the provider list, and there is no cleanup path. An ingest endpoint that can add rows to that table is an endpoint that can corrupt it from outside. A geolocation probe has no business defining the world's cities, so the ingest path now resolves against rows that already exist and creates none. model.MatchExistingLocation walks country -> region -> city, trying an exact, fully-indexed match at each level first (the common case: the winning source usually spells it the way the mmdb import did) and falling back to a normalized comparison -- lowercased with punctuation and whitespace dropped -- so the trivial variants land on the row that is already there. Standard library only: a transliteration/fuzzy-match dependency is a lot of new behaviour to take on for a path whose failure mode is already "use the country". Deliberately conservative -- "Frankfurt/Main" folds to "frankfurtmain", does not match "frankfurtammain", and falls back rather than guessing at the wrong row. When nothing resolves, the submission is stored at country granularity instead of being rejected: country is the granularity this design treats as trustworthy, and losing city precision for one probe beats polluting shared data permanently. The country fallback still goes through CreateLocation -- country rows are keyed on country_code, so a variant *name* cannot produce a second row for the same country the way a variant city name can, and a probe from a country not yet in the table must not be dropped. city_confident now records the granularity actually stored rather than what the probe claimed, keeping the schema's documented invariant intact: location_id is a city row exactly when city_confident is set. Tests: an unmatched city stores country granularity and leaves the location row count unchanged; five spelling/case/punctuation variants of one city all resolve to the single existing row and add none. Before this change the unmatched city took the table 3 -> 4 rows and stored as a city, and the variants took it 3 -> 7. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01QtgqtCmKJRXdsQ5ktiqwkg (cherry picked from commit 682b29b4dccbb30db46542087e7d1708b8970055) --- .../provider_egress_location_controller.go | 54 +++- ...rovider_egress_location_controller_test.go | 173 ++++++++++- model/provider_egress_location_model.go | 280 ++++++++++++++++++ 3 files changed, 494 insertions(+), 13 deletions(-) diff --git a/controller/provider_egress_location_controller.go b/controller/provider_egress_location_controller.go index 4c688150..c8c64832 100644 --- a/controller/provider_egress_location_controller.go +++ b/controller/provider_egress_location_controller.go @@ -126,23 +126,53 @@ func SubmitProviderEgressLocation( } } - // resolve to a canonical location row. city granularity only when the - // probe agreed on a city; otherwise country. - location := &model.Location{ - LocationType: model.LocationTypeCountry, - Country: country, - CountryCode: countryCode, - } + // resolve to a location row. City granularity only when the probe agreed + // on a city AND that city already exists in the location table. + // + // The probe MUST NOT define new cities or regions. model.CreateLocation + // dedupes a city on its exact location_name, so an unrecognised spelling + // does not fail -- it silently inserts a new permanent row into the shared + // `location` table and adds it to the search index. The three free + // geolocation sources the prober reaches consensus over demonstrably + // disagree on spelling ("Frankfurt am Main (Innenstadt I)" vs "Frankfurt am + // Main" for the same host, observed), and the consensus keeps the winning + // source's original display string -- so "Frankfurt am Main", "Frankfurt Am + // Main" and "Frankfurt/Main" would each become their own row. Those rows + // survive a code revert and there is no cleanup path. + // + // model.MatchExistingLocation therefore matches only, never creates, + // case-insensitively and ignoring punctuation/whitespace so the ordinary + // variants land on the row that is already there. When it does not resolve, + // this submission falls back to country granularity: country is the + // granularity this design treats as trustworthy anyway, and losing city + // precision for one probe is strictly better than permanently polluting a + // table shared with the provider list and the location search. + var location *model.Location if args.CityConfident { + location = model.MatchExistingLocation(ctx, countryCode, region, city) + } + + // city_confident records the granularity of the row actually stored, not + // what the probe claimed. The schema's documented invariant is that + // location_id is a city row exactly when city_confident is set (see the + // provider_egress_location migration), and a city-confident probe whose + // city did not resolve is stored at country granularity. + cityConfident := location != nil + + if location == nil { + // country granularity. This still goes through CreateLocation: a + // country row is keyed on country_code, so a variant *name* can never + // produce a second row for the same country the way a variant city name + // can -- the pollution this guards against is not reachable here. The + // country row is also the whole point of the fallback, so a probe from + // a country not yet in the table must not be dropped. location = &model.Location{ - LocationType: model.LocationTypeCity, - City: city, - Region: region, + LocationType: model.LocationTypeCountry, Country: country, CountryCode: countryCode, } + model.CreateLocation(ctx, location) } - model.CreateLocation(ctx, location) model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ ClientId: args.ClientId, @@ -153,7 +183,7 @@ func SubmitProviderEgressLocation( Hosting: args.Hosting, Proxy: args.Proxy, Mobile: args.Mobile, - CityConfident: args.CityConfident, + CityConfident: cityConfident, ObservedAt: args.ObservedAt, }) diff --git a/controller/provider_egress_location_controller_test.go b/controller/provider_egress_location_controller_test.go index d11c08c8..31c7d1cb 100644 --- a/controller/provider_egress_location_controller_test.go +++ b/controller/provider_egress_location_controller_test.go @@ -105,6 +105,18 @@ func TestSubmitProviderEgressLocationCityConfidentStoresCity(t *testing.T) { clientId := server.NewId() model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + // the ingest path resolves against locations that ALREADY exist and + // never creates one, so the city has to be in the table first -- as it + // would be from the mmdb import + denver := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Denver", + Region: "Colorado", + Country: "United States", + CountryCode: "us", + } + model.CreateLocation(ctx, denver) + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ ClientId: clientId, CountryCode: "us", @@ -123,7 +135,9 @@ func TestSubmitProviderEgressLocationCityConfidentStoresCity(t *testing.T) { } connect.AssertEqual(t, stored.CityConfident, true) - // the resolved location must be the city-granular row + // the resolved location must be the city-granular row that already + // existed, not a new one + connect.AssertEqual(t, stored.LocationId, denver.LocationId) loc := model.GetLocation(ctx, stored.LocationId) if loc == nil { t.Fatal("expected the resolved location row to exist") @@ -387,3 +401,160 @@ func TestSubmitProviderEgressLocationAcceptsLargeAsn(t *testing.T) { connect.AssertEqual(t, stored.ASN, largeAsn) }) } + +// testing_countLocations is the whole point of the two tests below: the +// `location` table is shared with the provider list and the location search, +// its rows are permanent, and nothing cleans up a bad one. An ingest endpoint +// that can add to it is an endpoint that can corrupt it from outside. +func testing_countLocations(ctx context.Context) int64 { + var count int64 + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query(ctx, `SELECT COUNT(*) FROM location`) + server.WithPgResult(result, err, func() { + if result.Next() { + server.Raise(result.Scan(&count)) + } + }) + }) + return count +} + +// A city-confident submission whose city is not already in the location table +// must fall back to country granularity and must NOT create a location row. +// +// model.CreateLocation dedupes a city on its exact location_name, so before +// this fix an unrecognised spelling did not fail -- it silently inserted a new +// permanent row and indexed it for search. The prober's consensus stores the +// winning source's original display string and the three free geolocation +// sources demonstrably disagree on spelling, so "Frankfurt am Main", +// "Frankfurt Am Main" and "Frankfurt/Main" would each have become their own +// row. Those rows outlive a code revert and there is no cleanup path. +// +// Reverting the MatchExistingLocation call in SubmitProviderEgressLocation must +// fail this test: the row count goes up and the stored location is a city. +func TestSubmitProviderEgressLocationUnknownCityDoesNotCreateALocation(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + // Germany, and one real German city, already exist -- as they would + // from the mmdb import. The submission below names a DIFFERENT city + // that has never been seen. + model.CreateLocation(ctx, &model.Location{ + LocationType: model.LocationTypeCity, + City: "Frankfurt am Main", + Region: "Hesse", + Country: "Germany", + CountryCode: "de", + }) + + before := testing_countLocations(ctx) + + res, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "de", + Country: "Germany", + Region: "Hesse", + City: "Kleinstadt Nirgendwo", + CountryConfident: true, + CityConfident: true, + ObservedAt: server.NowUtc(), + }) + connect.AssertEqual(t, err, nil) + + // nothing was added to the shared table + after := testing_countLocations(ctx) + if after != before { + t.Errorf("location row count went from %d to %d; an unmatched city must not create a permanent row in the shared location table", before, after) + } + + // and the submission was stored at country granularity instead + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected the submission to be stored") + } + loc := model.GetLocation(ctx, stored.LocationId) + if loc == nil { + t.Fatal("expected the resolved location row to exist") + } + if loc.LocationType != model.LocationTypeCountry { + t.Errorf("stored location_type = %q, want %q: an unmatched city must fall back to country granularity", loc.LocationType, model.LocationTypeCountry) + } + connect.AssertEqual(t, res.LocationId, stored.LocationId) + + // city_confident tracks the granularity actually stored, so the row + // stays internally consistent: location_id is a city row exactly when + // city_confident is set + if stored.CityConfident { + t.Error("city_confident must be false when the submission was stored at country granularity") + } + }) +} + +// The variants that matter are trivial: the same city spelled with different +// case, punctuation or spacing. Those must resolve to the row that is already +// there -- discarding them to country would throw away real precision for no +// reason, and creating a row for each is the bug this guards against. +func TestSubmitProviderEgressLocationMatchesCitySpellingVariant(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), server.NewId(), "", "") + + frankfurt := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Frankfurt am Main", + Region: "Hesse", + Country: "Germany", + CountryCode: "de", + } + model.CreateLocation(ctx, frankfurt) + + before := testing_countLocations(ctx) + + // each of these is a real disagreement between the geolocation sources + // over the same place + variants := []struct{ region, city string }{ + {"Hesse", "Frankfurt am Main"}, // exact + {"Hesse", "Frankfurt Am Main"}, // case + {"hesse", "FRANKFURT AM MAIN"}, // case, both levels + {"Hesse", "Frankfurt-am-Main"}, // punctuation + {"Hesse", " Frankfurt am Main "}, + } + for _, variant := range variants { + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "DE", + Country: "Germany", + Region: variant.region, + City: variant.city, + CountryConfident: true, + CityConfident: true, + ObservedAt: server.NowUtc(), + }) + connect.AssertEqual(t, err, nil) + + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatalf("%q: expected the submission to be stored", variant.city) + } + if stored.LocationId != frankfurt.LocationId { + t.Errorf("%q resolved to %s, want the existing Frankfurt row %s", variant.city, stored.LocationId, frankfurt.LocationId) + } + if !stored.CityConfident { + t.Errorf("%q: city_confident must stay set when the city resolved", variant.city) + } + } + + if after := testing_countLocations(ctx); after != before { + t.Errorf("location row count went from %d to %d; spelling variants must reuse the existing row, not add new ones", before, after) + } + }) +} diff --git a/model/provider_egress_location_model.go b/model/provider_egress_location_model.go index 55d0aaf3..1bb13eed 100644 --- a/model/provider_egress_location_model.go +++ b/model/provider_egress_location_model.go @@ -4,6 +4,7 @@ import ( "context" "strings" "time" + "unicode" "github.com/urnetwork/server" ) @@ -365,6 +366,252 @@ func GetLocation(ctx context.Context, locationId server.Id) *Location { return loc } +// normalizeLocationName folds a location name to a comparison key: lowercased, +// with every rune that is not a letter or a digit dropped. So +// "Frankfurt am Main", "Frankfurt Am Main" and "FRANKFURT AM MAIN" all fold to +// "frankfurtammain" and match the one row that already exists. +// +// This is deliberately a comparison key only -- it is never stored, and never +// used to build a location_name. It exists so a trivial spelling variant from a +// geolocation source resolves to the existing row instead of being treated as a +// different place. +// +// Punctuation is dropped rather than mapped to a space because the disagreement +// is over whether the separator exists at all ("Washington, D.C." vs +// "Washington DC"). Note this deliberately does not fold "Frankfurt/Main" onto +// "Frankfurt am Main": dropping the separator gives "frankfurtmain" != +// "frankfurtammain", so that one falls back to country granularity rather than +// matching the wrong row. Falling back is the safe outcome; guessing is not. +// Stdlib only, by design -- a transliteration/fuzzy-match dependency is a large +// amount of new behaviour to take on for an ingest path whose failure mode is +// already "use the country". +func normalizeLocationName(name string) string { + var b strings.Builder + b.Grow(len(name)) + for _, r := range strings.ToLower(name) { + if unicode.IsLetter(r) || unicode.IsDigit(r) { + b.WriteRune(r) + } + } + return b.String() +} + +// matchLocationNameInTx returns the location_id of the row in `candidates` +// whose location_name matches `name`, preferring an exact match and falling +// back to a normalized one (see normalizeLocationName), or nil for no match. +// Candidates must already be ordered deterministically by the caller so that +// two rows folding to the same key always resolve the same way. +func matchLocationName(name string, candidateIds []server.Id, candidateNames []string) *server.Id { + for i, candidateName := range candidateNames { + if candidateName == name { + return &candidateIds[i] + } + } + normalized := normalizeLocationName(name) + if normalized == "" { + // nothing comparable survives folding (e.g. a name of only + // punctuation); an empty key would match any other such row + return nil + } + for i, candidateName := range candidateNames { + if normalizeLocationName(candidateName) == normalized { + return &candidateIds[i] + } + } + return nil +} + +// MatchExistingLocation resolves (countryCode, region, city) against location +// rows that ALREADY EXIST and returns the city-granular row, or nil if any +// level of the hierarchy does not resolve. It never inserts anything. +// +// This is the resolver the provider egress ingest path uses instead of +// CreateLocation. CreateLocation deduplicates a city on its exact +// location_name, so an unrecognised spelling does not fail -- it silently +// creates a new, permanent row in the shared `location` table and indexes it +// for search. A geolocation probe has no business defining the world's cities: +// the three free sources the prober reaches consensus over demonstrably +// disagree on spelling (we observed "Frankfurt am Main (Innenstadt I)" against +// "Frankfurt am Main" for one host), and the consensus stores the winning +// source's original display string. Each variant would become its own row, +// those rows outlive a code revert, and there is no cleanup path. +// +// Matching is case-insensitive and ignores punctuation and whitespace +// differences, so the ordinary variants resolve to the row that is already +// there. When nothing resolves the caller falls back to country granularity -- +// see SubmitProviderEgressLocation. Falling back loses precision for one +// submission; creating a row corrupts shared data permanently. +// +// Each level tries an exact, fully-indexed match first (the common case: the +// winning source usually spells it the way the mmdb import did) and only scans +// the level's candidates when that misses. +func MatchExistingLocation( + ctx context.Context, + countryCode string, + region string, + city string, +) *Location { + countryCode = strings.ToLower(strings.TrimSpace(countryCode)) + region = strings.TrimSpace(region) + city = strings.TrimSpace(city) + if countryCode == "" || region == "" || city == "" { + return nil + } + + var match *Location + server.Db(ctx, func(conn server.PgConn) { + // country: keyed on country_code alone, exactly as CreateLocation + // dedupes it, so there is no name to match here + var countryLocationId server.Id + var countryName string + found := false + result, err := conn.Query( + ctx, + ` + SELECT location_id, location_name + FROM location + WHERE location_type = $1 AND country_code = $2 + ORDER BY location_id + LIMIT 1 + `, + LocationTypeCountry, + countryCode, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + server.Raise(result.Scan(&countryLocationId, &countryName)) + found = true + } + }) + if !found { + return + } + + // region, within that country + regionLocationId := matchChildLocation( + ctx, + conn, + LocationTypeRegion, + countryCode, + region, + ` + SELECT location_id, location_name + FROM location + WHERE + location_type = $1 AND + country_code = $2 AND + location_name = $3 AND + country_location_id = $4 + `, + ` + SELECT location_id, location_name + FROM location + WHERE + location_type = $1 AND + country_code = $2 AND + country_location_id = $3 + ORDER BY location_id + `, + []any{countryLocationId}, + ) + if regionLocationId == nil { + return + } + + // city, within that region + cityLocationId := matchChildLocation( + ctx, + conn, + LocationTypeCity, + countryCode, + city, + ` + SELECT location_id, location_name + FROM location + WHERE + location_type = $1 AND + country_code = $2 AND + location_name = $3 AND + region_location_id = $4 AND + country_location_id = $5 + `, + ` + SELECT location_id, location_name + FROM location + WHERE + location_type = $1 AND + country_code = $2 AND + region_location_id = $3 AND + country_location_id = $4 + ORDER BY location_id + `, + []any{*regionLocationId, countryLocationId}, + ) + if cityLocationId == nil { + return + } + + match = &Location{ + LocationType: LocationTypeCity, + City: city, + Region: region, + Country: countryName, + CountryCode: countryCode, + LocationId: *cityLocationId, + CityLocationId: *cityLocationId, + RegionLocationId: *regionLocationId, + CountryLocationId: countryLocationId, + } + }) + return match +} + +// matchChildLocation runs the exact-match query first and only falls back to +// scanning the level's candidates when it misses. `parents` are the parent +// location ids the two queries scope on: the exact query binds them after +// (location_type, country_code, name), the candidate query after +// (location_type, country_code). +func matchChildLocation( + ctx context.Context, + conn server.PgConn, + locationType LocationType, + countryCode string, + name string, + exactSql string, + candidatesSql string, + parents []any, +) *server.Id { + exactArgs := append([]any{locationType, countryCode, name}, parents...) + var exactId *server.Id + result, err := conn.Query(ctx, exactSql, exactArgs...) + server.WithPgResult(result, err, func() { + if result.Next() { + var locationId server.Id + var locationName string + server.Raise(result.Scan(&locationId, &locationName)) + exactId = &locationId + } + }) + if exactId != nil { + return exactId + } + + candidateArgs := append([]any{locationType, countryCode}, parents...) + candidateIds := []server.Id{} + candidateNames := []string{} + result, err = conn.Query(ctx, candidatesSql, candidateArgs...) + server.WithPgResult(result, err, func() { + for result.Next() { + var locationId server.Id + var locationName string + server.Raise(result.Scan(&locationId, &locationName)) + candidateIds = append(candidateIds, locationId) + candidateNames = append(candidateNames, locationName) + } + }) + return matchLocationName(name, candidateIds, candidateNames) +} + // GetProviderEgressLocationDue returns the client ids of providers whose // egress location is due for a probe: no fresh success (newest probe older than // minObservedAt, or never probed) *and* no recent attempt (last attempt older @@ -407,6 +654,39 @@ func GetLocation(ctx context.Context, locationId server.Id) *Location { // observed_at and attempt_at are naive `timestamp` columns holding utc, and // comparing them against sql now() would cast through the session timezone and // silently skip a window. +// +// # Two passes, not one +// +// Expressed as a single statement this is a scan of +// network_client_location_reliability with two LEFT JOINs, sorted on +// observed_at from an outer-joined table. That sort cannot use an index: the +// column being ordered on does not exist for most of the rows being ordered. +// At beta's 40 providers that is free. At 100k it is a full scan plus an +// unindexable sort, on every poll. +// +// The ordering makes the split possible. `NULLS FIRST` means every never-probed +// provider sorts ahead of every probed one, so the result is always the +// concatenation of two independently ordered groups: +// +// 1. never probed -- no provider_egress_location row at all. This is the +// dominant group (it is why the ordering is NULLS FIRST), and within it +// every observed_at is equally absent, so the order is client_id alone. As +// an anti-join with no outer-joined column in the ORDER BY it is an ordered +// index scan over (valid, connected, client_id) with a LIMIT: no sort, and +// it stops as soon as the batch is full. +// 2. stale but probed -- has a row, older than minObservedAt. Only reached +// when pass 1 came up short of the limit. Driven from +// provider_egress_location itself, where observed_at is a real, indexable +// column: an ordered range scan over (observed_at, client_id). +// +// Both passes carry the same eligibility predicates, so the concatenation is +// row-for-row what the single statement returned, in the same order, under the +// same limit. `attempt_at IS NULL OR attempt_at < $n` becomes the equivalent +// `NOT EXISTS (... AND $n <= attempt_at)` -- equivalent because client_id is the +// primary key of provider_egress_probe_attempt, so there is at most one row to +// quantify over. The same holds for `observed_at IS NULL` on +// provider_egress_location, whose client_id is likewise a primary key and whose +// observed_at is NOT NULL: the only way that test is true is that no row exists. func GetProviderEgressLocationDue( ctx context.Context, minObservedAt time.Time, From 927f225bd22f090ca72d73822bee9393bbc113d7 Mon Sep 17 00:00:00 2001 From: Ryan Mello Date: Mon, 27 Jul 2026 03:23:25 +0100 Subject: [PATCH 18/30] perf(model): split the provider-egress due query so the common case avoids a sort GetProviderEgressLocationDue scanned network_client_location_reliability, added two LEFT JOINs and an EXISTS, then sorted on observed_at from an outer-joined table. That sort cannot use an index: the column being ordered on does not exist for most of the rows being ordered. At beta's 40 providers it is free. At 100k it is a full scan plus an unindexable sort on every poll. Measured on a 100k-provider dataset (30% probed, 5% recently attempted): before Parallel Seq Scan over all 100k rows, hash-joins both tables, quicksort of ~94k rows (3.3MB), 2282 shared buffers, 62.9 ms after pass 1: Index Only Scan + two Merge Anti Joins, no sort, 138 rows touched, 18 shared buffers, 0.34 ms pass 2: Index Only Scan on the new index, no sort, 350 shared buffers, 1.6 ms The ordering is what makes the split possible. NULLS FIRST means every never-probed provider sorts ahead of every probed one, so the result is always the concatenation of two independently ordered groups: 1. never probed -- the dominant group, which is why the ordering is NULLS FIRST. Within it observed_at is equally absent, so client_id alone orders it, and as an anti-join with no outer-joined column in the ORDER BY it is an ordered index scan over the existing (valid, connected, client_id) index that stops as soon as the batch is full. 2. stale but probed -- only queried when pass 1 came up short of the limit. Driven from provider_egress_location, where observed_at is a real, indexable column. `attempt_at IS NULL OR attempt_at < $n` becomes `NOT EXISTS (... AND $n <= attempt_at)`, equivalent because client_id is the primary key of provider_egress_probe_attempt. Same for `observed_at IS NULL`, whose table also keys on client_id and whose observed_at is NOT NULL -- the only way that test is true is that no row exists. The two passes are separate snapshots, so a client that gains its first egress row between them is screened out rather than handed to the prober twice; the single statement could not do that. Verified row-for-row identical to the single statement on the 100k dataset at limits 0, 1, 2, 50, 100, 1000, 20000, 69000, 69500, 70000, 70500 and 100000 -- the middle four straddle the pass-one/pass-two seam (the never-probed population is ~70138) and the last exhausts the eligible set at 94088. New migration adds (observed_at, client_id) on provider_egress_location so pass 2's predicate and its full ORDER BY, tie-break included, are one ordered index scan; without it the planner adds an Incremental Sort (936 buffers vs 350). Appended, never inserted -- migrations apply by slice index here, so editing or reordering an applied one corrupts live databases. Pass 1 needs no new index. TestGetProviderEgressLocationDueOrderingIsStableAcrossLimits asserts the full ordering and then that every limit from 0 past the end returns exactly its prefix, over a population covering never-probed, stale, fresh, recently-attempted, stale-AND-recently-attempted and non-public. It passes against the old single statement as well as the new pair -- it is a behaviour-preservation test, not a description of the new code -- and fails against a pass 2 that uses the full limit instead of the remainder, and against a pass 2 that drops the attempt backoff. The stale-AND-attempted provider exists because of that second check: without it nothing in the suite covered the only case pass 2's backoff predicate screens, and the broken variant passed. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01QtgqtCmKJRXdsQ5ktiqwkg (cherry picked from commit 37ca6cd7202afbb2327fae5681cdd9342cc9b88e) --- db_migrations.go | 27 +++++ model/provider_egress_location_model.go | 108 +++++++++++++---- model/provider_egress_location_model_test.go | 121 +++++++++++++++++++ 3 files changed, 235 insertions(+), 21 deletions(-) diff --git a/db_migrations.go b/db_migrations.go index 7bcb3344..09d920f9 100644 --- a/db_migrations.go +++ b/db_migrations.go @@ -4450,4 +4450,31 @@ var migrations = []any{ CREATE INDEX IF NOT EXISTS provider_egress_probe_attempt_attempt_at ON provider_egress_probe_attempt (attempt_at) `), + + // serves the stale-but-probed pass of GetProviderEgressLocationDue, which + // drives from provider_egress_location with `observed_at < $n ORDER BY + // observed_at, client_id LIMIT $m`. With client_id in the index the + // predicate and the whole ORDER BY -- tie-break included -- are one ordered + // index scan that stops when the batch is full: no sort, and no heap visit + // to resolve the tie. The pre-existing (observed_at) index alone leaves the + // client_id tie-break to a sort. + // + // The other pass (never-probed) needs no new index: it is an anti-join over + // network_client_location_reliability ordered by client_id, which the + // existing (valid, connected, client_id) index already serves as an ordered + // scan, and both anti-joins plus the provide_key EXISTS are primary-key + // probes. + // + // This supersedes provider_egress_location_observed_at, which is now a + // prefix of it -- including for the RemoveExpiredProviderEgressLocations + // sweep. The redundant index is left in place deliberately: dropping it is a + // separate decision with its own (small) risk, and this migration is meant + // to be purely additive. + // + // Appended, never inserted: migrations here apply by slice index, so + // editing or reordering an already-applied entry corrupts live databases. + newSqlMigration(` + CREATE INDEX IF NOT EXISTS provider_egress_location_observed_at_client_id + ON provider_egress_location (observed_at, client_id) + `), } diff --git a/model/provider_egress_location_model.go b/model/provider_egress_location_model.go index 1bb13eed..88ee8603 100644 --- a/model/provider_egress_location_model.go +++ b/model/provider_egress_location_model.go @@ -695,6 +695,13 @@ func GetProviderEgressLocationDue( ) []server.Id { clientIds := []server.Id{} server.Db(ctx, func(conn server.PgConn) { + // pass 1: never probed. Ordered by client_id alone -- every row in this + // group has no observed_at, so the ORDER BY's leading key is constant + // across it and the tie-break is the whole ordering. + // + // `limit` is passed through as given rather than clamped, so a + // nonsensical limit fails exactly as the single-statement version did + // (LIMIT 0 returns nothing; a negative limit is an error). result, err := conn.Query( ctx, ` @@ -702,12 +709,6 @@ func GetProviderEgressLocationDue( network_client_location_reliability.client_id FROM network_client_location_reliability - LEFT JOIN provider_egress_location ON - provider_egress_location.client_id = network_client_location_reliability.client_id - - LEFT JOIN provider_egress_probe_attempt ON - provider_egress_probe_attempt.client_id = network_client_location_reliability.client_id - WHERE network_client_location_reliability.connected = true AND network_client_location_reliability.valid = true AND @@ -717,35 +718,100 @@ func GetProviderEgressLocationDue( provide_key.client_id = network_client_location_reliability.client_id AND provide_key.provide_mode = $1 ) AND - ( - provider_egress_location.observed_at IS NULL OR - provider_egress_location.observed_at < $2 + NOT EXISTS ( + SELECT 1 FROM provider_egress_location + WHERE + provider_egress_location.client_id = network_client_location_reliability.client_id + ) AND + NOT EXISTS ( + SELECT 1 FROM provider_egress_probe_attempt + WHERE + provider_egress_probe_attempt.client_id = network_client_location_reliability.client_id AND + $2 <= provider_egress_probe_attempt.attempt_at + ) + + ORDER BY network_client_location_reliability.client_id ASC + LIMIT $3 + `, + ProvideModePublic, + minAttemptAt.UTC(), + limit, + ) + server.WithPgResult(result, err, func() { + for result.Next() { + var clientId server.Id + server.Raise(result.Scan(&clientId)) + clientIds = append(clientIds, clientId) + } + }) + + remaining := limit - len(clientIds) + if remaining <= 0 { + // the batch is full from never-probed providers alone, which is the + // steady state until the population has been swept once. The + // single-statement version would have returned exactly these rows + // too: they all sort ahead of anything with an observed_at. + return + } + + // pass 2: stale but probed. Driven from provider_egress_location, so + // observed_at is a real column of the driving table and the ORDER BY is + // an ordered index scan rather than a sort. + result, err = conn.Query( + ctx, + ` + SELECT + provider_egress_location.client_id + FROM provider_egress_location + + INNER JOIN network_client_location_reliability ON + network_client_location_reliability.client_id = provider_egress_location.client_id + + WHERE + provider_egress_location.observed_at < $2 AND + network_client_location_reliability.connected = true AND + network_client_location_reliability.valid = true AND + EXISTS ( + SELECT 1 FROM provide_key + WHERE + provide_key.client_id = provider_egress_location.client_id AND + provide_key.provide_mode = $1 ) AND - ( - provider_egress_probe_attempt.attempt_at IS NULL OR - provider_egress_probe_attempt.attempt_at < $3 + NOT EXISTS ( + SELECT 1 FROM provider_egress_probe_attempt + WHERE + provider_egress_probe_attempt.client_id = provider_egress_location.client_id AND + $3 <= provider_egress_probe_attempt.attempt_at ) - -- never-probed sorts ahead of merely stale: a missing observed_at - -- is infinitely old. client_id breaks the tie so batch composition - -- is deterministic instead of plan-dependent -- otherwise the whole - -- never-probed population ties on NULL and which slice of it the - -- prober gets back under a limit is whatever order the executor - -- happened to produce. + -- oldest probe first, client_id breaking the tie, so batch + -- composition is deterministic instead of plan-dependent ORDER BY - provider_egress_location.observed_at ASC NULLS FIRST, - network_client_location_reliability.client_id ASC + provider_egress_location.observed_at ASC, + provider_egress_location.client_id ASC LIMIT $4 `, ProvideModePublic, minObservedAt.UTC(), minAttemptAt.UTC(), - limit, + remaining, ) server.WithPgResult(result, err, func() { + // the two passes are separate statements and so separate snapshots. + // A provider that gains its first provider_egress_location row + // between them would be never-probed to pass 1 and stale to pass 2; + // the single-statement version could not do that, so screen it out + // rather than hand the prober the same client twice. + seen := map[server.Id]bool{} + for _, clientId := range clientIds { + seen[clientId] = true + } for result.Next() { var clientId server.Id server.Raise(result.Scan(&clientId)) + if seen[clientId] { + continue + } clientIds = append(clientIds, clientId) } }) diff --git a/model/provider_egress_location_model_test.go b/model/provider_egress_location_model_test.go index 892f01b3..f3acd4c3 100644 --- a/model/provider_egress_location_model_test.go +++ b/model/provider_egress_location_model_test.go @@ -2,6 +2,7 @@ package model import ( "context" + "fmt" "slices" "testing" "time" @@ -543,3 +544,123 @@ func TestRemoveExpiredProviderEgressProbeAttempts(t *testing.T) { } }) } + +// GetProviderEgressLocationDue is served by two statements -- never-probed +// first, then stale-but-probed only when the first came up short -- because the +// single-statement form sorts on observed_at from an outer-joined table, which +// cannot use an index and becomes a full scan plus an unindexable sort at 100k +// providers. +// +// The split is only safe if the concatenation is row-for-row what one statement +// returned, at every limit. That is what this asserts: it builds one population +// covering every eligibility case and then walks the limit from 0 past the end, +// requiring each result to be exactly the prefix of the full ordering. Limit 3 +// is the seam (pass one exactly fills the batch) and limit 4 is the first that +// crosses into pass two. +func TestGetProviderEgressLocationDueOrderingIsStableAcrossLimits(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + + city := &Location{ + LocationType: LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, city) + + // three never-probed providers: the dominant group, and the reason the + // ordering is NULLS FIRST + never := []server.Id{server.NewId(), server.NewId(), server.NewId()} + // two stale ones, at different ages -- the older must be handed out first + staleOlder := server.NewId() + staleNewer := server.NewId() + // probed an hour ago: not due + fresh := server.NewId() + // never probed, but attempted seconds ago: deferred by the backoff, and + // the case that would otherwise starve the queue + attempted := server.NewId() + // probed long ago AND attempted seconds ago. This one is only screened + // by the backoff predicate on the stale-but-probed pass -- the + // never-probed pass never sees it, because it has an egress row. Drop + // that predicate and it reappears in the batch. + staleAttempted := server.NewId() + // no Public provide key: unprobeable at any freshness + nonPublic := server.NewId() + + address := 0 + connectProvider := func(clientId server.Id, provideMode ProvideMode) { + address += 1 + testing_connectProbeableProvider( + t, ctx, clientId, city.LocationId, + fmt.Sprintf("0.0.%d.1:0", address), provideMode, + ) + } + for _, clientId := range never { + connectProvider(clientId, ProvideModePublic) + } + connectProvider(staleOlder, ProvideModePublic) + connectProvider(staleNewer, ProvideModePublic) + connectProvider(fresh, ProvideModePublic) + connectProvider(attempted, ProvideModePublic) + connectProvider(staleAttempted, ProvideModePublic) + connectProvider(nonPublic, ProvideModeNetwork) + + UpdateClientLocationReliabilities(ctx, now.Add(-time.Hour), now) + + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: staleOlder, LocationId: city.LocationId, + CountryCode: "us", ObservedAt: now.Add(-100 * time.Hour), + }) + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: staleNewer, LocationId: city.LocationId, + CountryCode: "us", ObservedAt: now.Add(-50 * time.Hour), + }) + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: fresh, LocationId: city.LocationId, + CountryCode: "us", ObservedAt: now.Add(-1 * time.Hour), + }) + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: staleAttempted, LocationId: city.LocationId, + CountryCode: "us", ObservedAt: now.Add(-200 * time.Hour), + }) + SetProviderEgressProbeAttempt(ctx, &ProviderEgressProbeAttempt{ + ClientId: attempted, AttemptAt: now.Add(-5 * time.Second), + ProbeFailure: "tunnel_failed", + }) + // oldest observed_at of all, so it would sort to the head of the + // stale group if the backoff did not exclude it + SetProviderEgressProbeAttempt(ctx, &ProviderEgressProbeAttempt{ + ClientId: staleAttempted, AttemptAt: now.Add(-5 * time.Second), + ProbeFailure: "tunnel_failed", + }) + + minObservedAt := now.Add(-24 * time.Hour) + minAttemptAt := now.Add(-ProviderEgressProbeAttemptBackoff) + + // the never-probed group ties on a missing observed_at, so client_id + // alone orders it -- and postgres orders uuid by bytes, which is what + // server.Id.Cmp does + expected := slices.Clone(never) + slices.SortFunc(expected, func(a server.Id, b server.Id) int { return a.Cmp(b) }) + // ... then the probed group, oldest probe first + expected = append(expected, staleOlder, staleNewer) + + due := GetProviderEgressLocationDue(ctx, minObservedAt, minAttemptAt, 100) + if !slices.Equal(due, expected) { + t.Fatalf("due = %v, want %v (never-probed by client_id, then stale oldest-first; fresh/attempted/stale-attempted/non-public excluded)", due, expected) + } + + // every limit must return exactly the prefix of that ordering. limit 3 + // is the pass-one/pass-two seam; 4 is the first to cross it. + for limit := 0; limit <= len(expected)+2; limit += 1 { + want := expected[:min(limit, len(expected))] + got := GetProviderEgressLocationDue(ctx, minObservedAt, minAttemptAt, limit) + if !slices.Equal(got, want) { + t.Errorf("limit %d: due = %v, want %v", limit, got, want) + } + } + }) +} From 6913b87afa855926991fe6423bd9411c7ad350ca Mon Sep 17 00:00:00 2001 From: ryanmello07 Date: Mon, 27 Jul 2026 06:09:19 +0100 Subject: [PATCH 19/30] fix(egress): fold accents and district qualifiers, and never coarsen a probed location MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two review findings on the provider-egress probe ingest path. 1. The location-name normaliser dropped the cases it exists for. normalizeLocationName kept accented letters, so "São Paulo" did not fold onto "Sao Paulo", "Zürich" onto "Zurich", or "Kraków" onto "Krakow" -- every one of those missed and fell back to country. The motivating example missed too: "Frankfurt am Main (Innenstadt I)" folded to "frankfurtammaininnenstadti", because dropping "(" and ")" as punctuation leaves the qualifier's letters in the key. The fold now NFD-decomposes and drops the combining marks (unicode.Mn), which covers the whole accent class instead of a hand-rolled é->e table that would keep missing whatever it forgot. golang.org/x/text was already a dependency of this module and is now required directly; the stdlib-only constraint the previous revision was written under belongs to the prober repo's `geolocate` package, not to the server. matchLocationName gains a third pass that strips parenthesised qualifiers from both sides. That pass is the only one that can plausibly land on the wrong row -- "Springfield (IL)" and "Springfield (MA)" both reduce to "springfield" -- so it requires the stripped key to identify exactly one candidate and declines on any ambiguity. Falling back to country is the safe outcome; guessing is not. Letters carrying a stroke rather than a combining mark (Å‚, ø, Ä‘) still do not fold. That is the documented residual: closing it needs the transliteration table this deliberately avoids, and the failure mode is country granularity, not a wrong answer. The existing variant test only covered foldings the implementation already handled, which is why it could not catch any of this. It now carries the qualifier cases, and accents get their own test covering both directions -- accent on the probe, accent on the stored row -- since which side carries it depends on how the row happened to be seeded. Both fail against the previous matcher. 2. An unmatched city made a provider worse off than never being probed. SubmitProviderEgressLocation stores country granularity whenever the probed city does not match an existing location row, and SetConnectionLocation then let that probed row win unconditionally over the mmdb lookup. A provider the mmdb had placed in a city was therefore demoted to a country row and disappeared from every city filter: being probed made it less discoverable than being ignored. This is the common case, not a rare one. Cities are not seeded -- AddDefaultLocations runs with cityLimit = 0 -- so a probed city can only match rows organic traffic already created. SetConnectionLocation now resolves the mmdb answer first (an in-process mmdb/ARIN read, no db round trip) and asks probedLocationPreferred whether replacing it is actually an improvement. The rule is that a probe may correct a location but never coarsen it: - a city-confident probe wins; it loses nothing; - a probe wins when mmdb has no answer, or only a country; - a probe wins when mmdb's city is in a DIFFERENT country -- that city is not more precise, it is precisely wrong, and correcting it is the entire point of the feature; - mmdb wins when its city or region is in the SAME country the probe reports, because the probe then adds nothing but a loss of granularity. The reasoning is written out at the function, because the naive reading ("a probe is better evidence, so it always wins") is what produced the regression. Three tests cover it: the anti-coarsening case fails against the old unconditional preference, and the two correction cases fail against a rule that merely keeps whichever answer is finer. Fix 1 materially reduces how often fix 2 is reached, but neither depends on the other. Co-Authored-By: Claude Claude-Session: https://claude.ai/code/session_01QtgqtCmKJRXdsQ5ktiqwkg --- controller/network_client_controller.go | 105 +++++++-- controller/network_client_controller_test.go | 208 ++++++++++++++++++ controller/probed_location_preferred_test.go | 75 +++++++ .../provider_egress_location_controller.go | 6 +- ...rovider_egress_location_controller_test.go | 198 ++++++++++++++++- go.mod | 2 +- model/provider_egress_location_model.go | 123 +++++++++-- 7 files changed, 677 insertions(+), 40 deletions(-) create mode 100644 controller/probed_location_preferred_test.go diff --git a/controller/network_client_controller.go b/controller/network_client_controller.go index 4a723b6e..3556cdb1 100644 --- a/controller/network_client_controller.go +++ b/controller/network_client_controller.go @@ -3,7 +3,7 @@ package controller import ( "context" "net/netip" - // "strings" + "strings" "time" "github.com/urnetwork/glog" @@ -57,13 +57,24 @@ func SetConnectionLocation( connectionId server.Id, clientIp string, ) error { + // the mmdb lookup on the control ip. This is resolved up front even when + // the probed path is about to win, because the probed path has to know how + // precise the mmdb answer is before it can decide whether replacing it is + // an improvement (see probedLocationPreferred), and because it costs no db + // round trip -- it is an in-process maxminddb read plus an ARIN lookup. + // `err` is deliberately not returned yet: a failed mmdb lookup is not a + // reason to discard a perfectly good probed location. + location, connectionLocationScores, err := GetLocationForIp(ctx, clientIp) + // a provider probed through its own egress is located from that probe, not // from a lookup on its control-connection ip: the egress is where user // traffic actually exits, and an operator-run prober learns it by routing // geolocation lookups through the provider itself and cross-checking them // across several sources, then submits the result here. When a fresh // probed entry exists we prefer it over the built-in mmdb lookup on the - // control ip. GetFreshProviderEgressLocationForConnection is + // control ip -- subject to probedLocationPreferred, which is what stops the + // probe making a provider *less* locatable than it was. + // GetFreshProviderEgressLocationForConnection is // a single query joining network_client_connection to // provider_egress_location: this runs for every connection (provider or // not) on the connect-announce path and inside a retry loop, so it must @@ -73,7 +84,7 @@ func SetConnectionLocation( ctx, connectionId, model.ProviderEgressLocationMaxAge, - ); egress != nil { + ); egress != nil && probedLocationPreferred(egress, location) { scores := &model.ConnectionLocationScores{} if egress.Hosting { scores.NetTypeHosting = 1 @@ -92,30 +103,34 @@ func SetConnectionLocation( // the same parity reasoning applied to net_type_foreign). Mobile // stays on the model/wire contract as metadata; it just does not // feed the ranking score. + // keep the ARIN org-vs-country foreign check on the probed path too, // so a probed provider is ranked on equal terms with an equivalent - // unprobed one (net_type_foreign feeds the ranking columns). Compute - // it exactly as the mmdb path does in GetLocationForIp: the ARIN org - // country of the control ip against the mmdb country of that SAME - // control ip -- not the probed country, which is a different + // unprobed one (net_type_foreign feeds the ranking columns). It is + // computed exactly as the mmdb path computes it in GetLocationForIp: + // the ARIN org country of the control ip against the mmdb country of + // that SAME control ip -- not the probed country, which is a different // question (whether probing changed the answer) and must not be - // silently folded into this ranking penalty. Any lookup failure - // just leaves NetTypeForeign at 0; it must never fail or panic this - // path. + // silently folded into this ranking penalty. It is recomputed here + // rather than lifted off connectionLocationScores because that struct + // is nil whenever GetLocationForIp failed, including the case where + // mmdb resolved the ip fine but GuessLocationType could not classify + // the result -- the foreign check is still meaningful there. Any + // lookup failure just leaves NetTypeForeign at 0; it must never fail + // or panic this path. if addr, err := netip.ParseAddr(clientIp); err == nil { if ipInfo, err := server.GetIpInfo(addr); err == nil { scores.NetTypeForeign = arinForeignScore(addr, ipInfo.CountryCode) } } - err := model.SetConnectionLocation(ctx, connectionId, egress.LocationId, scores) - if err == nil { + setErr := model.SetConnectionLocation(ctx, connectionId, egress.LocationId, scores) + if setErr == nil { return nil } // fall through to the mmdb path on a storage error - glog.Infof("[ncc][%s]could not set probed egress location. err = %s\n", connectionId, err) + glog.Infof("[ncc][%s]could not set probed egress location. err = %s\n", connectionId, setErr) } - location, connectionLocationScores, err := GetLocationForIp(ctx, clientIp) if err != nil { // server.Logger().Printf("Get ip for location error: %s", err) glog.Infof("[ncc][%s]could not find client location. err = %s\n", connectionId, err) @@ -132,6 +147,68 @@ func SetConnectionLocation( return nil } +// probedLocationPreferred decides whether the probed egress location should be +// written for this connection instead of the mmdb location resolved from the +// control ip. mmdbLocation is nil when the mmdb lookup failed. +// +// Read this before changing it: the naive rule -- "a probe is better evidence +// than mmdb, so the probe always wins" -- is wrong, and produced a live +// regression. The probed location is not always as *precise* as the mmdb one. +// SubmitProviderEgressLocation only stores a city when the probed city matches +// a location row that already exists; anything else is stored at country +// granularity, deliberately, so that a probe can never mint new city rows in +// the shared `location` table. Cities are not seeded either -- AddDefaultLocations +// runs with cityLimit = 0 -- so the pool a probed city can match against is only +// the rows organic traffic happened to create, and a country-granular fallback +// is the common case, not a rare one. +// +// Letting that country row overwrite an mmdb *city* row would drop the provider +// out of every city filter in FindProviders2 and GetProviderLocations. Being +// probed would make a provider less discoverable than never having been probed +// at all -- a penalty for participating. +// +// So the rule is: a probe may CORRECT the location, but never COARSEN it. +// +// - The probe stored a city (CityConfident): it is at least as precise as +// anything mmdb has, and it is better evidence. It wins. +// - No usable mmdb answer: the probe is the only evidence there is. It wins. +// - The mmdb answer is itself country-granular: nothing to lose. The probe +// wins, and this is the case country-level correction exists for. +// - The mmdb answer is city- or region-granular in a DIFFERENT country: the +// mmdb row is not more precise, it is precisely wrong, and its city is a +// city in the wrong country. The probe wins. This is the other half of +// country-level correction and the reason this is not just "keep whichever +// is finer". +// - The mmdb answer is city- or region-granular in the SAME country the probe +// reports: the probe agrees with mmdb and adds nothing except a loss of +// granularity. mmdb wins. +// +// CityConfident is used as the probed row's granularity because the schema +// invariant is that provider_egress_location.location_id is a city row exactly +// when city_confident is set (see the provider_egress_location migration and +// SubmitProviderEgressLocation). Reading it off the flag keeps this on the hot +// connect-announce path without a second query for the location row's type. +func probedLocationPreferred( + egress *model.ProviderEgressLocation, + mmdbLocation *model.Location, +) bool { + if egress.CityConfident { + return true + } + if mmdbLocation == nil { + return true + } + if mmdbLocation.LocationType != model.LocationTypeCity && + mmdbLocation.LocationType != model.LocationTypeRegion { + return true + } + // both country codes are stored lowercased -- SubmitProviderEgressLocation + // lowercases the probed one and GetLocationForIp takes mmdb's, which the + // location table also stores lowercased -- but fold anyway rather than let + // a casing difference read as a country disagreement and silently coarsen. + return !strings.EqualFold(egress.CountryCode, mmdbLocation.CountryCode) +} + /* func SetMissingConnectionLocations(ctx context.Context, minTime time.Time) { connectionIpStrs := map[server.Id]string{} diff --git a/controller/network_client_controller_test.go b/controller/network_client_controller_test.go index f6561fb1..1276bf9d 100644 --- a/controller/network_client_controller_test.go +++ b/controller/network_client_controller_test.go @@ -333,3 +333,211 @@ func TestSetConnectionLocationMapsProbedFlagsToScores(t *testing.T) { connect.AssertEqual(t, netTypeVirtual, 0) }) } + +// testing_connectionLocationIds reads back the granularity actually stored for +// a connection. +func testing_connectionLocationIds(ctx context.Context, connectionId server.Id) ( + cityLocationId server.Id, + regionLocationId server.Id, + countryLocationId server.Id, +) { + server.Db(ctx, func(conn server.PgConn) { + result, qerr := conn.Query( + ctx, + ` + SELECT city_location_id, region_location_id, country_location_id + FROM network_client_location + WHERE connection_id = $1 + `, + connectionId, + ) + server.WithPgResult(result, qerr, func() { + if result.Next() { + server.Raise(result.Scan( + &cityLocationId, + ®ionLocationId, + &countryLocationId, + )) + } + }) + }) + return +} + +// A probe must never make a provider LESS locatable than leaving it unprobed +// would have. +// +// SubmitProviderEgressLocation stores country granularity whenever the probed +// city does not match a location row that already exists, which is the common +// case -- cities are not seeded, so the match pool is only what organic traffic +// created. If that country row then overwrote the mmdb city row unconditionally, +// the provider would fall out of every city filter: probed providers would be +// harder to find than unprobed ones, which is the exact opposite of the point. +// +// mmdb resolves 24.48.0.1 to Montreal, Quebec, CA at city granularity. A +// country-only probe agreeing that the provider is in CA adds nothing, so the +// mmdb city has to survive. +func TestSetConnectionLocationProbedCountryDoesNotCoarsenMmdbCity(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + clientIp := "24.48.0.1" + + mmdbLocation, _, err := GetLocationForIp(ctx, clientIp) + connect.AssertEqual(t, err, nil) + // the fixture is only meaningful if mmdb really has a city here + connect.AssertEqual(t, mmdbLocation.LocationType, model.LocationTypeCity) + model.CreateLocation(ctx, mmdbLocation) + + // the probed location: the same country, but only the country -- + // exactly what an unmatched city name falls back to + probed := &model.Location{ + LocationType: model.LocationTypeCountry, + Country: "Canada", + CountryCode: "ca", + } + model.CreateLocation(ctx, probed) + connect.AssertEqual(t, probed.LocationId, mmdbLocation.CountryLocationId) + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientIp+":0", handlerId) + connect.AssertEqual(t, err, nil) + + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: clientId, + LocationId: probed.LocationId, + CountryCode: "ca", + CityConfident: false, + ObservedAt: server.NowUtc(), + }) + + err = SetConnectionLocation(ctx, connectionId, clientIp) + connect.AssertEqual(t, err, nil) + + cityLocationId, regionLocationId, countryLocationId := testing_connectionLocationIds(ctx, connectionId) + if cityLocationId != mmdbLocation.CityLocationId { + t.Errorf( + "city_location_id = %s, want the mmdb city %s: a country-only probe must not coarsen a connection the mmdb already placed in a city", + cityLocationId, + mmdbLocation.CityLocationId, + ) + } + connect.AssertEqual(t, regionLocationId, mmdbLocation.RegionLocationId) + connect.AssertEqual(t, countryLocationId, mmdbLocation.CountryLocationId) + // and the collapse this guards against, stated directly + if cityLocationId == countryLocationId { + t.Errorf("city/region/country all collapsed to %s; the provider is no longer in any city filter", countryLocationId) + } + }) +} + +// The other half of the rule: country-level CORRECTION still wins. When the +// probe says the egress is in a different country than the control ip's mmdb +// city, that city is a city in the wrong country -- it is not more precise, it +// is precisely wrong -- so the probed country replaces it. This is the whole +// reason the feature exists and must not be lost to the anti-coarsening rule +// above. +func TestSetConnectionLocationProbedCountryCorrectsMmdbCityInAnotherCountry(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + clientIp := "24.48.0.1" + + mmdbLocation, _, err := GetLocationForIp(ctx, clientIp) + connect.AssertEqual(t, err, nil) + connect.AssertEqual(t, mmdbLocation.LocationType, model.LocationTypeCity) + connect.AssertEqual(t, mmdbLocation.CountryCode, "ca") + model.CreateLocation(ctx, mmdbLocation) + + probed := &model.Location{ + LocationType: model.LocationTypeCountry, + Country: "Japan", + CountryCode: "jp", + } + model.CreateLocation(ctx, probed) + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientIp+":0", handlerId) + connect.AssertEqual(t, err, nil) + + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: clientId, + LocationId: probed.LocationId, + CountryCode: "jp", + CityConfident: false, + ObservedAt: server.NowUtc(), + }) + + err = SetConnectionLocation(ctx, connectionId, clientIp) + connect.AssertEqual(t, err, nil) + + _, _, countryLocationId := testing_connectionLocationIds(ctx, connectionId) + if countryLocationId != probed.CountryLocationId { + t.Errorf( + "country_location_id = %s, want the probed country %s: a probe that disagrees with mmdb about the country must still correct it", + countryLocationId, + probed.CountryLocationId, + ) + } + if countryLocationId == mmdbLocation.CountryLocationId { + t.Errorf("kept the mmdb country %s; the provider is still advertised in the wrong country", mmdbLocation.CountryLocationId) + } + }) +} + +// A city-confident probe is at least as precise as anything mmdb has and is +// better evidence, so it wins outright -- including against an mmdb city in +// another country. Nothing is coarsened, so the anti-coarsening rule does not +// apply. +func TestSetConnectionLocationProbedCityWinsOverMmdbCity(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + clientIp := "24.48.0.1" + + mmdbLocation, _, err := GetLocationForIp(ctx, clientIp) + connect.AssertEqual(t, err, nil) + connect.AssertEqual(t, mmdbLocation.LocationType, model.LocationTypeCity) + model.CreateLocation(ctx, mmdbLocation) + + probed := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Tokyo", + Region: "Tokyo", + Country: "Japan", + CountryCode: "jp", + } + model.CreateLocation(ctx, probed) + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientIp+":0", handlerId) + connect.AssertEqual(t, err, nil) + + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: clientId, + LocationId: probed.LocationId, + CountryCode: "jp", + CityConfident: true, + ObservedAt: server.NowUtc(), + }) + + err = SetConnectionLocation(ctx, connectionId, clientIp) + connect.AssertEqual(t, err, nil) + + cityLocationId, _, countryLocationId := testing_connectionLocationIds(ctx, connectionId) + connect.AssertEqual(t, cityLocationId, probed.CityLocationId) + connect.AssertEqual(t, countryLocationId, probed.CountryLocationId) + }) +} diff --git a/controller/probed_location_preferred_test.go b/controller/probed_location_preferred_test.go new file mode 100644 index 00000000..33ac26a2 --- /dev/null +++ b/controller/probed_location_preferred_test.go @@ -0,0 +1,75 @@ +package controller + +import ( + "testing" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" +) + +// The decision rule on its own, with no database and no mmdb file: a probe may +// CORRECT a location but must never COARSEN it. The end-to-end behaviour is +// covered by the TestSetConnectionLocation* tests; this pins the rule itself so +// each clause is exercised independently of whichever mmdb the environment has. +// +// Every row here fails against at least one plausible wrong rule -- "the probe +// always wins" (the bug), or "keep whichever answer is finer" (the overcorrection +// that would throw away country correction). +func TestProbedLocationPreferred(t *testing.T) { + probedCity := func(countryCode string) *model.ProviderEgressLocation { + return &model.ProviderEgressLocation{ + LocationId: server.NewId(), + CountryCode: countryCode, + CityConfident: true, + } + } + probedCountry := func(countryCode string) *model.ProviderEgressLocation { + return &model.ProviderEgressLocation{ + LocationId: server.NewId(), + CountryCode: countryCode, + CityConfident: false, + } + } + mmdb := func(locationType string, countryCode string) *model.Location { + return &model.Location{ + LocationId: server.NewId(), + LocationType: locationType, + CountryCode: countryCode, + } + } + + tests := []struct { + name string + egress *model.ProviderEgressLocation + mmdb *model.Location + want bool + }{ + // the regression: a country-only probe must not replace an mmdb city + // in the same country. This is the row that fails against "the probe + // always wins". + {"country probe vs mmdb city, same country", probedCountry("ca"), mmdb(model.LocationTypeCity, "ca"), false}, + {"country probe vs mmdb region, same country", probedCountry("ca"), mmdb(model.LocationTypeRegion, "ca"), false}, + // casing must not read as a country disagreement + {"country probe vs mmdb city, same country cased", probedCountry("CA"), mmdb(model.LocationTypeCity, "ca"), false}, + + // country correction: these fail against "keep whichever is finer" + {"country probe vs mmdb city, other country", probedCountry("jp"), mmdb(model.LocationTypeCity, "ca"), true}, + {"country probe vs mmdb region, other country", probedCountry("jp"), mmdb(model.LocationTypeRegion, "ca"), true}, + + // nothing to lose + {"country probe vs mmdb country, same", probedCountry("ca"), mmdb(model.LocationTypeCountry, "ca"), true}, + {"country probe vs mmdb country, other", probedCountry("jp"), mmdb(model.LocationTypeCountry, "ca"), true}, + {"country probe vs no mmdb answer", probedCountry("jp"), nil, true}, + + // a city-confident probe is never a downgrade + {"city probe vs mmdb city, same country", probedCity("ca"), mmdb(model.LocationTypeCity, "ca"), true}, + {"city probe vs mmdb city, other country", probedCity("jp"), mmdb(model.LocationTypeCity, "ca"), true}, + {"city probe vs mmdb region", probedCity("ca"), mmdb(model.LocationTypeRegion, "ca"), true}, + {"city probe vs no mmdb answer", probedCity("jp"), nil, true}, + } + for _, test := range tests { + if got := probedLocationPreferred(test.egress, test.mmdb); got != test.want { + t.Errorf("%s: probedLocationPreferred = %v, want %v", test.name, got, test.want) + } + } +} diff --git a/controller/provider_egress_location_controller.go b/controller/provider_egress_location_controller.go index c8c64832..8cac0dcc 100644 --- a/controller/provider_egress_location_controller.go +++ b/controller/provider_egress_location_controller.go @@ -141,8 +141,10 @@ func SubmitProviderEgressLocation( // survive a code revert and there is no cleanup path. // // model.MatchExistingLocation therefore matches only, never creates, - // case-insensitively and ignoring punctuation/whitespace so the ordinary - // variants land on the row that is already there. When it does not resolve, + // case-insensitively and ignoring punctuation/whitespace/accents and + // parenthesised district qualifiers, so the ordinary variants -- the + // "(Innenstadt I)" case above included -- land on the row that is already + // there. When it does not resolve, // this submission falls back to country granularity: country is the // granularity this design treats as trustworthy anyway, and losing city // precision for one probe is strictly better than permanently polluting a diff --git a/controller/provider_egress_location_controller_test.go b/controller/provider_egress_location_controller_test.go index 31c7d1cb..45b28d22 100644 --- a/controller/provider_egress_location_controller_test.go +++ b/controller/provider_egress_location_controller_test.go @@ -2,6 +2,7 @@ package controller import ( "context" + "fmt" "strings" "testing" "time" @@ -494,10 +495,17 @@ func TestSubmitProviderEgressLocationUnknownCityDoesNotCreateALocation(t *testin }) } -// The variants that matter are trivial: the same city spelled with different -// case, punctuation or spacing. Those must resolve to the row that is already -// there -- discarding them to country would throw away real precision for no -// reason, and creating a row for each is the bug this guards against. +// The variants that matter are the ones the geolocation sources actually +// produce for one place: different case, punctuation or spacing, and -- the +// case this feature was built for -- a parenthesised district qualifier +// ("Frankfurt am Main (Innenstadt I)"). Those must resolve to the row that is +// already there; discarding them to country throws away real precision, and +// creating a row for each is the bug this guards against. +// +// An earlier revision of this list only contained foldings the implementation +// already handled, so it could not fail. The qualifier case below fails against +// a matcher that only drops punctuation, which is what makes this test worth +// running. Diacritics -- the other class that failed -- have their own test. func TestSubmitProviderEgressLocationMatchesCitySpellingVariant(t *testing.T) { server.DefaultTestEnv().Run(t, func(t testing.TB) { ctx := context.Background() @@ -524,8 +532,17 @@ func TestSubmitProviderEgressLocationMatchesCitySpellingVariant(t *testing.T) { {"hesse", "FRANKFURT AM MAIN"}, // case, both levels {"Hesse", "Frankfurt-am-Main"}, // punctuation {"Hesse", " Frankfurt am Main "}, + // the observed disagreement that motivated matching at all: one + // source appends the district. Dropping "(" and ")" as punctuation + // is not enough -- the qualifier's letters stay in the key -- so + // this misses unless the qualifier itself is stripped. + {"Hesse", "Frankfurt am Main (Innenstadt I)"}, + {"Hesse", "Frankfurt am Main (Innenstadt I) "}, + // qualifier on the region too + {"Hesse (Regierungsbezirk Darmstadt)", "Frankfurt am Main"}, } for _, variant := range variants { + label := fmt.Sprintf("%q/%q", variant.region, variant.city) clientId := server.NewId() model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") @@ -543,13 +560,13 @@ func TestSubmitProviderEgressLocationMatchesCitySpellingVariant(t *testing.T) { stored := model.GetProviderEgressLocation(ctx, clientId) if stored == nil { - t.Fatalf("%q: expected the submission to be stored", variant.city) + t.Fatalf("%s: expected the submission to be stored", label) } if stored.LocationId != frankfurt.LocationId { - t.Errorf("%q resolved to %s, want the existing Frankfurt row %s", variant.city, stored.LocationId, frankfurt.LocationId) + t.Errorf("%s resolved to %s, want the existing Frankfurt row %s", label, stored.LocationId, frankfurt.LocationId) } if !stored.CityConfident { - t.Errorf("%q: city_confident must stay set when the city resolved", variant.city) + t.Errorf("%s: city_confident must stay set when the city resolved", label) } } @@ -558,3 +575,170 @@ func TestSubmitProviderEgressLocationMatchesCitySpellingVariant(t *testing.T) { } }) } + +// Diacritics are the largest single class of spelling disagreement between the +// geolocation sources: one emits the local spelling ("São Paulo", "Zürich", +// "Kraków"), another an ASCII transliteration ("Sao Paulo", "Zurich", +// "Krakow"), and the mmdb import that seeded the existing rows picked one of +// the two per city with no way to know which. Every one of these missed before +// the NFD fold and fell back to country -- and, per +// TestSetConnectionLocationProbedCountryDoesNotCoarsenMmdbCity, a fallback on a +// provider the mmdb already placed in a city used to be an outright +// discoverability regression. +// +// Both directions are exercised, because which side carries the accent depends +// on how the existing row happened to be seeded. +func TestSubmitProviderEgressLocationMatchesAccentedCityVariant(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + + // existing rows, seeded the way the mmdb import would have: some + // accented, some already transliterated + saoPaulo := &model.Location{ + LocationType: model.LocationTypeCity, + City: "São Paulo", + Region: "São Paulo", + Country: "Brazil", + CountryCode: "br", + } + model.CreateLocation(ctx, saoPaulo) + + zurich := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Zürich", + Region: "Zürich", + Country: "Switzerland", + CountryCode: "ch", + } + model.CreateLocation(ctx, zurich) + + // seeded WITHOUT the accent, so the probe is the side carrying it + krakow := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Krakow", + Region: "Lesser Poland", + Country: "Poland", + CountryCode: "pl", + } + model.CreateLocation(ctx, krakow) + + before := testing_countLocations(ctx) + + variants := []struct { + countryCode string + country string + region string + city string + want *model.Location + }{ + // accent dropped by the probing source + {"BR", "Brazil", "Sao Paulo", "Sao Paulo", saoPaulo}, + {"CH", "Switzerland", "Zurich", "Zurich", zurich}, + // accent present in the probe, absent from the stored row + {"PL", "Poland", "Lesser Poland", "Kraków", krakow}, + // accent on one level only + {"BR", "Brazil", "São Paulo", "Sao Paulo", saoPaulo}, + // accent plus the other foldings at once + {"CH", "Switzerland", "zurich", " ZURICH ", zurich}, + } + for _, variant := range variants { + label := fmt.Sprintf("%q/%q", variant.region, variant.city) + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: variant.countryCode, + Country: variant.country, + Region: variant.region, + City: variant.city, + CountryConfident: true, + CityConfident: true, + ObservedAt: server.NowUtc(), + }) + connect.AssertEqual(t, err, nil) + + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatalf("%s: expected the submission to be stored", label) + } + if stored.LocationId != variant.want.LocationId { + t.Errorf("%s resolved to %s, want the existing row %s", label, stored.LocationId, variant.want.LocationId) + } + if !stored.CityConfident { + t.Errorf("%s: city_confident must stay set when the city resolved", label) + } + } + + if after := testing_countLocations(ctx); after != before { + t.Errorf("location row count went from %d to %d; accented variants must reuse the existing row, not add new ones", before, after) + } + }) +} + +// The qualifier-stripping pass is the only one that can pick the wrong row, so +// it must decline rather than guess. Two same-region rows that differ ONLY in +// their parenthesised qualifier both reduce to the same key; a probe carrying a +// third qualifier matches neither exactly and must fall back to country instead +// of silently landing on whichever row the candidate ordering put first. +func TestSubmitProviderEgressLocationAmbiguousQualifierFallsBackToCountry(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + springfieldIl := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Springfield (IL)", + Region: "Midwest", + Country: "United States", + CountryCode: "us", + } + model.CreateLocation(ctx, springfieldIl) + + springfieldMa := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Springfield (MA)", + Region: "Midwest", + Country: "United States", + CountryCode: "us", + } + model.CreateLocation(ctx, springfieldMa) + + before := testing_countLocations(ctx) + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "US", + Country: "United States", + Region: "Midwest", + City: "Springfield (OH)", + CountryConfident: true, + CityConfident: true, + ObservedAt: server.NowUtc(), + }) + connect.AssertEqual(t, err, nil) + + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected the submission to be stored") + } + if stored.CityConfident { + t.Errorf("an ambiguous qualifier must not resolve to a city; got city_confident with location %s", stored.LocationId) + } + if stored.LocationId == springfieldIl.LocationId || stored.LocationId == springfieldMa.LocationId { + t.Errorf("guessed a Springfield (%s) instead of falling back to country", stored.LocationId) + } + // the country fallback goes through CreateLocation, but the US country + // row already exists (the two Springfield fixtures created it), and + // CreateLocation dedupes a country on country_code -- so declining to + // guess must add no rows at all, least of all a third Springfield + if after := testing_countLocations(ctx); after != before { + t.Errorf("location row count went from %d to %d; the fallback must reuse the existing country row and never create a city", before, after) + } + }) +} diff --git a/go.mod b/go.mod index e4bddacf..284749b2 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( golang.org/x/crypto v0.53.0 golang.org/x/net v0.56.0 golang.org/x/sys v0.46.0 + golang.org/x/text v0.40.0 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 @@ -135,7 +136,6 @@ require ( golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/term v0.44.0 // indirect - golang.org/x/text v0.40.0 // indirect golang.org/x/time v0.15.0 // indirect gopkg.in/ini.v1 v1.67.2 // indirect src.agwa.name/tlshacks v0.0.3 // indirect diff --git a/model/provider_egress_location_model.go b/model/provider_egress_location_model.go index 88ee8603..c0ad7780 100644 --- a/model/provider_egress_location_model.go +++ b/model/provider_egress_location_model.go @@ -6,6 +6,8 @@ import ( "time" "unicode" + "golang.org/x/text/unicode/norm" + "github.com/urnetwork/server" ) @@ -367,28 +369,52 @@ func GetLocation(ctx context.Context, locationId server.Id) *Location { } // normalizeLocationName folds a location name to a comparison key: lowercased, -// with every rune that is not a letter or a digit dropped. So +// accent-stripped, with every rune that is not a letter or a digit dropped. So // "Frankfurt am Main", "Frankfurt Am Main" and "FRANKFURT AM MAIN" all fold to -// "frankfurtammain" and match the one row that already exists. +// "frankfurtammain", and "São Paulo", "Zürich" and "Kraków" fold to the same +// keys as "Sao Paulo", "Zurich" and "Krakow". // // This is deliberately a comparison key only -- it is never stored, and never // used to build a location_name. It exists so a trivial spelling variant from a // geolocation source resolves to the existing row instead of being treated as a // different place. // +// Diacritics are the single biggest source of these variants: the free +// geolocation sources disagree over whether to emit the local spelling or an +// ASCII transliteration for the same city, and the mmdb import that seeded most +// existing rows made its own choice. Folding is done with an NFD decomposition +// followed by dropping the combining marks (unicode.Mn), which covers the whole +// accent class at once -- a hand-rolled é->e table would have to enumerate the +// world's diacritics and would silently keep missing the ones it forgot. +// +// golang.org/x/text is already a dependency of this module, so this costs no +// new supply-chain surface. (An earlier revision of this function was +// stdlib-only by mistake: that constraint belongs to the prober repo's +// `geolocate` package, not to the server.) +// +// Letters that carry a stroke rather than a combining mark -- Å‚, ø, Ä‘ -- do not +// decompose and therefore still do not fold onto l/o/d. Those fall back to +// country granularity, which is the safe outcome; the alternative is the +// transliteration table this function deliberately avoids. +// // Punctuation is dropped rather than mapped to a space because the disagreement // is over whether the separator exists at all ("Washington, D.C." vs // "Washington DC"). Note this deliberately does not fold "Frankfurt/Main" onto // "Frankfurt am Main": dropping the separator gives "frankfurtmain" != // "frankfurtammain", so that one falls back to country granularity rather than // matching the wrong row. Falling back is the safe outcome; guessing is not. -// Stdlib only, by design -- a transliteration/fuzzy-match dependency is a large -// amount of new behaviour to take on for an ingest path whose failure mode is -// already "use the country". func normalizeLocationName(name string) string { + // NFD splits a precomposed letter ("ü") into its base letter plus a + // combining mark ("u" + U+0308). The mark is category Mn, which is neither + // a letter nor a digit, so the filter below drops it; the explicit Mn skip + // is there to say so rather than to leave it to a category coincidence. + decomposed := norm.NFD.String(strings.ToLower(name)) var b strings.Builder - b.Grow(len(name)) - for _, r := range strings.ToLower(name) { + b.Grow(len(decomposed)) + for _, r := range decomposed { + if unicode.Is(unicode.Mn, r) { + continue + } if unicode.IsLetter(r) || unicode.IsDigit(r) { b.WriteRune(r) } @@ -396,11 +422,59 @@ func normalizeLocationName(name string) string { return b.String() } -// matchLocationNameInTx returns the location_id of the row in `candidates` -// whose location_name matches `name`, preferring an exact match and falling -// back to a normalized one (see normalizeLocationName), or nil for no match. -// Candidates must already be ordered deterministically by the caller so that -// two rows folding to the same key always resolve the same way. +// stripParentheticals removes every parenthesised span from name, so +// "Frankfurt am Main (Innenstadt I)" becomes "Frankfurt am Main ". Nesting is +// tracked, and an unclosed "(" swallows the rest of the string -- a qualifier +// that was truncated by a length limit is still a qualifier. +// +// This is only ever applied to a comparison key, never to anything stored. +func stripParentheticals(name string) string { + if !strings.ContainsRune(name, '(') { + return name + } + var b strings.Builder + b.Grow(len(name)) + depth := 0 + for _, r := range name { + switch r { + case '(': + depth += 1 + case ')': + if 0 < depth { + depth -= 1 + } + default: + if depth == 0 { + b.WriteRune(r) + } + } + } + return b.String() +} + +// matchLocationName returns the location_id of the row in `candidates` whose +// location_name matches `name`, or nil for no match. Candidates must already be +// ordered deterministically by the caller so that two rows folding to the same +// key always resolve the same way. +// +// Three passes, narrowest first: +// +// 1. exact string equality -- the common case, since the winning source usually +// spells it the way the mmdb import did; +// 2. the normalized fold (see normalizeLocationName): case, punctuation, +// whitespace and accents; +// 3. the normalized fold with parenthesised qualifiers stripped from BOTH +// sides. This is the case that motivated the whole feature: one source +// reports "Frankfurt am Main (Innenstadt I)" -- a district qualifier -- for +// a host another source calls "Frankfurt am Main". Pass 2 cannot see +// through that, because dropping the parentheses as punctuation leaves the +// qualifier's letters in the key. +// +// Pass 3 is the only pass that can plausibly match the wrong row -- two +// same-region rows "Springfield (IL)" and "Springfield (MA)" both reduce to +// "springfield" -- so it requires the stripped key to identify exactly ONE +// candidate and returns nil on any ambiguity. Falling back to country +// granularity is the safe outcome; guessing is not. func matchLocationName(name string, candidateIds []server.Id, candidateNames []string) *server.Id { for i, candidateName := range candidateNames { if candidateName == name { @@ -418,7 +492,22 @@ func matchLocationName(name string, candidateIds []server.Id, candidateNames []s return &candidateIds[i] } } - return nil + + base := normalizeLocationName(stripParentheticals(name)) + if base == "" { + // the name was nothing but a qualifier + return nil + } + var unique *server.Id + for i, candidateName := range candidateNames { + if normalizeLocationName(stripParentheticals(candidateName)) == base { + if unique != nil { + return nil + } + unique = &candidateIds[i] + } + } + return unique } // MatchExistingLocation resolves (countryCode, region, city) against location @@ -436,9 +525,11 @@ func matchLocationName(name string, candidateIds []server.Id, candidateNames []s // source's original display string. Each variant would become its own row, // those rows outlive a code revert, and there is no cleanup path. // -// Matching is case-insensitive and ignores punctuation and whitespace -// differences, so the ordinary variants resolve to the row that is already -// there. When nothing resolves the caller falls back to country granularity -- +// Matching is case-insensitive and ignores punctuation, whitespace, accents and +// parenthesised district qualifiers (see matchLocationName), so the ordinary +// variants resolve to the row that is already there -- including the +// "Frankfurt am Main (Innenstadt I)" case above, which is what this exists for. +// When nothing resolves the caller falls back to country granularity -- // see SubmitProviderEgressLocation. Falling back loses precision for one // submission; creating a row corrupts shared data permanently. // From 734e26dea19a0d44abd8b458023c7b7a03cd03bf Mon Sep 17 00:00:00 2001 From: Ryan Mello Date: Thu, 30 Jul 2026 15:07:45 +0100 Subject: [PATCH 20/30] feat(model): derive provider bandwidth from settled contract bytes transfer_escrow already records real bytes delivered per contract as a byproduct of billing. Deriving throughput from it costs nothing additional and cannot be gamed selectively -- a provider cannot inflate real user traffic without actually being fast for real users. Excludes companion (return-traffic) contracts: a client's return leg settles with the client as destination, which would otherwise misread ordinary users as fast providers. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QtgqtCmKJRXdsQ5ktiqwkg --- model/provider_bandwidth_model.go | 120 ++++++++++++++++ model/provider_bandwidth_model_test.go | 185 +++++++++++++++++++++++++ 2 files changed, 305 insertions(+) create mode 100644 model/provider_bandwidth_model.go create mode 100644 model/provider_bandwidth_model_test.go diff --git a/model/provider_bandwidth_model.go b/model/provider_bandwidth_model.go new file mode 100644 index 00000000..f2854c10 --- /dev/null +++ b/model/provider_bandwidth_model.go @@ -0,0 +1,120 @@ +package model + +import ( + "context" + "time" + + "github.com/urnetwork/server" +) + +// bandwidth sources. A stored figure is tagged with the source that produced +// it so consumers never have to know which one did. +const ( + // ProviderBandwidthSourcePassive is derived from already-settled contract + // bytes: zero additional cost, and it cannot be gamed selectively. + ProviderBandwidthSourcePassive = "passive" + // ProviderBandwidthSourceActive is a sampled download over the provider's + // tunnel, used only where passive history does not exist yet. + ProviderBandwidthSourceActive = "active" +) + +// ProviderBandwidth is one throughput figure for a provider. It is advisory: +// nothing may gate provider selection on it. +type ProviderBandwidth struct { + ClientId server.Id + BytesPerSecond float64 + Source string + SampleByteCount ByteCount + WindowStart time.Time + WindowEnd time.Time +} + +// ComputePassiveProviderBandwidth derives a provider's throughput from bytes it +// has already been paid to carry. `transfer_escrow`/`contract_close` record +// settled bytes per contract as a byproduct of billing, so reading them costs +// no additional bandwidth, and a provider cannot inflate the figure selectively +// -- it cannot move more real user traffic without actually being fast for real +// users. +// +// The rate is the total settled bytes in the window over the wall-clock span +// those contracts covered, so it is an average over the sampled traffic rather +// than a peak. Returns nil, nil when the provider settled no bytes in the +// window: no history, which is not the same as measured-zero throughput. +func ComputePassiveProviderBandwidth( + ctx context.Context, + clientId server.Id, + window time.Duration, +) (*ProviderBandwidth, error) { + windowStart := server.NowUtc().Add(-window) + + var contractCount int + var sampleByteCount ByteCount + // null whenever no contract matched + var minCreateTime *time.Time + var maxCloseTime *time.Time + + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT + COUNT(*), + COALESCE(SUM(contract_close.used_transfer_byte_count), 0)::bigint, + MIN(transfer_contract.create_time), + MAX(contract_close.close_time) + + FROM contract_close + + INNER JOIN transfer_contract ON + transfer_contract.contract_id = contract_close.contract_id + + WHERE + transfer_contract.destination_id = $1 AND + contract_close.party = 'destination' AND + -- companion_contract_id IS NULL excludes return-traffic legs: a + -- client's return traffic settles as a contract where the CLIENT + -- is the destination, which would otherwise be misread as that + -- client acting as a fast provider. See + -- docs/superpowers/specs/2026-07-25-enforced-provider-geo-probing-design.md + -- "Threat model" -- confirmed empirically: on beta, every + -- non-Public-key "earner" turned out to be exactly this. + transfer_contract.companion_contract_id IS NULL AND + $2 <= contract_close.close_time + `, + clientId, + windowStart, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + server.Raise(result.Scan( + &contractCount, + &sampleByteCount, + &minCreateTime, + &maxCloseTime, + )) + } + }) + }) + + if contractCount == 0 || sampleByteCount <= 0 || minCreateTime == nil || maxCloseTime == nil { + return nil, nil + } + + elapsed := maxCloseTime.Sub(*minCreateTime) + if elapsed <= 0 { + // no usable denominator (a single instantaneous close, or skew between + // the create and close writers). A rate is undefined here, and dividing + // by zero or a negative span would report an absurd one. + return nil, nil + } + + return &ProviderBandwidth{ + ClientId: clientId, + BytesPerSecond: float64(sampleByteCount) / elapsed.Seconds(), + Source: ProviderBandwidthSourcePassive, + SampleByteCount: sampleByteCount, + // the span actually measured, which is at most `window` wide + WindowStart: *minCreateTime, + WindowEnd: *maxCloseTime, + }, nil +} diff --git a/model/provider_bandwidth_model_test.go b/model/provider_bandwidth_model_test.go new file mode 100644 index 00000000..46077c66 --- /dev/null +++ b/model/provider_bandwidth_model_test.go @@ -0,0 +1,185 @@ +package model + +import ( + "context" + "testing" + "time" + + "github.com/urnetwork/connect" + + "github.com/urnetwork/server" +) + +func TestComputePassiveProviderBandwidthDerivesFromSettledBytes(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + sourceNetworkId := server.NewId() + sourceId := server.NewId() + destNetworkId := server.NewId() + destId := server.NewId() + Testing_CreateDevice(ctx, sourceNetworkId, server.NewId(), sourceId, "", "") + Testing_CreateDevice(ctx, destNetworkId, server.NewId(), destId, "", "") + + // a contract that settled 32 MiB over exactly 10 seconds of wall time + windowStart := server.NowUtc().Add(-1 * time.Hour) + contractId := Testing_CreateSettledContract(ctx, sourceId, destId, + windowStart, windowStart.Add(10*time.Second), 32*1024*1024) + + bw, err := ComputePassiveProviderBandwidth(ctx, destId, 2*time.Hour) + connect.AssertEqual(t, err, nil) + if bw == nil { + t.Fatal("expected a passive bandwidth result, got nil") + } + connect.AssertEqual(t, bw.Source, "passive") + // 32 MiB / 10s ~= 3355443 bytes/sec + if bw.BytesPerSecond < 3_000_000 || 3_700_000 < bw.BytesPerSecond { + t.Errorf("BytesPerSecond = %.0f, want ~3355443", bw.BytesPerSecond) + } + _ = contractId + }) +} + +func TestComputePassiveProviderBandwidthNilWhenNoHistory(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + bw, err := ComputePassiveProviderBandwidth(ctx, server.NewId(), 2*time.Hour) + connect.AssertEqual(t, err, nil) + if bw != nil { + t.Errorf("expected nil for a provider with no settled bytes, got %+v", bw) + } + }) +} + +// TestComputePassiveProviderBandwidthExcludesCompanionContracts is the load +// bearing case: a client's return traffic settles as a companion contract where +// the CLIENT is the destination. Counting it would read an ordinary user as a +// very fast provider. Only the companion leg exists here, so the correct answer +// is nil (no provider history at all) -- a merely smaller number would prove +// only dilution, not exclusion. +func TestComputePassiveProviderBandwidthExcludesCompanionContracts(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + providerNetworkId := server.NewId() + providerId := server.NewId() + clientNetworkId := server.NewId() + clientId := server.NewId() + Testing_CreateDevice(ctx, providerNetworkId, server.NewId(), providerId, "", "") + Testing_CreateDevice(ctx, clientNetworkId, server.NewId(), clientId, "", "") + + // the client's own contract, which the companion leg pairs with + createTime := server.NowUtc().Add(-1 * time.Hour) + primaryContractId := Testing_CreateSettledContract(ctx, clientId, providerId, + createTime, createTime.Add(10*time.Second), 1024) + + // the return leg: provider -> client, with the client as destination + Testing_CreateSettledCompanionContract(ctx, providerId, clientId, + createTime, createTime.Add(1*time.Second), 64*1024*1024, primaryContractId) + + bw, err := ComputePassiveProviderBandwidth(ctx, clientId, 2*time.Hour) + connect.AssertEqual(t, err, nil) + if bw != nil { + t.Errorf( + "return traffic must not be read as provider egress: got %.0f bytes/sec for a client that never provided", + bw.BytesPerSecond, + ) + } + }) +} + +// Testing_CreateSettledContract inserts a closed contract and its +// destination-party close row, matching the shape real settlement writes +// (`CloseContract` in subscription_model.go). Returns the contract id. +func Testing_CreateSettledContract( + ctx context.Context, + sourceId server.Id, + destinationId server.Id, + createTime time.Time, + closeTime time.Time, + usedByteCount ByteCount, +) server.Id { + return testingCreateSettledContract( + ctx, sourceId, destinationId, createTime, closeTime, usedByteCount, nil, + ) +} + +// Testing_CreateSettledCompanionContract is Testing_CreateSettledContract for +// the return-traffic leg of `companionContractId`. +func Testing_CreateSettledCompanionContract( + ctx context.Context, + sourceId server.Id, + destinationId server.Id, + createTime time.Time, + closeTime time.Time, + usedByteCount ByteCount, + companionContractId server.Id, +) server.Id { + return testingCreateSettledContract( + ctx, sourceId, destinationId, createTime, closeTime, usedByteCount, &companionContractId, + ) +} + +func testingCreateSettledContract( + ctx context.Context, + sourceId server.Id, + destinationId server.Id, + createTime time.Time, + closeTime time.Time, + usedByteCount ByteCount, + companionContractId *server.Id, +) server.Id { + contractId := server.NewId() + + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO transfer_contract ( + contract_id, + source_network_id, + source_id, + destination_network_id, + destination_id, + transfer_byte_count, + create_time, + close_time, + outcome, + companion_contract_id + ) + VALUES ( + $1, + (SELECT network_id FROM network_client WHERE client_id = $2), + $2, + (SELECT network_id FROM network_client WHERE client_id = $3), + $3, + $4, + $5, + $6, + 'success', + $7 + ) + `, + contractId, + sourceId, + destinationId, + usedByteCount, + createTime.UTC(), + closeTime.UTC(), + companionContractId, + )) + + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO contract_close (contract_id, close_time, party, used_transfer_byte_count) + VALUES ($1, $2, 'destination', $3) + `, + contractId, + closeTime.UTC(), + usedByteCount, + )) + }) + + return contractId +} From 71f8d4a36dbfa1003926aea2271d1c9b5524dcb9 Mon Sep 17 00:00:00 2001 From: Ryan Mello Date: Thu, 30 Jul 2026 15:28:05 +0100 Subject: [PATCH 21/30] feat(schema): provider_bandwidth table and egress verdict columns Appends three migrations and the storage path for a measured bandwidth figure. - provider_bandwidth: one row per provider, keyed on client_id, tagged with the source that produced it (passive aggregation of settled bytes, or an active sample). A new measurement overwrites the old one -- this is the current figure a consumer reads, not a history, and keeping every sample would grow without bound for a value only ever read as "the current one". Indexed on window_end for staleness scans; single-provider lookups go through the primary key. - provider_egress_location gains verdict / verdict_reason / assurance. All three are additive with safe defaults ('unverified' / '' / 'direct'), so existing rows read as an unjudged direct probe and every existing reader of the table is unaffected. Nothing writes a non-default verdict yet. - StoreProviderBandwidth upserts through the client_id primary key, so both sources write one figure per provider. The migrations are appended at the tail of the slice and never inserted: migrations apply by slice index (`for i := DbVersion(ctx); i < upTo; i++`), so editing or reordering an applied entry corrupts a live database. Every statement is IF NOT EXISTS, making a re-run -- or a duplicated merge resolution -- a no-op. SetProviderEgressLocation names every column explicitly, which bypasses the new columns' DEFAULTs, so it normalizes an empty Verdict to 'unverified' and an empty Assurance to 'direct' the same way it already lowercases the country code. Without that, the additive-with-safe-defaults claim would be false for every row the current ingest path writes: it computes no judgement, and Go's zero value '' is a legal varchar. Both readers are updated -- the plain getter and GetFreshProviderEgressLocationForConnection, which is the connect-announce hot path -- so neither returns a zero-valued verdict that looks like a real one. Tests: the table and the three columns exist (both failed first against the un-migrated schema); StoreProviderBandwidth round-trips and a second measurement replaces the first in place; the write path stores the defaults for an unjudged submission and an explicit judgement verbatim. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QtgqtCmKJRXdsQ5ktiqwkg --- db_migrations.go | 49 ++++++++ model/provider_bandwidth_model.go | 45 ++++++++ model/provider_bandwidth_model_test.go | 113 +++++++++++++++++++ model/provider_egress_location_model.go | 62 +++++++++- model/provider_egress_location_model_test.go | 81 +++++++++++++ 5 files changed, 348 insertions(+), 2 deletions(-) diff --git a/db_migrations.go b/db_migrations.go index 09d920f9..9de598c2 100644 --- a/db_migrations.go +++ b/db_migrations.go @@ -4477,4 +4477,53 @@ var migrations = []any{ CREATE INDEX IF NOT EXISTS provider_egress_location_observed_at_client_id ON provider_egress_location (observed_at, client_id) `), + + // The recorded judgement for a probed egress location: `verdict` is + // verified/unverified/suspect, `verdict_reason` the short failure class that + // produced it (see probeverdict), and `assurance` how the probe reached the + // provider (`direct` until multi-hop lands in P3). + // + // All three are additive with safe defaults, so every existing row reads as + // an unjudged direct probe and every existing reader of + // provider_egress_location is unaffected. Nothing writes a non-default + // verdict until the ingest path computes one. + // + // Appended, never inserted: migrations here apply by slice index + // (`for i := DbVersion(ctx); i < upTo; i++`), so editing or reordering an + // already-applied entry corrupts live databases. IF NOT EXISTS on every + // statement makes a re-run -- or a duplicated merge resolution -- a no-op. + newSqlMigration(` + ALTER TABLE provider_egress_location + ADD COLUMN IF NOT EXISTS verdict varchar(16) NOT NULL DEFAULT 'unverified', + ADD COLUMN IF NOT EXISTS verdict_reason varchar(64) NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS assurance varchar(16) NOT NULL DEFAULT 'direct' + `), + + // One measured throughput figure per provider, from either source (passive + // aggregation of settled bytes, or an active sampled download) -- consumers + // read one number and the `source` column says which produced it. + // + // client_id is the primary key, so a new measurement overwrites the old one + // (mirrors provider_egress_location's own shape). This is a ranking input, + // not a history: keeping every sample would grow without bound for a value + // only ever read as "the current figure". + newSqlMigration(` + CREATE TABLE IF NOT EXISTS provider_bandwidth ( + client_id uuid NOT NULL PRIMARY KEY, + bytes_per_second double precision NOT NULL, + source varchar(16) NOT NULL, + sample_byte_count bigint NOT NULL, + window_start timestamp NOT NULL, + window_end timestamp NOT NULL, + update_time timestamp NOT NULL + ) + `), + + // serves staleness sweeps and "who needs a fresh measurement" scans, which + // range on window_end. Lookups of a single provider's figure go through the + // primary key and need no index of their own. + newSqlMigration(` + CREATE INDEX IF NOT EXISTS provider_bandwidth_window_end + ON provider_bandwidth (window_end) + `), } diff --git a/model/provider_bandwidth_model.go b/model/provider_bandwidth_model.go index f2854c10..1e4522ac 100644 --- a/model/provider_bandwidth_model.go +++ b/model/provider_bandwidth_model.go @@ -118,3 +118,48 @@ func ComputePassiveProviderBandwidth( WindowEnd: *maxCloseTime, }, nil } + +// StoreProviderBandwidth records a provider's current throughput figure, +// whatever produced it. The row is keyed on client_id, so a new measurement +// replaces the previous one: this is the current figure a consumer reads, not a +// history (see the provider_bandwidth migration). +// +// Both sources write through here, tagged by bw.Source, so nothing downstream +// has to know whether a figure came from settled traffic or an active sample. +// The figure is advisory -- storing one must never gate provider selection. +func StoreProviderBandwidth(ctx context.Context, bw *ProviderBandwidth) { + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO provider_bandwidth ( + client_id, + bytes_per_second, + source, + sample_byte_count, + window_start, + window_end, + update_time + ) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (client_id) DO UPDATE + SET + bytes_per_second = $2, + source = $3, + sample_byte_count = $4, + window_start = $5, + window_end = $6, + update_time = $7 + `, + bw.ClientId, + bw.BytesPerSecond, + bw.Source, + bw.SampleByteCount, + // window_start/window_end are naive timestamp columns holding utc, + // as everywhere else in this schema + bw.WindowStart.UTC(), + bw.WindowEnd.UTC(), + server.NowUtc(), + )) + }) +} diff --git a/model/provider_bandwidth_model_test.go b/model/provider_bandwidth_model_test.go index 46077c66..f04b1acb 100644 --- a/model/provider_bandwidth_model_test.go +++ b/model/provider_bandwidth_model_test.go @@ -183,3 +183,116 @@ func testingCreateSettledContract( return contractId } + +// TestProviderBandwidthTableExists asserts the schema shape the storage path +// depends on, separately from the storage path itself: a missing table here is a +// migration that was never appended, not a bug in StoreProviderBandwidth. +func TestProviderBandwidthTableExists(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + var exists bool + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query(ctx, `SELECT to_regclass('provider_bandwidth') IS NOT NULL`) + connect.AssertEqual(t, err, nil) + server.WithPgResult(result, err, func() { + if result.Next() { + server.Raise(result.Scan(&exists)) + } + }) + }) + if !exists { + t.Fatal("provider_bandwidth table does not exist") + } + }) +} + +// TestStoreProviderBandwidthUpsertsOneRowPerProvider covers the round trip and +// the primary-key overwrite: a second measurement for the same provider replaces +// the first rather than accumulating history. +func TestStoreProviderBandwidthUpsertsOneRowPerProvider(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + clientId := server.NewId() + windowStart := server.NowUtc().Add(-2 * time.Hour) + windowEnd := server.NowUtc().Add(-1 * time.Hour) + + StoreProviderBandwidth(ctx, &ProviderBandwidth{ + ClientId: clientId, + BytesPerSecond: 1024 * 1024, + Source: ProviderBandwidthSourcePassive, + SampleByteCount: ByteCount(32 * 1024 * 1024), + WindowStart: windowStart, + WindowEnd: windowEnd, + }) + + rowCount, stored := testingReadProviderBandwidth(ctx, clientId) + connect.AssertEqual(t, rowCount, 1) + connect.AssertEqual(t, stored.BytesPerSecond, float64(1024*1024)) + connect.AssertEqual(t, stored.Source, ProviderBandwidthSourcePassive) + connect.AssertEqual(t, stored.SampleByteCount, ByteCount(32*1024*1024)) + connect.AssertEqual(t, stored.WindowStart.UTC().Unix(), windowStart.Unix()) + connect.AssertEqual(t, stored.WindowEnd.UTC().Unix(), windowEnd.Unix()) + + // a later measurement from the other source replaces it in place + activeWindowStart := server.NowUtc().Add(-1 * time.Minute) + activeWindowEnd := server.NowUtc() + StoreProviderBandwidth(ctx, &ProviderBandwidth{ + ClientId: clientId, + BytesPerSecond: 4 * 1024 * 1024, + Source: ProviderBandwidthSourceActive, + SampleByteCount: ByteCount(8 * 1024 * 1024), + WindowStart: activeWindowStart, + WindowEnd: activeWindowEnd, + }) + + rowCount, stored = testingReadProviderBandwidth(ctx, clientId) + connect.AssertEqual(t, rowCount, 1) + connect.AssertEqual(t, stored.BytesPerSecond, float64(4*1024*1024)) + connect.AssertEqual(t, stored.Source, ProviderBandwidthSourceActive) + connect.AssertEqual(t, stored.SampleByteCount, ByteCount(8*1024*1024)) + connect.AssertEqual(t, stored.WindowEnd.UTC().Unix(), activeWindowEnd.Unix()) + }) +} + +// testingReadProviderBandwidth reads the stored row back with sql, so the test +// asserts what is actually in the table rather than trusting a model reader that +// StoreProviderBandwidth's own writer would share. +func testingReadProviderBandwidth( + ctx context.Context, + clientId server.Id, +) (rowCount int, bw *ProviderBandwidth) { + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT + client_id, + bytes_per_second, + source, + sample_byte_count, + window_start, + window_end + FROM provider_bandwidth + WHERE client_id = $1 + `, + clientId, + ) + server.WithPgResult(result, err, func() { + for result.Next() { + rowCount += 1 + bw = &ProviderBandwidth{} + server.Raise(result.Scan( + &bw.ClientId, + &bw.BytesPerSecond, + &bw.Source, + &bw.SampleByteCount, + &bw.WindowStart, + &bw.WindowEnd, + )) + } + }) + }) + return rowCount, bw +} diff --git a/model/provider_egress_location_model.go b/model/provider_egress_location_model.go index c0ad7780..d67adccc 100644 --- a/model/provider_egress_location_model.go +++ b/model/provider_egress_location_model.go @@ -26,8 +26,27 @@ const ProviderEgressLocationMaxAge = 7 * 24 * time.Hour // of the queue. const ProviderEgressProbeAttemptBackoff = 6 * time.Hour +// the verdict/assurance values a provider_egress_location row can hold. These +// mirror the column defaults, and are what an unjudged submission is normalized +// to on write -- see SetProviderEgressLocation. +const ( + // ProviderEgressVerdictUnverified is the default: no judgement recorded. + // Every row written before the ingest path computed verdicts reads as this. + ProviderEgressVerdictUnverified = "unverified" + // ProviderEgressAssuranceDirect means the probe reached the provider over a + // single tunnel from the prober. It is the only assurance in use; multi-hop + // is P3's concern. + ProviderEgressAssuranceDirect = "direct" +) + // ProviderEgressLocation is a provider location learned by probing the // provider's own egress, rather than by looking up its control-connection ip. +// +// Verdict/VerdictReason/Assurance carry the recorded judgement for the probe +// that produced this location. They are advisory: nothing in provider selection +// or scoring reads them. An empty Verdict or Assurance is normalized to the +// column default on write, so a caller that does not compute a judgement stores +// an unjudged direct probe rather than an empty string. type ProviderEgressLocation struct { ClientId server.Id LocationId server.Id @@ -39,6 +58,9 @@ type ProviderEgressLocation struct { Mobile bool CityConfident bool ObservedAt time.Time + Verdict string + VerdictReason string + Assurance string UpdateTime time.Time } @@ -52,6 +74,21 @@ func SetProviderEgressLocation(ctx context.Context, e *ProviderEgressLocation) { // return uppercase codes (e.g. "US"), so normalize before writing. countryCode := strings.ToLower(e.CountryCode) + // the verdict columns are NOT NULL with defaults, and this INSERT names + // every column explicitly -- which bypasses those defaults. A caller that + // computes no judgement would otherwise store '' rather than 'unverified' + // and '' rather than 'direct', so normalize here the way countryCode is + // normalized above. VerdictReason has no default value to fall back to: "" + // means "no reason", which is exactly the column default. + verdict := e.Verdict + if verdict == "" { + verdict = ProviderEgressVerdictUnverified + } + assurance := e.Assurance + if assurance == "" { + assurance = ProviderEgressAssuranceDirect + } + server.Tx(ctx, func(tx server.PgTx) { server.RaisePgResult(tx.Exec( ctx, @@ -67,9 +104,12 @@ func SetProviderEgressLocation(ctx context.Context, e *ProviderEgressLocation) { mobile, city_confident, observed_at, + verdict, + verdict_reason, + assurance, update_time ) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) ON CONFLICT (client_id) DO UPDATE SET location_id = $2, @@ -81,7 +121,10 @@ func SetProviderEgressLocation(ctx context.Context, e *ProviderEgressLocation) { mobile = $8, city_confident = $9, observed_at = $10, - update_time = $11 + verdict = $11, + verdict_reason = $12, + assurance = $13, + update_time = $14 WHERE provider_egress_location.observed_at < EXCLUDED.observed_at `, e.ClientId, @@ -94,6 +137,9 @@ func SetProviderEgressLocation(ctx context.Context, e *ProviderEgressLocation) { e.Mobile, e.CityConfident, e.ObservedAt.UTC(), + verdict, + e.VerdictReason, + assurance, server.NowUtc(), )) }) @@ -198,6 +244,9 @@ func GetProviderEgressLocation(ctx context.Context, clientId server.Id) *Provide mobile, city_confident, observed_at, + verdict, + verdict_reason, + assurance, update_time FROM provider_egress_location WHERE client_id = $1 @@ -218,6 +267,9 @@ func GetProviderEgressLocation(ctx context.Context, clientId server.Id) *Provide &e.Mobile, &e.CityConfident, &e.ObservedAt, + &e.Verdict, + &e.VerdictReason, + &e.Assurance, &e.UpdateTime, )) } @@ -278,6 +330,9 @@ func GetFreshProviderEgressLocationForConnection( pel.mobile, pel.city_confident, pel.observed_at, + pel.verdict, + pel.verdict_reason, + pel.assurance, pel.update_time FROM network_client_connection ncc INNER JOIN provider_egress_location pel ON pel.client_id = ncc.client_id @@ -299,6 +354,9 @@ func GetFreshProviderEgressLocationForConnection( &e.Mobile, &e.CityConfident, &e.ObservedAt, + &e.Verdict, + &e.VerdictReason, + &e.Assurance, &e.UpdateTime, )) } diff --git a/model/provider_egress_location_model_test.go b/model/provider_egress_location_model_test.go index f3acd4c3..53dc29b2 100644 --- a/model/provider_egress_location_model_test.go +++ b/model/provider_egress_location_model_test.go @@ -664,3 +664,84 @@ func TestGetProviderEgressLocationDueOrderingIsStableAcrossLimits(t *testing.T) } }) } + +// TestProviderEgressLocationHasVerdictColumns asserts the three verdict columns +// exist. They are additive with safe defaults, so no existing reader or row is +// affected -- but nothing can record a verdict until they are there. +func TestProviderEgressLocationHasVerdictColumns(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + for _, col := range []string{"verdict", "verdict_reason", "assurance"} { + var exists bool + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'provider_egress_location' AND column_name = $1 + ) + `, + col, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + server.Raise(result.Scan(&exists)) + } + }) + }) + if !exists { + t.Errorf("provider_egress_location missing column %q", col) + } + } + }) +} + +// TestProviderEgressLocationVerdictDefaults pins the write path's normalization. +// SetProviderEgressLocation names every column explicitly, which bypasses the +// column defaults, so a caller that computes no judgement -- every caller until +// the ingest path does -- must still store unverified/direct, not the empty +// string. +func TestProviderEgressLocationVerdictDefaults(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + clientId := server.NewId() + observedAt := server.NowUtc() + + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: clientId, + LocationId: server.NewId(), + CountryCode: "es", + ObservedAt: observedAt, + }) + + stored := GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected a stored egress location") + } + connect.AssertEqual(t, stored.Verdict, ProviderEgressVerdictUnverified) + connect.AssertEqual(t, stored.VerdictReason, "") + connect.AssertEqual(t, stored.Assurance, ProviderEgressAssuranceDirect) + + // an explicit judgement is stored verbatim + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: clientId, + LocationId: server.NewId(), + CountryCode: "de", + ObservedAt: observedAt.Add(time.Hour), + Verdict: "suspect", + VerdictReason: "unstable", + Assurance: ProviderEgressAssuranceDirect, + }) + + stored = GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected a stored egress location") + } + connect.AssertEqual(t, stored.Verdict, "suspect") + connect.AssertEqual(t, stored.VerdictReason, "unstable") + connect.AssertEqual(t, stored.Assurance, ProviderEgressAssuranceDirect) + }) +} From aacdbb6fcead800289f206c9c544f398fc6a880e Mon Sep 17 00:00:00 2001 From: Ryan Mello Date: Thu, 30 Jul 2026 16:29:41 +0100 Subject: [PATCH 22/30] feat(model): hourly byte budget for active bandwidth probes Active bandwidth probing pulls real data through a provider's tunnel, which is a real paid contract: a zero-cost balance code does not make it free, because the payout planner sums paid and unpaid traffic identically before computing payouts. Probe bytes therefore need a spend limit unconditionally, on every deployment, not only where payouts happen to be live. The budget is split into fixed, non-overlapping UTC hourly buckets rather than a rolling trailing window. A reservation goes to the earliest bucket with room, starting from the current hour; if the current hour is full the probe is deferred to a later one rather than rejected, up to a 24-hour lookahead, after which the deployment's daily budget is genuinely exhausted and the probe is refused. Fixed buckets are what make "defer to the next hour" well-defined -- there needs to be a discrete boundary to wait for. A caller that reserves budget it cannot ultimately use (the provider went offline between reserving and dialing) releases it explicitly via CancelProviderBandwidthReservation, and a reaper drops ledger rows whose bucket is safely in the past. The reservation is a plain read-then-insert in one transaction, with no row locking. SELECT ... FOR UPDATE would not help: the contended quantity is a bucket's aggregate, and an empty bucket has no row to lock -- serializing properly would need a materialized per-bucket row or an advisory lock, which this ledger shape deliberately doesn't have. Two concurrent reservations can overshoot the ceiling by at most their combined byte counts, which is acceptable for a coarse deployment-wide spend cap fed by a small number of prober processes. MaxActiveBandwidthProbesPerBucket = 40 is derived from the population this probes, not picked arbitrarily. Active sampling only ever runs against providers with no passive bandwidth history -- the trickle of newly-joined providers before their first settled contract on a mature deployment, and effectively the whole fleet on a new one. 40/hour x 5 MB per probe = 200 MB/hour, 4.8 GB/day worst case. One value chosen to behave sensibly at both scales rather than a per-environment knob; tuned, not structural, and worth revisiting against real production data. The two migrations are appended at the tail (535 -> 537 entries, zero deletions in the diff) and every statement is IF NOT EXISTS, since migrations apply by slice index and reordering an applied entry corrupts live databases. --- db_migrations.go | 34 ++++ model/provider_bandwidth_rate_limit.go | 187 ++++++++++++++++++++ model/provider_bandwidth_rate_limit_test.go | 120 +++++++++++++ 3 files changed, 341 insertions(+) create mode 100644 model/provider_bandwidth_rate_limit.go create mode 100644 model/provider_bandwidth_rate_limit_test.go diff --git a/db_migrations.go b/db_migrations.go index 9de598c2..b0030cf7 100644 --- a/db_migrations.go +++ b/db_migrations.go @@ -4526,4 +4526,38 @@ var migrations = []any{ CREATE INDEX IF NOT EXISTS provider_bandwidth_window_end ON provider_bandwidth (window_end) `), + + // The deployment-wide byte budget for active bandwidth probing: one row per + // admitted reservation, counting against a fixed hourly bucket. See + // ReserveProviderBandwidthSlot for why active probe bytes need a spend + // limit at all -- they are real, paid contract traffic on any deployment + // where payouts are planned, regardless of the balance code used. + // + // byte_count is bigint, not int: a single bucket's budget is measured in + // hundreds of megabytes, so an int would overflow well inside the range + // this ledger has to sum over a day. client_id is stored for observability + // only -- the limit itself is global, not scoped per provider, so nothing + // reads this column to make an admission decision. + // + // Appended, never inserted: migrations here apply by slice index + // (`for i := DbVersion(ctx); i < upTo; i++`), so editing or reordering an + // already-applied entry corrupts live databases. IF NOT EXISTS on every + // statement makes a re-run -- or a duplicated merge resolution -- a no-op. + newSqlMigration(` + CREATE TABLE IF NOT EXISTS provider_bandwidth_quota ( + provider_bandwidth_quota_id uuid NOT NULL PRIMARY KEY, + client_id uuid NOT NULL, + byte_count bigint NOT NULL, + bucket_start timestamp NOT NULL, + create_time timestamp NOT NULL + ) + `), + + // every read of this table is a range over the lookahead window + // (`$1 <= bucket_start AND bucket_start < $2`), and the reaper deletes by + // the same column, so bucket_start is the only index it needs. + newSqlMigration(` + CREATE INDEX IF NOT EXISTS provider_bandwidth_quota_bucket_start + ON provider_bandwidth_quota (bucket_start) + `), } diff --git a/model/provider_bandwidth_rate_limit.go b/model/provider_bandwidth_rate_limit.go new file mode 100644 index 00000000..6dee56ce --- /dev/null +++ b/model/provider_bandwidth_rate_limit.go @@ -0,0 +1,187 @@ +package model + +import ( + "context" + "fmt" + "time" + + "github.com/urnetwork/server" +) + +// A deployment-wide budget on the bytes active bandwidth probing is allowed +// to spend, counted across all providers together -- this does not +// distinguish which provider is being probed, only how much total probe +// traffic has been admitted. Active probing pulls real data through a +// provider's tunnel, which is a real paid contract: a zero-cost balance code +// does not make it free, because the payout planner sums paid and unpaid +// traffic identically before computing payouts +// (account_payment_model_plan.go). This is therefore a spend limit +// unconditionally, on every deployment, not only where payouts happen to be +// live. +// +// The budget is split into fixed, non-overlapping hourly buckets (UTC), +// rather than a rolling trailing window: a reservation is made against the +// earliest bucket -- starting with the current hour -- that has room for it. +// If the current hour is full, the probe is deferred to the next hour instead +// of rejected, and so on, up to MaxProviderBandwidthLookaheadBuckets hours +// out. Only once no bucket in that whole lookahead window has room -- meaning +// the deployment-wide daily budget (MaxProviderBandwidthBytesPerDay) is +// genuinely exhausted -- is the probe rejected. Fixed buckets (rather than a +// rolling window) are what make "defer to the next hour" well-defined: there +// needs to be a discrete boundary to wait for. +// +// This intentionally does NOT apply to the passive bandwidth signal, which is +// aggregated from already-settled transfer_escrow bytes and costs nothing +// additional -- there is no spend to budget there. +const ProviderBandwidthBucketDuration = time.Hour +const MaxProviderBandwidthLookaheadBuckets = 24 + +// MaxActiveBandwidthProbesPerBucket is derived from the population this +// probes, not picked arbitrarily. Active sampling only ever runs against +// providers with no passive history -- on beta today that is the entire fleet +// (nothing has settled a contract yet), and on a mature deployment it is the +// trickle of newly-joined providers before their first settled contract. 40 +// per hour comfortably covers a beta-sized fleet in a single pass, and stays a +// small, bounded spend on a mature deployment where the no-passive-history +// population is naturally small. It is one value chosen to behave sensibly at +// both scales, not a per-environment knob. +// +// This is a tuned value, not a structural one: revisit it against real +// production data once active probing has run for a week. +const MaxActiveBandwidthProbesPerBucket = 40 + +// One active probe downloads 5 MB (the spec's per-probe figure), so the +// hourly budget is 40 probes x 5 MB = 200 MB/hour, and the daily cap is +// 24 x 200 MB = 4.8 GB worst case -- the ceiling only reached if every bucket +// in the lookahead window fills. +// +// Explicitly int64: a byte budget is not a row count, and callers pass an +// int64 byteCount. +const MaxProviderBandwidthBytesPerProbe int64 = 5 * 1024 * 1024 +const MaxProviderBandwidthBytesPerBucket int64 = MaxActiveBandwidthProbesPerBucket * MaxProviderBandwidthBytesPerProbe +const MaxProviderBandwidthBytesPerDay int64 = MaxProviderBandwidthLookaheadBuckets * MaxProviderBandwidthBytesPerBucket + +func maxProviderBandwidthError() error { + return fmt.Errorf( + "The active bandwidth probe budget (%d bytes per hour, %d bytes per day) has been reached for this deployment. Please try again later.", + MaxProviderBandwidthBytesPerBucket, + MaxProviderBandwidthBytesPerDay, + ) +} + +// providerBandwidthBucketStart truncates t to the start of its UTC hourly +// bucket. Truncate operates on the absolute duration since the zero time, +// so this only aligns to true UTC hour boundaries when t is already UTC. +func providerBandwidthBucketStart(t time.Time) time.Time { + return t.UTC().Truncate(ProviderBandwidthBucketDuration) +} + +// ReserveProviderBandwidthSlot finds the earliest hourly bucket -- starting +// with the current hour -- that has room for `byteCount` more probe bytes, +// and reserves them there. It returns the reservation's id (for +// CancelProviderBandwidthReservation, if the caller can't ultimately use the +// budget it just reserved -- e.g. the provider went offline between reserving +// and dialing) and the bucket's start time, which the caller should use as +// the probe's RunAt when the bucket isn't the current hour. A probe deferred +// to a future hour should have its RunAt jittered randomly across that hour +// rather than firing at the hour's top: otherwise every probe pushed into the +// same future bucket becomes eligible at the identical instant, converting a +// budget that was meant to spread load into an hourly thundering herd. +// +// If no bucket within MaxProviderBandwidthLookaheadBuckets hours has room, +// nothing is reserved and an error is returned -- this is the deployment's +// daily cap. A single probe can never itself be too large to fit an empty +// bucket (one probe is MaxProviderBandwidthBytesPerProbe, a fortieth of a +// bucket), so this only happens under genuine contention from other probes. +// +// Concurrency: this is a plain read-then-insert in one transaction, with no +// row locking. `SELECT ... FOR UPDATE` would not help here -- the thing being +// contended is a bucket's *aggregate*, and an empty or partly-filled bucket +// has no single row to lock; serializing reservations properly would need a +// materialized per-bucket row or an advisory lock, which this ledger shape +// deliberately doesn't have. Under the default RepeatableRead isolation two +// concurrent reservations can therefore both read the same bucket total and +// both insert (their rows are distinct, so there's no write-write conflict +// and no serialization failure), overshooting the ceiling by at most +// (concurrent reservers x their byte counts). That is acceptable: this is a +// coarse deployment-wide spend cap, not an accounting invariant, and probe +// reservations arrive at a low rate from a small number of prober processes. +func ReserveProviderBandwidthSlot( + ctx context.Context, + clientId server.Id, + byteCount int64, +) (reservationId server.Id, bucketStart time.Time, err error) { + windowStart := providerBandwidthBucketStart(server.NowUtc()) + windowEnd := windowStart.Add(MaxProviderBandwidthLookaheadBuckets * ProviderBandwidthBucketDuration) + + server.Tx(ctx, func(tx server.PgTx) { + usedByBucket := map[time.Time]int64{} + result, qerr := tx.Query( + ctx, + ` + SELECT bucket_start, SUM(byte_count) FROM provider_bandwidth_quota + WHERE $1 <= bucket_start AND bucket_start < $2 + GROUP BY bucket_start + `, + windowStart, windowEnd, + ) + server.WithPgResult(result, qerr, func() { + for result.Next() { + var bucket time.Time + var used int64 + server.Raise(result.Scan(&bucket, &used)) + usedByBucket[bucket] = used + } + }) + + for i := 0; i < MaxProviderBandwidthLookaheadBuckets; i++ { + candidate := windowStart.Add(time.Duration(i) * ProviderBandwidthBucketDuration) + if usedByBucket[candidate]+byteCount <= MaxProviderBandwidthBytesPerBucket { + id := server.NewId() + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO provider_bandwidth_quota (provider_bandwidth_quota_id, client_id, byte_count, bucket_start, create_time) + VALUES ($1, $2, $3, $4, $5) + `, + id, clientId, byteCount, candidate, server.NowUtc(), + )) + reservationId = id + bucketStart = candidate + return + } + } + err = maxProviderBandwidthError() + }) + return reservationId, bucketStart, err +} + +// CancelProviderBandwidthReservation releases a reservation made by +// ReserveProviderBandwidthSlot that the caller ultimately couldn't use -- e.g. +// the provider turned out to be unreachable, discovered only after the budget +// was reserved (the target bucket has to be known before the probe can be +// scheduled, so reservation necessarily happens before that check). +func CancelProviderBandwidthReservation(ctx context.Context, reservationId server.Id) { + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + `DELETE FROM provider_bandwidth_quota WHERE provider_bandwidth_quota_id = $1`, + reservationId, + )) + }) +} + +// RemoveExpiredProviderBandwidthQuota deletes quota ledger rows whose bucket +// is entirely in the past relative to minBucketStart. Callers should pass a +// cutoff safely behind the lookahead window (e.g. now minus a couple of +// days), since a bucket up to MaxProviderBandwidthLookaheadBuckets hours in +// the future can still be actively reserved against. +func RemoveExpiredProviderBandwidthQuota(ctx context.Context, minBucketStart time.Time) { + server.MaintenanceTx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + `DELETE FROM provider_bandwidth_quota WHERE bucket_start < $1`, + minBucketStart.UTC(), + )) + }) +} diff --git a/model/provider_bandwidth_rate_limit_test.go b/model/provider_bandwidth_rate_limit_test.go new file mode 100644 index 00000000..9bf78fdf --- /dev/null +++ b/model/provider_bandwidth_rate_limit_test.go @@ -0,0 +1,120 @@ +package model + +import ( + "context" + "testing" + "time" + + "github.com/urnetwork/connect" + "github.com/urnetwork/server" +) + +// The budget is global, not per-provider: two different providers draw from +// the same shared per-bucket byte ceiling. +func TestReserveProviderBandwidthSlotFillsBucketThenSpillsToNext(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + clientIdA := server.NewId() + clientIdB := server.NewId() + + currentBucket := providerBandwidthBucketStart(server.NowUtc()) + + half := MaxProviderBandwidthBytesPerBucket / 2 + + _, bucket1, err := ReserveProviderBandwidthSlot(ctx, clientIdA, half) + connect.AssertEqual(t, err, nil) + connect.AssertEqual(t, bucket1, currentBucket) + + // the remaining half fits exactly at the current bucket's ceiling, + // even though it's a different provider + _, bucketStart, err := ReserveProviderBandwidthSlot(ctx, clientIdB, MaxProviderBandwidthBytesPerBucket-half) + connect.AssertEqual(t, err, nil) + connect.AssertEqual(t, bucketStart, currentBucket) + + // the current bucket is now fully spent; the next reservation must + // spill into the next hour instead of being rejected + _, bucket2, err := ReserveProviderBandwidthSlot(ctx, clientIdA, 1) + connect.AssertEqual(t, err, nil) + connect.AssertEqual(t, bucket2, currentBucket.Add(ProviderBandwidthBucketDuration)) + }) +} + +// Once every bucket in the whole lookahead window (the deployment's daily +// byte budget) is spent, a new reservation must be rejected outright rather +// than queued indefinitely -- this is the hard daily cap. +func TestReserveProviderBandwidthSlotErrorsWhenAllBucketsFull(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + for i := 0; i < MaxProviderBandwidthLookaheadBuckets; i++ { + _, _, err := ReserveProviderBandwidthSlot(ctx, server.NewId(), MaxProviderBandwidthBytesPerBucket) + connect.AssertEqual(t, err, nil) + } + + // every bucket in the lookahead window is now full + _, _, err := ReserveProviderBandwidthSlot(ctx, server.NewId(), 1) + connect.AssertNotEqual(t, err, nil) + }) +} + +// A cancelled reservation must free its bytes back up -- e.g. an active probe +// that was never actually run because the provider went offline between +// reserving the budget and dialing it. +func TestCancelProviderBandwidthReservationFreesSlot(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + clientId := server.NewId() + + currentBucket := providerBandwidthBucketStart(server.NowUtc()) + + reservationId, bucketStart, err := ReserveProviderBandwidthSlot(ctx, clientId, MaxProviderBandwidthBytesPerBucket) + connect.AssertEqual(t, err, nil) + connect.AssertEqual(t, bucketStart, currentBucket) + + CancelProviderBandwidthReservation(ctx, reservationId) + + // the current bucket's full ceiling must be available again + _, bucketStart, err = ReserveProviderBandwidthSlot(ctx, clientId, MaxProviderBandwidthBytesPerBucket) + connect.AssertEqual(t, err, nil) + connect.AssertEqual(t, bucketStart, currentBucket) + }) +} + +// RemoveExpiredProviderBandwidthQuota must only remove buckets safely before +// the cutoff, leaving buckets at or after it untouched. +func TestRemoveExpiredProviderBandwidthQuota(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + currentBucket := providerBandwidthBucketStart(server.NowUtc()) + oldBucket := currentBucket.Add(-72 * time.Hour) + + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO provider_bandwidth_quota (provider_bandwidth_quota_id, client_id, byte_count, bucket_start, create_time) + VALUES ($1, $2, $3, $4, $5) + `, + server.NewId(), server.NewId(), 5, oldBucket, server.NowUtc(), + )) + }) + + _, _, err := ReserveProviderBandwidthSlot(ctx, server.NewId(), 7) + connect.AssertEqual(t, err, nil) + + RemoveExpiredProviderBandwidthQuota(ctx, currentBucket.Add(-48*time.Hour)) + + var total int64 + server.Db(ctx, func(conn server.PgConn) { + result, qerr := conn.Query(ctx, `SELECT COALESCE(SUM(byte_count), 0) FROM provider_bandwidth_quota`) + server.WithPgResult(result, qerr, func() { + if result.Next() { + server.Raise(result.Scan(&total)) + } + }) + }) + // the old (72h back) row must be gone; the current-bucket row (7) + // from ReserveProviderBandwidthSlot must remain + connect.AssertEqual(t, total, int64(7)) + }) +} From 8125129a835bc73bdc778cfa4b489d256160cca4 Mon Sep 17 00:00:00 2001 From: Ryan Mello Date: Thu, 30 Jul 2026 16:55:37 +0100 Subject: [PATCH 23/30] feat(api): operator endpoints for the active bandwidth probe Three operator-secret-gated routes the active bandwidth probe needs: GET /network/provider-bandwidth-test?bytes=N POST /network/provider-bandwidth-result POST /network/provider-bandwidth-reserve The download endpoint is the target the prober pulls through a provider's tunnel to measure its throughput. It streams min(N, 5MB) from a small repeating block through an io.LimitReader, so the byte count is never allocated, and the clamp keeps a single request from being an open-ended resource commitment. A `bytes` that is absent, malformed, or non-positive falls back to 1MiB: `bytes=0` parses cleanly, and an empty body would hand the prober a zero-byte sample to divide by. The result endpoint stores the measurement with Source "active" and window_start == window_end (an active probe is a point measurement, not an aggregate over a window). A non-positive rate or sample size is rejected before anything is written -- a zero measurement is not usable and must never overwrite a real figure. The reserve endpoint wraps ReserveProviderBandwidthSlot. That function is built for a caller that can set a RunAt, so when the current hour is full it succeeds with a reservation in a later bucket; the prober measures over a tunnel it already has open, right now, so later-hour budget is useless to it and a 200 would let it spend the whole daily ceiling inside one hour. A deferred reservation is therefore cancelled again and the request answered 429 with Retry-After, which the prober treats as "skip this provider". All three reuse the existing operator auth unchanged: same X-UR-Operator-Secret header, same constant-time hmac.Equal compare, same fail-closed memoized vault read as the egress location ingest. --- api/api.go | 7 + api/handlers/provider_bandwidth_handlers.go | 269 +++++++++ .../provider_bandwidth_handlers_test.go | 564 ++++++++++++++++++ 3 files changed, 840 insertions(+) create mode 100644 api/handlers/provider_bandwidth_handlers.go create mode 100644 api/handlers/provider_bandwidth_handlers_test.go diff --git a/api/api.go b/api/api.go index bb9d6c21..252e42d5 100644 --- a/api/api.go +++ b/api/api.go @@ -54,6 +54,13 @@ func Routes() []*router.Route { router.NewRoute("POST", "/network/provider-egress-location", handlers.ProviderEgressLocationSubmit), router.NewRoute("GET", "/network/provider-egress-due", handlers.ProviderEgressLocationDue), router.NewRoute("POST", "/network/provider-egress-attempt", handlers.ProviderEgressLocationAttempt), + // operator-to-server, gated by the same operator secret as the egress + // location ingest above: the active bandwidth probe's download target, + // its result submission, and the byte-budget reservation the prober + // takes before spending any probe bytes + router.NewRoute("GET", "/network/provider-bandwidth-test", handlers.ProviderBandwidthTest), + router.NewRoute("POST", "/network/provider-bandwidth-result", handlers.ProviderBandwidthResult), + router.NewRoute("POST", "/network/provider-bandwidth-reserve", handlers.ProviderBandwidthReserve), router.NewRoute("GET", "/network/clients", handlers.NetworkClients), router.NewRoute("GET", "/network/peers", handlers.NetworkPeers), router.NewRoute("GET", "/network/provider-locations", handlers.NetworkGetProviderLocations), diff --git a/api/handlers/provider_bandwidth_handlers.go b/api/handlers/provider_bandwidth_handlers.go new file mode 100644 index 00000000..eabc7e90 --- /dev/null +++ b/api/handlers/provider_bandwidth_handlers.go @@ -0,0 +1,269 @@ +package handlers + +import ( + "crypto/hmac" + "encoding/json" + "io" + "net/http" + "strconv" + "time" + + "github.com/urnetwork/glog" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" +) + +// maxProviderBandwidthTestBytes bounds a single download. Without a clamp the +// endpoint is an open-ended resource commitment per request: a caller could +// ask for gigabytes and hold a connection (and the egress bytes that go with +// it) for as long as it liked. 5 MB matches the spec's per-probe sizing and +// the per-probe figure the byte budget reserves against +// (model.MaxProviderBandwidthBytesPerProbe). +const maxProviderBandwidthTestBytes = 5 * 1024 * 1024 + +// defaultProviderBandwidthTestBytes is used when `bytes` is absent, malformed, +// or non-positive. `bytes=0` and `bytes=-1` parse cleanly, so they are not +// caught by a malformed-input check -- and streaming an empty body for them +// would hand the prober a zero-byte sample to divide by. +const defaultProviderBandwidthTestBytes = 1024 * 1024 + +// maxProviderBandwidthBody bounds the request body of the two POST endpoints. +// Both carry a fixed handful of scalars. +const maxProviderBandwidthBody = 4 * 1024 + +// providerBandwidthTestBlock is the unit the download endpoint repeats. The +// content is irrelevant -- only the byte count is measured -- so this is one +// small shared block streamed over and over rather than a per-request +// allocation of the full byte count. +var providerBandwidthTestBlock = make([]byte, 32*1024) + +// repeatingReader yields providerBandwidthTestBlock endlessly. Bounded by an +// io.LimitReader at the call site, so it never needs an end of its own. +type repeatingReader struct { + block []byte + offset int +} + +func (self *repeatingReader) Read(p []byte) (int, error) { + n := copy(p, self.block[self.offset:]) + self.offset = (self.offset + n) % len(self.block) + return n, nil +} + +// authorizeOperator applies the same operator-secret check the provider egress +// location ingest endpoint uses: the shared secret from the vault +// (operatorIngestSecret, memoized and fail-closed) compared in constant time +// against the X-UR-Operator-Secret header. These are operator-to-server +// routes, not client routes -- there is no network jwt involved. +// +// An unconfigured vault leaves the secret empty, which rejects every request +// rather than accepting every request. +func authorizeOperator(r *http.Request) bool { + secret := operatorIngestSecret() + provided := r.Header.Get(operatorSecretHeader) + return secret != "" && provided != "" && hmac.Equal([]byte(secret), []byte(provided)) +} + +// readOperatorRequestBody reads a bounded operator request body, writing the +// error response itself and reporting whether the caller should continue. +func readOperatorRequestBody(w http.ResponseWriter, r *http.Request, out any) bool { + body, err := io.ReadAll(io.LimitReader(r.Body, maxProviderBandwidthBody+1)) + if err != nil { + http.Error(w, "Bad request", http.StatusBadRequest) + return false + } + if maxProviderBandwidthBody < len(body) { + http.Error(w, "Request too large", http.StatusRequestEntityTooLarge) + return false + } + if err := json.Unmarshal(body, out); err != nil { + http.Error(w, "Bad request", http.StatusBadRequest) + return false + } + return true +} + +// ProviderBandwidthTest streams a bounded number of bytes. The active +// bandwidth probe needs something to download *through* a provider's tunnel to +// measure that tunnel's throughput; this is that target. It is +// operator-to-server, gated by the operator secret, so ordinary clients cannot +// use the deployment as a free speed-test target. +// +// The content is arbitrary -- only the byte count matters -- and it is streamed +// from a small repeating block through an io.LimitReader, never materialized +// in full. +func ProviderBandwidthTest(w http.ResponseWriter, r *http.Request) { + if !authorizeOperator(r) { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + byteCount := int64(defaultProviderBandwidthTestBytes) + if requested, err := strconv.ParseInt(r.URL.Query().Get("bytes"), 10, 64); err == nil && 0 < requested { + byteCount = requested + } + if maxProviderBandwidthTestBytes < byteCount { + byteCount = maxProviderBandwidthTestBytes + } + + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Length", strconv.FormatInt(byteCount, 10)) + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(http.StatusOK) + + source := &repeatingReader{block: providerBandwidthTestBlock} + if _, err := io.Copy(w, io.LimitReader(source, byteCount)); err != nil { + // the prober hanging up mid-download is ordinary (it stops at its own + // time or byte cap), so this is not an error worth escalating + glog.Infof("[pbw]bandwidth test stream ended early. err = %s\n", err) + } +} + +// SubmitProviderBandwidthArgs is an active bandwidth measurement taken by the +// prober over a provider's tunnel. +type SubmitProviderBandwidthArgs struct { + ClientId server.Id `json:"client_id"` + BytesPerSecond float64 `json:"bytes_per_second"` + SampleByteCount int64 `json:"sample_byte_count"` +} + +// ProviderBandwidthResult stores an active bandwidth measurement. An active +// probe is a point measurement rather than an aggregate over a window, so +// window_start and window_end are both the arrival time. +// +// A non-positive rate or sample size is not a usable measurement, and storing +// one would overwrite a real figure with a meaningless one -- so those are +// rejected before anything is written. +func ProviderBandwidthResult(w http.ResponseWriter, r *http.Request) { + if !authorizeOperator(r) { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + var args SubmitProviderBandwidthArgs + if !readOperatorRequestBody(w, r, &args) { + return + } + + if args.ClientId == (server.Id{}) { + http.Error(w, "Missing client id.", http.StatusBadRequest) + return + } + if args.BytesPerSecond <= 0 { + http.Error(w, "bytes_per_second must be positive.", http.StatusBadRequest) + return + } + if args.SampleByteCount <= 0 { + http.Error(w, "sample_byte_count must be positive.", http.StatusBadRequest) + return + } + + now := server.NowUtc() + model.StoreProviderBandwidth(r.Context(), &model.ProviderBandwidth{ + ClientId: args.ClientId, + BytesPerSecond: args.BytesPerSecond, + Source: model.ProviderBandwidthSourceActive, + SampleByteCount: args.SampleByteCount, + WindowStart: now, + WindowEnd: now, + }) + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]any{}); err != nil { + glog.Infof("[pbw]could not write response. err = %s\n", err) + } +} + +// ReserveProviderBandwidthArgs asks for budget to run one active probe. +type ReserveProviderBandwidthArgs struct { + ClientId server.Id `json:"client_id"` + ByteCount int64 `json:"byte_count"` +} + +// ReserveProviderBandwidthResult carries the reservation the prober just took. +// BucketStart is always the current hourly bucket (see ProviderBandwidthReserve). +type ReserveProviderBandwidthResult struct { + ReservationId server.Id `json:"reservation_id"` + BucketStart time.Time `json:"bucket_start"` +} + +// ProviderBandwidthReserve reserves deployment-wide byte budget for one active +// bandwidth probe. Active probing pulls real data through a provider's tunnel, +// which is real paid contract traffic, so it is rationed +// (model.ReserveProviderBandwidthSlot). +// +// The prober measures over a tunnel it already has open, right now: it has no +// use for budget in a later hour. model.ReserveProviderBandwidthSlot will +// happily defer a reservation into a future bucket for callers that can +// schedule a RunAt, so when it does that here the reservation is cancelled +// again and the request answered 429 -- the hourly ceiling would otherwise be +// decorative, since the prober could spend the whole daily budget inside one +// hour. Retry-After points at the bucket that does have room. (The plan +// specifies 429 "when every lookahead bucket is full"; this returns 429 on a +// strict superset of that, for the same "skip this provider cleanly" reason.) +func ProviderBandwidthReserve(w http.ResponseWriter, r *http.Request) { + if !authorizeOperator(r) { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + var args ReserveProviderBandwidthArgs + if !readOperatorRequestBody(w, r, &args) { + return + } + + if args.ClientId == (server.Id{}) { + http.Error(w, "Missing client id.", http.StatusBadRequest) + return + } + if args.ByteCount <= 0 { + http.Error(w, "byte_count must be positive.", http.StatusBadRequest) + return + } + // the byte count is caller-supplied; a probe never legitimately needs more + // than the per-probe figure, and an oversized request must not be able to + // swallow a large slice of a bucket in one reservation + byteCount := args.ByteCount + if model.MaxProviderBandwidthBytesPerProbe < byteCount { + byteCount = model.MaxProviderBandwidthBytesPerProbe + } + + ctx := r.Context() + now := server.NowUtc() + currentBucketStart := now.UTC().Truncate(model.ProviderBandwidthBucketDuration) + + reservationId, bucketStart, err := model.ReserveProviderBandwidthSlot(ctx, args.ClientId, byteCount) + if err != nil { + // every bucket in the lookahead window is full: the deployment's daily + // budget is exhausted + writeProviderBandwidthBudgetExhausted(w, currentBucketStart.Add(model.ProviderBandwidthBucketDuration).Sub(now), err.Error()) + return + } + if bucketStart.After(currentBucketStart) { + // budget exists, but not until a later hour -- of no use to a probe + // that runs now, so give it back rather than burning it on a request + // that is about to be skipped + model.CancelProviderBandwidthReservation(ctx, reservationId) + writeProviderBandwidthBudgetExhausted(w, bucketStart.Sub(now), "The active bandwidth probe budget for this hour has been reached.") + return + } + + w.Header().Set("Content-Type", "application/json") + result := &ReserveProviderBandwidthResult{ + ReservationId: reservationId, + BucketStart: bucketStart, + } + if err := json.NewEncoder(w).Encode(result); err != nil { + glog.Infof("[pbw]could not write response. err = %s\n", err) + } +} + +func writeProviderBandwidthBudgetExhausted(w http.ResponseWriter, retryAfter time.Duration, message string) { + retryAfterSeconds := int64(retryAfter.Seconds()) + if retryAfterSeconds < 1 { + retryAfterSeconds = 1 + } + w.Header().Set("Retry-After", strconv.FormatInt(retryAfterSeconds, 10)) + http.Error(w, message, http.StatusTooManyRequests) +} diff --git a/api/handlers/provider_bandwidth_handlers_test.go b/api/handlers/provider_bandwidth_handlers_test.go new file mode 100644 index 00000000..0a8ae0bb --- /dev/null +++ b/api/handlers/provider_bandwidth_handlers_test.go @@ -0,0 +1,564 @@ +package handlers + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "testing" + "time" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" +) + +// --- download endpoint ------------------------------------------------------- +// +// These tests touch no database: the endpoint only streams bytes, so they run +// outside DefaultTestEnv. + +func TestProviderBandwidthTestRejectsMissingSecret(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/network/provider-bandwidth-test?bytes=1024", nil) + w := httptest.NewRecorder() + + ProviderBandwidthTest(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret header is absent", w.Code) + } + if 0 < w.Body.Len() && w.Body.Len() != len("Unauthorized\n") { + t.Fatalf("body length = %d, want no payload streamed to an unauthenticated caller", w.Body.Len()) + } +} + +func TestProviderBandwidthTestRejectsWrongSecret(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/network/provider-bandwidth-test?bytes=1024", nil) + req.Header.Set(operatorSecretHeader, "definitely-not-the-secret") + w := httptest.NewRecorder() + + ProviderBandwidthTest(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 on a wrong operator secret", w.Code) + } +} + +// TestProviderBandwidthTestRejectsAlteredSecret proves hmac.Equal is actually +// consulted once the vault is configured, rather than the endpoint accepting +// anything once secret != "". +func TestProviderBandwidthTestRejectsAlteredSecret(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + const wrongSecret = "correct-operator-secret-0123456780" // last char changed + defer withStubOperatorIngestSecret(secret)() + + req := httptest.NewRequest(http.MethodGet, "/network/provider-bandwidth-test?bytes=1024", nil) + req.Header.Set(operatorSecretHeader, wrongSecret) + w := httptest.NewRecorder() + + ProviderBandwidthTest(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the configured secret and the request's secret differ", w.Code) + } +} + +// TestProviderBandwidthTestStreamsRequestedByteCount is the accept-path test +// with teeth: a handler that authenticates correctly but streams nothing (or +// the wrong amount) fails here, and so would an unconditional 401. +func TestProviderBandwidthTestStreamsRequestedByteCount(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + req := httptest.NewRequest(http.MethodGet, "/network/provider-bandwidth-test?bytes=1048576", nil) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthTest(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 with the correct operator secret", w.Code) + } + if w.Body.Len() != 1048576 { + t.Fatalf("streamed %d bytes, want exactly the requested 1048576", w.Body.Len()) + } + if contentType := w.Header().Get("Content-Type"); contentType != "application/octet-stream" { + t.Fatalf("Content-Type = %q, want application/octet-stream", contentType) + } + // httptest.NewRecorder does not enforce Content-Length, so a handler that + // claims one size and writes another would otherwise pass the check above. + assertContentLengthMatchesBody(t, w) +} + +// TestProviderBandwidthTestClampsToMaximum: an unclamped stream is an +// open-ended resource commitment per request. +func TestProviderBandwidthTestClampsToMaximum(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + // two orders of magnitude beyond the cap + req := httptest.NewRequest(http.MethodGet, "/network/provider-bandwidth-test?bytes=536870912", nil) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthTest(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", w.Code) + } + if w.Body.Len() != maxProviderBandwidthTestBytes { + t.Fatalf("streamed %d bytes for a 536870912-byte request, want it clamped to %d", + w.Body.Len(), maxProviderBandwidthTestBytes) + } + assertContentLengthMatchesBody(t, w) +} + +// TestProviderBandwidthTestDefaultsUnusableByteCounts: `bytes=0` and +// `bytes=-1` parse cleanly, so they are not caught by a malformed-input check. +// Streaming an empty body for them would hand the prober a zero-byte sample to +// divide by. +func TestProviderBandwidthTestDefaultsUnusableByteCounts(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + for _, query := range []string{"", "?bytes=0", "?bytes=-1", "?bytes=not-a-number"} { + req := httptest.NewRequest(http.MethodGet, "/network/provider-bandwidth-test"+query, nil) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthTest(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("query %q: status = %d, want 200", query, w.Code) + } + if w.Body.Len() != defaultProviderBandwidthTestBytes { + t.Fatalf("query %q: streamed %d bytes, want the %d-byte default", + query, w.Body.Len(), defaultProviderBandwidthTestBytes) + } + } +} + +func assertContentLengthMatchesBody(t *testing.T, w *httptest.ResponseRecorder) { + t.Helper() + contentLength := w.Header().Get("Content-Length") + if contentLength == "" { + return + } + declared, err := strconv.Atoi(contentLength) + if err != nil { + t.Fatalf("Content-Length = %q, not a number", contentLength) + } + if declared != w.Body.Len() { + t.Fatalf("Content-Length = %d but %d bytes were written", declared, w.Body.Len()) + } +} + +// --- result endpoint --------------------------------------------------------- + +func TestProviderBandwidthResultRejectsMissingSecret(t *testing.T) { + body, _ := json.Marshal(map[string]any{ + "client_id": "019f8835-158d-6fd8-e9dd-fd0e4c6d6792", + "bytes_per_second": 1000000.0, + "sample_byte_count": 5242880, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-result", bytes.NewReader(body)) + w := httptest.NewRecorder() + + ProviderBandwidthResult(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret header is absent", w.Code) + } +} + +func TestProviderBandwidthResultRejectsWrongSecret(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + const wrongSecret = "correct-operator-secret-0123456780" + defer withStubOperatorIngestSecret(secret)() + + body, _ := json.Marshal(map[string]any{ + "client_id": "019f8835-158d-6fd8-e9dd-fd0e4c6d6792", + "bytes_per_second": 1000000.0, + "sample_byte_count": 5242880, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-result", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, wrongSecret) + w := httptest.NewRecorder() + + ProviderBandwidthResult(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 on a wrong operator secret", w.Code) + } +} + +// TestProviderBandwidthResultRejectsNonPositiveMeasurement: a zero or negative +// measurement is not a usable sample, and storing one would overwrite a real +// figure with a meaningless one. +func TestProviderBandwidthResultRejectsNonPositiveMeasurement(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + + cases := []struct { + name string + bytesPerSecond float64 + sampleByteCount int64 + }{ + {"zero rate", 0, 5 * 1024 * 1024}, + {"negative rate", -1, 5 * 1024 * 1024}, + {"zero sample", 1000000, 0}, + {"negative sample", 1000000, -1}, + } + for _, c := range cases { + clientId := server.NewId() + body, _ := json.Marshal(map[string]any{ + "client_id": clientId, + "bytes_per_second": c.bytesPerSecond, + "sample_byte_count": c.sampleByteCount, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-result", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthResult(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("%s: status = %d, want 400; body = %s", c.name, w.Code, w.Body.String()) + } + if count := countProviderBandwidthRows(ctx, clientId); count != 0 { + t.Fatalf("%s: %d provider_bandwidth rows written, want the submission rejected before storage", c.name, count) + } + } + }) +} + +// TestProviderBandwidthResultStoresAnActiveMeasurement is the accept-path test +// with teeth: it reads provider_bandwidth back with raw SQL, so a handler that +// authenticates correctly but never calls model.StoreProviderBandwidth fails +// here -- as would an unconditional 401. +func TestProviderBandwidthResultStoresAnActiveMeasurement(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + clientId := server.NewId() + const bytesPerSecond = 1234567.5 + const sampleByteCount int64 = 3 * 1024 * 1024 + + body, _ := json.Marshal(map[string]any{ + "client_id": clientId, + "bytes_per_second": bytesPerSecond, + "sample_byte_count": sampleByteCount, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-result", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthResult(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 for a valid submission; body = %s", w.Code, w.Body.String()) + } + + var gotSource string + var gotBytesPerSecond float64 + var gotSampleByteCount int64 + var found bool + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + `SELECT source, bytes_per_second, sample_byte_count FROM provider_bandwidth WHERE client_id = $1`, + clientId, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + found = true + server.Raise(result.Scan(&gotSource, &gotBytesPerSecond, &gotSampleByteCount)) + } + }) + }) + + if !found { + t.Fatal("no provider_bandwidth row for the submitted client; the handler never stored the measurement") + } + if gotSource != model.ProviderBandwidthSourceActive { + t.Fatalf("source = %q, want %q", gotSource, model.ProviderBandwidthSourceActive) + } + if gotBytesPerSecond != bytesPerSecond { + t.Fatalf("bytes_per_second = %f, want the submitted %f", gotBytesPerSecond, bytesPerSecond) + } + if gotSampleByteCount != sampleByteCount { + t.Fatalf("sample_byte_count = %d, want the submitted %d", gotSampleByteCount, sampleByteCount) + } + }) +} + +// TestProviderBandwidthPostsRejectMissingClientId: an absent client_id +// unmarshals to the zero id, which would otherwise be written as a real row +// keyed on the nil uuid. +func TestProviderBandwidthPostsRejectMissingClientId(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + resultBody, _ := json.Marshal(map[string]any{ + "bytes_per_second": 1000000.0, + "sample_byte_count": 5242880, + }) + reserveBody, _ := json.Marshal(map[string]any{ + "byte_count": 5 * 1024 * 1024, + }) + + cases := []struct { + name string + path string + body []byte + handler func(http.ResponseWriter, *http.Request) + }{ + {"result", "/network/provider-bandwidth-result", resultBody, ProviderBandwidthResult}, + {"reserve", "/network/provider-bandwidth-reserve", reserveBody, ProviderBandwidthReserve}, + } + for _, c := range cases { + req := httptest.NewRequest(http.MethodPost, c.path, bytes.NewReader(c.body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + c.handler(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("%s: status = %d, want 400 for a submission with no client id", c.name, w.Code) + } + } +} + +func countProviderBandwidthRows(ctx context.Context, clientId server.Id) int { + count := 0 + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + `SELECT COUNT(*) FROM provider_bandwidth WHERE client_id = $1`, + clientId, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + server.Raise(result.Scan(&count)) + } + }) + }) + return count +} + +// --- reservation endpoint ---------------------------------------------------- + +func TestProviderBandwidthReserveRejectsMissingSecret(t *testing.T) { + body, _ := json.Marshal(map[string]any{ + "client_id": "019f8835-158d-6fd8-e9dd-fd0e4c6d6792", + "byte_count": 5 * 1024 * 1024, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-reserve", bytes.NewReader(body)) + w := httptest.NewRecorder() + + ProviderBandwidthReserve(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret header is absent", w.Code) + } +} + +func TestProviderBandwidthReserveRejectsWrongSecret(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + const wrongSecret = "correct-operator-secret-0123456780" + defer withStubOperatorIngestSecret(secret)() + + body, _ := json.Marshal(map[string]any{ + "client_id": "019f8835-158d-6fd8-e9dd-fd0e4c6d6792", + "byte_count": 5 * 1024 * 1024, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-reserve", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, wrongSecret) + w := httptest.NewRecorder() + + ProviderBandwidthReserve(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 on a wrong operator secret", w.Code) + } +} + +// TestProviderBandwidthReserveReservesBudget is the accept-path test with +// teeth: it asserts the ledger row exists, so a handler that authenticates and +// returns 200 without reserving anything fails -- as would an unconditional +// 401. +func TestProviderBandwidthReserveReservesBudget(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + clientId := server.NewId() + + body, _ := json.Marshal(map[string]any{ + "client_id": clientId, + "byte_count": model.MaxProviderBandwidthBytesPerProbe, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-reserve", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthReserve(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 for a reservation against an empty budget; body = %s", w.Code, w.Body.String()) + } + + reservedCount, reservedBytes := providerBandwidthQuotaFor(ctx, clientId) + if reservedCount != 1 { + t.Fatalf("%d provider_bandwidth_quota rows, want exactly 1 -- the handler must actually reserve budget", reservedCount) + } + if reservedBytes != model.MaxProviderBandwidthBytesPerProbe { + t.Fatalf("reserved %d bytes, want %d", reservedBytes, model.MaxProviderBandwidthBytesPerProbe) + } + + var result ReserveProviderBandwidthResult + if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { + t.Fatalf("decode response: %s; body = %s", err, w.Body.String()) + } + if result.ReservationId == (server.Id{}) { + t.Fatal("response carried no reservation id") + } + }) +} + +// TestProviderBandwidthReserveClampsByteCount: the byte count is caller- +// supplied, so an oversized (or fat-fingered) request must not be able to +// consume a whole bucket in one reservation. +func TestProviderBandwidthReserveClampsByteCount(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + clientId := server.NewId() + + body, _ := json.Marshal(map[string]any{ + "client_id": clientId, + "byte_count": 100 * model.MaxProviderBandwidthBytesPerProbe, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-reserve", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthReserve(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body = %s", w.Code, w.Body.String()) + } + _, reservedBytes := providerBandwidthQuotaFor(ctx, clientId) + if reservedBytes != model.MaxProviderBandwidthBytesPerProbe { + t.Fatalf("reserved %d bytes for an oversized request, want it clamped to %d", + reservedBytes, model.MaxProviderBandwidthBytesPerProbe) + } + }) +} + +func TestProviderBandwidthReserveRejectsNonPositiveByteCount(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + for _, byteCount := range []int64{0, -1} { + clientId := server.NewId() + body, _ := json.Marshal(map[string]any{ + "client_id": clientId, + "byte_count": byteCount, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-reserve", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthReserve(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("byte_count %d: status = %d, want 400", byteCount, w.Code) + } + if count, _ := providerBandwidthQuotaFor(ctx, clientId); count != 0 { + t.Fatalf("byte_count %d: %d quota rows written for a rejected request", byteCount, count) + } + } + }) +} + +// TestProviderBandwidthReserveReturns429WhenCurrentBucketIsFull: the prober +// probes over an already-open tunnel right now, so a reservation deferred to a +// later hour is of no use to it -- the endpoint cancels that deferred +// reservation and answers 429, which the prober treats as "skip this +// provider". Leaving the deferred reservation in place would burn budget on +// every skipped provider. +func TestProviderBandwidthReserveReturns429WhenCurrentBucketIsFull(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + // fill the current hourly bucket exactly to its ceiling + bucketStart := server.NowUtc().UTC().Truncate(time.Hour) + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO provider_bandwidth_quota (provider_bandwidth_quota_id, client_id, byte_count, bucket_start, create_time) + VALUES ($1, $2, $3, $4, $5) + `, + server.NewId(), server.NewId(), model.MaxProviderBandwidthBytesPerBucket, bucketStart, server.NowUtc(), + )) + }) + + clientId := server.NewId() + body, _ := json.Marshal(map[string]any{ + "client_id": clientId, + "byte_count": model.MaxProviderBandwidthBytesPerProbe, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-reserve", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthReserve(w, req) + + if w.Code != http.StatusTooManyRequests { + t.Fatalf("status = %d, want 429 when the current hourly bucket has no room; body = %s", w.Code, w.Body.String()) + } + if count, _ := providerBandwidthQuotaFor(ctx, clientId); count != 0 { + t.Fatalf("%d quota rows left behind for a 429'd request, want 0 -- a deferred reservation must be cancelled", count) + } + if retryAfter := w.Header().Get("Retry-After"); retryAfter == "" { + t.Fatal("no Retry-After header on the 429; the prober cannot tell when budget frees up") + } + }) +} + +func providerBandwidthQuotaFor(ctx context.Context, clientId server.Id) (count int, byteCount int64) { + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + `SELECT COUNT(*), COALESCE(SUM(byte_count), 0)::bigint FROM provider_bandwidth_quota WHERE client_id = $1`, + clientId, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + server.Raise(result.Scan(&count, &byteCount)) + } + }) + }) + return count, byteCount +} From b000c11036f96fcb63259d84576ee3ce33feb614 Mon Sep 17 00:00:00 2001 From: Ryan Mello Date: Fri, 31 Jul 2026 13:26:55 +0100 Subject: [PATCH 24/30] feat(schema): key provider_bandwidth on (client_id, source) for two targets The active probe measures two independent targets per provider -- the operator's own download endpoint and a public CDN -- and the two figures are the point. A provider that prioritises one path and not the other is invisible in a single number and obvious in a pair. Keyed on client_id alone the two overwrite each other on every pass, so only one target could be stored at all; averaging them into the one row would lose the same signal more quietly. The source becomes part of the key rather than each target getting its own columns, because a further target then needs no migration: passive, active-operator and active-cdn are three rows in one shape, and a fourth would be a fourth row. Per-target columns would need a migration each. Migration is a strict tail append and is idempotent (DROP CONSTRAINT IF EXISTS paired with the ADD). No backfill: provider_bandwidth is empty on every deployment -- the active prober is its first writer and ships with this change -- so re-keying cannot orphan or collide with an existing row. The result endpoint now carries the source and VALIDATES it against the known active set instead of hardcoding one tag. Two failures are closed off. An unrecognised source is not a harmless label: it writes a row under a tag nothing will ever read or replace, while the submitter goes on looking like it works. And "passive" is refused specifically -- that figure is derived server-side from bytes the provider has already been paid to carry, which is exactly what makes it ungameable, so accepting a submitted one would let this endpoint overwrite a derived figure with an asserted one, through a secret that now travels over a provider-controlled path. Co-Authored-By: Claude Opus 5 Claude-Session: https://claude.ai/code/session_01QtgqtCmKJRXdsQ5ktiqwkg --- api/handlers/provider_bandwidth_handlers.go | 32 ++- .../provider_bandwidth_handlers_test.go | 216 ++++++++++++++---- db_migrations.go | 34 +++ model/provider_bandwidth_model.go | 79 +++++-- model/provider_bandwidth_model_test.go | 118 +++++++--- 5 files changed, 392 insertions(+), 87 deletions(-) diff --git a/api/handlers/provider_bandwidth_handlers.go b/api/handlers/provider_bandwidth_handlers.go index eabc7e90..cb67e88d 100644 --- a/api/handlers/provider_bandwidth_handlers.go +++ b/api/handlers/provider_bandwidth_handlers.go @@ -3,6 +3,7 @@ package handlers import ( "crypto/hmac" "encoding/json" + "fmt" "io" "net/http" "strconv" @@ -123,9 +124,14 @@ func ProviderBandwidthTest(w http.ResponseWriter, r *http.Request) { // SubmitProviderBandwidthArgs is an active bandwidth measurement taken by the // prober over a provider's tunnel. type SubmitProviderBandwidthArgs struct { - ClientId server.Id `json:"client_id"` - BytesPerSecond float64 `json:"bytes_per_second"` - SampleByteCount int64 `json:"sample_byte_count"` + ClientId server.Id `json:"client_id"` + // Source names which target produced this figure + // (model.ProviderBandwidthSourceActiveOperator or ...ActiveCDN). It is + // part of the storage key, so it is what keeps the two targets' figures in + // separate rows instead of overwriting each other. + Source string `json:"source"` + BytesPerSecond float64 `json:"bytes_per_second"` + SampleByteCount int64 `json:"sample_byte_count"` } // ProviderBandwidthResult stores an active bandwidth measurement. An active @@ -135,6 +141,16 @@ type SubmitProviderBandwidthArgs struct { // A non-positive rate or sample size is not a usable measurement, and storing // one would overwrite a real figure with a meaningless one -- so those are // rejected before anything is written. +// +// The source is validated against the known ACTIVE set rather than merely +// stored. Two distinct failures are being closed off. An unrecognised source +// is not a harmless label: the row is keyed on (client_id, source), so it +// creates a row nothing will ever read or replace, and a prober with a typo'd +// tag would look like it was working while writing to a tag no consumer knows. +// And "passive" is refused specifically: that figure is derived server-side +// from bytes the provider has already been paid to carry, which is exactly +// what makes it ungameable, so accepting a submitted one would let this +// endpoint overwrite the derived figure with an asserted one. func ProviderBandwidthResult(w http.ResponseWriter, r *http.Request) { if !authorizeOperator(r) { http.Error(w, "Unauthorized", http.StatusUnauthorized) @@ -158,12 +174,20 @@ func ProviderBandwidthResult(w http.ResponseWriter, r *http.Request) { http.Error(w, "sample_byte_count must be positive.", http.StatusBadRequest) return } + if !model.IsSubmittableProviderBandwidthSource(args.Source) { + http.Error(w, fmt.Sprintf( + "source must be one of %s, %s.", + model.ProviderBandwidthSourceActiveOperator, + model.ProviderBandwidthSourceActiveCDN, + ), http.StatusBadRequest) + return + } now := server.NowUtc() model.StoreProviderBandwidth(r.Context(), &model.ProviderBandwidth{ ClientId: args.ClientId, BytesPerSecond: args.BytesPerSecond, - Source: model.ProviderBandwidthSourceActive, + Source: args.Source, SampleByteCount: args.SampleByteCount, WindowStart: now, WindowEnd: now, diff --git a/api/handlers/provider_bandwidth_handlers_test.go b/api/handlers/provider_bandwidth_handlers_test.go index 0a8ae0bb..9313daa9 100644 --- a/api/handlers/provider_bandwidth_handlers_test.go +++ b/api/handlers/provider_bandwidth_handlers_test.go @@ -159,6 +159,7 @@ func assertContentLengthMatchesBody(t *testing.T, w *httptest.ResponseRecorder) func TestProviderBandwidthResultRejectsMissingSecret(t *testing.T) { body, _ := json.Marshal(map[string]any{ "client_id": "019f8835-158d-6fd8-e9dd-fd0e4c6d6792", + "source": model.ProviderBandwidthSourceActiveOperator, "bytes_per_second": 1000000.0, "sample_byte_count": 5242880, }) @@ -179,6 +180,7 @@ func TestProviderBandwidthResultRejectsWrongSecret(t *testing.T) { body, _ := json.Marshal(map[string]any{ "client_id": "019f8835-158d-6fd8-e9dd-fd0e4c6d6792", + "source": model.ProviderBandwidthSourceActiveOperator, "bytes_per_second": 1000000.0, "sample_byte_count": 5242880, }) @@ -218,6 +220,7 @@ func TestProviderBandwidthResultRejectsNonPositiveMeasurement(t *testing.T) { clientId := server.NewId() body, _ := json.Marshal(map[string]any{ "client_id": clientId, + "source": model.ProviderBandwidthSourceActiveOperator, "bytes_per_second": c.bytesPerSecond, "sample_byte_count": c.sampleByteCount, }) @@ -240,7 +243,8 @@ func TestProviderBandwidthResultRejectsNonPositiveMeasurement(t *testing.T) { // TestProviderBandwidthResultStoresAnActiveMeasurement is the accept-path test // with teeth: it reads provider_bandwidth back with raw SQL, so a handler that // authenticates correctly but never calls model.StoreProviderBandwidth fails -// here -- as would an unconditional 401. +// here -- as would an unconditional 401. Both active targets are covered, +// because the source now decides which row is written. func TestProviderBandwidthResultStoresAnActiveMeasurement(t *testing.T) { t.Setenv("WARP_ENV", "local") server.DefaultTestEnv().Run(t, func(t testing.TB) { @@ -248,58 +252,189 @@ func TestProviderBandwidthResultStoresAnActiveMeasurement(t *testing.T) { defer withStubOperatorIngestSecret(secret)() ctx := context.Background() - clientId := server.NewId() - const bytesPerSecond = 1234567.5 - const sampleByteCount int64 = 3 * 1024 * 1024 - body, _ := json.Marshal(map[string]any{ - "client_id": clientId, - "bytes_per_second": bytesPerSecond, - "sample_byte_count": sampleByteCount, - }) - req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-result", bytes.NewReader(body)) - req.Header.Set(operatorSecretHeader, secret) - w := httptest.NewRecorder() + cases := []struct { + name string + source string + }{ + {"operator target", model.ProviderBandwidthSourceActiveOperator}, + {"cdn target", model.ProviderBandwidthSourceActiveCDN}, + } + for _, c := range cases { + clientId := server.NewId() + const bytesPerSecond = 1234567.5 + const sampleByteCount int64 = 3 * 1024 * 1024 + + body, _ := json.Marshal(map[string]any{ + "client_id": clientId, + "source": c.source, + "bytes_per_second": bytesPerSecond, + "sample_byte_count": sampleByteCount, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-result", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthResult(w, req) - ProviderBandwidthResult(w, req) + if w.Code != http.StatusOK { + t.Fatalf("%s: status = %d, want 200 for a valid submission; body = %s", c.name, w.Code, w.Body.String()) + } - if w.Code != http.StatusOK { - t.Fatalf("status = %d, want 200 for a valid submission; body = %s", w.Code, w.Body.String()) + stored := readProviderBandwidthBySource(ctx, clientId) + row, ok := stored[c.source] + if !ok { + t.Fatalf("%s: no provider_bandwidth row under source %q; the handler never stored the measurement", c.name, c.source) + } + if row.bytesPerSecond != bytesPerSecond { + t.Fatalf("%s: bytes_per_second = %f, want the submitted %f", c.name, row.bytesPerSecond, bytesPerSecond) + } + if row.sampleByteCount != sampleByteCount { + t.Fatalf("%s: sample_byte_count = %d, want the submitted %d", c.name, row.sampleByteCount, sampleByteCount) + } } + }) +} - var gotSource string - var gotBytesPerSecond float64 - var gotSampleByteCount int64 - var found bool - server.Db(ctx, func(conn server.PgConn) { - result, err := conn.Query( - ctx, - `SELECT source, bytes_per_second, sample_byte_count FROM provider_bandwidth WHERE client_id = $1`, - clientId, - ) - server.WithPgResult(result, err, func() { - if result.Next() { - found = true - server.Raise(result.Scan(&gotSource, &gotBytesPerSecond, &gotSampleByteCount)) - } - }) - }) +// TestProviderBandwidthResultRejectsUnknownSource: the source is part of the +// storage key, so it is validated rather than merely stored. +// +// An unrecognised value is not a harmless label -- it creates a row under a tag +// nothing will ever read or replace, while the submitter goes on looking like +// it is working. "passive" is refused for a sharper reason: that figure is +// derived server-side from bytes the provider has already been paid to carry, +// which is precisely what makes it ungameable, and accepting a submitted one +// would let this endpoint overwrite the derived figure with an asserted one. +func TestProviderBandwidthResultRejectsUnknownSource(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() - if !found { - t.Fatal("no provider_bandwidth row for the submitted client; the handler never stored the measurement") + cases := []struct { + name string + source any + }{ + {"absent", nil}, + {"empty", ""}, + {"the pre-split active tag", "active"}, + {"a typo", "active-cnd"}, + {"wrong case", "ACTIVE-CDN"}, + {"server-derived passive", model.ProviderBandwidthSourcePassive}, } - if gotSource != model.ProviderBandwidthSourceActive { - t.Fatalf("source = %q, want %q", gotSource, model.ProviderBandwidthSourceActive) + for _, c := range cases { + clientId := server.NewId() + payload := map[string]any{ + "client_id": clientId, + "bytes_per_second": 1234567.5, + "sample_byte_count": 3 * 1024 * 1024, + } + if c.source != nil { + payload["source"] = c.source + } + body, _ := json.Marshal(payload) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-result", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthResult(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("%s (source=%v): status = %d, want 400; body = %s", c.name, c.source, w.Code, w.Body.String()) + } + if count := countProviderBandwidthRows(ctx, clientId); count != 0 { + t.Fatalf("%s (source=%v): %d provider_bandwidth rows written, want the submission rejected before storage", c.name, c.source, count) + } + } + }) +} + +// TestProviderBandwidthResultStoresTheTwoTargetsSeparately is the end-to-end +// form of the property the second target exists for: two submissions for ONE +// provider, one per target, must land in two rows carrying two figures. +// +// Against the old client_id-only key the second submission overwrote the +// first, so a provider prioritising the operator's own path while starving the +// public internet looked identical to one that was simply fast -- the exact +// divergence this is here to expose. +func TestProviderBandwidthResultStoresTheTwoTargetsSeparately(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + clientId := server.NewId() + + submitted := map[string]float64{ + model.ProviderBandwidthSourceActiveOperator: 12_000_000, + model.ProviderBandwidthSourceActiveCDN: 3_000_000, + } + for source, bytesPerSecond := range submitted { + body, _ := json.Marshal(map[string]any{ + "client_id": clientId, + "source": source, + "bytes_per_second": bytesPerSecond, + "sample_byte_count": 5 * 1024 * 1024, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-result", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthResult(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("source %q: status = %d, want 200; body = %s", source, w.Code, w.Body.String()) + } } - if gotBytesPerSecond != bytesPerSecond { - t.Fatalf("bytes_per_second = %f, want the submitted %f", gotBytesPerSecond, bytesPerSecond) + + stored := readProviderBandwidthBySource(ctx, clientId) + if len(stored) != 2 { + t.Fatalf("%d provider_bandwidth rows for one provider, want 2 (one per target) -- the two targets are overwriting each other", len(stored)) } - if gotSampleByteCount != sampleByteCount { - t.Fatalf("sample_byte_count = %d, want the submitted %d", gotSampleByteCount, sampleByteCount) + for source, want := range submitted { + row, ok := stored[source] + if !ok { + t.Fatalf("no row under source %q", source) + } + if row.bytesPerSecond != want { + t.Errorf("source %q stored %.0f B/s, want the submitted %.0f -- the figures are not being kept apart (an averaged pair would read %.0f)", + source, row.bytesPerSecond, want, (submitted[model.ProviderBandwidthSourceActiveOperator]+submitted[model.ProviderBandwidthSourceActiveCDN])/2) + } } }) } +type storedProviderBandwidth struct { + bytesPerSecond float64 + sampleByteCount int64 +} + +// readProviderBandwidthBySource reads the rows back with raw SQL, keyed by +// source, so the assertions are against the table rather than against the +// writer that produced it. +func readProviderBandwidthBySource(ctx context.Context, clientId server.Id) map[string]storedProviderBandwidth { + bySource := map[string]storedProviderBandwidth{} + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + `SELECT source, bytes_per_second, sample_byte_count FROM provider_bandwidth WHERE client_id = $1`, + clientId, + ) + server.WithPgResult(result, err, func() { + for result.Next() { + var source string + var row storedProviderBandwidth + server.Raise(result.Scan(&source, &row.bytesPerSecond, &row.sampleByteCount)) + bySource[source] = row + } + }) + }) + return bySource +} + // TestProviderBandwidthPostsRejectMissingClientId: an absent client_id // unmarshals to the zero id, which would otherwise be written as a real row // keyed on the nil uuid. @@ -308,6 +443,7 @@ func TestProviderBandwidthPostsRejectMissingClientId(t *testing.T) { defer withStubOperatorIngestSecret(secret)() resultBody, _ := json.Marshal(map[string]any{ + "source": model.ProviderBandwidthSourceActiveOperator, "bytes_per_second": 1000000.0, "sample_byte_count": 5242880, }) diff --git a/db_migrations.go b/db_migrations.go index b0030cf7..88aba23c 100644 --- a/db_migrations.go +++ b/db_migrations.go @@ -4507,6 +4507,10 @@ var migrations = []any{ // (mirrors provider_egress_location's own shape). This is a ranking input, // not a history: keeping every sample would grow without bound for a value // only ever read as "the current figure". + // + // SUPERSEDED: a later migration in this file re-keys the table on + // (client_id, source) so each source keeps its own current figure. The + // statement below is left exactly as applied -- see that migration for why. newSqlMigration(` CREATE TABLE IF NOT EXISTS provider_bandwidth ( client_id uuid NOT NULL PRIMARY KEY, @@ -4560,4 +4564,34 @@ var migrations = []any{ CREATE INDEX IF NOT EXISTS provider_bandwidth_quota_bucket_start ON provider_bandwidth_quota (bucket_start) `), + + // Re-key provider_bandwidth on (client_id, source). + // + // The active probe measures two independent targets per provider -- the + // operator's own download endpoint and a public CDN -- and the two figures + // are the point: a provider that prioritises one path and not the other is + // invisible in a single number and obvious in a pair. Keyed on client_id + // alone the two overwrite each other on every pass, so only one target can + // be stored at all. Averaging them into the one row would lose the same + // signal more quietly. + // + // The source becomes part of the key rather than each target getting its + // own columns, because a further target then needs no migration at all: + // 'passive', 'active-operator' and 'active-cdn' are three rows in the same + // shape, and a fourth would be a fourth row. + // + // No backfill: provider_bandwidth is empty on every deployment (nothing has + // written to it yet -- the active prober is the first writer and ships with + // this change), so re-keying cannot orphan or collide with an existing row. + // + // Appended, never inserted: migrations apply by slice index + // (`for i := DbVersion(ctx); i < upTo; i++`), so editing or reordering an + // already-applied entry corrupts live databases. DROP CONSTRAINT IF EXISTS + // paired with the ADD makes the whole statement idempotent -- a re-run + // drops whatever primary key is present and re-adds this one. + newSqlMigration(` + ALTER TABLE provider_bandwidth + DROP CONSTRAINT IF EXISTS provider_bandwidth_pkey, + ADD CONSTRAINT provider_bandwidth_pkey PRIMARY KEY (client_id, source) + `), } diff --git a/model/provider_bandwidth_model.go b/model/provider_bandwidth_model.go index 1e4522ac..3630db19 100644 --- a/model/provider_bandwidth_model.go +++ b/model/provider_bandwidth_model.go @@ -7,17 +7,63 @@ import ( "github.com/urnetwork/server" ) -// bandwidth sources. A stored figure is tagged with the source that produced -// it so consumers never have to know which one did. +// The bandwidth sources. A stored figure is tagged with the source that +// produced it, and the row is keyed on (client_id, source), so every source +// keeps its own figure and none overwrites another. +// +// Adding a further active target later needs a constant here and nothing else +// -- no migration, no new column. That generality is why the source is a key +// column rather than a set of per-target columns. const ( // ProviderBandwidthSourcePassive is derived from already-settled contract - // bytes: zero additional cost, and it cannot be gamed selectively. + // bytes: zero additional cost, and it cannot be gamed selectively. It is + // computed server-side by ComputePassiveProviderBandwidth and is the one + // source no prober may submit -- see IsSubmittableProviderBandwidthSource. ProviderBandwidthSourcePassive = "passive" - // ProviderBandwidthSourceActive is a sampled download over the provider's - // tunnel, used only where passive history does not exist yet. - ProviderBandwidthSourceActive = "active" + // ProviderBandwidthSourceActiveOperator is a sampled download from the + // operator's own endpoint, over the provider's tunnel. + ProviderBandwidthSourceActiveOperator = "active-operator" + // ProviderBandwidthSourceActiveCDN is the same sample taken against a + // public CDN over the same tunnel. + // + // The two active figures are stored separately and must never be averaged + // into one. A provider that prioritises the operator's own path while + // starving the internet at large is invisible in a combined number and + // obvious in a pair -- which is the entire reason a second target exists. + ProviderBandwidthSourceActiveCDN = "active-cdn" ) +// IsProviderBandwidthSource reports whether source is one this deployment +// knows. Storage is keyed on the source, so an unrecognised value is not a +// harmless label: it silently creates a row nothing will ever read or replace. +func IsProviderBandwidthSource(source string) bool { + switch source { + case ProviderBandwidthSourcePassive, + ProviderBandwidthSourceActiveOperator, + ProviderBandwidthSourceActiveCDN: + return true + } + return false +} + +// IsSubmittableProviderBandwidthSource reports whether source may arrive from +// outside, over the result endpoint. +// +// It is the active subset, deliberately: "passive" is derived server-side from +// bytes a provider has already been paid to carry, which is exactly what makes +// it ungameable. Accepting a submitted "passive" row would let the submitter +// overwrite that derived figure with an asserted one -- through an endpoint +// whose secret now travels over a provider-controlled path -- and destroy the +// one bandwidth signal in the system that cannot be gamed selectively. +func IsSubmittableProviderBandwidthSource(source string) bool { + switch source { + case ProviderBandwidthSourceActiveOperator, + ProviderBandwidthSourceActiveCDN: + return true + } + return false +} + // ProviderBandwidth is one throughput figure for a provider. It is advisory: // nothing may gate provider selection on it. type ProviderBandwidth struct { @@ -119,12 +165,20 @@ func ComputePassiveProviderBandwidth( }, nil } -// StoreProviderBandwidth records a provider's current throughput figure, -// whatever produced it. The row is keyed on client_id, so a new measurement -// replaces the previous one: this is the current figure a consumer reads, not a -// history (see the provider_bandwidth migration). +// StoreProviderBandwidth records a provider's current throughput figure for +// one source. The row is keyed on (client_id, source), so a new measurement +// replaces the previous one FROM THE SAME SOURCE and leaves the other sources +// alone: this is the current figure per source a consumer reads, not a history +// (see the provider_bandwidth migrations). +// +// The source is part of the key rather than a plain column, and that is what +// makes two active targets possible at all: keyed on client_id alone, the +// operator and cdn measurements for one provider would overwrite each other on +// every pass and the divergence between them -- the entire reason there is a +// second target -- would never be visible. Averaging them into one row would +// destroy the same signal more quietly. // -// Both sources write through here, tagged by bw.Source, so nothing downstream +// Every source writes through here, tagged by bw.Source, so nothing downstream // has to know whether a figure came from settled traffic or an active sample. // The figure is advisory -- storing one must never gate provider selection. func StoreProviderBandwidth(ctx context.Context, bw *ProviderBandwidth) { @@ -142,10 +196,9 @@ func StoreProviderBandwidth(ctx context.Context, bw *ProviderBandwidth) { update_time ) VALUES ($1, $2, $3, $4, $5, $6, $7) - ON CONFLICT (client_id) DO UPDATE + ON CONFLICT (client_id, source) DO UPDATE SET bytes_per_second = $2, - source = $3, sample_byte_count = $4, window_start = $5, window_end = $6, diff --git a/model/provider_bandwidth_model_test.go b/model/provider_bandwidth_model_test.go index f04b1acb..38b1cb16 100644 --- a/model/provider_bandwidth_model_test.go +++ b/model/provider_bandwidth_model_test.go @@ -207,10 +207,10 @@ func TestProviderBandwidthTableExists(t *testing.T) { }) } -// TestStoreProviderBandwidthUpsertsOneRowPerProvider covers the round trip and -// the primary-key overwrite: a second measurement for the same provider replaces +// TestStoreProviderBandwidthUpsertsOneRowPerSource covers the round trip and +// the primary-key overwrite: a second measurement from the SAME source replaces // the first rather than accumulating history. -func TestStoreProviderBandwidthUpsertsOneRowPerProvider(t *testing.T) { +func TestStoreProviderBandwidthUpsertsOneRowPerSource(t *testing.T) { server.DefaultTestEnv().Run(t, func(t testing.TB) { ctx := context.Background() @@ -227,42 +227,100 @@ func TestStoreProviderBandwidthUpsertsOneRowPerProvider(t *testing.T) { WindowEnd: windowEnd, }) - rowCount, stored := testingReadProviderBandwidth(ctx, clientId) - connect.AssertEqual(t, rowCount, 1) - connect.AssertEqual(t, stored.BytesPerSecond, float64(1024*1024)) - connect.AssertEqual(t, stored.Source, ProviderBandwidthSourcePassive) - connect.AssertEqual(t, stored.SampleByteCount, ByteCount(32*1024*1024)) - connect.AssertEqual(t, stored.WindowStart.UTC().Unix(), windowStart.Unix()) - connect.AssertEqual(t, stored.WindowEnd.UTC().Unix(), windowEnd.Unix()) - - // a later measurement from the other source replaces it in place - activeWindowStart := server.NowUtc().Add(-1 * time.Minute) - activeWindowEnd := server.NowUtc() + stored := testingReadProviderBandwidth(ctx, clientId) + connect.AssertEqual(t, len(stored), 1) + passive := stored[ProviderBandwidthSourcePassive] + connect.AssertEqual(t, passive.BytesPerSecond, float64(1024*1024)) + connect.AssertEqual(t, passive.SampleByteCount, ByteCount(32*1024*1024)) + connect.AssertEqual(t, passive.WindowStart.UTC().Unix(), windowStart.Unix()) + connect.AssertEqual(t, passive.WindowEnd.UTC().Unix(), windowEnd.Unix()) + + // a later measurement from the SAME source replaces it in place + laterWindowStart := server.NowUtc().Add(-1 * time.Minute) + laterWindowEnd := server.NowUtc() StoreProviderBandwidth(ctx, &ProviderBandwidth{ ClientId: clientId, BytesPerSecond: 4 * 1024 * 1024, - Source: ProviderBandwidthSourceActive, + Source: ProviderBandwidthSourcePassive, SampleByteCount: ByteCount(8 * 1024 * 1024), - WindowStart: activeWindowStart, - WindowEnd: activeWindowEnd, + WindowStart: laterWindowStart, + WindowEnd: laterWindowEnd, }) - rowCount, stored = testingReadProviderBandwidth(ctx, clientId) - connect.AssertEqual(t, rowCount, 1) - connect.AssertEqual(t, stored.BytesPerSecond, float64(4*1024*1024)) - connect.AssertEqual(t, stored.Source, ProviderBandwidthSourceActive) - connect.AssertEqual(t, stored.SampleByteCount, ByteCount(8*1024*1024)) - connect.AssertEqual(t, stored.WindowEnd.UTC().Unix(), activeWindowEnd.Unix()) + stored = testingReadProviderBandwidth(ctx, clientId) + connect.AssertEqual(t, len(stored), 1) + passive = stored[ProviderBandwidthSourcePassive] + connect.AssertEqual(t, passive.BytesPerSecond, float64(4*1024*1024)) + connect.AssertEqual(t, passive.SampleByteCount, ByteCount(8*1024*1024)) + connect.AssertEqual(t, passive.WindowEnd.UTC().Unix(), laterWindowEnd.Unix()) + }) +} + +// TestStoreProviderBandwidthKeepsEverySourceSeparate is the property the whole +// two-target design rests on: the operator and cdn measurements for ONE +// provider are stored as two rows carrying two figures, and neither overwrites +// the other or the passive figure. +// +// Keyed on client_id alone -- as this table was before the (client_id, source) +// migration -- the second measurement of every pass would silently replace the +// first, so only one target could ever be stored and the divergence between +// them, which is the only reason a second target exists, would be +// unobservable. +func TestStoreProviderBandwidthKeepsEverySourceSeparate(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + clientId := server.NewId() + now := server.NowUtc() + + figures := map[string]float64{ + ProviderBandwidthSourcePassive: 1 * 1024 * 1024, + ProviderBandwidthSourceActiveOperator: 12 * 1024 * 1024, + ProviderBandwidthSourceActiveCDN: 3 * 1024 * 1024, + } + for source, bytesPerSecond := range figures { + StoreProviderBandwidth(ctx, &ProviderBandwidth{ + ClientId: clientId, + BytesPerSecond: bytesPerSecond, + Source: source, + SampleByteCount: ByteCount(5 * 1024 * 1024), + WindowStart: now, + WindowEnd: now, + }) + } + + stored := testingReadProviderBandwidth(ctx, clientId) + if len(stored) != len(figures) { + t.Fatalf("%d rows stored for one provider, want %d (one per source) -- the sources are overwriting each other", len(stored), len(figures)) + } + for source, want := range figures { + row, ok := stored[source] + if !ok { + t.Fatalf("no row for source %q", source) + } + if row.BytesPerSecond != want { + t.Errorf("source %q stored %.0f B/s, want %.0f -- the figures are not being kept apart", + source, row.BytesPerSecond, want) + } + } + + // specifically: the two active targets diverge, and both divergent + // figures survive. An averaged pair would leave both at 7.5 MB/s. + operator := stored[ProviderBandwidthSourceActiveOperator].BytesPerSecond + cdn := stored[ProviderBandwidthSourceActiveCDN].BytesPerSecond + if operator == cdn { + t.Errorf("both active targets stored %.0f B/s; a provider prioritising one path is invisible once they collapse to one figure", operator) + } }) } -// testingReadProviderBandwidth reads the stored row back with sql, so the test -// asserts what is actually in the table rather than trusting a model reader that -// StoreProviderBandwidth's own writer would share. +// testingReadProviderBandwidth reads the stored rows back with sql, keyed by +// source, so the test asserts what is actually in the table rather than +// trusting a model reader that StoreProviderBandwidth's own writer would share. func testingReadProviderBandwidth( ctx context.Context, clientId server.Id, -) (rowCount int, bw *ProviderBandwidth) { +) map[string]*ProviderBandwidth { + bySource := map[string]*ProviderBandwidth{} server.Db(ctx, func(conn server.PgConn) { result, err := conn.Query( ctx, @@ -281,8 +339,7 @@ func testingReadProviderBandwidth( ) server.WithPgResult(result, err, func() { for result.Next() { - rowCount += 1 - bw = &ProviderBandwidth{} + bw := &ProviderBandwidth{} server.Raise(result.Scan( &bw.ClientId, &bw.BytesPerSecond, @@ -291,8 +348,9 @@ func testingReadProviderBandwidth( &bw.WindowStart, &bw.WindowEnd, )) + bySource[bw.Source] = bw } }) }) - return rowCount, bw + return bySource } From 60bfcf64329e1e475e1fde735f96bac00160f057 Mon Sep 17 00:00:00 2001 From: Ryan Mello Date: Fri, 31 Jul 2026 18:15:50 +0100 Subject: [PATCH 25/30] feat(model,api): reserve what a parallel bandwidth probe actually transfers The per-probe byte figure was 5 MiB because a probe was one download. It is now eight, and the reservation has to say so: a budget that under-counts by 3x is worse than no budget, because it reports a ceiling the deployment is quietly exceeding on every probe. The probe went parallel because a single TCP flow cannot exceed (send window / RTT), and connect's MaxWindowSize is scaledPow2WindowSize(mib(1), ...). The single-stream probe was therefore measuring 1 MiB / RTT rather than the provider: eleven of twelve beta providers came back with a bandwidth-delay product of ~1 MiB -- exactly one window -- and a provider independently measured at 79 MB/s on its own host reported 4.8 MB/s through the tunnel. Eight streams of 2 MiB lift the ceiling to ~94 MB/s at the fleet's 89 ms median, so the figure becomes the provider's. So MaxProviderBandwidthBytesPerProbe is 16 MiB, matching the prober's bandwidth.MaxSampleBytes. The two are halves of one number and must be changed together. maxProviderBandwidthTestBytes is left at 5 MiB but is no longer the same kind of quantity: it bounds ONE REQUEST (one stream, 2 MiB in practice) while the reservation bounds the whole probe. The new test pins the one direction they may not drift in -- a single request that could be served larger than a whole probe reserves would spend budget that was never admitted for it. --- api/handlers/provider_bandwidth_handlers.go | 18 +++++++++--- .../provider_bandwidth_handlers_test.go | 25 +++++++++++++++++ model/provider_bandwidth_rate_limit.go | 28 +++++++++++++++---- 3 files changed, 62 insertions(+), 9 deletions(-) diff --git a/api/handlers/provider_bandwidth_handlers.go b/api/handlers/provider_bandwidth_handlers.go index cb67e88d..71025571 100644 --- a/api/handlers/provider_bandwidth_handlers.go +++ b/api/handlers/provider_bandwidth_handlers.go @@ -15,12 +15,22 @@ import ( "github.com/urnetwork/server/model" ) -// maxProviderBandwidthTestBytes bounds a single download. Without a clamp the +// maxProviderBandwidthTestBytes bounds a single REQUEST. Without a clamp the // endpoint is an open-ended resource commitment per request: a caller could // ask for gigabytes and hold a connection (and the egress bytes that go with -// it) for as long as it liked. 5 MB matches the spec's per-probe sizing and -// the per-probe figure the byte budget reserves against -// (model.MaxProviderBandwidthBytesPerProbe). +// it) for as long as it liked. +// +// This is no longer the same thing as the per-probe figure. One probe is +// 8 parallel streams -- it has to be parallel, because a single TCP flow +// cannot exceed (connect's 1 MiB window / RTT) and the single-stream probe +// measured that ceiling rather than the provider -- so one probe is 8 requests +// of 2 MiB each, and the aggregate is bounded by the RESERVATION +// (model.MaxProviderBandwidthBytesPerProbe, 16 MiB), not by this clamp. +// +// The invariant to keep: the prober's per-stream byte count +// (bandwidth.StreamBytes, 2 MiB) must stay at or below this value. If it ever +// exceeds it, the endpoint silently truncates each stream and the probe +// transfers less than it reserved, understating every provider it measures. const maxProviderBandwidthTestBytes = 5 * 1024 * 1024 // defaultProviderBandwidthTestBytes is used when `bytes` is absent, malformed, diff --git a/api/handlers/provider_bandwidth_handlers_test.go b/api/handlers/provider_bandwidth_handlers_test.go index 9313daa9..54ef04f5 100644 --- a/api/handlers/provider_bandwidth_handlers_test.go +++ b/api/handlers/provider_bandwidth_handlers_test.go @@ -698,3 +698,28 @@ func providerBandwidthQuotaFor(ctx context.Context, clientId server.Id) (count i }) return count, byteCount } + +// TestProviderBandwidthTestClampFitsInsideOneProbesReservation guards the one +// invariant that ties the download endpoint to the byte budget now that they +// are no longer the same number. +// +// A probe is not one request any more: it is bandwidth.StreamCount parallel +// streams, because a single TCP flow cannot exceed (connect's 1 MiB window / +// RTT) and the single-stream probe was measuring that ceiling rather than the +// provider. So maxProviderBandwidthTestBytes bounds ONE STREAM and +// model.MaxProviderBandwidthBytesPerProbe bounds the whole probe. +// +// They may drift apart, but only in one direction. If a single request could +// be served larger than a whole probe reserves, one request could spend more +// budget than was ever admitted for it -- the reservation would be a number +// the deployment routinely exceeds, which is worse than having no budget at +// all. +func TestProviderBandwidthTestClampFitsInsideOneProbesReservation(t *testing.T) { + if model.MaxProviderBandwidthBytesPerProbe < int64(maxProviderBandwidthTestBytes) { + t.Fatalf( + "the download endpoint serves up to %d bytes in ONE request but a probe only reserves %d in total: "+ + "a single request could transfer more than the byte budget admitted for the whole probe", + maxProviderBandwidthTestBytes, model.MaxProviderBandwidthBytesPerProbe, + ) + } +} diff --git a/model/provider_bandwidth_rate_limit.go b/model/provider_bandwidth_rate_limit.go index 6dee56ce..81d4755e 100644 --- a/model/provider_bandwidth_rate_limit.go +++ b/model/provider_bandwidth_rate_limit.go @@ -50,14 +50,32 @@ const MaxProviderBandwidthLookaheadBuckets = 24 // production data once active probing has run for a week. const MaxActiveBandwidthProbesPerBucket = 40 -// One active probe downloads 5 MB (the spec's per-probe figure), so the -// hourly budget is 40 probes x 5 MB = 200 MB/hour, and the daily cap is -// 24 x 200 MB = 4.8 GB worst case -- the ceiling only reached if every bucket -// in the lookahead window fills. +// MaxProviderBandwidthBytesPerProbe is what ONE reservation admits, and it +// must equal what one measurement actually transfers -- a budget that +// under-counts is worse than no budget, because it reports a spend ceiling the +// deployment is quietly exceeding. +// +// A measurement is 8 parallel streams of 2 MiB = 16 MiB, per target. It is +// parallel because it has to be: a single TCP flow cannot exceed (send window +// / RTT), connect's MaxWindowSize is scaledPow2WindowSize(mib(1), ...), and +// the single-stream probe therefore measured 1 MiB / RTT for every provider on +// the fleet rather than the provider. Eleven of twelve beta providers came +// back with a bandwidth-delay product of ~1 MiB -- exactly one window -- and a +// provider independently measured at 79 MB/s on its own host reported +// 4.8 MB/s through the tunnel. N flows get N windows; the prober's +// bandwidth.MaxSampleBytes is the other half of this figure and the two must +// be changed together. +// +// So the hourly budget is 40 probes x 16 MiB = 640 MiB/hour, and the daily cap +// is 24 x 640 MiB = 15 GiB worst case -- the ceiling only reached if every +// bucket in the lookahead window fills. Note that a probe reserves per TARGET +// and there are two targets measured separately, so 40 reservations is 20 +// providers per hour, not 40. That is unchanged by this commit and is a +// property of MaxActiveBandwidthProbesPerBucket, which is left alone here. // // Explicitly int64: a byte budget is not a row count, and callers pass an // int64 byteCount. -const MaxProviderBandwidthBytesPerProbe int64 = 5 * 1024 * 1024 +const MaxProviderBandwidthBytesPerProbe int64 = 16 * 1024 * 1024 const MaxProviderBandwidthBytesPerBucket int64 = MaxActiveBandwidthProbesPerBucket * MaxProviderBandwidthBytesPerProbe const MaxProviderBandwidthBytesPerDay int64 = MaxProviderBandwidthLookaheadBuckets * MaxProviderBandwidthBytesPerBucket From 2ff068bffb33c78dac735411c3ece755191106df Mon Sep 17 00:00:00 2001 From: Ryan Mello Date: Sun, 2 Aug 2026 13:29:24 +0100 Subject: [PATCH 26/30] feat(model,api): persist the egress-health result the prober already computes The prober has run an egress-health check over every probed provider's tunnel since P2 and only ever logged the outcome, so `ok=131/131 dns=7/7 ... reputation=1/8` -- the one signal that says whether a provider carries traffic at all -- rolls off with the container logs. This stores it. provider_egress_health is keyed on client_id alone, so a run replaces the previous one: the current picture per provider, not a history, exactly as provider_egress_location behaves. class_results is jsonb rather than a column per class because the class set belongs to the prober, not the schema -- adding a destination class must not need a migration, and the per-class tally is read as a diagnostic ("dns=4/4 cdn=0/5 site=12/12" separates a datacenter refusal from a blackhole) rather than aggregated on in sql. Reputation is stored and never scored. reputation_ok/reputation_total sit beside ok_count/total_count and are never folded into them, the two failure name lists are kept apart, and a "reputation" key inside class_results is rejected outright rather than tolerated -- because if one ever arrived, the sum check would fail on every submission and the obvious "fix" is to relax the sum check, at which point reputation is silently inside the health score and nothing says so. The reputation class measures whether large vendors treat the exit ip as a datacenter address; nearly every honest hosted provider fails most of it because it IS hosted, so folding it in would score a provider that carried every byte it was asked for as partly broken. Everything is validated before anything is stored, because the row is an upsert: a bad submission does not sit beside the good one waiting to be noticed, it destroys the last good measurement for that provider. ok_count <= total_count, per-class ok <= total (checked separately -- {ok:5,total:2} and {ok:0,total:3} sum to a consistent 5/5 while describing an impossible class), exact sum equality against the class map, non-negative everything, and unknown fields rejected. The strict reader is a new one rather than a flag on readOperatorRequestBody: that is shared with the already-deployed bandwidth endpoints, and tightening a live parser as a side effect of adding a new endpoint is how a working fleet stops submitting overnight. measured_at is stamped on arrival, as the bandwidth result endpoint does, so a skewed prober clock cannot write a row that looks stale or future-dated. Auth is the same operator secret as the egress-location and bandwidth ingest routes, fail-closed on an unconfigured vault. Migration is a strict tail append: 542 -> 543, live index verified against migration_audit first. --- api/api.go | 5 + .../provider_egress_health_handlers.go | 218 +++++++++++++ .../provider_egress_health_handlers_test.go | 302 ++++++++++++++++++ db_migrations.go | 46 +++ model/provider_egress_health_model.go | 183 +++++++++++ model/provider_egress_health_model_test.go | 153 +++++++++ 6 files changed, 907 insertions(+) create mode 100644 api/handlers/provider_egress_health_handlers.go create mode 100644 api/handlers/provider_egress_health_handlers_test.go create mode 100644 model/provider_egress_health_model.go create mode 100644 model/provider_egress_health_model_test.go diff --git a/api/api.go b/api/api.go index 252e42d5..b235357b 100644 --- a/api/api.go +++ b/api/api.go @@ -61,6 +61,11 @@ func Routes() []*router.Route { router.NewRoute("GET", "/network/provider-bandwidth-test", handlers.ProviderBandwidthTest), router.NewRoute("POST", "/network/provider-bandwidth-result", handlers.ProviderBandwidthResult), router.NewRoute("POST", "/network/provider-bandwidth-reserve", handlers.ProviderBandwidthReserve), + // operator-to-server, same operator secret again: the egress-health + // run the prober takes over the tunnel the geolocation probe already + // opened. Until this existed the result was a log line and nothing + // else. + router.NewRoute("POST", "/network/provider-egress-health", handlers.ProviderEgressHealthResult), router.NewRoute("GET", "/network/clients", handlers.NetworkClients), router.NewRoute("GET", "/network/peers", handlers.NetworkPeers), router.NewRoute("GET", "/network/provider-locations", handlers.NetworkGetProviderLocations), diff --git a/api/handlers/provider_egress_health_handlers.go b/api/handlers/provider_egress_health_handlers.go new file mode 100644 index 00000000..b6bd773b --- /dev/null +++ b/api/handlers/provider_egress_health_handlers.go @@ -0,0 +1,218 @@ +package handlers + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/urnetwork/glog" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" +) + +// maxProviderEgressHealthBody bounds the request body. It is larger than the +// bandwidth endpoints' 4 KiB cap because this body carries a per-class map plus +// two joined destination-name lists, and a run against a wide table can name a +// dozen failed destinations. +const maxProviderEgressHealthBody = 16 * 1024 + +// providerEgressHealthReputationClass is the class name that must never appear +// inside class_results. See ProviderEgressHealthResult for why this is checked +// rather than tolerated. +const providerEgressHealthReputationClass = "reputation" + +// ProviderEgressHealthClassResult is one class's ok/total tally over the +// destinations the run SAMPLED. +type ProviderEgressHealthClassResult struct { + OK int `json:"ok"` + Total int `json:"total"` +} + +// SubmitProviderEgressHealthArgs is one egress-health run for one provider, as +// the prober measured it. +// +// There is deliberately no measured_at field: the server stamps arrival time, +// exactly as the bandwidth result endpoint does. A caller-supplied timestamp +// would be one more thing to validate and one more way for a skewed prober +// clock to write a row that looks stale or future-dated. +type SubmitProviderEgressHealthArgs struct { + ClientId server.Id `json:"client_id"` + // OKCount/TotalCount cover the SCORED classes only. Reputation is not part + // of them and must never be added to them. + OKCount int `json:"ok_count"` + TotalCount int `json:"total_count"` + // ClassResults is the per-class tally for the scored classes. Its ok and + // total must sum to exactly OKCount and TotalCount. + ClassResults map[string]ProviderEgressHealthClassResult `json:"class_results"` + // ReputationOK/ReputationTotal are stored beside the health figures and + // never inside them. + ReputationOK int `json:"reputation_ok"` + ReputationTotal int `json:"reputation_total"` + FailedNames string `json:"failed_names"` + ReputationFailedNames string `json:"reputation_failed_names"` +} + +// readStrictOperatorRequestBody reads a bounded operator request body and +// rejects any field the target struct does not declare. +// +// It is a separate reader from readOperatorRequestBody rather than a flag on +// it: that one is shared with the bandwidth endpoints, which are already +// deployed and already accept whatever their probers send, and tightening a +// live endpoint's parser as a side effect of adding a new one is how a working +// fleet stops submitting overnight. +// +// Rejecting unknown fields matters here specifically because this body is a +// set of counts that must agree with each other. A misspelled field silently +// decodes to zero, and a zero count is a perfectly valid, perfectly consistent +// payload -- so the failure would be a table full of plausible rows describing +// a measurement that never happened. +func readStrictOperatorRequestBody(w http.ResponseWriter, r *http.Request, out any) bool { + body, err := io.ReadAll(io.LimitReader(r.Body, maxProviderEgressHealthBody+1)) + if err != nil { + http.Error(w, "Bad request", http.StatusBadRequest) + return false + } + if maxProviderEgressHealthBody < len(body) { + http.Error(w, "Request too large", http.StatusRequestEntityTooLarge) + return false + } + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(out); err != nil { + http.Error(w, "Bad request", http.StatusBadRequest) + return false + } + return true +} + +// ProviderEgressHealthResult ingests one egress-health run from the operator's +// prober. The prober runs the check over the same tunnel the geolocation probe +// opened, and until now only logged the result -- so the one signal that says +// whether a provider carries traffic at all rolled off with the container +// logs. This stores it. +// +// The route is operator-to-server, gated by the same operator secret as the +// egress-location and bandwidth ingest endpoints. There is no network jwt. +// +// # Everything is validated before anything is stored +// +// The row is an upsert keyed on client_id, so a bad submission does not sit +// beside the good one waiting to be noticed -- it DESTROYS the last good +// measurement for that provider. That is why every rule below returns 400 +// before the store, and why none of them is a stored-then-flagged warning. +// +// # Reputation is not health +// +// reputation_ok/reputation_total are stored and never folded into +// ok_count/total_count, and a "reputation" key inside class_results is +// rejected outright. The reputation class measures whether large vendors treat +// the exit ip as a datacenter address; nearly every honest hosted provider +// fails most of it, because it IS hosted. Folding it in would score a provider +// that carried every byte it was asked for as partly broken, and would punish +// the well-run datacenter providers hardest. +// +// The explicit rejection exists because the alternative is worse than a +// rejection. If a caller ever put reputation inside class_results, the sum +// check would fail and every submission would 400 -- and the obvious "fix" is +// to relax the sum check, at which point reputation is silently inside the +// health score and nothing says so. The operator-proxy's egresshealth package +// calls this exact mistake "the one thing in this package most likely to be +// 'fixed' into a bug"; this is the server-side half of that guard. +func ProviderEgressHealthResult(w http.ResponseWriter, r *http.Request) { + if !authorizeOperator(r) { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + var args SubmitProviderEgressHealthArgs + if !readStrictOperatorRequestBody(w, r, &args) { + return + } + + if args.ClientId == (server.Id{}) { + http.Error(w, "Missing client id.", http.StatusBadRequest) + return + } + if args.OKCount < 0 || args.TotalCount < 0 { + http.Error(w, "ok_count and total_count must be non-negative.", http.StatusBadRequest) + return + } + if args.ReputationOK < 0 || args.ReputationTotal < 0 { + http.Error(w, "reputation_ok and reputation_total must be non-negative.", http.StatusBadRequest) + return + } + if args.TotalCount < args.OKCount { + // more destinations passed than were attempted: the submitter is not + // measuring what it thinks it is, and storing this would overwrite a + // real measurement with an impossible one + http.Error(w, "ok_count must not exceed total_count.", http.StatusBadRequest) + return + } + if args.ReputationTotal < args.ReputationOK { + http.Error(w, "reputation_ok must not exceed reputation_total.", http.StatusBadRequest) + return + } + + sumOK, sumTotal := 0, 0 + for class, tally := range args.ClassResults { + if class == providerEgressHealthReputationClass { + http.Error(w, fmt.Sprintf( + "%q is not a scored class: it is reported in reputation_ok/reputation_total and must never be part of ok_count/total_count.", + providerEgressHealthReputationClass, + ), http.StatusBadRequest) + return + } + if tally.OK < 0 || tally.Total < 0 { + http.Error(w, fmt.Sprintf("class %q: ok and total must be non-negative.", class), http.StatusBadRequest) + return + } + if tally.Total < tally.OK { + // checked per class as well as in aggregate: {ok:5,total:2} and + // {ok:0,total:3} sum to a consistent 5/5 while describing a class + // where more destinations passed than ran + http.Error(w, fmt.Sprintf("class %q: ok must not exceed total.", class), http.StatusBadRequest) + return + } + sumOK += tally.OK + sumTotal += tally.Total + } + // exact equality, not <=: the classes ARE the score. A total that does not + // decompose into its classes means the two halves of the payload were + // produced by different runs, or that something not in class_results was + // counted into the score -- which is precisely how reputation would get in. + if sumOK != args.OKCount || sumTotal != args.TotalCount { + http.Error(w, fmt.Sprintf( + "class_results sum to %d/%d but ok_count/total_count are %d/%d.", + sumOK, sumTotal, args.OKCount, args.TotalCount, + ), http.StatusBadRequest) + return + } + + classResults := map[string]model.ProviderEgressHealthClassResult{} + for class, tally := range args.ClassResults { + classResults[class] = model.ProviderEgressHealthClassResult{ + OK: tally.OK, + Total: tally.Total, + } + } + + model.SetProviderEgressHealth(r.Context(), &model.ProviderEgressHealth{ + ClientId: args.ClientId, + MeasuredAt: server.NowUtc(), + OKCount: args.OKCount, + Total: args.TotalCount, + ClassResults: classResults, + ReputationOK: args.ReputationOK, + ReputationTotal: args.ReputationTotal, + FailedNames: args.FailedNames, + ReputationFailedNames: args.ReputationFailedNames, + }) + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]any{}); err != nil { + glog.Infof("[pegh]could not write response. err = %s\n", err) + } +} diff --git a/api/handlers/provider_egress_health_handlers_test.go b/api/handlers/provider_egress_health_handlers_test.go new file mode 100644 index 00000000..945628b3 --- /dev/null +++ b/api/handlers/provider_egress_health_handlers_test.go @@ -0,0 +1,302 @@ +package handlers + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" +) + +// validEgressHealthBody is a well-formed submission: the four scored classes +// sum to exactly ok_count/total_count, and the reputation figures sit outside +// them. Tests mutate a copy of this to exercise one rule at a time. +func validEgressHealthBody(clientId server.Id) map[string]any { + return map[string]any{ + "client_id": clientId.String(), + "ok_count": 25, + "total_count": 26, + "class_results": map[string]any{ + "dns": map[string]any{"ok": 4, "total": 4}, + "connectivity": map[string]any{"ok": 5, "total": 5}, + "cdn": map[string]any{"ok": 4, "total": 5}, + "site": map[string]any{"ok": 12, "total": 12}, + }, + "reputation_ok": 1, + "reputation_total": 4, + "failed_names": "cachefly", + "reputation_failed_names": "akamai,etsy,canva", + } +} + +func postEgressHealth(t testing.TB, secret string, body map[string]any) *httptest.ResponseRecorder { + t.Helper() + buf, err := json.Marshal(body) + if err != nil { + t.Fatalf("marshal body: %s", err) + } + req := httptest.NewRequest(http.MethodPost, "/network/provider-egress-health", bytes.NewReader(buf)) + if secret != "" { + req.Header.Set(operatorSecretHeader, secret) + } + w := httptest.NewRecorder() + ProviderEgressHealthResult(w, req) + return w +} + +func TestProviderEgressHealthResultRejectsMissingSecret(t *testing.T) { + w := postEgressHealth(t, "", validEgressHealthBody(server.NewId())) + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret header is absent", w.Code) + } +} + +// TestProviderEgressHealthResultRejectsWrongSecret proves hmac.Equal is +// actually consulted once the vault IS configured. Without a configured +// secret the handler takes the secret=="" short-circuit, and the missing-header +// test above would pass even if the comparison were never made. +func TestProviderEgressHealthResultRejectsWrongSecret(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + const wrongSecret = "correct-operator-secret-0123456780" // last char changed + defer withStubOperatorIngestSecret(secret)() + + w := postEgressHealth(t, wrongSecret, validEgressHealthBody(server.NewId())) + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the configured secret and the request's secret differ", w.Code) + } +} + +// TestProviderEgressHealthResultValidationRejections walks every rule that +// must fire BEFORE anything is written. The row is an upsert keyed on +// client_id, so an accepted bad submission does not sit beside the good one -- +// it destroys the last good measurement for that provider. +// +// Each case asserts no row was stored, not merely that the status was 400: a +// store-then-flag implementation would pass a status-only assertion. +func TestProviderEgressHealthResultValidationRejections(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + ctx := context.Background() + + cases := []struct { + name string + mutate func(body map[string]any) + }{ + { + // the headline rule: more destinations passed than ran + name: "ok_count exceeds total_count", + mutate: func(body map[string]any) { + body["ok_count"] = 27 + body["total_count"] = 26 + body["class_results"] = map[string]any{ + "dns": map[string]any{"ok": 27, "total": 26}, + } + }, + }, + { + name: "class results do not sum to the totals", + mutate: func(body map[string]any) { + // drops the site class from the map but not from the total + body["class_results"] = map[string]any{ + "dns": map[string]any{"ok": 4, "total": 4}, + "connectivity": map[string]any{"ok": 5, "total": 5}, + "cdn": map[string]any{"ok": 4, "total": 5}, + } + }, + }, + { + // {ok:5,total:2} + {ok:0,total:3} sums to a consistent 5/5 + // while describing a class where more passed than ran, so the + // aggregate check alone does not catch this + name: "a single class has ok above total while the sums agree", + mutate: func(body map[string]any) { + body["ok_count"] = 5 + body["total_count"] = 5 + body["class_results"] = map[string]any{ + "dns": map[string]any{"ok": 5, "total": 2}, + "site": map[string]any{"ok": 0, "total": 3}, + } + }, + }, + { + // reputation is not health: it must never arrive as a scored + // class, because the only ways to accept it are to break the + // sum check or to fold it into the score + name: "reputation smuggled in as a scored class", + mutate: func(body map[string]any) { + body["ok_count"] = 26 + body["total_count"] = 30 + body["class_results"] = map[string]any{ + "dns": map[string]any{"ok": 4, "total": 4}, + "connectivity": map[string]any{"ok": 5, "total": 5}, + "cdn": map[string]any{"ok": 4, "total": 5}, + "site": map[string]any{"ok": 12, "total": 12}, + "reputation": map[string]any{"ok": 1, "total": 4}, + } + }, + }, + { + name: "unknown field", + mutate: func(body map[string]any) { + body["okay_count"] = 25 + }, + }, + { + name: "negative count", + mutate: func(body map[string]any) { + body["ok_count"] = -1 + body["total_count"] = -1 + body["class_results"] = map[string]any{} + }, + }, + { + name: "negative reputation count", + mutate: func(body map[string]any) { + body["reputation_ok"] = -1 + }, + }, + { + name: "reputation ok exceeds reputation total", + mutate: func(body map[string]any) { + body["reputation_ok"] = 5 + body["reputation_total"] = 4 + }, + }, + { + name: "negative class count", + mutate: func(body map[string]any) { + body["ok_count"] = 0 + body["total_count"] = -1 + body["class_results"] = map[string]any{ + "dns": map[string]any{"ok": 0, "total": -1}, + } + }, + }, + { + name: "missing client id", + mutate: func(body map[string]any) { + delete(body, "client_id") + }, + }, + } + + for _, c := range cases { + // a fresh client id per case, so "no row stored" is unambiguous + clientId := server.NewId() + body := validEgressHealthBody(clientId) + c.mutate(body) + + w := postEgressHealth(t, secret, body) + if w.Code != http.StatusBadRequest { + t.Errorf("%s: status = %d, want 400; body = %s", c.name, w.Code, w.Body.String()) + continue + } + if health := model.GetProviderEgressHealth(ctx, clientId); health != nil { + t.Errorf("%s: a rejected submission stored a row anyway: %+v", c.name, health) + } + } + }) +} + +// TestProviderEgressHealthResultRejectsOKAboveTotal pins the top-level +// ok_count <= total_count rule specifically. +// +// It asserts the REASON, not just the 400, and that is the whole point of the +// test. Under the full rule set the aggregate check is defence in depth: any +// payload with ok_count > total_count whose classes sum exactly to those two +// figures must contain a class with ok > total, so removing the aggregate +// check still gets a 400 -- from the per-class rule, describing a different +// invariant. A status-only assertion therefore cannot tell the two apart and +// would pass with the rule deleted. +// +// The rule earns its place by being checked BEFORE the payload is decomposed: +// it is a statement about the submission as a whole, it holds for any future +// class set, and it fires whether or not the class map agrees with itself. +func TestProviderEgressHealthResultRejectsOKAboveTotal(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + ctx := context.Background() + + clientId := server.NewId() + body := validEgressHealthBody(clientId) + body["ok_count"] = 27 + body["total_count"] = 26 + body["class_results"] = map[string]any{ + "dns": map[string]any{"ok": 27, "total": 26}, + } + + w := postEgressHealth(t, secret, body) + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 when ok_count exceeds total_count; body = %s", w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), "ok_count must not exceed total_count") { + t.Fatalf("body = %q, want the rejection to come from the ok_count <= total_count rule", strings.TrimSpace(w.Body.String())) + } + if health := model.GetProviderEgressHealth(ctx, clientId); health != nil { + t.Fatalf("a rejected submission stored a row anyway: %+v", health) + } + }) +} + +// TestProviderEgressHealthResultStoresAValidRun is the accept path: a correct +// secret clears auth, a consistent payload passes validation, and the row +// lands with the reputation figures stored beside the health figures rather +// than inside them. +func TestProviderEgressHealthResultStoresAValidRun(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + ctx := context.Background() + + clientId := server.NewId() + w := postEgressHealth(t, secret, validEgressHealthBody(clientId)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body = %s", w.Code, w.Body.String()) + } + + health := model.GetProviderEgressHealth(ctx, clientId) + if health == nil { + t.Fatal("a 200 stored no row") + } + if health.OKCount != 25 || health.Total != 26 { + t.Errorf("ok/total = %d/%d, want 25/26", health.OKCount, health.Total) + } + // reputation stored... + if health.ReputationOK != 1 || health.ReputationTotal != 4 { + t.Errorf("reputation = %d/%d, want 1/4", health.ReputationOK, health.ReputationTotal) + } + if health.ReputationFailedNames != "akamai,etsy,canva" { + t.Errorf("reputation_failed_names = %q", health.ReputationFailedNames) + } + // ...and excluded from the health figures. 26/30 would be the shape of + // a reputation-folded-in regression. + if health.OKCount == 26 || health.Total == 30 { + t.Errorf("reputation was folded into ok/total: %d/%d", health.OKCount, health.Total) + } + if _, present := health.ClassResults["reputation"]; present { + t.Error("reputation was stored as a scored class") + } + if health.FailedNames != "cachefly" { + t.Errorf("failed_names = %q, want the scored failures only", health.FailedNames) + } + if got := health.ClassResults["cdn"]; got.OK != 4 || got.Total != 5 { + t.Errorf("class_results[cdn] = %+v, want 4/5", got) + } + if len(health.ClassResults) != 4 { + t.Errorf("class_results has %d classes, want 4", len(health.ClassResults)) + } + if health.MeasuredAt.IsZero() { + t.Error("measured_at was not stamped on arrival") + } + }) +} diff --git a/db_migrations.go b/db_migrations.go index 88aba23c..b2647c67 100644 --- a/db_migrations.go +++ b/db_migrations.go @@ -4594,4 +4594,50 @@ var migrations = []any{ DROP CONSTRAINT IF EXISTS provider_bandwidth_pkey, ADD CONSTRAINT provider_bandwidth_pkey PRIMARY KEY (client_id, source) `), + + // The latest egress-health run per provider: does this provider actually + // carry traffic to the real internet, across several independent classes + // of destination. The prober has computed this every pass since P2 and + // only ever logged it, so the signal rolls off with the container logs. + // + // Keyed on client_id alone, so a run replaces the previous one -- the + // current picture per provider, not a history, exactly as + // provider_egress_location behaves. Trending, if it is ever wanted, + // belongs in a separate partitioned append table rather than a second key + // column here. + // + // class_results is jsonb rather than a column per class because the class + // set is the prober's, not the schema's: adding a destination class must + // not need a migration, and the per-class tally is read as a diagnostic + // document ("dns=4/4 cdn=0/5 site=12/12" separates a datacenter-refusal + // from a blackhole) rather than filtered or aggregated on in sql. + // + // reputation_ok/reputation_total and reputation_failed_names are stored + // SEPARATELY from ok_count/total_count and must never be folded into them. + // The reputation class measures whether big vendors treat the exit ip as a + // datacenter address; nearly every honest hosted provider fails most of it + // because it IS hosted. Summing it into the health figure would score a + // provider that carried every byte it was asked for as partly broken. The + // ingest endpoint rejects a 'reputation' key inside class_results for the + // same reason. + // + // Appended, never inserted: migrations here apply by slice index + // (`for i := DbVersion(ctx); i < upTo; i++`), so editing or reordering an + // already-applied entry corrupts live databases. IF NOT EXISTS makes a + // re-run -- or a duplicated merge resolution -- a no-op. + newSqlMigration(` + CREATE TABLE IF NOT EXISTS provider_egress_health ( + client_id uuid NOT NULL, + measured_at timestamp NOT NULL, + ok_count int NOT NULL, + total_count int NOT NULL, + class_results jsonb NOT NULL, + reputation_ok int NOT NULL, + reputation_total int NOT NULL, + failed_names text NOT NULL DEFAULT '', + reputation_failed_names text NOT NULL DEFAULT '', + + PRIMARY KEY (client_id) + ) + `), } diff --git a/model/provider_egress_health_model.go b/model/provider_egress_health_model.go new file mode 100644 index 00000000..3a69ccf8 --- /dev/null +++ b/model/provider_egress_health_model.go @@ -0,0 +1,183 @@ +package model + +import ( + "context" + "encoding/json" + "time" + + "github.com/urnetwork/server" +) + +// ProviderEgressHealthClassResult is one class's ok/total tally over the +// destinations a single run SAMPLED, not over the whole destination table. The +// prober draws a bounded random subset of each class per run (see the +// operator-proxy's egresshealth package), so `{"cdn":{"ok":4,"total":5}}` means +// four of the five drawn this pass, out of a much larger table. +type ProviderEgressHealthClassResult struct { + OK int `json:"ok"` + Total int `json:"total"` +} + +// ProviderEgressHealth is one egress-health run for one provider: does this +// provider actually carry traffic to the real internet, across several +// independent classes of destination. +// +// # Reputation is not health +// +// ReputationOK/ReputationTotal are stored because they are measured in the +// same pass, and they are deliberately NOT part of OKCount/Total. This mirrors +// the operator-proxy's egresshealth package comment, which calls its own +// version of this "the one thing in this package most likely to be 'fixed' +// into a bug", and the reasoning holds identically server-side. +// +// The reputation class measures whether large vendors treat the exit ip as a +// datacenter address. Nearly every honest hosted provider fails most of it, +// because it IS hosted -- that is a fact about the vendor's ip intelligence +// feed, not about whether the provider carries traffic. Folding it into +// OKCount/Total would take a provider that carried every byte it was asked for +// and score it as partly broken, and the providers it would punish hardest are +// the well-run datacenter ones. Nothing downstream may add these figures into +// OKCount, Total, or any health score derived from them. +// +// The two failure name lists are kept apart for the same reason: FailedNames +// is destinations that mean the provider is not carrying traffic, while +// ReputationFailedNames is vendors that refused a datacenter ip. Merged, they +// would read as one longer failure list. +type ProviderEgressHealth struct { + ClientId server.Id + MeasuredAt time.Time + // OKCount and Total cover the SCORED classes only (dns, connectivity, cdn, + // site), over this run's sample. + OKCount int + Total int + // ClassResults is the per-class tally for the scored classes only. A + // partial failure is only diagnosable per class: "ok=14/26" alone says + // nothing, while "dns=4/4 cdn=0/5 site=12/12" says the tunnel carries + // bytes and resolves names but is being refused by content providers -- + // a completely different fault from a total blackhole. + ClassResults map[string]ProviderEgressHealthClassResult + // ReputationOK/ReputationTotal: stored, never scored. See the type comment. + ReputationOK int + ReputationTotal int + // FailedNames is the comma-joined names of the scored destinations that + // failed. It is the only record of WHICH destinations a given provider was + // asked for on a given pass, since the sample is drawn fresh each run. + FailedNames string + // ReputationFailedNames is the comma-joined names of the reputation + // destinations that refused. Separate from FailedNames, deliberately. + ReputationFailedNames string +} + +// SetProviderEgressHealth records a provider's latest egress-health run. +// +// The row is keyed on client_id alone, so a new run REPLACES the previous one: +// this is the current picture per provider that a consumer reads, not a +// history. That mirrors provider_egress_location's lifecycle exactly. If +// trending is wanted later it belongs in a separate partitioned append table, +// not in a second key column here -- the read path for "is this provider +// carrying traffic right now" wants one row per provider and nothing else. +// +// Nothing here folds reputation into the health figures; see the +// ProviderEgressHealth comment for why that must stay true. +func SetProviderEgressHealth(ctx context.Context, health *ProviderEgressHealth) { + classResults := health.ClassResults + if classResults == nil { + classResults = map[string]ProviderEgressHealthClassResult{} + } + // marshalled here rather than handed to pgx as a map, so the column always + // receives a jsonb document of a known shape (an absent map becomes `{}`, + // not sql NULL, and the column is NOT NULL) + classResultsJson, err := json.Marshal(classResults) + server.Raise(err) + + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO provider_egress_health ( + client_id, + measured_at, + ok_count, + total_count, + class_results, + reputation_ok, + reputation_total, + failed_names, + reputation_failed_names + ) + VALUES ($1, $2, $3, $4, $5::jsonb, $6, $7, $8, $9) + ON CONFLICT (client_id) DO UPDATE + SET + measured_at = $2, + ok_count = $3, + total_count = $4, + class_results = $5::jsonb, + reputation_ok = $6, + reputation_total = $7, + failed_names = $8, + reputation_failed_names = $9 + `, + health.ClientId, + // measured_at is a naive timestamp column holding utc, as + // everywhere else in this schema + health.MeasuredAt.UTC(), + health.OKCount, + health.Total, + string(classResultsJson), + health.ReputationOK, + health.ReputationTotal, + health.FailedNames, + health.ReputationFailedNames, + )) + }) +} + +// GetProviderEgressHealth reads a provider's latest egress-health run, or nil +// when the provider has never been measured. Never measured is not the same as +// measured-unhealthy, so it is a nil result rather than a zero-valued one: +// a caller that cannot tell those apart would read every unprobed provider as +// a total blackhole. +func GetProviderEgressHealth(ctx context.Context, clientId server.Id) *ProviderEgressHealth { + var health *ProviderEgressHealth + + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT + measured_at, + ok_count, + total_count, + class_results, + reputation_ok, + reputation_total, + failed_names, + reputation_failed_names + FROM provider_egress_health + WHERE client_id = $1 + `, + clientId, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + h := &ProviderEgressHealth{ClientId: clientId} + var classResultsJson []byte + server.Raise(result.Scan( + &h.MeasuredAt, + &h.OKCount, + &h.Total, + &classResultsJson, + &h.ReputationOK, + &h.ReputationTotal, + &h.FailedNames, + &h.ReputationFailedNames, + )) + h.ClassResults = map[string]ProviderEgressHealthClassResult{} + server.Raise(json.Unmarshal(classResultsJson, &h.ClassResults)) + health = h + } + }) + }) + + return health +} diff --git a/model/provider_egress_health_model_test.go b/model/provider_egress_health_model_test.go new file mode 100644 index 00000000..94147f63 --- /dev/null +++ b/model/provider_egress_health_model_test.go @@ -0,0 +1,153 @@ +package model + +import ( + "context" + "testing" + "time" + + "github.com/urnetwork/connect" + + "github.com/urnetwork/server" +) + +func TestSetProviderEgressHealthStoresAndReadsBack(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + clientId := server.NewId() + measuredAt := server.NowUtc().Truncate(time.Millisecond) + + SetProviderEgressHealth(ctx, &ProviderEgressHealth{ + ClientId: clientId, + MeasuredAt: measuredAt, + OKCount: 25, + Total: 26, + ClassResults: map[string]ProviderEgressHealthClassResult{ + "dns": {OK: 4, Total: 4}, + "connectivity": {OK: 5, Total: 5}, + "cdn": {OK: 4, Total: 5}, + "site": {OK: 12, Total: 12}, + }, + ReputationOK: 1, + ReputationTotal: 4, + FailedNames: "cachefly", + ReputationFailedNames: "akamai,etsy,canva", + }) + + health := GetProviderEgressHealth(ctx, clientId) + if health == nil { + t.Fatal("expected a stored egress health row, got nil") + } + connect.AssertEqual(t, health.ClientId, clientId) + connect.AssertEqual(t, health.OKCount, 25) + connect.AssertEqual(t, health.Total, 26) + connect.AssertEqual(t, health.ReputationOK, 1) + connect.AssertEqual(t, health.ReputationTotal, 4) + connect.AssertEqual(t, health.FailedNames, "cachefly") + connect.AssertEqual(t, health.ReputationFailedNames, "akamai,etsy,canva") + if !health.MeasuredAt.UTC().Equal(measuredAt) { + t.Errorf("MeasuredAt = %s, want %s", health.MeasuredAt.UTC(), measuredAt) + } + + // asserted as a parsed map, never as json text: key order is not stable + connect.AssertEqual(t, len(health.ClassResults), 4) + connect.AssertEqual(t, health.ClassResults["dns"], ProviderEgressHealthClassResult{OK: 4, Total: 4}) + connect.AssertEqual(t, health.ClassResults["cdn"], ProviderEgressHealthClassResult{OK: 4, Total: 5}) + connect.AssertEqual(t, health.ClassResults["site"], ProviderEgressHealthClassResult{OK: 12, Total: 12}) + + // the reputation figures are stored beside the health figures and + // never inside them: 25/26 is the scored classes only, and the + // per-class tallies sum to exactly that. If reputation were ever + // folded in, this would read 26/30. + sumOK, sumTotal := 0, 0 + for _, c := range health.ClassResults { + sumOK += c.OK + sumTotal += c.Total + } + connect.AssertEqual(t, sumOK, health.OKCount) + connect.AssertEqual(t, sumTotal, health.Total) + if _, present := health.ClassResults["reputation"]; present { + t.Error("reputation must never appear as a scored class") + } + }) +} + +func TestGetProviderEgressHealthNilWhenNeverMeasured(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + // never measured is not the same as measured-unhealthy; a zero-valued + // row here would read as a total blackhole for every unprobed provider + if health := GetProviderEgressHealth(ctx, server.NewId()); health != nil { + t.Errorf("expected nil for a never-probed provider, got %+v", health) + } + }) +} + +// TestSetProviderEgressHealthUpsertReplaces is the lifecycle the table exists +// for: one row per provider carrying the LATEST run. A second run for the same +// client_id must replace the first, not accumulate beside it -- otherwise a +// consumer reading "the provider's health" gets an arbitrary one of N rows. +func TestSetProviderEgressHealthUpsertReplaces(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + clientId := server.NewId() + + SetProviderEgressHealth(ctx, &ProviderEgressHealth{ + ClientId: clientId, + MeasuredAt: server.NowUtc().Add(-1 * time.Hour), + OKCount: 0, + Total: 26, + ClassResults: map[string]ProviderEgressHealthClassResult{ + "dns": {OK: 0, Total: 26}, + }, + ReputationOK: 0, + ReputationTotal: 4, + FailedNames: "everything", + ReputationFailedNames: "akamai", + }) + + later := server.NowUtc() + SetProviderEgressHealth(ctx, &ProviderEgressHealth{ + ClientId: clientId, + MeasuredAt: later, + OKCount: 26, + Total: 26, + ClassResults: map[string]ProviderEgressHealthClassResult{ + "dns": {OK: 26, Total: 26}, + }, + ReputationOK: 2, + ReputationTotal: 4, + // the recovered run has no failures at all: an upsert that only + // wrote the non-empty columns would leave "everything" behind + FailedNames: "", + ReputationFailedNames: "", + }) + + var rowCount int + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + `SELECT COUNT(*) FROM provider_egress_health WHERE client_id = $1`, + clientId, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + server.Raise(result.Scan(&rowCount)) + } + }) + }) + connect.AssertEqual(t, rowCount, 1) + + health := GetProviderEgressHealth(ctx, clientId) + if health == nil { + t.Fatal("expected a stored egress health row, got nil") + } + connect.AssertEqual(t, health.OKCount, 26) + connect.AssertEqual(t, health.ReputationOK, 2) + connect.AssertEqual(t, health.FailedNames, "") + connect.AssertEqual(t, health.ReputationFailedNames, "") + connect.AssertEqual(t, health.ClassResults["dns"], ProviderEgressHealthClassResult{OK: 26, Total: 26}) + if health.MeasuredAt.UTC().Before(later.Add(-time.Minute)) { + t.Errorf("MeasuredAt = %s, want the later run's %s", health.MeasuredAt.UTC(), later) + } + }) +} From b47bf4596ee0a3a1ff8f970d9cfa0fb85f09a770 Mon Sep 17 00:00:00 2001 From: Ryan Mello Date: Thu, 30 Jul 2026 16:09:14 +0100 Subject: [PATCH 27/30] feat(probeverdict): pure verdict logic for geolocation probe submissions Turns a probe submission into verified / unverified / suspect from the only two corroboration rules that need no external reference point: absence of country consensus, and country instability inside a 24h window. Two omissions from Input are load-bearing and structural, not oversights: - There is no mmdb-derived country field. A probed country differing from what the free mmdb would have said is the entire point of this project, so with no field for it a caller cannot get the rule wrong. - There are no RTT or coordinate fields. An RTT-distance floor was designed and dropped before implementation: it needs a fixed reference point for where the RTT was measured from, and network_client_connection.connection_block exists precisely because more than one such point can exist. A wrong reference point does not fail safe -- it can flag an honest, correctly-placed provider as suspect. suspect("rtt_impossible") therefore does not exist. Both omissions are pinned by a reflection test over Input's field set, since neither can be asserted behaviourally. No I/O, no dependencies beyond time, so the package is testable independent of the transport that produced the submission. --- probeverdict/probeverdict.go | 55 ++++++++++++++++++ probeverdict/probeverdict_test.go | 97 +++++++++++++++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 probeverdict/probeverdict.go create mode 100644 probeverdict/probeverdict_test.go diff --git a/probeverdict/probeverdict.go b/probeverdict/probeverdict.go new file mode 100644 index 00000000..736ed5a7 --- /dev/null +++ b/probeverdict/probeverdict.go @@ -0,0 +1,55 @@ +// Package probeverdict turns a geolocation probe submission into a verdict: +// verified, unverified, or suspect. It is pure decision logic with no I/O, so +// it is fully table-testable independent of how a submission arrived. +// +// Deliberately absent from Input: the mmdb-derived country for the same +// connection, and any RTT or coordinate fields. A probed country differing +// from what the free mmdb would have said is the entire point of this +// project and must never be treated as suspicious -- keeping that field off +// Input makes the omission structural. An RTT-distance corroboration was +// designed and dropped before implementation (see the spec's "The RTT floor +// was designed, then dropped"): it needs a fixed reference point that this +// system does not have a single answer for once more than one deployment +// instance exists, and a wrong reference point does not fail safe -- it can +// flag an honest provider as suspect. Do not reintroduce ObservedRTT or +// coordinate fields here without first resolving that reference-point +// problem in the spec. +package probeverdict + +import "time" + +type Input struct { + CountryConfident bool + CountryCode string + + PreviousCountryCode string + PreviousObservedAt time.Time + Now time.Time +} + +type Verdict struct { + State string + Reason string +} + +// unstableWindow is how recently a prior, different country counts as a +// flip-flop rather than a legitimate correction. +const unstableWindow = 24 * time.Hour + +func Evaluate(in Input) Verdict { + if !in.CountryConfident { + return Verdict{State: "unverified", Reason: "no_consensus"} + } + + if in.PreviousCountryCode != "" && in.PreviousCountryCode != in.CountryCode { + now := in.Now + if now.IsZero() { + now = time.Now() + } + if now.Sub(in.PreviousObservedAt) < unstableWindow { + return Verdict{State: "suspect", Reason: "unstable"} + } + } + + return Verdict{State: "verified"} +} diff --git a/probeverdict/probeverdict_test.go b/probeverdict/probeverdict_test.go new file mode 100644 index 00000000..730967bd --- /dev/null +++ b/probeverdict/probeverdict_test.go @@ -0,0 +1,97 @@ +package probeverdict + +import ( + "reflect" + "testing" + "time" +) + +func TestEvaluateNoConsensusIsUnverified(t *testing.T) { + v := Evaluate(Input{CountryConfident: false}) + if v.State != "unverified" || v.Reason != "no_consensus" { + t.Errorf("got %+v, want unverified/no_consensus", v) + } +} + +func TestEvaluateCountryFlipFlopIsSuspect(t *testing.T) { + now := time.Now() + v := Evaluate(Input{ + CountryConfident: true, + CountryCode: "de", + PreviousCountryCode: "es", + PreviousObservedAt: now.Add(-2 * time.Hour), + Now: now, + }) + if v.State != "suspect" || v.Reason != "unstable" { + t.Errorf("got %+v, want suspect/unstable", v) + } +} + +func TestEvaluateCountryChangeOutsideWindowIsVerified(t *testing.T) { + // a country change is only "unstable" within the 24h window -- after it, + // a changed country is a legitimate correction, not a flip-flop + now := time.Now() + v := Evaluate(Input{ + CountryConfident: true, + CountryCode: "de", + PreviousCountryCode: "es", + PreviousObservedAt: now.Add(-25 * time.Hour), + Now: now, + }) + if v.State != "verified" { + t.Errorf("got %+v, want verified (change is outside the 24h window)", v) + } +} + +func TestEvaluateMmdbDivergenceAloneIsNotSuspect(t *testing.T) { + // this test asserts the single most important safety property in the + // package: a country that differs from what mmdb would have said is NOT + // an input to Evaluate at all -- there is no field for it on Input, so a + // clean first-time probe always verifies regardless of what mmdb would + // have said about the same connection. + v := Evaluate(Input{ + CountryConfident: true, + CountryCode: "es", + }) + if v.State != "verified" { + t.Errorf("a clean probe with no prior history must verify regardless of what mmdb would have said, got %+v", v) + } +} + +// TestInputHasNoMmdbRttOrCoordinateFields locks the two structural omissions +// this package depends on for correctness. Neither can be asserted +// behaviourally -- they are the absence of inputs, so the only way to test +// them is to pin the field set itself. +// +// 1. No mmdb-derived country. A probed country diverging from what the free +// mmdb would have said is the entire point of this project, and with no +// field for it a caller cannot get the rule wrong. +// 2. No RTT or coordinate fields. An RTT-distance floor was designed and +// deliberately dropped (see the package doc comment and the spec's "The +// RTT floor was designed, then dropped"): it needs a fixed reference +// point this system has no single answer for, and a wrong reference point +// does not fail safe -- it can flag an honest provider as suspect. +// +// If this test fails because a field was added, that is the point: resolve +// the reference-point problem in the spec first. +func TestInputHasNoMmdbRttOrCoordinateFields(t *testing.T) { + allowed := map[string]bool{ + "CountryConfident": true, + "CountryCode": true, + "PreviousCountryCode": true, + "PreviousObservedAt": true, + "Now": true, + } + + inputType := reflect.TypeOf(Input{}) + for i := range inputType.NumField() { + name := inputType.Field(i).Name + if !allowed[name] { + t.Errorf("Input has an unexpected field %q: mmdb-country, RTT and coordinate "+ + "fields are deliberately absent from Input; see the package doc comment", name) + } + } + if inputType.NumField() != len(allowed) { + t.Errorf("Input has %d fields, want exactly %d", inputType.NumField(), len(allowed)) + } +} From 4a79023dda7795a666e795156fdbadc7d4aead33 Mon Sep 17 00:00:00 2001 From: Ryan Mello Date: Sun, 2 Aug 2026 14:23:37 +0100 Subject: [PATCH 28/30] feat(controller): compute a real verdict on every egress-location submission probeverdict shipped complete and table-tested with zero call sites, so every provider_egress_location row reads verdict='unverified' -- the column default, which records the absence of a judgement and is indistinguishable from a judgement of "could not verify". Wire it into SubmitProviderEgressLocation, the one function every geolocation submission already funnels through, so verdicts fall out of the existing probe cadence with no new scheduler and no new endpoint. The upsert is about to overwrite the previous row, so the previous country/observed_at are read once, immediately before the write, and passed straight through to probeverdict. The marshalling lives in a small helper rather than inline: it is the seam a regression would land on (hardcoding CountryConfident, dropping the history, comparing an uppercased submitted code against a lowercased stored one), and a helper is directly unit-testable for the no-consensus case that SubmitProviderEgressLocation rejects before it can ever be stored. Deliberately not passed: the mmdb country for the provider's control ip. A probed country that disagrees with mmdb is the finding this project exists to produce, never a fault. probeverdict.Input has no field for it, and the new end-to-end test pins the property against a client whose control ip the mmdb places in Canada and whose probe says Japan. Assurance stays at the model default 'direct'; multi-hop is P3. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01QtgqtCmKJRXdsQ5ktiqwkg --- .../provider_egress_location_controller.go | 58 ++++++ ...rovider_egress_location_controller_test.go | 182 ++++++++++++++++++ 2 files changed, 240 insertions(+) diff --git a/controller/provider_egress_location_controller.go b/controller/provider_egress_location_controller.go index 8cac0dcc..b29feedc 100644 --- a/controller/provider_egress_location_controller.go +++ b/controller/provider_egress_location_controller.go @@ -8,6 +8,7 @@ import ( "github.com/urnetwork/server" "github.com/urnetwork/server/model" + "github.com/urnetwork/server/probeverdict" ) // MaxProviderEgressLocationSubmissionAge rejects a submission whose probe is @@ -58,6 +59,45 @@ type SubmitProviderEgressLocationResult struct { LocationId server.Id `json:"location_id"` } +// providerEgressVerdict marshals a submission, and the row it is about to +// replace, into probeverdict's Input. It is the only place the two are joined; +// the rules themselves live in the probeverdict package and are not restated +// here. +// +// previous is the row currently stored for this provider, or nil when the +// provider has never been probed successfully. A nil previous leaves +// PreviousCountryCode empty, which probeverdict reads as "no history to +// contradict" -- a first probe is judged on its consensus alone. +// +// Note what is NOT passed: the mmdb country for the provider's control ip. A +// probed country that disagrees with mmdb is the finding this project exists to +// produce, never a fault, and probeverdict.Input structurally has no field for +// it. Do not add one. +func providerEgressVerdict( + args *SubmitProviderEgressLocationArgs, + previous *model.ProviderEgressLocation, +) probeverdict.Verdict { + in := probeverdict.Input{ + CountryConfident: args.CountryConfident, + // stored country codes are lowercased (see + // model.SetProviderEgressLocation), so normalize the submitted one the + // same way before comparing it against the stored history -- otherwise + // a prober sending "US" against a stored "us" reads as a country change + // and every second probe would be suspect. + CountryCode: strings.ToLower(strings.TrimSpace(args.CountryCode)), + // the server's clock, not the prober's: the age of the stored history + // is a server-side judgement, and args.ObservedAt is attacker-adjacent + // input. Its skew is already bounded, but the bound is a rejection + // rule, not a licence to measure history against it. + Now: server.NowUtc(), + } + if previous != nil { + in.PreviousCountryCode = previous.CountryCode + in.PreviousObservedAt = previous.ObservedAt + } + return probeverdict.Evaluate(in) +} + // SubmitProviderEgressLocation records a probed egress location for a provider. // Only country-confident submissions are accepted; city/region are stored only // when the probe was also city-confident (free geolocation sources disagree on @@ -176,6 +216,20 @@ func SubmitProviderEgressLocation( model.CreateLocation(ctx, location) } + // judge the submission against the history it is about to replace. This is + // the only call site probeverdict has and the only place a verdict is + // computed: every geolocation submission already funnels through here, so + // verdicts fall out of the existing probe cadence with no separate + // scheduler and no separate endpoint. Before this, every row in the table + // read the column default `unverified` -- the absence of a judgement, which + // is indistinguishable from a judgement of "could not verify". + // + // The read is the only one: SetProviderEgressLocation below is an upsert, + // so the previous row has to be fetched before it is overwritten, and the + // verdict is the only thing that needs it. + previous := model.GetProviderEgressLocation(ctx, args.ClientId) + verdict := providerEgressVerdict(args, previous) + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ ClientId: args.ClientId, LocationId: location.LocationId, @@ -187,6 +241,10 @@ func SubmitProviderEgressLocation( Mobile: args.Mobile, CityConfident: cityConfident, ObservedAt: args.ObservedAt, + Verdict: verdict.State, + VerdictReason: verdict.Reason, + // assurance stays at the model default (`direct`): this probe reached + // the provider over a single tunnel from the prober. Multi-hop is P3. }) return &SubmitProviderEgressLocationResult{LocationId: location.LocationId}, nil diff --git a/controller/provider_egress_location_controller_test.go b/controller/provider_egress_location_controller_test.go index 45b28d22..efd591ec 100644 --- a/controller/provider_egress_location_controller_test.go +++ b/controller/provider_egress_location_controller_test.go @@ -742,3 +742,185 @@ func TestSubmitProviderEgressLocationAmbiguousQualifierFallsBackToCountry(t *tes } }) } + +// The verdict tests below cover the wiring of the probeverdict package into +// this ingest path. probeverdict itself is table-tested in its own package; +// what is asserted here is that a submission actually reaches it and that its +// answer reaches the stored row -- before this wiring every row in production +// read the column default, `unverified`, which is the absence of a judgement +// rather than a judgement. + +// A clean, consensus-backed submission with no conflicting history must store +// `verified`, not the `unverified` column default. +func TestSubmitProviderEgressLocationVerdictVerified(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "US", + Country: "United States", + CountryConfident: true, + ObservedAt: server.NowUtc(), + }) + connect.AssertEqual(t, err, nil) + + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected the submission to be stored") + } + if stored.Verdict != "verified" { + t.Errorf("verdict = %q, want %q: a consensus-backed submission with no conflicting history is verified", stored.Verdict, "verified") + } + connect.AssertEqual(t, stored.VerdictReason, "") + // assurance is unrelated to the verdict and stays at its default + connect.AssertEqual(t, stored.Assurance, model.ProviderEgressAssuranceDirect) + }) +} + +// A country flip-flop inside probeverdict's instability window must store +// `suspect` with the `unstable` reason. +// +// The two submissions go backwards in time rather than forwards: +// MaxProviderEgressLocationSubmissionSkew rejects an observed_at more than five +// minutes in the future, so the earlier probe is placed two hours ago and the +// later one now. That keeps both inside the 24h submission-age bound, satisfies +// the monotonic observed_at upsert, and puts the gap well inside the +// instability window. +func TestSubmitProviderEgressLocationVerdictSuspectOnCountryFlipFlop(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "US", + Country: "United States", + CountryConfident: true, + ObservedAt: server.NowUtc().Add(-2 * time.Hour), + }) + connect.AssertEqual(t, err, nil) + + first := model.GetProviderEgressLocation(ctx, clientId) + if first == nil { + t.Fatal("expected the first submission to be stored") + } + if first.Verdict != "verified" { + t.Fatalf("first verdict = %q, want %q; the fixture is only meaningful if the flip is a change", first.Verdict, "verified") + } + + // the same provider now probes from a different country + _, err = SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "JP", + Country: "Japan", + CountryConfident: true, + ObservedAt: server.NowUtc(), + }) + connect.AssertEqual(t, err, nil) + + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected the second submission to be stored") + } + connect.AssertEqual(t, stored.CountryCode, "jp") + if stored.Verdict != "suspect" { + t.Errorf("verdict = %q, want %q: a country change inside the instability window is a flip-flop", stored.Verdict, "suspect") + } + if stored.VerdictReason != "unstable" { + t.Errorf("verdict_reason = %q, want %q", stored.VerdictReason, "unstable") + } + }) +} + +// Absence of consensus is `unverified`/`no_consensus`. This is asserted against +// the marshalling helper rather than end-to-end because +// SubmitProviderEgressLocation rejects a submission that is not +// country-confident before anything is written (see +// TestSubmitProviderEgressLocationRejectsNotCountryConfident), so no row can +// ever be stored for one. The regression this guards against is the wiring +// hardcoding CountryConfident instead of passing the submission's own value +// through. +func TestSubmitProviderEgressLocationVerdictNoConsensus(t *testing.T) { + verdict := providerEgressVerdict(&SubmitProviderEgressLocationArgs{ + ClientId: server.NewId(), + CountryCode: "us", + CountryConfident: false, + ObservedAt: server.NowUtc(), + }, nil) + if verdict.State != "unverified" { + t.Errorf("state = %q, want %q", verdict.State, "unverified") + } + if verdict.Reason != "no_consensus" { + t.Errorf("reason = %q, want %q", verdict.Reason, "no_consensus") + } +} + +// The property this whole project exists for: a probed country that disagrees +// with what the free mmdb says about the provider's control ip is NOT +// suspicious. That divergence is the finding, not the fault -- a provider whose +// control connection looks Canadian while its egress is demonstrably Japanese +// is exactly the case probing was built to surface, and flagging it would +// invert the feature. +// +// probeverdict makes the omission structural: Input has no mmdb field at all, +// so there is no argument to toggle. The property is therefore asserted +// end-to-end -- give the client a connection from an ip the mmdb places in +// Canada, submit a stable, consensus-backed probe that says Japan, and require +// `verified`. +func TestSubmitProviderEgressLocationVerdictMmdbDivergenceIsNotSuspect(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + clientIp := "24.48.0.1" + + mmdbLocation, _, err := GetLocationForIp(ctx, clientIp) + connect.AssertEqual(t, err, nil) + // the fixture is only meaningful if mmdb disagrees with the probe below + connect.AssertEqual(t, mmdbLocation.CountryCode, "ca") + model.CreateLocation(ctx, mmdbLocation) + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientIp+":0", handlerId) + connect.AssertEqual(t, err, nil) + err = SetConnectionLocation(ctx, connectionId, clientIp) + connect.AssertEqual(t, err, nil) + + _, err = SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "JP", + Country: "Japan", + CountryConfident: true, + ObservedAt: server.NowUtc(), + }) + connect.AssertEqual(t, err, nil) + + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected the submission to be stored") + } + connect.AssertEqual(t, stored.CountryCode, "jp") + if stored.Verdict != "verified" { + t.Errorf( + "verdict = %q, want %q: the probed country (jp) differing from the mmdb country (%s) is the point of probing and must never be suspicious on its own", + stored.Verdict, + "verified", + mmdbLocation.CountryCode, + ) + } + if stored.VerdictReason != "" { + t.Errorf("verdict_reason = %q, want empty", stored.VerdictReason) + } + }) +} From 12921b425e642909a9383204bbed27941c0114c2 Mon Sep 17 00:00:00 2001 From: Ryan Mello Date: Sun, 2 Aug 2026 15:53:37 +0100 Subject: [PATCH 29/30] feat(model,api): receive client blackhole verdicts, quorum reprioritises Clients already detect a blackholed provider and remove it (connect's detectBlackhole); the verdict had nowhere to go. This adds the receiver and the aggregation. POST /network/provider-verdict is authed by client session, not by the operator secret: the reporter is a real network, and the network is the unit the quorum counts. reporter_network_id is taken from the jwt and never from the body -- a body-supplied reporter would make the count free to fake, which is the only thing the quorum rests on. The args type decodes strictly (unknown field -> 400), because a misspelled receive_ack_count decodes to zero and zero receive acks is precisely the value that means "egress dead": a typo would be a counting error, not a malformed request. Everything validates before any store. Aggregation is pure, read-time, and lives in one place (ProviderClientVerdictQuorumMet): egress-dead only (receive acks zero, tested on the counts and never on the reason string), decay outside a 15-minute window, and AT MOST ONE COUNTED VERDICT PER REPORTER NETWORK. That last rule is why the reporter collection is a set and not a counter: the table is append-only on purpose, so one network may say anything as often as it likes and still move the count by one. A met quorum ONLY brings the provider's next egress probe forward -- it never demotes, excludes, or touches filter sets, scores, PassesMinimums or find-providers2. This is a deliberate amendment to the source spec, which excluded on quorum. Client verdicts are the harder signal to game, but the spec paired exclusion with "the prober rehabilitates immediately", and that pair is a laundering mechanism: blackhole real users, get reported, get probed by the prober you already special-case, pass, repeat. And with NetworkCreateDailyLimit = 5 a three-network quorum is about fifteen minutes of sybil work, so exclusion would also let a griefer take an honest provider offline for three accounts. Separating the trigger (client verdicts) from the punishment (the prober, the sole authority) makes griefing cost one probe and no more. The prober confirming is what feeds the existing probation machinery; this adds no new gate. The reprioritise moves observed_at back to three quarters of the max age: past the due cutoff (half the max age) so the prober is offered the provider, and short of the max age so the location stays trusted by the connection path and survives the expiry sweep. It is non-ratcheting (`$2 < observed_at`), so repeated quorums cannot walk a row out of the freshness window, and it deliberately does not touch provider_egress_probe_attempt -- so the most any quorum can buy is one probe per provider per 6h backoff. Migration is a strict tail append on this file (530 -> 532): the append-only table, plus a (provider_client_id, create_time) index, because the window read is on the submit path and the table grows without bound per reporter. Deliberately not renumbered to match the beta deployment's 543 -> 545; migration indices legitimately differ per deployment, and these entries are appended, never inserted. Tests: 12, all against the local stack. Three distinct reporters meet quorum; one reporter twelve times does not; a verdict with receive acks does not count; a decayed verdict does not count; session auth fails closed; a body naming its own reporter_network_id is rejected; unknown reason / unknown field / negative counts 400 with nothing stored; and quorum makes the provider due using the endpoint's own due arithmetic while the stored location, country and probe verdict are unchanged. Co-Authored-By: Claude Opus 4.8 Claude-Session: https://claude.ai/code/session_01QtgqtCmKJRXdsQ5ktiqwkg --- api/api.go | 7 + .../provider_client_verdict_handlers.go | 25 + .../provider_client_verdict_handlers_test.go | 426 +++++++++++++++ db_migrations.go | 62 +++ model/provider_client_verdict_model.go | 501 ++++++++++++++++++ model/provider_client_verdict_model_test.go | 311 +++++++++++ 6 files changed, 1332 insertions(+) create mode 100644 api/handlers/provider_client_verdict_handlers.go create mode 100644 api/handlers/provider_client_verdict_handlers_test.go create mode 100644 model/provider_client_verdict_model.go create mode 100644 model/provider_client_verdict_model_test.go diff --git a/api/api.go b/api/api.go index b235357b..d08ae27f 100644 --- a/api/api.go +++ b/api/api.go @@ -66,6 +66,13 @@ func Routes() []*router.Route { // opened. Until this existed the result was a log line and nothing // else. router.NewRoute("POST", "/network/provider-egress-health", handlers.ProviderEgressHealthResult), + // client-to-server, and the only route in this group that is NOT + // operator-secret authed: a real client network reporting that a + // provider carried nothing. The reporting network is taken from the + // session jwt, never from the body, because the quorum counts distinct + // networks. A met quorum only brings the provider's next probe + // forward -- see model.ProviderClientVerdictQuorumMet. + router.NewRoute("POST", "/network/provider-verdict", handlers.ProviderClientVerdictSubmit), router.NewRoute("GET", "/network/clients", handlers.NetworkClients), router.NewRoute("GET", "/network/peers", handlers.NetworkPeers), router.NewRoute("GET", "/network/provider-locations", handlers.NetworkGetProviderLocations), diff --git a/api/handlers/provider_client_verdict_handlers.go b/api/handlers/provider_client_verdict_handlers.go new file mode 100644 index 00000000..a315173f --- /dev/null +++ b/api/handlers/provider_client_verdict_handlers.go @@ -0,0 +1,25 @@ +package handlers + +import ( + "net/http" + + "github.com/urnetwork/server/model" + "github.com/urnetwork/server/router" +) + +// ProviderClientVerdictSubmit receives one client-reported blackhole verdict. +// +// Unlike the other provider-probing ingest endpoints in this package, this one +// is NOT operator-secret authed: the reporter is a real client network, and the +// network is the unit the quorum counts. WrapWithInputRequireAuth fails closed +// with a 401 when the jwt is missing or unparseable, and hands the impl a +// session whose ByJwt.NetworkId is the reporter -- which is the only place the +// reporter identity may come from. +// +// All validation, the append-only store and the quorum aggregation live in +// model.SubmitProviderClientVerdict; the strict (unknown-field-rejecting) +// decode is attached to the args type, because the router's decoder is shared +// by every endpoint and must not be tightened globally. +func ProviderClientVerdictSubmit(w http.ResponseWriter, r *http.Request) { + router.WrapWithInputRequireAuth(model.SubmitProviderClientVerdict, w, r) +} diff --git a/api/handlers/provider_client_verdict_handlers_test.go b/api/handlers/provider_client_verdict_handlers_test.go new file mode 100644 index 00000000..de611bcc --- /dev/null +++ b/api/handlers/provider_client_verdict_handlers_test.go @@ -0,0 +1,426 @@ +package handlers + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "slices" + "testing" + "time" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/jwt" + "github.com/urnetwork/server/model" +) + +// validClientVerdictBody is a well-formed egress-dead report: the client sent +// and was acknowledged, and nothing came back. Tests mutate a copy to exercise +// one rule at a time. +// +// Note what is NOT in it: reporter_network_id. The reporter is the session. +func validClientVerdictBody(exitClientId server.Id) map[string]any { + return map[string]any{ + "exit_client_id": exitClientId.String(), + "reason": model.ProviderClientVerdictReasonNoReceiveAck, + "send_ack_count": 64, + "send_ack_bytes": 8192, + "receive_ack_count": 0, + "receive_ack_bytes": 0, + "syn_sent": 3, + "syn_received": 0, + "window_seconds": 30, + } +} + +// postClientVerdict posts as networkId, or unauthenticated when networkId is +// the zero id. +func postClientVerdict( + t testing.TB, + networkId server.Id, + body map[string]any, +) *httptest.ResponseRecorder { + t.Helper() + buf, err := json.Marshal(body) + if err != nil { + t.Fatalf("marshal body: %s", err) + } + req := httptest.NewRequest(http.MethodPost, "/network/provider-verdict", bytes.NewReader(buf)) + if networkId != (server.Id{}) { + byJwt := jwt.NewByJwt(networkId, server.NewId(), "test", false, false) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", byJwt.Sign())) + } + w := httptest.NewRecorder() + ProviderClientVerdictSubmit(w, req) + return w +} + +// Fails closed. Without a session there is no reporter network, and a verdict +// with no reporter is a verdict that cannot be counted or capped. +func TestProviderClientVerdictRejectsUnauthenticated(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + exitClientId := server.NewId() + + w := postClientVerdict(t, server.Id{}, validClientVerdictBody(exitClientId)) + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 for an unauthenticated report", w.Code) + } + + verdicts := model.GetProviderClientVerdictsInWindow( + context.Background(), + exitClientId, + server.NowUtc(), + ) + if len(verdicts) != 0 { + t.Fatalf("stored %d verdicts from an unauthenticated request, want 0", len(verdicts)) + } + }) +} + +// THE REPORTER IS THE SESSION. Two sessions posting a byte-identical body must +// store two different reporter networks -- and a body that tries to name its +// own reporter never gets in at all. +// +// The second half is a deviation worth being explicit about: because the args +// type decodes strictly, a body carrying reporter_network_id is REJECTED with a +// 400 rather than silently ignored. Either way it can never be honoured, and a +// loud rejection beats a client that believes it is choosing its own reporter +// id and is quietly overruled. +func TestProviderClientVerdictReporterComesFromTheSession(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + exitClientId := server.NewId() + reporterA := server.NewId() + reporterB := server.NewId() + liar := server.NewId() + + if w := postClientVerdict(t, reporterA, validClientVerdictBody(exitClientId)); w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + if w := postClientVerdict(t, reporterB, validClientVerdictBody(exitClientId)); w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + + // a body naming another network, posted by reporterA + lying := validClientVerdictBody(exitClientId) + lying["reporter_network_id"] = liar.String() + if w := postClientVerdict(t, reporterA, lying); w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 for a body naming its own reporter_network_id", w.Code) + } + + verdicts := model.GetProviderClientVerdictsInWindow(ctx, exitClientId, server.NowUtc()) + reporters := []server.Id{} + for _, verdict := range verdicts { + reporters = append(reporters, verdict.ReporterNetworkId) + } + if len(reporters) != 2 { + t.Fatalf("stored %d verdicts, want 2 (the lying body must not have been stored)", len(reporters)) + } + if !slices.Contains(reporters, reporterA) || !slices.Contains(reporters, reporterB) { + t.Fatalf("reporters = %v, want exactly the two session networks %s and %s", + reporters, reporterA, reporterB) + } + if slices.Contains(reporters, liar) { + t.Fatalf("reporters = %v: the body's reporter_network_id was honoured", reporters) + } + }) +} + +func TestProviderClientVerdictRejectsBadBodies(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + reporter := server.NewId() + + cases := []struct { + name string + mutate func(body map[string]any) + }{ + { + // an open reason column fills with whatever a client build + // happens to send, and then nothing can be counted by it + name: "unknown reason", + mutate: func(body map[string]any) { + body["reason"] = "slow" + }, + }, + { + name: "empty reason", + mutate: func(body map[string]any) { + body["reason"] = "" + }, + }, + { + // a misspelled count decodes to zero, and zero receive acks is + // exactly the value that means "egress dead" -- so a typo would + // not be a malformed report, it would be a counting one + name: "unknown field", + mutate: func(body map[string]any) { + body["recieve_ack_count"] = 0 + }, + }, + { + name: "negative receive ack count", + mutate: func(body map[string]any) { + body["receive_ack_count"] = -1 + }, + }, + { + name: "negative send ack bytes", + mutate: func(body map[string]any) { + body["send_ack_bytes"] = -8192 + }, + }, + { + name: "negative syn received", + mutate: func(body map[string]any) { + body["syn_received"] = -1 + }, + }, + { + name: "negative window", + mutate: func(body map[string]any) { + body["window_seconds"] = -30 + }, + }, + { + name: "missing exit client id", + mutate: func(body map[string]any) { + delete(body, "exit_client_id") + }, + }, + } + + for _, testCase := range cases { + exitClientId := server.NewId() + body := validClientVerdictBody(exitClientId) + testCase.mutate(body) + + w := postClientVerdict(t, reporter, body) + if w.Code != http.StatusBadRequest { + t.Errorf("%s: status = %d, want 400: %s", testCase.name, w.Code, w.Body.String()) + } + + // 400 BEFORE any store, never store-then-flag: a row that is in + // this table is a row that counts + verdicts := model.GetProviderClientVerdictsInWindow( + context.Background(), + exitClientId, + server.NowUtc(), + ) + if len(verdicts) != 0 { + t.Errorf("%s: stored %d verdicts on a rejected body, want 0", testCase.name, len(verdicts)) + } + } + }) +} + +func TestProviderClientVerdictStoresValidReport(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + exitClientId := server.NewId() + reporter := server.NewId() + + w := postClientVerdict(t, reporter, validClientVerdictBody(exitClientId)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + + verdicts := model.GetProviderClientVerdictsInWindow(ctx, exitClientId, server.NowUtc()) + if len(verdicts) != 1 { + t.Fatalf("stored %d verdicts, want 1", len(verdicts)) + } + verdict := verdicts[0] + if verdict.ReporterNetworkId != reporter { + t.Fatalf("reporter = %s, want the session network %s", verdict.ReporterNetworkId, reporter) + } + if verdict.Reason != model.ProviderClientVerdictReasonNoReceiveAck { + t.Fatalf("reason = %q", verdict.Reason) + } + if verdict.SendAckCount != 64 || verdict.SendAckBytes != 8192 { + t.Fatalf("send acks = %d/%dB", verdict.SendAckCount, verdict.SendAckBytes) + } + if verdict.ReceiveAckCount != 0 || verdict.ReceiveAckBytes != 0 { + t.Fatalf("receive acks = %d/%dB", verdict.ReceiveAckCount, verdict.ReceiveAckBytes) + } + if verdict.WindowSeconds != 30 { + t.Fatalf("window seconds = %d", verdict.WindowSeconds) + } + }) +} + +// testing_connectProbeableProvider is the api/handlers copy of the model test +// fixture: a connected, valid provider holding a Public provide key, which is +// what GetProviderEgressLocationDue requires before it will offer a provider. +func testing_connectProbeableProvider( + t testing.TB, + ctx context.Context, + clientId server.Id, + locationId server.Id, + clientAddress string, +) { + t.Helper() + model.Testing_CreateDevice(ctx, server.NewId(), server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientAddress, handlerId) + if err != nil { + t.Fatalf("connect client: %s", err) + } + if err := model.SetConnectionLocation(ctx, connectionId, locationId, &model.ConnectionLocationScores{}); err != nil { + t.Fatalf("set connection location: %s", err) + } + model.SetProvide(ctx, clientId, map[model.ProvideMode][]byte{ + model.ProvideModePublic: []byte("provide-secret"), + }) +} + +// End to end: three distinct networks report a provider egress-dead and the +// prober is offered that provider on its next poll -- and nothing else moves. +// +// The due cutoff here is the ENDPOINT's own arithmetic (providerEgressDueAge), +// not a hand-picked one. model cannot import that constant, so this is the test +// that fails if the reprioritise target and the due cutoff ever drift apart, +// instead of the feature silently doing nothing. +func TestProviderClientVerdictQuorumMakesTheProviderDue(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + + city := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + model.CreateLocation(ctx, city) + + reported := server.NewId() + quiet := server.NewId() + testing_connectProbeableProvider(t, ctx, reported, city.LocationId, "0.0.0.1:0") + testing_connectProbeableProvider(t, ctx, quiet, city.LocationId, "0.0.0.2:0") + model.UpdateClientLocationReliabilities(ctx, now.Add(-time.Hour), now) + + // both probed an hour ago: neither is due, and neither has a probe + // attempt row, so nothing is deferred by the attempt backoff either + for _, clientId := range []server.Id{reported, quiet} { + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: clientId, + LocationId: city.LocationId, + CountryCode: "us", + ObservedAt: now.Add(-time.Hour), + Verdict: "verified", + }) + } + + due := func() []server.Id { + return model.GetProviderEgressLocationDue( + ctx, + server.NowUtc().Add(-providerEgressDueAge), + server.NowUtc().Add(-model.ProviderEgressProbeAttemptBackoff), + 100, + ) + } + if slices.Contains(due(), reported) { + t.Fatal("the provider was already due before any verdict") + } + + // two reporters are not a quorum + for range 2 { + w := postClientVerdict(t, server.NewId(), validClientVerdictBody(reported)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body.String()) + } + } + if slices.Contains(due(), reported) { + t.Fatal("two reporters made the provider due, want three") + } + + w := postClientVerdict(t, server.NewId(), validClientVerdictBody(reported)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body.String()) + } + + if !slices.Contains(due(), reported) { + t.Fatal("a met quorum did not make the provider due for probing") + } + // the effect is scoped to the reported provider + if slices.Contains(due(), quiet) { + t.Fatal("an unreported provider became due") + } + + // AND NOTHING IN THE SELECTION PATH MOVED. The location still resolves, + // with the same location id, country and verdict -- a met quorum + // schedules a probe and does not demote, exclude or rescore anything. + // (Nothing in selection reads observed_at at all; the freshness lookup + // below is the closest thing to it that exists.) + fresh := model.GetFreshProviderEgressLocation(ctx, reported, model.ProviderEgressLocationMaxAge) + if fresh == nil { + t.Fatal("the reported provider's location stopped being fresh") + } + if fresh.LocationId != city.LocationId { + t.Fatalf("location id = %s, want %s", fresh.LocationId, city.LocationId) + } + if fresh.CountryCode != "us" { + t.Fatalf("country code = %q, want us", fresh.CountryCode) + } + if fresh.Verdict != "verified" { + t.Fatalf("verdict = %q, want verified: a client verdict must not touch the probe verdict", fresh.Verdict) + } + }) +} + +// The quorum brings the next probe forward; it does not override the attempt +// backoff. A provider the prober tried minutes ago stays deferred, so the most +// a quorum can buy -- honest or manufactured -- is one probe per provider per +// ProviderEgressProbeAttemptBackoff. That bound is what keeps the cost of +// griefing at one probe, and it is why the test above deliberately has no +// attempt row. +func TestProviderClientVerdictQuorumDoesNotOverrideTheAttemptBackoff(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + + city := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + model.CreateLocation(ctx, city) + + clientId := server.NewId() + testing_connectProbeableProvider(t, ctx, clientId, city.LocationId, "0.0.0.3:0") + model.UpdateClientLocationReliabilities(ctx, now.Add(-time.Hour), now) + + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: clientId, + LocationId: city.LocationId, + CountryCode: "us", + ObservedAt: now.Add(-time.Hour), + }) + model.SetProviderEgressProbeAttempt(ctx, &model.ProviderEgressProbeAttempt{ + ClientId: clientId, + AttemptAt: now.Add(-5 * time.Minute), + }) + + for range model.ProviderClientVerdictQuorum { + w := postClientVerdict(t, server.NewId(), validClientVerdictBody(clientId)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body.String()) + } + } + + due := model.GetProviderEgressLocationDue( + ctx, + server.NowUtc().Add(-providerEgressDueAge), + server.NowUtc().Add(-model.ProviderEgressProbeAttemptBackoff), + 100, + ) + if slices.Contains(due, clientId) { + t.Fatal("a met quorum overrode the probe attempt backoff") + } + }) +} diff --git a/db_migrations.go b/db_migrations.go index b2647c67..791ee2bb 100644 --- a/db_migrations.go +++ b/db_migrations.go @@ -4640,4 +4640,66 @@ var migrations = []any{ PRIMARY KEY (client_id) ) `), + + // Blackhole verdicts reported by real clients: one row per report, per + // provider, per reporting network. A client that removes a provider for + // carrying nothing (see connect's detectBlackhole) says so here. + // + // APPEND-ONLY ON PURPOSE. A reporter may say anything as often as it likes; + // the cap is on what a reporter can COUNT FOR -- at most one verdict per + // reporter network per provider per aggregation window -- and it is applied + // at read time, in ProviderClientVerdictQuorumMet, never on the write path. + // Capping the writes instead would make the table lie about what was + // actually reported, and would put a rate-limit decision in front of the + // one signal that says a provider is dead. + // + // reporter_network_id comes from the authenticated session and never from + // the request body. It is the entire basis of the quorum: distinct networks + // are what a griefer has to buy, and a body-supplied reporter id would cost + // nothing at all. + // + // A met quorum only REPRIORITISES the provider for probing -- it never + // demotes, excludes, or touches filter sets, scores, PassesMinimums or + // find-providers2. See ProviderClientVerdictQuorumMet for why the trigger + // (client verdicts) and the punishment (the prober) are separated. + // + // syn_sent/syn_received are accepted and validated by the endpoint but + // deliberately not columns here: aggregation keys on receive_ack_count + // alone, and a column nothing reads is a column that drifts. + // + // Appended, never inserted: migrations here apply by slice index + // (`for i := DbVersion(ctx); i < upTo; i++`), so editing or reordering an + // already-applied entry corrupts live databases. IF NOT EXISTS makes a + // re-run -- or a duplicated merge resolution -- a no-op. + newSqlMigration(` + CREATE TABLE IF NOT EXISTS provider_client_verdict ( + provider_client_id uuid NOT NULL, + reporter_network_id uuid NOT NULL, + reason text NOT NULL, + send_ack_count bigint NOT NULL, + send_ack_bytes bigint NOT NULL, + receive_ack_count bigint NOT NULL, + receive_ack_bytes bigint NOT NULL, + window_seconds int NOT NULL, + create_time timestamp NOT NULL, + + PRIMARY KEY (provider_client_id, reporter_network_id, create_time) + ) + `), + + // serves the window read (GetProviderClientVerdictsInWindow), which is + // `provider_client_id = $1 AND $2 <= create_time ORDER BY create_time`. + // The primary key's leading column alone would find the provider's rows and + // then filter and sort every verdict ever written about it -- and this table + // is append-only and unbounded per reporter, so that set only grows. With + // create_time second the window is an ordered range scan that stops at the + // scan limit. + // + // reporter_network_id is deliberately not in the index: the read does not + // filter on it, and the one-verdict-per-reporter cap is applied in Go, not + // by the database. + newSqlMigration(` + CREATE INDEX IF NOT EXISTS provider_client_verdict_provider_create_time + ON provider_client_verdict (provider_client_id, create_time) + `), } diff --git a/model/provider_client_verdict_model.go b/model/provider_client_verdict_model.go new file mode 100644 index 00000000..33e56fc7 --- /dev/null +++ b/model/provider_client_verdict_model.go @@ -0,0 +1,501 @@ +package model + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/session" +) + +// The blackhole reasons a client may report. This is a CLOSED set, derived from +// the three conditions that make connect's detectBlackhole fire +// (ip_remote_multi_client.go, `blackhole := func() bool`): +// +// - sendAckCount <= 0 within BlackholeTimeout of the first un-acked send +// - receiveAckCount <= 0 within the same window +// - receiveSynCount <= 0 within BlackholeConnectTimeout of the first syn +// +// It is an allowlist rather than free text because the reason is stored and +// read by humans: an open string column fills with whatever a client build +// happens to send, and then nothing can be counted by it. +// +// Note what the reason is NOT used for: aggregation keys on receive_ack_count +// alone (see ProviderClientVerdictQuorumMet). A client and a server that +// disagree about reason names must not be able to silently break the quorum. +const ( + // the provider acknowledged nothing the client sent + ProviderClientVerdictReasonNoSendAck = "no-send-ack" + // the client sent and was acknowledged, but nothing came back -- the + // egress-dead case, and the only one the quorum counts + ProviderClientVerdictReasonNoReceiveAck = "no-receive-ack" + // no syn came back inside the connect timeout + ProviderClientVerdictReasonNoReceiveSyn = "no-receive-syn" +) + +var providerClientVerdictReasons = map[string]bool{ + ProviderClientVerdictReasonNoSendAck: true, + ProviderClientVerdictReasonNoReceiveAck: true, + ProviderClientVerdictReasonNoReceiveSyn: true, +} + +// IsProviderClientVerdictReason reports whether reason is one of the known +// blackhole reasons. +func IsProviderClientVerdictReason(reason string) bool { + return providerClientVerdictReasons[reason] +} + +const ( + // ProviderClientVerdictQuorum is how many DISTINCT reporter networks must + // call a provider egress-dead inside the window before anything happens. + // + // Three mirrors the client-side dial-strike shape (3 strikes / 60s / any + // success clears). It is small on purpose: the consequence of a met quorum + // is a probe, not a punishment. NetworkCreateDailyLimit is 5, so three + // sybil networks cost a griefer roughly fifteen minutes -- which is exactly + // why a met quorum must never do more than schedule a probe. + ProviderClientVerdictQuorum = 3 + + // ProviderClientVerdictWindow is how long a verdict counts for. Outside it + // a verdict has decayed and contributes nothing, so a provider cannot + // accumulate a quorum out of unrelated reports spread over a day. + ProviderClientVerdictWindow = 15 * time.Minute + + // providerClientVerdictScanLimit bounds the window read. The table is + // append-only and unbounded per reporter, so without a limit one network + // writing in a loop would make every aggregation read its whole flood. + // + // The rows are read OLDEST FIRST, so the limit can only ever suppress a + // quorum, never manufacture one: eviction drops the newest reports, and the + // count is of distinct networks, which no volume of writes from one network + // can increase. Suppression is the safe direction -- it costs a probe that + // would have been scheduled, not an honest provider's traffic. + providerClientVerdictScanLimit = 4096 +) + +// ProviderClientVerdict is one client-reported blackhole verdict. +// +// ReporterNetworkId is always taken from the reporting session. Nothing may +// populate it from a request body -- see SubmitProviderClientVerdict. +type ProviderClientVerdict struct { + ProviderClientId server.Id + ReporterNetworkId server.Id + Reason string + SendAckCount int64 + SendAckBytes int64 + ReceiveAckCount int64 + ReceiveAckBytes int64 + WindowSeconds int + CreateTime time.Time +} + +// AddProviderClientVerdict appends one verdict row. +// +// ON CONFLICT DO NOTHING because the primary key includes create_time: two +// reports from the same reporter about the same provider inside the same clock +// tick would otherwise raise a duplicate-key error inside server.Tx, which +// retries the failed commit blindly for a minute before surfacing a 500. A +// dropped duplicate is exactly right anyway: the second one could not have +// counted for anything the first did not already count for. +func AddProviderClientVerdict(ctx context.Context, verdict *ProviderClientVerdict) { + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO provider_client_verdict ( + provider_client_id, + reporter_network_id, + reason, + send_ack_count, + send_ack_bytes, + receive_ack_count, + receive_ack_bytes, + window_seconds, + create_time + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (provider_client_id, reporter_network_id, create_time) DO NOTHING + `, + verdict.ProviderClientId, + verdict.ReporterNetworkId, + verdict.Reason, + verdict.SendAckCount, + verdict.SendAckBytes, + verdict.ReceiveAckCount, + verdict.ReceiveAckBytes, + verdict.WindowSeconds, + verdict.CreateTime.UTC(), + )) + }) +} + +// GetProviderClientVerdictsInWindow reads the verdicts about one provider that +// are recent enough to still count, oldest first. +// +// This is a bounded READ ONLY. It applies no policy beyond the window: not the +// egress-dead test, and above all not the one-per-reporter cap. Those live in +// ProviderClientVerdictQuorumMet, in Go, in one place, where they are unit +// testable and where breaking them fails a test rather than quietly changing an +// index plan. The database's job here is to hand back a bounded number of rows. +// +// create_time is a naive `timestamp` holding utc, so the cutoff is computed in +// Go and bound as a parameter rather than compared against sql now(), which +// would cast through the session timezone. +func GetProviderClientVerdictsInWindow( + ctx context.Context, + providerClientId server.Id, + now time.Time, +) []ProviderClientVerdict { + verdicts := []ProviderClientVerdict{} + minCreateTime := now.UTC().Add(-ProviderClientVerdictWindow) + + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT + reporter_network_id, + reason, + send_ack_count, + send_ack_bytes, + receive_ack_count, + receive_ack_bytes, + window_seconds, + create_time + FROM provider_client_verdict + WHERE + provider_client_id = $1 AND + $2 <= create_time + + -- oldest first: see providerClientVerdictScanLimit. A flood of + -- writes must not be able to evict the reports that arrived before + -- it, because that is the only direction in which the limit could + -- change a quorum into a non-quorum for a real provider. + ORDER BY create_time ASC + LIMIT $3 + `, + providerClientId, + minCreateTime, + providerClientVerdictScanLimit, + ) + server.WithPgResult(result, err, func() { + for result.Next() { + verdict := ProviderClientVerdict{ + ProviderClientId: providerClientId, + } + server.Raise(result.Scan( + &verdict.ReporterNetworkId, + &verdict.Reason, + &verdict.SendAckCount, + &verdict.SendAckBytes, + &verdict.ReceiveAckCount, + &verdict.ReceiveAckBytes, + &verdict.WindowSeconds, + &verdict.CreateTime, + )) + verdict.CreateTime = verdict.CreateTime.UTC() + verdicts = append(verdicts, verdict) + } + }) + }) + + return verdicts +} + +// ProviderClientVerdictQuorumMet is the whole aggregation policy: pure, read +// time, and the only place any of these three rules exists. +// +// 1. egress-dead only. receive_ack_count == 0 means the client sent and got +// nothing back. A verdict with any receive acks describes a provider that +// carried traffic, whatever reason string it came with, so it counts for +// nothing. The test is on the COUNTS, never on the reason string: a client +// build and a server that disagree about reason names must not be able to +// silently disable the quorum. +// 2. decay. A verdict older than ProviderClientVerdictWindow is gone. Without +// this, verdicts accumulate forever and every provider eventually reaches +// quorum from unrelated incidents months apart. +// 3. ONE VERDICT PER REPORTER NETWORK. This is the anti-griefing property, and +// it is why `reporters` is a SET and not a counter. One network can write +// as many rows as it likes -- the table is append-only on purpose -- and +// still move the count by exactly one. Turning this back into a counter +// lets a single network manufacture a quorum on its own, which is the +// precise failure this function exists to prevent. +// +// # What a met quorum is allowed to do +// +// Reprioritise the provider for probing. Nothing else. It must never demote, +// exclude, or touch filter sets, scores, PassesMinimums or find-providers2. +// +// That is a deliberate amendment to the source spec, which had a met quorum +// exclude the provider outright. Client verdicts are the harder signal to game +// -- real destinations, real vantage points, many uncoordinated reporters -- +// but the spec paired immediate exclusion with "the prober rehabilitates +// immediately", and that pair is a laundering mechanism: blackhole real users, +// get reported, get probed by the one prober you already special-case, pass, +// get rehabilitated, repeat. And with NetworkCreateDailyLimit = 5 a three +// network quorum is about fifteen minutes of sybil work, so immediate exclusion +// would also let a griefer take an honest provider offline for the price of +// three accounts. +// +// Separating the trigger (client verdicts, fast, hard to fake in aggregate) +// from the punishment (the prober, the sole authority) makes griefing cost one +// probe and nothing more -- which is what the spec wanted to be true. If the +// prober then confirms, the existing probation machinery does the gating; this +// adds no new gate. +func ProviderClientVerdictQuorumMet(verdicts []ProviderClientVerdict, now time.Time) bool { + minCreateTime := now.UTC().Add(-ProviderClientVerdictWindow) + + reporters := map[server.Id]bool{} + for _, verdict := range verdicts { + if verdict.ReceiveAckCount != 0 { + // the provider carried traffic back + continue + } + if verdict.CreateTime.UTC().Before(minCreateTime) { + // decayed + continue + } + // a set, not a counter: the cap is one counted verdict per reporter + // network per provider per window + reporters[verdict.ReporterNetworkId] = true + } + + return ProviderClientVerdictQuorum <= len(reporters) +} + +// providerClientVerdictProbeDueAge is how far back a quorum-met provider's +// observed_at is moved: far enough to be due for a probe, not so far that +// anything else changes. +// +// The column has two thresholds on it, and the target has to sit strictly +// between them: +// +// - ProviderEgressLocationMaxAge / 2 (3.5 days) -- the due cutoff +// (providerEgressDueAge in api/handlers). Older than this and the prober is +// offered the provider. +// - ProviderEgressLocationMaxAge (7 days) -- past this the stored location +// stops being trusted at all (GetFreshProviderEgressLocation*, which falls +// back to the mmdb lookup) and RemoveExpiredProviderEgressLocations deletes +// the row. +// +// Backdating past the second one would turn a met quorum into a selection-path +// effect plus data loss, which is exactly what this design forbids. Three +// quarters of the max age is comfortably inside both, and is derived from the +// one constant so it cannot drift away from them. +// +// api/handlers may not be imported here (it imports model), so the due cutoff +// cannot be referenced directly; the handler-side test asserts the provider is +// actually due using the handler's own arithmetic, so a future change to either +// constant breaks a test instead of silently disabling this. +const providerClientVerdictProbeDueAge = ProviderEgressLocationMaxAge * 3 / 4 + +// ReprioritiseProviderEgressProbe brings a provider's next egress probe +// forward, by moving its stored observed_at back to +// providerClientVerdictProbeDueAge ago. It reports whether a row moved. +// +// This is the ONLY effect a met client-verdict quorum has. It touches exactly +// one meaningful column of one row. No filter set, no score, no PassesMinimums, +// no find-providers2 path reads observed_at -- the selection path reads the +// location through GetFreshProviderEgressLocation*, which still resolves the +// same location, because the new observed_at is still inside +// ProviderEgressLocationMaxAge. +// +// `$2 < observed_at` makes this idempotent and, more importantly, non +// ratcheting: repeated quorums cannot walk a provider's observed_at further and +// further back until the row expires out of the freshness window. A row already +// at or past the target is left exactly where it is. +// +// A provider with no provider_egress_location row at all is not a miss: it has +// never been probed successfully, and the due queue already sorts it ahead of +// every probed provider (GetProviderEgressLocationDue pass 1). There is nothing +// to bring forward. +// +// What this deliberately does NOT touch is provider_egress_probe_attempt. A +// provider tried within ProviderEgressProbeAttemptBackoff (6h) stays deferred, +// so the most a quorum can buy -- honest or manufactured -- is one probe per +// provider per backoff window. That bound is the point: it is what keeps the +// cost of griefing at one probe. +func ReprioritiseProviderEgressProbe( + ctx context.Context, + providerClientId server.Id, + now time.Time, +) bool { + moved := false + dueAt := now.UTC().Add(-providerClientVerdictProbeDueAge) + + server.Tx(ctx, func(tx server.PgTx) { + tag, err := tx.Exec( + ctx, + ` + UPDATE provider_egress_location + SET + observed_at = $2, + update_time = $3 + WHERE + client_id = $1 AND + $2 < observed_at + `, + providerClientId, + dueAt, + server.NowUtc(), + ) + server.Raise(err) + moved = 0 < tag.RowsAffected() + }) + + return moved +} + +// SubmitProviderClientVerdictArgs is one client-reported blackhole verdict as +// the reporting client sends it. +// +// There is deliberately no reporter_network_id field. The reporter is the +// authenticated session and nothing else; declaring the field would both invite +// the lie and, with strict decoding, turn an honest client that echoes it into +// a 400. +// +// SynSent/SynReceived are accepted and validated but not stored: the schema +// carries the ack counters the aggregation actually keys on, and a column +// nothing reads is a column that drifts. They are declared here because the +// client sends them -- with strict decoding, an undeclared field is a 400, so +// silently dropping them from the struct would reject every real submission. +type SubmitProviderClientVerdictArgs struct { + ExitClientId server.Id `json:"exit_client_id"` + Reason string `json:"reason"` + SendAckCount int64 `json:"send_ack_count"` + SendAckBytes int64 `json:"send_ack_bytes"` + ReceiveAckCount int64 `json:"receive_ack_count"` + ReceiveAckBytes int64 `json:"receive_ack_bytes"` + SynSent int64 `json:"syn_sent"` + SynReceived int64 `json:"syn_received"` + WindowSeconds int `json:"window_seconds"` +} + +// UnmarshalJSON decodes strictly: an unknown field is an error, which the +// router turns into a 400 before the impl ever runs. +// +// The router's generic decoder is a plain json.Unmarshal shared by every +// endpoint, so strictness has to be attached to the type rather than switched +// on globally -- tightening the shared decoder would change the behaviour of +// every deployed endpoint at once. +// +// Strictness matters here for the same reason it does on the health ingest: the +// body is a set of counts that are read as a judgement. A misspelled +// receive_ack_count decodes to zero, and zero is precisely the value that means +// "egress dead" -- so a typo would not be a malformed report, it would be a +// counting one. +func (self *SubmitProviderClientVerdictArgs) UnmarshalJSON(b []byte) error { + // a defined type with the same layout, minus the methods, so decoding it + // does not recurse back into this function + type strictArgs SubmitProviderClientVerdictArgs + + decoder := json.NewDecoder(bytes.NewReader(b)) + decoder.DisallowUnknownFields() + + var parsed strictArgs + if err := decoder.Decode(&parsed); err != nil { + return err + } + + *self = SubmitProviderClientVerdictArgs(parsed) + return nil +} + +// SubmitProviderClientVerdictResult is deliberately empty. +// +// Telling the reporter whether its verdict met the quorum would hand a griefer +// a progress bar for its sybil campaign -- and would tell any client how many +// other networks are currently reporting a given provider. The reporter needs +// to know its report was accepted, which is the 200. +type SubmitProviderClientVerdictResult struct { +} + +// SubmitProviderClientVerdict records one client-reported blackhole verdict and, +// if the report completes a quorum, brings the provider's next egress probe +// forward. +// +// # The reporter is the session +// +// reporter_network_id comes from the authenticated jwt. This is the security +// property the whole aggregation rests on: the quorum counts distinct networks, +// networks cost account creation (NetworkCreateDailyLimit = 5 per day), and a +// body-supplied reporter id would make the count free to fake. +// +// # Validation happens before any store +// +// Every rule below returns 400 without writing. A stored-then-flagged row is +// not an option for a table whose only consumer is a count: a row that is in +// the table is a row that counts. +// +// # A met quorum only reprioritises +// +// See ProviderClientVerdictQuorumMet for the full reasoning. Nothing here +// touches filter sets, scores, PassesMinimums or find-providers2, and this adds +// no new gate: if the prober confirms the provider is dead, the existing +// probation machinery is what acts on it. +func SubmitProviderClientVerdict( + verdict *SubmitProviderClientVerdictArgs, + clientSession *session.ClientSession, +) (*SubmitProviderClientVerdictResult, error) { + // the route is wrapped in RequireAuth, so a session without a jwt cannot + // reach here. Checked anyway: the reporter identity is the security + // property, and a nil ByJwt must fail closed rather than panic or, worse, + // write a zero reporter network that every other zero reporter dedups with. + if clientSession == nil || clientSession.ByJwt == nil { + return nil, fmt.Errorf("%d Not authorized.", http.StatusUnauthorized) + } + reporterNetworkId := clientSession.ByJwt.NetworkId + + if verdict.ExitClientId == (server.Id{}) { + return nil, fmt.Errorf("%d Missing exit_client_id.", http.StatusBadRequest) + } + if !IsProviderClientVerdictReason(verdict.Reason) { + return nil, fmt.Errorf("%d Unknown verdict reason.", http.StatusBadRequest) + } + // a negative count is not a measurement. It is also the one input that + // could make the egress-dead test read strangely if it were ever changed + // from `!= 0` to `<= 0`, so it is rejected at the door. + if verdict.SendAckCount < 0 || + verdict.SendAckBytes < 0 || + verdict.ReceiveAckCount < 0 || + verdict.ReceiveAckBytes < 0 || + verdict.SynSent < 0 || + verdict.SynReceived < 0 { + return nil, fmt.Errorf("%d Counts must be non-negative.", http.StatusBadRequest) + } + if verdict.WindowSeconds < 0 { + return nil, fmt.Errorf("%d window_seconds must be non-negative.", http.StatusBadRequest) + } + + // the server stamps the time, exactly as the operator ingest endpoints do. + // A client-supplied timestamp would be one more thing to validate and one + // more way for a skewed or hostile clock to place a verdict inside a window + // it does not belong to -- and the window is the decay rule. + now := server.NowUtc() + + AddProviderClientVerdict(clientSession.Ctx, &ProviderClientVerdict{ + ProviderClientId: verdict.ExitClientId, + ReporterNetworkId: reporterNetworkId, + Reason: verdict.Reason, + SendAckCount: verdict.SendAckCount, + SendAckBytes: verdict.SendAckBytes, + ReceiveAckCount: verdict.ReceiveAckCount, + ReceiveAckBytes: verdict.ReceiveAckBytes, + WindowSeconds: verdict.WindowSeconds, + CreateTime: now, + }) + + // read-time aggregation, evaluated on the submit that might have completed + // the quorum. No scheduler and no second cadence: the only moment the + // answer can change is a write. + inWindow := GetProviderClientVerdictsInWindow(clientSession.Ctx, verdict.ExitClientId, now) + if ProviderClientVerdictQuorumMet(inWindow, now) { + ReprioritiseProviderEgressProbe(clientSession.Ctx, verdict.ExitClientId, now) + } + + return &SubmitProviderClientVerdictResult{}, nil +} diff --git a/model/provider_client_verdict_model_test.go b/model/provider_client_verdict_model_test.go new file mode 100644 index 00000000..d7ef259d --- /dev/null +++ b/model/provider_client_verdict_model_test.go @@ -0,0 +1,311 @@ +package model + +import ( + "context" + "testing" + "time" + + "github.com/urnetwork/connect" + + "github.com/urnetwork/server" +) + +// testing_addEgressDeadVerdict writes one egress-dead verdict (receive acks +// zero) at an explicit time. The time is explicit because the primary key +// includes create_time: three reports written in the same clock tick would +// collapse to one row and the same-reporter test would pass for the wrong +// reason. +func testing_addEgressDeadVerdict( + ctx context.Context, + providerClientId server.Id, + reporterNetworkId server.Id, + createTime time.Time, +) { + AddProviderClientVerdict(ctx, &ProviderClientVerdict{ + ProviderClientId: providerClientId, + ReporterNetworkId: reporterNetworkId, + Reason: ProviderClientVerdictReasonNoReceiveAck, + SendAckCount: 64, + SendAckBytes: 8192, + ReceiveAckCount: 0, + ReceiveAckBytes: 0, + WindowSeconds: 30, + CreateTime: createTime, + }) +} + +func testing_quorumMet(ctx context.Context, providerClientId server.Id, now time.Time) bool { + return ProviderClientVerdictQuorumMet( + GetProviderClientVerdictsInWindow(ctx, providerClientId, now), + now, + ) +} + +// Three distinct reporter networks, all inside the window, all egress-dead: +// this is the case the quorum exists to detect. +func TestProviderClientVerdictQuorumMetByThreeDistinctReporters(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + providerClientId := server.NewId() + + if testing_quorumMet(ctx, providerClientId, now) { + t.Fatal("quorum met with no verdicts at all") + } + + testing_addEgressDeadVerdict(ctx, providerClientId, server.NewId(), now.Add(-10*time.Minute)) + testing_addEgressDeadVerdict(ctx, providerClientId, server.NewId(), now.Add(-5*time.Minute)) + if testing_quorumMet(ctx, providerClientId, now) { + t.Fatal("quorum met with two reporters, want three") + } + + testing_addEgressDeadVerdict(ctx, providerClientId, server.NewId(), now.Add(-time.Minute)) + if !testing_quorumMet(ctx, providerClientId, now) { + t.Fatal("quorum not met with three distinct reporters inside the window") + } + + // verdicts about a different provider are not this provider's problem + connect.AssertEqual(t, testing_quorumMet(ctx, server.NewId(), now), false) + }) +} + +// THE ANTI-GRIEFING PROPERTY. One network, reporting as many times as it likes, +// moves the count by exactly one. +// +// The table is append-only on purpose -- a reporter may SAY anything, as often +// as it likes -- so this cap is the only thing standing between a single +// account and a manufactured quorum. Turn the reporter set in +// ProviderClientVerdictQuorumMet into a counter and this test must fail; if it +// still passes, the cap is not being enforced anywhere. +func TestProviderClientVerdictQuorumNotMetByOneReporterRepeating(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + providerClientId := server.NewId() + griefer := server.NewId() + + // well past the quorum in raw rows + for i := range 12 { + testing_addEgressDeadVerdict( + ctx, + providerClientId, + griefer, + now.Add(-time.Duration(i+1)*time.Minute), + ) + } + + // every row was written -- the cap is on what a reporter can count for, + // never on what it can write + verdicts := GetProviderClientVerdictsInWindow(ctx, providerClientId, now) + connect.AssertEqual(t, len(verdicts), 12) + + if ProviderClientVerdictQuorumMet(verdicts, now) { + t.Fatal("one reporter network met the quorum by repeating itself") + } + + // and two honest reporters on top of the flood are still only three + // networks short of nothing: 1 + 2 = 3 distinct, which IS a quorum. The + // flood contributed exactly one. + testing_addEgressDeadVerdict(ctx, providerClientId, server.NewId(), now.Add(-time.Minute)) + if testing_quorumMet(ctx, providerClientId, now) { + t.Fatal("two distinct networks met the quorum") + } + testing_addEgressDeadVerdict(ctx, providerClientId, server.NewId(), now.Add(-time.Minute)) + if !testing_quorumMet(ctx, providerClientId, now) { + t.Fatal("three distinct networks did not meet the quorum") + } + }) +} + +// A verdict that reports received acks describes a provider that carried +// traffic back, whatever reason string it carries. It counts for nothing. +func TestProviderClientVerdictReceivingProviderDoesNotCount(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + providerClientId := server.NewId() + + testing_addEgressDeadVerdict(ctx, providerClientId, server.NewId(), now.Add(-time.Minute)) + testing_addEgressDeadVerdict(ctx, providerClientId, server.NewId(), now.Add(-time.Minute)) + + // third reporter, inside the window, but the provider acknowledged and + // returned traffic. Note the reason still says no-receive-ack: the + // counts are what the aggregation reads, never the reason string. + AddProviderClientVerdict(ctx, &ProviderClientVerdict{ + ProviderClientId: providerClientId, + ReporterNetworkId: server.NewId(), + Reason: ProviderClientVerdictReasonNoReceiveAck, + SendAckCount: 64, + SendAckBytes: 8192, + ReceiveAckCount: 17, + ReceiveAckBytes: 4096, + WindowSeconds: 30, + CreateTime: now.Add(-time.Minute), + }) + + verdicts := GetProviderClientVerdictsInWindow(ctx, providerClientId, now) + // the row is stored and readable -- it is just not counted + connect.AssertEqual(t, len(verdicts), 3) + if ProviderClientVerdictQuorumMet(verdicts, now) { + t.Fatal("a verdict with receive acks counted toward the quorum") + } + }) +} + +// Decay: a verdict older than the window contributes nothing. Without this a +// provider accumulates a quorum out of unrelated incidents days apart. +func TestProviderClientVerdictOutsideWindowDoesNotCount(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + providerClientId := server.NewId() + stale := server.NewId() + + testing_addEgressDeadVerdict(ctx, providerClientId, server.NewId(), now.Add(-time.Minute)) + testing_addEgressDeadVerdict(ctx, providerClientId, server.NewId(), now.Add(-2*time.Minute)) + // one minute past the window + testing_addEgressDeadVerdict( + ctx, + providerClientId, + stale, + now.Add(-ProviderClientVerdictWindow-time.Minute), + ) + + // the window read already drops it, so the third reporter is not even + // offered to the aggregation + verdicts := GetProviderClientVerdictsInWindow(ctx, providerClientId, now) + connect.AssertEqual(t, len(verdicts), 2) + if ProviderClientVerdictQuorumMet(verdicts, now) { + t.Fatal("a decayed verdict counted toward the quorum") + } + + // and the pure aggregation drops it on its own, handed the row + // directly: the two layers are checked independently on purpose, since + // the window read is a scan bound and the aggregation is the policy + stalePlusTwo := append(verdicts, ProviderClientVerdict{ + ProviderClientId: providerClientId, + ReporterNetworkId: stale, + Reason: ProviderClientVerdictReasonNoReceiveAck, + ReceiveAckCount: 0, + CreateTime: now.Add(-ProviderClientVerdictWindow - time.Minute), + }) + if ProviderClientVerdictQuorumMet(stalePlusTwo, now) { + t.Fatal("the aggregation counted a verdict from outside the window") + } + + // the same reporter, inside the window, is the third network + testing_addEgressDeadVerdict(ctx, providerClientId, stale, now.Add(-time.Minute)) + if !testing_quorumMet(ctx, providerClientId, now) { + t.Fatal("a fresh third verdict did not meet the quorum") + } + }) +} + +// The reason allowlist is closed, and is exactly the three conditions that make +// connect's detectBlackhole fire. +func TestProviderClientVerdictReasonAllowlistIsClosed(t *testing.T) { + connect.AssertEqual(t, IsProviderClientVerdictReason(ProviderClientVerdictReasonNoSendAck), true) + connect.AssertEqual(t, IsProviderClientVerdictReason(ProviderClientVerdictReasonNoReceiveAck), true) + connect.AssertEqual(t, IsProviderClientVerdictReason(ProviderClientVerdictReasonNoReceiveSyn), true) + connect.AssertEqual(t, IsProviderClientVerdictReason("slow"), false) + connect.AssertEqual(t, IsProviderClientVerdictReason(""), false) +} + +// The one and only effect of a met quorum: the provider's stored observed_at +// moves back far enough that the prober is offered it, and NOTHING else about +// the row changes. +// +// The two-sided bound is the point. Too little and the provider is never +// offered; past ProviderEgressLocationMaxAge and the location stops being +// trusted by the connection path and the sweep deletes the row -- which would +// turn a client-verdict quorum into a selection-path effect, exactly what this +// design forbids. +func TestReprioritiseProviderEgressProbeMovesOnlyTheDueTime(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + clientId := server.NewId() + + city := &Location{ + LocationType: LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, city) + + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: clientId, + LocationId: city.LocationId, + CountryCode: "us", + ASN: 64500, + Org: "Example Hosting", + Hosting: true, + CityConfident: true, + ObservedAt: now.Add(-time.Hour), + Verdict: "verified", + VerdictReason: "", + }) + before := GetProviderEgressLocation(ctx, clientId) + if before == nil { + t.Fatal("expected a stored egress location") + } + + if !ReprioritiseProviderEgressProbe(ctx, clientId, now) { + t.Fatal("reprioritise did not move a fresh row") + } + + after := GetProviderEgressLocation(ctx, clientId) + if after == nil { + t.Fatal("reprioritise removed the row") + } + + age := now.Sub(after.ObservedAt.UTC()) + // due: older than the api handler's cutoff, which is half the max age + if age <= ProviderEgressLocationMaxAge/2 { + t.Fatalf("observed_at is %s old, not old enough to be due (> %s)", + age, ProviderEgressLocationMaxAge/2) + } + // still fresh: inside the max age, so the connection path still + // resolves this location and the expiry sweep does not delete it + if ProviderEgressLocationMaxAge <= age { + t.Fatalf("observed_at is %s old, past the max age %s: quorum must not expire a location", + age, ProviderEgressLocationMaxAge) + } + fresh := GetFreshProviderEgressLocation(ctx, clientId, ProviderEgressLocationMaxAge) + if fresh == nil { + t.Fatal("the location stopped being fresh: a met quorum must not change what selection sees") + } + + // everything selection could read is byte-identical + connect.AssertEqual(t, after.LocationId, before.LocationId) + connect.AssertEqual(t, after.CountryCode, before.CountryCode) + connect.AssertEqual(t, after.ASN, before.ASN) + connect.AssertEqual(t, after.Org, before.Org) + connect.AssertEqual(t, after.Hosting, before.Hosting) + connect.AssertEqual(t, after.Proxy, before.Proxy) + connect.AssertEqual(t, after.Mobile, before.Mobile) + connect.AssertEqual(t, after.CityConfident, before.CityConfident) + connect.AssertEqual(t, after.Verdict, before.Verdict) + connect.AssertEqual(t, after.VerdictReason, before.VerdictReason) + connect.AssertEqual(t, after.Assurance, before.Assurance) + + // NON-RATCHETING: a second quorum must not walk the row further back. + // Repeated quorums that each subtracted a fixed age would eventually + // push the location out of the freshness window, which is a slow + // version of the exclusion this design forbids. + if ReprioritiseProviderEgressProbe(ctx, clientId, now) { + t.Fatal("a second reprioritise moved an already-due row") + } + again := GetProviderEgressLocation(ctx, clientId) + if !again.ObservedAt.UTC().Equal(after.ObservedAt.UTC()) { + t.Fatalf("observed_at ratcheted from %s to %s", after.ObservedAt.UTC(), again.ObservedAt.UTC()) + } + + // a provider with no row at all is not a miss: it has never been probed + // successfully, so the due queue already sorts it ahead of every probed + // provider and there is nothing to bring forward + connect.AssertEqual(t, ReprioritiseProviderEgressProbe(ctx, server.NewId(), now), false) + }) +} From f882531787449333d643bb6120e9cdb62e863823 Mon Sep 17 00:00:00 2001 From: Ryan Mello Date: Sun, 2 Aug 2026 17:31:07 +0100 Subject: [PATCH 30/30] feat(model,work): observe geolocation source certificate pins directly On 2026-08-02 the entire beta fleet failed every provider with no_consensus. Two geolocation sources dropped out at once -- ip.pn moved its json endpoint to a different host, and ipinfo.io rotated BOTH its leaf and its intermediate, so the prober's hardcoded pin failed closed -- leaving one working source against MinSources = 2. Nothing raised. A pinned source that fails closed just shrinks the source set; the fleet could not tell that apart from "fewer sources answered". This is the server half of the fix: a six-hourly taskworker job that observes what the certificates actually are, so a CA change self-heals instead of becoming an outage. The prober switches over to consuming it in the next change; nothing reads the table yet. A pin may only ever be recorded from a DIRECT server-side connection that passed full WebPKI validation. The server reaches these hosts on its own network with no provider in the path, which is what makes its observation a valid trust anchor -- a provider cannot influence what the server sees. If a pin could be learned from a connection that traversed a provider tunnel, or from one whose chain was not verified, the provider under test could teach the server its own forged certificate, turning the mechanism that exists to catch a lying provider into the thing that trusts it. So: InsecureSkipVerify is false, ServerName is set, RootCAs is nil (system roots), and observeGeolocationSourcePin takes a host and an ADDRESS -- there is no parameter through which any caller could hand it a transport, a connection, a tls.Config or a root pool. Direct-only is structural, not a convention. The pins are read from ConnectionState().VerifiedChains, never PeerCertificates: the latter is whatever the peer sent, and a peer may pad its Certificate message with a real, publicly-downloadable intermediate that the validated path never used. That is the same reasoning behind the prober's checkPin, and recording from anything else would also record a pin the prober could never match. A host that fails validation leaves its stored row EXACTLY as it was, logs loudly, and the loop continues to the next host. One bad host must never blank the set -- that shortfall is precisely what took consensus offline. And the task returns nil even when hosts failed: an errored task is rescheduled with exponential backoff and its Post, which re-arms the six-hourly chain, does not run, so returning an error on one flaky host would eventually strand the refresh for all of them. Failures are loud in the log, not in the return value. Every change is logged with BOTH old and new values. SetGeolocationSourcePin returns the row it replaced, in the same transaction as the write, so the log line cannot drift from what was actually stored -- and so the property is testable rather than only asserted. The first run is scheduled immediately rather than one interval out, unlike the other Schedule* jobs: the prober's correct response to a missing pin is to refuse to probe, so an empty table means no probing at all. GeolocationSourceHosts duplicates `geolocate/sources.go` in the operator-proxy repo, and cannot import it: that is a separate Go module in a separate repository which depends on this server, so the import would invert the dependency. The comment names its counterpart and says which direction of drift is dangerous. Two backstops make it more than a promise: a test that compares the two lists whenever both repos are checked out side by side (it catches the exact ip.pn -> api.i.pn drift that caused this), and at runtime the prober treats a source host with no served pin as a hard error. Migration appends at slice index 546; the live beta database is at 545 (SELECT max(end_version_number) FROM migration_audit WHERE status='success'). --- db_migrations.go | 38 ++ model/geolocation_source_pin_model.go | 228 ++++++++++++ model/geolocation_source_pin_model_test.go | 162 ++++++++ taskworker/taskworker.go | 5 + .../work/geolocation_source_pin_work.go | 348 ++++++++++++++++++ .../work/geolocation_source_pin_work_test.go | 261 +++++++++++++ 6 files changed, 1042 insertions(+) create mode 100644 model/geolocation_source_pin_model.go create mode 100644 model/geolocation_source_pin_model_test.go create mode 100644 taskworker/work/geolocation_source_pin_work.go create mode 100644 taskworker/work/geolocation_source_pin_work_test.go diff --git a/db_migrations.go b/db_migrations.go index 791ee2bb..ae9b81f5 100644 --- a/db_migrations.go +++ b/db_migrations.go @@ -4702,4 +4702,42 @@ var migrations = []any{ CREATE INDEX IF NOT EXISTS provider_client_verdict_provider_create_time ON provider_client_verdict (provider_client_id, create_time) `), + + // the certificate pins this server has OBSERVED for the geolocation source + // hosts, by connecting to each host directly -- on the server's own + // network, no provider in the path -- and validating the chain under full + // WebPKI. See model.GeolocationSourcePin for why a direct, verified + // observation is the only thing that may ever write this table: a pin + // learned through a provider tunnel would let the provider under test teach + // the server its own forged certificate. + // + // One row per host, upserted: this is the current observation, not a + // history. A rotation is recorded in the refresh job's log line (old and + // new values), which is what was missing when the hardcoded pins went stale + // and silently took every source out of the consensus set. + // + // Both spki columns are NOT NULL and never written empty. An empty pin is + // not "no constraint", it is a pin that matches nothing, so a half-observed + // row would fail the prober closed for that host just as surely as a wrong + // one. The observation job leaves the previous row untouched rather than + // writing a partial one. + // + // No secondary index: the primary key on host serves both access paths -- + // the per-host upsert, and the unqualified read of every row (three rows, + // one per source host). + // + // Appended, never inserted: migrations here apply by slice index + // (`for i := DbVersion(ctx); i < upTo; i++`), so editing or reordering an + // already-applied entry corrupts live databases. IF NOT EXISTS makes a + // re-run -- or a duplicated merge resolution -- a no-op. + newSqlMigration(` + CREATE TABLE IF NOT EXISTS geolocation_source_pin ( + host text NOT NULL, + leaf_spki text NOT NULL, + intermediate_spki text NOT NULL, + observed_at timestamp NOT NULL, + + PRIMARY KEY (host) + ) + `), } diff --git a/model/geolocation_source_pin_model.go b/model/geolocation_source_pin_model.go new file mode 100644 index 00000000..10307d49 --- /dev/null +++ b/model/geolocation_source_pin_model.go @@ -0,0 +1,228 @@ +package model + +import ( + "context" + "time" + + "github.com/urnetwork/server" +) + +// GeolocationSourceHosts is the set of hosts the geolocation prober reaches +// through a provider tunnel, and therefore the set this server observes +// certificate pins for. +// +// # This list has a counterpart in another repository +// +// Its counterpart is `geolocate/sources.go` in the operator-proxy repo +// (github.com/urnetwork/urnetwork-operator-proxy), whose `sources` table is the +// authority on which endpoints are actually queried; `geolocate.SourceHosts()` +// derives the host set from it. The two CANNOT be linked in code: the prober is +// a separate Go module in a separate repository, and it depends on this server +// rather than the other way round -- importing the prober from the server would +// invert that dependency. So this is a deliberate second copy, and the comment +// is the only thing keeping it honest. When a source endpoint changes there, +// change it here in the same pass. +// +// # Drift is caught at runtime, fail-closed, not silently +// +// That is not merely a promise. The prober treats a source host with no served +// pin as a hard error and refuses to probe rather than probing unpinned, so a +// host added to `sources.go` but not to this list stops the prober at startup +// instead of quietly leaving one source unprotected. A host removed there but +// left here only costs a pointless observation. The dangerous direction is the +// one that fails loudly. +// +// This list is a compile-time constant on purpose: it is a trust decision about +// which hosts the server will vouch for, and nothing outside a code change -- +// no request, no database row, and above all no provider -- may add to it. +var GeolocationSourceHosts = []string{ + // ip.pn -- moved from ip.pn to api.i.pn on 2026-08-02, which is exactly + // the drift this comment block exists for. + "api.i.pn", + "free.freeipapi.com", + "ipinfo.io", +} + +// GeolocationSourcePin is the certificate pin this server observed for one +// geolocation source host, by connecting to it DIRECTLY -- on the server's own +// network, with no provider anywhere in the path -- and validating the chain +// under full WebPKI. +// +// That direct, validated observation is the entire basis for trusting the +// value. The geolocation lookup itself is issued THROUGH a provider's tunnel +// precisely so a provider forging its own location can be caught; if a pin +// could ever be learned from a connection that traversed a tunnel, or from one +// whose chain was not verified, the provider under test could teach this server +// its own forged certificate and the pin would authenticate the attacker. See +// the observation job for the code that upholds this. +// +// Both a leaf and an intermediate pin are recorded because the prober's check +// (providertunnel.checkPin) accepts a match anywhere in the verified chain: the +// intermediate is what absorbs routine leaf renewal without a redeploy, and the +// leaf is the tighter of the two while it lasts. +type GeolocationSourcePin struct { + Host string + LeafSpki string + IntermediateSpki string + ObservedAt time.Time +} + +// SetGeolocationSourcePin upserts the observed pin for one host and returns the +// row it REPLACED, or nil when the host had never been observed. +// +// Returning the previous row is what makes rotation visible. The outage this +// mechanism exists to prevent was invisible: a pin that had gone stale failed +// closed, the source dropped out of the set, and the fleet reported +// "no_consensus" -- indistinguishable from "fewer sources answered". The caller +// logs old and new from this return value, so a rotation leaves a record even +// though the table itself keeps no history. +// +// The read and the write are in one transaction so the returned "old" value is +// the one this write actually replaced, not whatever a separate earlier read +// happened to see. +// +// One row per host, replaced in place: this is the current picture, and history +// belongs in the log line rather than in a second key column that every reader +// would then have to filter on. +func SetGeolocationSourcePin(ctx context.Context, pin *GeolocationSourcePin) *GeolocationSourcePin { + var previous *GeolocationSourcePin + + server.Tx(ctx, func(tx server.PgTx) { + // server.Tx may rerun this closure, so start from a clean slate rather + // than carrying an abandoned attempt's reading forward. This does not + // cover the case where a commit succeeded but was REPORTED as failed: + // the rerun's select then finds the row it just wrote and the rotation + // goes unlogged once. That is rare, self-corrects at the next pass + // (the certificate genuinely has not changed by then), and fixing it + // properly means changing server.Tx's retry semantics for everyone. + previous = nil + + result, err := tx.Query( + ctx, + ` + SELECT + leaf_spki, + intermediate_spki, + observed_at + FROM geolocation_source_pin + WHERE host = $1 + FOR UPDATE + `, + pin.Host, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + p := &GeolocationSourcePin{Host: pin.Host} + server.Raise(result.Scan( + &p.LeafSpki, + &p.IntermediateSpki, + &p.ObservedAt, + )) + previous = p + } + }) + + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO geolocation_source_pin ( + host, + leaf_spki, + intermediate_spki, + observed_at + ) + VALUES ($1, $2, $3, $4) + ON CONFLICT (host) DO UPDATE + SET + leaf_spki = $2, + intermediate_spki = $3, + observed_at = $4 + `, + pin.Host, + pin.LeafSpki, + pin.IntermediateSpki, + // naive timestamp column holding utc, as everywhere else in this + // schema + pin.ObservedAt.UTC(), + )) + }) + + return previous +} + +// GetGeolocationSourcePin reads one host's observed pin, or nil when the host +// has never been successfully observed. +// +// Never observed is not the same as observed-and-empty: a caller that could not +// tell those apart would be one step from serving an empty pin set, which is +// the same as serving no pin at all. +func GetGeolocationSourcePin(ctx context.Context, host string) *GeolocationSourcePin { + var pin *GeolocationSourcePin + + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT + leaf_spki, + intermediate_spki, + observed_at + FROM geolocation_source_pin + WHERE host = $1 + `, + host, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + p := &GeolocationSourcePin{Host: host} + server.Raise(result.Scan( + &p.LeafSpki, + &p.IntermediateSpki, + &p.ObservedAt, + )) + pin = p + } + }) + }) + + return pin +} + +// GetGeolocationSourcePins reads every observed pin, keyed by host. +// +// It returns exactly what has been observed and never synthesizes a row for a +// host in GeolocationSourceHosts that has not been observed yet. A missing host +// must stay missing all the way to the consumer, because the consumer's correct +// response to a missing pin is to refuse to probe -- and a placeholder row here +// would take that decision away from it. +func GetGeolocationSourcePins(ctx context.Context) map[string]*GeolocationSourcePin { + pins := map[string]*GeolocationSourcePin{} + + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT + host, + leaf_spki, + intermediate_spki, + observed_at + FROM geolocation_source_pin + `, + ) + server.WithPgResult(result, err, func() { + for result.Next() { + p := &GeolocationSourcePin{} + server.Raise(result.Scan( + &p.Host, + &p.LeafSpki, + &p.IntermediateSpki, + &p.ObservedAt, + )) + pins[p.Host] = p + } + }) + }) + + return pins +} diff --git a/model/geolocation_source_pin_model_test.go b/model/geolocation_source_pin_model_test.go new file mode 100644 index 00000000..7a66ceee --- /dev/null +++ b/model/geolocation_source_pin_model_test.go @@ -0,0 +1,162 @@ +package model + +import ( + "context" + "os" + "path/filepath" + "regexp" + "sort" + "testing" + "time" + + "github.com/urnetwork/connect" + + "github.com/urnetwork/server" +) + +func TestSetGeolocationSourcePinStoresAndReplacesPerHost(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + observedAt := server.NowUtc().Truncate(time.Millisecond) + + // first observation of a host: nothing replaced + previous := SetGeolocationSourcePin(ctx, &GeolocationSourcePin{ + Host: "ipinfo.io", + LeafSpki: "leaf-ipinfo-1", + IntermediateSpki: "int-ipinfo-1", + ObservedAt: observedAt, + }) + if previous != nil { + t.Fatalf("expected no previous pin on first observation, got %+v", previous) + } + + // a second host is stored independently + SetGeolocationSourcePin(ctx, &GeolocationSourcePin{ + Host: "api.i.pn", + LeafSpki: "leaf-ipn-1", + IntermediateSpki: "int-ipn-1", + ObservedAt: observedAt, + }) + + stored := GetGeolocationSourcePin(ctx, "ipinfo.io") + if stored == nil { + t.Fatal("expected a stored pin for ipinfo.io, got nil") + } + connect.AssertEqual(t, stored.Host, "ipinfo.io") + connect.AssertEqual(t, stored.LeafSpki, "leaf-ipinfo-1") + connect.AssertEqual(t, stored.IntermediateSpki, "int-ipinfo-1") + connect.AssertEqual(t, stored.ObservedAt.UTC().Equal(observedAt.UTC()), true) + + // re-observing one host REPLACES that host's row and returns what it + // replaced -- the return value is what makes a rotation loggable with + // both old and new + rotatedAt := observedAt.Add(6 * time.Hour) + previous = SetGeolocationSourcePin(ctx, &GeolocationSourcePin{ + Host: "ipinfo.io", + LeafSpki: "leaf-ipinfo-2", + IntermediateSpki: "int-ipinfo-2", + ObservedAt: rotatedAt, + }) + if previous == nil { + t.Fatal("expected the replaced pin to be returned, got nil") + } + connect.AssertEqual(t, previous.LeafSpki, "leaf-ipinfo-1") + connect.AssertEqual(t, previous.IntermediateSpki, "int-ipinfo-1") + + pins := GetGeolocationSourcePins(ctx) + connect.AssertEqual(t, len(pins), 2) + connect.AssertEqual(t, pins["ipinfo.io"].LeafSpki, "leaf-ipinfo-2") + connect.AssertEqual(t, pins["ipinfo.io"].IntermediateSpki, "int-ipinfo-2") + // replacing one host must not touch another: a per-host write that + // disturbed its neighbours would be the same class of bug as a failing + // host blanking the whole set + connect.AssertEqual(t, pins["api.i.pn"].LeafSpki, "leaf-ipn-1") + connect.AssertEqual(t, pins["api.i.pn"].IntermediateSpki, "int-ipn-1") + + // a host that has never been observed reads as absent, not as an empty + // pin -- the consumer's correct response to absent is to refuse to + // probe, and a zero-valued row would take that decision away from it + if GetGeolocationSourcePin(ctx, "free.freeipapi.com") != nil { + t.Fatal("expected nil for a host that has never been observed") + } + if _, ok := pins["free.freeipapi.com"]; ok { + t.Fatal("expected an unobserved host to be absent from the pin map") + } + }) +} + +var sourceUrlPattern = regexp.MustCompile(`URL:\s*"https://([^/"]+)`) + +// TestGeolocationSourceHostsMatchProberSourcesWhenCheckedOutAlongside is the +// only mechanism available for linking GeolocationSourceHosts to its +// counterpart in the operator-proxy repo. The prober is a separate Go module in +// a separate repository that depends on this server, so the list genuinely +// cannot be imported -- see the GeolocationSourceHosts comment. +// +// What this covers: a machine that has both repositories checked out as +// siblings (or names one via URNETWORK_OPERATOR_PROXY) will fail this test the +// moment the two lists disagree, which is the case that matters -- whoever +// changes a source endpoint is working in both trees. +// +// What it does NOT cover: CI, or any checkout without the prober beside it. It +// skips there rather than passing vacuously. The real backstop for drift is at +// runtime and fails closed: the prober treats a source host with no served pin +// as a hard error and refuses to probe. +func TestGeolocationSourceHostsMatchProberSourcesWhenCheckedOutAlongside(t *testing.T) { + candidates := []string{ + os.Getenv("URNETWORK_OPERATOR_PROXY"), + filepath.Join("..", "..", "urnetwork-operator-proxy"), + } + + var sourcesPath string + for _, candidate := range candidates { + if candidate == "" { + continue + } + path := filepath.Join(candidate, "geolocate", "sources.go") + if _, err := os.Stat(path); err == nil { + sourcesPath = path + break + } + } + if sourcesPath == "" { + t.Skip("operator-proxy checkout not found beside this one; set URNETWORK_OPERATOR_PROXY to enable this check") + } + + b, err := os.ReadFile(sourcesPath) + if err != nil { + t.Fatalf("read %s: %v", sourcesPath, err) + } + + matches := sourceUrlPattern.FindAllStringSubmatch(string(b), -1) + if len(matches) == 0 { + // the file exists but no longer looks the way this test assumes. Fail + // rather than skip: a silently-inert drift check is worse than none, + // because it reads as coverage. + t.Fatalf("no source URLs found in %s -- the prober's source table has changed shape and this check needs updating", sourcesPath) + } + + seen := map[string]bool{} + proberHosts := []string{} + for _, match := range matches { + host := match[1] + if seen[host] { + continue + } + seen[host] = true + proberHosts = append(proberHosts, host) + } + + serverHosts := append([]string{}, GeolocationSourceHosts...) + sort.Strings(proberHosts) + sort.Strings(serverHosts) + + if len(proberHosts) != len(serverHosts) { + t.Fatalf("host lists disagree: prober %v (%s), server %v", proberHosts, sourcesPath, serverHosts) + } + for i := range proberHosts { + if proberHosts[i] != serverHosts[i] { + t.Fatalf("host lists disagree: prober %v (%s), server %v", proberHosts, sourcesPath, serverHosts) + } + } +} diff --git a/taskworker/taskworker.go b/taskworker/taskworker.go index 9cbc1c05..7f798349 100644 --- a/taskworker/taskworker.go +++ b/taskworker/taskworker.go @@ -53,6 +53,7 @@ func InitTasks(ctx context.Context) { work.ScheduleRemoveExpiredWalletAuthChallenges(clientSession, tx) work.ScheduleRemoveExpiredWalletNonces(clientSession, tx) work.ScheduleRemoveExpiredProviderEgressLocations(clientSession, tx) + work.ScheduleRefreshGeolocationSourcePins(clientSession, tx) work.ScheduleRemoveOldAuditNetworkEvents(clientSession, tx) work.ScheduleRemoveOldAuditEvents(clientSession, tx) work.ScheduleRemoveOldClientReliabilityStats(clientSession, tx) @@ -241,6 +242,10 @@ func InitTaskWorkerWithSettings(ctx context.Context, settings *task.TaskWorkerSe work.RemoveExpiredProviderEgressLocations, work.RemoveExpiredProviderEgressLocationsPost, ), + task.NewTaskTargetWithPost( + work.RefreshGeolocationSourcePins, + work.RefreshGeolocationSourcePinsPost, + ), task.NewTaskTargetWithPost( work.RemoveOldAuditNetworkEvents, work.RemoveOldAuditNetworkEventsPost, diff --git a/taskworker/work/geolocation_source_pin_work.go b/taskworker/work/geolocation_source_pin_work.go new file mode 100644 index 00000000..52d107d4 --- /dev/null +++ b/taskworker/work/geolocation_source_pin_work.go @@ -0,0 +1,348 @@ +package work + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "fmt" + "net" + "time" + + "github.com/urnetwork/glog" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" + "github.com/urnetwork/server/session" + "github.com/urnetwork/server/task" +) + +// GeolocationSourcePinRefreshTimeout is how often every source host is +// re-observed. Six hours is far shorter than any certificate's life and far +// longer than the handful of TLS handshakes it costs, so a rotation is picked +// up within a quarter of a day without the job ever being a load consideration. +const GeolocationSourcePinRefreshTimeout = 6 * time.Hour + +// geolocationSourceDialTimeout bounds one host's dial + handshake. A host that +// blackholes rather than refusing is a failure mode too, and an untimed dial +// would let one such host stall the whole pass -- which would leave the +// remaining hosts un-refreshed and look exactly like the silent staleness this +// job exists to prevent. +const geolocationSourceDialTimeout = 15 * time.Second + +// geolocationSourceTarget is one host to observe and the address to reach it +// at. Production always sets Addr to Host:443 (see productionGeolocationSourceTargets). +// +// Addr is a parameter ONLY so tests can point a host at a local listener. Note +// what is and is not injectable here: an address, and nothing else. The dialer, +// the *tls.Config, and the root pool are all built inside this package by +// sourceTLSConfig and observeGeolocationSourcePin, and there is deliberately no +// way for a caller to supply a transport, a connection, a tls.Config or a +// RootCAs override. That is the line this whole feature rests on -- a pin is +// only ever recorded from a connection this package itself made and verified +// against the system roots. A caller-supplied transport could be a provider +// tunnel, and a caller-supplied root pool could be a provider's own CA; either +// would let the provider under test choose the pin that is supposed to catch +// it. +type geolocationSourceTarget struct { + Host string + Addr string +} + +// productionGeolocationSourceTargets is the real target list: every known +// source host, dialed on 443 at its own name. +func productionGeolocationSourceTargets() []geolocationSourceTarget { + targets := make([]geolocationSourceTarget, 0, len(model.GeolocationSourceHosts)) + for _, host := range model.GeolocationSourceHosts { + targets = append(targets, geolocationSourceTarget{ + Host: host, + Addr: net.JoinHostPort(host, "443"), + }) + } + return targets +} + +// GeolocationSourcePinChange is one host's pin changing between two +// observations, carrying both the old and the new value. +// +// This is a return value rather than only a log line so the "rotation is +// visible" property is testable: a test can assert the change record carries +// what it replaced, which is the part that was missing during the outage. The +// job logs each record; nothing else consumes them. +// +// OldLeaf/OldIntermediate are empty for a host's first ever observation, which +// is a first sighting rather than a rotation and is logged as such. +type GeolocationSourcePinChange struct { + Host string + OldLeaf string + NewLeaf string + OldIntermediate string + NewIntermediate string + FirstObservation bool +} + +// sourceTLSConfig builds the tls.Config used for every observation. +// +// InsecureSkipVerify is false and ServerName is set, and both are +// load-bearing rather than boilerplate: +// +// - InsecureSkipVerify false is what makes the observation mean anything. It +// is also what populates ConnectionState().VerifiedChains at all: crypto/tls +// leaves the verified chains empty when verification is skipped, so the +// observation code below structurally cannot record a pin from an +// unverified handshake -- it has nothing to read. +// - ServerName pins the identity being verified to the host being recorded. +// Without it, hostname verification would have nothing to check against and +// any chain-valid certificate for any name would satisfy the handshake. +// +// No RootCAs is set, so the system root pool is used. There is no override and +// there must never be one, not even for tests: a settable root pool is exactly +// the injection point that would let something other than the public WebPKI +// decide what this server vouches for. +func sourceTLSConfig(host string) *tls.Config { + return &tls.Config{ + ServerName: host, + MinVersion: tls.VersionTLS12, + InsecureSkipVerify: false, + } +} + +// spkiPin is the base64 sha-256 of a certificate's subject public key info, +// byte-for-byte the same form the prober's providertunnel.SPKIPin computes and +// compares against. Hashing the key rather than the certificate is what lets a +// pin survive a renewal that keeps the same key. +func spkiPin(cert *x509.Certificate) string { + sum := sha256.Sum256(cert.RawSubjectPublicKeyInfo) + return base64.StdEncoding.EncodeToString(sum[:]) +} + +// observeGeolocationSourcePin dials addr directly, completes a fully verified +// TLS handshake as host, and returns the leaf and issuing-intermediate SPKI +// pins from the VERIFIED chain. +// +// It reads ConnectionState().VerifiedChains, never PeerCertificates, and that +// is a security property rather than a preference. PeerCertificates is whatever +// the peer sent; crypto/tls only ever uses the entries past the leaf as a pool +// to build a path from, and never promises the path it built used them. So a +// peer may pad its Certificate message with any publicly downloadable +// certificate it likes, and a PeerCertificates[1]-based reading would happily +// record that inert padding as this host's intermediate pin. VerifiedChains +// contains only certificates on a path crypto/tls actually validated to a +// system root. The prober's checkPin matches against verified chains for the +// same reason (see providertunnel/pinning.go), so recording from anything else +// would also record a pin the prober could never match. +// +// Any failure -- dial, handshake, hostname mismatch, expiry, an untrusted +// issuer, or a chain too short to have an issuer in it -- returns an error and +// no pin. The caller must leave the stored row alone in that case. +func observeGeolocationSourcePin(ctx context.Context, host string, addr string) (leafSpki string, intermediateSpki string, err error) { + dialer := &tls.Dialer{ + NetDialer: &net.Dialer{Timeout: geolocationSourceDialTimeout}, + Config: sourceTLSConfig(host), + } + + dialCtx, cancel := context.WithTimeout(ctx, geolocationSourceDialTimeout) + defer cancel() + + conn, err := dialer.DialContext(dialCtx, "tcp", addr) + if err != nil { + return "", "", err + } + defer conn.Close() + + tlsConn, ok := conn.(*tls.Conn) + if !ok { + // tls.Dialer always returns a *tls.Conn; this is here so a future + // change cannot silently turn into a pin read from a plain connection. + return "", "", fmt.Errorf("expected a tls connection to %s (%s), got %T", host, addr, conn) + } + + chains := tlsConn.ConnectionState().VerifiedChains + if len(chains) == 0 { + // unreachable with InsecureSkipVerify false -- crypto/tls aborts the + // handshake before this point when verification fails -- and it stays + // unreachable only as long as sourceTLSConfig keeps verification on. + // Failing here means a verification-disabled config could never write a + // pin even if one were somehow introduced. + return "", "", fmt.Errorf("no verified certificate chain for %s (%s)", host, addr) + } + chain := chains[0] + if len(chain) < 2 { + // a leaf with no issuer on the verified path: nothing to record as the + // intermediate pin. Writing an empty intermediate would be worse than + // writing nothing, since an empty pin matches no certificate at all. + return "", "", fmt.Errorf("verified chain for %s (%s) has %d certificates, need at least a leaf and its issuer", host, addr, len(chain)) + } + + // chain[0] is the leaf and chain[1] is its issuer on the validated path. + // For a two-certificate chain the issuer is the root itself, which is a + // broader pin than an intermediate would be -- but no broader than the + // trust the handshake already placed in that root, and the prober accepts a + // match anywhere in the chain regardless. + return spkiPin(chain[0]), spkiPin(chain[1]), nil +} + +// refreshGeolocationSourcePins observes every target and stores what it saw. +// +// Per-host isolation is the point. A host that fails validation leaves its +// stored row exactly as it was and contributes an error; the loop continues to +// the next host. Nothing is deleted, nothing is blanked, and no host's failure +// can cost another host its pin -- the failure mode that started this was one +// stale pin quietly shrinking the usable source set below the consensus +// minimum, and a job that abandoned the remaining hosts on the first error +// would reproduce it exactly. +// +// The store call is only reached after a successful, verified observation, so +// the "leave the previous row untouched" guarantee is structural rather than a +// branch that has to be remembered. +func refreshGeolocationSourcePins( + ctx context.Context, + targets []geolocationSourceTarget, +) (changes []GeolocationSourcePinChange, errs []error) { + for _, target := range targets { + leafSpki, intermediateSpki, err := observeGeolocationSourcePin(ctx, target.Host, target.Addr) + if err != nil { + errs = append(errs, fmt.Errorf("observe %s (%s): %w", target.Host, target.Addr, err)) + continue + } + + previous := model.SetGeolocationSourcePin(ctx, &model.GeolocationSourcePin{ + Host: target.Host, + LeafSpki: leafSpki, + IntermediateSpki: intermediateSpki, + ObservedAt: server.NowUtc(), + }) + + if previous == nil { + changes = append(changes, GeolocationSourcePinChange{ + Host: target.Host, + NewLeaf: leafSpki, + NewIntermediate: intermediateSpki, + FirstObservation: true, + }) + continue + } + if previous.LeafSpki != leafSpki || previous.IntermediateSpki != intermediateSpki { + changes = append(changes, GeolocationSourcePinChange{ + Host: target.Host, + OldLeaf: previous.LeafSpki, + NewLeaf: leafSpki, + OldIntermediate: previous.IntermediateSpki, + NewIntermediate: intermediateSpki, + }) + } + } + + return changes, errs +} + +type RefreshGeolocationSourcePinsArgs struct{} + +type RefreshGeolocationSourcePinsResult struct { + // counts only; the detail is in the log, and a task result is not a place + // anything should be reading pins back out of + Observed int `json:"observed"` + Changed int `json:"changed"` + Failed int `json:"failed"` +} + +// ScheduleRefreshGeolocationSourcePins schedules the first observation to run +// IMMEDIATELY, which is a deliberate departure from the usual Schedule* pattern +// of arming the first run one interval out. +// +// This job is not a cleanup job whose first pass can wait. Its table is what +// the prober pins against, and the prober's correct response to a missing pin +// is to refuse to probe -- so an empty table means no probing at all. Arming +// the first run six hours out would leave a freshly-migrated deployment unable +// to probe anything for those six hours, for no benefit. The recurring six-hour +// cadence is set in the Post below. +// +// RunOnce merges on conflict with `run_at = LEAST(existing, new)`, so a +// taskworker restart pulls a pending observation forward to now rather than +// stacking a second one. That costs one TLS handshake per source host per +// restart, which is nothing, and it means a restart after a rotation picks the +// new certificate up at once. +func ScheduleRefreshGeolocationSourcePins(clientSession *session.ClientSession, tx server.PgTx) { + scheduleRefreshGeolocationSourcePinsAt(clientSession, tx, server.NowUtc()) +} + +func scheduleRefreshGeolocationSourcePinsAt(clientSession *session.ClientSession, tx server.PgTx, runAt time.Time) { + task.ScheduleTaskInTx( + tx, + RefreshGeolocationSourcePins, + &RefreshGeolocationSourcePinsArgs{}, + clientSession, + task.RunOnce("refresh_geolocation_source_pins"), + task.RunAt(runAt), + ) +} + +// RefreshGeolocationSourcePins re-observes every geolocation source host and +// records what it saw, so the prober's pins track the real certificates instead +// of a constant someone pasted in weeks ago. +// +// It returns nil even when hosts failed, and that is deliberate. A task that +// returns an error is rescheduled with exponential backoff and its Post -- the +// function that re-arms the six-hourly chain -- does not run. So returning an +// error whenever any single host was unreachable would let one flaky host slow +// and eventually strand the refresh for ALL hosts: a job that quietly stops is +// the precise failure shape this feature exists to remove. Failures are loud in +// the log and counted in the result instead, and the chain always re-arms. +func RefreshGeolocationSourcePins( + _ *RefreshGeolocationSourcePinsArgs, + clientSession *session.ClientSession, +) (*RefreshGeolocationSourcePinsResult, error) { + targets := productionGeolocationSourceTargets() + changes, errs := refreshGeolocationSourcePins(clientSession.Ctx, targets) + + for _, change := range changes { + if change.FirstObservation { + glog.Infof( + "[gsp]first observation for %s: leaf=%s intermediate=%s\n", + change.Host, + change.NewLeaf, + change.NewIntermediate, + ) + continue + } + // old AND new, always. A rotation that is only visible as "the pin is + // different now" is the same as no record at all when someone is trying + // to work out why a source dropped out. + glog.Errorf( + "[gsp]pin ROTATED for %s: leaf %s -> %s, intermediate %s -> %s\n", + change.Host, + change.OldLeaf, + change.NewLeaf, + change.OldIntermediate, + change.NewIntermediate, + ) + } + for _, err := range errs { + // loud: a host that cannot be validated keeps serving its previous pin + // to the prober, which is correct but is also exactly how a pin goes + // stale without anyone noticing + glog.Errorf("[gsp]observation FAILED, previous pin left in place: %v\n", err) + } + + return &RefreshGeolocationSourcePinsResult{ + Observed: len(targets) - len(errs), + Changed: len(changes), + Failed: len(errs), + }, nil +} + +func RefreshGeolocationSourcePinsPost( + _ *RefreshGeolocationSourcePinsArgs, + _ *RefreshGeolocationSourcePinsResult, + clientSession *session.ClientSession, + tx server.PgTx, +) error { + // the recurring cadence; only the very first run is immediate + scheduleRefreshGeolocationSourcePinsAt( + clientSession, + tx, + server.NowUtc().Add(GeolocationSourcePinRefreshTimeout), + ) + return nil +} diff --git a/taskworker/work/geolocation_source_pin_work_test.go b/taskworker/work/geolocation_source_pin_work_test.go new file mode 100644 index 00000000..1f610283 --- /dev/null +++ b/taskworker/work/geolocation_source_pin_work_test.go @@ -0,0 +1,261 @@ +package work + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/urnetwork/connect" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" +) + +// TestSourceTLSConfigMandatesWebPKI is the cheap, direct assertion of the line +// this whole feature rests on. A pin may only ever be recorded from a +// connection that passed full WebPKI validation, and these three fields are +// what enforce that: verification on, an identity to verify against, and the +// system root pool rather than one someone handed us. +// +// The signature of observeGeolocationSourcePin carries the other half: it +// accepts a host and an address, and there is no parameter through which a +// caller could supply a transport, a connection, a tls.Config or a root pool. +// A provider tunnel therefore cannot be the path a pin is learned over, because +// there is no way to pass one in. +func TestSourceTLSConfigMandatesWebPKI(t *testing.T) { + cfg := sourceTLSConfig("ipinfo.io") + + if cfg.InsecureSkipVerify { + t.Fatal("InsecureSkipVerify must be false: an unverified chain would let the host being probed choose its own pin") + } + connect.AssertEqual(t, cfg.ServerName, "ipinfo.io") + if cfg.RootCAs != nil { + t.Fatal("RootCAs must be nil so the system roots are used; an overridable root pool is the injection point this design exists to close") + } + if cfg.VerifyPeerCertificate != nil { + t.Fatal("no custom peer verification: the standard WebPKI check is the whole point") + } +} + +// selfSignedTLSServer starts an httptest TLS server whose certificate is signed +// by its own throwaway CA, which is in no system root store. This is the +// hostile case: a server presenting a certificate that does not validate. +func selfSignedTLSServer(t testing.TB) (addr string, stop func()) { + t.Helper() + ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + u, err := url.Parse(ts.URL) + if err != nil { + ts.Close() + t.Fatalf("parse httptest url: %v", err) + } + return u.Host, ts.Close +} + +// requireRealSourceReachable skips when this machine has no outbound path to a +// real source host. The tests that use it are the only ones that can +// demonstrate a SUCCESSFUL observation: a success requires a chain that +// validates against the system roots, and manufacturing one hermetically would +// mean injecting a root pool -- which is exactly what this design forbids. So +// the hermetic tests cover the hostile cases and these cover the good path. +// +// The gate is a plain TCP dial, deliberately NOT a call to +// observeGeolocationSourcePin. Gating on the function under test would make +// these tests SKIP rather than FAIL whenever observation itself broke, which is +// the vacuous-coverage trap: no network skips, broken observation fails. +func requireRealSourceReachable(t testing.TB, host string) (addr string) { + t.Helper() + addr = net.JoinHostPort(host, "443") + conn, err := net.DialTimeout("tcp", addr, 10*time.Second) + if err != nil { + t.Skipf("no outbound path to %s (%v); skipping the real-observation case", addr, err) + } + conn.Close() + return addr +} + +// TestRefreshGeolocationSourcePinsLeavesPriorRowUnchangedOnSelfSignedServer is +// the most important test here. A host that fails validation must leave what +// was already stored EXACTLY as it was and raise -- never overwrite a good pin +// with an unverified one, and never blank it either. Overwriting would let a +// hostile server choose the pin; blanking would shrink the usable source set, +// which is the failure that took the whole fleet's consensus offline. +func TestRefreshGeolocationSourcePinsLeavesPriorRowUnchangedOnSelfSignedServer(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + addr, stop := selfSignedTLSServer(t) + defer stop() + + observedAt := server.NowUtc().Add(-time.Hour).Truncate(time.Millisecond) + good := &model.GeolocationSourcePin{ + Host: "ipinfo.io", + LeafSpki: "good-leaf", + IntermediateSpki: "good-intermediate", + ObservedAt: observedAt, + } + model.SetGeolocationSourcePin(ctx, good) + + changes, errs := refreshGeolocationSourcePins(ctx, []geolocationSourceTarget{ + {Host: "ipinfo.io", Addr: addr}, + }) + + if len(errs) != 1 { + t.Fatalf("expected the self-signed server to be rejected, got errs=%v", errs) + } + connect.AssertEqual(t, len(changes), 0) + + after := model.GetGeolocationSourcePin(ctx, "ipinfo.io") + if after == nil { + t.Fatal("the previously stored pin was removed by a failed observation") + } + connect.AssertEqual(t, after.LeafSpki, good.LeafSpki) + connect.AssertEqual(t, after.IntermediateSpki, good.IntermediateSpki) + connect.AssertEqual(t, after.ObservedAt.UTC().Equal(observedAt.UTC()), true) + }) +} + +// TestRefreshGeolocationSourcePinsLeavesPriorRowUnchangedOnHostnameMismatch is +// the same guarantee for the other half of WebPKI. Here the chain is genuinely +// valid and genuinely trusted -- it is a real source host's real certificate -- +// but it is not for the name being asked for. Without ServerName set and +// verification on, this would be accepted and one host's certificate would +// become another host's pin. +func TestRefreshGeolocationSourcePinsLeavesPriorRowUnchangedOnHostnameMismatch(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + addr := requireRealSourceReachable(t, "ipinfo.io") + + observedAt := server.NowUtc().Add(-time.Hour).Truncate(time.Millisecond) + model.SetGeolocationSourcePin(ctx, &model.GeolocationSourcePin{ + Host: "free.freeipapi.com", + LeafSpki: "good-leaf", + IntermediateSpki: "good-intermediate", + ObservedAt: observedAt, + }) + + // ipinfo.io's real address, asked for under a name its certificate does + // not cover + changes, errs := refreshGeolocationSourcePins(ctx, []geolocationSourceTarget{ + {Host: "free.freeipapi.com", Addr: addr}, + }) + + if len(errs) != 1 { + t.Fatalf("expected a hostname mismatch to be rejected, got errs=%v", errs) + } + connect.AssertEqual(t, len(changes), 0) + + after := model.GetGeolocationSourcePin(ctx, "free.freeipapi.com") + if after == nil { + t.Fatal("the previously stored pin was removed by a failed observation") + } + connect.AssertEqual(t, after.LeafSpki, "good-leaf") + connect.AssertEqual(t, after.IntermediateSpki, "good-intermediate") + connect.AssertEqual(t, after.ObservedAt.UTC().Equal(observedAt.UTC()), true) + }) +} + +// TestRefreshGeolocationSourcePinsReportsOldAndNewOnRotation covers the +// "rotation must be visible" requirement. The change record carries both the +// value that was there and the value that replaced it, which is what the job +// logs. A record that carried only the new value would be no better than the +// silence that made the original outage undiagnosable. +func TestRefreshGeolocationSourcePinsReportsOldAndNewOnRotation(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + host := "ipinfo.io" + addr := requireRealSourceReachable(t, host) + + // stand in for a pin observed before the certificate rotated + model.SetGeolocationSourcePin(ctx, &model.GeolocationSourcePin{ + Host: host, + LeafSpki: "stale-leaf", + IntermediateSpki: "stale-intermediate", + ObservedAt: server.NowUtc().Add(-24 * time.Hour), + }) + + changes, errs := refreshGeolocationSourcePins(ctx, []geolocationSourceTarget{ + {Host: host, Addr: addr}, + }) + connect.AssertEqual(t, len(errs), 0) + if len(changes) != 1 { + t.Fatalf("expected exactly one change record, got %+v", changes) + } + + change := changes[0] + connect.AssertEqual(t, change.Host, host) + connect.AssertEqual(t, change.FirstObservation, false) + connect.AssertEqual(t, change.OldLeaf, "stale-leaf") + connect.AssertEqual(t, change.OldIntermediate, "stale-intermediate") + if change.NewLeaf == "" || change.NewIntermediate == "" { + t.Fatalf("expected observed leaf and intermediate pins, got %+v", change) + } + if change.NewLeaf == change.OldLeaf || change.NewIntermediate == change.OldIntermediate { + t.Fatalf("expected the observed pins to differ from the stale ones, got %+v", change) + } + // what was logged is what was stored + stored := model.GetGeolocationSourcePin(ctx, host) + connect.AssertEqual(t, stored.LeafSpki, change.NewLeaf) + connect.AssertEqual(t, stored.IntermediateSpki, change.NewIntermediate) + + // re-observing an unchanged certificate is not a rotation and must not + // be reported as one, or the log stops meaning anything + changes, errs = refreshGeolocationSourcePins(ctx, []geolocationSourceTarget{ + {Host: host, Addr: addr}, + }) + connect.AssertEqual(t, len(errs), 0) + connect.AssertEqual(t, len(changes), 0) + }) +} + +// TestRefreshGeolocationSourcePinsContinuesPastAFailingHost: one bad host must +// never cost the others their observation. The original outage was two sources +// dropping out at once and leaving one against a minimum of two; a refresh that +// abandoned the remaining hosts after the first failure would manufacture that +// same shortfall from a single unreachable endpoint. +func TestRefreshGeolocationSourcePinsContinuesPastAFailingHost(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + goodHost := "ipinfo.io" + goodAddr := requireRealSourceReachable(t, goodHost) + badAddr, stop := selfSignedTLSServer(t) + defer stop() + + // the failing host is FIRST, so a loop that stopped on the first error + // would never reach the good one + changes, errs := refreshGeolocationSourcePins(ctx, []geolocationSourceTarget{ + {Host: "free.freeipapi.com", Addr: badAddr}, + {Host: goodHost, Addr: goodAddr}, + }) + + connect.AssertEqual(t, len(errs), 1) + if len(changes) != 1 { + t.Fatalf("expected the good host to still be observed, got %+v", changes) + } + connect.AssertEqual(t, changes[0].Host, goodHost) + connect.AssertEqual(t, changes[0].FirstObservation, true) + + if model.GetGeolocationSourcePin(ctx, goodHost) == nil { + t.Fatal("the good host was not stored") + } + if model.GetGeolocationSourcePin(ctx, "free.freeipapi.com") != nil { + t.Fatal("a failed observation must not write a row") + } + }) +} + +// TestProductionGeolocationSourceTargetsCoverEverySourceHost: the real target +// list is derived from model.GeolocationSourceHosts and dials each host on 443 +// under its own name. Nothing else may become a target. +func TestProductionGeolocationSourceTargetsCoverEverySourceHost(t *testing.T) { + targets := productionGeolocationSourceTargets() + connect.AssertEqual(t, len(targets), len(model.GeolocationSourceHosts)) + for i, host := range model.GeolocationSourceHosts { + connect.AssertEqual(t, targets[i].Host, host) + connect.AssertEqual(t, targets[i].Addr, net.JoinHostPort(host, "443")) + } +}