diff --git a/api/api.go b/api/api.go index fef528b8..d08ae27f 100644 --- a/api/api.go +++ b/api/api.go @@ -51,6 +51,28 @@ func Routes() []*router.Route { router.NewRoute("POST", "/auth/upgrade-guest-existing", handlers.UpgradeGuestExisting), router.NewRoute("POST", "/network/auth-client", handlers.AuthNetworkClient), router.NewRoute("POST", "/network/remove-client", handlers.RemoveNetworkClient), + router.NewRoute("POST", "/network/provider-egress-location", handlers.ProviderEgressLocationSubmit), + router.NewRoute("GET", "/network/provider-egress-due", handlers.ProviderEgressLocationDue), + router.NewRoute("POST", "/network/provider-egress-attempt", handlers.ProviderEgressLocationAttempt), + // operator-to-server, gated by the same operator secret as the egress + // location ingest above: the active bandwidth probe's download target, + // its result submission, and the byte-budget reservation the prober + // takes before spending any probe bytes + router.NewRoute("GET", "/network/provider-bandwidth-test", handlers.ProviderBandwidthTest), + router.NewRoute("POST", "/network/provider-bandwidth-result", handlers.ProviderBandwidthResult), + router.NewRoute("POST", "/network/provider-bandwidth-reserve", handlers.ProviderBandwidthReserve), + // operator-to-server, same operator secret again: the egress-health + // run the prober takes over the tunnel the geolocation probe already + // opened. Until this existed the result was a log line and nothing + // else. + router.NewRoute("POST", "/network/provider-egress-health", handlers.ProviderEgressHealthResult), + // client-to-server, and the only route in this group that is NOT + // operator-secret authed: a real client network reporting that a + // provider carried nothing. The reporting network is taken from the + // session jwt, never from the body, because the quorum counts distinct + // networks. A met quorum only brings the provider's next probe + // forward -- see model.ProviderClientVerdictQuorumMet. + router.NewRoute("POST", "/network/provider-verdict", handlers.ProviderClientVerdictSubmit), router.NewRoute("GET", "/network/clients", handlers.NetworkClients), router.NewRoute("GET", "/network/peers", handlers.NetworkPeers), router.NewRoute("GET", "/network/provider-locations", handlers.NetworkGetProviderLocations), diff --git a/api/handlers/provider_bandwidth_handlers.go b/api/handlers/provider_bandwidth_handlers.go new file mode 100644 index 00000000..71025571 --- /dev/null +++ b/api/handlers/provider_bandwidth_handlers.go @@ -0,0 +1,303 @@ +package handlers + +import ( + "crypto/hmac" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "time" + + "github.com/urnetwork/glog" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" +) + +// maxProviderBandwidthTestBytes bounds a single REQUEST. Without a clamp the +// endpoint is an open-ended resource commitment per request: a caller could +// ask for gigabytes and hold a connection (and the egress bytes that go with +// it) for as long as it liked. +// +// This is no longer the same thing as the per-probe figure. One probe is +// 8 parallel streams -- it has to be parallel, because a single TCP flow +// cannot exceed (connect's 1 MiB window / RTT) and the single-stream probe +// measured that ceiling rather than the provider -- so one probe is 8 requests +// of 2 MiB each, and the aggregate is bounded by the RESERVATION +// (model.MaxProviderBandwidthBytesPerProbe, 16 MiB), not by this clamp. +// +// The invariant to keep: the prober's per-stream byte count +// (bandwidth.StreamBytes, 2 MiB) must stay at or below this value. If it ever +// exceeds it, the endpoint silently truncates each stream and the probe +// transfers less than it reserved, understating every provider it measures. +const maxProviderBandwidthTestBytes = 5 * 1024 * 1024 + +// defaultProviderBandwidthTestBytes is used when `bytes` is absent, malformed, +// or non-positive. `bytes=0` and `bytes=-1` parse cleanly, so they are not +// caught by a malformed-input check -- and streaming an empty body for them +// would hand the prober a zero-byte sample to divide by. +const defaultProviderBandwidthTestBytes = 1024 * 1024 + +// maxProviderBandwidthBody bounds the request body of the two POST endpoints. +// Both carry a fixed handful of scalars. +const maxProviderBandwidthBody = 4 * 1024 + +// providerBandwidthTestBlock is the unit the download endpoint repeats. The +// content is irrelevant -- only the byte count is measured -- so this is one +// small shared block streamed over and over rather than a per-request +// allocation of the full byte count. +var providerBandwidthTestBlock = make([]byte, 32*1024) + +// repeatingReader yields providerBandwidthTestBlock endlessly. Bounded by an +// io.LimitReader at the call site, so it never needs an end of its own. +type repeatingReader struct { + block []byte + offset int +} + +func (self *repeatingReader) Read(p []byte) (int, error) { + n := copy(p, self.block[self.offset:]) + self.offset = (self.offset + n) % len(self.block) + return n, nil +} + +// authorizeOperator applies the same operator-secret check the provider egress +// location ingest endpoint uses: the shared secret from the vault +// (operatorIngestSecret, memoized and fail-closed) compared in constant time +// against the X-UR-Operator-Secret header. These are operator-to-server +// routes, not client routes -- there is no network jwt involved. +// +// An unconfigured vault leaves the secret empty, which rejects every request +// rather than accepting every request. +func authorizeOperator(r *http.Request) bool { + secret := operatorIngestSecret() + provided := r.Header.Get(operatorSecretHeader) + return secret != "" && provided != "" && hmac.Equal([]byte(secret), []byte(provided)) +} + +// readOperatorRequestBody reads a bounded operator request body, writing the +// error response itself and reporting whether the caller should continue. +func readOperatorRequestBody(w http.ResponseWriter, r *http.Request, out any) bool { + body, err := io.ReadAll(io.LimitReader(r.Body, maxProviderBandwidthBody+1)) + if err != nil { + http.Error(w, "Bad request", http.StatusBadRequest) + return false + } + if maxProviderBandwidthBody < len(body) { + http.Error(w, "Request too large", http.StatusRequestEntityTooLarge) + return false + } + if err := json.Unmarshal(body, out); err != nil { + http.Error(w, "Bad request", http.StatusBadRequest) + return false + } + return true +} + +// ProviderBandwidthTest streams a bounded number of bytes. The active +// bandwidth probe needs something to download *through* a provider's tunnel to +// measure that tunnel's throughput; this is that target. It is +// operator-to-server, gated by the operator secret, so ordinary clients cannot +// use the deployment as a free speed-test target. +// +// The content is arbitrary -- only the byte count matters -- and it is streamed +// from a small repeating block through an io.LimitReader, never materialized +// in full. +func ProviderBandwidthTest(w http.ResponseWriter, r *http.Request) { + if !authorizeOperator(r) { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + byteCount := int64(defaultProviderBandwidthTestBytes) + if requested, err := strconv.ParseInt(r.URL.Query().Get("bytes"), 10, 64); err == nil && 0 < requested { + byteCount = requested + } + if maxProviderBandwidthTestBytes < byteCount { + byteCount = maxProviderBandwidthTestBytes + } + + w.Header().Set("Content-Type", "application/octet-stream") + w.Header().Set("Content-Length", strconv.FormatInt(byteCount, 10)) + w.Header().Set("Cache-Control", "no-store") + w.WriteHeader(http.StatusOK) + + source := &repeatingReader{block: providerBandwidthTestBlock} + if _, err := io.Copy(w, io.LimitReader(source, byteCount)); err != nil { + // the prober hanging up mid-download is ordinary (it stops at its own + // time or byte cap), so this is not an error worth escalating + glog.Infof("[pbw]bandwidth test stream ended early. err = %s\n", err) + } +} + +// SubmitProviderBandwidthArgs is an active bandwidth measurement taken by the +// prober over a provider's tunnel. +type SubmitProviderBandwidthArgs struct { + ClientId server.Id `json:"client_id"` + // Source names which target produced this figure + // (model.ProviderBandwidthSourceActiveOperator or ...ActiveCDN). It is + // part of the storage key, so it is what keeps the two targets' figures in + // separate rows instead of overwriting each other. + Source string `json:"source"` + BytesPerSecond float64 `json:"bytes_per_second"` + SampleByteCount int64 `json:"sample_byte_count"` +} + +// ProviderBandwidthResult stores an active bandwidth measurement. An active +// probe is a point measurement rather than an aggregate over a window, so +// window_start and window_end are both the arrival time. +// +// A non-positive rate or sample size is not a usable measurement, and storing +// one would overwrite a real figure with a meaningless one -- so those are +// rejected before anything is written. +// +// The source is validated against the known ACTIVE set rather than merely +// stored. Two distinct failures are being closed off. An unrecognised source +// is not a harmless label: the row is keyed on (client_id, source), so it +// creates a row nothing will ever read or replace, and a prober with a typo'd +// tag would look like it was working while writing to a tag no consumer knows. +// And "passive" is refused specifically: that figure is derived server-side +// from bytes the provider has already been paid to carry, which is exactly +// what makes it ungameable, so accepting a submitted one would let this +// endpoint overwrite the derived figure with an asserted one. +func ProviderBandwidthResult(w http.ResponseWriter, r *http.Request) { + if !authorizeOperator(r) { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + var args SubmitProviderBandwidthArgs + if !readOperatorRequestBody(w, r, &args) { + return + } + + if args.ClientId == (server.Id{}) { + http.Error(w, "Missing client id.", http.StatusBadRequest) + return + } + if args.BytesPerSecond <= 0 { + http.Error(w, "bytes_per_second must be positive.", http.StatusBadRequest) + return + } + if args.SampleByteCount <= 0 { + http.Error(w, "sample_byte_count must be positive.", http.StatusBadRequest) + return + } + if !model.IsSubmittableProviderBandwidthSource(args.Source) { + http.Error(w, fmt.Sprintf( + "source must be one of %s, %s.", + model.ProviderBandwidthSourceActiveOperator, + model.ProviderBandwidthSourceActiveCDN, + ), http.StatusBadRequest) + return + } + + now := server.NowUtc() + model.StoreProviderBandwidth(r.Context(), &model.ProviderBandwidth{ + ClientId: args.ClientId, + BytesPerSecond: args.BytesPerSecond, + Source: args.Source, + SampleByteCount: args.SampleByteCount, + WindowStart: now, + WindowEnd: now, + }) + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]any{}); err != nil { + glog.Infof("[pbw]could not write response. err = %s\n", err) + } +} + +// ReserveProviderBandwidthArgs asks for budget to run one active probe. +type ReserveProviderBandwidthArgs struct { + ClientId server.Id `json:"client_id"` + ByteCount int64 `json:"byte_count"` +} + +// ReserveProviderBandwidthResult carries the reservation the prober just took. +// BucketStart is always the current hourly bucket (see ProviderBandwidthReserve). +type ReserveProviderBandwidthResult struct { + ReservationId server.Id `json:"reservation_id"` + BucketStart time.Time `json:"bucket_start"` +} + +// ProviderBandwidthReserve reserves deployment-wide byte budget for one active +// bandwidth probe. Active probing pulls real data through a provider's tunnel, +// which is real paid contract traffic, so it is rationed +// (model.ReserveProviderBandwidthSlot). +// +// The prober measures over a tunnel it already has open, right now: it has no +// use for budget in a later hour. model.ReserveProviderBandwidthSlot will +// happily defer a reservation into a future bucket for callers that can +// schedule a RunAt, so when it does that here the reservation is cancelled +// again and the request answered 429 -- the hourly ceiling would otherwise be +// decorative, since the prober could spend the whole daily budget inside one +// hour. Retry-After points at the bucket that does have room. (The plan +// specifies 429 "when every lookahead bucket is full"; this returns 429 on a +// strict superset of that, for the same "skip this provider cleanly" reason.) +func ProviderBandwidthReserve(w http.ResponseWriter, r *http.Request) { + if !authorizeOperator(r) { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + var args ReserveProviderBandwidthArgs + if !readOperatorRequestBody(w, r, &args) { + return + } + + if args.ClientId == (server.Id{}) { + http.Error(w, "Missing client id.", http.StatusBadRequest) + return + } + if args.ByteCount <= 0 { + http.Error(w, "byte_count must be positive.", http.StatusBadRequest) + return + } + // the byte count is caller-supplied; a probe never legitimately needs more + // than the per-probe figure, and an oversized request must not be able to + // swallow a large slice of a bucket in one reservation + byteCount := args.ByteCount + if model.MaxProviderBandwidthBytesPerProbe < byteCount { + byteCount = model.MaxProviderBandwidthBytesPerProbe + } + + ctx := r.Context() + now := server.NowUtc() + currentBucketStart := now.UTC().Truncate(model.ProviderBandwidthBucketDuration) + + reservationId, bucketStart, err := model.ReserveProviderBandwidthSlot(ctx, args.ClientId, byteCount) + if err != nil { + // every bucket in the lookahead window is full: the deployment's daily + // budget is exhausted + writeProviderBandwidthBudgetExhausted(w, currentBucketStart.Add(model.ProviderBandwidthBucketDuration).Sub(now), err.Error()) + return + } + if bucketStart.After(currentBucketStart) { + // budget exists, but not until a later hour -- of no use to a probe + // that runs now, so give it back rather than burning it on a request + // that is about to be skipped + model.CancelProviderBandwidthReservation(ctx, reservationId) + writeProviderBandwidthBudgetExhausted(w, bucketStart.Sub(now), "The active bandwidth probe budget for this hour has been reached.") + return + } + + w.Header().Set("Content-Type", "application/json") + result := &ReserveProviderBandwidthResult{ + ReservationId: reservationId, + BucketStart: bucketStart, + } + if err := json.NewEncoder(w).Encode(result); err != nil { + glog.Infof("[pbw]could not write response. err = %s\n", err) + } +} + +func writeProviderBandwidthBudgetExhausted(w http.ResponseWriter, retryAfter time.Duration, message string) { + retryAfterSeconds := int64(retryAfter.Seconds()) + if retryAfterSeconds < 1 { + retryAfterSeconds = 1 + } + w.Header().Set("Retry-After", strconv.FormatInt(retryAfterSeconds, 10)) + http.Error(w, message, http.StatusTooManyRequests) +} diff --git a/api/handlers/provider_bandwidth_handlers_test.go b/api/handlers/provider_bandwidth_handlers_test.go new file mode 100644 index 00000000..54ef04f5 --- /dev/null +++ b/api/handlers/provider_bandwidth_handlers_test.go @@ -0,0 +1,725 @@ +package handlers + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strconv" + "testing" + "time" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" +) + +// --- download endpoint ------------------------------------------------------- +// +// These tests touch no database: the endpoint only streams bytes, so they run +// outside DefaultTestEnv. + +func TestProviderBandwidthTestRejectsMissingSecret(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/network/provider-bandwidth-test?bytes=1024", nil) + w := httptest.NewRecorder() + + ProviderBandwidthTest(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret header is absent", w.Code) + } + if 0 < w.Body.Len() && w.Body.Len() != len("Unauthorized\n") { + t.Fatalf("body length = %d, want no payload streamed to an unauthenticated caller", w.Body.Len()) + } +} + +func TestProviderBandwidthTestRejectsWrongSecret(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/network/provider-bandwidth-test?bytes=1024", nil) + req.Header.Set(operatorSecretHeader, "definitely-not-the-secret") + w := httptest.NewRecorder() + + ProviderBandwidthTest(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 on a wrong operator secret", w.Code) + } +} + +// TestProviderBandwidthTestRejectsAlteredSecret proves hmac.Equal is actually +// consulted once the vault is configured, rather than the endpoint accepting +// anything once secret != "". +func TestProviderBandwidthTestRejectsAlteredSecret(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + const wrongSecret = "correct-operator-secret-0123456780" // last char changed + defer withStubOperatorIngestSecret(secret)() + + req := httptest.NewRequest(http.MethodGet, "/network/provider-bandwidth-test?bytes=1024", nil) + req.Header.Set(operatorSecretHeader, wrongSecret) + w := httptest.NewRecorder() + + ProviderBandwidthTest(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the configured secret and the request's secret differ", w.Code) + } +} + +// TestProviderBandwidthTestStreamsRequestedByteCount is the accept-path test +// with teeth: a handler that authenticates correctly but streams nothing (or +// the wrong amount) fails here, and so would an unconditional 401. +func TestProviderBandwidthTestStreamsRequestedByteCount(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + req := httptest.NewRequest(http.MethodGet, "/network/provider-bandwidth-test?bytes=1048576", nil) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthTest(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 with the correct operator secret", w.Code) + } + if w.Body.Len() != 1048576 { + t.Fatalf("streamed %d bytes, want exactly the requested 1048576", w.Body.Len()) + } + if contentType := w.Header().Get("Content-Type"); contentType != "application/octet-stream" { + t.Fatalf("Content-Type = %q, want application/octet-stream", contentType) + } + // httptest.NewRecorder does not enforce Content-Length, so a handler that + // claims one size and writes another would otherwise pass the check above. + assertContentLengthMatchesBody(t, w) +} + +// TestProviderBandwidthTestClampsToMaximum: an unclamped stream is an +// open-ended resource commitment per request. +func TestProviderBandwidthTestClampsToMaximum(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + // two orders of magnitude beyond the cap + req := httptest.NewRequest(http.MethodGet, "/network/provider-bandwidth-test?bytes=536870912", nil) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthTest(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200", w.Code) + } + if w.Body.Len() != maxProviderBandwidthTestBytes { + t.Fatalf("streamed %d bytes for a 536870912-byte request, want it clamped to %d", + w.Body.Len(), maxProviderBandwidthTestBytes) + } + assertContentLengthMatchesBody(t, w) +} + +// TestProviderBandwidthTestDefaultsUnusableByteCounts: `bytes=0` and +// `bytes=-1` parse cleanly, so they are not caught by a malformed-input check. +// Streaming an empty body for them would hand the prober a zero-byte sample to +// divide by. +func TestProviderBandwidthTestDefaultsUnusableByteCounts(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + for _, query := range []string{"", "?bytes=0", "?bytes=-1", "?bytes=not-a-number"} { + req := httptest.NewRequest(http.MethodGet, "/network/provider-bandwidth-test"+query, nil) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthTest(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("query %q: status = %d, want 200", query, w.Code) + } + if w.Body.Len() != defaultProviderBandwidthTestBytes { + t.Fatalf("query %q: streamed %d bytes, want the %d-byte default", + query, w.Body.Len(), defaultProviderBandwidthTestBytes) + } + } +} + +func assertContentLengthMatchesBody(t *testing.T, w *httptest.ResponseRecorder) { + t.Helper() + contentLength := w.Header().Get("Content-Length") + if contentLength == "" { + return + } + declared, err := strconv.Atoi(contentLength) + if err != nil { + t.Fatalf("Content-Length = %q, not a number", contentLength) + } + if declared != w.Body.Len() { + t.Fatalf("Content-Length = %d but %d bytes were written", declared, w.Body.Len()) + } +} + +// --- result endpoint --------------------------------------------------------- + +func TestProviderBandwidthResultRejectsMissingSecret(t *testing.T) { + body, _ := json.Marshal(map[string]any{ + "client_id": "019f8835-158d-6fd8-e9dd-fd0e4c6d6792", + "source": model.ProviderBandwidthSourceActiveOperator, + "bytes_per_second": 1000000.0, + "sample_byte_count": 5242880, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-result", bytes.NewReader(body)) + w := httptest.NewRecorder() + + ProviderBandwidthResult(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret header is absent", w.Code) + } +} + +func TestProviderBandwidthResultRejectsWrongSecret(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + const wrongSecret = "correct-operator-secret-0123456780" + defer withStubOperatorIngestSecret(secret)() + + body, _ := json.Marshal(map[string]any{ + "client_id": "019f8835-158d-6fd8-e9dd-fd0e4c6d6792", + "source": model.ProviderBandwidthSourceActiveOperator, + "bytes_per_second": 1000000.0, + "sample_byte_count": 5242880, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-result", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, wrongSecret) + w := httptest.NewRecorder() + + ProviderBandwidthResult(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 on a wrong operator secret", w.Code) + } +} + +// TestProviderBandwidthResultRejectsNonPositiveMeasurement: a zero or negative +// measurement is not a usable sample, and storing one would overwrite a real +// figure with a meaningless one. +func TestProviderBandwidthResultRejectsNonPositiveMeasurement(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + + cases := []struct { + name string + bytesPerSecond float64 + sampleByteCount int64 + }{ + {"zero rate", 0, 5 * 1024 * 1024}, + {"negative rate", -1, 5 * 1024 * 1024}, + {"zero sample", 1000000, 0}, + {"negative sample", 1000000, -1}, + } + for _, c := range cases { + clientId := server.NewId() + body, _ := json.Marshal(map[string]any{ + "client_id": clientId, + "source": model.ProviderBandwidthSourceActiveOperator, + "bytes_per_second": c.bytesPerSecond, + "sample_byte_count": c.sampleByteCount, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-result", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthResult(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("%s: status = %d, want 400; body = %s", c.name, w.Code, w.Body.String()) + } + if count := countProviderBandwidthRows(ctx, clientId); count != 0 { + t.Fatalf("%s: %d provider_bandwidth rows written, want the submission rejected before storage", c.name, count) + } + } + }) +} + +// TestProviderBandwidthResultStoresAnActiveMeasurement is the accept-path test +// with teeth: it reads provider_bandwidth back with raw SQL, so a handler that +// authenticates correctly but never calls model.StoreProviderBandwidth fails +// here -- as would an unconditional 401. Both active targets are covered, +// because the source now decides which row is written. +func TestProviderBandwidthResultStoresAnActiveMeasurement(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + + cases := []struct { + name string + source string + }{ + {"operator target", model.ProviderBandwidthSourceActiveOperator}, + {"cdn target", model.ProviderBandwidthSourceActiveCDN}, + } + for _, c := range cases { + clientId := server.NewId() + const bytesPerSecond = 1234567.5 + const sampleByteCount int64 = 3 * 1024 * 1024 + + body, _ := json.Marshal(map[string]any{ + "client_id": clientId, + "source": c.source, + "bytes_per_second": bytesPerSecond, + "sample_byte_count": sampleByteCount, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-result", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthResult(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("%s: status = %d, want 200 for a valid submission; body = %s", c.name, w.Code, w.Body.String()) + } + + stored := readProviderBandwidthBySource(ctx, clientId) + row, ok := stored[c.source] + if !ok { + t.Fatalf("%s: no provider_bandwidth row under source %q; the handler never stored the measurement", c.name, c.source) + } + if row.bytesPerSecond != bytesPerSecond { + t.Fatalf("%s: bytes_per_second = %f, want the submitted %f", c.name, row.bytesPerSecond, bytesPerSecond) + } + if row.sampleByteCount != sampleByteCount { + t.Fatalf("%s: sample_byte_count = %d, want the submitted %d", c.name, row.sampleByteCount, sampleByteCount) + } + } + }) +} + +// TestProviderBandwidthResultRejectsUnknownSource: the source is part of the +// storage key, so it is validated rather than merely stored. +// +// An unrecognised value is not a harmless label -- it creates a row under a tag +// nothing will ever read or replace, while the submitter goes on looking like +// it is working. "passive" is refused for a sharper reason: that figure is +// derived server-side from bytes the provider has already been paid to carry, +// which is precisely what makes it ungameable, and accepting a submitted one +// would let this endpoint overwrite the derived figure with an asserted one. +func TestProviderBandwidthResultRejectsUnknownSource(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + + cases := []struct { + name string + source any + }{ + {"absent", nil}, + {"empty", ""}, + {"the pre-split active tag", "active"}, + {"a typo", "active-cnd"}, + {"wrong case", "ACTIVE-CDN"}, + {"server-derived passive", model.ProviderBandwidthSourcePassive}, + } + for _, c := range cases { + clientId := server.NewId() + payload := map[string]any{ + "client_id": clientId, + "bytes_per_second": 1234567.5, + "sample_byte_count": 3 * 1024 * 1024, + } + if c.source != nil { + payload["source"] = c.source + } + body, _ := json.Marshal(payload) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-result", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthResult(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("%s (source=%v): status = %d, want 400; body = %s", c.name, c.source, w.Code, w.Body.String()) + } + if count := countProviderBandwidthRows(ctx, clientId); count != 0 { + t.Fatalf("%s (source=%v): %d provider_bandwidth rows written, want the submission rejected before storage", c.name, c.source, count) + } + } + }) +} + +// TestProviderBandwidthResultStoresTheTwoTargetsSeparately is the end-to-end +// form of the property the second target exists for: two submissions for ONE +// provider, one per target, must land in two rows carrying two figures. +// +// Against the old client_id-only key the second submission overwrote the +// first, so a provider prioritising the operator's own path while starving the +// public internet looked identical to one that was simply fast -- the exact +// divergence this is here to expose. +func TestProviderBandwidthResultStoresTheTwoTargetsSeparately(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + clientId := server.NewId() + + submitted := map[string]float64{ + model.ProviderBandwidthSourceActiveOperator: 12_000_000, + model.ProviderBandwidthSourceActiveCDN: 3_000_000, + } + for source, bytesPerSecond := range submitted { + body, _ := json.Marshal(map[string]any{ + "client_id": clientId, + "source": source, + "bytes_per_second": bytesPerSecond, + "sample_byte_count": 5 * 1024 * 1024, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-result", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthResult(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("source %q: status = %d, want 200; body = %s", source, w.Code, w.Body.String()) + } + } + + stored := readProviderBandwidthBySource(ctx, clientId) + if len(stored) != 2 { + t.Fatalf("%d provider_bandwidth rows for one provider, want 2 (one per target) -- the two targets are overwriting each other", len(stored)) + } + for source, want := range submitted { + row, ok := stored[source] + if !ok { + t.Fatalf("no row under source %q", source) + } + if row.bytesPerSecond != want { + t.Errorf("source %q stored %.0f B/s, want the submitted %.0f -- the figures are not being kept apart (an averaged pair would read %.0f)", + source, row.bytesPerSecond, want, (submitted[model.ProviderBandwidthSourceActiveOperator]+submitted[model.ProviderBandwidthSourceActiveCDN])/2) + } + } + }) +} + +type storedProviderBandwidth struct { + bytesPerSecond float64 + sampleByteCount int64 +} + +// readProviderBandwidthBySource reads the rows back with raw SQL, keyed by +// source, so the assertions are against the table rather than against the +// writer that produced it. +func readProviderBandwidthBySource(ctx context.Context, clientId server.Id) map[string]storedProviderBandwidth { + bySource := map[string]storedProviderBandwidth{} + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + `SELECT source, bytes_per_second, sample_byte_count FROM provider_bandwidth WHERE client_id = $1`, + clientId, + ) + server.WithPgResult(result, err, func() { + for result.Next() { + var source string + var row storedProviderBandwidth + server.Raise(result.Scan(&source, &row.bytesPerSecond, &row.sampleByteCount)) + bySource[source] = row + } + }) + }) + return bySource +} + +// TestProviderBandwidthPostsRejectMissingClientId: an absent client_id +// unmarshals to the zero id, which would otherwise be written as a real row +// keyed on the nil uuid. +func TestProviderBandwidthPostsRejectMissingClientId(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + resultBody, _ := json.Marshal(map[string]any{ + "source": model.ProviderBandwidthSourceActiveOperator, + "bytes_per_second": 1000000.0, + "sample_byte_count": 5242880, + }) + reserveBody, _ := json.Marshal(map[string]any{ + "byte_count": 5 * 1024 * 1024, + }) + + cases := []struct { + name string + path string + body []byte + handler func(http.ResponseWriter, *http.Request) + }{ + {"result", "/network/provider-bandwidth-result", resultBody, ProviderBandwidthResult}, + {"reserve", "/network/provider-bandwidth-reserve", reserveBody, ProviderBandwidthReserve}, + } + for _, c := range cases { + req := httptest.NewRequest(http.MethodPost, c.path, bytes.NewReader(c.body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + c.handler(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("%s: status = %d, want 400 for a submission with no client id", c.name, w.Code) + } + } +} + +func countProviderBandwidthRows(ctx context.Context, clientId server.Id) int { + count := 0 + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + `SELECT COUNT(*) FROM provider_bandwidth WHERE client_id = $1`, + clientId, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + server.Raise(result.Scan(&count)) + } + }) + }) + return count +} + +// --- reservation endpoint ---------------------------------------------------- + +func TestProviderBandwidthReserveRejectsMissingSecret(t *testing.T) { + body, _ := json.Marshal(map[string]any{ + "client_id": "019f8835-158d-6fd8-e9dd-fd0e4c6d6792", + "byte_count": 5 * 1024 * 1024, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-reserve", bytes.NewReader(body)) + w := httptest.NewRecorder() + + ProviderBandwidthReserve(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret header is absent", w.Code) + } +} + +func TestProviderBandwidthReserveRejectsWrongSecret(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + const wrongSecret = "correct-operator-secret-0123456780" + defer withStubOperatorIngestSecret(secret)() + + body, _ := json.Marshal(map[string]any{ + "client_id": "019f8835-158d-6fd8-e9dd-fd0e4c6d6792", + "byte_count": 5 * 1024 * 1024, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-reserve", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, wrongSecret) + w := httptest.NewRecorder() + + ProviderBandwidthReserve(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 on a wrong operator secret", w.Code) + } +} + +// TestProviderBandwidthReserveReservesBudget is the accept-path test with +// teeth: it asserts the ledger row exists, so a handler that authenticates and +// returns 200 without reserving anything fails -- as would an unconditional +// 401. +func TestProviderBandwidthReserveReservesBudget(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + clientId := server.NewId() + + body, _ := json.Marshal(map[string]any{ + "client_id": clientId, + "byte_count": model.MaxProviderBandwidthBytesPerProbe, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-reserve", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthReserve(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200 for a reservation against an empty budget; body = %s", w.Code, w.Body.String()) + } + + reservedCount, reservedBytes := providerBandwidthQuotaFor(ctx, clientId) + if reservedCount != 1 { + t.Fatalf("%d provider_bandwidth_quota rows, want exactly 1 -- the handler must actually reserve budget", reservedCount) + } + if reservedBytes != model.MaxProviderBandwidthBytesPerProbe { + t.Fatalf("reserved %d bytes, want %d", reservedBytes, model.MaxProviderBandwidthBytesPerProbe) + } + + var result ReserveProviderBandwidthResult + if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { + t.Fatalf("decode response: %s; body = %s", err, w.Body.String()) + } + if result.ReservationId == (server.Id{}) { + t.Fatal("response carried no reservation id") + } + }) +} + +// TestProviderBandwidthReserveClampsByteCount: the byte count is caller- +// supplied, so an oversized (or fat-fingered) request must not be able to +// consume a whole bucket in one reservation. +func TestProviderBandwidthReserveClampsByteCount(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + clientId := server.NewId() + + body, _ := json.Marshal(map[string]any{ + "client_id": clientId, + "byte_count": 100 * model.MaxProviderBandwidthBytesPerProbe, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-reserve", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthReserve(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body = %s", w.Code, w.Body.String()) + } + _, reservedBytes := providerBandwidthQuotaFor(ctx, clientId) + if reservedBytes != model.MaxProviderBandwidthBytesPerProbe { + t.Fatalf("reserved %d bytes for an oversized request, want it clamped to %d", + reservedBytes, model.MaxProviderBandwidthBytesPerProbe) + } + }) +} + +func TestProviderBandwidthReserveRejectsNonPositiveByteCount(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + for _, byteCount := range []int64{0, -1} { + clientId := server.NewId() + body, _ := json.Marshal(map[string]any{ + "client_id": clientId, + "byte_count": byteCount, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-reserve", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthReserve(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("byte_count %d: status = %d, want 400", byteCount, w.Code) + } + if count, _ := providerBandwidthQuotaFor(ctx, clientId); count != 0 { + t.Fatalf("byte_count %d: %d quota rows written for a rejected request", byteCount, count) + } + } + }) +} + +// TestProviderBandwidthReserveReturns429WhenCurrentBucketIsFull: the prober +// probes over an already-open tunnel right now, so a reservation deferred to a +// later hour is of no use to it -- the endpoint cancels that deferred +// reservation and answers 429, which the prober treats as "skip this +// provider". Leaving the deferred reservation in place would burn budget on +// every skipped provider. +func TestProviderBandwidthReserveReturns429WhenCurrentBucketIsFull(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + // fill the current hourly bucket exactly to its ceiling + bucketStart := server.NowUtc().UTC().Truncate(time.Hour) + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO provider_bandwidth_quota (provider_bandwidth_quota_id, client_id, byte_count, bucket_start, create_time) + VALUES ($1, $2, $3, $4, $5) + `, + server.NewId(), server.NewId(), model.MaxProviderBandwidthBytesPerBucket, bucketStart, server.NowUtc(), + )) + }) + + clientId := server.NewId() + body, _ := json.Marshal(map[string]any{ + "client_id": clientId, + "byte_count": model.MaxProviderBandwidthBytesPerProbe, + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-bandwidth-reserve", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderBandwidthReserve(w, req) + + if w.Code != http.StatusTooManyRequests { + t.Fatalf("status = %d, want 429 when the current hourly bucket has no room; body = %s", w.Code, w.Body.String()) + } + if count, _ := providerBandwidthQuotaFor(ctx, clientId); count != 0 { + t.Fatalf("%d quota rows left behind for a 429'd request, want 0 -- a deferred reservation must be cancelled", count) + } + if retryAfter := w.Header().Get("Retry-After"); retryAfter == "" { + t.Fatal("no Retry-After header on the 429; the prober cannot tell when budget frees up") + } + }) +} + +func providerBandwidthQuotaFor(ctx context.Context, clientId server.Id) (count int, byteCount int64) { + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + `SELECT COUNT(*), COALESCE(SUM(byte_count), 0)::bigint FROM provider_bandwidth_quota WHERE client_id = $1`, + clientId, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + server.Raise(result.Scan(&count, &byteCount)) + } + }) + }) + return count, byteCount +} + +// TestProviderBandwidthTestClampFitsInsideOneProbesReservation guards the one +// invariant that ties the download endpoint to the byte budget now that they +// are no longer the same number. +// +// A probe is not one request any more: it is bandwidth.StreamCount parallel +// streams, because a single TCP flow cannot exceed (connect's 1 MiB window / +// RTT) and the single-stream probe was measuring that ceiling rather than the +// provider. So maxProviderBandwidthTestBytes bounds ONE STREAM and +// model.MaxProviderBandwidthBytesPerProbe bounds the whole probe. +// +// They may drift apart, but only in one direction. If a single request could +// be served larger than a whole probe reserves, one request could spend more +// budget than was ever admitted for it -- the reservation would be a number +// the deployment routinely exceeds, which is worse than having no budget at +// all. +func TestProviderBandwidthTestClampFitsInsideOneProbesReservation(t *testing.T) { + if model.MaxProviderBandwidthBytesPerProbe < int64(maxProviderBandwidthTestBytes) { + t.Fatalf( + "the download endpoint serves up to %d bytes in ONE request but a probe only reserves %d in total: "+ + "a single request could transfer more than the byte budget admitted for the whole probe", + maxProviderBandwidthTestBytes, model.MaxProviderBandwidthBytesPerProbe, + ) + } +} diff --git a/api/handlers/provider_client_verdict_handlers.go b/api/handlers/provider_client_verdict_handlers.go new file mode 100644 index 00000000..a315173f --- /dev/null +++ b/api/handlers/provider_client_verdict_handlers.go @@ -0,0 +1,25 @@ +package handlers + +import ( + "net/http" + + "github.com/urnetwork/server/model" + "github.com/urnetwork/server/router" +) + +// ProviderClientVerdictSubmit receives one client-reported blackhole verdict. +// +// Unlike the other provider-probing ingest endpoints in this package, this one +// is NOT operator-secret authed: the reporter is a real client network, and the +// network is the unit the quorum counts. WrapWithInputRequireAuth fails closed +// with a 401 when the jwt is missing or unparseable, and hands the impl a +// session whose ByJwt.NetworkId is the reporter -- which is the only place the +// reporter identity may come from. +// +// All validation, the append-only store and the quorum aggregation live in +// model.SubmitProviderClientVerdict; the strict (unknown-field-rejecting) +// decode is attached to the args type, because the router's decoder is shared +// by every endpoint and must not be tightened globally. +func ProviderClientVerdictSubmit(w http.ResponseWriter, r *http.Request) { + router.WrapWithInputRequireAuth(model.SubmitProviderClientVerdict, w, r) +} diff --git a/api/handlers/provider_client_verdict_handlers_test.go b/api/handlers/provider_client_verdict_handlers_test.go new file mode 100644 index 00000000..de611bcc --- /dev/null +++ b/api/handlers/provider_client_verdict_handlers_test.go @@ -0,0 +1,426 @@ +package handlers + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "slices" + "testing" + "time" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/jwt" + "github.com/urnetwork/server/model" +) + +// validClientVerdictBody is a well-formed egress-dead report: the client sent +// and was acknowledged, and nothing came back. Tests mutate a copy to exercise +// one rule at a time. +// +// Note what is NOT in it: reporter_network_id. The reporter is the session. +func validClientVerdictBody(exitClientId server.Id) map[string]any { + return map[string]any{ + "exit_client_id": exitClientId.String(), + "reason": model.ProviderClientVerdictReasonNoReceiveAck, + "send_ack_count": 64, + "send_ack_bytes": 8192, + "receive_ack_count": 0, + "receive_ack_bytes": 0, + "syn_sent": 3, + "syn_received": 0, + "window_seconds": 30, + } +} + +// postClientVerdict posts as networkId, or unauthenticated when networkId is +// the zero id. +func postClientVerdict( + t testing.TB, + networkId server.Id, + body map[string]any, +) *httptest.ResponseRecorder { + t.Helper() + buf, err := json.Marshal(body) + if err != nil { + t.Fatalf("marshal body: %s", err) + } + req := httptest.NewRequest(http.MethodPost, "/network/provider-verdict", bytes.NewReader(buf)) + if networkId != (server.Id{}) { + byJwt := jwt.NewByJwt(networkId, server.NewId(), "test", false, false) + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", byJwt.Sign())) + } + w := httptest.NewRecorder() + ProviderClientVerdictSubmit(w, req) + return w +} + +// Fails closed. Without a session there is no reporter network, and a verdict +// with no reporter is a verdict that cannot be counted or capped. +func TestProviderClientVerdictRejectsUnauthenticated(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + exitClientId := server.NewId() + + w := postClientVerdict(t, server.Id{}, validClientVerdictBody(exitClientId)) + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 for an unauthenticated report", w.Code) + } + + verdicts := model.GetProviderClientVerdictsInWindow( + context.Background(), + exitClientId, + server.NowUtc(), + ) + if len(verdicts) != 0 { + t.Fatalf("stored %d verdicts from an unauthenticated request, want 0", len(verdicts)) + } + }) +} + +// THE REPORTER IS THE SESSION. Two sessions posting a byte-identical body must +// store two different reporter networks -- and a body that tries to name its +// own reporter never gets in at all. +// +// The second half is a deviation worth being explicit about: because the args +// type decodes strictly, a body carrying reporter_network_id is REJECTED with a +// 400 rather than silently ignored. Either way it can never be honoured, and a +// loud rejection beats a client that believes it is choosing its own reporter +// id and is quietly overruled. +func TestProviderClientVerdictReporterComesFromTheSession(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + exitClientId := server.NewId() + reporterA := server.NewId() + reporterB := server.NewId() + liar := server.NewId() + + if w := postClientVerdict(t, reporterA, validClientVerdictBody(exitClientId)); w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + if w := postClientVerdict(t, reporterB, validClientVerdictBody(exitClientId)); w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + + // a body naming another network, posted by reporterA + lying := validClientVerdictBody(exitClientId) + lying["reporter_network_id"] = liar.String() + if w := postClientVerdict(t, reporterA, lying); w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 for a body naming its own reporter_network_id", w.Code) + } + + verdicts := model.GetProviderClientVerdictsInWindow(ctx, exitClientId, server.NowUtc()) + reporters := []server.Id{} + for _, verdict := range verdicts { + reporters = append(reporters, verdict.ReporterNetworkId) + } + if len(reporters) != 2 { + t.Fatalf("stored %d verdicts, want 2 (the lying body must not have been stored)", len(reporters)) + } + if !slices.Contains(reporters, reporterA) || !slices.Contains(reporters, reporterB) { + t.Fatalf("reporters = %v, want exactly the two session networks %s and %s", + reporters, reporterA, reporterB) + } + if slices.Contains(reporters, liar) { + t.Fatalf("reporters = %v: the body's reporter_network_id was honoured", reporters) + } + }) +} + +func TestProviderClientVerdictRejectsBadBodies(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + reporter := server.NewId() + + cases := []struct { + name string + mutate func(body map[string]any) + }{ + { + // an open reason column fills with whatever a client build + // happens to send, and then nothing can be counted by it + name: "unknown reason", + mutate: func(body map[string]any) { + body["reason"] = "slow" + }, + }, + { + name: "empty reason", + mutate: func(body map[string]any) { + body["reason"] = "" + }, + }, + { + // a misspelled count decodes to zero, and zero receive acks is + // exactly the value that means "egress dead" -- so a typo would + // not be a malformed report, it would be a counting one + name: "unknown field", + mutate: func(body map[string]any) { + body["recieve_ack_count"] = 0 + }, + }, + { + name: "negative receive ack count", + mutate: func(body map[string]any) { + body["receive_ack_count"] = -1 + }, + }, + { + name: "negative send ack bytes", + mutate: func(body map[string]any) { + body["send_ack_bytes"] = -8192 + }, + }, + { + name: "negative syn received", + mutate: func(body map[string]any) { + body["syn_received"] = -1 + }, + }, + { + name: "negative window", + mutate: func(body map[string]any) { + body["window_seconds"] = -30 + }, + }, + { + name: "missing exit client id", + mutate: func(body map[string]any) { + delete(body, "exit_client_id") + }, + }, + } + + for _, testCase := range cases { + exitClientId := server.NewId() + body := validClientVerdictBody(exitClientId) + testCase.mutate(body) + + w := postClientVerdict(t, reporter, body) + if w.Code != http.StatusBadRequest { + t.Errorf("%s: status = %d, want 400: %s", testCase.name, w.Code, w.Body.String()) + } + + // 400 BEFORE any store, never store-then-flag: a row that is in + // this table is a row that counts + verdicts := model.GetProviderClientVerdictsInWindow( + context.Background(), + exitClientId, + server.NowUtc(), + ) + if len(verdicts) != 0 { + t.Errorf("%s: stored %d verdicts on a rejected body, want 0", testCase.name, len(verdicts)) + } + } + }) +} + +func TestProviderClientVerdictStoresValidReport(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + exitClientId := server.NewId() + reporter := server.NewId() + + w := postClientVerdict(t, reporter, validClientVerdictBody(exitClientId)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200: %s", w.Code, w.Body.String()) + } + + verdicts := model.GetProviderClientVerdictsInWindow(ctx, exitClientId, server.NowUtc()) + if len(verdicts) != 1 { + t.Fatalf("stored %d verdicts, want 1", len(verdicts)) + } + verdict := verdicts[0] + if verdict.ReporterNetworkId != reporter { + t.Fatalf("reporter = %s, want the session network %s", verdict.ReporterNetworkId, reporter) + } + if verdict.Reason != model.ProviderClientVerdictReasonNoReceiveAck { + t.Fatalf("reason = %q", verdict.Reason) + } + if verdict.SendAckCount != 64 || verdict.SendAckBytes != 8192 { + t.Fatalf("send acks = %d/%dB", verdict.SendAckCount, verdict.SendAckBytes) + } + if verdict.ReceiveAckCount != 0 || verdict.ReceiveAckBytes != 0 { + t.Fatalf("receive acks = %d/%dB", verdict.ReceiveAckCount, verdict.ReceiveAckBytes) + } + if verdict.WindowSeconds != 30 { + t.Fatalf("window seconds = %d", verdict.WindowSeconds) + } + }) +} + +// testing_connectProbeableProvider is the api/handlers copy of the model test +// fixture: a connected, valid provider holding a Public provide key, which is +// what GetProviderEgressLocationDue requires before it will offer a provider. +func testing_connectProbeableProvider( + t testing.TB, + ctx context.Context, + clientId server.Id, + locationId server.Id, + clientAddress string, +) { + t.Helper() + model.Testing_CreateDevice(ctx, server.NewId(), server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientAddress, handlerId) + if err != nil { + t.Fatalf("connect client: %s", err) + } + if err := model.SetConnectionLocation(ctx, connectionId, locationId, &model.ConnectionLocationScores{}); err != nil { + t.Fatalf("set connection location: %s", err) + } + model.SetProvide(ctx, clientId, map[model.ProvideMode][]byte{ + model.ProvideModePublic: []byte("provide-secret"), + }) +} + +// End to end: three distinct networks report a provider egress-dead and the +// prober is offered that provider on its next poll -- and nothing else moves. +// +// The due cutoff here is the ENDPOINT's own arithmetic (providerEgressDueAge), +// not a hand-picked one. model cannot import that constant, so this is the test +// that fails if the reprioritise target and the due cutoff ever drift apart, +// instead of the feature silently doing nothing. +func TestProviderClientVerdictQuorumMakesTheProviderDue(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + + city := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + model.CreateLocation(ctx, city) + + reported := server.NewId() + quiet := server.NewId() + testing_connectProbeableProvider(t, ctx, reported, city.LocationId, "0.0.0.1:0") + testing_connectProbeableProvider(t, ctx, quiet, city.LocationId, "0.0.0.2:0") + model.UpdateClientLocationReliabilities(ctx, now.Add(-time.Hour), now) + + // both probed an hour ago: neither is due, and neither has a probe + // attempt row, so nothing is deferred by the attempt backoff either + for _, clientId := range []server.Id{reported, quiet} { + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: clientId, + LocationId: city.LocationId, + CountryCode: "us", + ObservedAt: now.Add(-time.Hour), + Verdict: "verified", + }) + } + + due := func() []server.Id { + return model.GetProviderEgressLocationDue( + ctx, + server.NowUtc().Add(-providerEgressDueAge), + server.NowUtc().Add(-model.ProviderEgressProbeAttemptBackoff), + 100, + ) + } + if slices.Contains(due(), reported) { + t.Fatal("the provider was already due before any verdict") + } + + // two reporters are not a quorum + for range 2 { + w := postClientVerdict(t, server.NewId(), validClientVerdictBody(reported)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body.String()) + } + } + if slices.Contains(due(), reported) { + t.Fatal("two reporters made the provider due, want three") + } + + w := postClientVerdict(t, server.NewId(), validClientVerdictBody(reported)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body.String()) + } + + if !slices.Contains(due(), reported) { + t.Fatal("a met quorum did not make the provider due for probing") + } + // the effect is scoped to the reported provider + if slices.Contains(due(), quiet) { + t.Fatal("an unreported provider became due") + } + + // AND NOTHING IN THE SELECTION PATH MOVED. The location still resolves, + // with the same location id, country and verdict -- a met quorum + // schedules a probe and does not demote, exclude or rescore anything. + // (Nothing in selection reads observed_at at all; the freshness lookup + // below is the closest thing to it that exists.) + fresh := model.GetFreshProviderEgressLocation(ctx, reported, model.ProviderEgressLocationMaxAge) + if fresh == nil { + t.Fatal("the reported provider's location stopped being fresh") + } + if fresh.LocationId != city.LocationId { + t.Fatalf("location id = %s, want %s", fresh.LocationId, city.LocationId) + } + if fresh.CountryCode != "us" { + t.Fatalf("country code = %q, want us", fresh.CountryCode) + } + if fresh.Verdict != "verified" { + t.Fatalf("verdict = %q, want verified: a client verdict must not touch the probe verdict", fresh.Verdict) + } + }) +} + +// The quorum brings the next probe forward; it does not override the attempt +// backoff. A provider the prober tried minutes ago stays deferred, so the most +// a quorum can buy -- honest or manufactured -- is one probe per provider per +// ProviderEgressProbeAttemptBackoff. That bound is what keeps the cost of +// griefing at one probe, and it is why the test above deliberately has no +// attempt row. +func TestProviderClientVerdictQuorumDoesNotOverrideTheAttemptBackoff(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + + city := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + model.CreateLocation(ctx, city) + + clientId := server.NewId() + testing_connectProbeableProvider(t, ctx, clientId, city.LocationId, "0.0.0.3:0") + model.UpdateClientLocationReliabilities(ctx, now.Add(-time.Hour), now) + + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: clientId, + LocationId: city.LocationId, + CountryCode: "us", + ObservedAt: now.Add(-time.Hour), + }) + model.SetProviderEgressProbeAttempt(ctx, &model.ProviderEgressProbeAttempt{ + ClientId: clientId, + AttemptAt: now.Add(-5 * time.Minute), + }) + + for range model.ProviderClientVerdictQuorum { + w := postClientVerdict(t, server.NewId(), validClientVerdictBody(clientId)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d: %s", w.Code, w.Body.String()) + } + } + + due := model.GetProviderEgressLocationDue( + ctx, + server.NowUtc().Add(-providerEgressDueAge), + server.NowUtc().Add(-model.ProviderEgressProbeAttemptBackoff), + 100, + ) + if slices.Contains(due, clientId) { + t.Fatal("a met quorum overrode the probe attempt backoff") + } + }) +} diff --git a/api/handlers/provider_egress_health_handlers.go b/api/handlers/provider_egress_health_handlers.go new file mode 100644 index 00000000..b6bd773b --- /dev/null +++ b/api/handlers/provider_egress_health_handlers.go @@ -0,0 +1,218 @@ +package handlers + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "net/http" + + "github.com/urnetwork/glog" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" +) + +// maxProviderEgressHealthBody bounds the request body. It is larger than the +// bandwidth endpoints' 4 KiB cap because this body carries a per-class map plus +// two joined destination-name lists, and a run against a wide table can name a +// dozen failed destinations. +const maxProviderEgressHealthBody = 16 * 1024 + +// providerEgressHealthReputationClass is the class name that must never appear +// inside class_results. See ProviderEgressHealthResult for why this is checked +// rather than tolerated. +const providerEgressHealthReputationClass = "reputation" + +// ProviderEgressHealthClassResult is one class's ok/total tally over the +// destinations the run SAMPLED. +type ProviderEgressHealthClassResult struct { + OK int `json:"ok"` + Total int `json:"total"` +} + +// SubmitProviderEgressHealthArgs is one egress-health run for one provider, as +// the prober measured it. +// +// There is deliberately no measured_at field: the server stamps arrival time, +// exactly as the bandwidth result endpoint does. A caller-supplied timestamp +// would be one more thing to validate and one more way for a skewed prober +// clock to write a row that looks stale or future-dated. +type SubmitProviderEgressHealthArgs struct { + ClientId server.Id `json:"client_id"` + // OKCount/TotalCount cover the SCORED classes only. Reputation is not part + // of them and must never be added to them. + OKCount int `json:"ok_count"` + TotalCount int `json:"total_count"` + // ClassResults is the per-class tally for the scored classes. Its ok and + // total must sum to exactly OKCount and TotalCount. + ClassResults map[string]ProviderEgressHealthClassResult `json:"class_results"` + // ReputationOK/ReputationTotal are stored beside the health figures and + // never inside them. + ReputationOK int `json:"reputation_ok"` + ReputationTotal int `json:"reputation_total"` + FailedNames string `json:"failed_names"` + ReputationFailedNames string `json:"reputation_failed_names"` +} + +// readStrictOperatorRequestBody reads a bounded operator request body and +// rejects any field the target struct does not declare. +// +// It is a separate reader from readOperatorRequestBody rather than a flag on +// it: that one is shared with the bandwidth endpoints, which are already +// deployed and already accept whatever their probers send, and tightening a +// live endpoint's parser as a side effect of adding a new one is how a working +// fleet stops submitting overnight. +// +// Rejecting unknown fields matters here specifically because this body is a +// set of counts that must agree with each other. A misspelled field silently +// decodes to zero, and a zero count is a perfectly valid, perfectly consistent +// payload -- so the failure would be a table full of plausible rows describing +// a measurement that never happened. +func readStrictOperatorRequestBody(w http.ResponseWriter, r *http.Request, out any) bool { + body, err := io.ReadAll(io.LimitReader(r.Body, maxProviderEgressHealthBody+1)) + if err != nil { + http.Error(w, "Bad request", http.StatusBadRequest) + return false + } + if maxProviderEgressHealthBody < len(body) { + http.Error(w, "Request too large", http.StatusRequestEntityTooLarge) + return false + } + decoder := json.NewDecoder(bytes.NewReader(body)) + decoder.DisallowUnknownFields() + if err := decoder.Decode(out); err != nil { + http.Error(w, "Bad request", http.StatusBadRequest) + return false + } + return true +} + +// ProviderEgressHealthResult ingests one egress-health run from the operator's +// prober. The prober runs the check over the same tunnel the geolocation probe +// opened, and until now only logged the result -- so the one signal that says +// whether a provider carries traffic at all rolled off with the container +// logs. This stores it. +// +// The route is operator-to-server, gated by the same operator secret as the +// egress-location and bandwidth ingest endpoints. There is no network jwt. +// +// # Everything is validated before anything is stored +// +// The row is an upsert keyed on client_id, so a bad submission does not sit +// beside the good one waiting to be noticed -- it DESTROYS the last good +// measurement for that provider. That is why every rule below returns 400 +// before the store, and why none of them is a stored-then-flagged warning. +// +// # Reputation is not health +// +// reputation_ok/reputation_total are stored and never folded into +// ok_count/total_count, and a "reputation" key inside class_results is +// rejected outright. The reputation class measures whether large vendors treat +// the exit ip as a datacenter address; nearly every honest hosted provider +// fails most of it, because it IS hosted. Folding it in would score a provider +// that carried every byte it was asked for as partly broken, and would punish +// the well-run datacenter providers hardest. +// +// The explicit rejection exists because the alternative is worse than a +// rejection. If a caller ever put reputation inside class_results, the sum +// check would fail and every submission would 400 -- and the obvious "fix" is +// to relax the sum check, at which point reputation is silently inside the +// health score and nothing says so. The operator-proxy's egresshealth package +// calls this exact mistake "the one thing in this package most likely to be +// 'fixed' into a bug"; this is the server-side half of that guard. +func ProviderEgressHealthResult(w http.ResponseWriter, r *http.Request) { + if !authorizeOperator(r) { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + var args SubmitProviderEgressHealthArgs + if !readStrictOperatorRequestBody(w, r, &args) { + return + } + + if args.ClientId == (server.Id{}) { + http.Error(w, "Missing client id.", http.StatusBadRequest) + return + } + if args.OKCount < 0 || args.TotalCount < 0 { + http.Error(w, "ok_count and total_count must be non-negative.", http.StatusBadRequest) + return + } + if args.ReputationOK < 0 || args.ReputationTotal < 0 { + http.Error(w, "reputation_ok and reputation_total must be non-negative.", http.StatusBadRequest) + return + } + if args.TotalCount < args.OKCount { + // more destinations passed than were attempted: the submitter is not + // measuring what it thinks it is, and storing this would overwrite a + // real measurement with an impossible one + http.Error(w, "ok_count must not exceed total_count.", http.StatusBadRequest) + return + } + if args.ReputationTotal < args.ReputationOK { + http.Error(w, "reputation_ok must not exceed reputation_total.", http.StatusBadRequest) + return + } + + sumOK, sumTotal := 0, 0 + for class, tally := range args.ClassResults { + if class == providerEgressHealthReputationClass { + http.Error(w, fmt.Sprintf( + "%q is not a scored class: it is reported in reputation_ok/reputation_total and must never be part of ok_count/total_count.", + providerEgressHealthReputationClass, + ), http.StatusBadRequest) + return + } + if tally.OK < 0 || tally.Total < 0 { + http.Error(w, fmt.Sprintf("class %q: ok and total must be non-negative.", class), http.StatusBadRequest) + return + } + if tally.Total < tally.OK { + // checked per class as well as in aggregate: {ok:5,total:2} and + // {ok:0,total:3} sum to a consistent 5/5 while describing a class + // where more destinations passed than ran + http.Error(w, fmt.Sprintf("class %q: ok must not exceed total.", class), http.StatusBadRequest) + return + } + sumOK += tally.OK + sumTotal += tally.Total + } + // exact equality, not <=: the classes ARE the score. A total that does not + // decompose into its classes means the two halves of the payload were + // produced by different runs, or that something not in class_results was + // counted into the score -- which is precisely how reputation would get in. + if sumOK != args.OKCount || sumTotal != args.TotalCount { + http.Error(w, fmt.Sprintf( + "class_results sum to %d/%d but ok_count/total_count are %d/%d.", + sumOK, sumTotal, args.OKCount, args.TotalCount, + ), http.StatusBadRequest) + return + } + + classResults := map[string]model.ProviderEgressHealthClassResult{} + for class, tally := range args.ClassResults { + classResults[class] = model.ProviderEgressHealthClassResult{ + OK: tally.OK, + Total: tally.Total, + } + } + + model.SetProviderEgressHealth(r.Context(), &model.ProviderEgressHealth{ + ClientId: args.ClientId, + MeasuredAt: server.NowUtc(), + OKCount: args.OKCount, + Total: args.TotalCount, + ClassResults: classResults, + ReputationOK: args.ReputationOK, + ReputationTotal: args.ReputationTotal, + FailedNames: args.FailedNames, + ReputationFailedNames: args.ReputationFailedNames, + }) + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(map[string]any{}); err != nil { + glog.Infof("[pegh]could not write response. err = %s\n", err) + } +} diff --git a/api/handlers/provider_egress_health_handlers_test.go b/api/handlers/provider_egress_health_handlers_test.go new file mode 100644 index 00000000..945628b3 --- /dev/null +++ b/api/handlers/provider_egress_health_handlers_test.go @@ -0,0 +1,302 @@ +package handlers + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" +) + +// validEgressHealthBody is a well-formed submission: the four scored classes +// sum to exactly ok_count/total_count, and the reputation figures sit outside +// them. Tests mutate a copy of this to exercise one rule at a time. +func validEgressHealthBody(clientId server.Id) map[string]any { + return map[string]any{ + "client_id": clientId.String(), + "ok_count": 25, + "total_count": 26, + "class_results": map[string]any{ + "dns": map[string]any{"ok": 4, "total": 4}, + "connectivity": map[string]any{"ok": 5, "total": 5}, + "cdn": map[string]any{"ok": 4, "total": 5}, + "site": map[string]any{"ok": 12, "total": 12}, + }, + "reputation_ok": 1, + "reputation_total": 4, + "failed_names": "cachefly", + "reputation_failed_names": "akamai,etsy,canva", + } +} + +func postEgressHealth(t testing.TB, secret string, body map[string]any) *httptest.ResponseRecorder { + t.Helper() + buf, err := json.Marshal(body) + if err != nil { + t.Fatalf("marshal body: %s", err) + } + req := httptest.NewRequest(http.MethodPost, "/network/provider-egress-health", bytes.NewReader(buf)) + if secret != "" { + req.Header.Set(operatorSecretHeader, secret) + } + w := httptest.NewRecorder() + ProviderEgressHealthResult(w, req) + return w +} + +func TestProviderEgressHealthResultRejectsMissingSecret(t *testing.T) { + w := postEgressHealth(t, "", validEgressHealthBody(server.NewId())) + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret header is absent", w.Code) + } +} + +// TestProviderEgressHealthResultRejectsWrongSecret proves hmac.Equal is +// actually consulted once the vault IS configured. Without a configured +// secret the handler takes the secret=="" short-circuit, and the missing-header +// test above would pass even if the comparison were never made. +func TestProviderEgressHealthResultRejectsWrongSecret(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + const wrongSecret = "correct-operator-secret-0123456780" // last char changed + defer withStubOperatorIngestSecret(secret)() + + w := postEgressHealth(t, wrongSecret, validEgressHealthBody(server.NewId())) + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the configured secret and the request's secret differ", w.Code) + } +} + +// TestProviderEgressHealthResultValidationRejections walks every rule that +// must fire BEFORE anything is written. The row is an upsert keyed on +// client_id, so an accepted bad submission does not sit beside the good one -- +// it destroys the last good measurement for that provider. +// +// Each case asserts no row was stored, not merely that the status was 400: a +// store-then-flag implementation would pass a status-only assertion. +func TestProviderEgressHealthResultValidationRejections(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + ctx := context.Background() + + cases := []struct { + name string + mutate func(body map[string]any) + }{ + { + // the headline rule: more destinations passed than ran + name: "ok_count exceeds total_count", + mutate: func(body map[string]any) { + body["ok_count"] = 27 + body["total_count"] = 26 + body["class_results"] = map[string]any{ + "dns": map[string]any{"ok": 27, "total": 26}, + } + }, + }, + { + name: "class results do not sum to the totals", + mutate: func(body map[string]any) { + // drops the site class from the map but not from the total + body["class_results"] = map[string]any{ + "dns": map[string]any{"ok": 4, "total": 4}, + "connectivity": map[string]any{"ok": 5, "total": 5}, + "cdn": map[string]any{"ok": 4, "total": 5}, + } + }, + }, + { + // {ok:5,total:2} + {ok:0,total:3} sums to a consistent 5/5 + // while describing a class where more passed than ran, so the + // aggregate check alone does not catch this + name: "a single class has ok above total while the sums agree", + mutate: func(body map[string]any) { + body["ok_count"] = 5 + body["total_count"] = 5 + body["class_results"] = map[string]any{ + "dns": map[string]any{"ok": 5, "total": 2}, + "site": map[string]any{"ok": 0, "total": 3}, + } + }, + }, + { + // reputation is not health: it must never arrive as a scored + // class, because the only ways to accept it are to break the + // sum check or to fold it into the score + name: "reputation smuggled in as a scored class", + mutate: func(body map[string]any) { + body["ok_count"] = 26 + body["total_count"] = 30 + body["class_results"] = map[string]any{ + "dns": map[string]any{"ok": 4, "total": 4}, + "connectivity": map[string]any{"ok": 5, "total": 5}, + "cdn": map[string]any{"ok": 4, "total": 5}, + "site": map[string]any{"ok": 12, "total": 12}, + "reputation": map[string]any{"ok": 1, "total": 4}, + } + }, + }, + { + name: "unknown field", + mutate: func(body map[string]any) { + body["okay_count"] = 25 + }, + }, + { + name: "negative count", + mutate: func(body map[string]any) { + body["ok_count"] = -1 + body["total_count"] = -1 + body["class_results"] = map[string]any{} + }, + }, + { + name: "negative reputation count", + mutate: func(body map[string]any) { + body["reputation_ok"] = -1 + }, + }, + { + name: "reputation ok exceeds reputation total", + mutate: func(body map[string]any) { + body["reputation_ok"] = 5 + body["reputation_total"] = 4 + }, + }, + { + name: "negative class count", + mutate: func(body map[string]any) { + body["ok_count"] = 0 + body["total_count"] = -1 + body["class_results"] = map[string]any{ + "dns": map[string]any{"ok": 0, "total": -1}, + } + }, + }, + { + name: "missing client id", + mutate: func(body map[string]any) { + delete(body, "client_id") + }, + }, + } + + for _, c := range cases { + // a fresh client id per case, so "no row stored" is unambiguous + clientId := server.NewId() + body := validEgressHealthBody(clientId) + c.mutate(body) + + w := postEgressHealth(t, secret, body) + if w.Code != http.StatusBadRequest { + t.Errorf("%s: status = %d, want 400; body = %s", c.name, w.Code, w.Body.String()) + continue + } + if health := model.GetProviderEgressHealth(ctx, clientId); health != nil { + t.Errorf("%s: a rejected submission stored a row anyway: %+v", c.name, health) + } + } + }) +} + +// TestProviderEgressHealthResultRejectsOKAboveTotal pins the top-level +// ok_count <= total_count rule specifically. +// +// It asserts the REASON, not just the 400, and that is the whole point of the +// test. Under the full rule set the aggregate check is defence in depth: any +// payload with ok_count > total_count whose classes sum exactly to those two +// figures must contain a class with ok > total, so removing the aggregate +// check still gets a 400 -- from the per-class rule, describing a different +// invariant. A status-only assertion therefore cannot tell the two apart and +// would pass with the rule deleted. +// +// The rule earns its place by being checked BEFORE the payload is decomposed: +// it is a statement about the submission as a whole, it holds for any future +// class set, and it fires whether or not the class map agrees with itself. +func TestProviderEgressHealthResultRejectsOKAboveTotal(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + ctx := context.Background() + + clientId := server.NewId() + body := validEgressHealthBody(clientId) + body["ok_count"] = 27 + body["total_count"] = 26 + body["class_results"] = map[string]any{ + "dns": map[string]any{"ok": 27, "total": 26}, + } + + w := postEgressHealth(t, secret, body) + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 when ok_count exceeds total_count; body = %s", w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), "ok_count must not exceed total_count") { + t.Fatalf("body = %q, want the rejection to come from the ok_count <= total_count rule", strings.TrimSpace(w.Body.String())) + } + if health := model.GetProviderEgressHealth(ctx, clientId); health != nil { + t.Fatalf("a rejected submission stored a row anyway: %+v", health) + } + }) +} + +// TestProviderEgressHealthResultStoresAValidRun is the accept path: a correct +// secret clears auth, a consistent payload passes validation, and the row +// lands with the reputation figures stored beside the health figures rather +// than inside them. +func TestProviderEgressHealthResultStoresAValidRun(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + ctx := context.Background() + + clientId := server.NewId() + w := postEgressHealth(t, secret, validEgressHealthBody(clientId)) + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body = %s", w.Code, w.Body.String()) + } + + health := model.GetProviderEgressHealth(ctx, clientId) + if health == nil { + t.Fatal("a 200 stored no row") + } + if health.OKCount != 25 || health.Total != 26 { + t.Errorf("ok/total = %d/%d, want 25/26", health.OKCount, health.Total) + } + // reputation stored... + if health.ReputationOK != 1 || health.ReputationTotal != 4 { + t.Errorf("reputation = %d/%d, want 1/4", health.ReputationOK, health.ReputationTotal) + } + if health.ReputationFailedNames != "akamai,etsy,canva" { + t.Errorf("reputation_failed_names = %q", health.ReputationFailedNames) + } + // ...and excluded from the health figures. 26/30 would be the shape of + // a reputation-folded-in regression. + if health.OKCount == 26 || health.Total == 30 { + t.Errorf("reputation was folded into ok/total: %d/%d", health.OKCount, health.Total) + } + if _, present := health.ClassResults["reputation"]; present { + t.Error("reputation was stored as a scored class") + } + if health.FailedNames != "cachefly" { + t.Errorf("failed_names = %q, want the scored failures only", health.FailedNames) + } + if got := health.ClassResults["cdn"]; got.OK != 4 || got.Total != 5 { + t.Errorf("class_results[cdn] = %+v, want 4/5", got) + } + if len(health.ClassResults) != 4 { + t.Errorf("class_results has %d classes, want 4", len(health.ClassResults)) + } + if health.MeasuredAt.IsZero() { + t.Error("measured_at was not stamped on arrival") + } + }) +} diff --git a/api/handlers/provider_egress_location_handlers.go b/api/handlers/provider_egress_location_handlers.go new file mode 100644 index 00000000..264da746 --- /dev/null +++ b/api/handlers/provider_egress_location_handlers.go @@ -0,0 +1,229 @@ +package handlers + +import ( + "crypto/hmac" + "encoding/json" + "io" + "net/http" + "strconv" + "sync" + + "github.com/urnetwork/glog" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/controller" + "github.com/urnetwork/server/model" +) + +// operatorSecretHeader carries the operator ingest secret. This endpoint is +// operator-to-server, not a client route: it is authenticated by a shared +// secret from the vault, not by a network jwt. +const operatorSecretHeader = "X-UR-Operator-Secret" + +// maxProviderEgressLocationBody bounds the request body. +const maxProviderEgressLocationBody = 16 * 1024 + +// operatorIngestSecret memoizes readOperatorIngestSecret for the life of the +// process. It is a package-level var (not a plain sync.OnceValue call site) +// so tests can swap it for a stub and restore it with defer; production code +// never reassigns it. +var operatorIngestSecret func() string = sync.OnceValue(readOperatorIngestSecret) + +// readOperatorIngestSecret reads the operator ingest secret from the vault +// resource "provider_egress.yml", key "ingest_secret". It returns "" +// when the vault resource is absent, the key is absent, or the key is empty, +// which makes the endpoint fail closed (every request is rejected) rather +// than open. `SimpleResource`/`String` are the non-panicking lookups (unlike +// `RequireSimpleResource`/`RequireString`), so a missing vault resource +// disables the endpoint instead of panicking the api process at startup or +// per-request. +func readOperatorIngestSecret() string { + res, err := server.Vault.SimpleResource("provider_egress.yml") + if err != nil { + glog.Infof("[pegl]no provider_egress.yml in the vault; ingest endpoint disabled\n") + return "" + } + values := res.String("ingest_secret") + if len(values) != 1 || values[0] == "" { + glog.Infof("[pegl]no ingest_secret in provider_egress.yml; ingest endpoint disabled\n") + return "" + } + return values[0] +} + +// ProviderEgressLocationSubmit ingests a probed provider egress location from +// the operator's prober. The prober routes geolocation lookups through a +// provider's own egress -- rather than relying on a lookup against the +// provider's control-connection ip -- and submits the result here so the +// server can prefer it over the built-in mmdb lookup. The route is +// operator-to-server, authenticated by the shared secret above rather than a +// network jwt. +func ProviderEgressLocationSubmit(w http.ResponseWriter, r *http.Request) { + secret := operatorIngestSecret() + provided := r.Header.Get(operatorSecretHeader) + if secret == "" || provided == "" || !hmac.Equal([]byte(secret), []byte(provided)) { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + body, err := io.ReadAll(io.LimitReader(r.Body, maxProviderEgressLocationBody+1)) + if err != nil { + http.Error(w, "Bad request", http.StatusBadRequest) + return + } + if len(body) > maxProviderEgressLocationBody { + http.Error(w, "Request too large", http.StatusRequestEntityTooLarge) + return + } + + var args controller.SubmitProviderEgressLocationArgs + if err := json.Unmarshal(body, &args); err != nil { + http.Error(w, "Bad request", http.StatusBadRequest) + return + } + + result, err := controller.SubmitProviderEgressLocation(r.Context(), &args) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(result); err != nil { + glog.Infof("[pegl]could not write response. err = %s\n", err) + } +} + +// ProviderEgressLocationAttempt records that the operator's prober tried to +// probe a provider, whether or not the try produced a location. +// +// The prober reports a *failure* here; a success is reported by +// ProviderEgressLocationSubmit above, whose provider_egress_location row +// already defers the provider for the full staleness window. Reporting a +// success here as well is harmless -- the attempt backoff is far shorter than +// that window -- but redundant. +// +// This exists because ProviderEgressLocationDue would otherwise be starved by +// providers that can never be probed successfully: they never get an egress +// row, so they sort to the head of the queue on every poll forever. See +// model.GetProviderEgressLocationDue. +// +// Same auth as the two endpoints around it: operator-to-server, the shared +// secret header rather than a network jwt, fail-closed when the vault resource +// is missing. +func ProviderEgressLocationAttempt(w http.ResponseWriter, r *http.Request) { + secret := operatorIngestSecret() + provided := r.Header.Get(operatorSecretHeader) + if secret == "" || provided == "" || !hmac.Equal([]byte(secret), []byte(provided)) { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + body, err := io.ReadAll(io.LimitReader(r.Body, maxProviderEgressLocationBody+1)) + if err != nil { + http.Error(w, "Bad request", http.StatusBadRequest) + return + } + if len(body) > maxProviderEgressLocationBody { + http.Error(w, "Request too large", http.StatusRequestEntityTooLarge) + return + } + + var args controller.RecordProviderEgressProbeAttemptArgs + if err := json.Unmarshal(body, &args); err != nil { + http.Error(w, "Bad request", http.StatusBadRequest) + return + } + + result, err := controller.RecordProviderEgressProbeAttempt(r.Context(), &args) + if err != nil { + http.Error(w, err.Error(), http.StatusBadRequest) + return + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(result); err != nil { + glog.Infof("[pegl]could not write response. err = %s\n", err) + } +} + +const ( + // defaultProviderEgressDueLimit is the batch size when the caller does not + // ask for one. + defaultProviderEgressDueLimit = 100 + // maxProviderEgressDueLimit bounds the batch size regardless of what the + // caller asks for, so one request cannot ask the database for the entire + // provider population. + maxProviderEgressDueLimit = 500 +) + +// providerEgressDueAge is how stale a stored probe must be before its provider +// is offered up for re-probing. It is deliberately shorter than +// model.ProviderEgressLocationMaxAge -- the age past which a stored location +// stops being trusted at all. If the two were equal, every location would lapse +// to the mmdb fallback at the exact moment it became due and stay lapsed until +// the prober worked its way around to it; at half the max age the prober has a +// full max-age/2 window to refresh a location before it expires. +const providerEgressDueAge = model.ProviderEgressLocationMaxAge / 2 + +// ProviderEgressLocationDueResult is the response body of +// ProviderEgressLocationDue. +type ProviderEgressLocationDueResult struct { + ClientIds []server.Id `json:"client_ids"` +} + +// ProviderEgressLocationDue tells the operator's prober which providers to +// probe next: those whose egress location has gone stale, and those that have +// never been probed at all, oldest first. +// +// This moves the probe schedule from the prober's memory into the database. +// The prober used to decide what to probe from an in-memory ttl cache, so a +// restart re-probed the whole population and nothing durable recorded what was +// actually due; observed_at already carries that information server-side, and +// this exposes it. +// +// A provider is skipped if it has a fresh success *or* a recent attempt. The +// second cutoff, ProviderEgressProbeAttemptBackoff, is much shorter than the +// first: a provider that failed to probe should be retried within hours, but +// must not be handed back on every poll, which is what would starve the rest of +// the queue (see ProviderEgressLocationAttempt above). +// +// Same auth as ProviderEgressLocationSubmit above: operator-to-server, the +// shared secret header rather than a network jwt, fail-closed when the vault +// resource is missing. +func ProviderEgressLocationDue(w http.ResponseWriter, r *http.Request) { + secret := operatorIngestSecret() + provided := r.Header.Get(operatorSecretHeader) + if secret == "" || provided == "" || !hmac.Equal([]byte(secret), []byte(provided)) { + http.Error(w, "Unauthorized", http.StatusUnauthorized) + return + } + + limit := defaultProviderEgressDueLimit + if raw := r.URL.Query().Get("limit"); raw != "" { + parsed, err := strconv.Atoi(raw) + if err != nil || parsed < 1 { + // not clamped up to 1: `limit=0` would come back as an empty list, + // which the prober cannot distinguish from "nothing is due" + http.Error(w, "Bad request", http.StatusBadRequest) + return + } + limit = min(parsed, maxProviderEgressDueLimit) + } + + // both cutoffs are computed here and passed as arguments; observed_at and + // attempt_at are naive timestamps holding utc, so comparing them to sql + // now() in the query would cast through the session timezone + now := server.NowUtc() + minObservedAt := now.Add(-providerEgressDueAge) + minAttemptAt := now.Add(-model.ProviderEgressProbeAttemptBackoff) + + result := &ProviderEgressLocationDueResult{ + ClientIds: model.GetProviderEgressLocationDue(r.Context(), minObservedAt, minAttemptAt, limit), + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(result); err != nil { + glog.Infof("[pegl]could not write response. err = %s\n", err) + } +} diff --git a/api/handlers/provider_egress_location_handlers_test.go b/api/handlers/provider_egress_location_handlers_test.go new file mode 100644 index 00000000..0f094ca0 --- /dev/null +++ b/api/handlers/provider_egress_location_handlers_test.go @@ -0,0 +1,552 @@ +package handlers + +import ( + "bytes" + "context" + "encoding/json" + "net/http" + "net/http/httptest" + "slices" + "strings" + "testing" + "time" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/controller" + "github.com/urnetwork/server/model" +) + +func TestProviderEgressLocationSubmitRejectsMissingSecret(t *testing.T) { + body, _ := json.Marshal(map[string]any{ + "client_id": "019f8835-158d-6fd8-e9dd-fd0e4c6d6792", + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-egress-location", bytes.NewReader(body)) + w := httptest.NewRecorder() + + ProviderEgressLocationSubmit(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret header is absent", w.Code) + } +} + +func TestProviderEgressLocationSubmitRejectsWrongSecret(t *testing.T) { + body, _ := json.Marshal(map[string]any{ + "client_id": "019f8835-158d-6fd8-e9dd-fd0e4c6d6792", + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-egress-location", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, "definitely-not-the-secret") + w := httptest.NewRecorder() + + ProviderEgressLocationSubmit(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 on a wrong operator secret", w.Code) + } +} + +// withStubOperatorIngestSecret swaps the package-level operatorIngestSecret +// memo for a stub that always returns secret, and returns a func to restore +// the original (real, still-memoized) reader. This lets a test exercise the +// "configured vault" path without the sync.OnceValue in the real reader ever +// touching the vault, and without the reject tests above (which rely on an +// unconfigured vault) observing any change. +func withStubOperatorIngestSecret(secret string) (restore func()) { + prev := operatorIngestSecret + operatorIngestSecret = func() string { return secret } + return func() { operatorIngestSecret = prev } +} + +// TestProviderEgressLocationSubmitAcceptsCorrectSecret proves the auth gate +// can ACCEPT a correct secret and hand off to the controller. Without this +// test, the two reject tests above (which both run with the vault +// unconfigured and take the secret=="" short-circuit) would pass unchanged +// even if the handler's entire body were replaced with an unconditional 401 - +// hmac.Equal would never be proven to run on a real match. +// +// Clearing auth hands the request to controller.SubmitProviderEgressLocation, +// which looks up the client in the database before it can return "Unknown +// client.", so this test needs a real (throwaway) test database - see +// server.DefaultTestEnv, the same harness +// controller/provider_egress_location_controller_test.go uses. t.Setenv makes +// it self-sufficient under a plain `go test`, matching the pattern in +// router/warp_handlers_status_test.go. +func TestProviderEgressLocationSubmitAcceptsCorrectSecret(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + // A syntactically valid, semantically unregistered submission: once + // auth clears, the controller looks up the client and (since it does + // not exist in the fresh test database) returns "Unknown client.", + // surfaced by the handler as 400. That 400 is proof the request + // reached the controller, i.e. proof auth passed. + args := controller.SubmitProviderEgressLocationArgs{ + ClientId: server.NewId(), + CountryCode: "US", + Country: "United States", + CountryConfident: true, + ObservedAt: server.NowUtc(), + } + body, err := json.Marshal(args) + if err != nil { + t.Fatalf("marshal args: %s", err) + } + + req := httptest.NewRequest(http.MethodPost, "/network/provider-egress-location", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderEgressLocationSubmit(w, req) + + if w.Code == http.StatusUnauthorized { + t.Fatalf("status = %d, want the correct secret to clear auth (not 401)", w.Code) + } + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 (Unknown client.) once auth clears for an unregistered client id; body = %s", w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), "Unknown client.") { + t.Fatalf("body = %q, want it to report the unknown client", w.Body.String()) + } + }) +} + +// TestProviderEgressLocationSubmitRejectsAlteredSecret proves that once the +// vault is configured, hmac.Equal is actually consulted rather than the +// endpoint accepting any request once secret != "". Same configured secret as +// the accept test above, but the request carries a one-character-altered +// value. +func TestProviderEgressLocationSubmitRejectsAlteredSecret(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + const wrongSecret = "correct-operator-secret-0123456780" // last char changed + defer withStubOperatorIngestSecret(secret)() + + body, _ := json.Marshal(map[string]any{ + "client_id": "019f8835-158d-6fd8-e9dd-fd0e4c6d6792", + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-egress-location", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, wrongSecret) + w := httptest.NewRecorder() + + ProviderEgressLocationSubmit(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret is configured but the request's secret is wrong", w.Code) + } +} + +// TestProviderEgressLocationSubmitReadsSecretFromVault proves the vault +// plumbing itself - not the test stub - returns the configured secret: +// readOperatorIngestSecret (the un-memoized reader) reads a +// PushSimpleResource-injected provider_egress.yml. +func TestProviderEgressLocationSubmitReadsSecretFromVault(t *testing.T) { + const secret = "vault-provisioned-secret-abcdef" + pop := server.Vault.PushSimpleResource( + "provider_egress.yml", + []byte(`ingest_secret: "`+secret+`"`), + ) + defer pop() + + if got := readOperatorIngestSecret(); got != secret { + t.Fatalf("readOperatorIngestSecret() = %q, want %q", got, secret) + } +} + +func TestProviderEgressLocationDueRejectsMissingSecret(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/network/provider-egress-due", nil) + w := httptest.NewRecorder() + + ProviderEgressLocationDue(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret header is absent", w.Code) + } +} + +func TestProviderEgressLocationDueRejectsWrongSecret(t *testing.T) { + req := httptest.NewRequest(http.MethodGet, "/network/provider-egress-due", nil) + req.Header.Set(operatorSecretHeader, "definitely-not-the-secret") + w := httptest.NewRecorder() + + ProviderEgressLocationDue(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 on a wrong operator secret", w.Code) + } +} + +// TestProviderEgressLocationDueRejectsAlteredSecret is the reject case with +// the vault *configured*, so the request gets past the secret == "" fail-closed +// short-circuit and hmac.Equal is what does the rejecting. +func TestProviderEgressLocationDueRejectsAlteredSecret(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + const wrongSecret = "correct-operator-secret-0123456780" // last char changed + defer withStubOperatorIngestSecret(secret)() + + req := httptest.NewRequest(http.MethodGet, "/network/provider-egress-due", nil) + req.Header.Set(operatorSecretHeader, wrongSecret) + w := httptest.NewRecorder() + + ProviderEgressLocationDue(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret is configured but the request's secret is wrong", w.Code) + } +} + +// testing_connectDueProvider stands up a connected + valid provider holding a +// Public provide key and no probe result, i.e. a provider the due query must +// return. The caller runs model.UpdateClientLocationReliabilities afterward. +func testing_connectDueProvider( + t testing.TB, + ctx context.Context, + clientId server.Id, + locationId server.Id, + clientAddress string, +) { + model.Testing_CreateDevice(ctx, server.NewId(), server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientAddress, handlerId) + if err != nil { + t.Fatalf("connect client: %s", err) + } + if err := model.SetConnectionLocation(ctx, connectionId, locationId, &model.ConnectionLocationScores{}); err != nil { + t.Fatalf("set connection location: %s", err) + } + model.SetProvide(ctx, clientId, map[model.ProvideMode][]byte{ + model.ProvideModePublic: []byte("provide-secret"), + }) +} + +// TestProviderEgressLocationDueAcceptsCorrectSecret proves the auth gate can +// ACCEPT. This is the test that gives the three reject tests above their +// meaning: without it, a handler whose entire body was replaced with an +// unconditional `http.Error(w, "Unauthorized", 401)` would still pass all +// three, because a suite that only ever asserts rejections cannot tell a +// working auth check from a broken-shut one. +// +// It deliberately asserts more than "not 401": a real, never-probed provider +// is stood up in the test database and must come back in the response body, so +// the test also fails if the handler clears auth but never reaches the model +// query or writes the wrong json shape. +func TestProviderEgressLocationDueAcceptsCorrectSecret(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + + city := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + model.CreateLocation(ctx, city) + + due := server.NewId() + testing_connectDueProvider(t, ctx, due, city.LocationId, "0.0.0.1:0") + model.UpdateClientLocationReliabilities(ctx, server.NowUtc().Add(-time.Hour), server.NowUtc()) + + req := httptest.NewRequest(http.MethodGet, "/network/provider-egress-due?limit=10", nil) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderEgressLocationDue(w, req) + + if w.Code == http.StatusUnauthorized { + t.Fatalf("status = %d, want the correct secret to clear auth (not 401)", w.Code) + } + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body = %s", w.Code, w.Body.String()) + } + + var result ProviderEgressLocationDueResult + if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { + t.Fatalf("decode body %q: %s", w.Body.String(), err) + } + if !slices.Contains(result.ClientIds, due) { + t.Fatalf("client_ids = %v, want it to contain the never-probed provider %s", result.ClientIds, due) + } + // the wire name the prober reads + if !strings.Contains(w.Body.String(), `"client_ids"`) { + t.Fatalf("body = %s, want a client_ids field", w.Body.String()) + } + }) +} + +// The prober asks for a batch; the server must not hand back more than asked. +func TestProviderEgressLocationDueHonoursLimit(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + + city := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + model.CreateLocation(ctx, city) + + testing_connectDueProvider(t, ctx, server.NewId(), city.LocationId, "0.0.0.1:0") + testing_connectDueProvider(t, ctx, server.NewId(), city.LocationId, "0.0.0.2:0") + model.UpdateClientLocationReliabilities(ctx, server.NowUtc().Add(-time.Hour), server.NowUtc()) + + req := httptest.NewRequest(http.MethodGet, "/network/provider-egress-due?limit=1", nil) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderEgressLocationDue(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body = %s", w.Code, w.Body.String()) + } + var result ProviderEgressLocationDueResult + if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { + t.Fatalf("decode body %q: %s", w.Body.String(), err) + } + if len(result.ClientIds) != 1 { + t.Fatalf("len(client_ids) = %d, want 1 for limit=1; body = %s", len(result.ClientIds), w.Body.String()) + } + }) +} + +// Every other due test in this file stands up never-probed providers, which +// come back regardless of what cutoff the handler computes -- so nothing here +// actually exercised providerEgressDueAge. This one does: a provider probed +// just now must be held back, and one probed past the cutoff must come through. +// Defeating the cutoff (dropping it, computing it in the wrong direction, +// comparing against sql now() through the session timezone) fails this. +func TestProviderEgressLocationDueHonoursStalenessCutoff(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + + city := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + model.CreateLocation(ctx, city) + + fresh := server.NewId() + stale := server.NewId() + testing_connectDueProvider(t, ctx, fresh, city.LocationId, "0.0.0.1:0") + testing_connectDueProvider(t, ctx, stale, city.LocationId, "0.0.0.2:0") + model.UpdateClientLocationReliabilities(ctx, server.NowUtc().Add(-time.Hour), server.NowUtc()) + + now := server.NowUtc() + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: fresh, LocationId: city.LocationId, + CountryCode: "us", ObservedAt: now, + }) + // comfortably past providerEgressDueAge, which is half + // model.ProviderEgressLocationMaxAge + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: stale, LocationId: city.LocationId, + CountryCode: "us", ObservedAt: now.Add(-providerEgressDueAge - time.Hour), + }) + + req := httptest.NewRequest(http.MethodGet, "/network/provider-egress-due?limit=100", nil) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderEgressLocationDue(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body = %s", w.Code, w.Body.String()) + } + var result ProviderEgressLocationDueResult + if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { + t.Fatalf("decode body %q: %s", w.Body.String(), err) + } + if slices.Contains(result.ClientIds, fresh) { + t.Fatalf("client_ids = %v, must not contain the just-probed provider %s", result.ClientIds, fresh) + } + if !slices.Contains(result.ClientIds, stale) { + t.Fatalf("client_ids = %v, must contain the provider probed past the cutoff %s", result.ClientIds, stale) + } + }) +} + +func TestProviderEgressLocationAttemptRejectsMissingSecret(t *testing.T) { + body, _ := json.Marshal(map[string]any{ + "client_id": "019f8835-158d-6fd8-e9dd-fd0e4c6d6792", + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-egress-attempt", bytes.NewReader(body)) + w := httptest.NewRecorder() + + ProviderEgressLocationAttempt(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret header is absent", w.Code) + } +} + +// The reject case with the vault *configured*, so the request gets past the +// secret == "" fail-closed short-circuit and hmac.Equal is what rejects. +func TestProviderEgressLocationAttemptRejectsAlteredSecret(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + const wrongSecret = "correct-operator-secret-0123456780" // last char changed + defer withStubOperatorIngestSecret(secret)() + + body, _ := json.Marshal(map[string]any{ + "client_id": "019f8835-158d-6fd8-e9dd-fd0e4c6d6792", + }) + req := httptest.NewRequest(http.MethodPost, "/network/provider-egress-attempt", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, wrongSecret) + w := httptest.NewRecorder() + + ProviderEgressLocationAttempt(w, req) + + if w.Code != http.StatusUnauthorized { + t.Fatalf("status = %d, want 401 when the operator secret is configured but the request's secret is wrong", w.Code) + } +} + +// The whole point of the attempt endpoint, end to end over http: a provider +// that has never been probed successfully is due; the prober reports that it +// tried and failed; the provider stops being due. Without that, a provider +// whose probes always fail sits at the head of the queue on every poll forever +// (observed_at IS NULL sorts first) and starves every provider behind it. +func TestProviderEgressLocationAttemptDefersProvider(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + ctx := context.Background() + + city := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + model.CreateLocation(ctx, city) + + dead := server.NewId() + testing_connectDueProvider(t, ctx, dead, city.LocationId, "0.0.0.1:0") + model.UpdateClientLocationReliabilities(ctx, server.NowUtc().Add(-time.Hour), server.NowUtc()) + + if !slices.Contains(due(t, secret), dead) { + t.Fatalf("the never-probed provider %s must be due before any attempt is reported", dead) + } + + attemptBody, err := json.Marshal(controller.RecordProviderEgressProbeAttemptArgs{ + ClientId: dead, + ProbeFailure: "tunnel_failed", + }) + if err != nil { + t.Fatalf("marshal attempt: %s", err) + } + req := httptest.NewRequest(http.MethodPost, "/network/provider-egress-attempt", bytes.NewReader(attemptBody)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderEgressLocationAttempt(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("status = %d, want 200; body = %s", w.Code, w.Body.String()) + } + + attempt := model.GetProviderEgressProbeAttempt(ctx, dead) + if attempt == nil { + t.Fatal("expected the attempt to be recorded") + } + if attempt.ProbeFailure != "tunnel_failed" { + t.Fatalf("probe_failure = %q, want %q", attempt.ProbeFailure, "tunnel_failed") + } + + if slices.Contains(due(t, secret), dead) { + t.Fatalf("the provider %s must not be due again immediately after a failed attempt", dead) + } + }) +} + +// due drives the due endpoint over http and returns the batch. +func due(t testing.TB, secret string) []server.Id { + req := httptest.NewRequest(http.MethodGet, "/network/provider-egress-due?limit=100", nil) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderEgressLocationDue(w, req) + + if w.Code != http.StatusOK { + t.Fatalf("due: status = %d, want 200; body = %s", w.Code, w.Body.String()) + } + var result ProviderEgressLocationDueResult + if err := json.Unmarshal(w.Body.Bytes(), &result); err != nil { + t.Fatalf("due: decode body %q: %s", w.Body.String(), err) + } + return result.ClientIds +} + +// An unknown client id must be rejected rather than writing an attempt row +// keyed to a client that does not exist, which nothing would ever read. +func TestProviderEgressLocationAttemptRejectsUnknownClient(t *testing.T) { + t.Setenv("WARP_ENV", "local") + server.DefaultTestEnv().Run(t, func(t testing.TB) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + body, err := json.Marshal(controller.RecordProviderEgressProbeAttemptArgs{ + ClientId: server.NewId(), + ProbeFailure: "tunnel_failed", + }) + if err != nil { + t.Fatalf("marshal attempt: %s", err) + } + req := httptest.NewRequest(http.MethodPost, "/network/provider-egress-attempt", bytes.NewReader(body)) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderEgressLocationAttempt(w, req) + + if w.Code == http.StatusUnauthorized { + t.Fatalf("status = %d, want the correct secret to clear auth (not 401)", w.Code) + } + if w.Code != http.StatusBadRequest { + t.Fatalf("status = %d, want 400 for an unregistered client id; body = %s", w.Code, w.Body.String()) + } + if !strings.Contains(w.Body.String(), "Unknown client.") { + t.Fatalf("body = %q, want it to report the unknown client", w.Body.String()) + } + }) +} + +// A limit that is not a positive integer is a caller bug. Silently clamping it +// to 1 (or to the default) would answer a question the prober did not ask -- +// `limit=0` would come back as an empty list, indistinguishable from "nothing +// is due" -- so it is rejected instead. +func TestProviderEgressLocationDueRejectsBadLimit(t *testing.T) { + const secret = "correct-operator-secret-0123456789" + defer withStubOperatorIngestSecret(secret)() + + for _, raw := range []string{"0", "-1", "abc", "1.5"} { + req := httptest.NewRequest(http.MethodGet, "/network/provider-egress-due?limit="+raw, nil) + req.Header.Set(operatorSecretHeader, secret) + w := httptest.NewRecorder() + + ProviderEgressLocationDue(w, req) + + if w.Code != http.StatusBadRequest { + t.Fatalf("limit=%q: status = %d, want 400", raw, w.Code) + } + } +} diff --git a/controller/network_client_controller.go b/controller/network_client_controller.go index 5ecd7252..3556cdb1 100644 --- a/controller/network_client_controller.go +++ b/controller/network_client_controller.go @@ -2,7 +2,8 @@ package controller import ( "context" - // "strings" + "net/netip" + "strings" "time" "github.com/urnetwork/glog" @@ -56,7 +57,80 @@ func SetConnectionLocation( connectionId server.Id, clientIp string, ) error { + // the mmdb lookup on the control ip. This is resolved up front even when + // the probed path is about to win, because the probed path has to know how + // precise the mmdb answer is before it can decide whether replacing it is + // an improvement (see probedLocationPreferred), and because it costs no db + // round trip -- it is an in-process maxminddb read plus an ARIN lookup. + // `err` is deliberately not returned yet: a failed mmdb lookup is not a + // reason to discard a perfectly good probed location. location, connectionLocationScores, err := GetLocationForIp(ctx, clientIp) + + // a provider probed through its own egress is located from that probe, not + // from a lookup on its control-connection ip: the egress is where user + // traffic actually exits, and an operator-run prober learns it by routing + // geolocation lookups through the provider itself and cross-checking them + // across several sources, then submits the result here. When a fresh + // probed entry exists we prefer it over the built-in mmdb lookup on the + // control ip -- subject to probedLocationPreferred, which is what stops the + // probe making a provider *less* locatable than it was. + // GetFreshProviderEgressLocationForConnection is + // a single query joining network_client_connection to + // provider_egress_location: this runs for every connection (provider or + // not) on the connect-announce path and inside a retry loop, so it must + // not cost the two round trips (client lookup, then egress lookup) the + // naive version would. + if egress := model.GetFreshProviderEgressLocationForConnection( + ctx, + connectionId, + model.ProviderEgressLocationMaxAge, + ); egress != nil && probedLocationPreferred(egress, location) { + scores := &model.ConnectionLocationScores{} + if egress.Hosting { + scores.NetTypeHosting = 1 + } + if egress.Proxy { + scores.NetTypePrivacy = 1 + } + // egress.Mobile deliberately does NOT feed NetTypeVirtual: unlike + // Hosting/Proxy, Mobile has no mmdb-path equivalent (IpInfo has no + // Mobile concept; NetTypeVirtual is set from the ipinfo schema's + // is_satellite field only, see GetLocationForIp, and never from + // DB-IP). Deriving NetTypeVirtual from Mobile here would penalize a + // probed mobile provider's ranking with no equivalent penalty for an + // otherwise-identical unprobed one -- the opposite of the parity + // this feature is meant to preserve (see arinForeignScore's doc for + // the same parity reasoning applied to net_type_foreign). Mobile + // stays on the model/wire contract as metadata; it just does not + // feed the ranking score. + + // keep the ARIN org-vs-country foreign check on the probed path too, + // so a probed provider is ranked on equal terms with an equivalent + // unprobed one (net_type_foreign feeds the ranking columns). It is + // computed exactly as the mmdb path computes it in GetLocationForIp: + // the ARIN org country of the control ip against the mmdb country of + // that SAME control ip -- not the probed country, which is a different + // question (whether probing changed the answer) and must not be + // silently folded into this ranking penalty. It is recomputed here + // rather than lifted off connectionLocationScores because that struct + // is nil whenever GetLocationForIp failed, including the case where + // mmdb resolved the ip fine but GuessLocationType could not classify + // the result -- the foreign check is still meaningful there. Any + // lookup failure just leaves NetTypeForeign at 0; it must never fail + // or panic this path. + if addr, err := netip.ParseAddr(clientIp); err == nil { + if ipInfo, err := server.GetIpInfo(addr); err == nil { + scores.NetTypeForeign = arinForeignScore(addr, ipInfo.CountryCode) + } + } + setErr := model.SetConnectionLocation(ctx, connectionId, egress.LocationId, scores) + if setErr == nil { + return nil + } + // fall through to the mmdb path on a storage error + glog.Infof("[ncc][%s]could not set probed egress location. err = %s\n", connectionId, setErr) + } + if err != nil { // server.Logger().Printf("Get ip for location error: %s", err) glog.Infof("[ncc][%s]could not find client location. err = %s\n", connectionId, err) @@ -73,6 +147,68 @@ func SetConnectionLocation( return nil } +// probedLocationPreferred decides whether the probed egress location should be +// written for this connection instead of the mmdb location resolved from the +// control ip. mmdbLocation is nil when the mmdb lookup failed. +// +// Read this before changing it: the naive rule -- "a probe is better evidence +// than mmdb, so the probe always wins" -- is wrong, and produced a live +// regression. The probed location is not always as *precise* as the mmdb one. +// SubmitProviderEgressLocation only stores a city when the probed city matches +// a location row that already exists; anything else is stored at country +// granularity, deliberately, so that a probe can never mint new city rows in +// the shared `location` table. Cities are not seeded either -- AddDefaultLocations +// runs with cityLimit = 0 -- so the pool a probed city can match against is only +// the rows organic traffic happened to create, and a country-granular fallback +// is the common case, not a rare one. +// +// Letting that country row overwrite an mmdb *city* row would drop the provider +// out of every city filter in FindProviders2 and GetProviderLocations. Being +// probed would make a provider less discoverable than never having been probed +// at all -- a penalty for participating. +// +// So the rule is: a probe may CORRECT the location, but never COARSEN it. +// +// - The probe stored a city (CityConfident): it is at least as precise as +// anything mmdb has, and it is better evidence. It wins. +// - No usable mmdb answer: the probe is the only evidence there is. It wins. +// - The mmdb answer is itself country-granular: nothing to lose. The probe +// wins, and this is the case country-level correction exists for. +// - The mmdb answer is city- or region-granular in a DIFFERENT country: the +// mmdb row is not more precise, it is precisely wrong, and its city is a +// city in the wrong country. The probe wins. This is the other half of +// country-level correction and the reason this is not just "keep whichever +// is finer". +// - The mmdb answer is city- or region-granular in the SAME country the probe +// reports: the probe agrees with mmdb and adds nothing except a loss of +// granularity. mmdb wins. +// +// CityConfident is used as the probed row's granularity because the schema +// invariant is that provider_egress_location.location_id is a city row exactly +// when city_confident is set (see the provider_egress_location migration and +// SubmitProviderEgressLocation). Reading it off the flag keeps this on the hot +// connect-announce path without a second query for the location row's type. +func probedLocationPreferred( + egress *model.ProviderEgressLocation, + mmdbLocation *model.Location, +) bool { + if egress.CityConfident { + return true + } + if mmdbLocation == nil { + return true + } + if mmdbLocation.LocationType != model.LocationTypeCity && + mmdbLocation.LocationType != model.LocationTypeRegion { + return true + } + // both country codes are stored lowercased -- SubmitProviderEgressLocation + // lowercases the probed one and GetLocationForIp takes mmdb's, which the + // location table also stores lowercased -- but fold anyway rather than let + // a casing difference read as a country disagreement and silently coarsen. + return !strings.EqualFold(egress.CountryCode, mmdbLocation.CountryCode) +} + /* func SetMissingConnectionLocations(ctx context.Context, minTime time.Time) { connectionIpStrs := map[server.Id]string{} diff --git a/controller/network_client_controller_test.go b/controller/network_client_controller_test.go new file mode 100644 index 00000000..1276bf9d --- /dev/null +++ b/controller/network_client_controller_test.go @@ -0,0 +1,543 @@ +package controller + +import ( + "context" + "testing" + "time" + + "github.com/urnetwork/connect" + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" +) + +// A provider with a fresh probed egress location must be located from that +// entry, not from the mmdb lookup on its control ip. +func TestSetConnectionLocationPrefersEgressLocation(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + // the probed egress location: japan + probed := &model.Location{ + LocationType: model.LocationTypeCountry, + Country: "Japan", + CountryCode: "jp", + } + model.CreateLocation(ctx, probed) + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, "8.8.8.8:0", handlerId) + connect.AssertEqual(t, err, nil) + + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: clientId, + LocationId: probed.LocationId, + CountryCode: "jp", + ObservedAt: server.NowUtc(), + }) + + err = SetConnectionLocation(ctx, connectionId, "8.8.8.8") + connect.AssertEqual(t, err, nil) + + var countryLocationId server.Id + server.Db(ctx, func(conn server.PgConn) { + result, qerr := conn.Query( + ctx, + `SELECT country_location_id FROM network_client_location WHERE connection_id = $1`, + connectionId, + ) + server.WithPgResult(result, qerr, func() { + if result.Next() { + server.Raise(result.Scan(&countryLocationId)) + } + }) + }) + connect.AssertEqual(t, countryLocationId, probed.CountryLocationId) + }) +} + +// With no probed entry, the existing mmdb path still applies. +func TestSetConnectionLocationFallsBackToMmdb(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, "8.8.8.8:0", handlerId) + connect.AssertEqual(t, err, nil) + + // no SetProviderEgressLocation call -> mmdb path + err = SetConnectionLocation(ctx, connectionId, "8.8.8.8") + connect.AssertEqual(t, err, nil) + + var count int + server.Db(ctx, func(conn server.PgConn) { + result, qerr := conn.Query( + ctx, + `SELECT COUNT(*) FROM network_client_location WHERE connection_id = $1`, + connectionId, + ) + server.WithPgResult(result, qerr, func() { + if result.Next() { + server.Raise(result.Scan(&count)) + } + }) + }) + connect.AssertEqual(t, count, 1) + }) +} + +// A probed egress location observed longer ago than ProviderEgressLocationMaxAge +// must be ignored, falling back to the mmdb lookup exactly as if there were no +// probed entry at all. +func TestSetConnectionLocationStaleProbedFallsBackToMmdb(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + clientIp := "8.8.8.8" + + // the mmdb location this ip actually resolves to; created up front so + // its canonical (deduped) location id is known for the assertion. + mmdbLocation, _, err := GetLocationForIp(ctx, clientIp) + connect.AssertEqual(t, err, nil) + model.CreateLocation(ctx, mmdbLocation) + + // a stale probed location, deliberately a different country than + // whatever the mmdb lookup returns, so a wrongly-preferred probed + // value would be caught by the assertion below. + probed := &model.Location{ + LocationType: model.LocationTypeCountry, + Country: "Japan", + CountryCode: "jp", + } + model.CreateLocation(ctx, probed) + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientIp+":0", handlerId) + connect.AssertEqual(t, err, nil) + + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: clientId, + LocationId: probed.LocationId, + CountryCode: "jp", + ObservedAt: server.NowUtc().Add(-(model.ProviderEgressLocationMaxAge + time.Hour)), + }) + + err = SetConnectionLocation(ctx, connectionId, clientIp) + connect.AssertEqual(t, err, nil) + + var countryLocationId server.Id + server.Db(ctx, func(conn server.PgConn) { + result, qerr := conn.Query( + ctx, + `SELECT country_location_id FROM network_client_location WHERE connection_id = $1`, + connectionId, + ) + server.WithPgResult(result, qerr, func() { + if result.Next() { + server.Raise(result.Scan(&countryLocationId)) + } + }) + }) + connect.AssertEqual(t, countryLocationId, mmdbLocation.CountryLocationId) + if countryLocationId == probed.CountryLocationId { + t.Fatal("stale probed location must not be used") + } + }) +} + +// A probed egress location whose location_id points at no existing location +// row makes the storage write fail (SetConnectionLocation returns an error +// rather than panicking, see model.SetConnectionLocation). The connection +// must still end up located via the mmdb path, and SetConnectionLocation +// itself must not panic or error out on this. +func TestSetConnectionLocationProbedWriteErrorFallsBackToMmdb(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + clientIp := "8.8.8.8" + + mmdbLocation, _, err := GetLocationForIp(ctx, clientIp) + connect.AssertEqual(t, err, nil) + model.CreateLocation(ctx, mmdbLocation) + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientIp+":0", handlerId) + connect.AssertEqual(t, err, nil) + + // a fresh probed entry pointing at a location id that was never + // created via CreateLocation -- the storage write in + // model.SetConnectionLocation must fail cleanly on this, not panic. + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: clientId, + LocationId: server.NewId(), + CountryCode: "jp", + ObservedAt: server.NowUtc(), + }) + + err = SetConnectionLocation(ctx, connectionId, clientIp) + connect.AssertEqual(t, err, nil) + + var countryLocationId server.Id + server.Db(ctx, func(conn server.PgConn) { + result, qerr := conn.Query( + ctx, + `SELECT country_location_id FROM network_client_location WHERE connection_id = $1`, + connectionId, + ) + server.WithPgResult(result, qerr, func() { + if result.Next() { + server.Raise(result.Scan(&countryLocationId)) + } + }) + }) + connect.AssertEqual(t, countryLocationId, mmdbLocation.CountryLocationId) + }) +} + +// net_type_foreign on the probed path must be computed the same way as the +// mmdb path: the ARIN org country of the control ip against the mmdb country +// of that SAME control ip, not against the probed country. 8.8.8.8 resolves +// (both via mmdb and ARIN org registration) to "us", so it is non-foreign by +// construction -- this is the same ip used by the parity tests above. The +// probed country here is deliberately "jp", a different country than the +// control ip's mmdb country: probing having changed the answer must not, by +// itself, flip net_type_foreign, or a probed provider is penalized a full +// ranking tier precisely for the reason this feature exists. A prior version +// of this code compared the ARIN org country against the probed country +// instead of the control ip's mmdb country, which made this exact scenario +// (egress country differs from control-ip country) foreign=1 instead of the +// correct foreign=0. +func TestSetConnectionLocationProbedNetTypeForeignMatchesMmdbParity(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + clientIp := "8.8.8.8" + + probed := &model.Location{ + LocationType: model.LocationTypeCountry, + Country: "Japan", + CountryCode: "jp", + } + model.CreateLocation(ctx, probed) + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientIp+":0", handlerId) + connect.AssertEqual(t, err, nil) + + // probed country ("jp") deliberately differs from the control ip's + // mmdb country ("us" for 8.8.8.8). + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: clientId, + LocationId: probed.LocationId, + CountryCode: "jp", + ObservedAt: server.NowUtc(), + }) + + err = SetConnectionLocation(ctx, connectionId, clientIp) + connect.AssertEqual(t, err, nil) + + var netTypeForeign int + server.Db(ctx, func(conn server.PgConn) { + result, qerr := conn.Query( + ctx, + `SELECT net_type_foreign FROM network_client_location WHERE connection_id = $1`, + connectionId, + ) + server.WithPgResult(result, qerr, func() { + if result.Next() { + server.Raise(result.Scan(&netTypeForeign)) + } + }) + }) + connect.AssertEqual(t, netTypeForeign, 0) + }) +} + +// A fresh probed location's Hosting/Proxy flags must map onto the stored +// connection's net_type_hosting/net_type_privacy scores. Mobile must NOT map +// onto net_type_virtual: Hosting/Proxy have direct mmdb-path equivalents +// (ipInfo.Hosting/ipInfo.Privacy, see GetLocationForIp), but Mobile has none +// (IpInfo has no Mobile concept at all, and NetTypeVirtual is only ever set +// from the ipinfo schema's is_satellite field, never from DB-IP or from +// anything Mobile-shaped) -- deriving NetTypeVirtual from Mobile would give +// a probed mobile provider a ranking penalty an identical unprobed mobile +// provider never takes, breaking the parity this feature promises. +func TestSetConnectionLocationMapsProbedFlagsToScores(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + probed := &model.Location{ + LocationType: model.LocationTypeCountry, + Country: "Japan", + CountryCode: "jp", + } + model.CreateLocation(ctx, probed) + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, "8.8.8.8:0", handlerId) + connect.AssertEqual(t, err, nil) + + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: clientId, + LocationId: probed.LocationId, + CountryCode: "jp", + Hosting: true, + Proxy: true, + Mobile: true, + ObservedAt: server.NowUtc(), + }) + + err = SetConnectionLocation(ctx, connectionId, "8.8.8.8") + connect.AssertEqual(t, err, nil) + + var netTypeHosting int + var netTypePrivacy int + var netTypeVirtual int + server.Db(ctx, func(conn server.PgConn) { + result, qerr := conn.Query( + ctx, + `SELECT net_type_hosting, net_type_privacy, net_type_virtual FROM network_client_location WHERE connection_id = $1`, + connectionId, + ) + server.WithPgResult(result, qerr, func() { + if result.Next() { + server.Raise(result.Scan(&netTypeHosting, &netTypePrivacy, &netTypeVirtual)) + } + }) + }) + connect.AssertEqual(t, netTypeHosting, 1) + connect.AssertEqual(t, netTypePrivacy, 1) + connect.AssertEqual(t, netTypeVirtual, 0) + }) +} + +// testing_connectionLocationIds reads back the granularity actually stored for +// a connection. +func testing_connectionLocationIds(ctx context.Context, connectionId server.Id) ( + cityLocationId server.Id, + regionLocationId server.Id, + countryLocationId server.Id, +) { + server.Db(ctx, func(conn server.PgConn) { + result, qerr := conn.Query( + ctx, + ` + SELECT city_location_id, region_location_id, country_location_id + FROM network_client_location + WHERE connection_id = $1 + `, + connectionId, + ) + server.WithPgResult(result, qerr, func() { + if result.Next() { + server.Raise(result.Scan( + &cityLocationId, + ®ionLocationId, + &countryLocationId, + )) + } + }) + }) + return +} + +// A probe must never make a provider LESS locatable than leaving it unprobed +// would have. +// +// SubmitProviderEgressLocation stores country granularity whenever the probed +// city does not match a location row that already exists, which is the common +// case -- cities are not seeded, so the match pool is only what organic traffic +// created. If that country row then overwrote the mmdb city row unconditionally, +// the provider would fall out of every city filter: probed providers would be +// harder to find than unprobed ones, which is the exact opposite of the point. +// +// mmdb resolves 24.48.0.1 to Montreal, Quebec, CA at city granularity. A +// country-only probe agreeing that the provider is in CA adds nothing, so the +// mmdb city has to survive. +func TestSetConnectionLocationProbedCountryDoesNotCoarsenMmdbCity(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + clientIp := "24.48.0.1" + + mmdbLocation, _, err := GetLocationForIp(ctx, clientIp) + connect.AssertEqual(t, err, nil) + // the fixture is only meaningful if mmdb really has a city here + connect.AssertEqual(t, mmdbLocation.LocationType, model.LocationTypeCity) + model.CreateLocation(ctx, mmdbLocation) + + // the probed location: the same country, but only the country -- + // exactly what an unmatched city name falls back to + probed := &model.Location{ + LocationType: model.LocationTypeCountry, + Country: "Canada", + CountryCode: "ca", + } + model.CreateLocation(ctx, probed) + connect.AssertEqual(t, probed.LocationId, mmdbLocation.CountryLocationId) + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientIp+":0", handlerId) + connect.AssertEqual(t, err, nil) + + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: clientId, + LocationId: probed.LocationId, + CountryCode: "ca", + CityConfident: false, + ObservedAt: server.NowUtc(), + }) + + err = SetConnectionLocation(ctx, connectionId, clientIp) + connect.AssertEqual(t, err, nil) + + cityLocationId, regionLocationId, countryLocationId := testing_connectionLocationIds(ctx, connectionId) + if cityLocationId != mmdbLocation.CityLocationId { + t.Errorf( + "city_location_id = %s, want the mmdb city %s: a country-only probe must not coarsen a connection the mmdb already placed in a city", + cityLocationId, + mmdbLocation.CityLocationId, + ) + } + connect.AssertEqual(t, regionLocationId, mmdbLocation.RegionLocationId) + connect.AssertEqual(t, countryLocationId, mmdbLocation.CountryLocationId) + // and the collapse this guards against, stated directly + if cityLocationId == countryLocationId { + t.Errorf("city/region/country all collapsed to %s; the provider is no longer in any city filter", countryLocationId) + } + }) +} + +// The other half of the rule: country-level CORRECTION still wins. When the +// probe says the egress is in a different country than the control ip's mmdb +// city, that city is a city in the wrong country -- it is not more precise, it +// is precisely wrong -- so the probed country replaces it. This is the whole +// reason the feature exists and must not be lost to the anti-coarsening rule +// above. +func TestSetConnectionLocationProbedCountryCorrectsMmdbCityInAnotherCountry(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + clientIp := "24.48.0.1" + + mmdbLocation, _, err := GetLocationForIp(ctx, clientIp) + connect.AssertEqual(t, err, nil) + connect.AssertEqual(t, mmdbLocation.LocationType, model.LocationTypeCity) + connect.AssertEqual(t, mmdbLocation.CountryCode, "ca") + model.CreateLocation(ctx, mmdbLocation) + + probed := &model.Location{ + LocationType: model.LocationTypeCountry, + Country: "Japan", + CountryCode: "jp", + } + model.CreateLocation(ctx, probed) + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientIp+":0", handlerId) + connect.AssertEqual(t, err, nil) + + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: clientId, + LocationId: probed.LocationId, + CountryCode: "jp", + CityConfident: false, + ObservedAt: server.NowUtc(), + }) + + err = SetConnectionLocation(ctx, connectionId, clientIp) + connect.AssertEqual(t, err, nil) + + _, _, countryLocationId := testing_connectionLocationIds(ctx, connectionId) + if countryLocationId != probed.CountryLocationId { + t.Errorf( + "country_location_id = %s, want the probed country %s: a probe that disagrees with mmdb about the country must still correct it", + countryLocationId, + probed.CountryLocationId, + ) + } + if countryLocationId == mmdbLocation.CountryLocationId { + t.Errorf("kept the mmdb country %s; the provider is still advertised in the wrong country", mmdbLocation.CountryLocationId) + } + }) +} + +// A city-confident probe is at least as precise as anything mmdb has and is +// better evidence, so it wins outright -- including against an mmdb city in +// another country. Nothing is coarsened, so the anti-coarsening rule does not +// apply. +func TestSetConnectionLocationProbedCityWinsOverMmdbCity(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + clientIp := "24.48.0.1" + + mmdbLocation, _, err := GetLocationForIp(ctx, clientIp) + connect.AssertEqual(t, err, nil) + connect.AssertEqual(t, mmdbLocation.LocationType, model.LocationTypeCity) + model.CreateLocation(ctx, mmdbLocation) + + probed := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Tokyo", + Region: "Tokyo", + Country: "Japan", + CountryCode: "jp", + } + model.CreateLocation(ctx, probed) + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientIp+":0", handlerId) + connect.AssertEqual(t, err, nil) + + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: clientId, + LocationId: probed.LocationId, + CountryCode: "jp", + CityConfident: true, + ObservedAt: server.NowUtc(), + }) + + err = SetConnectionLocation(ctx, connectionId, clientIp) + connect.AssertEqual(t, err, nil) + + cityLocationId, _, countryLocationId := testing_connectionLocationIds(ctx, connectionId) + connect.AssertEqual(t, cityLocationId, probed.CityLocationId) + connect.AssertEqual(t, countryLocationId, probed.CountryLocationId) + }) +} diff --git a/controller/network_client_location_controller.go b/controller/network_client_location_controller.go index 3376a9af..a8796b8b 100644 --- a/controller/network_client_location_controller.go +++ b/controller/network_client_location_controller.go @@ -55,21 +55,35 @@ func GetLocationForIp(ctx context.Context, clientIp string) (*model.Location, *m connectionLocationScores.NetTypeVirtual = 1 } + connectionLocationScores.NetTypeForeign = arinForeignScore(addr, ipInfo.CountryCode) + + return location, connectionLocationScores, nil +} + +// arinForeignScore cross-checks the ARIN org registration country for addr +// against countryCode. Both the ordinary path (GetLocationForIp) and the +// provider-egress path (SetConnectionLocation, in network_client_controller.go) pass the +// mmdb-resolved country of addr here, not the probed egress country: this is +// deliberate parity, so a probed and an unprobed provider on the same +// control ip are scored on the same basis and probing does not, by itself, +// change this ranking signal. If the org's registered country differs, the +// use case is considered foreign (VPN/proxy-like), matching the heuristic +// previously inlined in GetLocationForIp. +// +// If the ARIN lookup fails, this returns 0 without error: it must never fail +// or panic a caller on the connect-announce hot path over a missing/failed +// foreign check. +func arinForeignScore(addr netip.Addr, countryCode string) int { arinInfo, err := server.GetArinInfo(addr) - if err == nil { - // if the org ownership does not match the ip country, - // we consider the use case of the ip to be virtual - foreign := false - for _, orgCountryCode := range arinInfo.OrgCountryCodes { - if orgCountryCode != ipInfo.CountryCode { - foreign = true - break - } - } - if foreign { - connectionLocationScores.NetTypeForeign = 1 + if err != nil { + return 0 + } + // if the org ownership does not match the claimed country, + // we consider the use case of the ip to be foreign + for _, orgCountryCode := range arinInfo.OrgCountryCodes { + if orgCountryCode != countryCode { + return 1 } } - - return location, connectionLocationScores, nil + return 0 } diff --git a/controller/probed_location_preferred_test.go b/controller/probed_location_preferred_test.go new file mode 100644 index 00000000..33ac26a2 --- /dev/null +++ b/controller/probed_location_preferred_test.go @@ -0,0 +1,75 @@ +package controller + +import ( + "testing" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" +) + +// The decision rule on its own, with no database and no mmdb file: a probe may +// CORRECT a location but must never COARSEN it. The end-to-end behaviour is +// covered by the TestSetConnectionLocation* tests; this pins the rule itself so +// each clause is exercised independently of whichever mmdb the environment has. +// +// Every row here fails against at least one plausible wrong rule -- "the probe +// always wins" (the bug), or "keep whichever answer is finer" (the overcorrection +// that would throw away country correction). +func TestProbedLocationPreferred(t *testing.T) { + probedCity := func(countryCode string) *model.ProviderEgressLocation { + return &model.ProviderEgressLocation{ + LocationId: server.NewId(), + CountryCode: countryCode, + CityConfident: true, + } + } + probedCountry := func(countryCode string) *model.ProviderEgressLocation { + return &model.ProviderEgressLocation{ + LocationId: server.NewId(), + CountryCode: countryCode, + CityConfident: false, + } + } + mmdb := func(locationType string, countryCode string) *model.Location { + return &model.Location{ + LocationId: server.NewId(), + LocationType: locationType, + CountryCode: countryCode, + } + } + + tests := []struct { + name string + egress *model.ProviderEgressLocation + mmdb *model.Location + want bool + }{ + // the regression: a country-only probe must not replace an mmdb city + // in the same country. This is the row that fails against "the probe + // always wins". + {"country probe vs mmdb city, same country", probedCountry("ca"), mmdb(model.LocationTypeCity, "ca"), false}, + {"country probe vs mmdb region, same country", probedCountry("ca"), mmdb(model.LocationTypeRegion, "ca"), false}, + // casing must not read as a country disagreement + {"country probe vs mmdb city, same country cased", probedCountry("CA"), mmdb(model.LocationTypeCity, "ca"), false}, + + // country correction: these fail against "keep whichever is finer" + {"country probe vs mmdb city, other country", probedCountry("jp"), mmdb(model.LocationTypeCity, "ca"), true}, + {"country probe vs mmdb region, other country", probedCountry("jp"), mmdb(model.LocationTypeRegion, "ca"), true}, + + // nothing to lose + {"country probe vs mmdb country, same", probedCountry("ca"), mmdb(model.LocationTypeCountry, "ca"), true}, + {"country probe vs mmdb country, other", probedCountry("jp"), mmdb(model.LocationTypeCountry, "ca"), true}, + {"country probe vs no mmdb answer", probedCountry("jp"), nil, true}, + + // a city-confident probe is never a downgrade + {"city probe vs mmdb city, same country", probedCity("ca"), mmdb(model.LocationTypeCity, "ca"), true}, + {"city probe vs mmdb city, other country", probedCity("jp"), mmdb(model.LocationTypeCity, "ca"), true}, + {"city probe vs mmdb region", probedCity("ca"), mmdb(model.LocationTypeRegion, "ca"), true}, + {"city probe vs no mmdb answer", probedCity("jp"), nil, true}, + } + for _, test := range tests { + if got := probedLocationPreferred(test.egress, test.mmdb); got != test.want { + t.Errorf("%s: probedLocationPreferred = %v, want %v", test.name, got, test.want) + } + } +} diff --git a/controller/provider_egress_location_controller.go b/controller/provider_egress_location_controller.go new file mode 100644 index 00000000..b29feedc --- /dev/null +++ b/controller/provider_egress_location_controller.go @@ -0,0 +1,303 @@ +package controller + +import ( + "context" + "fmt" + "strings" + "time" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" + "github.com/urnetwork/server/probeverdict" +) + +// MaxProviderEgressLocationSubmissionAge rejects a submission whose probe is +// already older than this when it arrives. It bounds replay of an old probe. +const MaxProviderEgressLocationSubmissionAge = 24 * time.Hour + +// MaxProviderEgressLocationSubmissionSkew rejects a submission whose +// observed_at is further in the future than this. The prober and server +// clocks should be roughly in sync, so a few minutes of allowance covers +// ordinary clock drift without opening the door to a far-future timestamp. +// Without this bound, a future observed_at would defeat every other +// safeguard at once: it always wins the monotonic upsert in +// model.SetProviderEgressLocation (so no later, legitimate probe can ever +// overwrite it), it reads as "fresh" forever against +// ProviderEgressLocationMaxAge, and it outlives the taskworker sweep in +// RemoveExpiredProviderEgressLocations -- permanently pinning a provider to +// whatever location was submitted, with no API-side recovery. +const MaxProviderEgressLocationSubmissionSkew = 5 * time.Minute + +// maxLocationNameLen bounds country/city/region as submitted: these flow into +// model.CreateLocation, whose location_name column is varchar(128). Rejecting +// an over-long value here with a clear error is preferable to letting +// CreateLocation panic on a Postgres "value too long for type character +// varying(128)" error. +const maxLocationNameLen = 128 + +// maxOrgLen mirrors maxLocationNameLen for org, which is stored in +// provider_egress_location.org, a varchar(256) column. +const maxOrgLen = 256 + +type SubmitProviderEgressLocationArgs struct { + ClientId server.Id `json:"client_id"` + CountryCode string `json:"country_code"` + Country string `json:"country"` + Region string `json:"region,omitempty"` + City string `json:"city,omitempty"` + ASN int `json:"asn,omitempty"` + Org string `json:"org,omitempty"` + Hosting bool `json:"hosting,omitempty"` + Proxy bool `json:"proxy,omitempty"` + Mobile bool `json:"mobile,omitempty"` + CountryConfident bool `json:"country_confident"` + CityConfident bool `json:"city_confident,omitempty"` + ObservedAt time.Time `json:"observed_at"` +} + +type SubmitProviderEgressLocationResult struct { + LocationId server.Id `json:"location_id"` +} + +// providerEgressVerdict marshals a submission, and the row it is about to +// replace, into probeverdict's Input. It is the only place the two are joined; +// the rules themselves live in the probeverdict package and are not restated +// here. +// +// previous is the row currently stored for this provider, or nil when the +// provider has never been probed successfully. A nil previous leaves +// PreviousCountryCode empty, which probeverdict reads as "no history to +// contradict" -- a first probe is judged on its consensus alone. +// +// Note what is NOT passed: the mmdb country for the provider's control ip. A +// probed country that disagrees with mmdb is the finding this project exists to +// produce, never a fault, and probeverdict.Input structurally has no field for +// it. Do not add one. +func providerEgressVerdict( + args *SubmitProviderEgressLocationArgs, + previous *model.ProviderEgressLocation, +) probeverdict.Verdict { + in := probeverdict.Input{ + CountryConfident: args.CountryConfident, + // stored country codes are lowercased (see + // model.SetProviderEgressLocation), so normalize the submitted one the + // same way before comparing it against the stored history -- otherwise + // a prober sending "US" against a stored "us" reads as a country change + // and every second probe would be suspect. + CountryCode: strings.ToLower(strings.TrimSpace(args.CountryCode)), + // the server's clock, not the prober's: the age of the stored history + // is a server-side judgement, and args.ObservedAt is attacker-adjacent + // input. Its skew is already bounded, but the bound is a rejection + // rule, not a licence to measure history against it. + Now: server.NowUtc(), + } + if previous != nil { + in.PreviousCountryCode = previous.CountryCode + in.PreviousObservedAt = previous.ObservedAt + } + return probeverdict.Evaluate(in) +} + +// SubmitProviderEgressLocation records a probed egress location for a provider. +// Only country-confident submissions are accepted; city/region are stored only +// when the probe was also city-confident (free geolocation sources disagree on +// city often enough that an unconfirmed city is worse than none). +func SubmitProviderEgressLocation( + ctx context.Context, + args *SubmitProviderEgressLocationArgs, +) (*SubmitProviderEgressLocationResult, error) { + if !args.CountryConfident { + return nil, fmt.Errorf("Submission is not country-confident.") + } + countryCode := strings.ToLower(strings.TrimSpace(args.CountryCode)) + if len(countryCode) != 2 { + return nil, fmt.Errorf("Country code must be alpha-2.") + } + if args.ObservedAt.IsZero() { + return nil, fmt.Errorf("Missing observed_at.") + } + if args.ObservedAt.Before(server.NowUtc().Add(-MaxProviderEgressLocationSubmissionAge)) { + return nil, fmt.Errorf("Submission is too old.") + } + if server.NowUtc().Add(MaxProviderEgressLocationSubmissionSkew).Before(args.ObservedAt) { + return nil, fmt.Errorf("Submission is too far in the future.") + } + if networkId := model.GetNetworkClientNetwork(ctx, args.ClientId); networkId == nil { + return nil, fmt.Errorf("Unknown client.") + } + + // country is always used to resolve/create a location row (at minimum + // the country-granular one), and model.CreateLocation dedupes country + // rows on (location_type, country_code): an empty name here would create + // a canonical row with location_name='' that every later lookup for this + // country reuses forever, even after a subsequent real mmdb lookup. Reject + // rather than silently falling back, so the prober learns it sent a bad + // payload instead of the server permanently corrupting shared data. + country := strings.TrimSpace(args.Country) + if country == "" { + return nil, fmt.Errorf("Missing country.") + } + if maxLocationNameLen < len(country) { + return nil, fmt.Errorf("Country is too long.") + } + if maxOrgLen < len(args.Org) { + return nil, fmt.Errorf("Org is too long.") + } + + // city/region are only used (and their rows only created) when the probe + // was city-confident; the same empty-name corruption applies to them, so + // require both are present and reject rather than silently dropping to + // country granularity on a bad payload. + var city, region string + if args.CityConfident { + city = strings.TrimSpace(args.City) + region = strings.TrimSpace(args.Region) + if city == "" { + return nil, fmt.Errorf("Missing city for a city-confident submission.") + } + if region == "" { + return nil, fmt.Errorf("Missing region for a city-confident submission.") + } + if maxLocationNameLen < len(city) { + return nil, fmt.Errorf("City is too long.") + } + if maxLocationNameLen < len(region) { + return nil, fmt.Errorf("Region is too long.") + } + } + + // resolve to a location row. City granularity only when the probe agreed + // on a city AND that city already exists in the location table. + // + // The probe MUST NOT define new cities or regions. model.CreateLocation + // dedupes a city on its exact location_name, so an unrecognised spelling + // does not fail -- it silently inserts a new permanent row into the shared + // `location` table and adds it to the search index. The three free + // geolocation sources the prober reaches consensus over demonstrably + // disagree on spelling ("Frankfurt am Main (Innenstadt I)" vs "Frankfurt am + // Main" for the same host, observed), and the consensus keeps the winning + // source's original display string -- so "Frankfurt am Main", "Frankfurt Am + // Main" and "Frankfurt/Main" would each become their own row. Those rows + // survive a code revert and there is no cleanup path. + // + // model.MatchExistingLocation therefore matches only, never creates, + // case-insensitively and ignoring punctuation/whitespace/accents and + // parenthesised district qualifiers, so the ordinary variants -- the + // "(Innenstadt I)" case above included -- land on the row that is already + // there. When it does not resolve, + // this submission falls back to country granularity: country is the + // granularity this design treats as trustworthy anyway, and losing city + // precision for one probe is strictly better than permanently polluting a + // table shared with the provider list and the location search. + var location *model.Location + if args.CityConfident { + location = model.MatchExistingLocation(ctx, countryCode, region, city) + } + + // city_confident records the granularity of the row actually stored, not + // what the probe claimed. The schema's documented invariant is that + // location_id is a city row exactly when city_confident is set (see the + // provider_egress_location migration), and a city-confident probe whose + // city did not resolve is stored at country granularity. + cityConfident := location != nil + + if location == nil { + // country granularity. This still goes through CreateLocation: a + // country row is keyed on country_code, so a variant *name* can never + // produce a second row for the same country the way a variant city name + // can -- the pollution this guards against is not reachable here. The + // country row is also the whole point of the fallback, so a probe from + // a country not yet in the table must not be dropped. + location = &model.Location{ + LocationType: model.LocationTypeCountry, + Country: country, + CountryCode: countryCode, + } + model.CreateLocation(ctx, location) + } + + // judge the submission against the history it is about to replace. This is + // the only call site probeverdict has and the only place a verdict is + // computed: every geolocation submission already funnels through here, so + // verdicts fall out of the existing probe cadence with no separate + // scheduler and no separate endpoint. Before this, every row in the table + // read the column default `unverified` -- the absence of a judgement, which + // is indistinguishable from a judgement of "could not verify". + // + // The read is the only one: SetProviderEgressLocation below is an upsert, + // so the previous row has to be fetched before it is overwritten, and the + // verdict is the only thing that needs it. + previous := model.GetProviderEgressLocation(ctx, args.ClientId) + verdict := providerEgressVerdict(args, previous) + + model.SetProviderEgressLocation(ctx, &model.ProviderEgressLocation{ + ClientId: args.ClientId, + LocationId: location.LocationId, + CountryCode: countryCode, + ASN: args.ASN, + Org: args.Org, + Hosting: args.Hosting, + Proxy: args.Proxy, + Mobile: args.Mobile, + CityConfident: cityConfident, + ObservedAt: args.ObservedAt, + Verdict: verdict.State, + VerdictReason: verdict.Reason, + // assurance stays at the model default (`direct`): this probe reached + // the provider over a single tunnel from the prober. Multi-hop is P3. + }) + + return &SubmitProviderEgressLocationResult{LocationId: location.LocationId}, nil +} + +// maxProbeFailureLen bounds the failure class as submitted: +// provider_egress_probe_attempt.probe_failure is a varchar(64), and rejecting +// an over-long value with a clear error beats letting the insert panic on a +// Postgres "value too long" error and spin in the retry loop. +const maxProbeFailureLen = 64 + +type RecordProviderEgressProbeAttemptArgs struct { + ClientId server.Id `json:"client_id"` + // ProbeFailure is "" when the attempt succeeded, otherwise a short failure + // class (`contract_failed`, `tunnel_failed`, `no_consensus`, ...). + ProbeFailure string `json:"probe_failure,omitempty"` +} + +type RecordProviderEgressProbeAttemptResult struct { + AttemptAt time.Time `json:"attempt_at"` +} + +// RecordProviderEgressProbeAttempt records that the prober tried this provider. +// A failed attempt defers the provider from the due queue for +// ProviderEgressProbeAttemptBackoff, exactly as a successful probe defers it +// for the (much longer) staleness window -- without this, a provider that +// always fails to probe never gets a provider_egress_location row and so stays +// permanently at the head of the queue, starving every other provider. See +// model.GetProviderEgressLocationDue. +// +// The attempt is timestamped by the server, not the prober: the prober is +// reporting something it just did, and a prober whose clock ran fast could +// otherwise defer a provider far past the backoff window. +func RecordProviderEgressProbeAttempt( + ctx context.Context, + args *RecordProviderEgressProbeAttemptArgs, +) (*RecordProviderEgressProbeAttemptResult, error) { + if maxProbeFailureLen < len(args.ProbeFailure) { + return nil, fmt.Errorf("Probe failure class is too long.") + } + // same check as SubmitProviderEgressLocation: without it a typo'd or stale + // client id writes a row keyed to a client that does not exist, which + // nothing ever reads and only the sweep ever removes. + if networkId := model.GetNetworkClientNetwork(ctx, args.ClientId); networkId == nil { + return nil, fmt.Errorf("Unknown client.") + } + + attemptAt := server.NowUtc() + model.SetProviderEgressProbeAttempt(ctx, &model.ProviderEgressProbeAttempt{ + ClientId: args.ClientId, + AttemptAt: attemptAt, + ProbeFailure: args.ProbeFailure, + }) + + return &RecordProviderEgressProbeAttemptResult{AttemptAt: attemptAt}, nil +} diff --git a/controller/provider_egress_location_controller_test.go b/controller/provider_egress_location_controller_test.go new file mode 100644 index 00000000..efd591ec --- /dev/null +++ b/controller/provider_egress_location_controller_test.go @@ -0,0 +1,926 @@ +package controller + +import ( + "context" + "fmt" + "strings" + "testing" + "time" + + "github.com/urnetwork/connect" + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" +) + +func TestSubmitProviderEgressLocationCountryOnly(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + res, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "US", + Country: "United States", + ASN: 401486, + Org: "RAVNIX LLC", + Hosting: true, + CountryConfident: true, + ObservedAt: server.NowUtc(), + }) + connect.AssertEqual(t, err, nil) + if res.LocationId == (server.Id{}) { + t.Fatal("expected a resolved location id") + } + + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected the submission to be stored") + } + connect.AssertEqual(t, stored.CountryCode, "us") + connect.AssertEqual(t, stored.ASN, 401486) + connect.AssertEqual(t, stored.Hosting, true) + connect.AssertEqual(t, stored.CityConfident, false) + + // the resolved location must be the country-granular row, with no + // city/region association + loc := model.GetLocation(ctx, stored.LocationId) + if loc == nil { + t.Fatal("expected the resolved location row to exist") + } + connect.AssertEqual(t, loc.LocationType, model.LocationTypeCountry) + if loc.CityLocationId != (server.Id{}) { + t.Fatal("a country-granularity row must not have a city association") + } + if loc.RegionLocationId != (server.Id{}) { + t.Fatal("a country-granularity row must not have a region association") + } + }) +} + +func TestSubmitProviderEgressLocationRejectsNotCountryConfident(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "us", + CountryConfident: false, + ObservedAt: server.NowUtc(), + }) + if err == nil { + t.Fatal("a submission that is not country-confident must be rejected") + } + if model.GetProviderEgressLocation(ctx, clientId) != nil { + t.Fatal("rejected submission must not be stored") + } + }) +} + +func TestSubmitProviderEgressLocationRejectsUnknownClient(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: server.NewId(), // never created + CountryCode: "us", + CountryConfident: true, + ObservedAt: server.NowUtc(), + }) + if err == nil { + t.Fatal("unknown client_id must be rejected") + } + }) +} + +func TestSubmitProviderEgressLocationCityConfidentStoresCity(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + // the ingest path resolves against locations that ALREADY exist and + // never creates one, so the city has to be in the table first -- as it + // would be from the mmdb import + denver := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Denver", + Region: "Colorado", + Country: "United States", + CountryCode: "us", + } + model.CreateLocation(ctx, denver) + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "us", + Country: "United States", + Region: "Colorado", + City: "Denver", + CountryConfident: true, + CityConfident: true, + ObservedAt: server.NowUtc(), + }) + connect.AssertEqual(t, err, nil) + + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected the submission to be stored") + } + connect.AssertEqual(t, stored.CityConfident, true) + + // the resolved location must be the city-granular row that already + // existed, not a new one + connect.AssertEqual(t, stored.LocationId, denver.LocationId) + loc := model.GetLocation(ctx, stored.LocationId) + if loc == nil { + t.Fatal("expected the resolved location row to exist") + } + connect.AssertEqual(t, loc.LocationType, model.LocationTypeCity) + }) +} + +// An empty Country must be rejected, not silently stored: model.CreateLocation +// dedupes country rows on (location_type, country_code), so an empty name +// would create a canonical, permanently-blank row that every later lookup for +// that country reuses forever. +func TestSubmitProviderEgressLocationRejectsEmptyCountry(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "us", + Country: " ", // blank after trimming + CountryConfident: true, + ObservedAt: server.NowUtc(), + }) + if err == nil { + t.Fatal("an empty country must be rejected") + } + if model.GetProviderEgressLocation(ctx, clientId) != nil { + t.Fatal("rejected submission must not be stored") + } + }) +} + +// A city-confident submission with an empty City must be rejected rather than +// silently falling back to country granularity: the same empty-canonical-row +// corruption applies to city/region rows. +func TestSubmitProviderEgressLocationRejectsEmptyCityWhenCityConfident(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "us", + Country: "United States", + Region: "Colorado", + City: "", + CountryConfident: true, + CityConfident: true, + ObservedAt: server.NowUtc(), + }) + if err == nil { + t.Fatal("a city-confident submission with an empty city must be rejected") + } + if model.GetProviderEgressLocation(ctx, clientId) != nil { + t.Fatal("rejected submission must not be stored") + } + }) +} + +// A city-confident submission with an empty Region must likewise be rejected: +// the region row is created with the same dedupe-on-empty-name hazard. +func TestSubmitProviderEgressLocationRejectsEmptyRegionWhenCityConfident(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "us", + Country: "United States", + Region: "", + City: "Denver", + CountryConfident: true, + CityConfident: true, + ObservedAt: server.NowUtc(), + }) + if err == nil { + t.Fatal("a city-confident submission with an empty region must be rejected") + } + if model.GetProviderEgressLocation(ctx, clientId) != nil { + t.Fatal("rejected submission must not be stored") + } + }) +} + +// An over-long Country must be rejected with a clear error instead of +// panicking inside model.CreateLocation on a Postgres "value too long for +// type character varying(128)" error. +func TestSubmitProviderEgressLocationRejectsOverLongCountry(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "us", + Country: strings.Repeat("a", 129), + CountryConfident: true, + ObservedAt: server.NowUtc(), + }) + if err == nil { + t.Fatal("an over-long country must be rejected") + } + if model.GetProviderEgressLocation(ctx, clientId) != nil { + t.Fatal("rejected submission must not be stored") + } + }) +} + +// An over-long Org must likewise be rejected rather than panicking inside +// storage. +func TestSubmitProviderEgressLocationRejectsOverLongOrg(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "us", + Country: "United States", + Org: strings.Repeat("a", 257), + CountryConfident: true, + ObservedAt: server.NowUtc(), + }) + if err == nil { + t.Fatal("an over-long org must be rejected") + } + if model.GetProviderEgressLocation(ctx, clientId) != nil { + t.Fatal("rejected submission must not be stored") + } + }) +} + +func TestSubmitProviderEgressLocationRejectsStaleObservedAt(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "us", + CountryConfident: true, + ObservedAt: server.NowUtc().Add(-30 * 24 * time.Hour), + }) + if err == nil { + t.Fatal("a submission observed long ago must be rejected") + } + }) +} + +// A far-future observed_at must be rejected, not just an old one: unchecked, +// it would defeat the monotonic upsert (it always "wins"), read as fresh +// forever, and outlive the taskworker sweep -- permanently pinning a +// provider's location with no API-side recovery. See +// MaxProviderEgressLocationSubmissionSkew. +func TestSubmitProviderEgressLocationRejectsFutureObservedAt(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "jp", + Country: "Japan", + CountryConfident: true, + ObservedAt: server.NowUtc().Add(10 * 365 * 24 * time.Hour), + }) + if err == nil { + t.Fatal("a far-future observed_at must be rejected") + } + if model.GetProviderEgressLocation(ctx, clientId) != nil { + t.Fatal("rejected submission must not be stored") + } + }) +} + +// A submission within the allowed clock-skew window must still be accepted: +// the future-timestamp rejection must not be so strict that ordinary clock +// drift between the prober and server breaks legitimate submissions. +func TestSubmitProviderEgressLocationAcceptsWithinSkewObservedAt(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + res, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "us", + Country: "United States", + CountryConfident: true, + ObservedAt: server.NowUtc().Add(1 * time.Minute), + }) + connect.AssertEqual(t, err, nil) + if res.LocationId == (server.Id{}) { + t.Fatal("expected a resolved location id") + } + + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected the submission to be stored") + } + }) +} + +// A large 32-bit ASN (e.g. from the private-use range, common on +// hosting/VPN infrastructure) must round-trip cleanly, not panic. The asn +// column used to be `int` (Postgres int4, max ~2.147e9); ASNs are 32-bit +// unsigned (max ~4.295e9), so a value above int4's range panicked deep in +// pgx's arg encoding. +func TestSubmitProviderEgressLocationAcceptsLargeAsn(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + const largeAsn = 4200000000 + + res, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "us", + Country: "United States", + ASN: largeAsn, + CountryConfident: true, + ObservedAt: server.NowUtc(), + }) + connect.AssertEqual(t, err, nil) + if res.LocationId == (server.Id{}) { + t.Fatal("expected a resolved location id") + } + + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected the submission to be stored") + } + connect.AssertEqual(t, stored.ASN, largeAsn) + }) +} + +// testing_countLocations is the whole point of the two tests below: the +// `location` table is shared with the provider list and the location search, +// its rows are permanent, and nothing cleans up a bad one. An ingest endpoint +// that can add to it is an endpoint that can corrupt it from outside. +func testing_countLocations(ctx context.Context) int64 { + var count int64 + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query(ctx, `SELECT COUNT(*) FROM location`) + server.WithPgResult(result, err, func() { + if result.Next() { + server.Raise(result.Scan(&count)) + } + }) + }) + return count +} + +// A city-confident submission whose city is not already in the location table +// must fall back to country granularity and must NOT create a location row. +// +// model.CreateLocation dedupes a city on its exact location_name, so before +// this fix an unrecognised spelling did not fail -- it silently inserted a new +// permanent row and indexed it for search. The prober's consensus stores the +// winning source's original display string and the three free geolocation +// sources demonstrably disagree on spelling, so "Frankfurt am Main", +// "Frankfurt Am Main" and "Frankfurt/Main" would each have become their own +// row. Those rows outlive a code revert and there is no cleanup path. +// +// Reverting the MatchExistingLocation call in SubmitProviderEgressLocation must +// fail this test: the row count goes up and the stored location is a city. +func TestSubmitProviderEgressLocationUnknownCityDoesNotCreateALocation(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + // Germany, and one real German city, already exist -- as they would + // from the mmdb import. The submission below names a DIFFERENT city + // that has never been seen. + model.CreateLocation(ctx, &model.Location{ + LocationType: model.LocationTypeCity, + City: "Frankfurt am Main", + Region: "Hesse", + Country: "Germany", + CountryCode: "de", + }) + + before := testing_countLocations(ctx) + + res, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "de", + Country: "Germany", + Region: "Hesse", + City: "Kleinstadt Nirgendwo", + CountryConfident: true, + CityConfident: true, + ObservedAt: server.NowUtc(), + }) + connect.AssertEqual(t, err, nil) + + // nothing was added to the shared table + after := testing_countLocations(ctx) + if after != before { + t.Errorf("location row count went from %d to %d; an unmatched city must not create a permanent row in the shared location table", before, after) + } + + // and the submission was stored at country granularity instead + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected the submission to be stored") + } + loc := model.GetLocation(ctx, stored.LocationId) + if loc == nil { + t.Fatal("expected the resolved location row to exist") + } + if loc.LocationType != model.LocationTypeCountry { + t.Errorf("stored location_type = %q, want %q: an unmatched city must fall back to country granularity", loc.LocationType, model.LocationTypeCountry) + } + connect.AssertEqual(t, res.LocationId, stored.LocationId) + + // city_confident tracks the granularity actually stored, so the row + // stays internally consistent: location_id is a city row exactly when + // city_confident is set + if stored.CityConfident { + t.Error("city_confident must be false when the submission was stored at country granularity") + } + }) +} + +// The variants that matter are the ones the geolocation sources actually +// produce for one place: different case, punctuation or spacing, and -- the +// case this feature was built for -- a parenthesised district qualifier +// ("Frankfurt am Main (Innenstadt I)"). Those must resolve to the row that is +// already there; discarding them to country throws away real precision, and +// creating a row for each is the bug this guards against. +// +// An earlier revision of this list only contained foldings the implementation +// already handled, so it could not fail. The qualifier case below fails against +// a matcher that only drops punctuation, which is what makes this test worth +// running. Diacritics -- the other class that failed -- have their own test. +func TestSubmitProviderEgressLocationMatchesCitySpellingVariant(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), server.NewId(), "", "") + + frankfurt := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Frankfurt am Main", + Region: "Hesse", + Country: "Germany", + CountryCode: "de", + } + model.CreateLocation(ctx, frankfurt) + + before := testing_countLocations(ctx) + + // each of these is a real disagreement between the geolocation sources + // over the same place + variants := []struct{ region, city string }{ + {"Hesse", "Frankfurt am Main"}, // exact + {"Hesse", "Frankfurt Am Main"}, // case + {"hesse", "FRANKFURT AM MAIN"}, // case, both levels + {"Hesse", "Frankfurt-am-Main"}, // punctuation + {"Hesse", " Frankfurt am Main "}, + // the observed disagreement that motivated matching at all: one + // source appends the district. Dropping "(" and ")" as punctuation + // is not enough -- the qualifier's letters stay in the key -- so + // this misses unless the qualifier itself is stripped. + {"Hesse", "Frankfurt am Main (Innenstadt I)"}, + {"Hesse", "Frankfurt am Main (Innenstadt I) "}, + // qualifier on the region too + {"Hesse (Regierungsbezirk Darmstadt)", "Frankfurt am Main"}, + } + for _, variant := range variants { + label := fmt.Sprintf("%q/%q", variant.region, variant.city) + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "DE", + Country: "Germany", + Region: variant.region, + City: variant.city, + CountryConfident: true, + CityConfident: true, + ObservedAt: server.NowUtc(), + }) + connect.AssertEqual(t, err, nil) + + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatalf("%s: expected the submission to be stored", label) + } + if stored.LocationId != frankfurt.LocationId { + t.Errorf("%s resolved to %s, want the existing Frankfurt row %s", label, stored.LocationId, frankfurt.LocationId) + } + if !stored.CityConfident { + t.Errorf("%s: city_confident must stay set when the city resolved", label) + } + } + + if after := testing_countLocations(ctx); after != before { + t.Errorf("location row count went from %d to %d; spelling variants must reuse the existing row, not add new ones", before, after) + } + }) +} + +// Diacritics are the largest single class of spelling disagreement between the +// geolocation sources: one emits the local spelling ("São Paulo", "Zürich", +// "Kraków"), another an ASCII transliteration ("Sao Paulo", "Zurich", +// "Krakow"), and the mmdb import that seeded the existing rows picked one of +// the two per city with no way to know which. Every one of these missed before +// the NFD fold and fell back to country -- and, per +// TestSetConnectionLocationProbedCountryDoesNotCoarsenMmdbCity, a fallback on a +// provider the mmdb already placed in a city used to be an outright +// discoverability regression. +// +// Both directions are exercised, because which side carries the accent depends +// on how the existing row happened to be seeded. +func TestSubmitProviderEgressLocationMatchesAccentedCityVariant(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + + // existing rows, seeded the way the mmdb import would have: some + // accented, some already transliterated + saoPaulo := &model.Location{ + LocationType: model.LocationTypeCity, + City: "São Paulo", + Region: "São Paulo", + Country: "Brazil", + CountryCode: "br", + } + model.CreateLocation(ctx, saoPaulo) + + zurich := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Zürich", + Region: "Zürich", + Country: "Switzerland", + CountryCode: "ch", + } + model.CreateLocation(ctx, zurich) + + // seeded WITHOUT the accent, so the probe is the side carrying it + krakow := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Krakow", + Region: "Lesser Poland", + Country: "Poland", + CountryCode: "pl", + } + model.CreateLocation(ctx, krakow) + + before := testing_countLocations(ctx) + + variants := []struct { + countryCode string + country string + region string + city string + want *model.Location + }{ + // accent dropped by the probing source + {"BR", "Brazil", "Sao Paulo", "Sao Paulo", saoPaulo}, + {"CH", "Switzerland", "Zurich", "Zurich", zurich}, + // accent present in the probe, absent from the stored row + {"PL", "Poland", "Lesser Poland", "Kraków", krakow}, + // accent on one level only + {"BR", "Brazil", "São Paulo", "Sao Paulo", saoPaulo}, + // accent plus the other foldings at once + {"CH", "Switzerland", "zurich", " ZURICH ", zurich}, + } + for _, variant := range variants { + label := fmt.Sprintf("%q/%q", variant.region, variant.city) + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: variant.countryCode, + Country: variant.country, + Region: variant.region, + City: variant.city, + CountryConfident: true, + CityConfident: true, + ObservedAt: server.NowUtc(), + }) + connect.AssertEqual(t, err, nil) + + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatalf("%s: expected the submission to be stored", label) + } + if stored.LocationId != variant.want.LocationId { + t.Errorf("%s resolved to %s, want the existing row %s", label, stored.LocationId, variant.want.LocationId) + } + if !stored.CityConfident { + t.Errorf("%s: city_confident must stay set when the city resolved", label) + } + } + + if after := testing_countLocations(ctx); after != before { + t.Errorf("location row count went from %d to %d; accented variants must reuse the existing row, not add new ones", before, after) + } + }) +} + +// The qualifier-stripping pass is the only one that can pick the wrong row, so +// it must decline rather than guess. Two same-region rows that differ ONLY in +// their parenthesised qualifier both reduce to the same key; a probe carrying a +// third qualifier matches neither exactly and must fall back to country instead +// of silently landing on whichever row the candidate ordering put first. +func TestSubmitProviderEgressLocationAmbiguousQualifierFallsBackToCountry(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + springfieldIl := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Springfield (IL)", + Region: "Midwest", + Country: "United States", + CountryCode: "us", + } + model.CreateLocation(ctx, springfieldIl) + + springfieldMa := &model.Location{ + LocationType: model.LocationTypeCity, + City: "Springfield (MA)", + Region: "Midwest", + Country: "United States", + CountryCode: "us", + } + model.CreateLocation(ctx, springfieldMa) + + before := testing_countLocations(ctx) + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "US", + Country: "United States", + Region: "Midwest", + City: "Springfield (OH)", + CountryConfident: true, + CityConfident: true, + ObservedAt: server.NowUtc(), + }) + connect.AssertEqual(t, err, nil) + + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected the submission to be stored") + } + if stored.CityConfident { + t.Errorf("an ambiguous qualifier must not resolve to a city; got city_confident with location %s", stored.LocationId) + } + if stored.LocationId == springfieldIl.LocationId || stored.LocationId == springfieldMa.LocationId { + t.Errorf("guessed a Springfield (%s) instead of falling back to country", stored.LocationId) + } + // the country fallback goes through CreateLocation, but the US country + // row already exists (the two Springfield fixtures created it), and + // CreateLocation dedupes a country on country_code -- so declining to + // guess must add no rows at all, least of all a third Springfield + if after := testing_countLocations(ctx); after != before { + t.Errorf("location row count went from %d to %d; the fallback must reuse the existing country row and never create a city", before, after) + } + }) +} + +// The verdict tests below cover the wiring of the probeverdict package into +// this ingest path. probeverdict itself is table-tested in its own package; +// what is asserted here is that a submission actually reaches it and that its +// answer reaches the stored row -- before this wiring every row in production +// read the column default, `unverified`, which is the absence of a judgement +// rather than a judgement. + +// A clean, consensus-backed submission with no conflicting history must store +// `verified`, not the `unverified` column default. +func TestSubmitProviderEgressLocationVerdictVerified(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "US", + Country: "United States", + CountryConfident: true, + ObservedAt: server.NowUtc(), + }) + connect.AssertEqual(t, err, nil) + + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected the submission to be stored") + } + if stored.Verdict != "verified" { + t.Errorf("verdict = %q, want %q: a consensus-backed submission with no conflicting history is verified", stored.Verdict, "verified") + } + connect.AssertEqual(t, stored.VerdictReason, "") + // assurance is unrelated to the verdict and stays at its default + connect.AssertEqual(t, stored.Assurance, model.ProviderEgressAssuranceDirect) + }) +} + +// A country flip-flop inside probeverdict's instability window must store +// `suspect` with the `unstable` reason. +// +// The two submissions go backwards in time rather than forwards: +// MaxProviderEgressLocationSubmissionSkew rejects an observed_at more than five +// minutes in the future, so the earlier probe is placed two hours ago and the +// later one now. That keeps both inside the 24h submission-age bound, satisfies +// the monotonic observed_at upsert, and puts the gap well inside the +// instability window. +func TestSubmitProviderEgressLocationVerdictSuspectOnCountryFlipFlop(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + _, err := SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "US", + Country: "United States", + CountryConfident: true, + ObservedAt: server.NowUtc().Add(-2 * time.Hour), + }) + connect.AssertEqual(t, err, nil) + + first := model.GetProviderEgressLocation(ctx, clientId) + if first == nil { + t.Fatal("expected the first submission to be stored") + } + if first.Verdict != "verified" { + t.Fatalf("first verdict = %q, want %q; the fixture is only meaningful if the flip is a change", first.Verdict, "verified") + } + + // the same provider now probes from a different country + _, err = SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "JP", + Country: "Japan", + CountryConfident: true, + ObservedAt: server.NowUtc(), + }) + connect.AssertEqual(t, err, nil) + + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected the second submission to be stored") + } + connect.AssertEqual(t, stored.CountryCode, "jp") + if stored.Verdict != "suspect" { + t.Errorf("verdict = %q, want %q: a country change inside the instability window is a flip-flop", stored.Verdict, "suspect") + } + if stored.VerdictReason != "unstable" { + t.Errorf("verdict_reason = %q, want %q", stored.VerdictReason, "unstable") + } + }) +} + +// Absence of consensus is `unverified`/`no_consensus`. This is asserted against +// the marshalling helper rather than end-to-end because +// SubmitProviderEgressLocation rejects a submission that is not +// country-confident before anything is written (see +// TestSubmitProviderEgressLocationRejectsNotCountryConfident), so no row can +// ever be stored for one. The regression this guards against is the wiring +// hardcoding CountryConfident instead of passing the submission's own value +// through. +func TestSubmitProviderEgressLocationVerdictNoConsensus(t *testing.T) { + verdict := providerEgressVerdict(&SubmitProviderEgressLocationArgs{ + ClientId: server.NewId(), + CountryCode: "us", + CountryConfident: false, + ObservedAt: server.NowUtc(), + }, nil) + if verdict.State != "unverified" { + t.Errorf("state = %q, want %q", verdict.State, "unverified") + } + if verdict.Reason != "no_consensus" { + t.Errorf("reason = %q, want %q", verdict.Reason, "no_consensus") + } +} + +// The property this whole project exists for: a probed country that disagrees +// with what the free mmdb says about the provider's control ip is NOT +// suspicious. That divergence is the finding, not the fault -- a provider whose +// control connection looks Canadian while its egress is demonstrably Japanese +// is exactly the case probing was built to surface, and flagging it would +// invert the feature. +// +// probeverdict makes the omission structural: Input has no mmdb field at all, +// so there is no argument to toggle. The property is therefore asserted +// end-to-end -- give the client a connection from an ip the mmdb places in +// Canada, submit a stable, consensus-backed probe that says Japan, and require +// `verified`. +func TestSubmitProviderEgressLocationVerdictMmdbDivergenceIsNotSuspect(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + clientIp := "24.48.0.1" + + mmdbLocation, _, err := GetLocationForIp(ctx, clientIp) + connect.AssertEqual(t, err, nil) + // the fixture is only meaningful if mmdb disagrees with the probe below + connect.AssertEqual(t, mmdbLocation.CountryCode, "ca") + model.CreateLocation(ctx, mmdbLocation) + + networkId := server.NewId() + clientId := server.NewId() + model.Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := model.CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := model.ConnectNetworkClient(ctx, clientId, clientIp+":0", handlerId) + connect.AssertEqual(t, err, nil) + err = SetConnectionLocation(ctx, connectionId, clientIp) + connect.AssertEqual(t, err, nil) + + _, err = SubmitProviderEgressLocation(ctx, &SubmitProviderEgressLocationArgs{ + ClientId: clientId, + CountryCode: "JP", + Country: "Japan", + CountryConfident: true, + ObservedAt: server.NowUtc(), + }) + connect.AssertEqual(t, err, nil) + + stored := model.GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected the submission to be stored") + } + connect.AssertEqual(t, stored.CountryCode, "jp") + if stored.Verdict != "verified" { + t.Errorf( + "verdict = %q, want %q: the probed country (jp) differing from the mmdb country (%s) is the point of probing and must never be suspicious on its own", + stored.Verdict, + "verified", + mmdbLocation.CountryCode, + ) + } + if stored.VerdictReason != "" { + t.Errorf("verdict_reason = %q, want empty", stored.VerdictReason) + } + }) +} diff --git a/db_migrations.go b/db_migrations.go index 063f091e..ae9b81f5 100644 --- a/db_migrations.go +++ b/db_migrations.go @@ -4385,4 +4385,359 @@ var migrations = []any{ CREATE INDEX IF NOT EXISTS network_client_top_level_contract_time ON network_client (contract_time) WHERE (active = true AND source_client_id IS NULL AND contract_time IS NOT NULL) `), + + // provider egress locations: locations learned by an operator-run prober + // that routes geolocation lookups through a provider's own egress rather + // than trusting a lookup on the provider's control-connection ip, since + // the egress is where user traffic actually exits and can differ from + // where the provider's control connection originates (e.g. behind a VPN + // or hosting network). Keyed by client_id, one row per provider, upserted + // by the operator's prober. location_id is the canonical country (or + // city, when the probe was city-confident) location row. observed_at is + // when the probe ran, and is what freshness is judged against. + newSqlMigration(` + CREATE TABLE IF NOT EXISTS provider_egress_location ( + client_id uuid NOT NULL PRIMARY KEY, + location_id uuid NOT NULL, + country_code varchar(2) NOT NULL, + asn bigint NOT NULL DEFAULT 0, + org varchar(256) NOT NULL DEFAULT '', + hosting bool NOT NULL DEFAULT false, + proxy bool NOT NULL DEFAULT false, + mobile bool NOT NULL DEFAULT false, + city_confident bool NOT NULL DEFAULT false, + observed_at timestamp NOT NULL, + update_time timestamp NOT NULL + ) + `), + newSqlMigration(` + CREATE INDEX IF NOT EXISTS provider_egress_location_observed_at + ON provider_egress_location (observed_at) + `), + + // provider egress probe attempts: when the prober last *tried* a provider, + // successful or not, and how the try failed. + // + // This cannot live on provider_egress_location, because the case it exists + // to handle is precisely a provider that has no row there. A provider that + // connects, holds a Public provide key and fails every probe (firewalled + // egress, dead upstream) never gets an egress row, so its observed_at stays + // NULL, so it sorts to the head of the due queue forever. Enough of them and + // every batch the prober asks for is the same set of permanently-dead + // providers, and no healthy provider's location is ever refreshed -- while + // the endpoint keeps returning a full, plausible-looking batch. + // GetProviderEgressLocationDue defers on a recent attempt as well as a fresh + // success, which needs somewhere to record the attempt. + // + // Pulled forward from the P2 verdict model + // (docs/superpowers/specs/2026-07-25-enforced-provider-geo-probing-design.md, + // probe_attempt_at / probe_failure) because the P1 schedule cannot function + // without it. Deliberately only the two columns the schedule reads, not the + // rest of that model. + newSqlMigration(` + CREATE TABLE IF NOT EXISTS provider_egress_probe_attempt ( + client_id uuid NOT NULL PRIMARY KEY, + attempt_at timestamp NOT NULL, + probe_failure varchar(64) NOT NULL DEFAULT '', + update_time timestamp NOT NULL + ) + `), + + // serves the sweep in RemoveExpiredProviderEgressProbeAttempts. The due + // query reaches this table by primary key through the left join, so it + // needs no index of its own. + newSqlMigration(` + CREATE INDEX IF NOT EXISTS provider_egress_probe_attempt_attempt_at + ON provider_egress_probe_attempt (attempt_at) + `), + + // serves the stale-but-probed pass of GetProviderEgressLocationDue, which + // drives from provider_egress_location with `observed_at < $n ORDER BY + // observed_at, client_id LIMIT $m`. With client_id in the index the + // predicate and the whole ORDER BY -- tie-break included -- are one ordered + // index scan that stops when the batch is full: no sort, and no heap visit + // to resolve the tie. The pre-existing (observed_at) index alone leaves the + // client_id tie-break to a sort. + // + // The other pass (never-probed) needs no new index: it is an anti-join over + // network_client_location_reliability ordered by client_id, which the + // existing (valid, connected, client_id) index already serves as an ordered + // scan, and both anti-joins plus the provide_key EXISTS are primary-key + // probes. + // + // This supersedes provider_egress_location_observed_at, which is now a + // prefix of it -- including for the RemoveExpiredProviderEgressLocations + // sweep. The redundant index is left in place deliberately: dropping it is a + // separate decision with its own (small) risk, and this migration is meant + // to be purely additive. + // + // Appended, never inserted: migrations here apply by slice index, so + // editing or reordering an already-applied entry corrupts live databases. + newSqlMigration(` + CREATE INDEX IF NOT EXISTS provider_egress_location_observed_at_client_id + ON provider_egress_location (observed_at, client_id) + `), + + // The recorded judgement for a probed egress location: `verdict` is + // verified/unverified/suspect, `verdict_reason` the short failure class that + // produced it (see probeverdict), and `assurance` how the probe reached the + // provider (`direct` until multi-hop lands in P3). + // + // All three are additive with safe defaults, so every existing row reads as + // an unjudged direct probe and every existing reader of + // provider_egress_location is unaffected. Nothing writes a non-default + // verdict until the ingest path computes one. + // + // Appended, never inserted: migrations here apply by slice index + // (`for i := DbVersion(ctx); i < upTo; i++`), so editing or reordering an + // already-applied entry corrupts live databases. IF NOT EXISTS on every + // statement makes a re-run -- or a duplicated merge resolution -- a no-op. + newSqlMigration(` + ALTER TABLE provider_egress_location + ADD COLUMN IF NOT EXISTS verdict varchar(16) NOT NULL DEFAULT 'unverified', + ADD COLUMN IF NOT EXISTS verdict_reason varchar(64) NOT NULL DEFAULT '', + ADD COLUMN IF NOT EXISTS assurance varchar(16) NOT NULL DEFAULT 'direct' + `), + + // One measured throughput figure per provider, from either source (passive + // aggregation of settled bytes, or an active sampled download) -- consumers + // read one number and the `source` column says which produced it. + // + // client_id is the primary key, so a new measurement overwrites the old one + // (mirrors provider_egress_location's own shape). This is a ranking input, + // not a history: keeping every sample would grow without bound for a value + // only ever read as "the current figure". + // + // SUPERSEDED: a later migration in this file re-keys the table on + // (client_id, source) so each source keeps its own current figure. The + // statement below is left exactly as applied -- see that migration for why. + newSqlMigration(` + CREATE TABLE IF NOT EXISTS provider_bandwidth ( + client_id uuid NOT NULL PRIMARY KEY, + bytes_per_second double precision NOT NULL, + source varchar(16) NOT NULL, + sample_byte_count bigint NOT NULL, + window_start timestamp NOT NULL, + window_end timestamp NOT NULL, + update_time timestamp NOT NULL + ) + `), + + // serves staleness sweeps and "who needs a fresh measurement" scans, which + // range on window_end. Lookups of a single provider's figure go through the + // primary key and need no index of their own. + newSqlMigration(` + CREATE INDEX IF NOT EXISTS provider_bandwidth_window_end + ON provider_bandwidth (window_end) + `), + + // The deployment-wide byte budget for active bandwidth probing: one row per + // admitted reservation, counting against a fixed hourly bucket. See + // ReserveProviderBandwidthSlot for why active probe bytes need a spend + // limit at all -- they are real, paid contract traffic on any deployment + // where payouts are planned, regardless of the balance code used. + // + // byte_count is bigint, not int: a single bucket's budget is measured in + // hundreds of megabytes, so an int would overflow well inside the range + // this ledger has to sum over a day. client_id is stored for observability + // only -- the limit itself is global, not scoped per provider, so nothing + // reads this column to make an admission decision. + // + // Appended, never inserted: migrations here apply by slice index + // (`for i := DbVersion(ctx); i < upTo; i++`), so editing or reordering an + // already-applied entry corrupts live databases. IF NOT EXISTS on every + // statement makes a re-run -- or a duplicated merge resolution -- a no-op. + newSqlMigration(` + CREATE TABLE IF NOT EXISTS provider_bandwidth_quota ( + provider_bandwidth_quota_id uuid NOT NULL PRIMARY KEY, + client_id uuid NOT NULL, + byte_count bigint NOT NULL, + bucket_start timestamp NOT NULL, + create_time timestamp NOT NULL + ) + `), + + // every read of this table is a range over the lookahead window + // (`$1 <= bucket_start AND bucket_start < $2`), and the reaper deletes by + // the same column, so bucket_start is the only index it needs. + newSqlMigration(` + CREATE INDEX IF NOT EXISTS provider_bandwidth_quota_bucket_start + ON provider_bandwidth_quota (bucket_start) + `), + + // Re-key provider_bandwidth on (client_id, source). + // + // The active probe measures two independent targets per provider -- the + // operator's own download endpoint and a public CDN -- and the two figures + // are the point: a provider that prioritises one path and not the other is + // invisible in a single number and obvious in a pair. Keyed on client_id + // alone the two overwrite each other on every pass, so only one target can + // be stored at all. Averaging them into the one row would lose the same + // signal more quietly. + // + // The source becomes part of the key rather than each target getting its + // own columns, because a further target then needs no migration at all: + // 'passive', 'active-operator' and 'active-cdn' are three rows in the same + // shape, and a fourth would be a fourth row. + // + // No backfill: provider_bandwidth is empty on every deployment (nothing has + // written to it yet -- the active prober is the first writer and ships with + // this change), so re-keying cannot orphan or collide with an existing row. + // + // Appended, never inserted: migrations apply by slice index + // (`for i := DbVersion(ctx); i < upTo; i++`), so editing or reordering an + // already-applied entry corrupts live databases. DROP CONSTRAINT IF EXISTS + // paired with the ADD makes the whole statement idempotent -- a re-run + // drops whatever primary key is present and re-adds this one. + newSqlMigration(` + ALTER TABLE provider_bandwidth + DROP CONSTRAINT IF EXISTS provider_bandwidth_pkey, + ADD CONSTRAINT provider_bandwidth_pkey PRIMARY KEY (client_id, source) + `), + + // The latest egress-health run per provider: does this provider actually + // carry traffic to the real internet, across several independent classes + // of destination. The prober has computed this every pass since P2 and + // only ever logged it, so the signal rolls off with the container logs. + // + // Keyed on client_id alone, so a run replaces the previous one -- the + // current picture per provider, not a history, exactly as + // provider_egress_location behaves. Trending, if it is ever wanted, + // belongs in a separate partitioned append table rather than a second key + // column here. + // + // class_results is jsonb rather than a column per class because the class + // set is the prober's, not the schema's: adding a destination class must + // not need a migration, and the per-class tally is read as a diagnostic + // document ("dns=4/4 cdn=0/5 site=12/12" separates a datacenter-refusal + // from a blackhole) rather than filtered or aggregated on in sql. + // + // reputation_ok/reputation_total and reputation_failed_names are stored + // SEPARATELY from ok_count/total_count and must never be folded into them. + // The reputation class measures whether big vendors treat the exit ip as a + // datacenter address; nearly every honest hosted provider fails most of it + // because it IS hosted. Summing it into the health figure would score a + // provider that carried every byte it was asked for as partly broken. The + // ingest endpoint rejects a 'reputation' key inside class_results for the + // same reason. + // + // Appended, never inserted: migrations here apply by slice index + // (`for i := DbVersion(ctx); i < upTo; i++`), so editing or reordering an + // already-applied entry corrupts live databases. IF NOT EXISTS makes a + // re-run -- or a duplicated merge resolution -- a no-op. + newSqlMigration(` + CREATE TABLE IF NOT EXISTS provider_egress_health ( + client_id uuid NOT NULL, + measured_at timestamp NOT NULL, + ok_count int NOT NULL, + total_count int NOT NULL, + class_results jsonb NOT NULL, + reputation_ok int NOT NULL, + reputation_total int NOT NULL, + failed_names text NOT NULL DEFAULT '', + reputation_failed_names text NOT NULL DEFAULT '', + + PRIMARY KEY (client_id) + ) + `), + + // Blackhole verdicts reported by real clients: one row per report, per + // provider, per reporting network. A client that removes a provider for + // carrying nothing (see connect's detectBlackhole) says so here. + // + // APPEND-ONLY ON PURPOSE. A reporter may say anything as often as it likes; + // the cap is on what a reporter can COUNT FOR -- at most one verdict per + // reporter network per provider per aggregation window -- and it is applied + // at read time, in ProviderClientVerdictQuorumMet, never on the write path. + // Capping the writes instead would make the table lie about what was + // actually reported, and would put a rate-limit decision in front of the + // one signal that says a provider is dead. + // + // reporter_network_id comes from the authenticated session and never from + // the request body. It is the entire basis of the quorum: distinct networks + // are what a griefer has to buy, and a body-supplied reporter id would cost + // nothing at all. + // + // A met quorum only REPRIORITISES the provider for probing -- it never + // demotes, excludes, or touches filter sets, scores, PassesMinimums or + // find-providers2. See ProviderClientVerdictQuorumMet for why the trigger + // (client verdicts) and the punishment (the prober) are separated. + // + // syn_sent/syn_received are accepted and validated by the endpoint but + // deliberately not columns here: aggregation keys on receive_ack_count + // alone, and a column nothing reads is a column that drifts. + // + // Appended, never inserted: migrations here apply by slice index + // (`for i := DbVersion(ctx); i < upTo; i++`), so editing or reordering an + // already-applied entry corrupts live databases. IF NOT EXISTS makes a + // re-run -- or a duplicated merge resolution -- a no-op. + newSqlMigration(` + CREATE TABLE IF NOT EXISTS provider_client_verdict ( + provider_client_id uuid NOT NULL, + reporter_network_id uuid NOT NULL, + reason text NOT NULL, + send_ack_count bigint NOT NULL, + send_ack_bytes bigint NOT NULL, + receive_ack_count bigint NOT NULL, + receive_ack_bytes bigint NOT NULL, + window_seconds int NOT NULL, + create_time timestamp NOT NULL, + + PRIMARY KEY (provider_client_id, reporter_network_id, create_time) + ) + `), + + // serves the window read (GetProviderClientVerdictsInWindow), which is + // `provider_client_id = $1 AND $2 <= create_time ORDER BY create_time`. + // The primary key's leading column alone would find the provider's rows and + // then filter and sort every verdict ever written about it -- and this table + // is append-only and unbounded per reporter, so that set only grows. With + // create_time second the window is an ordered range scan that stops at the + // scan limit. + // + // reporter_network_id is deliberately not in the index: the read does not + // filter on it, and the one-verdict-per-reporter cap is applied in Go, not + // by the database. + newSqlMigration(` + CREATE INDEX IF NOT EXISTS provider_client_verdict_provider_create_time + ON provider_client_verdict (provider_client_id, create_time) + `), + + // the certificate pins this server has OBSERVED for the geolocation source + // hosts, by connecting to each host directly -- on the server's own + // network, no provider in the path -- and validating the chain under full + // WebPKI. See model.GeolocationSourcePin for why a direct, verified + // observation is the only thing that may ever write this table: a pin + // learned through a provider tunnel would let the provider under test teach + // the server its own forged certificate. + // + // One row per host, upserted: this is the current observation, not a + // history. A rotation is recorded in the refresh job's log line (old and + // new values), which is what was missing when the hardcoded pins went stale + // and silently took every source out of the consensus set. + // + // Both spki columns are NOT NULL and never written empty. An empty pin is + // not "no constraint", it is a pin that matches nothing, so a half-observed + // row would fail the prober closed for that host just as surely as a wrong + // one. The observation job leaves the previous row untouched rather than + // writing a partial one. + // + // No secondary index: the primary key on host serves both access paths -- + // the per-host upsert, and the unqualified read of every row (three rows, + // one per source host). + // + // Appended, never inserted: migrations here apply by slice index + // (`for i := DbVersion(ctx); i < upTo; i++`), so editing or reordering an + // already-applied entry corrupts live databases. IF NOT EXISTS makes a + // re-run -- or a duplicated merge resolution -- a no-op. + newSqlMigration(` + CREATE TABLE IF NOT EXISTS geolocation_source_pin ( + host text NOT NULL, + leaf_spki text NOT NULL, + intermediate_spki text NOT NULL, + observed_at timestamp NOT NULL, + + PRIMARY KEY (host) + ) + `), } diff --git a/go.mod b/go.mod index e4bddacf..284749b2 100644 --- a/go.mod +++ b/go.mod @@ -41,6 +41,7 @@ require ( golang.org/x/crypto v0.53.0 golang.org/x/net v0.56.0 golang.org/x/sys v0.46.0 + golang.org/x/text v0.40.0 golang.zx2c4.com/wireguard/wgctrl v0.0.0-20241231184526-a9ab2273dd10 google.golang.org/protobuf v1.36.11 gopkg.in/yaml.v3 v3.0.1 @@ -135,7 +136,6 @@ require ( golang.org/x/oauth2 v0.36.0 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/term v0.44.0 // indirect - golang.org/x/text v0.40.0 // indirect golang.org/x/time v0.15.0 // indirect gopkg.in/ini.v1 v1.67.2 // indirect src.agwa.name/tlshacks v0.0.3 // indirect diff --git a/model/geolocation_source_pin_model.go b/model/geolocation_source_pin_model.go new file mode 100644 index 00000000..10307d49 --- /dev/null +++ b/model/geolocation_source_pin_model.go @@ -0,0 +1,228 @@ +package model + +import ( + "context" + "time" + + "github.com/urnetwork/server" +) + +// GeolocationSourceHosts is the set of hosts the geolocation prober reaches +// through a provider tunnel, and therefore the set this server observes +// certificate pins for. +// +// # This list has a counterpart in another repository +// +// Its counterpart is `geolocate/sources.go` in the operator-proxy repo +// (github.com/urnetwork/urnetwork-operator-proxy), whose `sources` table is the +// authority on which endpoints are actually queried; `geolocate.SourceHosts()` +// derives the host set from it. The two CANNOT be linked in code: the prober is +// a separate Go module in a separate repository, and it depends on this server +// rather than the other way round -- importing the prober from the server would +// invert that dependency. So this is a deliberate second copy, and the comment +// is the only thing keeping it honest. When a source endpoint changes there, +// change it here in the same pass. +// +// # Drift is caught at runtime, fail-closed, not silently +// +// That is not merely a promise. The prober treats a source host with no served +// pin as a hard error and refuses to probe rather than probing unpinned, so a +// host added to `sources.go` but not to this list stops the prober at startup +// instead of quietly leaving one source unprotected. A host removed there but +// left here only costs a pointless observation. The dangerous direction is the +// one that fails loudly. +// +// This list is a compile-time constant on purpose: it is a trust decision about +// which hosts the server will vouch for, and nothing outside a code change -- +// no request, no database row, and above all no provider -- may add to it. +var GeolocationSourceHosts = []string{ + // ip.pn -- moved from ip.pn to api.i.pn on 2026-08-02, which is exactly + // the drift this comment block exists for. + "api.i.pn", + "free.freeipapi.com", + "ipinfo.io", +} + +// GeolocationSourcePin is the certificate pin this server observed for one +// geolocation source host, by connecting to it DIRECTLY -- on the server's own +// network, with no provider anywhere in the path -- and validating the chain +// under full WebPKI. +// +// That direct, validated observation is the entire basis for trusting the +// value. The geolocation lookup itself is issued THROUGH a provider's tunnel +// precisely so a provider forging its own location can be caught; if a pin +// could ever be learned from a connection that traversed a tunnel, or from one +// whose chain was not verified, the provider under test could teach this server +// its own forged certificate and the pin would authenticate the attacker. See +// the observation job for the code that upholds this. +// +// Both a leaf and an intermediate pin are recorded because the prober's check +// (providertunnel.checkPin) accepts a match anywhere in the verified chain: the +// intermediate is what absorbs routine leaf renewal without a redeploy, and the +// leaf is the tighter of the two while it lasts. +type GeolocationSourcePin struct { + Host string + LeafSpki string + IntermediateSpki string + ObservedAt time.Time +} + +// SetGeolocationSourcePin upserts the observed pin for one host and returns the +// row it REPLACED, or nil when the host had never been observed. +// +// Returning the previous row is what makes rotation visible. The outage this +// mechanism exists to prevent was invisible: a pin that had gone stale failed +// closed, the source dropped out of the set, and the fleet reported +// "no_consensus" -- indistinguishable from "fewer sources answered". The caller +// logs old and new from this return value, so a rotation leaves a record even +// though the table itself keeps no history. +// +// The read and the write are in one transaction so the returned "old" value is +// the one this write actually replaced, not whatever a separate earlier read +// happened to see. +// +// One row per host, replaced in place: this is the current picture, and history +// belongs in the log line rather than in a second key column that every reader +// would then have to filter on. +func SetGeolocationSourcePin(ctx context.Context, pin *GeolocationSourcePin) *GeolocationSourcePin { + var previous *GeolocationSourcePin + + server.Tx(ctx, func(tx server.PgTx) { + // server.Tx may rerun this closure, so start from a clean slate rather + // than carrying an abandoned attempt's reading forward. This does not + // cover the case where a commit succeeded but was REPORTED as failed: + // the rerun's select then finds the row it just wrote and the rotation + // goes unlogged once. That is rare, self-corrects at the next pass + // (the certificate genuinely has not changed by then), and fixing it + // properly means changing server.Tx's retry semantics for everyone. + previous = nil + + result, err := tx.Query( + ctx, + ` + SELECT + leaf_spki, + intermediate_spki, + observed_at + FROM geolocation_source_pin + WHERE host = $1 + FOR UPDATE + `, + pin.Host, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + p := &GeolocationSourcePin{Host: pin.Host} + server.Raise(result.Scan( + &p.LeafSpki, + &p.IntermediateSpki, + &p.ObservedAt, + )) + previous = p + } + }) + + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO geolocation_source_pin ( + host, + leaf_spki, + intermediate_spki, + observed_at + ) + VALUES ($1, $2, $3, $4) + ON CONFLICT (host) DO UPDATE + SET + leaf_spki = $2, + intermediate_spki = $3, + observed_at = $4 + `, + pin.Host, + pin.LeafSpki, + pin.IntermediateSpki, + // naive timestamp column holding utc, as everywhere else in this + // schema + pin.ObservedAt.UTC(), + )) + }) + + return previous +} + +// GetGeolocationSourcePin reads one host's observed pin, or nil when the host +// has never been successfully observed. +// +// Never observed is not the same as observed-and-empty: a caller that could not +// tell those apart would be one step from serving an empty pin set, which is +// the same as serving no pin at all. +func GetGeolocationSourcePin(ctx context.Context, host string) *GeolocationSourcePin { + var pin *GeolocationSourcePin + + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT + leaf_spki, + intermediate_spki, + observed_at + FROM geolocation_source_pin + WHERE host = $1 + `, + host, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + p := &GeolocationSourcePin{Host: host} + server.Raise(result.Scan( + &p.LeafSpki, + &p.IntermediateSpki, + &p.ObservedAt, + )) + pin = p + } + }) + }) + + return pin +} + +// GetGeolocationSourcePins reads every observed pin, keyed by host. +// +// It returns exactly what has been observed and never synthesizes a row for a +// host in GeolocationSourceHosts that has not been observed yet. A missing host +// must stay missing all the way to the consumer, because the consumer's correct +// response to a missing pin is to refuse to probe -- and a placeholder row here +// would take that decision away from it. +func GetGeolocationSourcePins(ctx context.Context) map[string]*GeolocationSourcePin { + pins := map[string]*GeolocationSourcePin{} + + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT + host, + leaf_spki, + intermediate_spki, + observed_at + FROM geolocation_source_pin + `, + ) + server.WithPgResult(result, err, func() { + for result.Next() { + p := &GeolocationSourcePin{} + server.Raise(result.Scan( + &p.Host, + &p.LeafSpki, + &p.IntermediateSpki, + &p.ObservedAt, + )) + pins[p.Host] = p + } + }) + }) + + return pins +} diff --git a/model/geolocation_source_pin_model_test.go b/model/geolocation_source_pin_model_test.go new file mode 100644 index 00000000..7a66ceee --- /dev/null +++ b/model/geolocation_source_pin_model_test.go @@ -0,0 +1,162 @@ +package model + +import ( + "context" + "os" + "path/filepath" + "regexp" + "sort" + "testing" + "time" + + "github.com/urnetwork/connect" + + "github.com/urnetwork/server" +) + +func TestSetGeolocationSourcePinStoresAndReplacesPerHost(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + observedAt := server.NowUtc().Truncate(time.Millisecond) + + // first observation of a host: nothing replaced + previous := SetGeolocationSourcePin(ctx, &GeolocationSourcePin{ + Host: "ipinfo.io", + LeafSpki: "leaf-ipinfo-1", + IntermediateSpki: "int-ipinfo-1", + ObservedAt: observedAt, + }) + if previous != nil { + t.Fatalf("expected no previous pin on first observation, got %+v", previous) + } + + // a second host is stored independently + SetGeolocationSourcePin(ctx, &GeolocationSourcePin{ + Host: "api.i.pn", + LeafSpki: "leaf-ipn-1", + IntermediateSpki: "int-ipn-1", + ObservedAt: observedAt, + }) + + stored := GetGeolocationSourcePin(ctx, "ipinfo.io") + if stored == nil { + t.Fatal("expected a stored pin for ipinfo.io, got nil") + } + connect.AssertEqual(t, stored.Host, "ipinfo.io") + connect.AssertEqual(t, stored.LeafSpki, "leaf-ipinfo-1") + connect.AssertEqual(t, stored.IntermediateSpki, "int-ipinfo-1") + connect.AssertEqual(t, stored.ObservedAt.UTC().Equal(observedAt.UTC()), true) + + // re-observing one host REPLACES that host's row and returns what it + // replaced -- the return value is what makes a rotation loggable with + // both old and new + rotatedAt := observedAt.Add(6 * time.Hour) + previous = SetGeolocationSourcePin(ctx, &GeolocationSourcePin{ + Host: "ipinfo.io", + LeafSpki: "leaf-ipinfo-2", + IntermediateSpki: "int-ipinfo-2", + ObservedAt: rotatedAt, + }) + if previous == nil { + t.Fatal("expected the replaced pin to be returned, got nil") + } + connect.AssertEqual(t, previous.LeafSpki, "leaf-ipinfo-1") + connect.AssertEqual(t, previous.IntermediateSpki, "int-ipinfo-1") + + pins := GetGeolocationSourcePins(ctx) + connect.AssertEqual(t, len(pins), 2) + connect.AssertEqual(t, pins["ipinfo.io"].LeafSpki, "leaf-ipinfo-2") + connect.AssertEqual(t, pins["ipinfo.io"].IntermediateSpki, "int-ipinfo-2") + // replacing one host must not touch another: a per-host write that + // disturbed its neighbours would be the same class of bug as a failing + // host blanking the whole set + connect.AssertEqual(t, pins["api.i.pn"].LeafSpki, "leaf-ipn-1") + connect.AssertEqual(t, pins["api.i.pn"].IntermediateSpki, "int-ipn-1") + + // a host that has never been observed reads as absent, not as an empty + // pin -- the consumer's correct response to absent is to refuse to + // probe, and a zero-valued row would take that decision away from it + if GetGeolocationSourcePin(ctx, "free.freeipapi.com") != nil { + t.Fatal("expected nil for a host that has never been observed") + } + if _, ok := pins["free.freeipapi.com"]; ok { + t.Fatal("expected an unobserved host to be absent from the pin map") + } + }) +} + +var sourceUrlPattern = regexp.MustCompile(`URL:\s*"https://([^/"]+)`) + +// TestGeolocationSourceHostsMatchProberSourcesWhenCheckedOutAlongside is the +// only mechanism available for linking GeolocationSourceHosts to its +// counterpart in the operator-proxy repo. The prober is a separate Go module in +// a separate repository that depends on this server, so the list genuinely +// cannot be imported -- see the GeolocationSourceHosts comment. +// +// What this covers: a machine that has both repositories checked out as +// siblings (or names one via URNETWORK_OPERATOR_PROXY) will fail this test the +// moment the two lists disagree, which is the case that matters -- whoever +// changes a source endpoint is working in both trees. +// +// What it does NOT cover: CI, or any checkout without the prober beside it. It +// skips there rather than passing vacuously. The real backstop for drift is at +// runtime and fails closed: the prober treats a source host with no served pin +// as a hard error and refuses to probe. +func TestGeolocationSourceHostsMatchProberSourcesWhenCheckedOutAlongside(t *testing.T) { + candidates := []string{ + os.Getenv("URNETWORK_OPERATOR_PROXY"), + filepath.Join("..", "..", "urnetwork-operator-proxy"), + } + + var sourcesPath string + for _, candidate := range candidates { + if candidate == "" { + continue + } + path := filepath.Join(candidate, "geolocate", "sources.go") + if _, err := os.Stat(path); err == nil { + sourcesPath = path + break + } + } + if sourcesPath == "" { + t.Skip("operator-proxy checkout not found beside this one; set URNETWORK_OPERATOR_PROXY to enable this check") + } + + b, err := os.ReadFile(sourcesPath) + if err != nil { + t.Fatalf("read %s: %v", sourcesPath, err) + } + + matches := sourceUrlPattern.FindAllStringSubmatch(string(b), -1) + if len(matches) == 0 { + // the file exists but no longer looks the way this test assumes. Fail + // rather than skip: a silently-inert drift check is worse than none, + // because it reads as coverage. + t.Fatalf("no source URLs found in %s -- the prober's source table has changed shape and this check needs updating", sourcesPath) + } + + seen := map[string]bool{} + proberHosts := []string{} + for _, match := range matches { + host := match[1] + if seen[host] { + continue + } + seen[host] = true + proberHosts = append(proberHosts, host) + } + + serverHosts := append([]string{}, GeolocationSourceHosts...) + sort.Strings(proberHosts) + sort.Strings(serverHosts) + + if len(proberHosts) != len(serverHosts) { + t.Fatalf("host lists disagree: prober %v (%s), server %v", proberHosts, sourcesPath, serverHosts) + } + for i := range proberHosts { + if proberHosts[i] != serverHosts[i] { + t.Fatalf("host lists disagree: prober %v (%s), server %v", proberHosts, sourcesPath, serverHosts) + } + } +} diff --git a/model/network_client_location_model.go b/model/network_client_location_model.go index a2650fc4..811cc80e 100644 --- a/model/network_client_location_model.go +++ b/model/network_client_location_model.go @@ -1242,6 +1242,36 @@ func SetConnectionLocation( } }) + // geo databases vary in how deep their coverage goes: many IPs -- + // datacenter, mobile, and VPN egress especially -- only resolve to + // country or region granularity, with no city. network_client_location + // requires city_location_id and region_location_id NOT NULL, so a + // country-only location's NULL city/region made this INSERT panic + // inside server.Tx. That panic propagated out of the connection + // announce goroutine (connect/transport_announce.go), whose + // HandleError wrapper then cancelled the whole connection context -- + // tearing down every country-only client's connect connection right + // after auth (the app itself included, whichever egress it resolved + // to), and, because the panic hit before the disconnect-cleanup + // defer was registered, orphaning the connection row as + // connected=true forever. Fall back to the coarsest available + // granularity so the columns are always non-null: a country-only + // location stores its country id for city/region too, which keeps + // the provider locatable at country level instead of crashing the + // connection. If even the country id is missing (location row absent + // or malformed), return a clean error so the caller's existing + // graceful retry path handles it -- never panic here. + if countryLocationId == nil { + returnErr = fmt.Errorf("Location %s has no country granularity.", locationId) + return + } + if cityLocationId == nil { + cityLocationId = countryLocationId + } + if regionLocationId == nil { + regionLocationId = countryLocationId + } + server.RaisePgResult(tx.Exec( ctx, ` diff --git a/model/network_client_location_model_test.go b/model/network_client_location_model_test.go index ab91c7ee..888a1ce2 100644 --- a/model/network_client_location_model_test.go +++ b/model/network_client_location_model_test.go @@ -1198,3 +1198,62 @@ func TestFindProviders2ReliabilityDeployGap(t *testing.T) { connect.AssertEqual(t, len(res.Providers), n) }) } + +// SetConnectionLocation must not panic when the resolved location has no city +// (country-only). network_client_location requires city_location_id and +// region_location_id NOT NULL, but a country-granularity location row has them +// NULL, so the insert panicked inside server.Tx. That panic propagated out of +// the connection announce goroutine, whose HandleError wrapper cancelled the +// connection context -- tearing down the client's connection right after auth, +// and (because the panic hit before the disconnect-cleanup defer was +// registered) orphaning the connection row as connected=true. This asserts a +// country-only location is stored, falling back to country granularity, with +// no panic and no error. +func TestSetConnectionLocationToleratesCountryOnlyLocation(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + // country-only location -- its row has NULL city_location_id and + // NULL region_location_id, exactly what crashed the insert before + country := &Location{ + LocationType: LocationTypeCountry, + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, country) + + networkId := server.NewId() + clientId := server.NewId() + Testing_CreateDevice(ctx, networkId, server.NewId(), clientId, "", "") + + handlerId := CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := ConnectNetworkClient(ctx, clientId, "0.0.0.1:0", handlerId) + connect.AssertEqual(t, err, nil) + + // this call panicked before the fix; now it must succeed and store + // the connection at country granularity + err = SetConnectionLocation(ctx, connectionId, country.LocationId, &ConnectionLocationScores{}) + connect.AssertEqual(t, err, nil) + + var city, region, cty *server.Id + server.Db(ctx, func(conn server.PgConn) { + result, qerr := conn.Query( + ctx, + `SELECT city_location_id, region_location_id, country_location_id FROM network_client_location WHERE connection_id = $1`, + connectionId, + ) + server.WithPgResult(result, qerr, func() { + if result.Next() { + server.Raise(result.Scan(&city, ®ion, &cty)) + } + }) + }) + if city == nil || region == nil || cty == nil { + t.Fatal("expected all three location ids to be set (falling back to country), got a nil") + } + // city and region fall back to the country id for a country-only location + connect.AssertEqual(t, *city, country.CountryLocationId) + connect.AssertEqual(t, *region, country.CountryLocationId) + connect.AssertEqual(t, *cty, country.CountryLocationId) + }) +} diff --git a/model/provider_bandwidth_model.go b/model/provider_bandwidth_model.go new file mode 100644 index 00000000..3630db19 --- /dev/null +++ b/model/provider_bandwidth_model.go @@ -0,0 +1,218 @@ +package model + +import ( + "context" + "time" + + "github.com/urnetwork/server" +) + +// The bandwidth sources. A stored figure is tagged with the source that +// produced it, and the row is keyed on (client_id, source), so every source +// keeps its own figure and none overwrites another. +// +// Adding a further active target later needs a constant here and nothing else +// -- no migration, no new column. That generality is why the source is a key +// column rather than a set of per-target columns. +const ( + // ProviderBandwidthSourcePassive is derived from already-settled contract + // bytes: zero additional cost, and it cannot be gamed selectively. It is + // computed server-side by ComputePassiveProviderBandwidth and is the one + // source no prober may submit -- see IsSubmittableProviderBandwidthSource. + ProviderBandwidthSourcePassive = "passive" + // ProviderBandwidthSourceActiveOperator is a sampled download from the + // operator's own endpoint, over the provider's tunnel. + ProviderBandwidthSourceActiveOperator = "active-operator" + // ProviderBandwidthSourceActiveCDN is the same sample taken against a + // public CDN over the same tunnel. + // + // The two active figures are stored separately and must never be averaged + // into one. A provider that prioritises the operator's own path while + // starving the internet at large is invisible in a combined number and + // obvious in a pair -- which is the entire reason a second target exists. + ProviderBandwidthSourceActiveCDN = "active-cdn" +) + +// IsProviderBandwidthSource reports whether source is one this deployment +// knows. Storage is keyed on the source, so an unrecognised value is not a +// harmless label: it silently creates a row nothing will ever read or replace. +func IsProviderBandwidthSource(source string) bool { + switch source { + case ProviderBandwidthSourcePassive, + ProviderBandwidthSourceActiveOperator, + ProviderBandwidthSourceActiveCDN: + return true + } + return false +} + +// IsSubmittableProviderBandwidthSource reports whether source may arrive from +// outside, over the result endpoint. +// +// It is the active subset, deliberately: "passive" is derived server-side from +// bytes a provider has already been paid to carry, which is exactly what makes +// it ungameable. Accepting a submitted "passive" row would let the submitter +// overwrite that derived figure with an asserted one -- through an endpoint +// whose secret now travels over a provider-controlled path -- and destroy the +// one bandwidth signal in the system that cannot be gamed selectively. +func IsSubmittableProviderBandwidthSource(source string) bool { + switch source { + case ProviderBandwidthSourceActiveOperator, + ProviderBandwidthSourceActiveCDN: + return true + } + return false +} + +// ProviderBandwidth is one throughput figure for a provider. It is advisory: +// nothing may gate provider selection on it. +type ProviderBandwidth struct { + ClientId server.Id + BytesPerSecond float64 + Source string + SampleByteCount ByteCount + WindowStart time.Time + WindowEnd time.Time +} + +// ComputePassiveProviderBandwidth derives a provider's throughput from bytes it +// has already been paid to carry. `transfer_escrow`/`contract_close` record +// settled bytes per contract as a byproduct of billing, so reading them costs +// no additional bandwidth, and a provider cannot inflate the figure selectively +// -- it cannot move more real user traffic without actually being fast for real +// users. +// +// The rate is the total settled bytes in the window over the wall-clock span +// those contracts covered, so it is an average over the sampled traffic rather +// than a peak. Returns nil, nil when the provider settled no bytes in the +// window: no history, which is not the same as measured-zero throughput. +func ComputePassiveProviderBandwidth( + ctx context.Context, + clientId server.Id, + window time.Duration, +) (*ProviderBandwidth, error) { + windowStart := server.NowUtc().Add(-window) + + var contractCount int + var sampleByteCount ByteCount + // null whenever no contract matched + var minCreateTime *time.Time + var maxCloseTime *time.Time + + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT + COUNT(*), + COALESCE(SUM(contract_close.used_transfer_byte_count), 0)::bigint, + MIN(transfer_contract.create_time), + MAX(contract_close.close_time) + + FROM contract_close + + INNER JOIN transfer_contract ON + transfer_contract.contract_id = contract_close.contract_id + + WHERE + transfer_contract.destination_id = $1 AND + contract_close.party = 'destination' AND + -- companion_contract_id IS NULL excludes return-traffic legs: a + -- client's return traffic settles as a contract where the CLIENT + -- is the destination, which would otherwise be misread as that + -- client acting as a fast provider. See + -- docs/superpowers/specs/2026-07-25-enforced-provider-geo-probing-design.md + -- "Threat model" -- confirmed empirically: on beta, every + -- non-Public-key "earner" turned out to be exactly this. + transfer_contract.companion_contract_id IS NULL AND + $2 <= contract_close.close_time + `, + clientId, + windowStart, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + server.Raise(result.Scan( + &contractCount, + &sampleByteCount, + &minCreateTime, + &maxCloseTime, + )) + } + }) + }) + + if contractCount == 0 || sampleByteCount <= 0 || minCreateTime == nil || maxCloseTime == nil { + return nil, nil + } + + elapsed := maxCloseTime.Sub(*minCreateTime) + if elapsed <= 0 { + // no usable denominator (a single instantaneous close, or skew between + // the create and close writers). A rate is undefined here, and dividing + // by zero or a negative span would report an absurd one. + return nil, nil + } + + return &ProviderBandwidth{ + ClientId: clientId, + BytesPerSecond: float64(sampleByteCount) / elapsed.Seconds(), + Source: ProviderBandwidthSourcePassive, + SampleByteCount: sampleByteCount, + // the span actually measured, which is at most `window` wide + WindowStart: *minCreateTime, + WindowEnd: *maxCloseTime, + }, nil +} + +// StoreProviderBandwidth records a provider's current throughput figure for +// one source. The row is keyed on (client_id, source), so a new measurement +// replaces the previous one FROM THE SAME SOURCE and leaves the other sources +// alone: this is the current figure per source a consumer reads, not a history +// (see the provider_bandwidth migrations). +// +// The source is part of the key rather than a plain column, and that is what +// makes two active targets possible at all: keyed on client_id alone, the +// operator and cdn measurements for one provider would overwrite each other on +// every pass and the divergence between them -- the entire reason there is a +// second target -- would never be visible. Averaging them into one row would +// destroy the same signal more quietly. +// +// Every source writes through here, tagged by bw.Source, so nothing downstream +// has to know whether a figure came from settled traffic or an active sample. +// The figure is advisory -- storing one must never gate provider selection. +func StoreProviderBandwidth(ctx context.Context, bw *ProviderBandwidth) { + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO provider_bandwidth ( + client_id, + bytes_per_second, + source, + sample_byte_count, + window_start, + window_end, + update_time + ) + VALUES ($1, $2, $3, $4, $5, $6, $7) + ON CONFLICT (client_id, source) DO UPDATE + SET + bytes_per_second = $2, + sample_byte_count = $4, + window_start = $5, + window_end = $6, + update_time = $7 + `, + bw.ClientId, + bw.BytesPerSecond, + bw.Source, + bw.SampleByteCount, + // window_start/window_end are naive timestamp columns holding utc, + // as everywhere else in this schema + bw.WindowStart.UTC(), + bw.WindowEnd.UTC(), + server.NowUtc(), + )) + }) +} diff --git a/model/provider_bandwidth_model_test.go b/model/provider_bandwidth_model_test.go new file mode 100644 index 00000000..38b1cb16 --- /dev/null +++ b/model/provider_bandwidth_model_test.go @@ -0,0 +1,356 @@ +package model + +import ( + "context" + "testing" + "time" + + "github.com/urnetwork/connect" + + "github.com/urnetwork/server" +) + +func TestComputePassiveProviderBandwidthDerivesFromSettledBytes(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + sourceNetworkId := server.NewId() + sourceId := server.NewId() + destNetworkId := server.NewId() + destId := server.NewId() + Testing_CreateDevice(ctx, sourceNetworkId, server.NewId(), sourceId, "", "") + Testing_CreateDevice(ctx, destNetworkId, server.NewId(), destId, "", "") + + // a contract that settled 32 MiB over exactly 10 seconds of wall time + windowStart := server.NowUtc().Add(-1 * time.Hour) + contractId := Testing_CreateSettledContract(ctx, sourceId, destId, + windowStart, windowStart.Add(10*time.Second), 32*1024*1024) + + bw, err := ComputePassiveProviderBandwidth(ctx, destId, 2*time.Hour) + connect.AssertEqual(t, err, nil) + if bw == nil { + t.Fatal("expected a passive bandwidth result, got nil") + } + connect.AssertEqual(t, bw.Source, "passive") + // 32 MiB / 10s ~= 3355443 bytes/sec + if bw.BytesPerSecond < 3_000_000 || 3_700_000 < bw.BytesPerSecond { + t.Errorf("BytesPerSecond = %.0f, want ~3355443", bw.BytesPerSecond) + } + _ = contractId + }) +} + +func TestComputePassiveProviderBandwidthNilWhenNoHistory(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + bw, err := ComputePassiveProviderBandwidth(ctx, server.NewId(), 2*time.Hour) + connect.AssertEqual(t, err, nil) + if bw != nil { + t.Errorf("expected nil for a provider with no settled bytes, got %+v", bw) + } + }) +} + +// TestComputePassiveProviderBandwidthExcludesCompanionContracts is the load +// bearing case: a client's return traffic settles as a companion contract where +// the CLIENT is the destination. Counting it would read an ordinary user as a +// very fast provider. Only the companion leg exists here, so the correct answer +// is nil (no provider history at all) -- a merely smaller number would prove +// only dilution, not exclusion. +func TestComputePassiveProviderBandwidthExcludesCompanionContracts(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + providerNetworkId := server.NewId() + providerId := server.NewId() + clientNetworkId := server.NewId() + clientId := server.NewId() + Testing_CreateDevice(ctx, providerNetworkId, server.NewId(), providerId, "", "") + Testing_CreateDevice(ctx, clientNetworkId, server.NewId(), clientId, "", "") + + // the client's own contract, which the companion leg pairs with + createTime := server.NowUtc().Add(-1 * time.Hour) + primaryContractId := Testing_CreateSettledContract(ctx, clientId, providerId, + createTime, createTime.Add(10*time.Second), 1024) + + // the return leg: provider -> client, with the client as destination + Testing_CreateSettledCompanionContract(ctx, providerId, clientId, + createTime, createTime.Add(1*time.Second), 64*1024*1024, primaryContractId) + + bw, err := ComputePassiveProviderBandwidth(ctx, clientId, 2*time.Hour) + connect.AssertEqual(t, err, nil) + if bw != nil { + t.Errorf( + "return traffic must not be read as provider egress: got %.0f bytes/sec for a client that never provided", + bw.BytesPerSecond, + ) + } + }) +} + +// Testing_CreateSettledContract inserts a closed contract and its +// destination-party close row, matching the shape real settlement writes +// (`CloseContract` in subscription_model.go). Returns the contract id. +func Testing_CreateSettledContract( + ctx context.Context, + sourceId server.Id, + destinationId server.Id, + createTime time.Time, + closeTime time.Time, + usedByteCount ByteCount, +) server.Id { + return testingCreateSettledContract( + ctx, sourceId, destinationId, createTime, closeTime, usedByteCount, nil, + ) +} + +// Testing_CreateSettledCompanionContract is Testing_CreateSettledContract for +// the return-traffic leg of `companionContractId`. +func Testing_CreateSettledCompanionContract( + ctx context.Context, + sourceId server.Id, + destinationId server.Id, + createTime time.Time, + closeTime time.Time, + usedByteCount ByteCount, + companionContractId server.Id, +) server.Id { + return testingCreateSettledContract( + ctx, sourceId, destinationId, createTime, closeTime, usedByteCount, &companionContractId, + ) +} + +func testingCreateSettledContract( + ctx context.Context, + sourceId server.Id, + destinationId server.Id, + createTime time.Time, + closeTime time.Time, + usedByteCount ByteCount, + companionContractId *server.Id, +) server.Id { + contractId := server.NewId() + + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO transfer_contract ( + contract_id, + source_network_id, + source_id, + destination_network_id, + destination_id, + transfer_byte_count, + create_time, + close_time, + outcome, + companion_contract_id + ) + VALUES ( + $1, + (SELECT network_id FROM network_client WHERE client_id = $2), + $2, + (SELECT network_id FROM network_client WHERE client_id = $3), + $3, + $4, + $5, + $6, + 'success', + $7 + ) + `, + contractId, + sourceId, + destinationId, + usedByteCount, + createTime.UTC(), + closeTime.UTC(), + companionContractId, + )) + + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO contract_close (contract_id, close_time, party, used_transfer_byte_count) + VALUES ($1, $2, 'destination', $3) + `, + contractId, + closeTime.UTC(), + usedByteCount, + )) + }) + + return contractId +} + +// TestProviderBandwidthTableExists asserts the schema shape the storage path +// depends on, separately from the storage path itself: a missing table here is a +// migration that was never appended, not a bug in StoreProviderBandwidth. +func TestProviderBandwidthTableExists(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + var exists bool + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query(ctx, `SELECT to_regclass('provider_bandwidth') IS NOT NULL`) + connect.AssertEqual(t, err, nil) + server.WithPgResult(result, err, func() { + if result.Next() { + server.Raise(result.Scan(&exists)) + } + }) + }) + if !exists { + t.Fatal("provider_bandwidth table does not exist") + } + }) +} + +// TestStoreProviderBandwidthUpsertsOneRowPerSource covers the round trip and +// the primary-key overwrite: a second measurement from the SAME source replaces +// the first rather than accumulating history. +func TestStoreProviderBandwidthUpsertsOneRowPerSource(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + clientId := server.NewId() + windowStart := server.NowUtc().Add(-2 * time.Hour) + windowEnd := server.NowUtc().Add(-1 * time.Hour) + + StoreProviderBandwidth(ctx, &ProviderBandwidth{ + ClientId: clientId, + BytesPerSecond: 1024 * 1024, + Source: ProviderBandwidthSourcePassive, + SampleByteCount: ByteCount(32 * 1024 * 1024), + WindowStart: windowStart, + WindowEnd: windowEnd, + }) + + stored := testingReadProviderBandwidth(ctx, clientId) + connect.AssertEqual(t, len(stored), 1) + passive := stored[ProviderBandwidthSourcePassive] + connect.AssertEqual(t, passive.BytesPerSecond, float64(1024*1024)) + connect.AssertEqual(t, passive.SampleByteCount, ByteCount(32*1024*1024)) + connect.AssertEqual(t, passive.WindowStart.UTC().Unix(), windowStart.Unix()) + connect.AssertEqual(t, passive.WindowEnd.UTC().Unix(), windowEnd.Unix()) + + // a later measurement from the SAME source replaces it in place + laterWindowStart := server.NowUtc().Add(-1 * time.Minute) + laterWindowEnd := server.NowUtc() + StoreProviderBandwidth(ctx, &ProviderBandwidth{ + ClientId: clientId, + BytesPerSecond: 4 * 1024 * 1024, + Source: ProviderBandwidthSourcePassive, + SampleByteCount: ByteCount(8 * 1024 * 1024), + WindowStart: laterWindowStart, + WindowEnd: laterWindowEnd, + }) + + stored = testingReadProviderBandwidth(ctx, clientId) + connect.AssertEqual(t, len(stored), 1) + passive = stored[ProviderBandwidthSourcePassive] + connect.AssertEqual(t, passive.BytesPerSecond, float64(4*1024*1024)) + connect.AssertEqual(t, passive.SampleByteCount, ByteCount(8*1024*1024)) + connect.AssertEqual(t, passive.WindowEnd.UTC().Unix(), laterWindowEnd.Unix()) + }) +} + +// TestStoreProviderBandwidthKeepsEverySourceSeparate is the property the whole +// two-target design rests on: the operator and cdn measurements for ONE +// provider are stored as two rows carrying two figures, and neither overwrites +// the other or the passive figure. +// +// Keyed on client_id alone -- as this table was before the (client_id, source) +// migration -- the second measurement of every pass would silently replace the +// first, so only one target could ever be stored and the divergence between +// them, which is the only reason a second target exists, would be +// unobservable. +func TestStoreProviderBandwidthKeepsEverySourceSeparate(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + clientId := server.NewId() + now := server.NowUtc() + + figures := map[string]float64{ + ProviderBandwidthSourcePassive: 1 * 1024 * 1024, + ProviderBandwidthSourceActiveOperator: 12 * 1024 * 1024, + ProviderBandwidthSourceActiveCDN: 3 * 1024 * 1024, + } + for source, bytesPerSecond := range figures { + StoreProviderBandwidth(ctx, &ProviderBandwidth{ + ClientId: clientId, + BytesPerSecond: bytesPerSecond, + Source: source, + SampleByteCount: ByteCount(5 * 1024 * 1024), + WindowStart: now, + WindowEnd: now, + }) + } + + stored := testingReadProviderBandwidth(ctx, clientId) + if len(stored) != len(figures) { + t.Fatalf("%d rows stored for one provider, want %d (one per source) -- the sources are overwriting each other", len(stored), len(figures)) + } + for source, want := range figures { + row, ok := stored[source] + if !ok { + t.Fatalf("no row for source %q", source) + } + if row.BytesPerSecond != want { + t.Errorf("source %q stored %.0f B/s, want %.0f -- the figures are not being kept apart", + source, row.BytesPerSecond, want) + } + } + + // specifically: the two active targets diverge, and both divergent + // figures survive. An averaged pair would leave both at 7.5 MB/s. + operator := stored[ProviderBandwidthSourceActiveOperator].BytesPerSecond + cdn := stored[ProviderBandwidthSourceActiveCDN].BytesPerSecond + if operator == cdn { + t.Errorf("both active targets stored %.0f B/s; a provider prioritising one path is invisible once they collapse to one figure", operator) + } + }) +} + +// testingReadProviderBandwidth reads the stored rows back with sql, keyed by +// source, so the test asserts what is actually in the table rather than +// trusting a model reader that StoreProviderBandwidth's own writer would share. +func testingReadProviderBandwidth( + ctx context.Context, + clientId server.Id, +) map[string]*ProviderBandwidth { + bySource := map[string]*ProviderBandwidth{} + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT + client_id, + bytes_per_second, + source, + sample_byte_count, + window_start, + window_end + FROM provider_bandwidth + WHERE client_id = $1 + `, + clientId, + ) + server.WithPgResult(result, err, func() { + for result.Next() { + bw := &ProviderBandwidth{} + server.Raise(result.Scan( + &bw.ClientId, + &bw.BytesPerSecond, + &bw.Source, + &bw.SampleByteCount, + &bw.WindowStart, + &bw.WindowEnd, + )) + bySource[bw.Source] = bw + } + }) + }) + return bySource +} diff --git a/model/provider_bandwidth_rate_limit.go b/model/provider_bandwidth_rate_limit.go new file mode 100644 index 00000000..81d4755e --- /dev/null +++ b/model/provider_bandwidth_rate_limit.go @@ -0,0 +1,205 @@ +package model + +import ( + "context" + "fmt" + "time" + + "github.com/urnetwork/server" +) + +// A deployment-wide budget on the bytes active bandwidth probing is allowed +// to spend, counted across all providers together -- this does not +// distinguish which provider is being probed, only how much total probe +// traffic has been admitted. Active probing pulls real data through a +// provider's tunnel, which is a real paid contract: a zero-cost balance code +// does not make it free, because the payout planner sums paid and unpaid +// traffic identically before computing payouts +// (account_payment_model_plan.go). This is therefore a spend limit +// unconditionally, on every deployment, not only where payouts happen to be +// live. +// +// The budget is split into fixed, non-overlapping hourly buckets (UTC), +// rather than a rolling trailing window: a reservation is made against the +// earliest bucket -- starting with the current hour -- that has room for it. +// If the current hour is full, the probe is deferred to the next hour instead +// of rejected, and so on, up to MaxProviderBandwidthLookaheadBuckets hours +// out. Only once no bucket in that whole lookahead window has room -- meaning +// the deployment-wide daily budget (MaxProviderBandwidthBytesPerDay) is +// genuinely exhausted -- is the probe rejected. Fixed buckets (rather than a +// rolling window) are what make "defer to the next hour" well-defined: there +// needs to be a discrete boundary to wait for. +// +// This intentionally does NOT apply to the passive bandwidth signal, which is +// aggregated from already-settled transfer_escrow bytes and costs nothing +// additional -- there is no spend to budget there. +const ProviderBandwidthBucketDuration = time.Hour +const MaxProviderBandwidthLookaheadBuckets = 24 + +// MaxActiveBandwidthProbesPerBucket is derived from the population this +// probes, not picked arbitrarily. Active sampling only ever runs against +// providers with no passive history -- on beta today that is the entire fleet +// (nothing has settled a contract yet), and on a mature deployment it is the +// trickle of newly-joined providers before their first settled contract. 40 +// per hour comfortably covers a beta-sized fleet in a single pass, and stays a +// small, bounded spend on a mature deployment where the no-passive-history +// population is naturally small. It is one value chosen to behave sensibly at +// both scales, not a per-environment knob. +// +// This is a tuned value, not a structural one: revisit it against real +// production data once active probing has run for a week. +const MaxActiveBandwidthProbesPerBucket = 40 + +// MaxProviderBandwidthBytesPerProbe is what ONE reservation admits, and it +// must equal what one measurement actually transfers -- a budget that +// under-counts is worse than no budget, because it reports a spend ceiling the +// deployment is quietly exceeding. +// +// A measurement is 8 parallel streams of 2 MiB = 16 MiB, per target. It is +// parallel because it has to be: a single TCP flow cannot exceed (send window +// / RTT), connect's MaxWindowSize is scaledPow2WindowSize(mib(1), ...), and +// the single-stream probe therefore measured 1 MiB / RTT for every provider on +// the fleet rather than the provider. Eleven of twelve beta providers came +// back with a bandwidth-delay product of ~1 MiB -- exactly one window -- and a +// provider independently measured at 79 MB/s on its own host reported +// 4.8 MB/s through the tunnel. N flows get N windows; the prober's +// bandwidth.MaxSampleBytes is the other half of this figure and the two must +// be changed together. +// +// So the hourly budget is 40 probes x 16 MiB = 640 MiB/hour, and the daily cap +// is 24 x 640 MiB = 15 GiB worst case -- the ceiling only reached if every +// bucket in the lookahead window fills. Note that a probe reserves per TARGET +// and there are two targets measured separately, so 40 reservations is 20 +// providers per hour, not 40. That is unchanged by this commit and is a +// property of MaxActiveBandwidthProbesPerBucket, which is left alone here. +// +// Explicitly int64: a byte budget is not a row count, and callers pass an +// int64 byteCount. +const MaxProviderBandwidthBytesPerProbe int64 = 16 * 1024 * 1024 +const MaxProviderBandwidthBytesPerBucket int64 = MaxActiveBandwidthProbesPerBucket * MaxProviderBandwidthBytesPerProbe +const MaxProviderBandwidthBytesPerDay int64 = MaxProviderBandwidthLookaheadBuckets * MaxProviderBandwidthBytesPerBucket + +func maxProviderBandwidthError() error { + return fmt.Errorf( + "The active bandwidth probe budget (%d bytes per hour, %d bytes per day) has been reached for this deployment. Please try again later.", + MaxProviderBandwidthBytesPerBucket, + MaxProviderBandwidthBytesPerDay, + ) +} + +// providerBandwidthBucketStart truncates t to the start of its UTC hourly +// bucket. Truncate operates on the absolute duration since the zero time, +// so this only aligns to true UTC hour boundaries when t is already UTC. +func providerBandwidthBucketStart(t time.Time) time.Time { + return t.UTC().Truncate(ProviderBandwidthBucketDuration) +} + +// ReserveProviderBandwidthSlot finds the earliest hourly bucket -- starting +// with the current hour -- that has room for `byteCount` more probe bytes, +// and reserves them there. It returns the reservation's id (for +// CancelProviderBandwidthReservation, if the caller can't ultimately use the +// budget it just reserved -- e.g. the provider went offline between reserving +// and dialing) and the bucket's start time, which the caller should use as +// the probe's RunAt when the bucket isn't the current hour. A probe deferred +// to a future hour should have its RunAt jittered randomly across that hour +// rather than firing at the hour's top: otherwise every probe pushed into the +// same future bucket becomes eligible at the identical instant, converting a +// budget that was meant to spread load into an hourly thundering herd. +// +// If no bucket within MaxProviderBandwidthLookaheadBuckets hours has room, +// nothing is reserved and an error is returned -- this is the deployment's +// daily cap. A single probe can never itself be too large to fit an empty +// bucket (one probe is MaxProviderBandwidthBytesPerProbe, a fortieth of a +// bucket), so this only happens under genuine contention from other probes. +// +// Concurrency: this is a plain read-then-insert in one transaction, with no +// row locking. `SELECT ... FOR UPDATE` would not help here -- the thing being +// contended is a bucket's *aggregate*, and an empty or partly-filled bucket +// has no single row to lock; serializing reservations properly would need a +// materialized per-bucket row or an advisory lock, which this ledger shape +// deliberately doesn't have. Under the default RepeatableRead isolation two +// concurrent reservations can therefore both read the same bucket total and +// both insert (their rows are distinct, so there's no write-write conflict +// and no serialization failure), overshooting the ceiling by at most +// (concurrent reservers x their byte counts). That is acceptable: this is a +// coarse deployment-wide spend cap, not an accounting invariant, and probe +// reservations arrive at a low rate from a small number of prober processes. +func ReserveProviderBandwidthSlot( + ctx context.Context, + clientId server.Id, + byteCount int64, +) (reservationId server.Id, bucketStart time.Time, err error) { + windowStart := providerBandwidthBucketStart(server.NowUtc()) + windowEnd := windowStart.Add(MaxProviderBandwidthLookaheadBuckets * ProviderBandwidthBucketDuration) + + server.Tx(ctx, func(tx server.PgTx) { + usedByBucket := map[time.Time]int64{} + result, qerr := tx.Query( + ctx, + ` + SELECT bucket_start, SUM(byte_count) FROM provider_bandwidth_quota + WHERE $1 <= bucket_start AND bucket_start < $2 + GROUP BY bucket_start + `, + windowStart, windowEnd, + ) + server.WithPgResult(result, qerr, func() { + for result.Next() { + var bucket time.Time + var used int64 + server.Raise(result.Scan(&bucket, &used)) + usedByBucket[bucket] = used + } + }) + + for i := 0; i < MaxProviderBandwidthLookaheadBuckets; i++ { + candidate := windowStart.Add(time.Duration(i) * ProviderBandwidthBucketDuration) + if usedByBucket[candidate]+byteCount <= MaxProviderBandwidthBytesPerBucket { + id := server.NewId() + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO provider_bandwidth_quota (provider_bandwidth_quota_id, client_id, byte_count, bucket_start, create_time) + VALUES ($1, $2, $3, $4, $5) + `, + id, clientId, byteCount, candidate, server.NowUtc(), + )) + reservationId = id + bucketStart = candidate + return + } + } + err = maxProviderBandwidthError() + }) + return reservationId, bucketStart, err +} + +// CancelProviderBandwidthReservation releases a reservation made by +// ReserveProviderBandwidthSlot that the caller ultimately couldn't use -- e.g. +// the provider turned out to be unreachable, discovered only after the budget +// was reserved (the target bucket has to be known before the probe can be +// scheduled, so reservation necessarily happens before that check). +func CancelProviderBandwidthReservation(ctx context.Context, reservationId server.Id) { + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + `DELETE FROM provider_bandwidth_quota WHERE provider_bandwidth_quota_id = $1`, + reservationId, + )) + }) +} + +// RemoveExpiredProviderBandwidthQuota deletes quota ledger rows whose bucket +// is entirely in the past relative to minBucketStart. Callers should pass a +// cutoff safely behind the lookahead window (e.g. now minus a couple of +// days), since a bucket up to MaxProviderBandwidthLookaheadBuckets hours in +// the future can still be actively reserved against. +func RemoveExpiredProviderBandwidthQuota(ctx context.Context, minBucketStart time.Time) { + server.MaintenanceTx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + `DELETE FROM provider_bandwidth_quota WHERE bucket_start < $1`, + minBucketStart.UTC(), + )) + }) +} diff --git a/model/provider_bandwidth_rate_limit_test.go b/model/provider_bandwidth_rate_limit_test.go new file mode 100644 index 00000000..9bf78fdf --- /dev/null +++ b/model/provider_bandwidth_rate_limit_test.go @@ -0,0 +1,120 @@ +package model + +import ( + "context" + "testing" + "time" + + "github.com/urnetwork/connect" + "github.com/urnetwork/server" +) + +// The budget is global, not per-provider: two different providers draw from +// the same shared per-bucket byte ceiling. +func TestReserveProviderBandwidthSlotFillsBucketThenSpillsToNext(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + clientIdA := server.NewId() + clientIdB := server.NewId() + + currentBucket := providerBandwidthBucketStart(server.NowUtc()) + + half := MaxProviderBandwidthBytesPerBucket / 2 + + _, bucket1, err := ReserveProviderBandwidthSlot(ctx, clientIdA, half) + connect.AssertEqual(t, err, nil) + connect.AssertEqual(t, bucket1, currentBucket) + + // the remaining half fits exactly at the current bucket's ceiling, + // even though it's a different provider + _, bucketStart, err := ReserveProviderBandwidthSlot(ctx, clientIdB, MaxProviderBandwidthBytesPerBucket-half) + connect.AssertEqual(t, err, nil) + connect.AssertEqual(t, bucketStart, currentBucket) + + // the current bucket is now fully spent; the next reservation must + // spill into the next hour instead of being rejected + _, bucket2, err := ReserveProviderBandwidthSlot(ctx, clientIdA, 1) + connect.AssertEqual(t, err, nil) + connect.AssertEqual(t, bucket2, currentBucket.Add(ProviderBandwidthBucketDuration)) + }) +} + +// Once every bucket in the whole lookahead window (the deployment's daily +// byte budget) is spent, a new reservation must be rejected outright rather +// than queued indefinitely -- this is the hard daily cap. +func TestReserveProviderBandwidthSlotErrorsWhenAllBucketsFull(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + for i := 0; i < MaxProviderBandwidthLookaheadBuckets; i++ { + _, _, err := ReserveProviderBandwidthSlot(ctx, server.NewId(), MaxProviderBandwidthBytesPerBucket) + connect.AssertEqual(t, err, nil) + } + + // every bucket in the lookahead window is now full + _, _, err := ReserveProviderBandwidthSlot(ctx, server.NewId(), 1) + connect.AssertNotEqual(t, err, nil) + }) +} + +// A cancelled reservation must free its bytes back up -- e.g. an active probe +// that was never actually run because the provider went offline between +// reserving the budget and dialing it. +func TestCancelProviderBandwidthReservationFreesSlot(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + clientId := server.NewId() + + currentBucket := providerBandwidthBucketStart(server.NowUtc()) + + reservationId, bucketStart, err := ReserveProviderBandwidthSlot(ctx, clientId, MaxProviderBandwidthBytesPerBucket) + connect.AssertEqual(t, err, nil) + connect.AssertEqual(t, bucketStart, currentBucket) + + CancelProviderBandwidthReservation(ctx, reservationId) + + // the current bucket's full ceiling must be available again + _, bucketStart, err = ReserveProviderBandwidthSlot(ctx, clientId, MaxProviderBandwidthBytesPerBucket) + connect.AssertEqual(t, err, nil) + connect.AssertEqual(t, bucketStart, currentBucket) + }) +} + +// RemoveExpiredProviderBandwidthQuota must only remove buckets safely before +// the cutoff, leaving buckets at or after it untouched. +func TestRemoveExpiredProviderBandwidthQuota(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + currentBucket := providerBandwidthBucketStart(server.NowUtc()) + oldBucket := currentBucket.Add(-72 * time.Hour) + + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO provider_bandwidth_quota (provider_bandwidth_quota_id, client_id, byte_count, bucket_start, create_time) + VALUES ($1, $2, $3, $4, $5) + `, + server.NewId(), server.NewId(), 5, oldBucket, server.NowUtc(), + )) + }) + + _, _, err := ReserveProviderBandwidthSlot(ctx, server.NewId(), 7) + connect.AssertEqual(t, err, nil) + + RemoveExpiredProviderBandwidthQuota(ctx, currentBucket.Add(-48*time.Hour)) + + var total int64 + server.Db(ctx, func(conn server.PgConn) { + result, qerr := conn.Query(ctx, `SELECT COALESCE(SUM(byte_count), 0) FROM provider_bandwidth_quota`) + server.WithPgResult(result, qerr, func() { + if result.Next() { + server.Raise(result.Scan(&total)) + } + }) + }) + // the old (72h back) row must be gone; the current-bucket row (7) + // from ReserveProviderBandwidthSlot must remain + connect.AssertEqual(t, total, int64(7)) + }) +} diff --git a/model/provider_client_verdict_model.go b/model/provider_client_verdict_model.go new file mode 100644 index 00000000..33e56fc7 --- /dev/null +++ b/model/provider_client_verdict_model.go @@ -0,0 +1,501 @@ +package model + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "net/http" + "time" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/session" +) + +// The blackhole reasons a client may report. This is a CLOSED set, derived from +// the three conditions that make connect's detectBlackhole fire +// (ip_remote_multi_client.go, `blackhole := func() bool`): +// +// - sendAckCount <= 0 within BlackholeTimeout of the first un-acked send +// - receiveAckCount <= 0 within the same window +// - receiveSynCount <= 0 within BlackholeConnectTimeout of the first syn +// +// It is an allowlist rather than free text because the reason is stored and +// read by humans: an open string column fills with whatever a client build +// happens to send, and then nothing can be counted by it. +// +// Note what the reason is NOT used for: aggregation keys on receive_ack_count +// alone (see ProviderClientVerdictQuorumMet). A client and a server that +// disagree about reason names must not be able to silently break the quorum. +const ( + // the provider acknowledged nothing the client sent + ProviderClientVerdictReasonNoSendAck = "no-send-ack" + // the client sent and was acknowledged, but nothing came back -- the + // egress-dead case, and the only one the quorum counts + ProviderClientVerdictReasonNoReceiveAck = "no-receive-ack" + // no syn came back inside the connect timeout + ProviderClientVerdictReasonNoReceiveSyn = "no-receive-syn" +) + +var providerClientVerdictReasons = map[string]bool{ + ProviderClientVerdictReasonNoSendAck: true, + ProviderClientVerdictReasonNoReceiveAck: true, + ProviderClientVerdictReasonNoReceiveSyn: true, +} + +// IsProviderClientVerdictReason reports whether reason is one of the known +// blackhole reasons. +func IsProviderClientVerdictReason(reason string) bool { + return providerClientVerdictReasons[reason] +} + +const ( + // ProviderClientVerdictQuorum is how many DISTINCT reporter networks must + // call a provider egress-dead inside the window before anything happens. + // + // Three mirrors the client-side dial-strike shape (3 strikes / 60s / any + // success clears). It is small on purpose: the consequence of a met quorum + // is a probe, not a punishment. NetworkCreateDailyLimit is 5, so three + // sybil networks cost a griefer roughly fifteen minutes -- which is exactly + // why a met quorum must never do more than schedule a probe. + ProviderClientVerdictQuorum = 3 + + // ProviderClientVerdictWindow is how long a verdict counts for. Outside it + // a verdict has decayed and contributes nothing, so a provider cannot + // accumulate a quorum out of unrelated reports spread over a day. + ProviderClientVerdictWindow = 15 * time.Minute + + // providerClientVerdictScanLimit bounds the window read. The table is + // append-only and unbounded per reporter, so without a limit one network + // writing in a loop would make every aggregation read its whole flood. + // + // The rows are read OLDEST FIRST, so the limit can only ever suppress a + // quorum, never manufacture one: eviction drops the newest reports, and the + // count is of distinct networks, which no volume of writes from one network + // can increase. Suppression is the safe direction -- it costs a probe that + // would have been scheduled, not an honest provider's traffic. + providerClientVerdictScanLimit = 4096 +) + +// ProviderClientVerdict is one client-reported blackhole verdict. +// +// ReporterNetworkId is always taken from the reporting session. Nothing may +// populate it from a request body -- see SubmitProviderClientVerdict. +type ProviderClientVerdict struct { + ProviderClientId server.Id + ReporterNetworkId server.Id + Reason string + SendAckCount int64 + SendAckBytes int64 + ReceiveAckCount int64 + ReceiveAckBytes int64 + WindowSeconds int + CreateTime time.Time +} + +// AddProviderClientVerdict appends one verdict row. +// +// ON CONFLICT DO NOTHING because the primary key includes create_time: two +// reports from the same reporter about the same provider inside the same clock +// tick would otherwise raise a duplicate-key error inside server.Tx, which +// retries the failed commit blindly for a minute before surfacing a 500. A +// dropped duplicate is exactly right anyway: the second one could not have +// counted for anything the first did not already count for. +func AddProviderClientVerdict(ctx context.Context, verdict *ProviderClientVerdict) { + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO provider_client_verdict ( + provider_client_id, + reporter_network_id, + reason, + send_ack_count, + send_ack_bytes, + receive_ack_count, + receive_ack_bytes, + window_seconds, + create_time + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9) + ON CONFLICT (provider_client_id, reporter_network_id, create_time) DO NOTHING + `, + verdict.ProviderClientId, + verdict.ReporterNetworkId, + verdict.Reason, + verdict.SendAckCount, + verdict.SendAckBytes, + verdict.ReceiveAckCount, + verdict.ReceiveAckBytes, + verdict.WindowSeconds, + verdict.CreateTime.UTC(), + )) + }) +} + +// GetProviderClientVerdictsInWindow reads the verdicts about one provider that +// are recent enough to still count, oldest first. +// +// This is a bounded READ ONLY. It applies no policy beyond the window: not the +// egress-dead test, and above all not the one-per-reporter cap. Those live in +// ProviderClientVerdictQuorumMet, in Go, in one place, where they are unit +// testable and where breaking them fails a test rather than quietly changing an +// index plan. The database's job here is to hand back a bounded number of rows. +// +// create_time is a naive `timestamp` holding utc, so the cutoff is computed in +// Go and bound as a parameter rather than compared against sql now(), which +// would cast through the session timezone. +func GetProviderClientVerdictsInWindow( + ctx context.Context, + providerClientId server.Id, + now time.Time, +) []ProviderClientVerdict { + verdicts := []ProviderClientVerdict{} + minCreateTime := now.UTC().Add(-ProviderClientVerdictWindow) + + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT + reporter_network_id, + reason, + send_ack_count, + send_ack_bytes, + receive_ack_count, + receive_ack_bytes, + window_seconds, + create_time + FROM provider_client_verdict + WHERE + provider_client_id = $1 AND + $2 <= create_time + + -- oldest first: see providerClientVerdictScanLimit. A flood of + -- writes must not be able to evict the reports that arrived before + -- it, because that is the only direction in which the limit could + -- change a quorum into a non-quorum for a real provider. + ORDER BY create_time ASC + LIMIT $3 + `, + providerClientId, + minCreateTime, + providerClientVerdictScanLimit, + ) + server.WithPgResult(result, err, func() { + for result.Next() { + verdict := ProviderClientVerdict{ + ProviderClientId: providerClientId, + } + server.Raise(result.Scan( + &verdict.ReporterNetworkId, + &verdict.Reason, + &verdict.SendAckCount, + &verdict.SendAckBytes, + &verdict.ReceiveAckCount, + &verdict.ReceiveAckBytes, + &verdict.WindowSeconds, + &verdict.CreateTime, + )) + verdict.CreateTime = verdict.CreateTime.UTC() + verdicts = append(verdicts, verdict) + } + }) + }) + + return verdicts +} + +// ProviderClientVerdictQuorumMet is the whole aggregation policy: pure, read +// time, and the only place any of these three rules exists. +// +// 1. egress-dead only. receive_ack_count == 0 means the client sent and got +// nothing back. A verdict with any receive acks describes a provider that +// carried traffic, whatever reason string it came with, so it counts for +// nothing. The test is on the COUNTS, never on the reason string: a client +// build and a server that disagree about reason names must not be able to +// silently disable the quorum. +// 2. decay. A verdict older than ProviderClientVerdictWindow is gone. Without +// this, verdicts accumulate forever and every provider eventually reaches +// quorum from unrelated incidents months apart. +// 3. ONE VERDICT PER REPORTER NETWORK. This is the anti-griefing property, and +// it is why `reporters` is a SET and not a counter. One network can write +// as many rows as it likes -- the table is append-only on purpose -- and +// still move the count by exactly one. Turning this back into a counter +// lets a single network manufacture a quorum on its own, which is the +// precise failure this function exists to prevent. +// +// # What a met quorum is allowed to do +// +// Reprioritise the provider for probing. Nothing else. It must never demote, +// exclude, or touch filter sets, scores, PassesMinimums or find-providers2. +// +// That is a deliberate amendment to the source spec, which had a met quorum +// exclude the provider outright. Client verdicts are the harder signal to game +// -- real destinations, real vantage points, many uncoordinated reporters -- +// but the spec paired immediate exclusion with "the prober rehabilitates +// immediately", and that pair is a laundering mechanism: blackhole real users, +// get reported, get probed by the one prober you already special-case, pass, +// get rehabilitated, repeat. And with NetworkCreateDailyLimit = 5 a three +// network quorum is about fifteen minutes of sybil work, so immediate exclusion +// would also let a griefer take an honest provider offline for the price of +// three accounts. +// +// Separating the trigger (client verdicts, fast, hard to fake in aggregate) +// from the punishment (the prober, the sole authority) makes griefing cost one +// probe and nothing more -- which is what the spec wanted to be true. If the +// prober then confirms, the existing probation machinery does the gating; this +// adds no new gate. +func ProviderClientVerdictQuorumMet(verdicts []ProviderClientVerdict, now time.Time) bool { + minCreateTime := now.UTC().Add(-ProviderClientVerdictWindow) + + reporters := map[server.Id]bool{} + for _, verdict := range verdicts { + if verdict.ReceiveAckCount != 0 { + // the provider carried traffic back + continue + } + if verdict.CreateTime.UTC().Before(minCreateTime) { + // decayed + continue + } + // a set, not a counter: the cap is one counted verdict per reporter + // network per provider per window + reporters[verdict.ReporterNetworkId] = true + } + + return ProviderClientVerdictQuorum <= len(reporters) +} + +// providerClientVerdictProbeDueAge is how far back a quorum-met provider's +// observed_at is moved: far enough to be due for a probe, not so far that +// anything else changes. +// +// The column has two thresholds on it, and the target has to sit strictly +// between them: +// +// - ProviderEgressLocationMaxAge / 2 (3.5 days) -- the due cutoff +// (providerEgressDueAge in api/handlers). Older than this and the prober is +// offered the provider. +// - ProviderEgressLocationMaxAge (7 days) -- past this the stored location +// stops being trusted at all (GetFreshProviderEgressLocation*, which falls +// back to the mmdb lookup) and RemoveExpiredProviderEgressLocations deletes +// the row. +// +// Backdating past the second one would turn a met quorum into a selection-path +// effect plus data loss, which is exactly what this design forbids. Three +// quarters of the max age is comfortably inside both, and is derived from the +// one constant so it cannot drift away from them. +// +// api/handlers may not be imported here (it imports model), so the due cutoff +// cannot be referenced directly; the handler-side test asserts the provider is +// actually due using the handler's own arithmetic, so a future change to either +// constant breaks a test instead of silently disabling this. +const providerClientVerdictProbeDueAge = ProviderEgressLocationMaxAge * 3 / 4 + +// ReprioritiseProviderEgressProbe brings a provider's next egress probe +// forward, by moving its stored observed_at back to +// providerClientVerdictProbeDueAge ago. It reports whether a row moved. +// +// This is the ONLY effect a met client-verdict quorum has. It touches exactly +// one meaningful column of one row. No filter set, no score, no PassesMinimums, +// no find-providers2 path reads observed_at -- the selection path reads the +// location through GetFreshProviderEgressLocation*, which still resolves the +// same location, because the new observed_at is still inside +// ProviderEgressLocationMaxAge. +// +// `$2 < observed_at` makes this idempotent and, more importantly, non +// ratcheting: repeated quorums cannot walk a provider's observed_at further and +// further back until the row expires out of the freshness window. A row already +// at or past the target is left exactly where it is. +// +// A provider with no provider_egress_location row at all is not a miss: it has +// never been probed successfully, and the due queue already sorts it ahead of +// every probed provider (GetProviderEgressLocationDue pass 1). There is nothing +// to bring forward. +// +// What this deliberately does NOT touch is provider_egress_probe_attempt. A +// provider tried within ProviderEgressProbeAttemptBackoff (6h) stays deferred, +// so the most a quorum can buy -- honest or manufactured -- is one probe per +// provider per backoff window. That bound is the point: it is what keeps the +// cost of griefing at one probe. +func ReprioritiseProviderEgressProbe( + ctx context.Context, + providerClientId server.Id, + now time.Time, +) bool { + moved := false + dueAt := now.UTC().Add(-providerClientVerdictProbeDueAge) + + server.Tx(ctx, func(tx server.PgTx) { + tag, err := tx.Exec( + ctx, + ` + UPDATE provider_egress_location + SET + observed_at = $2, + update_time = $3 + WHERE + client_id = $1 AND + $2 < observed_at + `, + providerClientId, + dueAt, + server.NowUtc(), + ) + server.Raise(err) + moved = 0 < tag.RowsAffected() + }) + + return moved +} + +// SubmitProviderClientVerdictArgs is one client-reported blackhole verdict as +// the reporting client sends it. +// +// There is deliberately no reporter_network_id field. The reporter is the +// authenticated session and nothing else; declaring the field would both invite +// the lie and, with strict decoding, turn an honest client that echoes it into +// a 400. +// +// SynSent/SynReceived are accepted and validated but not stored: the schema +// carries the ack counters the aggregation actually keys on, and a column +// nothing reads is a column that drifts. They are declared here because the +// client sends them -- with strict decoding, an undeclared field is a 400, so +// silently dropping them from the struct would reject every real submission. +type SubmitProviderClientVerdictArgs struct { + ExitClientId server.Id `json:"exit_client_id"` + Reason string `json:"reason"` + SendAckCount int64 `json:"send_ack_count"` + SendAckBytes int64 `json:"send_ack_bytes"` + ReceiveAckCount int64 `json:"receive_ack_count"` + ReceiveAckBytes int64 `json:"receive_ack_bytes"` + SynSent int64 `json:"syn_sent"` + SynReceived int64 `json:"syn_received"` + WindowSeconds int `json:"window_seconds"` +} + +// UnmarshalJSON decodes strictly: an unknown field is an error, which the +// router turns into a 400 before the impl ever runs. +// +// The router's generic decoder is a plain json.Unmarshal shared by every +// endpoint, so strictness has to be attached to the type rather than switched +// on globally -- tightening the shared decoder would change the behaviour of +// every deployed endpoint at once. +// +// Strictness matters here for the same reason it does on the health ingest: the +// body is a set of counts that are read as a judgement. A misspelled +// receive_ack_count decodes to zero, and zero is precisely the value that means +// "egress dead" -- so a typo would not be a malformed report, it would be a +// counting one. +func (self *SubmitProviderClientVerdictArgs) UnmarshalJSON(b []byte) error { + // a defined type with the same layout, minus the methods, so decoding it + // does not recurse back into this function + type strictArgs SubmitProviderClientVerdictArgs + + decoder := json.NewDecoder(bytes.NewReader(b)) + decoder.DisallowUnknownFields() + + var parsed strictArgs + if err := decoder.Decode(&parsed); err != nil { + return err + } + + *self = SubmitProviderClientVerdictArgs(parsed) + return nil +} + +// SubmitProviderClientVerdictResult is deliberately empty. +// +// Telling the reporter whether its verdict met the quorum would hand a griefer +// a progress bar for its sybil campaign -- and would tell any client how many +// other networks are currently reporting a given provider. The reporter needs +// to know its report was accepted, which is the 200. +type SubmitProviderClientVerdictResult struct { +} + +// SubmitProviderClientVerdict records one client-reported blackhole verdict and, +// if the report completes a quorum, brings the provider's next egress probe +// forward. +// +// # The reporter is the session +// +// reporter_network_id comes from the authenticated jwt. This is the security +// property the whole aggregation rests on: the quorum counts distinct networks, +// networks cost account creation (NetworkCreateDailyLimit = 5 per day), and a +// body-supplied reporter id would make the count free to fake. +// +// # Validation happens before any store +// +// Every rule below returns 400 without writing. A stored-then-flagged row is +// not an option for a table whose only consumer is a count: a row that is in +// the table is a row that counts. +// +// # A met quorum only reprioritises +// +// See ProviderClientVerdictQuorumMet for the full reasoning. Nothing here +// touches filter sets, scores, PassesMinimums or find-providers2, and this adds +// no new gate: if the prober confirms the provider is dead, the existing +// probation machinery is what acts on it. +func SubmitProviderClientVerdict( + verdict *SubmitProviderClientVerdictArgs, + clientSession *session.ClientSession, +) (*SubmitProviderClientVerdictResult, error) { + // the route is wrapped in RequireAuth, so a session without a jwt cannot + // reach here. Checked anyway: the reporter identity is the security + // property, and a nil ByJwt must fail closed rather than panic or, worse, + // write a zero reporter network that every other zero reporter dedups with. + if clientSession == nil || clientSession.ByJwt == nil { + return nil, fmt.Errorf("%d Not authorized.", http.StatusUnauthorized) + } + reporterNetworkId := clientSession.ByJwt.NetworkId + + if verdict.ExitClientId == (server.Id{}) { + return nil, fmt.Errorf("%d Missing exit_client_id.", http.StatusBadRequest) + } + if !IsProviderClientVerdictReason(verdict.Reason) { + return nil, fmt.Errorf("%d Unknown verdict reason.", http.StatusBadRequest) + } + // a negative count is not a measurement. It is also the one input that + // could make the egress-dead test read strangely if it were ever changed + // from `!= 0` to `<= 0`, so it is rejected at the door. + if verdict.SendAckCount < 0 || + verdict.SendAckBytes < 0 || + verdict.ReceiveAckCount < 0 || + verdict.ReceiveAckBytes < 0 || + verdict.SynSent < 0 || + verdict.SynReceived < 0 { + return nil, fmt.Errorf("%d Counts must be non-negative.", http.StatusBadRequest) + } + if verdict.WindowSeconds < 0 { + return nil, fmt.Errorf("%d window_seconds must be non-negative.", http.StatusBadRequest) + } + + // the server stamps the time, exactly as the operator ingest endpoints do. + // A client-supplied timestamp would be one more thing to validate and one + // more way for a skewed or hostile clock to place a verdict inside a window + // it does not belong to -- and the window is the decay rule. + now := server.NowUtc() + + AddProviderClientVerdict(clientSession.Ctx, &ProviderClientVerdict{ + ProviderClientId: verdict.ExitClientId, + ReporterNetworkId: reporterNetworkId, + Reason: verdict.Reason, + SendAckCount: verdict.SendAckCount, + SendAckBytes: verdict.SendAckBytes, + ReceiveAckCount: verdict.ReceiveAckCount, + ReceiveAckBytes: verdict.ReceiveAckBytes, + WindowSeconds: verdict.WindowSeconds, + CreateTime: now, + }) + + // read-time aggregation, evaluated on the submit that might have completed + // the quorum. No scheduler and no second cadence: the only moment the + // answer can change is a write. + inWindow := GetProviderClientVerdictsInWindow(clientSession.Ctx, verdict.ExitClientId, now) + if ProviderClientVerdictQuorumMet(inWindow, now) { + ReprioritiseProviderEgressProbe(clientSession.Ctx, verdict.ExitClientId, now) + } + + return &SubmitProviderClientVerdictResult{}, nil +} diff --git a/model/provider_client_verdict_model_test.go b/model/provider_client_verdict_model_test.go new file mode 100644 index 00000000..d7ef259d --- /dev/null +++ b/model/provider_client_verdict_model_test.go @@ -0,0 +1,311 @@ +package model + +import ( + "context" + "testing" + "time" + + "github.com/urnetwork/connect" + + "github.com/urnetwork/server" +) + +// testing_addEgressDeadVerdict writes one egress-dead verdict (receive acks +// zero) at an explicit time. The time is explicit because the primary key +// includes create_time: three reports written in the same clock tick would +// collapse to one row and the same-reporter test would pass for the wrong +// reason. +func testing_addEgressDeadVerdict( + ctx context.Context, + providerClientId server.Id, + reporterNetworkId server.Id, + createTime time.Time, +) { + AddProviderClientVerdict(ctx, &ProviderClientVerdict{ + ProviderClientId: providerClientId, + ReporterNetworkId: reporterNetworkId, + Reason: ProviderClientVerdictReasonNoReceiveAck, + SendAckCount: 64, + SendAckBytes: 8192, + ReceiveAckCount: 0, + ReceiveAckBytes: 0, + WindowSeconds: 30, + CreateTime: createTime, + }) +} + +func testing_quorumMet(ctx context.Context, providerClientId server.Id, now time.Time) bool { + return ProviderClientVerdictQuorumMet( + GetProviderClientVerdictsInWindow(ctx, providerClientId, now), + now, + ) +} + +// Three distinct reporter networks, all inside the window, all egress-dead: +// this is the case the quorum exists to detect. +func TestProviderClientVerdictQuorumMetByThreeDistinctReporters(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + providerClientId := server.NewId() + + if testing_quorumMet(ctx, providerClientId, now) { + t.Fatal("quorum met with no verdicts at all") + } + + testing_addEgressDeadVerdict(ctx, providerClientId, server.NewId(), now.Add(-10*time.Minute)) + testing_addEgressDeadVerdict(ctx, providerClientId, server.NewId(), now.Add(-5*time.Minute)) + if testing_quorumMet(ctx, providerClientId, now) { + t.Fatal("quorum met with two reporters, want three") + } + + testing_addEgressDeadVerdict(ctx, providerClientId, server.NewId(), now.Add(-time.Minute)) + if !testing_quorumMet(ctx, providerClientId, now) { + t.Fatal("quorum not met with three distinct reporters inside the window") + } + + // verdicts about a different provider are not this provider's problem + connect.AssertEqual(t, testing_quorumMet(ctx, server.NewId(), now), false) + }) +} + +// THE ANTI-GRIEFING PROPERTY. One network, reporting as many times as it likes, +// moves the count by exactly one. +// +// The table is append-only on purpose -- a reporter may SAY anything, as often +// as it likes -- so this cap is the only thing standing between a single +// account and a manufactured quorum. Turn the reporter set in +// ProviderClientVerdictQuorumMet into a counter and this test must fail; if it +// still passes, the cap is not being enforced anywhere. +func TestProviderClientVerdictQuorumNotMetByOneReporterRepeating(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + providerClientId := server.NewId() + griefer := server.NewId() + + // well past the quorum in raw rows + for i := range 12 { + testing_addEgressDeadVerdict( + ctx, + providerClientId, + griefer, + now.Add(-time.Duration(i+1)*time.Minute), + ) + } + + // every row was written -- the cap is on what a reporter can count for, + // never on what it can write + verdicts := GetProviderClientVerdictsInWindow(ctx, providerClientId, now) + connect.AssertEqual(t, len(verdicts), 12) + + if ProviderClientVerdictQuorumMet(verdicts, now) { + t.Fatal("one reporter network met the quorum by repeating itself") + } + + // and two honest reporters on top of the flood are still only three + // networks short of nothing: 1 + 2 = 3 distinct, which IS a quorum. The + // flood contributed exactly one. + testing_addEgressDeadVerdict(ctx, providerClientId, server.NewId(), now.Add(-time.Minute)) + if testing_quorumMet(ctx, providerClientId, now) { + t.Fatal("two distinct networks met the quorum") + } + testing_addEgressDeadVerdict(ctx, providerClientId, server.NewId(), now.Add(-time.Minute)) + if !testing_quorumMet(ctx, providerClientId, now) { + t.Fatal("three distinct networks did not meet the quorum") + } + }) +} + +// A verdict that reports received acks describes a provider that carried +// traffic back, whatever reason string it carries. It counts for nothing. +func TestProviderClientVerdictReceivingProviderDoesNotCount(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + providerClientId := server.NewId() + + testing_addEgressDeadVerdict(ctx, providerClientId, server.NewId(), now.Add(-time.Minute)) + testing_addEgressDeadVerdict(ctx, providerClientId, server.NewId(), now.Add(-time.Minute)) + + // third reporter, inside the window, but the provider acknowledged and + // returned traffic. Note the reason still says no-receive-ack: the + // counts are what the aggregation reads, never the reason string. + AddProviderClientVerdict(ctx, &ProviderClientVerdict{ + ProviderClientId: providerClientId, + ReporterNetworkId: server.NewId(), + Reason: ProviderClientVerdictReasonNoReceiveAck, + SendAckCount: 64, + SendAckBytes: 8192, + ReceiveAckCount: 17, + ReceiveAckBytes: 4096, + WindowSeconds: 30, + CreateTime: now.Add(-time.Minute), + }) + + verdicts := GetProviderClientVerdictsInWindow(ctx, providerClientId, now) + // the row is stored and readable -- it is just not counted + connect.AssertEqual(t, len(verdicts), 3) + if ProviderClientVerdictQuorumMet(verdicts, now) { + t.Fatal("a verdict with receive acks counted toward the quorum") + } + }) +} + +// Decay: a verdict older than the window contributes nothing. Without this a +// provider accumulates a quorum out of unrelated incidents days apart. +func TestProviderClientVerdictOutsideWindowDoesNotCount(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + providerClientId := server.NewId() + stale := server.NewId() + + testing_addEgressDeadVerdict(ctx, providerClientId, server.NewId(), now.Add(-time.Minute)) + testing_addEgressDeadVerdict(ctx, providerClientId, server.NewId(), now.Add(-2*time.Minute)) + // one minute past the window + testing_addEgressDeadVerdict( + ctx, + providerClientId, + stale, + now.Add(-ProviderClientVerdictWindow-time.Minute), + ) + + // the window read already drops it, so the third reporter is not even + // offered to the aggregation + verdicts := GetProviderClientVerdictsInWindow(ctx, providerClientId, now) + connect.AssertEqual(t, len(verdicts), 2) + if ProviderClientVerdictQuorumMet(verdicts, now) { + t.Fatal("a decayed verdict counted toward the quorum") + } + + // and the pure aggregation drops it on its own, handed the row + // directly: the two layers are checked independently on purpose, since + // the window read is a scan bound and the aggregation is the policy + stalePlusTwo := append(verdicts, ProviderClientVerdict{ + ProviderClientId: providerClientId, + ReporterNetworkId: stale, + Reason: ProviderClientVerdictReasonNoReceiveAck, + ReceiveAckCount: 0, + CreateTime: now.Add(-ProviderClientVerdictWindow - time.Minute), + }) + if ProviderClientVerdictQuorumMet(stalePlusTwo, now) { + t.Fatal("the aggregation counted a verdict from outside the window") + } + + // the same reporter, inside the window, is the third network + testing_addEgressDeadVerdict(ctx, providerClientId, stale, now.Add(-time.Minute)) + if !testing_quorumMet(ctx, providerClientId, now) { + t.Fatal("a fresh third verdict did not meet the quorum") + } + }) +} + +// The reason allowlist is closed, and is exactly the three conditions that make +// connect's detectBlackhole fire. +func TestProviderClientVerdictReasonAllowlistIsClosed(t *testing.T) { + connect.AssertEqual(t, IsProviderClientVerdictReason(ProviderClientVerdictReasonNoSendAck), true) + connect.AssertEqual(t, IsProviderClientVerdictReason(ProviderClientVerdictReasonNoReceiveAck), true) + connect.AssertEqual(t, IsProviderClientVerdictReason(ProviderClientVerdictReasonNoReceiveSyn), true) + connect.AssertEqual(t, IsProviderClientVerdictReason("slow"), false) + connect.AssertEqual(t, IsProviderClientVerdictReason(""), false) +} + +// The one and only effect of a met quorum: the provider's stored observed_at +// moves back far enough that the prober is offered it, and NOTHING else about +// the row changes. +// +// The two-sided bound is the point. Too little and the provider is never +// offered; past ProviderEgressLocationMaxAge and the location stops being +// trusted by the connection path and the sweep deletes the row -- which would +// turn a client-verdict quorum into a selection-path effect, exactly what this +// design forbids. +func TestReprioritiseProviderEgressProbeMovesOnlyTheDueTime(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + clientId := server.NewId() + + city := &Location{ + LocationType: LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, city) + + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: clientId, + LocationId: city.LocationId, + CountryCode: "us", + ASN: 64500, + Org: "Example Hosting", + Hosting: true, + CityConfident: true, + ObservedAt: now.Add(-time.Hour), + Verdict: "verified", + VerdictReason: "", + }) + before := GetProviderEgressLocation(ctx, clientId) + if before == nil { + t.Fatal("expected a stored egress location") + } + + if !ReprioritiseProviderEgressProbe(ctx, clientId, now) { + t.Fatal("reprioritise did not move a fresh row") + } + + after := GetProviderEgressLocation(ctx, clientId) + if after == nil { + t.Fatal("reprioritise removed the row") + } + + age := now.Sub(after.ObservedAt.UTC()) + // due: older than the api handler's cutoff, which is half the max age + if age <= ProviderEgressLocationMaxAge/2 { + t.Fatalf("observed_at is %s old, not old enough to be due (> %s)", + age, ProviderEgressLocationMaxAge/2) + } + // still fresh: inside the max age, so the connection path still + // resolves this location and the expiry sweep does not delete it + if ProviderEgressLocationMaxAge <= age { + t.Fatalf("observed_at is %s old, past the max age %s: quorum must not expire a location", + age, ProviderEgressLocationMaxAge) + } + fresh := GetFreshProviderEgressLocation(ctx, clientId, ProviderEgressLocationMaxAge) + if fresh == nil { + t.Fatal("the location stopped being fresh: a met quorum must not change what selection sees") + } + + // everything selection could read is byte-identical + connect.AssertEqual(t, after.LocationId, before.LocationId) + connect.AssertEqual(t, after.CountryCode, before.CountryCode) + connect.AssertEqual(t, after.ASN, before.ASN) + connect.AssertEqual(t, after.Org, before.Org) + connect.AssertEqual(t, after.Hosting, before.Hosting) + connect.AssertEqual(t, after.Proxy, before.Proxy) + connect.AssertEqual(t, after.Mobile, before.Mobile) + connect.AssertEqual(t, after.CityConfident, before.CityConfident) + connect.AssertEqual(t, after.Verdict, before.Verdict) + connect.AssertEqual(t, after.VerdictReason, before.VerdictReason) + connect.AssertEqual(t, after.Assurance, before.Assurance) + + // NON-RATCHETING: a second quorum must not walk the row further back. + // Repeated quorums that each subtracted a fixed age would eventually + // push the location out of the freshness window, which is a slow + // version of the exclusion this design forbids. + if ReprioritiseProviderEgressProbe(ctx, clientId, now) { + t.Fatal("a second reprioritise moved an already-due row") + } + again := GetProviderEgressLocation(ctx, clientId) + if !again.ObservedAt.UTC().Equal(after.ObservedAt.UTC()) { + t.Fatalf("observed_at ratcheted from %s to %s", after.ObservedAt.UTC(), again.ObservedAt.UTC()) + } + + // a provider with no row at all is not a miss: it has never been probed + // successfully, so the due queue already sorts it ahead of every probed + // provider and there is nothing to bring forward + connect.AssertEqual(t, ReprioritiseProviderEgressProbe(ctx, server.NewId(), now), false) + }) +} diff --git a/model/provider_egress_health_model.go b/model/provider_egress_health_model.go new file mode 100644 index 00000000..3a69ccf8 --- /dev/null +++ b/model/provider_egress_health_model.go @@ -0,0 +1,183 @@ +package model + +import ( + "context" + "encoding/json" + "time" + + "github.com/urnetwork/server" +) + +// ProviderEgressHealthClassResult is one class's ok/total tally over the +// destinations a single run SAMPLED, not over the whole destination table. The +// prober draws a bounded random subset of each class per run (see the +// operator-proxy's egresshealth package), so `{"cdn":{"ok":4,"total":5}}` means +// four of the five drawn this pass, out of a much larger table. +type ProviderEgressHealthClassResult struct { + OK int `json:"ok"` + Total int `json:"total"` +} + +// ProviderEgressHealth is one egress-health run for one provider: does this +// provider actually carry traffic to the real internet, across several +// independent classes of destination. +// +// # Reputation is not health +// +// ReputationOK/ReputationTotal are stored because they are measured in the +// same pass, and they are deliberately NOT part of OKCount/Total. This mirrors +// the operator-proxy's egresshealth package comment, which calls its own +// version of this "the one thing in this package most likely to be 'fixed' +// into a bug", and the reasoning holds identically server-side. +// +// The reputation class measures whether large vendors treat the exit ip as a +// datacenter address. Nearly every honest hosted provider fails most of it, +// because it IS hosted -- that is a fact about the vendor's ip intelligence +// feed, not about whether the provider carries traffic. Folding it into +// OKCount/Total would take a provider that carried every byte it was asked for +// and score it as partly broken, and the providers it would punish hardest are +// the well-run datacenter ones. Nothing downstream may add these figures into +// OKCount, Total, or any health score derived from them. +// +// The two failure name lists are kept apart for the same reason: FailedNames +// is destinations that mean the provider is not carrying traffic, while +// ReputationFailedNames is vendors that refused a datacenter ip. Merged, they +// would read as one longer failure list. +type ProviderEgressHealth struct { + ClientId server.Id + MeasuredAt time.Time + // OKCount and Total cover the SCORED classes only (dns, connectivity, cdn, + // site), over this run's sample. + OKCount int + Total int + // ClassResults is the per-class tally for the scored classes only. A + // partial failure is only diagnosable per class: "ok=14/26" alone says + // nothing, while "dns=4/4 cdn=0/5 site=12/12" says the tunnel carries + // bytes and resolves names but is being refused by content providers -- + // a completely different fault from a total blackhole. + ClassResults map[string]ProviderEgressHealthClassResult + // ReputationOK/ReputationTotal: stored, never scored. See the type comment. + ReputationOK int + ReputationTotal int + // FailedNames is the comma-joined names of the scored destinations that + // failed. It is the only record of WHICH destinations a given provider was + // asked for on a given pass, since the sample is drawn fresh each run. + FailedNames string + // ReputationFailedNames is the comma-joined names of the reputation + // destinations that refused. Separate from FailedNames, deliberately. + ReputationFailedNames string +} + +// SetProviderEgressHealth records a provider's latest egress-health run. +// +// The row is keyed on client_id alone, so a new run REPLACES the previous one: +// this is the current picture per provider that a consumer reads, not a +// history. That mirrors provider_egress_location's lifecycle exactly. If +// trending is wanted later it belongs in a separate partitioned append table, +// not in a second key column here -- the read path for "is this provider +// carrying traffic right now" wants one row per provider and nothing else. +// +// Nothing here folds reputation into the health figures; see the +// ProviderEgressHealth comment for why that must stay true. +func SetProviderEgressHealth(ctx context.Context, health *ProviderEgressHealth) { + classResults := health.ClassResults + if classResults == nil { + classResults = map[string]ProviderEgressHealthClassResult{} + } + // marshalled here rather than handed to pgx as a map, so the column always + // receives a jsonb document of a known shape (an absent map becomes `{}`, + // not sql NULL, and the column is NOT NULL) + classResultsJson, err := json.Marshal(classResults) + server.Raise(err) + + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO provider_egress_health ( + client_id, + measured_at, + ok_count, + total_count, + class_results, + reputation_ok, + reputation_total, + failed_names, + reputation_failed_names + ) + VALUES ($1, $2, $3, $4, $5::jsonb, $6, $7, $8, $9) + ON CONFLICT (client_id) DO UPDATE + SET + measured_at = $2, + ok_count = $3, + total_count = $4, + class_results = $5::jsonb, + reputation_ok = $6, + reputation_total = $7, + failed_names = $8, + reputation_failed_names = $9 + `, + health.ClientId, + // measured_at is a naive timestamp column holding utc, as + // everywhere else in this schema + health.MeasuredAt.UTC(), + health.OKCount, + health.Total, + string(classResultsJson), + health.ReputationOK, + health.ReputationTotal, + health.FailedNames, + health.ReputationFailedNames, + )) + }) +} + +// GetProviderEgressHealth reads a provider's latest egress-health run, or nil +// when the provider has never been measured. Never measured is not the same as +// measured-unhealthy, so it is a nil result rather than a zero-valued one: +// a caller that cannot tell those apart would read every unprobed provider as +// a total blackhole. +func GetProviderEgressHealth(ctx context.Context, clientId server.Id) *ProviderEgressHealth { + var health *ProviderEgressHealth + + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT + measured_at, + ok_count, + total_count, + class_results, + reputation_ok, + reputation_total, + failed_names, + reputation_failed_names + FROM provider_egress_health + WHERE client_id = $1 + `, + clientId, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + h := &ProviderEgressHealth{ClientId: clientId} + var classResultsJson []byte + server.Raise(result.Scan( + &h.MeasuredAt, + &h.OKCount, + &h.Total, + &classResultsJson, + &h.ReputationOK, + &h.ReputationTotal, + &h.FailedNames, + &h.ReputationFailedNames, + )) + h.ClassResults = map[string]ProviderEgressHealthClassResult{} + server.Raise(json.Unmarshal(classResultsJson, &h.ClassResults)) + health = h + } + }) + }) + + return health +} diff --git a/model/provider_egress_health_model_test.go b/model/provider_egress_health_model_test.go new file mode 100644 index 00000000..94147f63 --- /dev/null +++ b/model/provider_egress_health_model_test.go @@ -0,0 +1,153 @@ +package model + +import ( + "context" + "testing" + "time" + + "github.com/urnetwork/connect" + + "github.com/urnetwork/server" +) + +func TestSetProviderEgressHealthStoresAndReadsBack(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + clientId := server.NewId() + measuredAt := server.NowUtc().Truncate(time.Millisecond) + + SetProviderEgressHealth(ctx, &ProviderEgressHealth{ + ClientId: clientId, + MeasuredAt: measuredAt, + OKCount: 25, + Total: 26, + ClassResults: map[string]ProviderEgressHealthClassResult{ + "dns": {OK: 4, Total: 4}, + "connectivity": {OK: 5, Total: 5}, + "cdn": {OK: 4, Total: 5}, + "site": {OK: 12, Total: 12}, + }, + ReputationOK: 1, + ReputationTotal: 4, + FailedNames: "cachefly", + ReputationFailedNames: "akamai,etsy,canva", + }) + + health := GetProviderEgressHealth(ctx, clientId) + if health == nil { + t.Fatal("expected a stored egress health row, got nil") + } + connect.AssertEqual(t, health.ClientId, clientId) + connect.AssertEqual(t, health.OKCount, 25) + connect.AssertEqual(t, health.Total, 26) + connect.AssertEqual(t, health.ReputationOK, 1) + connect.AssertEqual(t, health.ReputationTotal, 4) + connect.AssertEqual(t, health.FailedNames, "cachefly") + connect.AssertEqual(t, health.ReputationFailedNames, "akamai,etsy,canva") + if !health.MeasuredAt.UTC().Equal(measuredAt) { + t.Errorf("MeasuredAt = %s, want %s", health.MeasuredAt.UTC(), measuredAt) + } + + // asserted as a parsed map, never as json text: key order is not stable + connect.AssertEqual(t, len(health.ClassResults), 4) + connect.AssertEqual(t, health.ClassResults["dns"], ProviderEgressHealthClassResult{OK: 4, Total: 4}) + connect.AssertEqual(t, health.ClassResults["cdn"], ProviderEgressHealthClassResult{OK: 4, Total: 5}) + connect.AssertEqual(t, health.ClassResults["site"], ProviderEgressHealthClassResult{OK: 12, Total: 12}) + + // the reputation figures are stored beside the health figures and + // never inside them: 25/26 is the scored classes only, and the + // per-class tallies sum to exactly that. If reputation were ever + // folded in, this would read 26/30. + sumOK, sumTotal := 0, 0 + for _, c := range health.ClassResults { + sumOK += c.OK + sumTotal += c.Total + } + connect.AssertEqual(t, sumOK, health.OKCount) + connect.AssertEqual(t, sumTotal, health.Total) + if _, present := health.ClassResults["reputation"]; present { + t.Error("reputation must never appear as a scored class") + } + }) +} + +func TestGetProviderEgressHealthNilWhenNeverMeasured(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + // never measured is not the same as measured-unhealthy; a zero-valued + // row here would read as a total blackhole for every unprobed provider + if health := GetProviderEgressHealth(ctx, server.NewId()); health != nil { + t.Errorf("expected nil for a never-probed provider, got %+v", health) + } + }) +} + +// TestSetProviderEgressHealthUpsertReplaces is the lifecycle the table exists +// for: one row per provider carrying the LATEST run. A second run for the same +// client_id must replace the first, not accumulate beside it -- otherwise a +// consumer reading "the provider's health" gets an arbitrary one of N rows. +func TestSetProviderEgressHealthUpsertReplaces(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + clientId := server.NewId() + + SetProviderEgressHealth(ctx, &ProviderEgressHealth{ + ClientId: clientId, + MeasuredAt: server.NowUtc().Add(-1 * time.Hour), + OKCount: 0, + Total: 26, + ClassResults: map[string]ProviderEgressHealthClassResult{ + "dns": {OK: 0, Total: 26}, + }, + ReputationOK: 0, + ReputationTotal: 4, + FailedNames: "everything", + ReputationFailedNames: "akamai", + }) + + later := server.NowUtc() + SetProviderEgressHealth(ctx, &ProviderEgressHealth{ + ClientId: clientId, + MeasuredAt: later, + OKCount: 26, + Total: 26, + ClassResults: map[string]ProviderEgressHealthClassResult{ + "dns": {OK: 26, Total: 26}, + }, + ReputationOK: 2, + ReputationTotal: 4, + // the recovered run has no failures at all: an upsert that only + // wrote the non-empty columns would leave "everything" behind + FailedNames: "", + ReputationFailedNames: "", + }) + + var rowCount int + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + `SELECT COUNT(*) FROM provider_egress_health WHERE client_id = $1`, + clientId, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + server.Raise(result.Scan(&rowCount)) + } + }) + }) + connect.AssertEqual(t, rowCount, 1) + + health := GetProviderEgressHealth(ctx, clientId) + if health == nil { + t.Fatal("expected a stored egress health row, got nil") + } + connect.AssertEqual(t, health.OKCount, 26) + connect.AssertEqual(t, health.ReputationOK, 2) + connect.AssertEqual(t, health.FailedNames, "") + connect.AssertEqual(t, health.ReputationFailedNames, "") + connect.AssertEqual(t, health.ClassResults["dns"], ProviderEgressHealthClassResult{OK: 26, Total: 26}) + if health.MeasuredAt.UTC().Before(later.Add(-time.Minute)) { + t.Errorf("MeasuredAt = %s, want the later run's %s", health.MeasuredAt.UTC(), later) + } + }) +} diff --git a/model/provider_egress_location_model.go b/model/provider_egress_location_model.go new file mode 100644 index 00000000..d67adccc --- /dev/null +++ b/model/provider_egress_location_model.go @@ -0,0 +1,995 @@ +package model + +import ( + "context" + "strings" + "time" + "unicode" + + "golang.org/x/text/unicode/norm" + + "github.com/urnetwork/server" +) + +// ProviderEgressLocationMaxAge bounds how long a probed egress location is +// trusted. Past this, the location is ignored and the caller falls back to the +// mmdb lookup on the observed control ip. +const ProviderEgressLocationMaxAge = 7 * 24 * time.Hour + +// ProviderEgressProbeAttemptBackoff is how long a probe *attempt* defers a +// provider from being offered up again, whether or not the attempt succeeded. +// +// It is much shorter than the staleness window a successful probe buys +// (providerEgressDueAge in api/handlers, half ProviderEgressLocationMaxAge): a +// provider that fails to probe should be retried periodically -- the fault may +// be transient -- just not on every single poll, which is what starves the rest +// of the queue. +const ProviderEgressProbeAttemptBackoff = 6 * time.Hour + +// the verdict/assurance values a provider_egress_location row can hold. These +// mirror the column defaults, and are what an unjudged submission is normalized +// to on write -- see SetProviderEgressLocation. +const ( + // ProviderEgressVerdictUnverified is the default: no judgement recorded. + // Every row written before the ingest path computed verdicts reads as this. + ProviderEgressVerdictUnverified = "unverified" + // ProviderEgressAssuranceDirect means the probe reached the provider over a + // single tunnel from the prober. It is the only assurance in use; multi-hop + // is P3's concern. + ProviderEgressAssuranceDirect = "direct" +) + +// ProviderEgressLocation is a provider location learned by probing the +// provider's own egress, rather than by looking up its control-connection ip. +// +// Verdict/VerdictReason/Assurance carry the recorded judgement for the probe +// that produced this location. They are advisory: nothing in provider selection +// or scoring reads them. An empty Verdict or Assurance is normalized to the +// column default on write, so a caller that does not compute a judgement stores +// an unjudged direct probe rather than an empty string. +type ProviderEgressLocation struct { + ClientId server.Id + LocationId server.Id + CountryCode string + ASN int + Org string + Hosting bool + Proxy bool + Mobile bool + CityConfident bool + ObservedAt time.Time + Verdict string + VerdictReason string + Assurance string + UpdateTime time.Time +} + +// SetProviderEgressLocation upserts the probed location for a provider. The +// upsert is monotonic in observed_at: a replayed or out-of-order submission +// older than what is already stored is silently dropped rather than +// clobbering a newer probe result. +func SetProviderEgressLocation(ctx context.Context, e *ProviderEgressLocation) { + // country codes are stored/compared lowercased (see CreateLocation in + // network_client_location_model.go); the geolocation APIs that feed this + // return uppercase codes (e.g. "US"), so normalize before writing. + countryCode := strings.ToLower(e.CountryCode) + + // the verdict columns are NOT NULL with defaults, and this INSERT names + // every column explicitly -- which bypasses those defaults. A caller that + // computes no judgement would otherwise store '' rather than 'unverified' + // and '' rather than 'direct', so normalize here the way countryCode is + // normalized above. VerdictReason has no default value to fall back to: "" + // means "no reason", which is exactly the column default. + verdict := e.Verdict + if verdict == "" { + verdict = ProviderEgressVerdictUnverified + } + assurance := e.Assurance + if assurance == "" { + assurance = ProviderEgressAssuranceDirect + } + + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO provider_egress_location ( + client_id, + location_id, + country_code, + asn, + org, + hosting, + proxy, + mobile, + city_confident, + observed_at, + verdict, + verdict_reason, + assurance, + update_time + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14) + ON CONFLICT (client_id) DO UPDATE + SET + location_id = $2, + country_code = $3, + asn = $4, + org = $5, + hosting = $6, + proxy = $7, + mobile = $8, + city_confident = $9, + observed_at = $10, + verdict = $11, + verdict_reason = $12, + assurance = $13, + update_time = $14 + WHERE provider_egress_location.observed_at < EXCLUDED.observed_at + `, + e.ClientId, + e.LocationId, + countryCode, + e.ASN, + e.Org, + e.Hosting, + e.Proxy, + e.Mobile, + e.CityConfident, + e.ObservedAt.UTC(), + verdict, + e.VerdictReason, + assurance, + server.NowUtc(), + )) + }) +} + +// ProviderEgressProbeAttempt records that the prober tried a provider, whether +// or not the try produced a location. +// +// A provider that has never been probed successfully has no +// ProviderEgressLocation row at all, so an attempt cannot be recorded there -- +// see the provider_egress_probe_attempt migration for why that matters. +// ProbeFailure is "" for a successful attempt, otherwise a short failure class +// (`tunnel_failed`, `no_consensus`, ...). +type ProviderEgressProbeAttempt struct { + ClientId server.Id + AttemptAt time.Time + ProbeFailure string + UpdateTime time.Time +} + +// SetProviderEgressProbeAttempt upserts the last probe attempt for a provider. +// +// Like SetProviderEgressLocation the upsert is monotonic in its timestamp: a +// replayed or out-of-order report older than what is already stored is dropped +// rather than moving the provider's last-attempt time backwards, which would +// hand it back to the prober early. +func SetProviderEgressProbeAttempt(ctx context.Context, a *ProviderEgressProbeAttempt) { + server.Tx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + ` + INSERT INTO provider_egress_probe_attempt ( + client_id, + attempt_at, + probe_failure, + update_time + ) + VALUES ($1, $2, $3, $4) + ON CONFLICT (client_id) DO UPDATE + SET + attempt_at = $2, + probe_failure = $3, + update_time = $4 + WHERE provider_egress_probe_attempt.attempt_at < EXCLUDED.attempt_at + `, + a.ClientId, + a.AttemptAt.UTC(), + a.ProbeFailure, + server.NowUtc(), + )) + }) +} + +// GetProviderEgressProbeAttempt returns the last recorded probe attempt for a +// provider, or nil. +func GetProviderEgressProbeAttempt(ctx context.Context, clientId server.Id) *ProviderEgressProbeAttempt { + var a *ProviderEgressProbeAttempt + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT + client_id, + attempt_at, + probe_failure, + update_time + FROM provider_egress_probe_attempt + WHERE client_id = $1 + `, + clientId, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + a = &ProviderEgressProbeAttempt{} + server.Raise(result.Scan( + &a.ClientId, + &a.AttemptAt, + &a.ProbeFailure, + &a.UpdateTime, + )) + } + }) + }) + return a +} + +// GetProviderEgressLocation returns the stored location for a provider, or nil. +func GetProviderEgressLocation(ctx context.Context, clientId server.Id) *ProviderEgressLocation { + var e *ProviderEgressLocation + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT + client_id, + location_id, + country_code, + asn, + org, + hosting, + proxy, + mobile, + city_confident, + observed_at, + verdict, + verdict_reason, + assurance, + update_time + FROM provider_egress_location + WHERE client_id = $1 + `, + clientId, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + e = &ProviderEgressLocation{} + server.Raise(result.Scan( + &e.ClientId, + &e.LocationId, + &e.CountryCode, + &e.ASN, + &e.Org, + &e.Hosting, + &e.Proxy, + &e.Mobile, + &e.CityConfident, + &e.ObservedAt, + &e.Verdict, + &e.VerdictReason, + &e.Assurance, + &e.UpdateTime, + )) + } + }) + }) + return e +} + +// GetFreshProviderEgressLocation is GetProviderEgressLocation, filtered to +// entries probed within maxAge. The cutoff is computed in Go and bound as a +// parameter: observed_at is a naive timestamp holding utc, and comparing it +// against sql now() would cast through the session timezone. +func GetFreshProviderEgressLocation( + ctx context.Context, + clientId server.Id, + maxAge time.Duration, +) *ProviderEgressLocation { + e := GetProviderEgressLocation(ctx, clientId) + if e == nil { + return nil + } + if e.ObservedAt.Before(server.NowUtc().Add(-maxAge)) { + return nil + } + return e +} + +// GetFreshProviderEgressLocationForConnection resolves the probed provider +// egress location for a connection in a single query, joining +// network_client_connection to provider_egress_location on client_id. This +// exists for the connect-announce hot path (SetConnectionLocation), which +// previously spent two round trips per connection -- resolving the client id +// for the connection, then fetching its fresh egress location -- before ever +// reaching the mmdb fallback; collapsing to one query matters on a path that +// runs for every connection and inside a retry loop. +// +// As with GetFreshProviderEgressLocation, the maxAge cutoff is computed in Go +// and compared in Go: observed_at is a naive timestamp holding utc, and +// comparing it against sql now() would cast through the session timezone. +func GetFreshProviderEgressLocationForConnection( + ctx context.Context, + connectionId server.Id, + maxAge time.Duration, +) *ProviderEgressLocation { + var e *ProviderEgressLocation + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT + pel.client_id, + pel.location_id, + pel.country_code, + pel.asn, + pel.org, + pel.hosting, + pel.proxy, + pel.mobile, + pel.city_confident, + pel.observed_at, + pel.verdict, + pel.verdict_reason, + pel.assurance, + pel.update_time + FROM network_client_connection ncc + INNER JOIN provider_egress_location pel ON pel.client_id = ncc.client_id + WHERE ncc.connection_id = $1 + `, + connectionId, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + e = &ProviderEgressLocation{} + server.Raise(result.Scan( + &e.ClientId, + &e.LocationId, + &e.CountryCode, + &e.ASN, + &e.Org, + &e.Hosting, + &e.Proxy, + &e.Mobile, + &e.CityConfident, + &e.ObservedAt, + &e.Verdict, + &e.VerdictReason, + &e.Assurance, + &e.UpdateTime, + )) + } + }) + }) + if e == nil { + return nil + } + if e.ObservedAt.Before(server.NowUtc().Add(-maxAge)) { + return nil + } + return e +} + +// GetLocation returns the canonical location row, or nil. +// +// Note: the location table also has a location_name column, but it holds the +// name for whichever granularity that specific row represents (e.g. a city +// row's own name), not a single name field on the Location struct. Location +// instead splits City/Region/Country by joining sibling rows (see +// IndexSearchLocationsInTx in network_client_location_model.go). This helper +// only needs to resolve identity/type, so it selects the columns that map +// directly onto Location's fields and leaves City/Region/Country empty. +func GetLocation(ctx context.Context, locationId server.Id) *Location { + var loc *Location + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT location_id, location_type, city_location_id, region_location_id, country_location_id, country_code + FROM location + WHERE location_id = $1 + `, + locationId, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + loc = &Location{} + // city_location_id/region_location_id are only set once the + // row's hierarchy reaches that granularity (e.g. a country + // row has both NULL); server.Id.Scan errors on a nil source, + // so scan through nullable pointers as in + // IndexSearchLocationsInTx (network_client_location_model.go). + var cityLocationId *server.Id + var regionLocationId *server.Id + var countryLocationId *server.Id + server.Raise(result.Scan( + &loc.LocationId, + &loc.LocationType, + &cityLocationId, + ®ionLocationId, + &countryLocationId, + &loc.CountryCode, + )) + if cityLocationId != nil { + loc.CityLocationId = *cityLocationId + } + if regionLocationId != nil { + loc.RegionLocationId = *regionLocationId + } + if countryLocationId != nil { + loc.CountryLocationId = *countryLocationId + } + } + }) + }) + return loc +} + +// normalizeLocationName folds a location name to a comparison key: lowercased, +// accent-stripped, with every rune that is not a letter or a digit dropped. So +// "Frankfurt am Main", "Frankfurt Am Main" and "FRANKFURT AM MAIN" all fold to +// "frankfurtammain", and "São Paulo", "Zürich" and "Kraków" fold to the same +// keys as "Sao Paulo", "Zurich" and "Krakow". +// +// This is deliberately a comparison key only -- it is never stored, and never +// used to build a location_name. It exists so a trivial spelling variant from a +// geolocation source resolves to the existing row instead of being treated as a +// different place. +// +// Diacritics are the single biggest source of these variants: the free +// geolocation sources disagree over whether to emit the local spelling or an +// ASCII transliteration for the same city, and the mmdb import that seeded most +// existing rows made its own choice. Folding is done with an NFD decomposition +// followed by dropping the combining marks (unicode.Mn), which covers the whole +// accent class at once -- a hand-rolled é->e table would have to enumerate the +// world's diacritics and would silently keep missing the ones it forgot. +// +// golang.org/x/text is already a dependency of this module, so this costs no +// new supply-chain surface. (An earlier revision of this function was +// stdlib-only by mistake: that constraint belongs to the prober repo's +// `geolocate` package, not to the server.) +// +// Letters that carry a stroke rather than a combining mark -- Å‚, ø, Ä‘ -- do not +// decompose and therefore still do not fold onto l/o/d. Those fall back to +// country granularity, which is the safe outcome; the alternative is the +// transliteration table this function deliberately avoids. +// +// Punctuation is dropped rather than mapped to a space because the disagreement +// is over whether the separator exists at all ("Washington, D.C." vs +// "Washington DC"). Note this deliberately does not fold "Frankfurt/Main" onto +// "Frankfurt am Main": dropping the separator gives "frankfurtmain" != +// "frankfurtammain", so that one falls back to country granularity rather than +// matching the wrong row. Falling back is the safe outcome; guessing is not. +func normalizeLocationName(name string) string { + // NFD splits a precomposed letter ("ü") into its base letter plus a + // combining mark ("u" + U+0308). The mark is category Mn, which is neither + // a letter nor a digit, so the filter below drops it; the explicit Mn skip + // is there to say so rather than to leave it to a category coincidence. + decomposed := norm.NFD.String(strings.ToLower(name)) + var b strings.Builder + b.Grow(len(decomposed)) + for _, r := range decomposed { + if unicode.Is(unicode.Mn, r) { + continue + } + if unicode.IsLetter(r) || unicode.IsDigit(r) { + b.WriteRune(r) + } + } + return b.String() +} + +// stripParentheticals removes every parenthesised span from name, so +// "Frankfurt am Main (Innenstadt I)" becomes "Frankfurt am Main ". Nesting is +// tracked, and an unclosed "(" swallows the rest of the string -- a qualifier +// that was truncated by a length limit is still a qualifier. +// +// This is only ever applied to a comparison key, never to anything stored. +func stripParentheticals(name string) string { + if !strings.ContainsRune(name, '(') { + return name + } + var b strings.Builder + b.Grow(len(name)) + depth := 0 + for _, r := range name { + switch r { + case '(': + depth += 1 + case ')': + if 0 < depth { + depth -= 1 + } + default: + if depth == 0 { + b.WriteRune(r) + } + } + } + return b.String() +} + +// matchLocationName returns the location_id of the row in `candidates` whose +// location_name matches `name`, or nil for no match. Candidates must already be +// ordered deterministically by the caller so that two rows folding to the same +// key always resolve the same way. +// +// Three passes, narrowest first: +// +// 1. exact string equality -- the common case, since the winning source usually +// spells it the way the mmdb import did; +// 2. the normalized fold (see normalizeLocationName): case, punctuation, +// whitespace and accents; +// 3. the normalized fold with parenthesised qualifiers stripped from BOTH +// sides. This is the case that motivated the whole feature: one source +// reports "Frankfurt am Main (Innenstadt I)" -- a district qualifier -- for +// a host another source calls "Frankfurt am Main". Pass 2 cannot see +// through that, because dropping the parentheses as punctuation leaves the +// qualifier's letters in the key. +// +// Pass 3 is the only pass that can plausibly match the wrong row -- two +// same-region rows "Springfield (IL)" and "Springfield (MA)" both reduce to +// "springfield" -- so it requires the stripped key to identify exactly ONE +// candidate and returns nil on any ambiguity. Falling back to country +// granularity is the safe outcome; guessing is not. +func matchLocationName(name string, candidateIds []server.Id, candidateNames []string) *server.Id { + for i, candidateName := range candidateNames { + if candidateName == name { + return &candidateIds[i] + } + } + normalized := normalizeLocationName(name) + if normalized == "" { + // nothing comparable survives folding (e.g. a name of only + // punctuation); an empty key would match any other such row + return nil + } + for i, candidateName := range candidateNames { + if normalizeLocationName(candidateName) == normalized { + return &candidateIds[i] + } + } + + base := normalizeLocationName(stripParentheticals(name)) + if base == "" { + // the name was nothing but a qualifier + return nil + } + var unique *server.Id + for i, candidateName := range candidateNames { + if normalizeLocationName(stripParentheticals(candidateName)) == base { + if unique != nil { + return nil + } + unique = &candidateIds[i] + } + } + return unique +} + +// MatchExistingLocation resolves (countryCode, region, city) against location +// rows that ALREADY EXIST and returns the city-granular row, or nil if any +// level of the hierarchy does not resolve. It never inserts anything. +// +// This is the resolver the provider egress ingest path uses instead of +// CreateLocation. CreateLocation deduplicates a city on its exact +// location_name, so an unrecognised spelling does not fail -- it silently +// creates a new, permanent row in the shared `location` table and indexes it +// for search. A geolocation probe has no business defining the world's cities: +// the three free sources the prober reaches consensus over demonstrably +// disagree on spelling (we observed "Frankfurt am Main (Innenstadt I)" against +// "Frankfurt am Main" for one host), and the consensus stores the winning +// source's original display string. Each variant would become its own row, +// those rows outlive a code revert, and there is no cleanup path. +// +// Matching is case-insensitive and ignores punctuation, whitespace, accents and +// parenthesised district qualifiers (see matchLocationName), so the ordinary +// variants resolve to the row that is already there -- including the +// "Frankfurt am Main (Innenstadt I)" case above, which is what this exists for. +// When nothing resolves the caller falls back to country granularity -- +// see SubmitProviderEgressLocation. Falling back loses precision for one +// submission; creating a row corrupts shared data permanently. +// +// Each level tries an exact, fully-indexed match first (the common case: the +// winning source usually spells it the way the mmdb import did) and only scans +// the level's candidates when that misses. +func MatchExistingLocation( + ctx context.Context, + countryCode string, + region string, + city string, +) *Location { + countryCode = strings.ToLower(strings.TrimSpace(countryCode)) + region = strings.TrimSpace(region) + city = strings.TrimSpace(city) + if countryCode == "" || region == "" || city == "" { + return nil + } + + var match *Location + server.Db(ctx, func(conn server.PgConn) { + // country: keyed on country_code alone, exactly as CreateLocation + // dedupes it, so there is no name to match here + var countryLocationId server.Id + var countryName string + found := false + result, err := conn.Query( + ctx, + ` + SELECT location_id, location_name + FROM location + WHERE location_type = $1 AND country_code = $2 + ORDER BY location_id + LIMIT 1 + `, + LocationTypeCountry, + countryCode, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + server.Raise(result.Scan(&countryLocationId, &countryName)) + found = true + } + }) + if !found { + return + } + + // region, within that country + regionLocationId := matchChildLocation( + ctx, + conn, + LocationTypeRegion, + countryCode, + region, + ` + SELECT location_id, location_name + FROM location + WHERE + location_type = $1 AND + country_code = $2 AND + location_name = $3 AND + country_location_id = $4 + `, + ` + SELECT location_id, location_name + FROM location + WHERE + location_type = $1 AND + country_code = $2 AND + country_location_id = $3 + ORDER BY location_id + `, + []any{countryLocationId}, + ) + if regionLocationId == nil { + return + } + + // city, within that region + cityLocationId := matchChildLocation( + ctx, + conn, + LocationTypeCity, + countryCode, + city, + ` + SELECT location_id, location_name + FROM location + WHERE + location_type = $1 AND + country_code = $2 AND + location_name = $3 AND + region_location_id = $4 AND + country_location_id = $5 + `, + ` + SELECT location_id, location_name + FROM location + WHERE + location_type = $1 AND + country_code = $2 AND + region_location_id = $3 AND + country_location_id = $4 + ORDER BY location_id + `, + []any{*regionLocationId, countryLocationId}, + ) + if cityLocationId == nil { + return + } + + match = &Location{ + LocationType: LocationTypeCity, + City: city, + Region: region, + Country: countryName, + CountryCode: countryCode, + LocationId: *cityLocationId, + CityLocationId: *cityLocationId, + RegionLocationId: *regionLocationId, + CountryLocationId: countryLocationId, + } + }) + return match +} + +// matchChildLocation runs the exact-match query first and only falls back to +// scanning the level's candidates when it misses. `parents` are the parent +// location ids the two queries scope on: the exact query binds them after +// (location_type, country_code, name), the candidate query after +// (location_type, country_code). +func matchChildLocation( + ctx context.Context, + conn server.PgConn, + locationType LocationType, + countryCode string, + name string, + exactSql string, + candidatesSql string, + parents []any, +) *server.Id { + exactArgs := append([]any{locationType, countryCode, name}, parents...) + var exactId *server.Id + result, err := conn.Query(ctx, exactSql, exactArgs...) + server.WithPgResult(result, err, func() { + if result.Next() { + var locationId server.Id + var locationName string + server.Raise(result.Scan(&locationId, &locationName)) + exactId = &locationId + } + }) + if exactId != nil { + return exactId + } + + candidateArgs := append([]any{locationType, countryCode}, parents...) + candidateIds := []server.Id{} + candidateNames := []string{} + result, err = conn.Query(ctx, candidatesSql, candidateArgs...) + server.WithPgResult(result, err, func() { + for result.Next() { + var locationId server.Id + var locationName string + server.Raise(result.Scan(&locationId, &locationName)) + candidateIds = append(candidateIds, locationId) + candidateNames = append(candidateNames, locationName) + } + }) + return matchLocationName(name, candidateIds, candidateNames) +} + +// GetProviderEgressLocationDue returns the client ids of providers whose +// egress location is due for a probe: no fresh success (newest probe older than +// minObservedAt, or never probed) *and* no recent attempt (last attempt older +// than minAttemptAt, or never attempted). Oldest first, so the +// longest-unprobed are handed out first, capped at limit. +// +// This is the durable replacement for the prober's in-memory ttl cache: the +// schedule lives in the database, so a prober restart resumes where it left +// off instead of re-probing everything. +// +// Three things about the shape of this query matter. +// +// First, candidates are sourced from the live provider population +// (network_client_location_reliability, connected + valid) and the egress row +// is LEFT JOINed on. The dominant case by far is a provider that has *never* +// been probed and therefore has no provider_egress_location row at all; +// selecting from provider_egress_location would return exactly the providers +// that least need probing and none of the ones that most do. +// +// Second, only providers holding a Public provide key are returned. Probing +// tunnels through the provider itself, which means opening a contract from +// outside the provider's own network -- something a provider without a Public +// key refuses. Offering one to the prober would burn a probe slot on a +// guaranteed failure. This is the same filter UpdateClientLocations and +// UpdateClientScores apply (network_client_location_model.go). +// +// Third, a recent *attempt* defers a provider the same way a recent success +// does. Without that, a provider that connects and holds a Public provide key +// but always fails to probe -- for any reason other than the missing Public key +// screened for above -- never gets an egress row, so its observed_at stays +// NULL, so it sorts ahead of every stale-but-refreshable provider on every +// single poll, forever. Enough such providers to fill a batch and no healthy +// provider is ever refreshed again, while this endpoint goes on returning a +// full, plausible-looking batch. The in-memory ttl cache this replaced was +// incidentally immune, because it marked a provider probed whether or not the +// probe worked; moving the schedule server-side dropped that protection, and +// provider_egress_probe_attempt is what restores it. +// +// Both cutoffs are computed by the caller in Go and bound as parameters: +// observed_at and attempt_at are naive `timestamp` columns holding utc, and +// comparing them against sql now() would cast through the session timezone and +// silently skip a window. +// +// # Two passes, not one +// +// Expressed as a single statement this is a scan of +// network_client_location_reliability with two LEFT JOINs, sorted on +// observed_at from an outer-joined table. That sort cannot use an index: the +// column being ordered on does not exist for most of the rows being ordered. +// At beta's 40 providers that is free. At 100k it is a full scan plus an +// unindexable sort, on every poll. +// +// The ordering makes the split possible. `NULLS FIRST` means every never-probed +// provider sorts ahead of every probed one, so the result is always the +// concatenation of two independently ordered groups: +// +// 1. never probed -- no provider_egress_location row at all. This is the +// dominant group (it is why the ordering is NULLS FIRST), and within it +// every observed_at is equally absent, so the order is client_id alone. As +// an anti-join with no outer-joined column in the ORDER BY it is an ordered +// index scan over (valid, connected, client_id) with a LIMIT: no sort, and +// it stops as soon as the batch is full. +// 2. stale but probed -- has a row, older than minObservedAt. Only reached +// when pass 1 came up short of the limit. Driven from +// provider_egress_location itself, where observed_at is a real, indexable +// column: an ordered range scan over (observed_at, client_id). +// +// Both passes carry the same eligibility predicates, so the concatenation is +// row-for-row what the single statement returned, in the same order, under the +// same limit. `attempt_at IS NULL OR attempt_at < $n` becomes the equivalent +// `NOT EXISTS (... AND $n <= attempt_at)` -- equivalent because client_id is the +// primary key of provider_egress_probe_attempt, so there is at most one row to +// quantify over. The same holds for `observed_at IS NULL` on +// provider_egress_location, whose client_id is likewise a primary key and whose +// observed_at is NOT NULL: the only way that test is true is that no row exists. +func GetProviderEgressLocationDue( + ctx context.Context, + minObservedAt time.Time, + minAttemptAt time.Time, + limit int, +) []server.Id { + clientIds := []server.Id{} + server.Db(ctx, func(conn server.PgConn) { + // pass 1: never probed. Ordered by client_id alone -- every row in this + // group has no observed_at, so the ORDER BY's leading key is constant + // across it and the tie-break is the whole ordering. + // + // `limit` is passed through as given rather than clamped, so a + // nonsensical limit fails exactly as the single-statement version did + // (LIMIT 0 returns nothing; a negative limit is an error). + result, err := conn.Query( + ctx, + ` + SELECT + network_client_location_reliability.client_id + FROM network_client_location_reliability + + WHERE + network_client_location_reliability.connected = true AND + network_client_location_reliability.valid = true AND + EXISTS ( + SELECT 1 FROM provide_key + WHERE + provide_key.client_id = network_client_location_reliability.client_id AND + provide_key.provide_mode = $1 + ) AND + NOT EXISTS ( + SELECT 1 FROM provider_egress_location + WHERE + provider_egress_location.client_id = network_client_location_reliability.client_id + ) AND + NOT EXISTS ( + SELECT 1 FROM provider_egress_probe_attempt + WHERE + provider_egress_probe_attempt.client_id = network_client_location_reliability.client_id AND + $2 <= provider_egress_probe_attempt.attempt_at + ) + + ORDER BY network_client_location_reliability.client_id ASC + LIMIT $3 + `, + ProvideModePublic, + minAttemptAt.UTC(), + limit, + ) + server.WithPgResult(result, err, func() { + for result.Next() { + var clientId server.Id + server.Raise(result.Scan(&clientId)) + clientIds = append(clientIds, clientId) + } + }) + + remaining := limit - len(clientIds) + if remaining <= 0 { + // the batch is full from never-probed providers alone, which is the + // steady state until the population has been swept once. The + // single-statement version would have returned exactly these rows + // too: they all sort ahead of anything with an observed_at. + return + } + + // pass 2: stale but probed. Driven from provider_egress_location, so + // observed_at is a real column of the driving table and the ORDER BY is + // an ordered index scan rather than a sort. + result, err = conn.Query( + ctx, + ` + SELECT + provider_egress_location.client_id + FROM provider_egress_location + + INNER JOIN network_client_location_reliability ON + network_client_location_reliability.client_id = provider_egress_location.client_id + + WHERE + provider_egress_location.observed_at < $2 AND + network_client_location_reliability.connected = true AND + network_client_location_reliability.valid = true AND + EXISTS ( + SELECT 1 FROM provide_key + WHERE + provide_key.client_id = provider_egress_location.client_id AND + provide_key.provide_mode = $1 + ) AND + NOT EXISTS ( + SELECT 1 FROM provider_egress_probe_attempt + WHERE + provider_egress_probe_attempt.client_id = provider_egress_location.client_id AND + $3 <= provider_egress_probe_attempt.attempt_at + ) + + -- oldest probe first, client_id breaking the tie, so batch + -- composition is deterministic instead of plan-dependent + ORDER BY + provider_egress_location.observed_at ASC, + provider_egress_location.client_id ASC + LIMIT $4 + `, + ProvideModePublic, + minObservedAt.UTC(), + minAttemptAt.UTC(), + remaining, + ) + server.WithPgResult(result, err, func() { + // the two passes are separate statements and so separate snapshots. + // A provider that gains its first provider_egress_location row + // between them would be never-probed to pass 1 and stale to pass 2; + // the single-statement version could not do that, so screen it out + // rather than hand the prober the same client twice. + seen := map[server.Id]bool{} + for _, clientId := range clientIds { + seen[clientId] = true + } + for result.Next() { + var clientId server.Id + server.Raise(result.Scan(&clientId)) + if seen[clientId] { + continue + } + clientIds = append(clientIds, clientId) + } + }) + }) + return clientIds +} + +// RemoveExpiredProviderEgressLocations drops entries probed before +// minObservedAt. +func RemoveExpiredProviderEgressLocations(ctx context.Context, minObservedAt time.Time) { + server.MaintenanceTx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + `DELETE FROM provider_egress_location WHERE observed_at < $1`, + minObservedAt.UTC(), + )) + }) +} + +// RemoveExpiredProviderEgressProbeAttempts drops attempts older than +// minAttemptAt. An attempt only carries information for as long as it defers +// the provider (ProviderEgressProbeAttemptBackoff); past that the row is just +// storage held for a client id that may no longer exist. +func RemoveExpiredProviderEgressProbeAttempts(ctx context.Context, minAttemptAt time.Time) { + server.MaintenanceTx(ctx, func(tx server.PgTx) { + server.RaisePgResult(tx.Exec( + ctx, + `DELETE FROM provider_egress_probe_attempt WHERE attempt_at < $1`, + minAttemptAt.UTC(), + )) + }) +} diff --git a/model/provider_egress_location_model_test.go b/model/provider_egress_location_model_test.go new file mode 100644 index 00000000..53dc29b2 --- /dev/null +++ b/model/provider_egress_location_model_test.go @@ -0,0 +1,747 @@ +package model + +import ( + "context" + "fmt" + "slices" + "testing" + "time" + + "github.com/urnetwork/connect" + "github.com/urnetwork/server" +) + +func TestProviderEgressLocationUpsertAndGet(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + country := &Location{ + LocationType: LocationTypeCountry, + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, country) + + clientId := server.NewId() + now := server.NowUtc() + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: clientId, + LocationId: country.LocationId, + CountryCode: "us", + ASN: 401486, + Org: "RAVNIX LLC", + Hosting: true, + ObservedAt: now, + }) + + got := GetProviderEgressLocation(ctx, clientId) + if got == nil { + t.Fatal("expected a stored egress location") + } + connect.AssertEqual(t, got.LocationId, country.LocationId) + connect.AssertEqual(t, got.CountryCode, "us") + connect.AssertEqual(t, got.ASN, 401486) + connect.AssertEqual(t, got.Hosting, true) + connect.AssertEqual(t, got.Proxy, false) + + // upsert replaces, given a strictly newer observed_at: the upsert is + // monotonic (see TestProviderEgressLocationUpsertIgnoresOlderReplay below), + // so a second submission at the same observed_at would not win. + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: clientId, + LocationId: country.LocationId, + CountryCode: "us", + ASN: 999, + Hosting: false, + Proxy: true, + ObservedAt: now.Add(time.Minute), + }) + got = GetProviderEgressLocation(ctx, clientId) + connect.AssertEqual(t, got.ASN, 999) + connect.AssertEqual(t, got.Hosting, false) + connect.AssertEqual(t, got.Proxy, true) + }) +} + +// The upsert is monotonic in observed_at: a replayed submission older than +// what is already stored must not clobber the newer row. +func TestProviderEgressLocationUpsertIgnoresOlderReplay(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + usCountry := &Location{ + LocationType: LocationTypeCountry, + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, usCountry) + + jpCountry := &Location{ + LocationType: LocationTypeCountry, + Country: "Japan", + CountryCode: "jp", + } + CreateLocation(ctx, jpCountry) + + clientId := server.NewId() + newer := server.NowUtc() + older := newer.Add(-1 * time.Hour) + + // the newer probe lands first + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: clientId, + LocationId: jpCountry.LocationId, + CountryCode: "jp", + ASN: 111, + ObservedAt: newer, + }) + + // a stale/replayed older probe arrives afterward and must not win + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: clientId, + LocationId: usCountry.LocationId, + CountryCode: "us", + ASN: 222, + ObservedAt: older, + }) + + got := GetProviderEgressLocation(ctx, clientId) + if got == nil { + t.Fatal("expected a stored egress location") + } + connect.AssertEqual(t, got.CountryCode, "jp") + connect.AssertEqual(t, got.ASN, 111) + connect.AssertEqual(t, got.LocationId, jpCountry.LocationId) + }) +} + +func TestProviderEgressLocationCountryCodeLowercased(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + country := &Location{ + LocationType: LocationTypeCountry, + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, country) + + clientId := server.NewId() + // geolocation APIs return uppercase codes (e.g. "US"); the model must + // normalize to lowercase before storing, matching CreateLocation's + // established invariant that country codes are stored/compared lowercased. + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: clientId, + LocationId: country.LocationId, + CountryCode: "US", + ASN: 12345, + Org: "TEST ORG", + ObservedAt: server.NowUtc(), + }) + + got := GetProviderEgressLocation(ctx, clientId) + if got == nil { + t.Fatal("expected a stored egress location") + } + connect.AssertEqual(t, got.CountryCode, "us") + }) +} + +func TestProviderEgressLocationFreshness(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + country := &Location{ + LocationType: LocationTypeCountry, + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, country) + + fresh := server.NewId() + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: fresh, LocationId: country.LocationId, CountryCode: "us", + ObservedAt: server.NowUtc(), + }) + stale := server.NewId() + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: stale, LocationId: country.LocationId, CountryCode: "us", + ObservedAt: server.NowUtc().Add(-8 * 24 * time.Hour), + }) + + if GetFreshProviderEgressLocation(ctx, fresh, ProviderEgressLocationMaxAge) == nil { + t.Fatal("fresh entry must be returned") + } + if GetFreshProviderEgressLocation(ctx, stale, ProviderEgressLocationMaxAge) != nil { + t.Fatal("stale entry must not be returned") + } + // absent + if GetFreshProviderEgressLocation(ctx, server.NewId(), ProviderEgressLocationMaxAge) != nil { + t.Fatal("absent entry must return nil") + } + }) +} + +func TestRemoveExpiredProviderEgressLocations(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + country := &Location{ + LocationType: LocationTypeCountry, + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, country) + + keep := server.NewId() + drop := server.NewId() + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: keep, LocationId: country.LocationId, CountryCode: "us", + ObservedAt: server.NowUtc(), + }) + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: drop, LocationId: country.LocationId, CountryCode: "us", + ObservedAt: server.NowUtc().Add(-30 * 24 * time.Hour), + }) + + RemoveExpiredProviderEgressLocations(ctx, server.NowUtc().Add(-14*24*time.Hour)) + + if GetProviderEgressLocation(ctx, keep) == nil { + t.Fatal("recent entry must survive the sweep") + } + if GetProviderEgressLocation(ctx, drop) != nil { + t.Fatal("old entry must be swept") + } + }) +} + +// testing_connectProbeableProvider stands up the minimum a client needs to +// look like a live provider to the due-selection query: a device, a live +// connection with a resolved location, and a provide key of the given mode. +// The caller must run UpdateClientLocationReliabilities afterward -- that is +// what rolls the live connection tables up into the +// network_client_location_reliability row (connected + valid) the query reads. +// It returns the connection id, so a caller can disconnect the provider again. +func testing_connectProbeableProvider( + t testing.TB, + ctx context.Context, + clientId server.Id, + locationId server.Id, + clientAddress string, + provideMode ProvideMode, +) server.Id { + Testing_CreateDevice(ctx, server.NewId(), server.NewId(), clientId, "", "") + + handlerId := CreateNetworkClientHandler(ctx) + connectionId, _, _, _, err := ConnectNetworkClient(ctx, clientId, clientAddress, handlerId) + if err != nil { + t.Fatalf("connect client: %s", err) + } + + if err := SetConnectionLocation(ctx, connectionId, locationId, &ConnectionLocationScores{}); err != nil { + t.Fatalf("set connection location: %s", err) + } + + SetProvide(ctx, clientId, map[ProvideMode][]byte{ + provideMode: []byte("provide-secret"), + }) + + return connectionId +} + +// The prober asks the server what to probe next. The answer must be sourced +// from the live provider population and not from provider_egress_location, +// because the dominant case -- a provider that has never been probed at all -- +// has no row there. +func TestGetProviderEgressLocationDue(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + + city := &Location{ + LocationType: LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, city) + + fresh := server.NewId() + stale := server.NewId() + never := server.NewId() + // a provider that cannot serve a stranger is unprobeable: the tunnel + // contract would be refused, so it must never be offered to the prober + nonPublic := server.NewId() + + testing_connectProbeableProvider(t, ctx, fresh, city.LocationId, "0.0.0.1:0", ProvideModePublic) + testing_connectProbeableProvider(t, ctx, stale, city.LocationId, "0.0.0.2:0", ProvideModePublic) + testing_connectProbeableProvider(t, ctx, never, city.LocationId, "0.0.0.3:0", ProvideModePublic) + testing_connectProbeableProvider(t, ctx, nonPublic, city.LocationId, "0.0.0.4:0", ProvideModeNetwork) + + UpdateClientLocationReliabilities(ctx, now.Add(-time.Hour), now) + + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: fresh, LocationId: city.LocationId, + CountryCode: "us", ObservedAt: now.Add(-1 * time.Hour), + }) + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: stale, LocationId: city.LocationId, + CountryCode: "us", ObservedAt: now.Add(-72 * time.Hour), + }) + // `never` and `nonPublic` deliberately get no row at all + + // no attempt rows exist in this test, so the attempt cutoff never + // excludes anything; freshness is the only variable + due := GetProviderEgressLocationDue(ctx, now.Add(-24*time.Hour), now, 100) + + // a provider probed an hour ago must not be re-probed; one probed three + // days ago must be; one never probed must be + if slices.Contains(due, fresh) { + t.Fatalf("due = %v, must not contain the provider probed an hour ago (%s)", due, fresh) + } + if !slices.Contains(due, stale) { + t.Fatalf("due = %v, must contain the provider probed three days ago (%s)", due, stale) + } + if !slices.Contains(due, never) { + t.Fatalf("due = %v, must contain the never-probed provider (%s)", due, never) + } + // unprobeable regardless of freshness + if slices.Contains(due, nonPublic) { + t.Fatalf("due = %v, must not contain the provider without a Public provide key (%s)", due, nonPublic) + } + + // oldest first, so the longest-unprobed are probed first: the + // never-probed provider sorts ahead of the three-days-stale one + neverIndex := slices.Index(due, never) + staleIndex := slices.Index(due, stale) + if staleIndex < neverIndex { + t.Fatalf("never-probed provider at %d must sort before the stale one at %d", neverIndex, staleIndex) + } + + // limit is honoured + limited := GetProviderEgressLocationDue(ctx, now.Add(-24*time.Hour), now, 1) + if len(limited) != 1 { + t.Fatalf("len(due) = %d for limit 1, want 1", len(limited)) + } + if limited[0] != never { + t.Fatalf("due[0] = %s for limit 1, want the never-probed provider %s", limited[0], never) + } + }) +} + +// A provider that connects, holds a Public provide key and fails every probe +// never gets a provider_egress_location row, so its observed_at stays NULL, so +// it sorts ahead of every stale-but-refreshable provider -- forever, on every +// poll. Enough of them to fill a batch and no healthy provider's location is +// ever refreshed again, silently: the endpoint keeps returning a full, +// plausible-looking batch of the same dead providers. +// +// A recent attempt must therefore defer a provider exactly as a fresh success +// does. Deleting the attempt predicate from GetProviderEgressLocationDue must +// fail this test. +func TestGetProviderEgressLocationDueDefersRecentlyAttempted(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + + city := &Location{ + LocationType: LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, city) + + // never probed successfully, and the prober just tried it and failed + dead := server.NewId() + // probed successfully three days ago, never attempted since: the + // provider that actually needs the next probe slot + healthyStale := server.NewId() + + testing_connectProbeableProvider(t, ctx, dead, city.LocationId, "0.0.0.1:0", ProvideModePublic) + testing_connectProbeableProvider(t, ctx, healthyStale, city.LocationId, "0.0.0.2:0", ProvideModePublic) + + UpdateClientLocationReliabilities(ctx, now.Add(-time.Hour), now) + + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: healthyStale, LocationId: city.LocationId, + CountryCode: "us", ObservedAt: now.Add(-72 * time.Hour), + }) + // `dead` deliberately gets no location row -- it has never succeeded -- + // only a failed attempt seconds ago + SetProviderEgressProbeAttempt(ctx, &ProviderEgressProbeAttempt{ + ClientId: dead, + AttemptAt: now.Add(-5 * time.Second), + ProbeFailure: "tunnel_failed", + }) + + minObservedAt := now.Add(-24 * time.Hour) + minAttemptAt := now.Add(-ProviderEgressProbeAttemptBackoff) + + due := GetProviderEgressLocationDue(ctx, minObservedAt, minAttemptAt, 100) + + if slices.Contains(due, dead) { + t.Fatalf("due = %v, must not contain the provider attempted seconds ago (%s)", due, dead) + } + if !slices.Contains(due, healthyStale) { + t.Fatalf("due = %v, must contain the stale-but-refreshable provider (%s)", due, healthyStale) + } + + // the starvation itself: with a batch big enough for exactly one + // provider, the slot must go to the one that can actually be refreshed, + // not to the never-probed one that just failed. Without the attempt + // predicate `dead` wins this on observed_at IS NULL every single poll. + limited := GetProviderEgressLocationDue(ctx, minObservedAt, minAttemptAt, 1) + if len(limited) != 1 { + t.Fatalf("len(due) = %d for limit 1, want 1", len(limited)) + } + if limited[0] != healthyStale { + t.Fatalf("due[0] = %s for limit 1, want the refreshable provider %s, not the just-failed one %s", limited[0], healthyStale, dead) + } + + // ... and the deferral is a backoff, not a ban: once the backoff has + // elapsed the same provider is offered again. The caller computes the + // cutoff as (wall clock - backoff), so a poll exactly one backoff period + // after the attempt computes `now`. + afterBackoff := GetProviderEgressLocationDue(ctx, minObservedAt, now, 100) + if !slices.Contains(afterBackoff, dead) { + t.Fatalf("due = %v, must contain the failed provider (%s) again once the attempt backoff has elapsed", afterBackoff, dead) + } + }) +} + +// Only live, routable providers are probeable. A provider that has gone offline +// (connected = false) or that looks messed up from a routing perspective +// (valid = false, a generated column: more than one address hash or location on +// its live connections) must not be handed to the prober. Deleting either +// predicate from GetProviderEgressLocationDue must fail this test. +func TestGetProviderEgressLocationDueRequiresConnectedAndValid(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + + city := &Location{ + LocationType: LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, city) + + good := server.NewId() + disconnected := server.NewId() + invalid := server.NewId() + + testing_connectProbeableProvider(t, ctx, good, city.LocationId, "0.0.0.1:0", ProvideModePublic) + disconnectedConnectionId := testing_connectProbeableProvider(t, ctx, disconnected, city.LocationId, "0.0.0.2:0", ProvideModePublic) + + // `invalid` holds two simultaneous connections from two different + // addresses, which makes client_address_hash_count = 2 and so the + // generated `valid` column false. The two addresses must be in + // different /29s: server.ClientIpHash buckets ipv4 to the /29 network, + // so e.g. 0.0.0.3 and 0.0.0.4 would hash the same and count as one. + testing_connectProbeableProvider(t, ctx, invalid, city.LocationId, "0.0.0.3:0", ProvideModePublic) + secondHandlerId := CreateNetworkClientHandler(ctx) + secondConnectionId, _, _, _, err := ConnectNetworkClient(ctx, invalid, "0.0.8.3:0", secondHandlerId) + if err != nil { + t.Fatalf("connect second address: %s", err) + } + if err := SetConnectionLocation(ctx, secondConnectionId, city.LocationId, &ConnectionLocationScores{}); err != nil { + t.Fatalf("set second connection location: %s", err) + } + + // first roll-up: everything above is connected + UpdateClientLocationReliabilities(ctx, now.Add(-time.Hour), now) + + // `disconnected` drops off, and a second roll-up flips its reliability + // row's connected to false (the row itself survives) + if err := DisconnectNetworkClient(ctx, disconnectedConnectionId); err != nil { + t.Fatalf("disconnect client: %s", err) + } + UpdateClientLocationReliabilities(ctx, now.Add(-time.Hour), server.NowUtc()) + + due := GetProviderEgressLocationDue(ctx, now.Add(-24*time.Hour), now, 100) + + if !slices.Contains(due, good) { + t.Fatalf("due = %v, must contain the connected, valid provider (%s)", due, good) + } + if slices.Contains(due, disconnected) { + t.Fatalf("due = %v, must not contain the disconnected provider (%s)", due, disconnected) + } + if slices.Contains(due, invalid) { + t.Fatalf("due = %v, must not contain the provider whose reliability row is not valid (%s)", due, invalid) + } + }) +} + +// The attempt upsert is monotonic in attempt_at, for the same reason the +// location upsert is: a replayed or out-of-order report must not move the last +// attempt backwards and hand the provider back to the prober early. +func TestProviderEgressProbeAttemptUpsertIgnoresOlderReplay(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + clientId := server.NewId() + newer := server.NowUtc() + older := newer.Add(-time.Hour) + + SetProviderEgressProbeAttempt(ctx, &ProviderEgressProbeAttempt{ + ClientId: clientId, AttemptAt: newer, ProbeFailure: "no_consensus", + }) + SetProviderEgressProbeAttempt(ctx, &ProviderEgressProbeAttempt{ + ClientId: clientId, AttemptAt: older, ProbeFailure: "tunnel_failed", + }) + + got := GetProviderEgressProbeAttempt(ctx, clientId) + if got == nil { + t.Fatal("expected a stored probe attempt") + } + connect.AssertEqual(t, got.ProbeFailure, "no_consensus") + // postgres `timestamp` keeps microseconds, Go keeps nanoseconds, so + // compare with a tolerance rather than for equality + if delta := got.AttemptAt.Sub(newer); delta < -time.Millisecond || time.Millisecond < delta { + t.Fatalf("attempt_at = %s, want the newer attempt %s", got.AttemptAt, newer) + } + + // a strictly newer report does win + newest := newer.Add(time.Minute) + SetProviderEgressProbeAttempt(ctx, &ProviderEgressProbeAttempt{ + ClientId: clientId, AttemptAt: newest, ProbeFailure: "", + }) + got = GetProviderEgressProbeAttempt(ctx, clientId) + connect.AssertEqual(t, got.ProbeFailure, "") + + // absent + if GetProviderEgressProbeAttempt(ctx, server.NewId()) != nil { + t.Fatal("absent attempt must return nil") + } + }) +} + +func TestRemoveExpiredProviderEgressProbeAttempts(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + keep := server.NewId() + drop := server.NewId() + SetProviderEgressProbeAttempt(ctx, &ProviderEgressProbeAttempt{ + ClientId: keep, AttemptAt: server.NowUtc(), + }) + SetProviderEgressProbeAttempt(ctx, &ProviderEgressProbeAttempt{ + ClientId: drop, AttemptAt: server.NowUtc().Add(-30 * 24 * time.Hour), + }) + + RemoveExpiredProviderEgressProbeAttempts(ctx, server.NowUtc().Add(-24*time.Hour)) + + if GetProviderEgressProbeAttempt(ctx, keep) == nil { + t.Fatal("recent attempt must survive the sweep") + } + if GetProviderEgressProbeAttempt(ctx, drop) != nil { + t.Fatal("old attempt must be swept") + } + }) +} + +// GetProviderEgressLocationDue is served by two statements -- never-probed +// first, then stale-but-probed only when the first came up short -- because the +// single-statement form sorts on observed_at from an outer-joined table, which +// cannot use an index and becomes a full scan plus an unindexable sort at 100k +// providers. +// +// The split is only safe if the concatenation is row-for-row what one statement +// returned, at every limit. That is what this asserts: it builds one population +// covering every eligibility case and then walks the limit from 0 past the end, +// requiring each result to be exactly the prefix of the full ordering. Limit 3 +// is the seam (pass one exactly fills the batch) and limit 4 is the first that +// crosses into pass two. +func TestGetProviderEgressLocationDueOrderingIsStableAcrossLimits(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + now := server.NowUtc() + + city := &Location{ + LocationType: LocationTypeCity, + City: "Palo Alto", + Region: "California", + Country: "United States", + CountryCode: "us", + } + CreateLocation(ctx, city) + + // three never-probed providers: the dominant group, and the reason the + // ordering is NULLS FIRST + never := []server.Id{server.NewId(), server.NewId(), server.NewId()} + // two stale ones, at different ages -- the older must be handed out first + staleOlder := server.NewId() + staleNewer := server.NewId() + // probed an hour ago: not due + fresh := server.NewId() + // never probed, but attempted seconds ago: deferred by the backoff, and + // the case that would otherwise starve the queue + attempted := server.NewId() + // probed long ago AND attempted seconds ago. This one is only screened + // by the backoff predicate on the stale-but-probed pass -- the + // never-probed pass never sees it, because it has an egress row. Drop + // that predicate and it reappears in the batch. + staleAttempted := server.NewId() + // no Public provide key: unprobeable at any freshness + nonPublic := server.NewId() + + address := 0 + connectProvider := func(clientId server.Id, provideMode ProvideMode) { + address += 1 + testing_connectProbeableProvider( + t, ctx, clientId, city.LocationId, + fmt.Sprintf("0.0.%d.1:0", address), provideMode, + ) + } + for _, clientId := range never { + connectProvider(clientId, ProvideModePublic) + } + connectProvider(staleOlder, ProvideModePublic) + connectProvider(staleNewer, ProvideModePublic) + connectProvider(fresh, ProvideModePublic) + connectProvider(attempted, ProvideModePublic) + connectProvider(staleAttempted, ProvideModePublic) + connectProvider(nonPublic, ProvideModeNetwork) + + UpdateClientLocationReliabilities(ctx, now.Add(-time.Hour), now) + + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: staleOlder, LocationId: city.LocationId, + CountryCode: "us", ObservedAt: now.Add(-100 * time.Hour), + }) + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: staleNewer, LocationId: city.LocationId, + CountryCode: "us", ObservedAt: now.Add(-50 * time.Hour), + }) + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: fresh, LocationId: city.LocationId, + CountryCode: "us", ObservedAt: now.Add(-1 * time.Hour), + }) + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: staleAttempted, LocationId: city.LocationId, + CountryCode: "us", ObservedAt: now.Add(-200 * time.Hour), + }) + SetProviderEgressProbeAttempt(ctx, &ProviderEgressProbeAttempt{ + ClientId: attempted, AttemptAt: now.Add(-5 * time.Second), + ProbeFailure: "tunnel_failed", + }) + // oldest observed_at of all, so it would sort to the head of the + // stale group if the backoff did not exclude it + SetProviderEgressProbeAttempt(ctx, &ProviderEgressProbeAttempt{ + ClientId: staleAttempted, AttemptAt: now.Add(-5 * time.Second), + ProbeFailure: "tunnel_failed", + }) + + minObservedAt := now.Add(-24 * time.Hour) + minAttemptAt := now.Add(-ProviderEgressProbeAttemptBackoff) + + // the never-probed group ties on a missing observed_at, so client_id + // alone orders it -- and postgres orders uuid by bytes, which is what + // server.Id.Cmp does + expected := slices.Clone(never) + slices.SortFunc(expected, func(a server.Id, b server.Id) int { return a.Cmp(b) }) + // ... then the probed group, oldest probe first + expected = append(expected, staleOlder, staleNewer) + + due := GetProviderEgressLocationDue(ctx, minObservedAt, minAttemptAt, 100) + if !slices.Equal(due, expected) { + t.Fatalf("due = %v, want %v (never-probed by client_id, then stale oldest-first; fresh/attempted/stale-attempted/non-public excluded)", due, expected) + } + + // every limit must return exactly the prefix of that ordering. limit 3 + // is the pass-one/pass-two seam; 4 is the first to cross it. + for limit := 0; limit <= len(expected)+2; limit += 1 { + want := expected[:min(limit, len(expected))] + got := GetProviderEgressLocationDue(ctx, minObservedAt, minAttemptAt, limit) + if !slices.Equal(got, want) { + t.Errorf("limit %d: due = %v, want %v", limit, got, want) + } + } + }) +} + +// TestProviderEgressLocationHasVerdictColumns asserts the three verdict columns +// exist. They are additive with safe defaults, so no existing reader or row is +// affected -- but nothing can record a verdict until they are there. +func TestProviderEgressLocationHasVerdictColumns(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + for _, col := range []string{"verdict", "verdict_reason", "assurance"} { + var exists bool + server.Db(ctx, func(conn server.PgConn) { + result, err := conn.Query( + ctx, + ` + SELECT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = 'provider_egress_location' AND column_name = $1 + ) + `, + col, + ) + server.WithPgResult(result, err, func() { + if result.Next() { + server.Raise(result.Scan(&exists)) + } + }) + }) + if !exists { + t.Errorf("provider_egress_location missing column %q", col) + } + } + }) +} + +// TestProviderEgressLocationVerdictDefaults pins the write path's normalization. +// SetProviderEgressLocation names every column explicitly, which bypasses the +// column defaults, so a caller that computes no judgement -- every caller until +// the ingest path does -- must still store unverified/direct, not the empty +// string. +func TestProviderEgressLocationVerdictDefaults(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + + clientId := server.NewId() + observedAt := server.NowUtc() + + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: clientId, + LocationId: server.NewId(), + CountryCode: "es", + ObservedAt: observedAt, + }) + + stored := GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected a stored egress location") + } + connect.AssertEqual(t, stored.Verdict, ProviderEgressVerdictUnverified) + connect.AssertEqual(t, stored.VerdictReason, "") + connect.AssertEqual(t, stored.Assurance, ProviderEgressAssuranceDirect) + + // an explicit judgement is stored verbatim + SetProviderEgressLocation(ctx, &ProviderEgressLocation{ + ClientId: clientId, + LocationId: server.NewId(), + CountryCode: "de", + ObservedAt: observedAt.Add(time.Hour), + Verdict: "suspect", + VerdictReason: "unstable", + Assurance: ProviderEgressAssuranceDirect, + }) + + stored = GetProviderEgressLocation(ctx, clientId) + if stored == nil { + t.Fatal("expected a stored egress location") + } + connect.AssertEqual(t, stored.Verdict, "suspect") + connect.AssertEqual(t, stored.VerdictReason, "unstable") + connect.AssertEqual(t, stored.Assurance, ProviderEgressAssuranceDirect) + }) +} diff --git a/probeverdict/probeverdict.go b/probeverdict/probeverdict.go new file mode 100644 index 00000000..736ed5a7 --- /dev/null +++ b/probeverdict/probeverdict.go @@ -0,0 +1,55 @@ +// Package probeverdict turns a geolocation probe submission into a verdict: +// verified, unverified, or suspect. It is pure decision logic with no I/O, so +// it is fully table-testable independent of how a submission arrived. +// +// Deliberately absent from Input: the mmdb-derived country for the same +// connection, and any RTT or coordinate fields. A probed country differing +// from what the free mmdb would have said is the entire point of this +// project and must never be treated as suspicious -- keeping that field off +// Input makes the omission structural. An RTT-distance corroboration was +// designed and dropped before implementation (see the spec's "The RTT floor +// was designed, then dropped"): it needs a fixed reference point that this +// system does not have a single answer for once more than one deployment +// instance exists, and a wrong reference point does not fail safe -- it can +// flag an honest provider as suspect. Do not reintroduce ObservedRTT or +// coordinate fields here without first resolving that reference-point +// problem in the spec. +package probeverdict + +import "time" + +type Input struct { + CountryConfident bool + CountryCode string + + PreviousCountryCode string + PreviousObservedAt time.Time + Now time.Time +} + +type Verdict struct { + State string + Reason string +} + +// unstableWindow is how recently a prior, different country counts as a +// flip-flop rather than a legitimate correction. +const unstableWindow = 24 * time.Hour + +func Evaluate(in Input) Verdict { + if !in.CountryConfident { + return Verdict{State: "unverified", Reason: "no_consensus"} + } + + if in.PreviousCountryCode != "" && in.PreviousCountryCode != in.CountryCode { + now := in.Now + if now.IsZero() { + now = time.Now() + } + if now.Sub(in.PreviousObservedAt) < unstableWindow { + return Verdict{State: "suspect", Reason: "unstable"} + } + } + + return Verdict{State: "verified"} +} diff --git a/probeverdict/probeverdict_test.go b/probeverdict/probeverdict_test.go new file mode 100644 index 00000000..730967bd --- /dev/null +++ b/probeverdict/probeverdict_test.go @@ -0,0 +1,97 @@ +package probeverdict + +import ( + "reflect" + "testing" + "time" +) + +func TestEvaluateNoConsensusIsUnverified(t *testing.T) { + v := Evaluate(Input{CountryConfident: false}) + if v.State != "unverified" || v.Reason != "no_consensus" { + t.Errorf("got %+v, want unverified/no_consensus", v) + } +} + +func TestEvaluateCountryFlipFlopIsSuspect(t *testing.T) { + now := time.Now() + v := Evaluate(Input{ + CountryConfident: true, + CountryCode: "de", + PreviousCountryCode: "es", + PreviousObservedAt: now.Add(-2 * time.Hour), + Now: now, + }) + if v.State != "suspect" || v.Reason != "unstable" { + t.Errorf("got %+v, want suspect/unstable", v) + } +} + +func TestEvaluateCountryChangeOutsideWindowIsVerified(t *testing.T) { + // a country change is only "unstable" within the 24h window -- after it, + // a changed country is a legitimate correction, not a flip-flop + now := time.Now() + v := Evaluate(Input{ + CountryConfident: true, + CountryCode: "de", + PreviousCountryCode: "es", + PreviousObservedAt: now.Add(-25 * time.Hour), + Now: now, + }) + if v.State != "verified" { + t.Errorf("got %+v, want verified (change is outside the 24h window)", v) + } +} + +func TestEvaluateMmdbDivergenceAloneIsNotSuspect(t *testing.T) { + // this test asserts the single most important safety property in the + // package: a country that differs from what mmdb would have said is NOT + // an input to Evaluate at all -- there is no field for it on Input, so a + // clean first-time probe always verifies regardless of what mmdb would + // have said about the same connection. + v := Evaluate(Input{ + CountryConfident: true, + CountryCode: "es", + }) + if v.State != "verified" { + t.Errorf("a clean probe with no prior history must verify regardless of what mmdb would have said, got %+v", v) + } +} + +// TestInputHasNoMmdbRttOrCoordinateFields locks the two structural omissions +// this package depends on for correctness. Neither can be asserted +// behaviourally -- they are the absence of inputs, so the only way to test +// them is to pin the field set itself. +// +// 1. No mmdb-derived country. A probed country diverging from what the free +// mmdb would have said is the entire point of this project, and with no +// field for it a caller cannot get the rule wrong. +// 2. No RTT or coordinate fields. An RTT-distance floor was designed and +// deliberately dropped (see the package doc comment and the spec's "The +// RTT floor was designed, then dropped"): it needs a fixed reference +// point this system has no single answer for, and a wrong reference point +// does not fail safe -- it can flag an honest provider as suspect. +// +// If this test fails because a field was added, that is the point: resolve +// the reference-point problem in the spec first. +func TestInputHasNoMmdbRttOrCoordinateFields(t *testing.T) { + allowed := map[string]bool{ + "CountryConfident": true, + "CountryCode": true, + "PreviousCountryCode": true, + "PreviousObservedAt": true, + "Now": true, + } + + inputType := reflect.TypeOf(Input{}) + for i := range inputType.NumField() { + name := inputType.Field(i).Name + if !allowed[name] { + t.Errorf("Input has an unexpected field %q: mmdb-country, RTT and coordinate "+ + "fields are deliberately absent from Input; see the package doc comment", name) + } + } + if inputType.NumField() != len(allowed) { + t.Errorf("Input has %d fields, want exactly %d", inputType.NumField(), len(allowed)) + } +} diff --git a/taskworker/taskworker.go b/taskworker/taskworker.go index 3364dce0..7f798349 100644 --- a/taskworker/taskworker.go +++ b/taskworker/taskworker.go @@ -52,6 +52,8 @@ func InitTasks(ctx context.Context) { work.ScheduleRemoveExpiredAuthAttempts(clientSession, tx) work.ScheduleRemoveExpiredWalletAuthChallenges(clientSession, tx) work.ScheduleRemoveExpiredWalletNonces(clientSession, tx) + work.ScheduleRemoveExpiredProviderEgressLocations(clientSession, tx) + work.ScheduleRefreshGeolocationSourcePins(clientSession, tx) work.ScheduleRemoveOldAuditNetworkEvents(clientSession, tx) work.ScheduleRemoveOldAuditEvents(clientSession, tx) work.ScheduleRemoveOldClientReliabilityStats(clientSession, tx) @@ -236,6 +238,14 @@ func InitTaskWorkerWithSettings(ctx context.Context, settings *task.TaskWorkerSe work.RemoveExpiredWalletNoncesPost, "github.com/urnetwork/server/taskworker/work.RemoveExpiredWalletNonces", ), + task.NewTaskTargetWithPost( + work.RemoveExpiredProviderEgressLocations, + work.RemoveExpiredProviderEgressLocationsPost, + ), + task.NewTaskTargetWithPost( + work.RefreshGeolocationSourcePins, + work.RefreshGeolocationSourcePinsPost, + ), task.NewTaskTargetWithPost( work.RemoveOldAuditNetworkEvents, work.RemoveOldAuditNetworkEventsPost, diff --git a/taskworker/work/geolocation_source_pin_work.go b/taskworker/work/geolocation_source_pin_work.go new file mode 100644 index 00000000..52d107d4 --- /dev/null +++ b/taskworker/work/geolocation_source_pin_work.go @@ -0,0 +1,348 @@ +package work + +import ( + "context" + "crypto/sha256" + "crypto/tls" + "crypto/x509" + "encoding/base64" + "fmt" + "net" + "time" + + "github.com/urnetwork/glog" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" + "github.com/urnetwork/server/session" + "github.com/urnetwork/server/task" +) + +// GeolocationSourcePinRefreshTimeout is how often every source host is +// re-observed. Six hours is far shorter than any certificate's life and far +// longer than the handful of TLS handshakes it costs, so a rotation is picked +// up within a quarter of a day without the job ever being a load consideration. +const GeolocationSourcePinRefreshTimeout = 6 * time.Hour + +// geolocationSourceDialTimeout bounds one host's dial + handshake. A host that +// blackholes rather than refusing is a failure mode too, and an untimed dial +// would let one such host stall the whole pass -- which would leave the +// remaining hosts un-refreshed and look exactly like the silent staleness this +// job exists to prevent. +const geolocationSourceDialTimeout = 15 * time.Second + +// geolocationSourceTarget is one host to observe and the address to reach it +// at. Production always sets Addr to Host:443 (see productionGeolocationSourceTargets). +// +// Addr is a parameter ONLY so tests can point a host at a local listener. Note +// what is and is not injectable here: an address, and nothing else. The dialer, +// the *tls.Config, and the root pool are all built inside this package by +// sourceTLSConfig and observeGeolocationSourcePin, and there is deliberately no +// way for a caller to supply a transport, a connection, a tls.Config or a +// RootCAs override. That is the line this whole feature rests on -- a pin is +// only ever recorded from a connection this package itself made and verified +// against the system roots. A caller-supplied transport could be a provider +// tunnel, and a caller-supplied root pool could be a provider's own CA; either +// would let the provider under test choose the pin that is supposed to catch +// it. +type geolocationSourceTarget struct { + Host string + Addr string +} + +// productionGeolocationSourceTargets is the real target list: every known +// source host, dialed on 443 at its own name. +func productionGeolocationSourceTargets() []geolocationSourceTarget { + targets := make([]geolocationSourceTarget, 0, len(model.GeolocationSourceHosts)) + for _, host := range model.GeolocationSourceHosts { + targets = append(targets, geolocationSourceTarget{ + Host: host, + Addr: net.JoinHostPort(host, "443"), + }) + } + return targets +} + +// GeolocationSourcePinChange is one host's pin changing between two +// observations, carrying both the old and the new value. +// +// This is a return value rather than only a log line so the "rotation is +// visible" property is testable: a test can assert the change record carries +// what it replaced, which is the part that was missing during the outage. The +// job logs each record; nothing else consumes them. +// +// OldLeaf/OldIntermediate are empty for a host's first ever observation, which +// is a first sighting rather than a rotation and is logged as such. +type GeolocationSourcePinChange struct { + Host string + OldLeaf string + NewLeaf string + OldIntermediate string + NewIntermediate string + FirstObservation bool +} + +// sourceTLSConfig builds the tls.Config used for every observation. +// +// InsecureSkipVerify is false and ServerName is set, and both are +// load-bearing rather than boilerplate: +// +// - InsecureSkipVerify false is what makes the observation mean anything. It +// is also what populates ConnectionState().VerifiedChains at all: crypto/tls +// leaves the verified chains empty when verification is skipped, so the +// observation code below structurally cannot record a pin from an +// unverified handshake -- it has nothing to read. +// - ServerName pins the identity being verified to the host being recorded. +// Without it, hostname verification would have nothing to check against and +// any chain-valid certificate for any name would satisfy the handshake. +// +// No RootCAs is set, so the system root pool is used. There is no override and +// there must never be one, not even for tests: a settable root pool is exactly +// the injection point that would let something other than the public WebPKI +// decide what this server vouches for. +func sourceTLSConfig(host string) *tls.Config { + return &tls.Config{ + ServerName: host, + MinVersion: tls.VersionTLS12, + InsecureSkipVerify: false, + } +} + +// spkiPin is the base64 sha-256 of a certificate's subject public key info, +// byte-for-byte the same form the prober's providertunnel.SPKIPin computes and +// compares against. Hashing the key rather than the certificate is what lets a +// pin survive a renewal that keeps the same key. +func spkiPin(cert *x509.Certificate) string { + sum := sha256.Sum256(cert.RawSubjectPublicKeyInfo) + return base64.StdEncoding.EncodeToString(sum[:]) +} + +// observeGeolocationSourcePin dials addr directly, completes a fully verified +// TLS handshake as host, and returns the leaf and issuing-intermediate SPKI +// pins from the VERIFIED chain. +// +// It reads ConnectionState().VerifiedChains, never PeerCertificates, and that +// is a security property rather than a preference. PeerCertificates is whatever +// the peer sent; crypto/tls only ever uses the entries past the leaf as a pool +// to build a path from, and never promises the path it built used them. So a +// peer may pad its Certificate message with any publicly downloadable +// certificate it likes, and a PeerCertificates[1]-based reading would happily +// record that inert padding as this host's intermediate pin. VerifiedChains +// contains only certificates on a path crypto/tls actually validated to a +// system root. The prober's checkPin matches against verified chains for the +// same reason (see providertunnel/pinning.go), so recording from anything else +// would also record a pin the prober could never match. +// +// Any failure -- dial, handshake, hostname mismatch, expiry, an untrusted +// issuer, or a chain too short to have an issuer in it -- returns an error and +// no pin. The caller must leave the stored row alone in that case. +func observeGeolocationSourcePin(ctx context.Context, host string, addr string) (leafSpki string, intermediateSpki string, err error) { + dialer := &tls.Dialer{ + NetDialer: &net.Dialer{Timeout: geolocationSourceDialTimeout}, + Config: sourceTLSConfig(host), + } + + dialCtx, cancel := context.WithTimeout(ctx, geolocationSourceDialTimeout) + defer cancel() + + conn, err := dialer.DialContext(dialCtx, "tcp", addr) + if err != nil { + return "", "", err + } + defer conn.Close() + + tlsConn, ok := conn.(*tls.Conn) + if !ok { + // tls.Dialer always returns a *tls.Conn; this is here so a future + // change cannot silently turn into a pin read from a plain connection. + return "", "", fmt.Errorf("expected a tls connection to %s (%s), got %T", host, addr, conn) + } + + chains := tlsConn.ConnectionState().VerifiedChains + if len(chains) == 0 { + // unreachable with InsecureSkipVerify false -- crypto/tls aborts the + // handshake before this point when verification fails -- and it stays + // unreachable only as long as sourceTLSConfig keeps verification on. + // Failing here means a verification-disabled config could never write a + // pin even if one were somehow introduced. + return "", "", fmt.Errorf("no verified certificate chain for %s (%s)", host, addr) + } + chain := chains[0] + if len(chain) < 2 { + // a leaf with no issuer on the verified path: nothing to record as the + // intermediate pin. Writing an empty intermediate would be worse than + // writing nothing, since an empty pin matches no certificate at all. + return "", "", fmt.Errorf("verified chain for %s (%s) has %d certificates, need at least a leaf and its issuer", host, addr, len(chain)) + } + + // chain[0] is the leaf and chain[1] is its issuer on the validated path. + // For a two-certificate chain the issuer is the root itself, which is a + // broader pin than an intermediate would be -- but no broader than the + // trust the handshake already placed in that root, and the prober accepts a + // match anywhere in the chain regardless. + return spkiPin(chain[0]), spkiPin(chain[1]), nil +} + +// refreshGeolocationSourcePins observes every target and stores what it saw. +// +// Per-host isolation is the point. A host that fails validation leaves its +// stored row exactly as it was and contributes an error; the loop continues to +// the next host. Nothing is deleted, nothing is blanked, and no host's failure +// can cost another host its pin -- the failure mode that started this was one +// stale pin quietly shrinking the usable source set below the consensus +// minimum, and a job that abandoned the remaining hosts on the first error +// would reproduce it exactly. +// +// The store call is only reached after a successful, verified observation, so +// the "leave the previous row untouched" guarantee is structural rather than a +// branch that has to be remembered. +func refreshGeolocationSourcePins( + ctx context.Context, + targets []geolocationSourceTarget, +) (changes []GeolocationSourcePinChange, errs []error) { + for _, target := range targets { + leafSpki, intermediateSpki, err := observeGeolocationSourcePin(ctx, target.Host, target.Addr) + if err != nil { + errs = append(errs, fmt.Errorf("observe %s (%s): %w", target.Host, target.Addr, err)) + continue + } + + previous := model.SetGeolocationSourcePin(ctx, &model.GeolocationSourcePin{ + Host: target.Host, + LeafSpki: leafSpki, + IntermediateSpki: intermediateSpki, + ObservedAt: server.NowUtc(), + }) + + if previous == nil { + changes = append(changes, GeolocationSourcePinChange{ + Host: target.Host, + NewLeaf: leafSpki, + NewIntermediate: intermediateSpki, + FirstObservation: true, + }) + continue + } + if previous.LeafSpki != leafSpki || previous.IntermediateSpki != intermediateSpki { + changes = append(changes, GeolocationSourcePinChange{ + Host: target.Host, + OldLeaf: previous.LeafSpki, + NewLeaf: leafSpki, + OldIntermediate: previous.IntermediateSpki, + NewIntermediate: intermediateSpki, + }) + } + } + + return changes, errs +} + +type RefreshGeolocationSourcePinsArgs struct{} + +type RefreshGeolocationSourcePinsResult struct { + // counts only; the detail is in the log, and a task result is not a place + // anything should be reading pins back out of + Observed int `json:"observed"` + Changed int `json:"changed"` + Failed int `json:"failed"` +} + +// ScheduleRefreshGeolocationSourcePins schedules the first observation to run +// IMMEDIATELY, which is a deliberate departure from the usual Schedule* pattern +// of arming the first run one interval out. +// +// This job is not a cleanup job whose first pass can wait. Its table is what +// the prober pins against, and the prober's correct response to a missing pin +// is to refuse to probe -- so an empty table means no probing at all. Arming +// the first run six hours out would leave a freshly-migrated deployment unable +// to probe anything for those six hours, for no benefit. The recurring six-hour +// cadence is set in the Post below. +// +// RunOnce merges on conflict with `run_at = LEAST(existing, new)`, so a +// taskworker restart pulls a pending observation forward to now rather than +// stacking a second one. That costs one TLS handshake per source host per +// restart, which is nothing, and it means a restart after a rotation picks the +// new certificate up at once. +func ScheduleRefreshGeolocationSourcePins(clientSession *session.ClientSession, tx server.PgTx) { + scheduleRefreshGeolocationSourcePinsAt(clientSession, tx, server.NowUtc()) +} + +func scheduleRefreshGeolocationSourcePinsAt(clientSession *session.ClientSession, tx server.PgTx, runAt time.Time) { + task.ScheduleTaskInTx( + tx, + RefreshGeolocationSourcePins, + &RefreshGeolocationSourcePinsArgs{}, + clientSession, + task.RunOnce("refresh_geolocation_source_pins"), + task.RunAt(runAt), + ) +} + +// RefreshGeolocationSourcePins re-observes every geolocation source host and +// records what it saw, so the prober's pins track the real certificates instead +// of a constant someone pasted in weeks ago. +// +// It returns nil even when hosts failed, and that is deliberate. A task that +// returns an error is rescheduled with exponential backoff and its Post -- the +// function that re-arms the six-hourly chain -- does not run. So returning an +// error whenever any single host was unreachable would let one flaky host slow +// and eventually strand the refresh for ALL hosts: a job that quietly stops is +// the precise failure shape this feature exists to remove. Failures are loud in +// the log and counted in the result instead, and the chain always re-arms. +func RefreshGeolocationSourcePins( + _ *RefreshGeolocationSourcePinsArgs, + clientSession *session.ClientSession, +) (*RefreshGeolocationSourcePinsResult, error) { + targets := productionGeolocationSourceTargets() + changes, errs := refreshGeolocationSourcePins(clientSession.Ctx, targets) + + for _, change := range changes { + if change.FirstObservation { + glog.Infof( + "[gsp]first observation for %s: leaf=%s intermediate=%s\n", + change.Host, + change.NewLeaf, + change.NewIntermediate, + ) + continue + } + // old AND new, always. A rotation that is only visible as "the pin is + // different now" is the same as no record at all when someone is trying + // to work out why a source dropped out. + glog.Errorf( + "[gsp]pin ROTATED for %s: leaf %s -> %s, intermediate %s -> %s\n", + change.Host, + change.OldLeaf, + change.NewLeaf, + change.OldIntermediate, + change.NewIntermediate, + ) + } + for _, err := range errs { + // loud: a host that cannot be validated keeps serving its previous pin + // to the prober, which is correct but is also exactly how a pin goes + // stale without anyone noticing + glog.Errorf("[gsp]observation FAILED, previous pin left in place: %v\n", err) + } + + return &RefreshGeolocationSourcePinsResult{ + Observed: len(targets) - len(errs), + Changed: len(changes), + Failed: len(errs), + }, nil +} + +func RefreshGeolocationSourcePinsPost( + _ *RefreshGeolocationSourcePinsArgs, + _ *RefreshGeolocationSourcePinsResult, + clientSession *session.ClientSession, + tx server.PgTx, +) error { + // the recurring cadence; only the very first run is immediate + scheduleRefreshGeolocationSourcePinsAt( + clientSession, + tx, + server.NowUtc().Add(GeolocationSourcePinRefreshTimeout), + ) + return nil +} diff --git a/taskworker/work/geolocation_source_pin_work_test.go b/taskworker/work/geolocation_source_pin_work_test.go new file mode 100644 index 00000000..1f610283 --- /dev/null +++ b/taskworker/work/geolocation_source_pin_work_test.go @@ -0,0 +1,261 @@ +package work + +import ( + "context" + "net" + "net/http" + "net/http/httptest" + "net/url" + "testing" + "time" + + "github.com/urnetwork/connect" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" +) + +// TestSourceTLSConfigMandatesWebPKI is the cheap, direct assertion of the line +// this whole feature rests on. A pin may only ever be recorded from a +// connection that passed full WebPKI validation, and these three fields are +// what enforce that: verification on, an identity to verify against, and the +// system root pool rather than one someone handed us. +// +// The signature of observeGeolocationSourcePin carries the other half: it +// accepts a host and an address, and there is no parameter through which a +// caller could supply a transport, a connection, a tls.Config or a root pool. +// A provider tunnel therefore cannot be the path a pin is learned over, because +// there is no way to pass one in. +func TestSourceTLSConfigMandatesWebPKI(t *testing.T) { + cfg := sourceTLSConfig("ipinfo.io") + + if cfg.InsecureSkipVerify { + t.Fatal("InsecureSkipVerify must be false: an unverified chain would let the host being probed choose its own pin") + } + connect.AssertEqual(t, cfg.ServerName, "ipinfo.io") + if cfg.RootCAs != nil { + t.Fatal("RootCAs must be nil so the system roots are used; an overridable root pool is the injection point this design exists to close") + } + if cfg.VerifyPeerCertificate != nil { + t.Fatal("no custom peer verification: the standard WebPKI check is the whole point") + } +} + +// selfSignedTLSServer starts an httptest TLS server whose certificate is signed +// by its own throwaway CA, which is in no system root store. This is the +// hostile case: a server presenting a certificate that does not validate. +func selfSignedTLSServer(t testing.TB) (addr string, stop func()) { + t.Helper() + ts := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + u, err := url.Parse(ts.URL) + if err != nil { + ts.Close() + t.Fatalf("parse httptest url: %v", err) + } + return u.Host, ts.Close +} + +// requireRealSourceReachable skips when this machine has no outbound path to a +// real source host. The tests that use it are the only ones that can +// demonstrate a SUCCESSFUL observation: a success requires a chain that +// validates against the system roots, and manufacturing one hermetically would +// mean injecting a root pool -- which is exactly what this design forbids. So +// the hermetic tests cover the hostile cases and these cover the good path. +// +// The gate is a plain TCP dial, deliberately NOT a call to +// observeGeolocationSourcePin. Gating on the function under test would make +// these tests SKIP rather than FAIL whenever observation itself broke, which is +// the vacuous-coverage trap: no network skips, broken observation fails. +func requireRealSourceReachable(t testing.TB, host string) (addr string) { + t.Helper() + addr = net.JoinHostPort(host, "443") + conn, err := net.DialTimeout("tcp", addr, 10*time.Second) + if err != nil { + t.Skipf("no outbound path to %s (%v); skipping the real-observation case", addr, err) + } + conn.Close() + return addr +} + +// TestRefreshGeolocationSourcePinsLeavesPriorRowUnchangedOnSelfSignedServer is +// the most important test here. A host that fails validation must leave what +// was already stored EXACTLY as it was and raise -- never overwrite a good pin +// with an unverified one, and never blank it either. Overwriting would let a +// hostile server choose the pin; blanking would shrink the usable source set, +// which is the failure that took the whole fleet's consensus offline. +func TestRefreshGeolocationSourcePinsLeavesPriorRowUnchangedOnSelfSignedServer(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + addr, stop := selfSignedTLSServer(t) + defer stop() + + observedAt := server.NowUtc().Add(-time.Hour).Truncate(time.Millisecond) + good := &model.GeolocationSourcePin{ + Host: "ipinfo.io", + LeafSpki: "good-leaf", + IntermediateSpki: "good-intermediate", + ObservedAt: observedAt, + } + model.SetGeolocationSourcePin(ctx, good) + + changes, errs := refreshGeolocationSourcePins(ctx, []geolocationSourceTarget{ + {Host: "ipinfo.io", Addr: addr}, + }) + + if len(errs) != 1 { + t.Fatalf("expected the self-signed server to be rejected, got errs=%v", errs) + } + connect.AssertEqual(t, len(changes), 0) + + after := model.GetGeolocationSourcePin(ctx, "ipinfo.io") + if after == nil { + t.Fatal("the previously stored pin was removed by a failed observation") + } + connect.AssertEqual(t, after.LeafSpki, good.LeafSpki) + connect.AssertEqual(t, after.IntermediateSpki, good.IntermediateSpki) + connect.AssertEqual(t, after.ObservedAt.UTC().Equal(observedAt.UTC()), true) + }) +} + +// TestRefreshGeolocationSourcePinsLeavesPriorRowUnchangedOnHostnameMismatch is +// the same guarantee for the other half of WebPKI. Here the chain is genuinely +// valid and genuinely trusted -- it is a real source host's real certificate -- +// but it is not for the name being asked for. Without ServerName set and +// verification on, this would be accepted and one host's certificate would +// become another host's pin. +func TestRefreshGeolocationSourcePinsLeavesPriorRowUnchangedOnHostnameMismatch(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + addr := requireRealSourceReachable(t, "ipinfo.io") + + observedAt := server.NowUtc().Add(-time.Hour).Truncate(time.Millisecond) + model.SetGeolocationSourcePin(ctx, &model.GeolocationSourcePin{ + Host: "free.freeipapi.com", + LeafSpki: "good-leaf", + IntermediateSpki: "good-intermediate", + ObservedAt: observedAt, + }) + + // ipinfo.io's real address, asked for under a name its certificate does + // not cover + changes, errs := refreshGeolocationSourcePins(ctx, []geolocationSourceTarget{ + {Host: "free.freeipapi.com", Addr: addr}, + }) + + if len(errs) != 1 { + t.Fatalf("expected a hostname mismatch to be rejected, got errs=%v", errs) + } + connect.AssertEqual(t, len(changes), 0) + + after := model.GetGeolocationSourcePin(ctx, "free.freeipapi.com") + if after == nil { + t.Fatal("the previously stored pin was removed by a failed observation") + } + connect.AssertEqual(t, after.LeafSpki, "good-leaf") + connect.AssertEqual(t, after.IntermediateSpki, "good-intermediate") + connect.AssertEqual(t, after.ObservedAt.UTC().Equal(observedAt.UTC()), true) + }) +} + +// TestRefreshGeolocationSourcePinsReportsOldAndNewOnRotation covers the +// "rotation must be visible" requirement. The change record carries both the +// value that was there and the value that replaced it, which is what the job +// logs. A record that carried only the new value would be no better than the +// silence that made the original outage undiagnosable. +func TestRefreshGeolocationSourcePinsReportsOldAndNewOnRotation(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + host := "ipinfo.io" + addr := requireRealSourceReachable(t, host) + + // stand in for a pin observed before the certificate rotated + model.SetGeolocationSourcePin(ctx, &model.GeolocationSourcePin{ + Host: host, + LeafSpki: "stale-leaf", + IntermediateSpki: "stale-intermediate", + ObservedAt: server.NowUtc().Add(-24 * time.Hour), + }) + + changes, errs := refreshGeolocationSourcePins(ctx, []geolocationSourceTarget{ + {Host: host, Addr: addr}, + }) + connect.AssertEqual(t, len(errs), 0) + if len(changes) != 1 { + t.Fatalf("expected exactly one change record, got %+v", changes) + } + + change := changes[0] + connect.AssertEqual(t, change.Host, host) + connect.AssertEqual(t, change.FirstObservation, false) + connect.AssertEqual(t, change.OldLeaf, "stale-leaf") + connect.AssertEqual(t, change.OldIntermediate, "stale-intermediate") + if change.NewLeaf == "" || change.NewIntermediate == "" { + t.Fatalf("expected observed leaf and intermediate pins, got %+v", change) + } + if change.NewLeaf == change.OldLeaf || change.NewIntermediate == change.OldIntermediate { + t.Fatalf("expected the observed pins to differ from the stale ones, got %+v", change) + } + // what was logged is what was stored + stored := model.GetGeolocationSourcePin(ctx, host) + connect.AssertEqual(t, stored.LeafSpki, change.NewLeaf) + connect.AssertEqual(t, stored.IntermediateSpki, change.NewIntermediate) + + // re-observing an unchanged certificate is not a rotation and must not + // be reported as one, or the log stops meaning anything + changes, errs = refreshGeolocationSourcePins(ctx, []geolocationSourceTarget{ + {Host: host, Addr: addr}, + }) + connect.AssertEqual(t, len(errs), 0) + connect.AssertEqual(t, len(changes), 0) + }) +} + +// TestRefreshGeolocationSourcePinsContinuesPastAFailingHost: one bad host must +// never cost the others their observation. The original outage was two sources +// dropping out at once and leaving one against a minimum of two; a refresh that +// abandoned the remaining hosts after the first failure would manufacture that +// same shortfall from a single unreachable endpoint. +func TestRefreshGeolocationSourcePinsContinuesPastAFailingHost(t *testing.T) { + server.DefaultTestEnv().Run(t, func(t testing.TB) { + ctx := context.Background() + goodHost := "ipinfo.io" + goodAddr := requireRealSourceReachable(t, goodHost) + badAddr, stop := selfSignedTLSServer(t) + defer stop() + + // the failing host is FIRST, so a loop that stopped on the first error + // would never reach the good one + changes, errs := refreshGeolocationSourcePins(ctx, []geolocationSourceTarget{ + {Host: "free.freeipapi.com", Addr: badAddr}, + {Host: goodHost, Addr: goodAddr}, + }) + + connect.AssertEqual(t, len(errs), 1) + if len(changes) != 1 { + t.Fatalf("expected the good host to still be observed, got %+v", changes) + } + connect.AssertEqual(t, changes[0].Host, goodHost) + connect.AssertEqual(t, changes[0].FirstObservation, true) + + if model.GetGeolocationSourcePin(ctx, goodHost) == nil { + t.Fatal("the good host was not stored") + } + if model.GetGeolocationSourcePin(ctx, "free.freeipapi.com") != nil { + t.Fatal("a failed observation must not write a row") + } + }) +} + +// TestProductionGeolocationSourceTargetsCoverEverySourceHost: the real target +// list is derived from model.GeolocationSourceHosts and dials each host on 443 +// under its own name. Nothing else may become a target. +func TestProductionGeolocationSourceTargetsCoverEverySourceHost(t *testing.T) { + targets := productionGeolocationSourceTargets() + connect.AssertEqual(t, len(targets), len(model.GeolocationSourceHosts)) + for i, host := range model.GeolocationSourceHosts { + connect.AssertEqual(t, targets[i].Host, host) + connect.AssertEqual(t, targets[i].Addr, net.JoinHostPort(host, "443")) + } +} diff --git a/taskworker/work/provider_egress_location_work.go b/taskworker/work/provider_egress_location_work.go new file mode 100644 index 00000000..533d9609 --- /dev/null +++ b/taskworker/work/provider_egress_location_work.go @@ -0,0 +1,55 @@ +package work + +import ( + "time" + + "github.com/urnetwork/server" + "github.com/urnetwork/server/model" + "github.com/urnetwork/server/session" + "github.com/urnetwork/server/task" +) + +type RemoveExpiredProviderEgressLocationsArgs struct{} + +type RemoveExpiredProviderEgressLocationsResult struct{} + +func ScheduleRemoveExpiredProviderEgressLocations(clientSession *session.ClientSession, tx server.PgTx) { + task.ScheduleTaskInTx( + tx, + RemoveExpiredProviderEgressLocations, + &RemoveExpiredProviderEgressLocationsArgs{}, + clientSession, + task.RunOnce("remove_expired_provider_egress_locations"), + task.RunAt(server.NowUtc().Add(6*time.Hour)), + ) +} + +// RemoveExpiredProviderEgressLocations drops probed locations well past their +// trust window, so a provider that stops being probed eventually falls back to +// the mmdb path instead of being pinned to a stale location forever. The cutoff +// is deliberately looser than ProviderEgressLocationMaxAge: reads already +// ignore stale rows, so this is only reclaiming storage. +func RemoveExpiredProviderEgressLocations( + _ *RemoveExpiredProviderEgressLocationsArgs, + clientSession *session.ClientSession, +) (*RemoveExpiredProviderEgressLocationsResult, error) { + now := server.NowUtc() + minObservedAt := now.Add(-4 * model.ProviderEgressLocationMaxAge) + model.RemoveExpiredProviderEgressLocations(clientSession.Ctx, minObservedAt) + // probe attempts stop meaning anything once they no longer defer the + // provider; same reasoning as above, a looser multiple of the window that + // actually matters. + minAttemptAt := now.Add(-4 * model.ProviderEgressProbeAttemptBackoff) + model.RemoveExpiredProviderEgressProbeAttempts(clientSession.Ctx, minAttemptAt) + return &RemoveExpiredProviderEgressLocationsResult{}, nil +} + +func RemoveExpiredProviderEgressLocationsPost( + _ *RemoveExpiredProviderEgressLocationsArgs, + _ *RemoveExpiredProviderEgressLocationsResult, + clientSession *session.ClientSession, + tx server.PgTx, +) error { + ScheduleRemoveExpiredProviderEgressLocations(clientSession, tx) + return nil +}