IPFS peer-fetch durability layer (1.5a)

.env.example

@@ -355,3 +355,43 @@
 
 # OTEL Collector image tag
 # OTEL_COLLECTOR_IMAGE_TAG=0.119.0
+
+# ------------------------------------------------------------------------------
+# IPFS (named-content serving + fleet durability)
+# ------------------------------------------------------------------------------
+
+# Enable the IPFS subsystem (serve /ipfs/{cid} via a paired Kubo node). Requires
+# the `ipfs` docker-compose profile (the kubo sidecar). Default: false
+# IPFS_ENABLED=true
+
+# Kubo read-only gateway (:8080) and RPC API (:5001) endpoints. The RPC API is
+# required for local-only (offline) reads and for peer-fetch imports.
+# IPFS_KUBO_URL=http://kubo:8080
+# IPFS_KUBO_API_URL=http://kubo:5001
+
+# Pin ArNS-resolved (named) CIDs so read-only content survives Kubo GC. When
+# true, peer-fetched roots are pinned on import. Default: false
+# IPFS_PIN_ARNS_CONTENT=true
+
+# --- Peer-fetch durability layer -------------------------------------------
+# When a gateway lacks a named CID, fetch it from a peer AR.IO gateway that holds
+# it as a verifiable CAR; Kubo verifies every block against the CID on import, so
+# a lying/tampered peer is rejected. Named content then survives as long as ANY
+# fleet gateway holds it. Ships dark — false is a pure passthrough (no behavior
+# change). Enable only after validation. Default: false
+# IPFS_PEER_FETCH_ENABLED=true
+
+# Peers to try per CID before falling through to public IPFS. Default: 3
+# IPFS_PEER_FETCH_COUNT=3
+
+# Overall deadline for a peer-fetch attempt, ms (kept short — public IPFS is the
+# patient fallback). Default: 5000
+# IPFS_PEER_FETCH_TIMEOUT_MS=5000
+
+# Max CAR bytes accepted from a peer; above this, skip peers → public IPFS.
+# Default: 104857600 (100 MB)
+# IPFS_PEER_FETCH_MAX_CAR_BYTES=104857600
+
+# Optional comma-separated peer gateway base URLs (private fleets / testing).
+# When unset, peers come from the on-chain gateway registry (GAR). Default: unset
+# IPFS_PEER_FETCH_STATIC_PEERS=https://peer-a.example,https://peer-b.example

docker-compose.yaml

@@ -176,6 +176,11 @@ services:
       - IPFS_RATE_LIMITER_RESOURCE_TOKENS_PER_BUCKET=${IPFS_RATE_LIMITER_RESOURCE_TOKENS_PER_BUCKET:-}
       - IPFS_RATE_LIMITER_RESOURCE_REFILL_PER_SEC=${IPFS_RATE_LIMITER_RESOURCE_REFILL_PER_SEC:-}
       - IPFS_MAX_RESPONSE_SIZE_BYTES=${IPFS_MAX_RESPONSE_SIZE_BYTES:-}
+      - IPFS_PEER_FETCH_ENABLED=${IPFS_PEER_FETCH_ENABLED:-}
+      - IPFS_PEER_FETCH_COUNT=${IPFS_PEER_FETCH_COUNT:-}
+      - IPFS_PEER_FETCH_TIMEOUT_MS=${IPFS_PEER_FETCH_TIMEOUT_MS:-}
+      - IPFS_PEER_FETCH_MAX_CAR_BYTES=${IPFS_PEER_FETCH_MAX_CAR_BYTES:-}
+      - IPFS_PEER_FETCH_STATIC_PEERS=${IPFS_PEER_FETCH_STATIC_PEERS:-}
       - NODE_MAX_OLD_SPACE_SIZE=${NODE_MAX_OLD_SPACE_SIZE:-}
       - ENABLE_FS_HEADER_CACHE_CLEANUP=${ENABLE_FS_HEADER_CACHE_CLEANUP:-}
       - ON_DEMAND_RETRIEVAL_ORDER=${ON_DEMAND_RETRIEVAL_ORDER:-}

package.json

@@ -48,6 +48,7 @@
     "express-openapi-validator": "^5.4.2",
     "express-prom-bundle": "^7.0.0",
     "fastq": "^1.19.1",
+    "form-data": "^4.0.5",
     "fs-extra": "^11.3.2",
     "graphql": "^16.11.0",
     "ioredis": "^5.8.0",

src/config.ts

@@ -3320,6 +3320,44 @@ export const IPFS_MAX_RESPONSE_SIZE_BYTES = env.positiveIntOrDefault(
   1 * 1024 * 1024 * 1024, // 1 GB
 );
 
+//
+// IPFS peer-fetch (fleet durability layer)
+//
+
+// Master switch for peer-fetch. Ships dark: when false the IPFS composite is a
+// pure passthrough to Kubo (zero behavior change). Enable only after the
+// multi-node integration test passes.
+export const IPFS_PEER_FETCH_ENABLED =
+  env.varOrDefault('IPFS_PEER_FETCH_ENABLED', 'false') === 'true';
+
+// Peers to try per CID before falling through to public IPFS.
+export const IPFS_PEER_FETCH_COUNT = env.positiveIntOrDefault(
+  'IPFS_PEER_FETCH_COUNT',
+  3,
+);
+
+// Overall deadline for the whole peer-fetch attempt (kept short — public IPFS is
+// the patient fallback).
+export const IPFS_PEER_FETCH_TIMEOUT_MS = env.positiveIntOrDefault(
+  'IPFS_PEER_FETCH_TIMEOUT_MS',
+  5000,
+);
+
+// Max CAR bytes accepted from a peer; above this, skip peers → public IPFS.
+export const IPFS_PEER_FETCH_MAX_CAR_BYTES = env.positiveIntOrDefault(
+  'IPFS_PEER_FETCH_MAX_CAR_BYTES',
+  100 * 1024 * 1024, // 100 MB
+);
+
+// Optional deterministic peer override (comma-separated gateway base URLs) for
+// private fleets and integration tests; when unset, peers come from the GAR via
+// ArIOPeerManager.
+export const IPFS_PEER_FETCH_STATIC_PEERS = env
+  .varOrDefault('IPFS_PEER_FETCH_STATIC_PEERS', '')
+  .split(',')
+  .map((s) => s.trim())
+  .filter((s) => s.length > 0);
+
 //
 // StandaloneSqlite worker pools
 //

src/constants.ts

@@ -29,6 +29,11 @@ export const headerNames = {
   // Which retrieval source served the body (e.g. 'ipfs'). Declared centrally so
   // it's referenced consistently and is a candidate for HTTPSIG trigger headers.
   arIoSource: 'X-Ar-Io-Source',
+  // IPFS local-only serve mode: on a request, "serve only from the local
+  // blockstore, never touch public IPFS/DHT" (peer-fetch recursion guard +
+  // trustless holding probe); echoed on a local-only hit so a caller/observer
+  // can assert the server honored the mode.
+  ipfsLocalOnly: 'X-Ar-Io-Local-Only',
   origin: 'X-AR-IO-Origin',
   originNodeRelease: 'X-AR-IO-Origin-Node-Release',
   digest: 'X-AR-IO-Digest',

src/ipfs/ipfs-content-source.ts

@@ -0,0 +1,31 @@
+/**
+ * AR.IO Gateway
+ * Copyright (C) 2022-2025 Permanent Data Solutions, Inc. All Rights Reserved.
+ *
+ * SPDX-License-Identifier: AGPL-3.0-or-later
+ */
+import { Span } from '@opentelemetry/api';
+
+import { IpfsContentResult } from './kubo-data-source.js';
+
+// Shared shape for any IPFS content tier (local Kubo, fleet peers, public Kubo).
+// Mirrors KuboDataSource.getContent so the composite (SequentialIpfsSource) can
+// compose sources uniformly, and IpfsService can depend on the interface rather
+// than the concrete KuboDataSource.
+export interface IpfsContentSourceOptions {
+  cidString: string;
+  path?: string;
+  signal?: AbortSignal;
+  parentSpan?: Span;
+  range?: string;
+  // Trustless response format: a single verifiable block (`raw`) or a verifiable
+  // DAG archive (`car`). Absent = UnixFS proxy.
+  format?: 'raw' | 'car';
+  // Serve ONLY from the local blockstore (offline) — never peers, never public
+  // IPFS. The recursion guard + holding-measurement primitive.
+  localOnly?: boolean;
+}
+
+export interface IpfsContentSource {
+  getContent(opts: IpfsContentSourceOptions): Promise<IpfsContentResult>;
+}