Clear signing: track latest registry, provider blocklist, richer processing log

do_update.sh

@@ -14,8 +14,6 @@ fi
 
 # Update all submodules to their latest commit
 git submodule update --init --recursive --remote
-# Keep the pinned submodule at its fixed commit
-git submodule update -- "ethereum/clear-signing-erc7730-registry"
 
 SHOW_ADDED=""
 if [[ "$1" == "--show-added" ]]; then SHOW_ADDED="--show-added"; fi

eth_definitions/download.py

@@ -49,21 +49,12 @@
 TOKENS_PATH = ETHEREUM_LISTS / "tokens" / "tokens"
 DISPLAY_FORMATS_PATH = ERC7730_REGISTRY / "registry"
 
-ENABLED_PROVIDERS = frozenset(
-    {
-        "lifi",
-        "lido",
-        "morpho",
-        "kiln",
-        "poap",
-        "starkgate",
-        "walletconnect",
-        "yieldxyz",
-        "corestake",
-        "ethena",
-        "fellow-fund",
-    }
-)
+# Registry providers (top-level directory names) whose descriptors are NOT
+# parsed or emitted. Everything else in the registry goes through.
+BLOCKED_PROVIDERS = {
+    "uniswap",
+    "safe",
+}
 
 
 class CacheableError(Exception):
@@ -396,31 +387,36 @@ def _load_display_formats_from_repo(
     calldata-specific). `common-*.json` files reach us via the `includes`
     merge; `eip712-*.json` files aren't representable in our schema.
 
-    Skips files under `tests/` subdirectories and records for chain_ids
+    Skips files under `tests/` subdirectories, providers listed in
+    `BLOCKED_PROVIDERS` (noted in the log header), and records for chain_ids
     we don't otherwise know about.
 
-    A descriptor that uses any feature we can't faithfully represent is skipped
-    in full (we never emit a display format with a field missing). Every such
-    feature is collected and written to `definitions-latest.log`. The whole
-    registry is scanned for that inventory, but only enabled providers feed the
-    emitted records.
+    A display format that uses any feature we can't faithfully represent is
+    skipped whole (we never emit one with a field missing). Every such feature,
+    and every accepted-but-adjusted field, is collected and written to
+    `definitions-latest.log`.
 
     Deduplicates on `(chain_id, address, func_sig)`;
     later files override earlier ones.
     """
 
     known_chain_ids = {n["chain_id"] for n in networks}
     unsupported: list[tuple[str, str, str]] = []
-    loaded: list[tuple[str, bool, list[ERC20DisplayFormat]]] = []
+    adjustments: list[tuple[str, str, str]] = []
+    loaded: list[tuple[str, list[ERC20DisplayFormat]]] = []
 
     for path in sorted(DISPLAY_FORMATS_PATH.glob("*/calldata-*.json")):
         if "tests" in path.parts:
             continue
+        # `*/calldata-*.json` glob → the provider is the file's parent directory.
+        if path.parent.name in BLOCKED_PROVIDERS:
+            logging.info(f"blocked provider, skipping {path.relative_to(ROOT_DIR)}")
+            continue
 
         try:
-            # Scan every file so the unsupported-features log covers the whole
-            # registry, regardless of which providers are enabled below.
-            records = load_display_formats(path, unsupported=unsupported)
+            records = load_display_formats(
+                path, unsupported=unsupported, adjustments=adjustments
+            )
         except UnsupportedFeature as e:
             # File skipped — its features were already collected into `unsupported`.
             logging.info(f"skipping {path.relative_to(ROOT_DIR)} — {e}")
@@ -429,10 +425,7 @@ def _load_display_formats_from_repo(
             logging.warning(f"failed to parse {path.relative_to(ROOT_DIR)}")
             raise
 
-        rel = str(path.relative_to(ROOT_DIR))
-        # `*/calldata-*.json` glob → the provider is the file's parent directory.
-        gated = path.parent.name in ENABLED_PROVIDERS
-        loaded.append((rel, gated, records))
+        loaded.append((str(path.relative_to(ROOT_DIR)), records))
 
     display_formats, conflicts = _dedup_display_formats(loaded, known_chain_ids)
     for key_str, overridden, kept in conflicts:
@@ -441,7 +434,7 @@ def _load_display_formats_from_repo(
         )
 
     if loaded or unsupported:
-        _write_display_formats_log(unsupported, conflicts)
+        _write_display_formats_log(unsupported, conflicts, adjustments)
     else:
         # Nothing scanned at all — the registry submodule is most likely
         # uninitialized (e.g. a shallow checkout). Surface the probable cause
@@ -454,23 +447,22 @@ def _load_display_formats_from_repo(
 
 
 def _dedup_display_formats(
-    loaded: list[tuple[str, bool, list[ERC20DisplayFormat]]],
+    loaded: list[tuple[str, list[ERC20DisplayFormat]]],
     known_chain_ids: set[int],
 ) -> tuple[list[ERC20DisplayFormat], list[tuple[str, str, str]]]:
     """Deduplicate display formats on `(chain_id, address, func_sig)`.
 
-    `loaded` is a list of `(source, gated, records)` in processing order. Only
-    `gated` records feed the emitted output (later files override earlier ones),
-    but conflicts are detected registry-wide: whenever any two files define the
-    same key with a *different* payload it's reported as an override conflict
-    `(key, overridden_source, kept_source)` — even for not-yet-enabled providers.
-    Identical redefinitions are harmless duplicates and not reported.
+    `loaded` is a list of `(source, records)` in processing order; later files
+    override earlier ones. Whenever two files define the same key with a
+    *different* payload it's reported as an override conflict
+    `(key, overridden_source, kept_source)`. Identical redefinitions are
+    harmless duplicates and not reported.
     """
     dedup: dict[tuple[int, str, str], ERC20DisplayFormat] = {}
     seen: dict[tuple[int, str, str], tuple[ERC20DisplayFormat, str]] = {}
     conflicts: list[tuple[str, str, str]] = []
 
-    for source, gated, records in loaded:
+    for source, records in loaded:
         for r in records:
             if r["chain_id"] not in known_chain_ids:
                 continue
@@ -480,49 +472,75 @@ def _dedup_display_formats(
                 key_str = f"chain={key[0]} address={key[1]} selector={key[2]}"
                 conflicts.append((key_str, prev[1], source))
             seen[key] = (r, source)
-            if gated:
-                dedup[key] = r
+            dedup[key] = r
 
     return list(dedup.values()), conflicts
 
 
+def _group_by_kind_and_file(
+    entries: list[tuple[str, str, str]],
+) -> tuple[Counter[str], dict[str, set[str]], dict[str, list[tuple[str, str]]]]:
+    """Group `(source, kind, detail)` entries for the two log views."""
+    by_kind_count: Counter[str] = Counter(kind for _, kind, _ in entries)
+    by_kind_files: dict[str, set[str]] = defaultdict(set)
+    by_file: dict[str, list[tuple[str, str]]] = defaultdict(list)
+    for source, kind, detail in entries:
+        by_kind_files[kind].add(source)
+        by_file[source].append((kind, detail))
+    return by_kind_count, by_kind_files, by_file
+
+
+def _grouped_section_lines(
+    entries: list[tuple[str, str, str]], kind_header: str
+) -> list[str]:
+    by_kind_count, by_kind_files, by_file = _group_by_kind_and_file(entries)
+    lines = ["", kind_header]
+    if not entries:
+        lines.append("(none)")
+    for kind in sorted(by_kind_count, key=lambda k: (-by_kind_count[k], k)):
+        lines.append(
+            f"{by_kind_count[kind]:5d}  {len(by_kind_files[kind]):4d} file(s)  {kind}"
+        )
+    lines += ["", "## By file"]
+    for source in sorted(by_file):
+        lines.append(source)
+        for kind, detail in sorted(by_file[source]):
+            lines.append(f"    {kind}: {detail}")
+        lines.append("")
+    return lines
+
+
 def _write_display_formats_log(
     unsupported: list[tuple[str, str, str]],
     conflicts: list[tuple[str, str, str]],
+    adjustments: list[tuple[str, str, str]],
 ) -> None:
     """Write the ERC-7730 processing log (`definitions-latest.log`).
 
-    Two independent sections: skipped descriptors grouped by unsupported feature,
-    and conflicting overrides (the same key redefined differently by multiple
-    files). Conflicts are *not* unsupported features — they get their own section.
+    Headed by the provider blocklist, then three independent sections: dropped
+    display formats grouped by unsupported feature, accepted-but-adjusted
+    fields (formatter overrides, ABI retypes, materialized constants), and
+    conflicting overrides (the same key redefined differently by multiple
+    files). Blocked providers are excluded from all three (never parsed).
     """
-    by_feature_count: Counter[str] = Counter(feat for _, feat, _ in unsupported)
-    by_feature_files: dict[str, set[str]] = defaultdict(set)
-    by_file: dict[str, list[tuple[str, str]]] = defaultdict(list)
-    for source, feat, detail in unsupported:
-        by_feature_files[feat].add(source)
-        by_file[source].append((feat, detail))
-
+    blocked = ", ".join(sorted(BLOCKED_PROVIDERS)) if BLOCKED_PROVIDERS else "(none)"
+    affected_files = {src for src, _, _ in unsupported}
     lines = [
-        "# ERC-7730 unsupported features",
-        f"# {len(by_file)} file(s) skipped, "
+        f"# Providers blocked: {blocked}",
+        "",
+        "# ERC-7730 unsupported features (display format dropped)",
+        f"# {len(affected_files)} file(s) affected, "
         f"{len(unsupported)} feature occurrence(s)",
+    ]
+    lines += _grouped_section_lines(unsupported, "## By feature")
+
+    adjusted_files = {src for src, _, _ in adjustments}
+    lines += [
         "",
-        "## By feature",
+        "# Adjustments (field accepted with modification)",
+        f"# {len(adjusted_files)} file(s), {len(adjustments)} adjustment(s)",
     ]
-    if not unsupported:
-        lines.append("(none)")
-    for feat in sorted(by_feature_count, key=lambda f: (-by_feature_count[f], f)):
-        lines.append(
-            f"{by_feature_count[feat]:5d}  "
-            f"{len(by_feature_files[feat]):4d} file(s)  {feat}"
-        )
-    lines += ["", "## By file"]
-    for source in sorted(by_file):
-        lines.append(source)
-        for feat, detail in sorted(by_file[source]):
-            lines.append(f"    {feat}: {detail}")
-        lines.append("")
+    lines += _grouped_section_lines(adjustments, "## By kind")
 
     lines += [
         "",
@@ -541,7 +559,8 @@ def _write_display_formats_log(
     DISPLAY_FORMATS_LOG_PATH.write_text("\n".join(lines).rstrip() + "\n")
     logging.info(
         f"wrote {DISPLAY_FORMATS_LOG_PATH.name} "
-        f"({len(by_file)} file(s) skipped, {len(conflicts)} conflict(s))"
+        f"({len(affected_files)} file(s) with drops, {len(adjustments)} adjustment(s), "
+        f"{len(conflicts)} conflict(s))"
     )
 
 

eth_definitions/test_download.py

@@ -28,7 +28,7 @@ def test_dedup_detects_conflicting_override():
     a = _rec(intent="Swap")
     b = _rec(intent="Exchange")  # same key, different payload
     emitted, conflicts = _dedup_display_formats(
-        [("provA/f.json", True, [a]), ("provB/g.json", True, [b])], {1}
+        [("provA/f.json", [a]), ("provB/g.json", [b])], {1}
     )
     assert len(conflicts) == 1
     _key, overridden, kept = conflicts[0]
@@ -39,39 +39,36 @@ def test_dedup_detects_conflicting_override():
 
 def test_dedup_identical_definitions_not_a_conflict():
     emitted, conflicts = _dedup_display_formats(
-        [("x.json", True, [_rec()]), ("y.json", True, [_rec()])], {1}
+        [("x.json", [_rec()]), ("y.json", [_rec()])], {1}
     )
     assert conflicts == []
     assert len(emitted) == 1
 
 
-def test_dedup_conflicts_span_gated_and_ungated_but_only_gated_emitted():
-    ungated = _rec(intent="A")
-    gated = _rec(intent="B")
-    emitted, conflicts = _dedup_display_formats(
-        [("other/f.json", False, [ungated]), ("lifi/g.json", True, [gated])], {1}
-    )
-    assert len(conflicts) == 1
-    assert emitted == [gated]  # ungated provider does not feed output
-
-
 def test_dedup_ignores_unknown_chains():
     emitted, conflicts = _dedup_display_formats(
-        [("x.json", True, [_rec(chain_id=999)])], {1}
+        [("x.json", [_rec(chain_id=999)])], {1}
     )
     assert emitted == []
     assert conflicts == []
 
 
-def test_write_display_formats_log_has_both_sections(tmp_path, monkeypatch):
+def test_write_display_formats_log_has_all_sections(tmp_path, monkeypatch):
     monkeypatch.setattr(dl, "DISPLAY_FORMATS_LOG_PATH", tmp_path / "out.log")
     _write_display_formats_log(
         unsupported=[("provA/f.json", "unsupported-formatter", "enum (field 'X')")],
         conflicts=[("chain=1 address=0xabc selector=0xdead", "provA/f.json", "provB/g.json")],
+        adjustments=[
+            ("provC/h.json", "calldata-as-raw", "data shown as raw bytes (field 'Swap')")
+        ],
     )
     text = (tmp_path / "out.log").read_text()
+    assert text.startswith("# Providers blocked: ")
     assert "unsupported features" in text
     assert "unsupported-formatter" in text
+    assert "Adjustments" in text
+    assert "calldata-as-raw" in text
+    assert "provC/h.json" in text
     assert "Conflicting overrides" in text
     assert "kept:     provB/g.json" in text
     assert "overrode: provA/f.json" in text