diff --git a/do_update.sh b/do_update.sh index 0ca0c5f..783ae77 100755 --- a/do_update.sh +++ b/do_update.sh @@ -14,8 +14,6 @@ fi # Update all submodules to their latest commit git submodule update --init --recursive --remote -# Keep the pinned submodule at its fixed commit -git submodule update -- "ethereum/clear-signing-erc7730-registry" SHOW_ADDED="" if [[ "$1" == "--show-added" ]]; then SHOW_ADDED="--show-added"; fi diff --git a/eth_definitions/download.py b/eth_definitions/download.py index efb0d96..9b7ed90 100755 --- a/eth_definitions/download.py +++ b/eth_definitions/download.py @@ -49,21 +49,12 @@ TOKENS_PATH = ETHEREUM_LISTS / "tokens" / "tokens" DISPLAY_FORMATS_PATH = ERC7730_REGISTRY / "registry" -ENABLED_PROVIDERS = frozenset( - { - "lifi", - "lido", - "morpho", - "kiln", - "poap", - "starkgate", - "walletconnect", - "yieldxyz", - "corestake", - "ethena", - "fellow-fund", - } -) +# Registry providers (top-level directory names) whose descriptors are NOT +# parsed or emitted. Everything else in the registry goes through. +BLOCKED_PROVIDERS = { + "uniswap", + "safe", +} class CacheableError(Exception): @@ -396,14 +387,14 @@ def _load_display_formats_from_repo( calldata-specific). `common-*.json` files reach us via the `includes` merge; `eip712-*.json` files aren't representable in our schema. - Skips files under `tests/` subdirectories and records for chain_ids + Skips files under `tests/` subdirectories, providers listed in + `BLOCKED_PROVIDERS` (noted in the log header), and records for chain_ids we don't otherwise know about. - A descriptor that uses any feature we can't faithfully represent is skipped - in full (we never emit a display format with a field missing). Every such - feature is collected and written to `definitions-latest.log`. The whole - registry is scanned for that inventory, but only enabled providers feed the - emitted records. + A display format that uses any feature we can't faithfully represent is + skipped whole (we never emit one with a field missing). Every such feature, + and every accepted-but-adjusted field, is collected and written to + `definitions-latest.log`. Deduplicates on `(chain_id, address, func_sig)`; later files override earlier ones. @@ -411,16 +402,21 @@ def _load_display_formats_from_repo( known_chain_ids = {n["chain_id"] for n in networks} unsupported: list[tuple[str, str, str]] = [] - loaded: list[tuple[str, bool, list[ERC20DisplayFormat]]] = [] + adjustments: list[tuple[str, str, str]] = [] + loaded: list[tuple[str, list[ERC20DisplayFormat]]] = [] for path in sorted(DISPLAY_FORMATS_PATH.glob("*/calldata-*.json")): if "tests" in path.parts: continue + # `*/calldata-*.json` glob → the provider is the file's parent directory. + if path.parent.name in BLOCKED_PROVIDERS: + logging.info(f"blocked provider, skipping {path.relative_to(ROOT_DIR)}") + continue try: - # Scan every file so the unsupported-features log covers the whole - # registry, regardless of which providers are enabled below. - records = load_display_formats(path, unsupported=unsupported) + records = load_display_formats( + path, unsupported=unsupported, adjustments=adjustments + ) except UnsupportedFeature as e: # File skipped — its features were already collected into `unsupported`. logging.info(f"skipping {path.relative_to(ROOT_DIR)} — {e}") @@ -429,10 +425,7 @@ def _load_display_formats_from_repo( logging.warning(f"failed to parse {path.relative_to(ROOT_DIR)}") raise - rel = str(path.relative_to(ROOT_DIR)) - # `*/calldata-*.json` glob → the provider is the file's parent directory. - gated = path.parent.name in ENABLED_PROVIDERS - loaded.append((rel, gated, records)) + loaded.append((str(path.relative_to(ROOT_DIR)), records)) display_formats, conflicts = _dedup_display_formats(loaded, known_chain_ids) for key_str, overridden, kept in conflicts: @@ -441,7 +434,7 @@ def _load_display_formats_from_repo( ) if loaded or unsupported: - _write_display_formats_log(unsupported, conflicts) + _write_display_formats_log(unsupported, conflicts, adjustments) else: # Nothing scanned at all — the registry submodule is most likely # uninitialized (e.g. a shallow checkout). Surface the probable cause @@ -454,23 +447,22 @@ def _load_display_formats_from_repo( def _dedup_display_formats( - loaded: list[tuple[str, bool, list[ERC20DisplayFormat]]], + loaded: list[tuple[str, list[ERC20DisplayFormat]]], known_chain_ids: set[int], ) -> tuple[list[ERC20DisplayFormat], list[tuple[str, str, str]]]: """Deduplicate display formats on `(chain_id, address, func_sig)`. - `loaded` is a list of `(source, gated, records)` in processing order. Only - `gated` records feed the emitted output (later files override earlier ones), - but conflicts are detected registry-wide: whenever any two files define the - same key with a *different* payload it's reported as an override conflict - `(key, overridden_source, kept_source)` — even for not-yet-enabled providers. - Identical redefinitions are harmless duplicates and not reported. + `loaded` is a list of `(source, records)` in processing order; later files + override earlier ones. Whenever two files define the same key with a + *different* payload it's reported as an override conflict + `(key, overridden_source, kept_source)`. Identical redefinitions are + harmless duplicates and not reported. """ dedup: dict[tuple[int, str, str], ERC20DisplayFormat] = {} seen: dict[tuple[int, str, str], tuple[ERC20DisplayFormat, str]] = {} conflicts: list[tuple[str, str, str]] = [] - for source, gated, records in loaded: + for source, records in loaded: for r in records: if r["chain_id"] not in known_chain_ids: continue @@ -480,49 +472,75 @@ def _dedup_display_formats( key_str = f"chain={key[0]} address={key[1]} selector={key[2]}" conflicts.append((key_str, prev[1], source)) seen[key] = (r, source) - if gated: - dedup[key] = r + dedup[key] = r return list(dedup.values()), conflicts +def _group_by_kind_and_file( + entries: list[tuple[str, str, str]], +) -> tuple[Counter[str], dict[str, set[str]], dict[str, list[tuple[str, str]]]]: + """Group `(source, kind, detail)` entries for the two log views.""" + by_kind_count: Counter[str] = Counter(kind for _, kind, _ in entries) + by_kind_files: dict[str, set[str]] = defaultdict(set) + by_file: dict[str, list[tuple[str, str]]] = defaultdict(list) + for source, kind, detail in entries: + by_kind_files[kind].add(source) + by_file[source].append((kind, detail)) + return by_kind_count, by_kind_files, by_file + + +def _grouped_section_lines( + entries: list[tuple[str, str, str]], kind_header: str +) -> list[str]: + by_kind_count, by_kind_files, by_file = _group_by_kind_and_file(entries) + lines = ["", kind_header] + if not entries: + lines.append("(none)") + for kind in sorted(by_kind_count, key=lambda k: (-by_kind_count[k], k)): + lines.append( + f"{by_kind_count[kind]:5d} {len(by_kind_files[kind]):4d} file(s) {kind}" + ) + lines += ["", "## By file"] + for source in sorted(by_file): + lines.append(source) + for kind, detail in sorted(by_file[source]): + lines.append(f" {kind}: {detail}") + lines.append("") + return lines + + def _write_display_formats_log( unsupported: list[tuple[str, str, str]], conflicts: list[tuple[str, str, str]], + adjustments: list[tuple[str, str, str]], ) -> None: """Write the ERC-7730 processing log (`definitions-latest.log`). - Two independent sections: skipped descriptors grouped by unsupported feature, - and conflicting overrides (the same key redefined differently by multiple - files). Conflicts are *not* unsupported features — they get their own section. + Headed by the provider blocklist, then three independent sections: dropped + display formats grouped by unsupported feature, accepted-but-adjusted + fields (formatter overrides, ABI retypes, materialized constants), and + conflicting overrides (the same key redefined differently by multiple + files). Blocked providers are excluded from all three (never parsed). """ - by_feature_count: Counter[str] = Counter(feat for _, feat, _ in unsupported) - by_feature_files: dict[str, set[str]] = defaultdict(set) - by_file: dict[str, list[tuple[str, str]]] = defaultdict(list) - for source, feat, detail in unsupported: - by_feature_files[feat].add(source) - by_file[source].append((feat, detail)) - + blocked = ", ".join(sorted(BLOCKED_PROVIDERS)) if BLOCKED_PROVIDERS else "(none)" + affected_files = {src for src, _, _ in unsupported} lines = [ - "# ERC-7730 unsupported features", - f"# {len(by_file)} file(s) skipped, " + f"# Providers blocked: {blocked}", + "", + "# ERC-7730 unsupported features (display format dropped)", + f"# {len(affected_files)} file(s) affected, " f"{len(unsupported)} feature occurrence(s)", + ] + lines += _grouped_section_lines(unsupported, "## By feature") + + adjusted_files = {src for src, _, _ in adjustments} + lines += [ "", - "## By feature", + "# Adjustments (field accepted with modification)", + f"# {len(adjusted_files)} file(s), {len(adjustments)} adjustment(s)", ] - if not unsupported: - lines.append("(none)") - for feat in sorted(by_feature_count, key=lambda f: (-by_feature_count[f], f)): - lines.append( - f"{by_feature_count[feat]:5d} " - f"{len(by_feature_files[feat]):4d} file(s) {feat}" - ) - lines += ["", "## By file"] - for source in sorted(by_file): - lines.append(source) - for feat, detail in sorted(by_file[source]): - lines.append(f" {feat}: {detail}") - lines.append("") + lines += _grouped_section_lines(adjustments, "## By kind") lines += [ "", @@ -541,7 +559,8 @@ def _write_display_formats_log( DISPLAY_FORMATS_LOG_PATH.write_text("\n".join(lines).rstrip() + "\n") logging.info( f"wrote {DISPLAY_FORMATS_LOG_PATH.name} " - f"({len(by_file)} file(s) skipped, {len(conflicts)} conflict(s))" + f"({len(affected_files)} file(s) with drops, {len(adjustments)} adjustment(s), " + f"{len(conflicts)} conflict(s))" ) diff --git a/eth_definitions/test_download.py b/eth_definitions/test_download.py index f30f9cc..feeaadc 100644 --- a/eth_definitions/test_download.py +++ b/eth_definitions/test_download.py @@ -28,7 +28,7 @@ def test_dedup_detects_conflicting_override(): a = _rec(intent="Swap") b = _rec(intent="Exchange") # same key, different payload emitted, conflicts = _dedup_display_formats( - [("provA/f.json", True, [a]), ("provB/g.json", True, [b])], {1} + [("provA/f.json", [a]), ("provB/g.json", [b])], {1} ) assert len(conflicts) == 1 _key, overridden, kept = conflicts[0] @@ -39,39 +39,36 @@ def test_dedup_detects_conflicting_override(): def test_dedup_identical_definitions_not_a_conflict(): emitted, conflicts = _dedup_display_formats( - [("x.json", True, [_rec()]), ("y.json", True, [_rec()])], {1} + [("x.json", [_rec()]), ("y.json", [_rec()])], {1} ) assert conflicts == [] assert len(emitted) == 1 -def test_dedup_conflicts_span_gated_and_ungated_but_only_gated_emitted(): - ungated = _rec(intent="A") - gated = _rec(intent="B") - emitted, conflicts = _dedup_display_formats( - [("other/f.json", False, [ungated]), ("lifi/g.json", True, [gated])], {1} - ) - assert len(conflicts) == 1 - assert emitted == [gated] # ungated provider does not feed output - - def test_dedup_ignores_unknown_chains(): emitted, conflicts = _dedup_display_formats( - [("x.json", True, [_rec(chain_id=999)])], {1} + [("x.json", [_rec(chain_id=999)])], {1} ) assert emitted == [] assert conflicts == [] -def test_write_display_formats_log_has_both_sections(tmp_path, monkeypatch): +def test_write_display_formats_log_has_all_sections(tmp_path, monkeypatch): monkeypatch.setattr(dl, "DISPLAY_FORMATS_LOG_PATH", tmp_path / "out.log") _write_display_formats_log( unsupported=[("provA/f.json", "unsupported-formatter", "enum (field 'X')")], conflicts=[("chain=1 address=0xabc selector=0xdead", "provA/f.json", "provB/g.json")], + adjustments=[ + ("provC/h.json", "calldata-as-raw", "data shown as raw bytes (field 'Swap')") + ], ) text = (tmp_path / "out.log").read_text() + assert text.startswith("# Providers blocked: ") assert "unsupported features" in text assert "unsupported-formatter" in text + assert "Adjustments" in text + assert "calldata-as-raw" in text + assert "provC/h.json" in text assert "Conflicting overrides" in text assert "kept: provB/g.json" in text assert "overrode: provA/f.json" in text diff --git a/ethereum/clear-signing-erc7730-registry b/ethereum/clear-signing-erc7730-registry index f06846e..5a67786 160000 --- a/ethereum/clear-signing-erc7730-registry +++ b/ethereum/clear-signing-erc7730-registry @@ -1 +1 @@ -Subproject commit f06846e341fb25c91e3214d6a534a3f9f4b82fd1 +Subproject commit 5a677869bf544defad70c3080a2c51364d8671c7