Tests comparing against cardano-cli, on top of cardano-testnet and tx-generator

flake.nix

@@ -14,6 +14,13 @@
       inputs.nixpkgs.follows = "nixpkgs";
       inputs.hackage.follows = "hackageNix";
     };
+    cardano-node.url = "github:IntersectMBO/cardano-node/11.0.1";
+    # tx-generator only, for the integration suite's submission test: its
+    # Ogmios submission transport (IntersectMBO/cardano-node#6609) landed
+    # after 11.0.1. Separate from the input above so the node, cli and
+    # testnet binaries stay on the released tag the server is built against;
+    # fold it back into `cardano-node` once a release carries the transport.
+    cardano-node-tx-generator.url = "github:IntersectMBO/cardano-node/ec672fd082c7839792d0ec4a0bce27e96c9bb70f";
     iohkNix.url = "github:input-output-hk/iohk-nix";
     nixpkgs.follows = "haskellNix/nixpkgs-unstable";
     self.submodules = true;
@@ -23,11 +30,12 @@
     inherit ((import ./flake/lib.nix {inherit inputs;}).flake.lib) recursiveImports;
   in
     inputs.flake-parts.lib.mkFlake {inherit inputs;} {
-      imports = recursiveImports [./perSystem];
+      imports = recursiveImports [./perSystem] ++ [inputs.flake-parts.flakeModules.touchup];
+      touchup.attr.formatter.enable = false;
       systems = [
         "x86_64-linux"
         # "aarch64-linux"
-        # "aarch64-darwin"
+        "aarch64-darwin"
       ];
       perSystem = {system, ...}: {
         _module.args.pkgs = import inputs.nixpkgs {

perSystem/devShells.nix

@@ -1,11 +1,12 @@
 { inputs, ... }: {
-  perSystem = { shellFor, pkgs, ... }: {
+  perSystem = { shellFor, hsPkgs, pkgs, system, ... }: {
     devShells.default = shellFor {
       packages = p: [ p.ogmios ];
 
       nativeBuildInputs = [
         pkgs.jq
         pkgs.gh
+        pkgs.lmdb
       ];
 
       tools = {
@@ -29,5 +30,30 @@
 
       withHoogle = true;
     };
+
+    devShells.integration = let
+      cn = inputs.cardano-node.packages.${system};
+      cnTxGenerator = inputs.cardano-node-tx-generator.packages.${system};
+    in shellFor {
+      packages = p: [ p.ogmios p.ogmios-integration-tests ];
+
+      nativeBuildInputs = [
+        pkgs.jq
+        pkgs.lmdb
+        hsPkgs.ogmios.components.exes.ogmios
+        cn.cardano-node
+        cn.cardano-cli
+        cn.cardano-testnet
+        cnTxGenerator.tx-generator
+      ];
+
+      tools = {
+        cabal = "latest";
+      };
+
+      shellHook = ''
+        export LANG="en_US.UTF-8"
+      '';
+    };
   };
 }

perSystem/hydraJobs.nix

@@ -3,11 +3,26 @@
     {
       config,
       pkgs,
+      system,
       ...
     }: let
-      required    = {inherit (config) packages checks;};
-      nonRequired = {inherit (config) devShells;};
-      jobs        = required // nonRequired;
+      integrationChecks = [ "ogmios-integration" ];
+      ciChecks =
+        if lib.hasSuffix "-linux" system
+        then config.checks
+        else removeAttrs config.checks integrationChecks;
+      required = {
+        inherit (config) packages;
+        checks = removeAttrs ciChecks integrationChecks;
+      };
+      nonRequired = {
+        inherit (config) devShells;
+        checks = lib.filterAttrs (name: _: lib.elem name integrationChecks) ciChecks;
+      };
+      jobs = {
+        inherit (config) packages devShells;
+        checks = ciChecks;
+      };
     in
       jobs
       // {

perSystem/musl.nix

@@ -5,13 +5,19 @@
     lib,
     ...
   }: let
+    # Only offer the static build from a Linux host. Cross-compiling to
+    # linux-musl from darwin means asking haskell.nix for a whole
+    # aarch64-unknown-linux-musl GHC that no cache has a substitute for, so it
+    # gets built from source and times out on the CI builders.
+    hostIsLinux = lib.hasSuffix "-linux" system;
     muslProject = project.projectCross.${
       if system == "x86_64-linux"
       then "musl64"
       else "aarch64-multiplatform-musl"
     };
     muslExes = muslProject.hsPkgs.ogmios.components.exes;
-  in {
-    packages.ogmios-musl = muslExes.ogmios;
-  };
+  in
+    lib.optionalAttrs hostIsLinux {
+      packages.ogmios-musl = muslExes.ogmios;
+    };
 }

perSystem/packages.nix

@@ -1,15 +1,81 @@
-{
-  perSystem = { hsPkgs, ... }:
+{ inputs, ... }: {
+  perSystem = { hsPkgs, pkgs, lib, system, ... }:
     let
       ogmios = hsPkgs.ogmios;
+      integrationTestsExe =
+        hsPkgs.ogmios-integration-tests.components.exes.ogmios-integration-tests;
+      cn = inputs.cardano-node.packages.${system};
+      cnTxGenerator = inputs.cardano-node-tx-generator.packages.${system};
+
+      # The suite spawns ogmios, cardano-node, cardano-cli, cardano-testnet
+      # and tx-generator (Test.Integration.Env resolves them from PATH), so
+      # ship them as an execution dependency of the packaged binary:
+      # `nix run .#ogmios-integration-tests` is self-contained. Inside the
+      # `integration` devshell the same tools come from the shell's PATH
+      # instead, and `cabal run` builds are intentionally left unwrapped so
+      # they can be pointed at other tool versions.
+      integrationTests =
+        pkgs.runCommand "ogmios-integration-tests"
+          {
+            nativeBuildInputs = [ pkgs.makeWrapper ];
+            meta.mainProgram = "ogmios-integration-tests";
+          }
+          ''
+            makeWrapper \
+              ${integrationTestsExe}/bin/ogmios-integration-tests \
+              $out/bin/ogmios-integration-tests \
+              --prefix PATH : ${lib.makeBinPath [
+                ogmios.components.exes.ogmios
+                cn.cardano-node
+                cn.cardano-cli
+                cn.cardano-testnet
+                cnTxGenerator.tx-generator
+              ]}
+          '';
     in
     {
       packages = {
         default = ogmios.components.exes.ogmios;
         ogmios = ogmios.components.exes.ogmios;
         ogmios-lib = ogmios.components.library;
+        ogmios-integration-tests = integrationTests;
       };
 
-      checks.ogmios-unit = ogmios.checks.unit;
+      checks = {
+        ogmios-unit = ogmios.checks.unit;
+
+        # Actually executes the integration suite: a throwaway testnet,
+        # ogmios and tx-generator all run inside the build, talking over
+        # loopback. Defined for every system so it can always be run by
+        # hand (`nix build .#checks.<system>.ogmios-integration`, or
+        # `nix flake check`); hydra however only schedules it on Linux,
+        # where the sandbox's network namespace keeps the suite's fixed
+        # ogmios port private to the build - see hydraJobs.nix.
+        ogmios-integration =
+          pkgs.runCommand "ogmios-integration"
+            {
+              nativeBuildInputs = [ integrationTests ];
+              # Sandboxed darwin builds block networking by default; the
+              # suite only ever talks to 127.0.0.1.
+              __darwinAllowLocalNetworking = true;
+              meta = {
+                description = "run the ogmios integration suite against a local testnet";
+                # ~2 minutes when healthy; don't let a wedged daemon hold a
+                # builder for hours.
+                timeout = 1800;
+              };
+            }
+            ''
+              ogmios-integration-tests || {
+                echo "==== integration suite failed; daemon log tails follow"
+                for f in "$TMPDIR"/ogmios-integration-*/logs/*; do
+                  echo "==== $f"
+                  tail -n 40 "$f" || true
+                done
+                exit 1
+              }
+              touch $out
+            '';
+      };
     };
 }

perSystem/project.nix

@@ -48,14 +48,10 @@
             cp ${self}/server/ogmios.json .
             cp ${self}/server/cardano.json .
 
-            # Test.Path.Util.getProjectRoot uses TH `makeRelativeToProject ""`
-            # which bakes the compile-time absolute path
-            # /build/ogmios-src-test-unit-root/server into the binary. Recreate
-            # that path at runtime with the golden fixtures and the
-            # cardano-configurations submodule content.
-            mkdir -p /build/ogmios-src-test-unit-root/server/test
-            cp -r ${self}/server/test/golden /build/ogmios-src-test-unit-root/server/test/golden
-            cp -r ${self}/server/config /build/ogmios-src-test-unit-root/server/config
+            export OGMIOS_TEST_PROJECT_ROOT="$TMPDIR/ogmios-project-root"
+            mkdir -p "$OGMIOS_TEST_PROJECT_ROOT/test"
+            cp -r ${self}/server/test/golden "$OGMIOS_TEST_PROJECT_ROOT/test/golden"
+            cp -r ${self}/server/config "$OGMIOS_TEST_PROJECT_ROOT/config"
           '';
         }
         ({pkgs, ...}: {

server/cabal.project

@@ -24,7 +24,7 @@ constraints:
   , any.io-classes ^>= 1.8
   , any.formatting == 7.2.0
   , any.validation < 1.2
-  , any.text source
+  , any.text >= 2.1.2
 
   -- ogmios doesn't exercise io_uring code paths, so build `blockio` with the
   -- serial backend to drop the `liburing` system dependency.
@@ -46,6 +46,7 @@ packages:
   modules/hjsonschema
   modules/hspec-json-schema
   modules/json-rpc
+  test/integration
 
 tests: False
 

server/src/Ogmios/Data/Json/Byron.hs

@@ -44,6 +44,7 @@ import qualified Cardano.Crypto.Signing as By
 import qualified Cardano.Crypto.Wallet as CC
 import qualified Cardano.Ledger.Binary as Binary
 import qualified Cardano.Ledger.Core as Ledger
+import qualified Data.ByteString as BS
 
 encodeAddress
     :: By.Address
@@ -85,11 +86,11 @@ encodeABlockOrBoundary opts = encodeObject . \case
             )
         <>
         "issuer" .= encodeObject
-            ( "verificationKey" .= encodeVerificationKey (By.Dlg.issuerVK c)
+            ( "verificationKey" .= encodeExtendedVerificationKey (By.Dlg.issuerVK c)
             )
         <>
         "delegate" .= encodeObject
-            ( "verificationKey" .= encodeVerificationKey (By.Dlg.delegateVK c)
+            ( "verificationKey" .= encodeExtendedVerificationKey (By.Dlg.delegateVK c)
             )
       where
         h = By.blockHeader blk
@@ -126,11 +127,11 @@ encodeACertificate
 encodeACertificate x =
     encodeObject
         ( "issuer" .= encodeObject
-            ( "verificationKey" .= encodeVerificationKey (By.Dlg.issuerVK x)
+            ( "verificationKey" .= encodeExtendedVerificationKey (By.Dlg.issuerVK x)
             )
         <>
           "delegate" .= encodeObject
-            ( "verificationKey" .= encodeVerificationKey (By.Dlg.delegateVK x)
+            ( "verificationKey" .= encodeExtendedVerificationKey (By.Dlg.delegateVK x)
             )
         )
 
@@ -302,7 +303,7 @@ encodeAVote x = encodeObject
         )
     <> "voter" .= encodeObject
         ( "verificationKey" .=
-            encodeVerificationKey (By.Upd.Vote.voterVK x)
+            encodeExtendedVerificationKey (By.Upd.Vote.voterVK x)
         )
     )
 
@@ -535,9 +536,9 @@ encodeTxInWitness
     -> Json
 encodeTxInWitness = encodeObject . \case
     By.VKWitness key sig ->
-        "key" .=
-            encodeVerificationKey key <>
-        "signature" .=
+        mempty
+        <> encodeVerificationKey key
+        <> "signature" .=
             encodeSignature sig
     By.RedeemWitness key sig ->
         "key" .=
@@ -557,12 +558,22 @@ encodeTxProof x =
         encodeHash (By.txpWitnessesHash x)
     & encodeObject
 
-encodeVerificationKey
+encodeExtendedVerificationKey
     :: By.VerificationKey
     -> Json
-encodeVerificationKey =
+encodeExtendedVerificationKey =
     encodeByteStringBase16 . CC.unXPub . By.unVerificationKey
 
+encodeVerificationKey
+    :: By.VerificationKey
+    -> Series
+encodeVerificationKey x =
+    "key" .= encodeByteStringBase16 (BS.take 32 xpub)
+    <>
+    "chainCode" .= encodeByteStringBase16 (BS.drop 32 xpub)
+  where
+    xpub = CC.unXPub (By.unVerificationKey x)
+
 stringifyAddress
     :: By.Address
     -> Text