Make remote submission over SSH work, from the host and from the container

Dockerfile

@@ -2,7 +2,7 @@
 # The parent image is the base image that the Dockerfile builds upon.
 # The RMG installation instructions suggest Anaconda for installation by source, however, we use micromamba for the Docker image due to its smaller size and less overhead.
 # Installation of ARC will also be done in this stage.
 FROM --platform=linux/amd64 mambaorg/micromamba:2.2-ubuntu24.04 AS builder
 
 # Set ARGS
 ARG RMG_PY_BRANCH=main
@@ -34,11 +34,13 @@
 # which then dies with NoChannelsConfiguredError (non-fatally) and silently drops that pin.
 RUN printf 'channels:\n  - conda-forge\n' > /home/mambauser/.condarc
 
-# Set JuliaUp PATH and install Julia 1.10 as req. by RMG (mirrors RMG-Py's own Dockerfile)
+# Set JuliaUp PATH and install Julia as req. by RMG (mirrors RMG-Py's own Dockerfile).
+# Pinned to a patch version, not the 1.10 channel: juliacall segfaults on import under 1.10.12,
+# and the channel floats to the newest patch. RMG-Py pinned the same version in 62eb728c0.
 ENV PATH="/home/mambauser/.juliaup/bin:$PATH"
-RUN wget -qO- https://install.julialang.org | sh -s -- --yes --default-channel 1.10 && \
-    juliaup add 1.10 && \
-    juliaup default 1.10 && \
+RUN wget -qO- https://install.julialang.org | sh -s -- --yes --default-channel 1.10.11 && \
+    juliaup add 1.10.11 && \
+    juliaup default 1.10.11 && \
     juliaup list && \
     rm -rf /home/mambauser/.juliaup/downloads /home/mambauser/.juliaup/tmp
 
@@ -109,7 +111,7 @@
 # Stage 2: Final image
 # The final image is based on the same micromamba image, but we copy over the installed RMG and ARC from the builder stage.
 # This keeps the final image size smaller and avoids unnecessary layers.
 FROM --platform=linux/amd64 mambaorg/micromamba:2.2-ubuntu24.04
 
 # --- OCI image metadata -----------------------------------------------------
 # ARGs do not cross stages, so ARC_BRANCH is re-declared here.
@@ -153,7 +155,7 @@
 # any --entrypoint override) lands on root with HOME=/root rather than going through
 # entrywrapper.sh's `runuser -u mambauser`. juliaup then finds no config, falls back to the
 # `release` channel, and downloads a newer Julia over the network - silently bypassing the pinned
-# 1.10 and every pkgimage baked above, or hard-failing when the host is offline. Pinning the depot
+# 1.10.11 and every pkgimage baked above, or hard-failing when the host is offline. Pinning the depot
 # explicitly makes resolution HOME-independent. Note this is JULIAUP_DEPOT_PATH, not
 # JULIA_DEPOT_PATH; juliaup does not read the latter for channel lookup.
 ENV JULIAUP_DEPOT_PATH=/home/mambauser/.julia
@@ -166,7 +168,14 @@
         ca-certificates \
         nano \
         make \
+        openssh-client \
     && apt-get clean && rm -rf /var/lib/apt/lists/*
+# Drop the vestigial 'ubuntu' account the base image ships at 1000:1000. Nothing here uses it
+# (micromamba runs as mambauser, uid 57439), but it occupies exactly the IDs that the documented
+# `-e PUID=$(id -u) -e PGID=$(id -g)` asks for on a typical Linux desktop, where the entrypoint
+# would then refuse to remap mambauser and abort the container.
+RUN if getent passwd ubuntu >/dev/null; then userdel -r ubuntu 2>/dev/null || userdel ubuntu; fi && \
+    if getent group ubuntu >/dev/null; then groupdel ubuntu; fi
 USER mambauser
 
 COPY --from=builder --chown=mambauser:mambauser /opt/conda /opt/conda
@@ -187,6 +196,13 @@
 COPY --chmod=644  dockerfiles/aliases.sh       /etc/profile.d/aliases.sh
 COPY --chmod=755  dockerfiles/job_helpers.sh   /usr/local/bin/arc_job_helpers.sh
 COPY --chmod=755  dockerfiles/aliases_print.sh /usr/local/bin/aliases
+COPY --chmod=755  dockerfiles/arc_preflight.py /usr/local/bin/arc_preflight.py
+# Mount points for the user's SSH material (agent socket or read-only key/known_hosts mounts)
+# and for the personal ARC settings overlay, so both bind mounts land on an existing path
+# owned by mambauser.
+RUN mkdir -p /home/mambauser/.ssh && chmod 700 /home/mambauser/.ssh && \
+    mkdir -p /home/mambauser/.arc && chown mambauser:mambauser /home/mambauser/.arc
+
 RUN touch /home/mambauser/.bashrc && \
     grep -qxF 'source /etc/profile.d/aliases.sh' /home/mambauser/.bashrc || \
     echo 'source /etc/profile.d/aliases.sh' >> /home/mambauser/.bashrc

arc/common.py

@@ -56,7 +56,6 @@
 
 default_job_types, servers, supported_ess = settings['default_job_types'], settings['servers'], settings['supported_ess']
 
-
 def initialize_job_types(job_types: dict | None = None,
                          specific_job_type: str = '',
                          ) -> dict:
@@ -121,17 +120,25 @@ def initialize_job_types(job_types: dict | None = None,
     return job_types
 
 
-def check_ess_settings(ess_settings: dict | None = None) -> dict:
+def check_ess_settings(ess_settings: dict | None = None,
+                       ts_adapters: list[str] | None = None,
+                       ) -> dict:
     """
     A helper function to convert servers in the ess_settings dict to lists
     Assists in troubleshooting job and trying a different server
     Also check ESS and servers.
 
     Args:
         ess_settings (dict, optional): ARC's ESS settings dictionary.
+        ts_adapters (list, optional): The TS search adapters this run will use, used to check the
+                                      remote paths of the servers they will run on. ``None``
+                                      selects the default set from the settings.
 
     Returns: dict
         An updated ARC ESS dictionary.
+
+    Raises:
+        SettingsError: If an ESS, a server, or the remote path of a server is unusable.
     """
     if ess_settings is None or not ess_settings:
         return dict()
@@ -155,10 +162,55 @@ def check_ess_settings(ess_settings: dict | None = None) -> dict:
             if not isinstance(server, bool) and server.lower() not in [s.lower() for s in servers.keys()]:
                 server_names = [name for name in servers.keys()]
                 raise SettingsError(f'Recognized servers are {server_names}. Got: {server}')
+    check_remote_paths_of_path_naming_adapters(ess_settings=settings_dict, ts_adapters=ts_adapters)
     logger.info(f'\nUsing the following ESS settings:\n{pprint.pformat(settings_dict)}\n')
     return settings_dict
 
 
+def check_remote_paths_of_path_naming_adapters(ess_settings: dict,
+                                               ts_adapters: list[str] | None = None,
+                                               ) -> None:
+    """
+    Check that every server an adapter which names a path on the server will run on has an
+    absolute ``path``.
+
+    Reported here, before any calculation is spawned, because the alternative is reaching it when
+    the first reaction gets to its TS search: the adapter cannot build a usable input file for
+    such a server, so the failure arrives once per run either way, and arriving at startup is the
+    difference between a settings error the reader can act on and a run that has already spent
+    time on jobs it will not be able to use.
+
+    A server that is not in the ``servers`` settings is not reported here, since
+    :func:`check_ess_settings` has already rejected it by name.
+
+    Args:
+        ess_settings (dict): ARC's ESS settings dictionary, each ESS mapped to a list of servers.
+        ts_adapters (list, optional): The TS search adapters this run will use. ``None`` selects
+                                      the default set from the settings.
+
+    Raises:
+        SettingsError: If such an adapter would run on a server whose ``path`` is missing or not
+                       absolute.
+    """
+    adapters = settings.get('ts_adapters', list()) if ts_adapters is None else ts_adapters
+    adapters = [adapter.lower() for adapter in adapters if isinstance(adapter, str)]
+    if 'orca_neb' not in adapters:
+        return
+    for server in ess_settings.get('orca', list()):
+        if not isinstance(server, str) or server.lower() == 'local':
+            continue
+        server_key = next((key for key in servers.keys() if key.lower() == server.lower()), None)
+        if server_key is None:
+            continue
+        path = servers[server_key].get('path')
+        if not path or not os.path.isabs(path):
+            raise SettingsError(f'Server "{server}" has no absolute "path" entry in the settings, '
+                                f'got {path!r}, but the "orca_neb" adapter will run on it and its '
+                                f'input file must name a path on the server. Set "path" for this '
+                                f'server to the absolute directory holding the user directories, '
+                                f'e.g. "/home", or remove "orca_neb" from "ts_adapters".')
+
+
 def initialize_log(log_file: str,
                    project: str,
                    project_directory: str | None = None,

arc/common_test.py

@@ -1559,5 +1559,77 @@ def test_repeated_initialize_log_does_not_re_log_a_drained_warning(self):
         self.assertEqual(self._read_log().count('only flushed once'), 1)
 
 
+class TestCheckRemotePathsOfPathNamingAdapters(unittest.TestCase):
+    """
+    An adapter whose input file names a path on the server cannot run on a server that has no
+    absolute ``path``, and the shipped remote server examples had no ``path`` at all, so this is
+    reported at startup rather than when the first TS search reaches it.
+    """
+
+    def _use_servers(self, servers_dict):
+        """Point arc.common at ``servers_dict`` for the duration of one test."""
+        original = common.servers
+        common.servers = servers_dict
+        self.addCleanup(setattr, common, 'servers', original)
+
+    def test_a_server_without_a_path_is_reported(self):
+        """The ordinary remote configuration, since no shipped server example defined a path."""
+        self._use_servers({'remote': {'cluster_soft': 'PBS', 'address': 'host.edu', 'un': 'user'}})
+        with self.assertRaises(SettingsError) as raised:
+            common.check_remote_paths_of_path_naming_adapters(ess_settings={'orca': ['remote']},
+                                                              ts_adapters=['heuristics', 'orca_neb'])
+        self.assertIn('remote', str(raised.exception))
+        self.assertIn('path', str(raised.exception))
+
+    def test_an_absolute_path_is_accepted(self):
+        """A correctly configured server must not be reported."""
+        self._use_servers({'remote': {'cluster_soft': 'PBS', 'address': 'host.edu', 'un': 'user',
+                                      'path': '/home'}})
+        common.check_remote_paths_of_path_naming_adapters(ess_settings={'orca': ['remote']},
+                                                          ts_adapters=['orca_neb'])
+
+    def test_a_relative_path_is_reported(self):
+        """A relative path is what the remote directories are rooted at, and Orca cannot follow it."""
+        self._use_servers({'remote': {'cluster_soft': 'PBS', 'address': 'host.edu', 'un': 'user',
+                                      'path': 'runs'}})
+        with self.assertRaises(SettingsError):
+            common.check_remote_paths_of_path_naming_adapters(ess_settings={'orca': ['remote']},
+                                                              ts_adapters=['orca_neb'])
+
+    def test_an_adapter_that_is_not_used_is_not_checked(self):
+        """A server that no path-naming adapter runs on must not block a run that never uses one."""
+        self._use_servers({'remote': {'cluster_soft': 'PBS', 'address': 'host.edu', 'un': 'user'}})
+        common.check_remote_paths_of_path_naming_adapters(ess_settings={'orca': ['remote']},
+                                                          ts_adapters=['heuristics', 'autotst'])
+
+    def test_the_local_server_is_not_checked(self):
+        """A local job reads its files from the directory ARC wrote them to, with no remote path."""
+        self._use_servers({'local': {'cluster_soft': 'local', 'un': 'user'}})
+        common.check_remote_paths_of_path_naming_adapters(ess_settings={'orca': ['local']},
+                                                          ts_adapters=['orca_neb'])
+
+    def test_the_adapter_is_checked_on_the_ess_key_it_resolves_its_server_from(self):
+        """
+        OrcaNEBAdapter is given its server while it is still an OrcaAdapter, so it runs wherever
+        orca runs. Keying the check on "orca_neb" instead would never match a real configuration.
+        """
+        self._use_servers({'remote': {'cluster_soft': 'PBS', 'address': 'host.edu', 'un': 'user'}})
+        common.check_remote_paths_of_path_naming_adapters(ess_settings={'orca_neb': ['remote']},
+                                                          ts_adapters=['orca_neb'])
+
+    def test_check_ess_settings_runs_the_check(self):
+        """The startup path must reach it, which is the whole point of validating there."""
+        self._use_servers({'remote': {'cluster_soft': 'PBS', 'address': 'host.edu', 'un': 'user'}})
+        with self.assertRaises(SettingsError):
+            common.check_ess_settings(ess_settings={'orca': ['remote']}, ts_adapters=['orca_neb'])
+
+    def test_the_shipped_server_examples_define_an_absolute_path(self):
+        """The documented configuration must be a working one, not the one that fails."""
+        for name in ['server1', 'server2', 'server3']:
+            path = settings['servers'][name].get('path')
+            self.assertIsNotNone(path, f'{name} has no "path" entry')
+            self.assertTrue(os.path.isabs(path), f'{name} has a relative path: {path!r}')
+
+
 if __name__ == '__main__':
     unittest.main(testRunner=unittest.TextTestRunner(verbosity=2))

arc/imports.py

@@ -56,6 +56,49 @@ def resolve_overridden_dependents(settings: dict, local_settings_dict: dict) ->
                 queue_deferred_warning(msg)
 
 
+_UNUSABLE_OVERLAYS_REPORTED = set()
+
+
+def _report_unusable_overlay(path: str, module: str, error: ImportError, what: str = '') -> None:
+    """
+    Report that a local ~/.arc overlay could not supply a setting, so ARC's default is used.
+
+    An overlay that fails to load leaves ARC running on the repository defaults, which is a
+    working configuration and therefore produces no other symptom: the submit script or the
+    server definitions the user wrote are simply not the ones in use, and a run goes to the
+    wrong cluster or with the wrong resources. Naming the file and the error is what makes that
+    visible instead of silent.
+
+    Whether the overlay file itself loaded decides how loud the report is, since the two cases
+    mean opposite things. A file that loaded but does not define the name is a partial overlay,
+    which is the ordinary way to override one setting and leave the rest alone, and is reported
+    at the debug level. A file that did not load at all, most often because something it imports
+    is not installed, loses every setting in it; that is reported once at the warning level, and
+    queued so it survives the log being initialized later in the run.
+
+    Note that a syntax error in an overlay is not reported here and never reaches this function:
+    it is a ``SyntaxError`` rather than an ``ImportError``, and it propagates out of this module
+    and stops ARC from starting at all.
+
+    Args:
+        path (str): The overlay file that could not be used.
+        module (str): The module name the overlay is imported under.
+        error (ImportError): The import failure.
+        what (str, optional): The name that could not be imported, when the file itself loaded.
+    """
+    if module in sys.modules:
+        logger.debug(f'{path} does not define "{what}", so ARC\'s default is used. '
+                     f'Got {type(error).__name__}: {error}')
+        return
+    if path in _UNUSABLE_OVERLAYS_REPORTED:
+        return
+    _UNUSABLE_OVERLAYS_REPORTED.add(path)
+    msg = f'Could not import {path}, so none of the settings in it are used and ARC\'s defaults ' \
+          f'are used instead. Got {type(error).__name__}: {error}'
+    logger.warning(msg)
+    queue_deferred_warning(msg)
+
+
 # Common imports where the user can optionally put a modified copy of settings.py or submit.py file under ~/.arc
 home = os.getenv("HOME") or os.path.expanduser("~")
 local_arc_path = os.path.join(home, '.arc')
@@ -68,8 +111,8 @@ def resolve_overridden_dependents(settings: dict, local_settings_dict: dict) ->
         sys.path.insert(1, local_arc_path)
     try:
         import settings as local_settings
-    except ImportError:
-        pass
+    except ImportError as e:
+        _report_unusable_overlay(local_arc_settings_path, 'settings', e)
     if local_settings:
         local_settings_dict = {key: val for key, val in vars(local_settings).items() if '__' not in key}
         settings.update(local_settings_dict)
@@ -85,16 +128,16 @@ def resolve_overridden_dependents(settings: dict, local_settings_dict: dict) ->
         sys.path.insert(1, local_arc_path)
     try:
         from submit import incore_commands as local_incore_commands
-    except ImportError:
-        pass
+    except ImportError as e:
+        _report_unusable_overlay(local_arc_submit_path, 'submit', e, 'incore_commands')
     try:
         from submit import pipe_submit as local_pipe_submit
-    except ImportError:
-        pass
+    except ImportError as e:
+        _report_unusable_overlay(local_arc_submit_path, 'submit', e, 'pipe_submit')
     try:
         from submit import submit_scripts as local_submit_scripts
-    except ImportError:
-        pass
+    except ImportError as e:
+        _report_unusable_overlay(local_arc_submit_path, 'submit', e, 'submit_scripts')
     if local_incore_commands:
         incore_commands.update(local_incore_commands)
     if local_pipe_submit:
@@ -109,7 +152,7 @@ def resolve_overridden_dependents(settings: dict, local_settings_dict: dict) ->
         sys.path.insert(1, local_arc_path)
     try:
         from inputs import input_files as local_input_files
-    except ImportError:
-        pass
+    except ImportError as e:
+        _report_unusable_overlay(local_arc_inputs_path, 'inputs', e, 'input_files')
     if local_input_files:
         input_files.update(local_input_files)