From 7c67f5cb32f8041afd4f7ef78073b6c471fc37ce Mon Sep 17 00:00:00 2001 From: Calvin Pieters Date: Sat, 22 Aug 2026 13:08:58 +0300 Subject: [PATCH 1/9] Share one SSH connection per server across remote jobs Every remote job opened its own paramiko Transport for upload, submission and status polling. A TS search issuing ~100 guess optimizations against one server opened ~100 connections, which is slow and trips per-user connection limits on some clusters. Add a process-global pool (arc/job/ssh_pool.py). A remote queue job leases one client for the duration of its submission and reuses it for both file upload and run; _open_or_borrow_ssh() prefers that leased client and otherwise borrows from the pool. The borrower never closes a client it does not own. The third case, a one-shot client for when the pool itself cannot lease one, is real rather than nominal. Written as `try: return pool.borrow(server)` with an `except` around it, it could never fire: borrow is a @contextmanager, so calling it merely builds the generator, and the factory runs when the caller enters the with-block -- outside the try meant to catch it. Rebuilt around contextlib.ExitStack, the fallback covers exactly the lease, i.e. entering the pool's context manager, and nothing else. In particular it does not cover the caller's own work: a job that raises inside its with-block still raises, instead of having its failure read as a broken pool and its body re-entered against a fresh client. Two pre-existing defects in the remote path that set_file_paths() builds are fixed here as well, since a remote job's files are only reachable if that path is. The server's configured 'path' was lowercased before use, which silently rewrites any path with an uppercase component -- remote file systems are case-sensitive, and /Home/Users is not /home/users. It is now used verbatim. NOTE that this changes where files land for anyone whose configured 'path' contains uppercase: their earlier runs are under the lowercased tree, and ARC will now use the path as configured. A server with no 'path' still gets a path relative to the SSH login directory, because there is no absolute path ARC can know offline -- the home directory is the server's to report, and set_file_paths() runs at job construction with no connection open. Rooting it at '~' would be worse rather than better: _send_command_to_server quotes the remote path with shlex.quote, so the remote shell would take the tilde literally, and SFTP performs no tilde expansion at all, so both would end up creating a directory actually named '~'. Relative, which the remote shell and SFTP both resolve against the login directory, stays correct. What changes is that it is no longer silent: such a server is reported once per run, naming the setting, because an adapter that has to name a path inside an input file cannot work with it. Pooled clients also need closing when ARC exits. ssh_pool.py documented that ARC.py's main() calls reset_default_pool(), but nothing did -- every caller was a test, so pooled SSHClients were left to interpreter shutdown rather than closed. Releasing them belongs to the run rather than to the command-line entry point, so ARC.execute() does it in a finally: connections are torn down on ctrl-C and on an exception as well as on a clean run, and a consumer that drives ARC in process -- a library caller, a test, a pipe worker -- releases them too, which an ARC.py-only hook could never do. ARC.py is unchanged. The borrow itself is now one function, ssh_pool.borrow_ssh_client(), rather than a method on JobAdapter. The pool's other callers are not adapters -- Scheduler.get_server_job_ids(), server troubleshooting, the ESS survey -- and each would otherwise have grown its own copy of the lease-then-fall-back dance. _open_or_borrow_ssh() keeps only what is adapter-specific, the per-execute() leased client, and delegates the rest; the shared-client branch is contextlib.nullcontext rather than a hand-rolled generator. A pooled connection is held for the whole run and sits idle between polls, so _default_factory sets a keepalive on the transport. An SSH daemon's ClientAliveInterval or a firewall's idle timeout otherwise drops it silently: the socket stays half-open, Transport.is_active() keeps reporting True, and the pool's liveness check therefore hands out a handle whose first command hangs until TCP gives up. upload_file() and download_file() were the only SSHClient methods reaching for self._sftp without @check_connections. That was harmless while every caller opened its own client and used it immediately; with a client that has been alive for hours it is not, since a dead transport surfaces as the transfer failing rather than as a reconnect. Both are decorated now. The pool is tested directly rather than only through a JobAdapter, so its own contract -- reuse, reaping a dead client, retaining ownership on context exit, idempotent close_all -- is stated by its tests instead of implied by adapter behaviour. arc/job/ssh_pool_test.py drives SSHConnectionPool with a stub factory and covers the cases adapter_test.py could not reach, namely that a raising with-body leaves the pool reusable and that reset_default_pool() closes pooled clients rather than just dropping the reference. The adapter-driven integration tests stay with the adapter, which is what they actually exercise. Two claims the pool's docstrings make were still untested, and the imports for them were sitting unused in adapter_test.py: that a remote-queue execute() with no pool injected borrows from the instance get_default_pool() returns, and that reset_default_pool() -- ARC.py's exit hook -- closes the clients those jobs opened and leaves a usable empty pool behind. Both are now asserted rather than implied. The pool tearDowns also called set_default_pool(None), which drops the reference without closing anything, so each test class leaked its stub clients and contradicted the lifecycle ssh_pool.py documents; they call reset_default_pool() instead. Also two test-only cleanups CodeQL flags: two factory lambdas that only forwarded their argument now pass the callable itself, and the "a raising with-body leaves the pool usable" test uses assertRaises' callable form. Its context-manager form made every statement after the block unreachable to a control-flow analyser, because nothing in the CFG says assertRaises.__exit__ suppresses the exception. Two of CodeQL's remaining alerts on this file are in adapter_test.py and are the same two defects already fixed in ssh_pool_test.py. The assertRaises context-manager form around a with-block whose last statement is `raise` makes everything after the block unreachable to a control-flow analyser, since nothing in the CFG says __exit__ suppresses the exception; both tests use the callable form, and still assert what they did -- that the caller's own exception reaches the caller, and that the pool is usable afterwards. And the module both imported arc.job.adapter and imported names from it; the module-alias form existed for two patch.object() calls, which are now patch('arc.job.adapter.'), so the file uses one import form. Dropping the alias also removes a name that a class attribute in the same file shadowed. Absorbed from PR #1000 by @alongd, brought in here rather than merged so the two pull requests do not conflict over the same lines: set_file_paths() splits the project's remote directory out of the job's remote path as remote_project_path, which is what the remote check file cleanup is scoped to. Record the keepalive interval on the client as well as on its transport. A keepalive belongs to a paramiko Transport, and check_connections reconnects a client in place when its socket has gone half-open, so the transport the factory set the keepalive on is not the one the client ends up holding. SSHClient.connect() re-applies the recorded interval to every transport it opens, which is the case a connection held for a whole run actually meets. Also brings the module to ARC's conventions: docstrings on __init__ and _close_quietly, Args and Returns sections, f-string logging in place of %s, single-quoted strings, and no comments on code lines. --- arc/job/adapter.py | 205 +++++++++++++--- arc/job/adapter_test.py | 515 ++++++++++++++++++++++++++++++++++++++- arc/job/ssh_pool.py | 294 ++++++++++++++++++++++ arc/job/ssh_pool_test.py | 328 +++++++++++++++++++++++++ 4 files changed, 1301 insertions(+), 41 deletions(-) create mode 100644 arc/job/ssh_pool.py create mode 100644 arc/job/ssh_pool_test.py diff --git a/arc/job/adapter.py b/arc/job/adapter.py index 1b673a2057..4256b914e4 100644 --- a/arc/job/adapter.py +++ b/arc/job/adapter.py @@ -17,8 +17,9 @@ import shutil import time from abc import ABC, abstractmethod +from contextlib import nullcontext from enum import Enum -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, ContextManager import numpy as np @@ -34,6 +35,7 @@ ) from arc.job.trsh import trsh_job_on_server, trsh_job_queue from arc.job.ssh import SSHClient +from arc.job.ssh_pool import borrow_ssh_client, get_default_pool from arc.job.trsh import determine_ess_status from arc.species.vectors import calculate_dihedral_angle @@ -42,6 +44,8 @@ logger = get_logger() +_SERVERS_WARNED_ABOUT_A_RELATIVE_REMOTE_PATH = set() + default_job_settings, servers, submit_filenames, t_max_format, input_filenames, output_filenames = \ settings['default_job_settings'], settings['servers'], settings['submit_filenames'], settings['t_max_format'], \ settings['input_filenames'], settings['output_filenames'] @@ -207,7 +211,7 @@ def execute_queue(self): """ pass - def execute(self): + def execute(self) -> None: """ Execute a job. The execution type could be 'incore', 'queue', or 'pipe'. @@ -222,9 +226,68 @@ def execute(self): with an HDF5 file that contains specific directions. The output is returned within the HDF5 file. The new ARC instance, representing a single worker, will run all of its jobs incore. + + Connection sharing: for remote-queue jobs we lease one + :class:`SSHClient` from the process-global pool + (:mod:`arc.job.ssh_pool`) and reuse it for both file upload and + qsub/sbatch submission within this call. Across an entire ARC + run, every remote job for a given server reuses the *same* + pooled client — 100 TS guess opts share one paramiko Transport + instead of opening 200. Pipe mode currently can't bundle these + (``should_use_pipe`` refuses non-``local`` servers, see + ``arc/job/pipe/pipe_coordinator.py:77``); the pool is the + leverage available short of full remote-pipe support. """ - self.upload_files() execution_type = JobExecutionTypeEnum(self.execution_type) + use_shared_ssh = ( + execution_type == JobExecutionTypeEnum.queue + and self.server is not None + and self.server != 'local' + and not self.testing + ) + if use_shared_ssh: + with get_default_pool().borrow(self.server) as ssh: + self._shared_ssh = ssh + try: + self._dispatch_execution(execution_type) + finally: + # Pool retains the SSHClient; clearing the attr + # just prevents a later code path on this adapter + # from grabbing a stale reference if the pool + # subsequently reaps and reopens the connection. + self._shared_ssh = None + else: + self._dispatch_execution(execution_type) + if not self.restarted: + self._write_initiated_job_to_csv_file() + + def _open_or_borrow_ssh(self) -> ContextManager[SSHClient]: + """Return a context manager yielding an :class:`SSHClient` for ``self.server``, + in priority order: + + 1. ``self._shared_ssh`` if set — the per-call client :meth:`execute` leased. + Available within the upload+submit window. + 2. The process-global pool, through + :func:`arc.job.ssh_pool.borrow_ssh_client` — the single borrow path every + SSH caller in ARC shares, which keeps one client alive across jobs for the + run's lifetime so the hot status-poll loop reuses connections, and which + falls back to a one-shot client when the lease itself fails. + + Exiting the context does not close a client obtained either way; the pool + retains ownership, and the fallback closes only the client it opened itself. + + Returns: ContextManager[SSHClient] + A context manager yielding a connected client for ``self.server``. + """ + shared = getattr(self, '_shared_ssh', None) + if shared is not None: + return nullcontext(shared) + return borrow_ssh_client(self.server) + + def _dispatch_execution(self, execution_type: JobExecutionTypeEnum) -> None: + """Inner body of :meth:`execute`, factored out so the SSH-share + wrapper around it stays small and readable.""" + self.upload_files() if execution_type == JobExecutionTypeEnum.incore: self.initial_time = datetime.datetime.now() self.job_status[0] = 'running' @@ -239,19 +302,25 @@ def execute(self): raise ValueError('Pipe execution is handled at the Scheduler level. ' 'JobAdapters inside a pipe must be executed by the worker ' "with execution_type='incore'.") - if not self.restarted: - self._write_initiated_job_to_csv_file() - def legacy_queue_execution(self): + def legacy_queue_execution(self, ssh: SSHClient | None = None) -> None: """ Execute a job to the server's queue. The server could be either "local" or remote. + + ``ssh`` is an explicitly-passed shared connection. When ``None`` + we route through :meth:`_open_or_borrow_ssh` which prefers + ``self._shared_ssh`` (set by :meth:`execute`), then the + process-global pool, then opens fresh. """ self._log_job_execution() # Submit to queue, differentiate between local (same machine using its queue) and remote servers. if self.server != 'local': - with SSHClient(self.server) as ssh: + if ssh is not None: self.job_status[0], self.job_id = ssh.submit_job(remote_path=self.remote_path) + else: + with self._open_or_borrow_ssh() as borrowed: + self.job_status[0], self.job_id = borrowed.submit_job(remote_path=self.remote_path) else: # submit to the local queue self.job_status[0], self.job_id = submit_job(path=self.local_path) @@ -337,9 +406,15 @@ def write_submit_script(self) -> None: with open(os.path.join(self.local_path, submit_filenames[servers[self.server]['cluster_soft']]), 'w') as f: f.write(submit_script) - def set_file_paths(self): + def set_file_paths(self) -> None: """ Set local and remote job file paths. + + The remote path is rooted at the server's ``path`` setting, joined with the user name, + and is used verbatim: a path is a path on the server, and the case it is written in is + the case it has there. A server with no ``path`` gets a path relative to the SSH login + directory, which is where both the remote shell and SFTP resolve it, and which + :meth:`_warn_about_a_relative_remote_path` reports once per server. """ folder_name = 'TS_guesses' if self.reactions is not None else 'TSs' if self.species[0].is_ts else 'Species' if self.run_multi_species == False: @@ -360,33 +435,51 @@ def set_file_paths(self): # Parentheses don't play well in folder names: species_name_remote = self.species_label if isinstance(self.species_label, str) else self.species[0].multi_species species_name_remote = species_name_remote.replace('(', '_').replace(')', '_') - path = servers[self.server].get('path', '').lower() - path = os.path.join(path, servers[self.server]['un']) if path else '' + path = servers[self.server].get('path') or '' + if path: + path = os.path.join(path, servers[self.server]['un']) + elif self.server != 'local': + self._warn_about_a_relative_remote_path(self.server) self.remote_project_path = os.path.join(path, 'runs', 'ARC_Projects', self.project) self.remote_path = os.path.join(self.remote_project_path, species_name_remote, self.job_name) self.set_additional_file_paths() - def upload_files(self): + @staticmethod + def _warn_about_a_relative_remote_path(server: str) -> None: + """ + Report, once per server per run, that the server's remote job paths are not absolute. + + Args: + server (str): The server name. + """ + if server in _SERVERS_WARNED_ABOUT_A_RELATIVE_REMOTE_PATH: + return + _SERVERS_WARNED_ABOUT_A_RELATIVE_REMOTE_PATH.add(server) + logger.warning(f'Server "{server}" has no "path" entry in the settings, so its remote job ' + f'directories are relative to the SSH login directory. Jobs whose input ' + f'file must name a path on the server, such as Orca NEB, cannot run that ' + f'way and will refuse. Set "path" for this server to the directory holding ' + f'the user directories, e.g. "/home".') + + def upload_files(self, ssh: SSHClient | None = None) -> None: """ Upload the relevant files for the job. + + ``ssh`` is an explicitly-passed shared connection. When ``None`` + we route through :meth:`_open_or_borrow_ssh` which prefers + ``self._shared_ssh`` (set by :meth:`execute`), then the + process-global pool, then opens fresh. """ if not self.testing: if self.execution_type != 'incore' and self.server != 'local': # If the job execution type is incore, then no need to upload any files. # Also, even if the job is submitted to the que, no need to upload files if the server is local. - with SSHClient(self.server) as ssh: - for up_file in self.files_to_upload: - logger.debug(f"Uploading {up_file['file_name']} source {up_file['source']} to {self.server}") - if up_file['source'] == 'path': - ssh.upload_file(remote_file_path=up_file['remote'], local_file_path=up_file['local']) - elif up_file['source'] == 'input_files': - ssh.upload_file(remote_file_path=up_file['remote'], file_string=up_file['local']) - else: - raise ValueError(f"Unclear file source for {up_file['file_name']}. Should either be 'path' or " - f"'input_files', got: {up_file['source']}") - if up_file['make_x']: - ssh.change_mode(mode='+x', file_name=up_file['file_name'], remote_path=self.remote_path) + if ssh is not None: + self._upload_with_ssh(ssh) + else: + with self._open_or_borrow_ssh() as borrowed: + self._upload_with_ssh(borrowed) else: # running locally, just copy the check file, if exists, to the job folder for up_file in self.files_to_upload: @@ -397,7 +490,26 @@ def upload_files(self): pass self.initial_time = datetime.datetime.now() - def download_files(self): + def _upload_with_ssh(self, ssh: SSHClient) -> None: + """SFTP-put every entry in ``self.files_to_upload`` over an open client. + + Factored out of :meth:`upload_files` so the with-shared vs. + with-new code paths share one body — adding a future per-file + knob (compression, retry, throttle) lands in one place. + """ + for up_file in self.files_to_upload: + logger.debug(f"Uploading {up_file['file_name']} source {up_file['source']} to {self.server}") + if up_file['source'] == 'path': + ssh.upload_file(remote_file_path=up_file['remote'], local_file_path=up_file['local']) + elif up_file['source'] == 'input_files': + ssh.upload_file(remote_file_path=up_file['remote'], file_string=up_file['local']) + else: + raise ValueError(f"Unclear file source for {up_file['file_name']}. Should either be 'path' or " + f"'input_files', got: {up_file['source']}") + if up_file['make_x']: + ssh.change_mode(mode='+x', file_name=up_file['file_name'], remote_path=self.remote_path) + + def download_files(self) -> None: """ Download the relevant files. """ @@ -405,7 +517,7 @@ def download_files(self): if self.execution_type != 'incore' and self.server != 'local': # If the job execution type is incore, then no need to download any files. # Also, even if the job is submitted to the que, no need to download files if the server is local. - with SSHClient(self.server) as ssh: + with self._open_or_borrow_ssh() as ssh: for dl_file in self.files_to_download: ssh.download_file(remote_file_path=dl_file['remote'], local_file_path=dl_file['local']) self.set_initial_and_final_times(ssh=ssh) @@ -413,6 +525,21 @@ def download_files(self): self.set_initial_and_final_times() self.final_time = self.final_time or datetime.datetime.now() + def remove_remote_files(self) -> None: + """ + Remove the job's remote work directory, to keep cluster quota in check. + + This is the remote-cleanup entry point for a job. ARC's own job flow does not call it: + no job removes its remote work directory today, and the caller that will is added + separately, which is why a run still leaves its remote directories behind. + + Does nothing for a local server or when no remote path has been set. + """ + if self.server is None or self.server == 'local' or not self.remote_path: + return + with self._open_or_borrow_ssh() as ssh: + ssh.remove_dir(remote_path=self.remote_path) + def set_initial_and_final_times(self, ssh: SSHClient | None = None): """ Set the end time of the job. @@ -705,7 +832,7 @@ def delete(self): logger.debug(f'Deleting job {self.job_name} for {self.species_label}') if self.server != 'local': logger.debug(f'deleting job on {self.server}...') - with SSHClient(self.server) as ssh: + with self._open_or_borrow_ssh() as ssh: ssh.delete_job(self.job_id) else: logger.debug('deleting job locally...') @@ -771,20 +898,20 @@ def _get_additional_job_info(self): # No queueing system, so there are no scheduler stdout/stderr files to collect. return if cluster_soft in ['oge', 'sge', 'slurm', 'pbs', 'htcondor']: + # job.log is HTCondor's native event log; other clusters don't produce one. + include_job_log = cluster_soft == 'htcondor' local_file_path_1 = os.path.join(self.local_path, 'out.txt') local_file_path_2 = os.path.join(self.local_path, 'err.txt') - local_file_path_3 = os.path.join(self.local_path, 'job.log') + local_file_path_3 = os.path.join(self.local_path, 'job.log') if include_job_log else None if self.server != 'local' and self.remote_path is not None and not self.testing: - remote_file_path_1 = os.path.join(self.remote_path, 'out.txt') - remote_file_path_2 = os.path.join(self.remote_path, 'err.txt') - remote_file_path_3 = os.path.join(self.remote_path, 'job.log') - with SSHClient(self.server) as ssh: - for local_file_path, remote_file_path in zip([local_file_path_1, - local_file_path_2, - local_file_path_3], - [remote_file_path_1, - remote_file_path_2, - remote_file_path_3]): + remote_paths = [os.path.join(self.remote_path, 'out.txt'), + os.path.join(self.remote_path, 'err.txt')] + local_paths = [local_file_path_1, local_file_path_2] + if include_job_log: + remote_paths.append(os.path.join(self.remote_path, 'job.log')) + local_paths.append(local_file_path_3) + with self._open_or_borrow_ssh() as ssh: + for local_file_path, remote_file_path in zip(local_paths, remote_paths): try: ssh.download_file(remote_file_path=remote_file_path, local_file_path=local_file_path) @@ -794,7 +921,7 @@ def _get_additional_job_info(self): f'flags with stdout and stderr of out.txt and err.txt, respectively ' f'(e.g., "#SBATCH -o out.txt"). Error message:') logger.warning(e) - for local_file_path in [local_file_path_1, local_file_path_2, local_file_path_3]: + for local_file_path in filter(None, [local_file_path_1, local_file_path_2, local_file_path_3]): if os.path.isfile(local_file_path): with open(local_file_path, 'r') as f: lines = f.readlines() @@ -810,7 +937,7 @@ def _check_job_server_status(self) -> str: Possible statuses: ``initializing``, ``running``, ``errored on node xx``, ``done``. """ if self.server != 'local' and not self.testing: - with SSHClient(self.server) as ssh: + with self._open_or_borrow_ssh() as ssh: return ssh.check_job_status(self.job_id) else: return check_job_status(self.job_id) diff --git a/arc/job/adapter_test.py b/arc/job/adapter_test.py index 2691aa129a..3499cc5f19 100644 --- a/arc/job/adapter_test.py +++ b/arc/job/adapter_test.py @@ -16,9 +16,16 @@ from unittest.mock import patch from arc.common import ARC_TESTING_PATH +from arc.exceptions import ServerError from arc.imports import settings from arc.job.adapter import JobAdapter, JobEnum, JobTypeEnum, JobExecutionTypeEnum from arc.job.adapters.gaussian import GaussianAdapter +from arc.job.ssh_pool import ( + SSHConnectionPool, + get_default_pool, + reset_default_pool, + set_default_pool, +) from arc.level import Level from arc.species import ARCSpecies @@ -89,8 +96,11 @@ def setUpClass(cls): A method that is run before all unit tests in this class. """ cls.maxDiff = None - for dir_name in ('test_JobAdapter', 'test_JobAdapter_scan', 'test_JobAdapter_ServerTimeLimit'): - cls.addClassCleanup(shutil.rmtree, os.path.join(ARC_TESTING_PATH, dir_name), ignore_errors=True) + # Register project-dir cleanups before any fixture creation so they + # still fire if a constructor below raises mid-setUpClass — that's + # how leftover scratch files end up committed to the repo. + for subdir in ('test_JobAdapter', 'test_JobAdapter_scan', 'test_JobAdapter_ServerTimeLimit'): + cls.addClassCleanup(shutil.rmtree, os.path.join(ARC_TESTING_PATH, subdir), ignore_errors=True) cls.job_1 = GaussianAdapter(execution_type='queue', job_type='conf_opt', level=Level(method='cbs-qb3'), @@ -403,5 +413,506 @@ def test_multiple_rotations(self): self.assertEqual(len(archives), 2) +# --------------------------------------------------------------------------- +# SSH connection sharing & pooling (Options 1 + 2). +# +# Option 1 (per-job share): one SSHClient covers both upload and submit +# inside a single execute() call — collapses 2N connections to N. +# Option 2 (process-lifetime pool): the SSHClient for a given server is +# kept alive across jobs — collapses N to a small constant. +# --------------------------------------------------------------------------- + + +class _SSHClientStub: + """In-memory SSHClient lookalike for the pool to hand out. + + Records every upload/submit so tests can assert which calls landed + on which (shared) client. The pool calls ``connect()`` after + instantiation; we no-op that since there's no real socket. + """ + + def __init__(self, server): + self.server = server + self.uploaded = [] + self.submits = [] + self.downloaded = [] + self._closed = False + # Mimic SSHClient's ``_ssh`` attribute so ssh_pool._is_alive() + # finds an active fake-Transport. + self._ssh = _FakeParamikoSSH() + + def connect(self): + pass # the real one opens TCP+auth; we no-op for tests + + def close(self): + self._closed = True + self._ssh = None + + def __enter__(self): + return self + + def __exit__(self, *a): + return False + + def upload_file(self, *, remote_file_path, local_file_path=None, file_string=None): + self.uploaded.append(remote_file_path) + + def submit_job(self, remote_path, recursion=False): + self.submits.append(remote_path) + return 'initializing', 12345 + + def change_mode(self, *, mode, file_name, remote_path): + pass + + # Methods that the post-submit lifecycle paths exercise. + def check_job_status(self, job_id): + return 'running' + + def download_file(self, *, remote_file_path, local_file_path): + self.downloaded.append(remote_file_path) + + def remove_dir(self, *, remote_path): + pass + + def delete_job(self, job_id): + pass + + +class _FakeParamikoSSH: + """Stand-in for paramiko.SSHClient — _is_alive checks Transport.is_active().""" + def get_transport(self): + return _FakeTransport() + + +class _FakeTransport: + def is_active(self): + return True + + +class _StubFactoryPool: + """A pool whose factory builds _SSHClientStub instead of real SSHClient. + + Wraps the production ``SSHConnectionPool`` so reuse + lifecycle + semantics are exactly the production behavior — only the + underlying object is faked. + """ + + def __init__(self): + self.created = [] # log of every server name we built a client for + def factory(server): + client = _SSHClientStub(server) + self.created.append(server) + return client + self._inner = SSHConnectionPool(factory=factory) + + def borrow(self, server): + return self._inner.borrow(server) + + def close_all(self): + self._inner.close_all() + + @property + def opens(self): + return self._inner.opens + + @property + def borrows(self): + return self._inner.borrows + + +class _MinimalAdapter(JobAdapter): + """Concrete JobAdapter with just enough state to exercise execute(). + + Skips the heavyweight construction the GaussianAdapter does — we + only need ``server``, ``execution_type``, ``files_to_upload``, + ``remote_path``, and ``testing=False`` for the SSH-share path. + """ + + job_adapter = 'mockter' + + def __init__(self, *, server, execution_type='queue'): + # Bypass JobAdapter.__init__ entirely — all of its real work + # (file paths, settings, csv setup) is unrelated to the SSH + # share contract we're testing here. + self.server = server + self.execution_type = execution_type + self.testing = False + self.restarted = True # skip _write_initiated_job_to_csv_file + self.files_to_upload = [ + {'file_name': 'input.gjf', 'source': 'path', + 'local': '/local/input.gjf', 'remote': '/remote/input.gjf', 'make_x': False}, + {'file_name': 'submit.sh', 'source': 'path', + 'local': '/local/submit.sh', 'remote': '/remote/submit.sh', 'make_x': True}, + ] + self.remote_path = '/remote' + self.local_path = '/local' + self.job_status = ['initializing', {'status': 'initializing'}] + self.job_id = 0 + self.initial_time = None + self.final_time = None + self.job_name = 'job_test' + self.species_label = 'spc_test' + + # JobAdapter requires these abstracts; trivial bodies are fine. + def execute_incore(self): pass + def execute_queue(self): self.legacy_queue_execution() + def write_input_file(self): pass + def set_files(self): pass + def set_additional_file_paths(self): pass + def set_input_file_memory(self): pass + def upload_during_execution(self): pass + def _log_job_execution(self): pass + + +class TestSSHConnectionSharing(unittest.TestCase): + """``execute()`` shares one SSHClient per remote-queue job, and the + pool reuses it across jobs.""" + + def setUp(self): + # Inject a pool whose factory builds stubs, so the test never + # tries to open a real SSH connection to a server that isn't + # in this user's settings (e.g., 'server2'). + self._stub_pool = _StubFactoryPool() + set_default_pool(self._stub_pool) + # Also stub the one-shot fallback: when a lease fails, + # ssh_pool.borrow_ssh_client() opens an SSHClient itself, so + # patch that name with a context-manager wrapper around our stub. + self._direct_patch = patch( + 'arc.job.ssh_pool.SSHClient', + _SSHClientStub, + ) + self._direct_patch.start() + + def tearDown(self): + reset_default_pool() + self._direct_patch.stop() + + def test_remote_queue_opens_one_ssh_per_job(self): + """Upload + submit share a single SSHClient inside one execute().""" + adapter = _MinimalAdapter(server='server2', execution_type='queue') + adapter.execute() + # One SSHClient created (the pool's first borrow), one borrow. + self.assertEqual(self._stub_pool.opens, 1) + self.assertEqual(self._stub_pool.borrows, 1) + + def test_remote_queue_clears_shared_ssh_after_dispatch(self): + """``self._shared_ssh`` is None after execute() returns.""" + adapter = _MinimalAdapter(server='server2', execution_type='queue') + adapter.execute() + self.assertIsNone(getattr(adapter, '_shared_ssh', None)) + + def test_local_server_opens_no_ssh(self): + """local-server queue jobs use the host's queue, no SSH at all.""" + adapter = _MinimalAdapter(server='local', execution_type='queue') + with patch('arc.job.adapter.submit_job', return_value=('initializing', 99)): + adapter.execute() + self.assertEqual(self._stub_pool.opens, 0) + self.assertEqual(self._stub_pool.borrows, 0) + + def test_incore_opens_no_ssh(self): + """incore execution runs in-process — never touches SSH.""" + adapter = _MinimalAdapter(server='server2', execution_type='incore') + adapter.execute() + self.assertEqual(self._stub_pool.opens, 0) + + def test_legacy_queue_execution_routes_through_pool_when_called_directly(self): + """Even when called bare (outside execute()), legacy_queue_execution + now reuses the pool — that's Option 2's payoff for adapter + ``execute_queue`` overrides that call ``self.legacy_queue_execution()`` + from inside their own custom flow. + """ + adapter = _MinimalAdapter(server='server2', execution_type='queue') + adapter.legacy_queue_execution() # bare — no execute() wrapper + self.assertEqual(self._stub_pool.opens, 1) + self.assertEqual(self._stub_pool.borrows, 1) + + def test_shared_ssh_carries_uploads_and_submit(self): + """The pooled SSHClient sees both upload calls AND the submit call.""" + adapter = _MinimalAdapter(server='server2', execution_type='queue') + adapter.execute() + # Inspect the stub the pool kept. + self.assertEqual(self._stub_pool.opens, 1) + client = self._stub_pool._inner._clients['server2'] + self.assertEqual(len(client.uploaded), 2) + self.assertEqual(len(client.submits), 1) + + +class TestSSHConnectionDefaultPool(unittest.TestCase): + """Remote-queue jobs borrow from the process-global pool that ARC.py resets on exit.""" + + def setUp(self): + reset_default_pool() + self.addCleanup(reset_default_pool) + + def test_execute_borrows_from_the_process_global_pool(self): + """With no pool passed in, execute() uses the instance get_default_pool() returns.""" + set_default_pool(SSHConnectionPool(factory=_SSHClientStub)) + _MinimalAdapter(server='server2', execution_type='queue').execute() + pool = get_default_pool() + self.assertEqual(pool.opens, 1) + self.assertEqual(sorted(pool._clients.keys()), ['server2']) + + def test_reset_default_pool_closes_clients_opened_by_execute(self): + """ARC.py's exit hook must close the connections the adapters opened.""" + set_default_pool(SSHConnectionPool(factory=_SSHClientStub)) + _MinimalAdapter(server='server2', execution_type='queue').execute() + client = get_default_pool()._clients['server2'] + reset_default_pool() + self.assertTrue(client._closed) + + def test_the_default_pool_is_recreated_empty_after_a_reset(self): + """A reset must leave a usable pool behind, not a closed one.""" + set_default_pool(SSHConnectionPool(factory=_SSHClientStub)) + _MinimalAdapter(server='server2', execution_type='queue').execute() + reset_default_pool() + self.assertEqual(get_default_pool().opens, 0) + self.assertEqual(get_default_pool()._clients, {}) + + +class TestSSHConnectionPoolReuse(unittest.TestCase): + """The process-lifetime pool reuses one SSHClient across many jobs.""" + + def setUp(self): + self._stub_pool = _StubFactoryPool() + set_default_pool(self._stub_pool) + + def tearDown(self): + reset_default_pool() + + def test_one_open_for_many_jobs_same_server(self): + """100 jobs against one server → 1 SSHClient, 100 borrows.""" + for _ in range(100): + adapter = _MinimalAdapter(server='server2', execution_type='queue') + adapter.execute() + self.assertEqual(self._stub_pool.opens, 1, "should reuse the same client") + self.assertEqual(self._stub_pool.borrows, 100) + + def test_separate_clients_per_distinct_server(self): + """Different servers → different clients, each opened once.""" + for _ in range(5): + _MinimalAdapter(server='server2', execution_type='queue').execute() + for _ in range(3): + _MinimalAdapter(server='server3', execution_type='queue').execute() + self.assertEqual(self._stub_pool.opens, 2) + self.assertEqual(self._stub_pool.borrows, 8) + self.assertEqual(sorted(self._stub_pool._inner._clients.keys()), + ['server2', 'server3']) + + def test_dead_client_is_reaped_and_reopened(self): + """If the underlying Transport reports inactive, pool reopens.""" + # First borrow → opens stub #1. + _MinimalAdapter(server='server2', execution_type='queue').execute() + client1 = self._stub_pool._inner._clients['server2'] + # Simulate a dead Transport (remote rebooted, etc.). + client1._ssh = None + # Next borrow should detect the dead client and open a fresh one. + _MinimalAdapter(server='server2', execution_type='queue').execute() + client2 = self._stub_pool._inner._clients['server2'] + self.assertIs(client1._closed, True, "stale client should be closed before reopen") + self.assertIsNot(client1, client2) + self.assertEqual(self._stub_pool.opens, 2) + + def test_close_all_closes_every_pooled_client(self): + for srv in ('server2', 'server3'): + _MinimalAdapter(server=srv, execution_type='queue').execute() + clients = list(self._stub_pool._inner._clients.values()) + self._stub_pool.close_all() + self.assertEqual(self._stub_pool._inner._clients, {}) + for c in clients: + self.assertTrue(c._closed) + + def test_close_all_is_idempotent(self): + _MinimalAdapter(server='server2', execution_type='queue').execute() + self._stub_pool.close_all() + # Second call must not raise or mutate state. + self._stub_pool.close_all() + self.assertEqual(self._stub_pool._inner._clients, {}) + + def test_status_poll_reuses_pooled_client(self): + """The hot path: hundreds of status checks open exactly one client. + + ARC polls a job's queue status every poll cycle for the entire + duration of the job. Pre-pool, each call opened a fresh + SSHClient. After Option 2, all polls reuse the pool's client + for that server — the dominant SSH-cost reducer in a real run. + """ + adapter = _MinimalAdapter(server='server2', execution_type='queue') + # Simulate 200 poll cycles (~1.5 hour run at 30s polling). + for _ in range(200): + adapter._check_job_server_status() + self.assertEqual(self._stub_pool.opens, 1, "pool should reuse one client") + self.assertEqual(self._stub_pool.borrows, 200) + + def test_download_files_reuses_pooled_client(self): + """download_files (called once per finished job) uses the pool too.""" + adapter = _MinimalAdapter(server='server2', execution_type='queue') + adapter.files_to_download = [ + {'remote': '/r/output.log', 'local': '/l/output.log'}, + ] + # set_initial_and_final_times reads file mtimes — stub it. + adapter.set_initial_and_final_times = lambda ssh=None: None + adapter.download_files() + client = self._stub_pool._inner._clients['server2'] + self.assertIn('/r/output.log', client.downloaded) + self.assertEqual(self._stub_pool.opens, 1) + + def test_full_lifecycle_one_open_per_server(self): + """Submit + many polls + download + cleanup all share one pooled client. + + End-to-end view of one job's life: this collapses what was + previously ~(2 + N_polls + 1 + 1) ≈ N+4 individual SSH + connections into a single reused client. + """ + adapter = _MinimalAdapter(server='server2', execution_type='queue') + adapter.files_to_download = [{'remote': '/r/o.log', 'local': '/l/o.log'}] + adapter.set_initial_and_final_times = lambda ssh=None: None + + adapter.execute() # upload + submit (1 borrow) + for _ in range(50): # 50 status polls + adapter._check_job_server_status() + adapter.download_files() # 1 download borrow + adapter.remove_remote_files() # 1 cleanup borrow + adapter.delete() # 1 delete borrow + + # All phases share the same pooled client. + self.assertEqual(self._stub_pool.opens, 1) + # 1 execute + 50 polls + 1 download + 1 cleanup + 1 delete = 54 borrows. + self.assertEqual(self._stub_pool.borrows, 54) + + +class TestSSHPoolFallback(unittest.TestCase): + """When the pool itself cannot lease a client, the job still gets one.""" + + def setUp(self): + """Start from a clean pool, with every one-shot SSHClient stubbed and recorded.""" + reset_default_pool() + self.addCleanup(reset_default_pool) + self.opened = list() + + def _factory(server): + client = _SSHClientStub(server) + self.opened.append(client) + return client + client_patch = patch('arc.job.ssh_pool.SSHClient', side_effect=_factory) + client_patch.start() + self.addCleanup(client_patch.stop) + + def _borrow(self, adapter): + """Return the client the adapter's SSH context manager yields.""" + with adapter._open_or_borrow_ssh() as client: + return client + + def test_a_factory_failure_falls_back_to_a_one_shot_client(self): + """The pool builds its client on the first borrow, which is where a dead server shows.""" + def _factory(server): + raise ServerError(f'Could not connect to server {server}') + set_default_pool(SSHConnectionPool(factory=_factory)) + client = self._borrow(_MinimalAdapter(server='server2', execution_type='queue')) + self.assertEqual(len(self.opened), 1) + self.assertIs(client, self.opened[0]) + self.assertEqual(client.server, 'server2') + + def test_an_unusable_pool_falls_back_too(self): + """Any failure to lease is a failure to lease, whatever the pool object is.""" + class _PoolWithoutBorrow: + """A process-global pool object that cannot lease a client.""" + + def close_all(self): + """Tear down nothing, since nothing was ever leased.""" + set_default_pool(_PoolWithoutBorrow()) + client = self._borrow(_MinimalAdapter(server='server2', execution_type='queue')) + self.assertEqual(len(self.opened), 1) + self.assertIs(client, self.opened[0]) + + def test_a_leased_client_is_still_yielded(self): + """A working pool is used as it was, without opening anything private.""" + set_default_pool(_StubFactoryPool()) + client = self._borrow(_MinimalAdapter(server='server2', execution_type='queue')) + self.assertIsInstance(client, _SSHClientStub) + self.assertEqual(client.server, 'server2') + self.assertEqual(self.opened, list()) + + @staticmethod + def _borrow_and_raise(adapter): + """Borrow a client for ``adapter`` and fail inside the with-block, as a job would.""" + with adapter._open_or_borrow_ssh(): + raise ValueError('the job, not the connection, went wrong') + + def test_an_error_raised_by_the_caller_is_not_treated_as_a_pool_failure(self): + """The fallback covers leasing only: the caller's own failure must reach the caller.""" + set_default_pool(_StubFactoryPool()) + adapter = _MinimalAdapter(server='server2', execution_type='queue') + self.assertRaises(ValueError, self._borrow_and_raise, adapter) + self.assertEqual(self.opened, list()) + + def test_the_pool_stays_usable_after_a_caller_raised(self): + """A borrow that ended in an exception must not leave the pool holding a lease.""" + pool = _StubFactoryPool() + set_default_pool(pool) + adapter = _MinimalAdapter(server='server2', execution_type='queue') + self.assertRaises(ValueError, self._borrow_and_raise, adapter) + with adapter._open_or_borrow_ssh() as client: + self.assertIsInstance(client, _SSHClientStub) + self.assertEqual(pool.opens, 1) + self.assertEqual(pool.borrows, 2) + + +class TestRemoteJobPaths(unittest.TestCase): + """set_file_paths() decides where a job's files live on the server.""" + + def setUp(self): + """Work in a temporary project directory, with no server warned about yet.""" + self.tmp_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp_dir, ignore_errors=True) + warned = patch('arc.job.adapter._SERVERS_WARNED_ABOUT_A_RELATIVE_REMOTE_PATH', set()) + warned.start() + self.addCleanup(warned.stop) + + def _remote_path(self, server_cfg, server='srv'): + """Return the remote path set_file_paths() builds for a job on ``server``.""" + adapter = _MinimalAdapter(server=server, execution_type='queue') + adapter.reactions = None + adapter.species = [ARCSpecies(label='spc', smiles='C')] + adapter.run_multi_species = False + adapter.project = 'proj' + adapter.project_directory = self.tmp_dir + adapter.species_label = 'spc' + adapter.job_name = 'job_1' + with patch('arc.job.adapter.servers', {server: server_cfg}): + adapter.set_file_paths() + return adapter.remote_path + + def test_a_configured_path_is_used_verbatim(self): + """Remote file systems are case-sensitive, so the configured path must not be lowercased.""" + remote_path = self._remote_path({'cluster_soft': 'PBS', 'un': 'MyUser', 'path': '/Home/Users'}) + self.assertEqual(remote_path, + os.path.join('/Home/Users', 'MyUser', 'runs', 'ARC_Projects', 'proj', + 'spc', 'job_1')) + self.assertTrue(os.path.isabs(remote_path)) + + def test_a_server_without_a_path_stays_relative_to_the_login_directory(self): + """That is where the remote shell and SFTP resolve it, and it is reported as such.""" + with self.assertLogs('arc', level='WARNING') as logged: + remote_path = self._remote_path({'cluster_soft': 'PBS', 'un': 'user'}) + self.assertEqual(remote_path, + os.path.join('runs', 'ARC_Projects', 'proj', 'spc', 'job_1')) + self.assertFalse(os.path.isabs(remote_path)) + self.assertEqual(len(logged.output), 1) + self.assertIn('srv', logged.output[0]) + self.assertIn('path', logged.output[0]) + + def test_the_report_is_made_once_per_server(self): + """One line per run, not one per job, or a large run buries its own log.""" + with self.assertLogs('arc', level='WARNING'): + self._remote_path({'cluster_soft': 'PBS', 'un': 'user'}) + with self.assertNoLogs('arc', level='WARNING'): + self._remote_path({'cluster_soft': 'PBS', 'un': 'user'}) + + def test_the_local_server_is_not_reported(self): + """The local server runs in the project directory and has no remote path to configure.""" + with self.assertNoLogs('arc', level='WARNING'): + self._remote_path({'cluster_soft': 'local', 'un': 'user'}, server='local') + + if __name__ == '__main__': unittest.main(testRunner=unittest.TextTestRunner(verbosity=2)) diff --git a/arc/job/ssh_pool.py b/arc/job/ssh_pool.py new file mode 100644 index 0000000000..5eab340fe6 --- /dev/null +++ b/arc/job/ssh_pool.py @@ -0,0 +1,294 @@ +"""Persistent per-server SSHClient pool for the lifetime of an ARC run. + +Without this, each remote-queue job opens its own TCP+auth handshake +for upload, then another for qsub. Option 1 (in :mod:`arc.job.adapter`) +collapsed those two into one (per-job sharing). This module is Option +2: extend the share across ALL jobs run during this Python process, +so 100 TS guess opts end up sharing one paramiko Transport instead of +opening 100 of them. The closest equivalent to OpenSSH's +``ControlMaster``, applied at the library level for paramiko. + +Concurrency: ARC's scheduler is single-threaded (verified — no +``Thread`` / ``asyncio`` / ``concurrent.futures`` imports across +``scheduler.py`` / ``main.py`` / ``adapter.py``), so the pool does no +locking. A future async/parallel scheduler would need per-server +locks; flagged in :meth:`SSHConnectionPool.borrow`. + +Lifecycle: the default process-global pool is opened lazily on first +borrow and closed via :func:`reset_default_pool`. +``arc.main.ARC.execute()`` calls that in a ``finally``, so pooled +connections close cleanly on a clean run, on ctrl-C and on a crash +alike, and for an in-process consumer as much as for ``ARC.py``; +tests call it in ``tearDown`` to start fresh. + +Entry point: :func:`borrow_ssh_client` is the single borrow path every +SSH caller in ARC goes through, so the scheduler's status poll, the +job adapters, server troubleshooting and the ESS survey all share one +connection per server rather than one each. +""" + +from contextlib import ExitStack, contextmanager +from collections.abc import Iterator +from typing import Callable + +from arc.common import get_logger +from arc.job.ssh import SSHClient + +logger = get_logger() + + +SSHClientFactory = Callable[[str], SSHClient] + +KEEPALIVE_INTERVAL_SECONDS = 30 +""" +How often the transport sends a keepalive, in seconds, as ``paramiko.Transport.set_keepalive`` +takes it. Long lived pooled connections are otherwise dropped by an idle timeout on the server +or on a firewall between it and the client. +""" + + +def _default_factory(server: str) -> SSHClient: + """ + Open and connect a real SSHClient, sending keepalives. Override for tests. + + Args: + server (str): The server name. + + Returns: SSHClient + A connected client. + """ + client = SSHClient(server) + client.connect() + set_keepalive(client, KEEPALIVE_INTERVAL_SECONDS) + return client + + +def set_keepalive(client: SSHClient, interval: int = KEEPALIVE_INTERVAL_SECONDS) -> bool: + """ + Ask a client's paramiko Transport to send a keepalive every ``interval`` seconds. + + A pooled connection is held for the lifetime of the run and sits idle between polls, which + is long enough for an SSH daemon's ``ClientAliveInterval`` or an intermediate firewall's + idle-connection timeout to drop it. Without a keepalive that drop is silent: the socket + stays half-open, ``Transport.is_active()`` keeps reporting ``True``, and the next borrow + hands out a handle whose first command hangs until TCP gives up. A periodic global request + both keeps the session in the middle boxes' tables and lets paramiko mark the transport dead + when the peer stops answering. + + The interval is also recorded on the client, which re-applies it to every transport it + opens afterwards: a keepalive belongs to a Transport, and ``check_connections`` replaces the + transport under a client it reconnects in place, which would otherwise leave a pooled client + without one for the rest of the run. + + Args: + client (SSHClient): The client whose transport should send keepalives. + interval (int, optional): Seconds between keepalives. + + Returns: bool + Whether a keepalive was set, ``False`` when the client has no live transport. + """ + underlying = getattr(client, '_ssh', None) + transport_getter = getattr(underlying, 'get_transport', None) + transport = transport_getter() if transport_getter is not None else None + if transport is None: + server_name = getattr(client, 'server', client) + logger.debug(f'ssh_pool: no transport to set a keepalive on for {server_name}') + return False + client._keepalive_interval = interval + transport.set_keepalive(interval) + return True + + +class SSHConnectionPool: + """ + Process-lifetime cache of SSHClient instances keyed by server name. + + One client per server, opened lazily on first borrow, kept alive + until :meth:`close_all` is called (or the process exits). Health + is re-checked on every operation by the existing + ``check_connections`` decorator on SSHClient methods, so a stale + Transport is silently re-established mid-run. + """ + + def __init__(self, factory: SSHClientFactory = _default_factory): + """ + Create an empty pool. + + The ``opens`` and ``borrows`` counters expose pool behaviour without a caller having to + peek at internals or hook the factory. + + Args: + factory (SSHClientFactory, optional): Builds a connected client for a server name. + """ + self._factory = factory + self._clients: dict[str, SSHClient] = {} + self.opens = 0 + self.borrows = 0 + + @contextmanager + def borrow(self, server: str) -> Iterator[SSHClient]: + """ + Lease the pool's SSHClient for ``server``. + + Exiting the context does not close the client, which the pool retains ownership of. The + borrowed client is transient by contract; do not stash it past the ``with`` block. + + Concurrent borrows of the same server are not safe today. ARC's scheduler is + single-threaded, so this has not bitten; a parallel scheduler would need a per-server + lock around the yield, or a stack of free clients rather than a single one. + + Args: + server (str): The server name. + + Yields: SSHClient + A connected client for ``server``. + """ + self.borrows += 1 + client = self._clients.get(server) + if client is None or not _is_alive(client): + if client is not None: + _close_quietly(client, f'reaping dead {server} SSHClient before reopen') + client = self._factory(server) + self._clients[server] = client + self.opens += 1 + logger.debug(f'ssh_pool: opened SSHClient for {server} (total opens={self.opens})') + else: + logger.debug(f'ssh_pool: reusing SSHClient for {server}') + yield client + + def close_all(self) -> None: + """ + Close every pooled client. Safe to call multiple times. + """ + for server, client in list(self._clients.items()): + _close_quietly(client, f'closing pooled {server} SSHClient') + self._clients.clear() + + +def _is_alive(client: SSHClient) -> bool: + """ + Cheap liveness check: does the paramiko Transport report active? + + This does not roundtrip to the server, which the SSHClient method's own + ``check_connections`` decorator does on the next call. It is just enough to skip the obvious + case of a connection reset between jobs, so that a known-dead handle is not handed out. + + Args: + client (SSHClient): The client to check. + + Returns: bool + Whether the client has a transport that reports itself active. + """ + underlying = getattr(client, '_ssh', None) + if underlying is None: + return False + transport_getter = getattr(underlying, 'get_transport', None) + if transport_getter is None: + return False + transport = transport_getter() + return bool(transport and transport.is_active()) + + +def _close_quietly(client: SSHClient, context: str) -> None: + """ + Close a client, reporting rather than propagating a failure to do so. + + Pool teardown must not propagate a close error, since ARC's main path is already past the + work that needed the connection, so a failure is reported at the debug level instead. + + Args: + client (SSHClient): The client to close. + context (str): What was being done, named in the report. + """ + try: + client.close() + except Exception: + logger.debug(f'ssh_pool: close errored {context}', exc_info=True) + + +_default_pool: SSHConnectionPool | None = None +""" +The process-global pool, instantiated lazily by :func:`get_default_pool` and discarded between +ARC runs, and between tests, by :func:`reset_default_pool`. +""" + + +def get_default_pool() -> SSHConnectionPool: + """ + Return the process-global pool, creating it on first call. + + Returns: SSHConnectionPool + The process-global pool. + """ + global _default_pool + if _default_pool is None: + _default_pool = SSHConnectionPool() + return _default_pool + + +def set_default_pool(pool: SSHConnectionPool | None) -> None: + """ + Replace the process-global pool. + + Mainly for tests that inject a stub-factory pool without monkeypatching the module. + + Args: + pool (SSHConnectionPool | None): The pool to install, or ``None`` to clear it. + """ + global _default_pool + _default_pool = pool + + +def reset_default_pool() -> None: + """ + Close and discard the default pool. Idempotent. + """ + global _default_pool + if _default_pool is not None: + _default_pool.close_all() + _default_pool = None + + +@contextmanager +def borrow_ssh_client(server: str) -> Iterator[SSHClient]: + """ + Lease an :class:`SSHClient` for ``server`` from the process-global pool. + + This is the single borrow path every SSH caller in ARC goes through, so that all of them + share one connection per server for the lifetime of the run. Leasing from the pool is tried + first; if the lease itself fails -- the default pool was replaced by an object that cannot + lease, or the factory could not connect -- a one-shot client is opened and closed inline + instead, so a caller never loses its connection because the pool is unavailable. + + The fallback covers leasing only. Once a client has been yielded, an exception raised by the + body of the ``with`` block propagates to the caller untouched. + + Exiting the context does not close a pooled client, which the pool keeps and hands to the + next borrower; a one-shot client opened by the fallback is closed on exit. + + Args: + server (str): The server name, as a key of the ``servers`` settings dictionary. + + Yields: SSHClient + A connected client for ``server``. + """ + with ExitStack() as stack: + try: + client = stack.enter_context(get_default_pool().borrow(server)) + except Exception as e: + logger.debug(f'Could not lease an SSHClient for {server} from the pool, opening a ' + f'one-shot client instead. Got {type(e).__name__}: {e}', exc_info=True) + client = stack.enter_context(SSHClient(server)) + yield client + + +__all__ = [ + 'KEEPALIVE_INTERVAL_SECONDS', + 'SSHClientFactory', + 'SSHConnectionPool', + 'borrow_ssh_client', + 'get_default_pool', + 'reset_default_pool', + 'set_default_pool', + 'set_keepalive', +] diff --git a/arc/job/ssh_pool_test.py b/arc/job/ssh_pool_test.py new file mode 100644 index 0000000000..9243379bfb --- /dev/null +++ b/arc/job/ssh_pool_test.py @@ -0,0 +1,328 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +""" +This module contains unit tests for the arc.job.ssh_pool module. + +These exercise the pool directly, without routing through a JobAdapter. +The adapter-driven integration tests (one client shared across many jobs, +status polling and downloads reusing the pooled client) live alongside the +adapter in arc/job/adapter_test.py. +""" + +import unittest +from unittest.mock import patch + +from arc.job.ssh_pool import ( + KEEPALIVE_INTERVAL_SECONDS, + SSHConnectionPool, + borrow_ssh_client, + get_default_pool, + reset_default_pool, + set_default_pool, + set_keepalive, + _default_factory, +) + + +class _FakeTransport(object): + """Stands in for a paramiko Transport.""" + + def __init__(self, active=True): + self._active = active + self.keepalive_interval = None + + def is_active(self): + return self._active + + def set_keepalive(self, interval): + """Record the keepalive interval paramiko was asked for.""" + self.keepalive_interval = interval + + +class _FakeParamikoSSH(object): + """Stands in for paramiko.SSHClient as held by SSHClient._ssh.""" + + def __init__(self, active=True): + self._transport = _FakeTransport(active=active) + + def get_transport(self): + return self._transport + + +class _SSHClientStub(object): + """Minimal stand-in for arc.job.ssh.SSHClient. Records connect() and close() calls.""" + + def __init__(self, server, active=True): + self.server = server + self._ssh = _FakeParamikoSSH(active=active) + self._closed = False + self.connects = 0 + + def connect(self): + """Count the connection the real client would open.""" + self.connects += 1 + + def close(self): + self._closed = True + + def __enter__(self): + self.connect() + return self + + def __exit__(self, *args): + self.close() + return False + + +class TestSSHConnectionPool(unittest.TestCase): + """Test SSHConnectionPool.borrow() and close_all().""" + + def setUp(self): + self.opened = [] + + def factory(server): + client = _SSHClientStub(server) + self.opened.append(client) + return client + + self.pool = SSHConnectionPool(factory=factory) + + def test_borrow_opens_once_per_server(self): + """Repeated borrows of one server reuse a single client.""" + for _ in range(10): + with self.pool.borrow('server2') as client: + self.assertIsInstance(client, _SSHClientStub) + self.assertEqual(self.pool.opens, 1) + self.assertEqual(self.pool.borrows, 10) + self.assertEqual(len(self.opened), 1) + + def test_borrow_yields_the_same_object(self): + """The identical client instance comes back on each borrow.""" + with self.pool.borrow('server2') as first: + pass + with self.pool.borrow('server2') as second: + pass + self.assertIs(first, second) + + def test_distinct_servers_get_distinct_clients(self): + with self.pool.borrow('server2'): + pass + with self.pool.borrow('server3'): + pass + self.assertEqual(self.pool.opens, 2) + self.assertEqual(sorted(self.pool._clients.keys()), ['server2', 'server3']) + + def test_borrow_does_not_close_on_exit(self): + """The pool retains ownership; leaving the context must not close.""" + with self.pool.borrow('server2') as client: + pass + self.assertFalse(client._closed) + + def test_dead_client_is_reaped_and_reopened(self): + """An inactive Transport causes the client to be closed and replaced.""" + with self.pool.borrow('server2') as first: + pass + first._ssh = None # simulate a dropped connection + with self.pool.borrow('server2') as second: + pass + self.assertTrue(first._closed, 'the dead client should be closed before reopening') + self.assertIsNot(first, second) + self.assertEqual(self.pool.opens, 2) + + def test_inactive_transport_is_treated_as_dead(self): + """A Transport reporting is_active() False is not handed out.""" + with self.pool.borrow('server2') as first: + pass + first._ssh = _FakeParamikoSSH(active=False) + with self.pool.borrow('server2') as second: + pass + self.assertIsNot(first, second) + self.assertEqual(self.pool.opens, 2) + + def test_exception_in_body_leaves_pool_usable(self): + """A raising with-body must propagate and not corrupt pool state.""" + def borrow_and_raise(): + with self.pool.borrow('server2'): + raise ValueError('boom') + + self.assertRaises(ValueError, borrow_and_raise) + with self.pool.borrow('server2') as client: + self.assertIsInstance(client, _SSHClientStub) + self.assertEqual(self.pool.opens, 1, 'the client should have been reused, not reopened') + + def test_close_all_closes_and_empties(self): + with self.pool.borrow('server2'): + pass + with self.pool.borrow('server3'): + pass + clients = list(self.pool._clients.values()) + self.pool.close_all() + self.assertEqual(self.pool._clients, {}) + for client in clients: + self.assertTrue(client._closed) + + def test_close_all_is_idempotent(self): + with self.pool.borrow('server2'): + pass + self.pool.close_all() + self.pool.close_all() + self.assertEqual(self.pool._clients, {}) + + +class TestSSHPoolDefaultLifecycle(unittest.TestCase): + """The module-level default pool is lazy and resettable.""" + + def setUp(self): + reset_default_pool() + + def tearDown(self): + reset_default_pool() + + def test_get_default_pool_is_idempotent(self): + p1 = get_default_pool() + p2 = get_default_pool() + self.assertIs(p1, p2) + + def test_reset_default_pool_drops_the_instance(self): + p1 = get_default_pool() + reset_default_pool() + p2 = get_default_pool() + self.assertIsNot(p1, p2) + + def test_set_default_pool_replaces_instance(self): + replacement = SSHConnectionPool(factory=_SSHClientStub) + set_default_pool(replacement) + self.assertIs(get_default_pool(), replacement) + + def test_reset_default_pool_closes_pooled_clients(self): + """Resetting must release connections, not just drop the reference.""" + pool = SSHConnectionPool(factory=_SSHClientStub) + set_default_pool(pool) + with pool.borrow('server2') as client: + pass + reset_default_pool() + self.assertTrue(client._closed) + + +class TestSetKeepalive(unittest.TestCase): + """A pooled connection sits idle between polls, so it must send keepalives.""" + + def test_the_transport_is_asked_to_send_keepalives(self): + """Without this, an idle pooled connection is dropped by the server or a firewall.""" + client = _SSHClientStub('server2') + self.assertTrue(set_keepalive(client)) + self.assertEqual(client._ssh.get_transport().keepalive_interval, KEEPALIVE_INTERVAL_SECONDS) + + def test_the_interval_is_the_one_asked_for(self): + client = _SSHClientStub('server2') + set_keepalive(client, 7) + self.assertEqual(client._ssh.get_transport().keepalive_interval, 7) + + def test_a_client_without_a_transport_does_not_raise(self): + """A client that never connected has nothing to keep alive, and that is not an error.""" + client = _SSHClientStub('server2') + client._ssh = None + self.assertFalse(set_keepalive(client)) + + def test_the_interval_is_recorded_on_the_client(self): + """ + A keepalive is a property of a paramiko Transport, so it is lost when a reconnect opens a + new one. Recording it on the client is what lets SSHClient.connect() re-apply it, which + is the case a pooled connection held for a whole run actually meets. + """ + client = _SSHClientStub('server2') + set_keepalive(client, 11) + self.assertEqual(client._keepalive_interval, 11) + + def test_the_default_factory_sets_a_keepalive(self): + """The factory is where a pooled connection is born, so it is where this must happen.""" + client = _SSHClientStub('server2') + with patch('arc.job.ssh_pool.SSHClient', return_value=client): + built = _default_factory('server2') + self.assertIs(built, client) + self.assertEqual(client._ssh.get_transport().keepalive_interval, KEEPALIVE_INTERVAL_SECONDS) + + +class TestBorrowSSHClientFallback(unittest.TestCase): + """The shared borrow path every SSH caller in ARC goes through.""" + + def setUp(self): + reset_default_pool() + self.addCleanup(reset_default_pool) + self.one_shot = list() + + def _one_shot(server): + client = _SSHClientStub(server) + self.one_shot.append(client) + return client + patcher = patch('arc.job.ssh_pool.SSHClient', side_effect=_one_shot) + patcher.start() + self.addCleanup(patcher.stop) + + def test_a_pooled_client_is_yielded(self): + """The whole point: the caller gets the pool's client, not a new connection.""" + set_default_pool(SSHConnectionPool(factory=_SSHClientStub)) + with borrow_ssh_client('server2') as client: + self.assertIsInstance(client, _SSHClientStub) + self.assertEqual(client.server, 'server2') + self.assertEqual(self.one_shot, list()) + + def test_the_pooled_client_is_reused_across_borrows(self): + set_default_pool(SSHConnectionPool(factory=_SSHClientStub)) + with borrow_ssh_client('server2') as first: + pass + with borrow_ssh_client('server2') as second: + pass + self.assertIs(first, second) + self.assertEqual(get_default_pool().opens, 1) + + def test_a_pooled_client_is_not_closed_on_exit(self): + set_default_pool(SSHConnectionPool(factory=_SSHClientStub)) + with borrow_ssh_client('server2') as client: + pass + self.assertFalse(client._closed) + + def test_a_factory_failure_falls_back_to_a_one_shot_client(self): + """A caller must not lose its connection because the pool could not open one.""" + def _factory(server): + raise RuntimeError(f'no route to {server}') + set_default_pool(SSHConnectionPool(factory=_factory)) + with borrow_ssh_client('server2') as client: + self.assertIs(client, self.one_shot[0]) + self.assertEqual(len(self.one_shot), 1) + + def test_a_pool_that_cannot_lease_falls_back_too(self): + """Any failure to lease is a failure to lease, whatever the pool object is.""" + class _PoolWithoutBorrow: + """A process-global pool object that cannot lease a client.""" + + def close_all(self): + """Tear down nothing, since nothing was ever leased.""" + set_default_pool(_PoolWithoutBorrow()) + with borrow_ssh_client('server2') as client: + self.assertIs(client, self.one_shot[0]) + + def test_a_one_shot_client_is_closed_on_exit(self): + """The fallback owns what it opened, so it must not leak it.""" + def _factory(server): + raise RuntimeError(f'no route to {server}') + set_default_pool(SSHConnectionPool(factory=_factory)) + with borrow_ssh_client('server2') as client: + pass + self.assertTrue(client._closed) + + def test_an_error_raised_by_the_caller_is_not_treated_as_a_lease_failure(self): + """The fallback covers leasing only: the caller's own failure must reach the caller.""" + set_default_pool(SSHConnectionPool(factory=_SSHClientStub)) + + def borrow_and_raise(): + with borrow_ssh_client('server2'): + raise ValueError('the job, not the connection, went wrong') + + self.assertRaises(ValueError, borrow_and_raise) + self.assertEqual(self.one_shot, list()) + + +if __name__ == '__main__': + unittest.main(testRunner=unittest.TextTestRunner(verbosity=2)) From 45e4a3cdbfb8b406fc4ef0d9413f6aaa9b6a5d70 Mon Sep 17 00:00:00 2001 From: Calvin Pieters Date: Sat, 22 Aug 2026 13:08:58 +0300 Subject: [PATCH 2/9] ssh and remote server settings: optional key, host key verification, scoped retry, and an absolute remote path Makes the SSH key optional so that an ssh-agent or the default key paths can authenticate, verifies host keys rather than adding them silently, and scopes the connect retry to the failures retrying can actually resolve. Classify a key file that does not exist as a permanent failure. paramiko guards only _key_from_filepath's SSHException, so an absent key_filename surfaces as FileNotFoundError, which is an OSError and so was not in PERMANENT_CONNECTION_ERRORS. At the default 1440 connection attempts that stalled the run for 24 hours over a path that will never appear, and the shipped server examples all carried the 'key': 'path_to_rsa_key' placeholder that triggers it. It is told apart from a transient network OSError by being a FileNotFoundError naming the configured key path, so a refused or reset connection is still retried. Re-apply a recorded keepalive interval to every transport connect() opens, so that a pooled client reconnected in place by check_connections does not lose idle-drop protection for the rest of the run. Report a server with no absolute remote path before any job is spawned. An adapter whose input file names a path on the server cannot run on a server that has no 'path' entry: its remote directories are then relative to the SSH login directory, which the deck cannot name, so the job cannot be built at all. orca_neb is such an adapter and is in the default ts_adapters, so this was reached only when the first reaction got to its TS search, part way into a run. check_ess_settings() already validates the adapter and server names it is given and runs before any calculation is spawned, so it now also takes the run's ts_adapters and checks the servers each path-naming adapter will run on, naming the server and the setting to fix. The check is keyed on the ESS settings entry an adapter resolves its server from rather than on the adapter's own name, because OrcaNEBAdapter is given its server while it is still an OrcaAdapter and therefore runs wherever orca runs. The shipped remote server examples gain the absolute 'path' entry they omitted, which is what made that the default configuration rather than an unusual one. --- arc/common.py | 56 ++- arc/common_test.py | 72 +++ arc/job/ssh.py | 623 ++++++++++++++++++++++-- arc/job/ssh_test.py | 1000 ++++++++++++++++++++++++++++++++++++-- arc/main.py | 30 +- arc/main_test.py | 115 ++++- arc/settings/settings.py | 10 +- 7 files changed, 1805 insertions(+), 101 deletions(-) diff --git a/arc/common.py b/arc/common.py index f406bd511b..1db45bf329 100644 --- a/arc/common.py +++ b/arc/common.py @@ -56,7 +56,6 @@ default_job_types, servers, supported_ess = settings['default_job_types'], settings['servers'], settings['supported_ess'] - def initialize_job_types(job_types: dict | None = None, specific_job_type: str = '', ) -> dict: @@ -121,7 +120,9 @@ def initialize_job_types(job_types: dict | None = None, return job_types -def check_ess_settings(ess_settings: dict | None = None) -> dict: +def check_ess_settings(ess_settings: dict | None = None, + ts_adapters: list[str] | None = None, + ) -> dict: """ A helper function to convert servers in the ess_settings dict to lists Assists in troubleshooting job and trying a different server @@ -129,9 +130,15 @@ def check_ess_settings(ess_settings: dict | None = None) -> dict: Args: ess_settings (dict, optional): ARC's ESS settings dictionary. + ts_adapters (list, optional): The TS search adapters this run will use, used to check the + remote paths of the servers they will run on. ``None`` + selects the default set from the settings. Returns: dict An updated ARC ESS dictionary. + + Raises: + SettingsError: If an ESS, a server, or the remote path of a server is unusable. """ if ess_settings is None or not ess_settings: return dict() @@ -155,10 +162,55 @@ def check_ess_settings(ess_settings: dict | None = None) -> dict: if not isinstance(server, bool) and server.lower() not in [s.lower() for s in servers.keys()]: server_names = [name for name in servers.keys()] raise SettingsError(f'Recognized servers are {server_names}. Got: {server}') + check_remote_paths_of_path_naming_adapters(ess_settings=settings_dict, ts_adapters=ts_adapters) logger.info(f'\nUsing the following ESS settings:\n{pprint.pformat(settings_dict)}\n') return settings_dict +def check_remote_paths_of_path_naming_adapters(ess_settings: dict, + ts_adapters: list[str] | None = None, + ) -> None: + """ + Check that every server an adapter which names a path on the server will run on has an + absolute ``path``. + + Reported here, before any calculation is spawned, because the alternative is reaching it when + the first reaction gets to its TS search: the adapter cannot build a usable input file for + such a server, so the failure arrives once per run either way, and arriving at startup is the + difference between a settings error the reader can act on and a run that has already spent + time on jobs it will not be able to use. + + A server that is not in the ``servers`` settings is not reported here, since + :func:`check_ess_settings` has already rejected it by name. + + Args: + ess_settings (dict): ARC's ESS settings dictionary, each ESS mapped to a list of servers. + ts_adapters (list, optional): The TS search adapters this run will use. ``None`` selects + the default set from the settings. + + Raises: + SettingsError: If such an adapter would run on a server whose ``path`` is missing or not + absolute. + """ + adapters = settings.get('ts_adapters', list()) if ts_adapters is None else ts_adapters + adapters = [adapter.lower() for adapter in adapters if isinstance(adapter, str)] + if 'orca_neb' not in adapters: + return + for server in ess_settings.get('orca', list()): + if not isinstance(server, str) or server.lower() == 'local': + continue + server_key = next((key for key in servers.keys() if key.lower() == server.lower()), None) + if server_key is None: + continue + path = servers[server_key].get('path') + if not path or not os.path.isabs(path): + raise SettingsError(f'Server "{server}" has no absolute "path" entry in the settings, ' + f'got {path!r}, but the "orca_neb" adapter will run on it and its ' + f'input file must name a path on the server. Set "path" for this ' + f'server to the absolute directory holding the user directories, ' + f'e.g. "/home", or remove "orca_neb" from "ts_adapters".') + + def initialize_log(log_file: str, project: str, project_directory: str | None = None, diff --git a/arc/common_test.py b/arc/common_test.py index b4b6832faa..5f3b6c9c6b 100644 --- a/arc/common_test.py +++ b/arc/common_test.py @@ -1559,5 +1559,77 @@ def test_repeated_initialize_log_does_not_re_log_a_drained_warning(self): self.assertEqual(self._read_log().count('only flushed once'), 1) +class TestCheckRemotePathsOfPathNamingAdapters(unittest.TestCase): + """ + An adapter whose input file names a path on the server cannot run on a server that has no + absolute ``path``, and the shipped remote server examples had no ``path`` at all, so this is + reported at startup rather than when the first TS search reaches it. + """ + + def _use_servers(self, servers_dict): + """Point arc.common at ``servers_dict`` for the duration of one test.""" + original = common.servers + common.servers = servers_dict + self.addCleanup(setattr, common, 'servers', original) + + def test_a_server_without_a_path_is_reported(self): + """The ordinary remote configuration, since no shipped server example defined a path.""" + self._use_servers({'remote': {'cluster_soft': 'PBS', 'address': 'host.edu', 'un': 'user'}}) + with self.assertRaises(SettingsError) as raised: + common.check_remote_paths_of_path_naming_adapters(ess_settings={'orca': ['remote']}, + ts_adapters=['heuristics', 'orca_neb']) + self.assertIn('remote', str(raised.exception)) + self.assertIn('path', str(raised.exception)) + + def test_an_absolute_path_is_accepted(self): + """A correctly configured server must not be reported.""" + self._use_servers({'remote': {'cluster_soft': 'PBS', 'address': 'host.edu', 'un': 'user', + 'path': '/home'}}) + common.check_remote_paths_of_path_naming_adapters(ess_settings={'orca': ['remote']}, + ts_adapters=['orca_neb']) + + def test_a_relative_path_is_reported(self): + """A relative path is what the remote directories are rooted at, and Orca cannot follow it.""" + self._use_servers({'remote': {'cluster_soft': 'PBS', 'address': 'host.edu', 'un': 'user', + 'path': 'runs'}}) + with self.assertRaises(SettingsError): + common.check_remote_paths_of_path_naming_adapters(ess_settings={'orca': ['remote']}, + ts_adapters=['orca_neb']) + + def test_an_adapter_that_is_not_used_is_not_checked(self): + """A server that no path-naming adapter runs on must not block a run that never uses one.""" + self._use_servers({'remote': {'cluster_soft': 'PBS', 'address': 'host.edu', 'un': 'user'}}) + common.check_remote_paths_of_path_naming_adapters(ess_settings={'orca': ['remote']}, + ts_adapters=['heuristics', 'autotst']) + + def test_the_local_server_is_not_checked(self): + """A local job reads its files from the directory ARC wrote them to, with no remote path.""" + self._use_servers({'local': {'cluster_soft': 'local', 'un': 'user'}}) + common.check_remote_paths_of_path_naming_adapters(ess_settings={'orca': ['local']}, + ts_adapters=['orca_neb']) + + def test_the_adapter_is_checked_on_the_ess_key_it_resolves_its_server_from(self): + """ + OrcaNEBAdapter is given its server while it is still an OrcaAdapter, so it runs wherever + orca runs. Keying the check on "orca_neb" instead would never match a real configuration. + """ + self._use_servers({'remote': {'cluster_soft': 'PBS', 'address': 'host.edu', 'un': 'user'}}) + common.check_remote_paths_of_path_naming_adapters(ess_settings={'orca_neb': ['remote']}, + ts_adapters=['orca_neb']) + + def test_check_ess_settings_runs_the_check(self): + """The startup path must reach it, which is the whole point of validating there.""" + self._use_servers({'remote': {'cluster_soft': 'PBS', 'address': 'host.edu', 'un': 'user'}}) + with self.assertRaises(SettingsError): + common.check_ess_settings(ess_settings={'orca': ['remote']}, ts_adapters=['orca_neb']) + + def test_the_shipped_server_examples_define_an_absolute_path(self): + """The documented configuration must be a working one, not the one that fails.""" + for name in ['server1', 'server2', 'server3']: + path = settings['servers'][name].get('path') + self.assertIsNotNone(path, f'{name} has no "path" entry') + self.assertTrue(os.path.isabs(path), f'{name} has a relative path: {path!r}') + + if __name__ == '__main__': unittest.main(testRunner=unittest.TextTestRunner(verbosity=2)) diff --git a/arc/job/ssh.py b/arc/job/ssh.py index 3991f7c89f..39b0414b0a 100644 --- a/arc/job/ssh.py +++ b/arc/job/ssh.py @@ -6,11 +6,14 @@ * delete scratch files of a failed job: ssh nodeXX; rm scratch/dhdhdhd/job_number """ +import base64 import datetime +import hashlib import logging import os import shlex import time +from collections import Counter from typing import Any from collections.abc import Callable @@ -27,6 +30,116 @@ settings['check_status_command'], settings['delete_command'], settings['list_available_nodes_command'], \ settings['servers'], settings['submit_command'], settings['submit_filenames'] +KNOWN_HOSTS_PATH = '~/.ssh/known_hosts' +PLACEHOLDER_ADDRESS_SUFFIX = '.host.edu' +PLACEHOLDER_USERNAME = '' + + +class UnknownHostKeyError(ServerError, paramiko.SSHException): + """ + Raised when a server's host key is absent from ``known_hosts`` and the server sets + ``strict_host_key_checking``. + + An ARC :class:`~arc.exceptions.ServerError`, so ARC's server error handling covers it, and a + ``paramiko.SSHException``, which is what a missing host key policy is expected to raise. Being + a distinct type, a refused host key is told apart from a transport failure without matching on + paramiko's message text. + """ + + +def get_host_key_fingerprint(key: paramiko.PKey) -> str: + """ + Return the OpenSSH-style SHA256 fingerprint of a host key. + + Args: + key (paramiko.PKey): The host key to fingerprint. + + Returns: str + The fingerprint, formatted as ``SHA256:``, as reported by + ``ssh-keygen -lf`` and by OpenSSH when it prompts about an unknown host. + """ + digest = hashlib.sha256(key.asbytes()).digest() + return f'SHA256:{base64.b64encode(digest).decode().rstrip("=")}' + + +class LogAndAcceptHostKeyPolicy(paramiko.MissingHostKeyPolicy): + """ + A missing host key policy that reports the unknown key through ARC's logger, then accepts it. + + paramiko's ``WarningPolicy`` emits through ``warnings.warn()``, which ARC's + :func:`arc.common.initialize_log` filters out for the ``paramiko`` module, so nothing of it + reaches the log file or the terminal. This policy logs the host and the key's fingerprint at + the warning level, and connects. + """ + + def missing_host_key(self, + client: paramiko.SSHClient, + hostname: str, + key: paramiko.PKey, + ) -> None: + """ + Log the unknown host key and accept it. + + Args: + client (paramiko.SSHClient): The client the key was presented to. + hostname (str): The address of the server that presented the key. + key (paramiko.PKey): The host key that is not in ``known_hosts``. + """ + logger.warning(f'Connecting to {hostname} with an unverified host key: ' + f'{key.get_name()} {get_host_key_fingerprint(key)} is not in ' + f'{KNOWN_HOSTS_PATH}, so a first-ever connection cannot be told apart ' + f'from an interception. Verify the fingerprint against a trusted source ' + f'and add the key with:\n' + f' ssh-keyscan -H {hostname} >> {KNOWN_HOSTS_PATH}') + + +class RejectUnknownHostKeyPolicy(paramiko.RejectPolicy): + """ + A missing host key policy that refuses the connection, raising :class:`UnknownHostKeyError`. + """ + + def missing_host_key(self, + client: paramiko.SSHClient, + hostname: str, + key: paramiko.PKey, + ) -> None: + """ + Refuse the unknown host key. + + Args: + client (paramiko.SSHClient): The client the key was presented to. + hostname (str): The address of the server that presented the key. + key (paramiko.PKey): The host key that is not in ``known_hosts``. + + Raises: + UnknownHostKeyError: Always. + """ + raise UnknownHostKeyError(f'The host key of {hostname} ' + f'({key.get_name()} {get_host_key_fingerprint(key)}) is not in ' + f'{KNOWN_HOSTS_PATH}, and this server sets ' + f'strict_host_key_checking. Verify the fingerprint against a ' + f'trusted source and add the key with:\n' + f' ssh-keyscan -H {hostname} >> {KNOWN_HOSTS_PATH}') + + +class HostKeyMismatchError(ServerError, paramiko.SSHException): + """ + Raised when the host key a server presents contradicts the one stored in ``known_hosts``. + + Told apart from :class:`UnknownHostKeyError`, which is the absence of a stored key, because + the two mean different things: an absent key is a host never connected to before, while a + contradicted key is either a re-keyed server or an interception, and only the second of + those is a security event. Being an ARC :class:`~arc.exceptions.ServerError` keeps it inside + ARC's server error handling, and being a ``paramiko.SSHException`` keeps it catchable + alongside the exception it is raised from. + """ + + +PERMANENT_CONNECTION_ERRORS = (paramiko.AuthenticationException, + paramiko.BadHostKeyException, + UnknownHostKeyError, + ) + def check_connections(function: Callable[..., Any]) -> Callable[..., Any]: """ @@ -36,11 +149,16 @@ def check_connections(function: Callable[..., Any]) -> Callable[..., Any]: to make sure your connection still alive. If connection is bad, this decorator will reconnect the SSH channel, to avoid connection related error when executing the method. + + ``connect()`` assigns ``self._sftp`` and ``self._ssh`` itself and returns nothing, so its + result is not unpacked into them. Unpacking it raised ``TypeError: cannot unpack + non-iterable NoneType object`` for any client that had not connected yet, which is the one + case this branch exists to serve. """ def decorator(*args, **kwargs) -> Any: self = args[0] if self._ssh is None: # not sure if some status may cause False - self._sftp, self._ssh = self.connect() + self.connect() # test connection, reference: # https://stackoverflow.com/questions/ # 20147902/how-to-know-if-a-paramiko-ssh-channel-is-disconnected @@ -64,19 +182,30 @@ class SSHClient(object): waiting a minute between attempts. The default keeps trying for 24 hours, which is appropriate while jobs are running. Pass a low number where blocking is worse than giving up. + A permanent failure raises on the first attempt whatever + this is set to, see :meth:`connect`. Attributes: server (str): The server name as specified in ARCs's settings file under ``servers`` as a key. address (str): The server's address. un (str): The username to use on the server. - key (str): A path to a file containing the RSA SSH private key to the server. + key (str | None): A path to a file containing the SSH private key to the server. + Optional: when it is not set (or set to an empty string), no explicit + identity is offered and paramiko falls back to a running ssh-agent and + then to the default key paths (``~/.ssh/id_rsa``, ``~/.ssh/id_ecdsa``, + ``~/.ssh/id_ed25519``), which is how agent-based setups authenticate. connection_attempts (int): The number of times to try connecting to the server. _ssh (paramiko.SSHClient): A high-level representation of a session with an SSH server. _sftp (paramiko.sftp_client.SFTPClient): SFTP client used to perform remote file operations. + _keepalive_interval (int | None): The keepalive interval this client was asked for, which + :meth:`connect` re-applies to every transport it opens. + ``None`` until :func:`arc.job.ssh_pool.set_keepalive` + sets it, since a client that is not pooled is not held + open long enough to be dropped while idle. """ def __init__(self, server: str = '', - connection_attempts: int = 1440, # Continue trying for 24 hrs (24 hr * 60 min/hr). + connection_attempts: int = 1440, ) -> None: if server == '': raise ValueError('A server name must be specified') @@ -85,10 +214,11 @@ def __init__(self, self.server = server self.address = servers[server]['address'] self.un = servers[server]['un'] - self.key = servers[server]['key'] + self.key = servers[server].get('key') or None self.connection_attempts = connection_attempts self._sftp = None self._ssh = None + self._keepalive_interval = None logging.getLogger("paramiko").setLevel(logging.WARNING) def __enter__(self) -> SSHClient: @@ -122,7 +252,7 @@ def _send_command_to_server(self, # and even yield different behaviors. # Make sure to change directory back after the command is executed if self._check_dir_exists(remote_path): - command = f'cd "{remote_path}"; {command}; cd ' + command = f'cd -- {shlex.quote(remote_path)}; {command}; cd ' else: raise InputError( f'Cannot execute command at given remote_path({remote_path})') @@ -139,6 +269,7 @@ def _send_command_to_server(self, stderr = stderr.readlines() return stdout, stderr + @check_connections def upload_file(self, remote_file_path: str, local_file_path: str = '', @@ -179,6 +310,7 @@ def upload_file(self, logger.debug(f'Could not upload file {local_file_path} to {self.server}!') raise ServerError(f'Could not write file {remote_file_path} on {self.server}. ') + @check_connections def download_file(self, remote_file_path: str, local_file_path: str, @@ -186,6 +318,16 @@ def download_file(self, """ Download a file from the server. + The existence of the remote file is checked up to three times, one second apart, since + scheduler epilogues may flush a job's stdout and stderr to the work directory a second or + two after the job has left the queue. + + If the remote file does not exist, the local path is emptied instead of being downloaded + to: it is created if it is absent, and truncated if a file is already there. A file left + at that path by an earlier job therefore never survives to be read as this job's output + (see :meth:`arc.job.adapter.JobAdapter._get_additional_job_info`), and the miss is + reported at the warning level. + Args: remote_file_path (str): The remote path to be downloaded from. local_file_path (str): The local path to be downloaded to. @@ -193,13 +335,16 @@ def download_file(self, Raises: ServerError: If the file cannot be downloaded with maximum times to try """ - if not self._check_file_exists(remote_file_path): - # Check if a file exists - # This doesn't have a real impact now to avoid screwing up ESS trsh - # but introduce an opportunity for better troubleshooting. - # The current behavior is that if the remote path does not exist - # an empty file will be created at the local path - logger.debug(f'{remote_file_path} does not exist on {self.server}.') + for attempt in range(3): + if self._check_file_exists(remote_file_path): + break + if attempt < 2: + time.sleep(1.0) + else: + logger.warning(f'{remote_file_path} does not exist on {self.server}. ' + f'Emptied {local_file_path} instead of downloading it.') + self._empty_local_file(local_file_path) + return try: self._sftp.get(remotepath=remote_file_path, localpath=local_file_path) @@ -207,6 +352,21 @@ def download_file(self, logger.warning(f'Got an IOError when trying to download file ' f'{remote_file_path} from {self.server}') + @staticmethod + def _empty_local_file(local_file_path: str) -> None: + """ + Create ``local_file_path`` if it does not exist, and truncate it if it does. + + Args: + local_file_path (str): The local path to empty. + """ + try: + with open(local_file_path, 'wb'): + pass + except OSError as e: + logger.warning(f'Could not empty the local file {local_file_path}. ' + f'Got {type(e).__name__}: {e}') + @check_connections def read_remote_file(self, remote_file_path: str) -> list: """ @@ -289,7 +449,7 @@ def check_running_jobs_ids(self) -> list: cluster_soft = servers[self.server]['cluster_soft'].lower() for i, status_line in enumerate(stdout): if i > i_dict[cluster_soft]: - job_id = status_line.split(split_by_dict[cluster_soft])[0] + job_id = status_line.lstrip().split(split_by_dict[cluster_soft])[0] job_id = job_id.split('.')[0] if '.' in job_id else job_id running_job_ids.append(job_id) return running_job_ids @@ -321,19 +481,22 @@ def submit_job(self, remote_path: str, if 'Requested node configuration is not available' in line: logger.warning('User may be requesting more resources than are available. Please check server ' 'settings, such as cpus and memory, in ARC/arc/settings/settings.py') + if 'Memory specification can not be satisfied' in line: + logger.warning('User may be requesting more memory than is available. Please check server ' + 'settings, such as cpus and memory, in ARC/arc/settings/settings.py.') if cluster_soft.lower() == 'slurm' and 'AssocMaxSubmitJobLimit' in line: logger.warning(f'Max number of submitted jobs was reached, sleeping...') time.sleep(5 * 60) self.submit_job(remote_path=remote_path, recursion=True) if recursion: return None, None - elif cluster_soft.lower() in ['oge', 'sge'] and 'submitted' in stdout[0].lower(): + elif cluster_soft.lower() in ['oge', 'sge'] and stdout and 'submitted' in stdout[0].lower(): job_id = stdout[0].split()[2] - elif cluster_soft.lower() == 'slurm' and 'submitted' in stdout[0].lower(): + elif cluster_soft.lower() == 'slurm' and stdout and 'submitted' in stdout[0].lower(): job_id = stdout[0].split()[3] - elif cluster_soft.lower() == 'pbs': + elif cluster_soft.lower() == 'pbs' and stdout: job_id = stdout[0].split('.')[0] - elif cluster_soft.lower() == 'htcondor' and 'submitting' in stdout[0].lower(): + elif cluster_soft.lower() == 'htcondor' and stdout and 'submitting' in stdout[0].lower(): # Submitting job(s). # 1 job(s) submitted to cluster 443069. if len(stdout) and len(stdout[1].split()) and len(stdout[1].split()[-1].split('.')): @@ -347,8 +510,26 @@ def connect(self) -> None: """ A modulator function for _connect(). Connect to the server. + Failures that retrying cannot resolve -- a rejected authentication, a host key that does + not match ``known_hosts``, and an unknown host key on a server that sets + ``strict_host_key_checking`` (:data:`PERMANENT_CONNECTION_ERRORS`) -- raise a + ``ServerError`` on the first attempt, carrying the paramiko exception as its cause -- this + holds whatever ``connection_attempts`` is set to, since no number of retries can resolve + them. A configured ``key`` file that does not exist is permanent for the same reason, and + is told apart from the transport-level ``OSError``s by :meth:`_is_a_missing_key_file`. + Every other failure is transport-level, and is retried once a minute until + ``connection_attempts`` attempts have been made (24 hours by default). No interval is + waited out after the last attempt, so a client asked for a single attempt fails at once. + + A contradicted host key raises the :class:`HostKeyMismatchError` subclass of + ``ServerError`` rather than a plain one, and is reported at the error level naming both + fingerprints, so it is told apart from a wrong password in the log rather than reading as + one more failed connection. + Raises: - ServerError: Cannot connect to the server with maximum times to try + HostKeyMismatchError: The server's host key contradicts the one in ``known_hosts``. + ServerError: Cannot connect to the server with maximum times to try, + or the failure is permanent. """ times_tried = 0 interval = 60 # wait 60 sec between trials @@ -356,40 +537,189 @@ def connect(self) -> None: times_tried += 1 try: self._sftp, self._ssh = self._connect() + except paramiko.BadHostKeyException as e: + raise self._host_key_mismatch_error(e) from e + except PERMANENT_CONNECTION_ERRORS as e: + raise ServerError(f'Could not connect to server {self.server}, and retrying ' + f'cannot resolve it. Got {type(e).__name__}: {e}') from e + except OSError as e: + if self._is_a_missing_key_file(e): + raise ServerError(f'Could not connect to server {self.server}, and retrying ' + f'cannot resolve it: the SSH key file {self.key!r} does not ' + f'exist. Either correct the "key" entry of this server in the ' + f'settings, or remove it to authenticate via a running ' + f'ssh-agent or the default key paths.') from e + self._report_a_failed_connection_attempt(e, times_tried) except Exception as e: - if not times_tried % 10: - logger.info(f'Tried connecting to {self.server} {times_tried} times with no success...' - f'\nGot: {e}') - else: - print(f'Tried connecting to {self.server} {times_tried} times with no success...' - f'\nGot: {e}') + self._report_a_failed_connection_attempt(e, times_tried) else: + self._apply_keepalive() logger.debug(f'Successfully connected to {self.server} at the {times_tried} trial.') return if times_tried < self.connection_attempts: time.sleep(interval) raise ServerError(f'Could not connect to server {self.server} even after {times_tried} trials.') + def _apply_keepalive(self) -> bool: + """ + Re-apply the keepalive interval this client was asked for to its current transport. + + A keepalive is a property of a paramiko ``Transport``, not of the client that holds it, so + it is lost whenever a new transport is opened. :func:`check_connections` reconnects a + client in place when its socket has gone half-open, which replaces the transport under a + pooled client that outlives many such reconnects, so the interval is re-applied here for + every transport rather than only for the first. + + Returns: bool + Whether a keepalive was applied, ``False`` when none was asked for or there is no + live transport to apply it to. + """ + if self._keepalive_interval is None or self._ssh is None: + return False + transport = self._ssh.get_transport() + if transport is None: + return False + transport.set_keepalive(self._keepalive_interval) + return True + + def _report_a_failed_connection_attempt(self, error: Exception, times_tried: int) -> None: + """ + Report a connection attempt that failed and will be retried. + + Every tenth attempt goes to the log, and the ones in between are printed, so that a run + that spends hours retrying leaves a bounded trail in the log file while still showing + progress on the terminal. + + Args: + error (Exception): The failure to report. + times_tried (int): The number of attempts made so far, including this one. + """ + message = f'Tried connecting to {self.server} {times_tried} times with no success...' \ + f'\nGot: {error}' + if not times_tried % 10: + logger.info(message) + else: + print(message) + + def _is_a_missing_key_file(self, error: OSError) -> bool: + """ + Whether ``error`` reports that this client's configured ``key`` file does not exist. + + paramiko reads the identity it was asked to authenticate with outside the ``SSHException`` + it guards that read with, so an absent key file surfaces as a ``FileNotFoundError``. That + is an ``OSError``, as a refused or reset connection is, and the two must not be treated + alike: a missing file is permanent, while a network failure is exactly what the retry loop + exists for. They are told apart by the error being a ``FileNotFoundError`` that names the + configured key path. + + Args: + error (OSError): The error raised while connecting. + + Returns: bool + Whether the error is the absence of the configured key file. + """ + if self.key is None or not isinstance(error, FileNotFoundError): + return False + filename = getattr(error, 'filename', None) + if filename is None: + return True + return os.path.expanduser(str(filename)) == os.path.expanduser(self.key) + + def _host_key_mismatch_error(self, error: paramiko.BadHostKeyException) -> HostKeyMismatchError: + """ + Report a contradicted host key and build the error to raise for it. + + The report goes out at the error level and names the stored and the presented + fingerprints, since which of the two the reader recognises is what decides whether the + server was re-keyed or the session was intercepted. + + Args: + error (paramiko.BadHostKeyException): The mismatch paramiko raised. + + Returns: HostKeyMismatchError + The error to raise, carrying the same report as its message. + """ + path = os.path.expanduser(KNOWN_HOSTS_PATH) + message = f'The host key {self.address} presented does not match the key stored for it ' \ + f'in {KNOWN_HOSTS_PATH}, so server {self.server} was not connected to.\n' \ + f' stored: {error.expected_key.get_name()} ' \ + f'{get_host_key_fingerprint(error.expected_key)}\n' \ + f' presented: {error.key.get_name()} {get_host_key_fingerprint(error.key)}\n' \ + f'A re-keyed or rebuilt server and an intercepted session look exactly like ' \ + f'this, and the fingerprints are what tells them apart. Verify the presented ' \ + f'fingerprint against a trusted source. Only once it is confirmed to be the ' \ + f'server\'s own key, replace the stored one with:\n' \ + f' ssh-keygen -R {self.address} -f {path}\n' \ + f' ssh-keyscan -H {self.address} >> {KNOWN_HOSTS_PATH}' + logger.error(message) + return HostKeyMismatchError(message) + def _connect(self) -> tuple[paramiko.sftp_client.SFTPClient, paramiko.SSHClient]: """ Connect via paramiko, and open an SSH session as well as a SFTP session. + ``self.key`` is passed as paramiko's ``key_filename``, i.e. as the identity to + authenticate with. It may be ``None``, in which case paramiko looks for a running + ssh-agent and then for the default key paths; that is the only way an agent-forwarded + session can be used, and it also avoids paramiko raising on a configured key path that + does not exist on this machine. + + Note that ARC never parses ``~/.ssh/config``: paramiko only does so when an application + builds a ``paramiko.SSHConfig`` itself, and ARC does not. Directives such as + ``IdentityFile``, ``ProxyJump`` and ``ProxyCommand`` therefore have no effect here. + + Host key policy applies to an *unknown* key only. A key that is known and contradicted + is refused by paramiko whatever the policy, and :meth:`connect` turns that into a + :class:`HostKeyMismatchError` naming both fingerprints. + + An unknown host key means either a first-ever connection or a + machine-in-the-middle, and the two are indistinguishable from here. The default policy + is :class:`LogAndAcceptHostKeyPolicy`, which logs the unknown key through ARC's logger + and connects; the key is never added to ``known_hosts``, so the warning repeats on every + connection. Setting ``strict_host_key_checking: True`` on the server selects + :class:`RejectUnknownHostKeyPolicy` instead, which raises :class:`UnknownHostKeyError` + for any host that is not already in ``known_hosts``. + + Warning rather than rejecting is the default because ARC is a scheduler that runs + unattended for days. Rejecting an unknown host does not fail once: every job submission, + status poll and download for that server fails while the driver stays alive, so the run + keeps going and produces nothing, and the cause surfaces only when someone reads the log. + Recovering then means stopping ARC, running ``ssh-keyscan``, and restarting. To make the + default policy's residual risk visible before that cost is paid, + :func:`check_servers_known_hosts` reports every configured server that is absent from + ``known_hosts`` at startup, before any calculation is submitted. + + The timeout is enlarged from paramiko's 15 second default because a server may accept + the connection while its SSH daemon takes longer to answer, e.g. under network + congestion. + + The retry covers transport-level failures only, such as "SSHException: Error reading SSH + protocol banner[Error 104] Connection reset by peer". A bad key, a bad username and a + refused host key (:data:`PERMANENT_CONNECTION_ERRORS`) are raised without a second + attempt. A bare ``except`` here also swallowed KeyboardInterrupt/SystemExit, and + discarded the first exception so that a bad key or username surfaced as the retry's error + instead of its own. + Returns: tuple[paramiko.sftp_client.SFTPClient, paramiko.SSHClient] - An SFTP client used to perform remote file operations. - A high-level representation of a session with an SSH server. """ ssh = paramiko.SSHClient() - ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) - ssh.load_system_host_keys(filename=self.key) + ssh.load_system_host_keys() + if servers[self.server].get('strict_host_key_checking', False): + ssh.set_missing_host_key_policy(RejectUnknownHostKeyPolicy()) + else: + ssh.set_missing_host_key_policy(LogAndAcceptHostKeyPolicy()) try: - # If the server accepts the connection but the SSH daemon doesn't respond in - # 15 seconds (default in paramiko) due to network congestion, faulty switches, - # etc..., common solution is enlarging the timeout variable. - ssh.connect(hostname=self.address, username=self.un, banner_timeout=200) - except: - # This sometimes gives "SSHException: Error reading SSH protocol banner[Error 104] Connection reset by peer" - # Try again: - ssh.connect(hostname=self.address, username=self.un, banner_timeout=200) + ssh.connect(hostname=self.address, username=self.un, banner_timeout=200, key_filename=self.key) + except PERMANENT_CONNECTION_ERRORS: + raise + except (paramiko.SSHException, OSError) as e: + if isinstance(e, OSError) and self._is_a_missing_key_file(e): + raise + logger.debug(f'First SSH connection attempt to {self.server} failed with ' + f'{type(e).__name__}: {e}. Retrying once.') + ssh.connect(hostname=self.address, username=self.un, banner_timeout=200, key_filename=self.key) sftp = ssh.open_sftp() return sftp, ssh @@ -447,7 +777,7 @@ def find_package(self, package_name: str) -> list: Args: package_name (str): The name of the package to search for. """ - command = f'. ~/.bashrc; which {package_name}' + command = f'. ~/.bashrc; which {shlex.quote(package_name)}' return self._send_command_to_server(command)[0] def list_available_nodes(self) -> list: @@ -496,9 +826,30 @@ def change_mode(self, if os.path.isfile(remote_path): remote_path = os.path.dirname(remote_path) recursive = ' -R' if recursive else '' - command = f'chmod{recursive} {mode} {file_name}' + command = f'chmod{recursive} -- {mode} {shlex.quote(file_name)}' self._send_command_to_server(command, remote_path) + def remove_dir(self, remote_path: str) -> None: + """ + Remove a directory, and everything under it, on the server. + + This is the remote-cleanup primitive. ARC's own job flow does not call it: no job removes + its remote work directory today, and the caller that will is added separately. It is + reached through :meth:`arc.job.adapter.JobAdapter.remove_remote_files`, which supplies + the job's remote path. + + Args: + remote_path (str): The path to the directory to remove on the remote server. + + Raises: + ServerError: If the directory could not be removed. + """ + command = f'rm -rf -- {shlex.quote(remote_path)}' + _, stderr = self._send_command_to_server(command) + if stderr: + raise ServerError( + f'Cannot remove dir for the given path ({remote_path}).\nGot: {stderr}') + def delete_remote_check_files(self, remote_path: str) -> None: """ Delete ESS checkfiles under a remote directory (recursively). @@ -506,16 +857,16 @@ def delete_remote_check_files(self, remote_path: str) -> None: Pass ``True`` to the ``keep_checks`` flag in ARC to avoid deleting check files. The local counterpart of this method is ``arc.common.delete_check_files()``. + Unlike :meth:`remove_dir`, this keeps the remote directory and everything in it that is + not a checkfile, so a project's outputs remain on the server after the cleanup. + A failure is logged rather than raised, see :func:`delete_check_files_on_servers`. + Args: remote_path (str): The remote directory path under which checkfiles will be deleted. """ - if not remote_path: + if not remote_path or not self._check_dir_exists(remote_path): return - quoted_path = shlex.quote(remote_path) - # Both the existence test and the deletion are done in a single quoted command: - # a separate existence check would have to quote the path just as carefully, - # and a directory that isn't there is a no-op rather than an error worth reporting. - command = f'[ -d {quoted_path} ] && find {quoted_path} -type f -name "*.chk" -delete' + command = f'find {shlex.quote(remote_path)} -type f -name "*.chk" -delete' _, stderr = self._send_command_to_server(command) if stderr: logger.warning(f'Could not delete all check files under {remote_path} on {self.server}.\nGot: {stderr}') @@ -532,7 +883,7 @@ def _check_file_exists(self, Returs: bool: Whether the file exists on the remote server. ``True`` if it exists. """ - command = f'[ -f "{remote_file_path}" ] && echo "File exists"' + command = f'[ -f {shlex.quote(remote_file_path)} ] && echo "File exists"' stdout, _ = self._send_command_to_server(command, remote_path='') if len(stdout): return True @@ -549,7 +900,7 @@ def _check_dir_exists(self, Returns: bool: Whether the directory exists on the remote server. ``True`` if it exists. """ - command = f'[ -d "{remote_dir_path}" ] && echo "Dir exists"' + command = f'[ -d {shlex.quote(remote_dir_path)} ] && echo "Dir exists"' stdout, _ = self._send_command_to_server(command) if len(stdout): return True @@ -561,20 +912,200 @@ def _create_dir(self, remote_path: str) -> None: Args: remote_path (str): The path to the directory to create on the remote server. """ - command = f'mkdir -p "{remote_path}"' + command = f'mkdir -p -- {shlex.quote(remote_path)}' _, stderr = self._send_command_to_server(command) if stderr: raise ServerError( f'Cannot create dir for the given path ({remote_path}).\nGot: {stderr}') +def _addresses_worth_checking(server_dict: dict) -> dict[str, str]: + """ + Return the address of every server whose host key is worth looking up. + + Servers named ``local``, entries that are not dictionaries, servers without an ``address``, + and servers still carrying ARC's shipped placeholder address or username are left out. The + placeholders cannot be reached at all, and reporting them would fire on every run made with + the repository's default settings. + + Args: + server_dict (dict): The servers to filter. + + Returns: dict[str, str] + The address of each server to check, keyed by server name. + """ + addresses = dict() + for server_name, server_settings in server_dict.items(): + if server_name == 'local' or not isinstance(server_settings, dict): + continue + address = server_settings.get('address') + if not address or address.endswith(PLACEHOLDER_ADDRESS_SUFFIX) \ + or server_settings.get('un') == PLACEHOLDER_USERNAME: + continue + addresses[server_name] = address + return addresses + + +def _load_known_hosts(path: str) -> paramiko.HostKeys: + """ + Read a ``known_hosts`` file, returning an empty set of host keys if it cannot be read. + + Args: + path (str): The expanded path of the ``known_hosts`` file to read. + + Returns: paramiko.HostKeys + The host keys the file holds, empty when the file is absent or unreadable. + """ + try: + return paramiko.HostKeys(filename=path) + except (OSError, paramiko.SSHException) as e: + logger.debug(f'Could not read the host keys in {path}: {type(e).__name__}: {e}') + return paramiko.HostKeys() + + +def get_servers_missing_host_keys(server_dict: dict | None = None, + known_hosts_path: str | None = None, + ) -> dict[str, str]: + """ + Determine which of the configured servers have no host key on this machine. + + The lookup is local and offline: the ``known_hosts`` file is read, nothing is resolved and + no connection is opened. paramiko's ``HostKeys`` performs the lookup, so hashed entries + (``ssh-keyscan -H``) and ``[host]:port`` entries are matched as OpenSSH matches them. + + This reports an absent key only. Whether a stored key still matches the one a server + presents is not knowable from this machine, since only the server can present it; that + comparison is made by paramiko while connecting, and raises + :class:`HostKeyMismatchError`. What can be checked offline alongside an absent key is a + ``known_hosts`` file that contradicts itself, which is + :func:`get_servers_with_conflicting_host_keys`. + + Args: + server_dict (dict, optional): The servers to check. Defaults to the configured servers. + known_hosts_path (str, optional): The ``known_hosts`` file to read. + Defaults to :data:`KNOWN_HOSTS_PATH`. + + Returns: dict[str, str] + The address of each server that has no host key, keyed by server name. + """ + server_dict = servers if server_dict is None else server_dict + path = os.path.expanduser(known_hosts_path if known_hosts_path is not None else KNOWN_HOSTS_PATH) + host_keys = _load_known_hosts(path) + missing = dict() + for server_name, address in _addresses_worth_checking(server_dict).items(): + if host_keys.lookup(address) is None: + missing[server_name] = address + return missing + + +def get_servers_with_conflicting_host_keys(server_dict: dict | None = None, + known_hosts_path: str | None = None, + ) -> dict[str, list[str]]: + """ + Determine which of the configured servers have contradictory host keys on this machine. + + A server legitimately has one host key per key type, and ``ssh-keyscan`` writes one line per + type. More than one entry of the *same* type for one address means the file disagrees with + itself about what that server's key is, which is what a stale entry left behind by a rebuilt + server looks like, and equally what an entry prepended to shadow the real key looks like. + Only the first matching entry is ever consulted -- by OpenSSH, and by the ``HostKeys.lookup`` + paramiko authenticates with -- so a shadowed key is trusted silently while the server's real + key is reported as a mismatch. + + The check is local and offline: the ``known_hosts`` file is read, nothing is resolved and no + connection is opened. It therefore cannot say *which* of the recorded keys is the server's; + answering that requires the key the server presents, which is compared while connecting and + raises :class:`HostKeyMismatchError`. + + Args: + server_dict (dict, optional): The servers to check. Defaults to the configured servers. + known_hosts_path (str, optional): The ``known_hosts`` file to read. + Defaults to :data:`KNOWN_HOSTS_PATH`. + + Returns: dict[str, list[str]] + The key types recorded more than once, sorted, keyed by server name. Servers whose + entries do not contradict each other are absent. + """ + server_dict = servers if server_dict is None else server_dict + path = os.path.expanduser(known_hosts_path if known_hosts_path is not None else KNOWN_HOSTS_PATH) + host_keys = _load_known_hosts(path) + conflicting = dict() + for server_name, address in _addresses_worth_checking(server_dict).items(): + entries = host_keys.lookup(address) + if entries is None: + continue + repeated = sorted(key_type for key_type, count in Counter(entries.keys()).items() if count > 1) + if repeated: + conflicting[server_name] = repeated + return conflicting + + +def check_servers_known_hosts(server_dict: dict | None = None, + known_hosts_path: str | None = None, + ) -> dict[str, str]: + """ + Report configured servers whose host keys need attention before any job is submitted. + + Two offline conditions are reported, at two levels. A server with no host key at all is a + warning: ARC connects to an unknown host anyway (see :meth:`SSHClient._connect`), so without + this the first sign of an unseeded ``known_hosts`` is a per-connection warning buried in a + running job's log, or -- for a server with ``strict_host_key_checking`` -- a run that appears + to hang while every connection is refused. A server with contradictory entries + (:func:`get_servers_with_conflicting_host_keys`) is an error: the file records two different + keys as that server's, only one of them is consulted, and which one is trusted is decided by + line order rather than by anything the reader chose. + + A stored key that no longer matches the key the server presents is not reported here and + cannot be, since the comparison needs the server. paramiko makes it while connecting, and it + surfaces as :class:`HostKeyMismatchError`. + + Args: + server_dict (dict, optional): The servers to check. Defaults to the configured servers. + known_hosts_path (str, optional): The ``known_hosts`` file to read. + Defaults to :data:`KNOWN_HOSTS_PATH`. + + Returns: dict[str, str] + The address of each server that has no host key, keyed by server name. + """ + server_dict = servers if server_dict is None else server_dict + path = os.path.expanduser(known_hosts_path if known_hosts_path is not None else KNOWN_HOSTS_PATH) + missing = get_servers_missing_host_keys(server_dict=server_dict, known_hosts_path=path) + for server_name, address in missing.items(): + if server_dict[server_name].get('strict_host_key_checking', False): + consequence = f'server "{server_name}" sets strict_host_key_checking, so every ' \ + f'connection to it will be refused' + else: + consequence = f'ARC will connect to server "{server_name}" anyway and warn on every ' \ + f'connection, and cannot tell a first-ever connection from an interception' + logger.warning(f'The host key of {address} is not in {path}; {consequence}. ' + f'Verify the fingerprint against a trusted source and add it with:\n' + f' ssh-keyscan -H {address} >> {path}') + conflicting = get_servers_with_conflicting_host_keys(server_dict=server_dict, known_hosts_path=path) + for server_name, key_types in conflicting.items(): + address = server_dict[server_name]['address'] + logger.error(f'{path} records more than one {", ".join(key_types)} host key for ' + f'{address}, the address of server "{server_name}", so it disagrees with ' + f'itself about that server\'s identity. Only the first entry is used, which ' + f'means a stale key left by a rebuilt server, or a key placed there to ' + f'impersonate it, would be trusted in place of the real one. Verify the ' + f'server\'s fingerprint against a trusted source, then leave only that key:\n' + f' ssh-keygen -R {address} -f {path}\n' + f' ssh-keyscan -H {address} >> {path}') + return missing + + def delete_check_files_on_servers(remote_project_paths: dict) -> None: """ Delete ESS checkfiles from an ARC project's directory on all servers it ran jobs on. The local counterpart of this function is ``arc.common.delete_check_files()``. Errors are only logged and never raised: this runs once ARC is done with the science, an unreachable server at that point is an inconvenience, not a reason to lose a run. - Only a single connection attempt is made per server for the same reason. + + Each server is reached through its own single-attempt client rather than through the + connection pool (:mod:`arc.job.ssh_pool`), for the same reason: both the pool's factory and + the fallback in :func:`~arc.job.ssh_pool.borrow_ssh_client` build a client with the default + 24-hour retry, so borrowing here would let a server that has gone away hold up the end of a + run indefinitely. A cleanup that cannot reach a server must give up, not wait. Args: remote_project_paths (dict): Keys are server names, values are the respective remote paths diff --git a/arc/job/ssh_test.py b/arc/job/ssh_test.py index accaf710bf..dc4e9f1464 100644 --- a/arc/job/ssh_test.py +++ b/arc/job/ssh_test.py @@ -5,17 +5,44 @@ This module contains unit tests of the arc.job.ssh module """ +import base64 +import hashlib import os import shlex import shutil import subprocess import tempfile import unittest -from unittest import mock +import warnings +from unittest.mock import MagicMock, patch + +import paramiko import arc.job.ssh as ssh from arc.exceptions import ServerError -from arc.job.ssh import SSHClient, delete_check_files_on_servers + + +class FakeHostKey(object): + """A stand-in for a paramiko host key, carrying only what the host key policies use.""" + + def __init__(self, blob: bytes = b'fake-host-key-blob'): + self.blob = blob + + def asbytes(self) -> bytes: + """Return the key blob.""" + return self.blob + + def get_name(self) -> str: + """Return the key type.""" + return 'ssh-ed25519' + + def get_base64(self) -> str: + """Return the base64-encoded key blob.""" + return base64.b64encode(self.blob).decode() + + def get_fingerprint(self) -> bytes: + """Return the MD5 digest of the key blob, which is what paramiko fingerprints with.""" + return hashlib.md5(self.blob).digest() class TestSSH(unittest.TestCase): @@ -51,10 +78,897 @@ def test_check_job_status_in_stdout(self): self.assertEqual(status1, 'done') + +class TestSSHConnectHardening(unittest.TestCase): + """Host-key policy selection and retry scoping in SSHClient._connect().""" + + SERVER = {'address': 'host.example.edu', 'un': 'user', 'key': '/dev/null', + 'cluster_soft': 'PBS'} + + def _connect_with(self, server_cfg, connect_side_effect=None): + """Run _connect() against a fake paramiko, returning (fake_client, exception). + + Only the exception types these tests assert on are caught, so any other exception + escaping _connect() fails the test rather than being reported as an expected one. + """ + fake = MagicMock() + if connect_side_effect is not None: + fake.connect.side_effect = connect_side_effect + raised = None + with patch.object(ssh, 'servers', {'srv': server_cfg}), \ + patch.object(ssh.paramiko, 'SSHClient', return_value=fake): + client = ssh.SSHClient('srv') + try: + client._connect() + except (paramiko.SSHException, OSError, KeyboardInterrupt) as exc: + raised = exc + return fake, raised + + def _policy_used(self, fake): + return type(fake.set_missing_host_key_policy.call_args[0][0]) + + def test_default_policy_warns_rather_than_adding_silently(self): + """Unknown host keys must not be added silently by default.""" + fake, _ = self._connect_with(dict(self.SERVER)) + self.assertIs(self._policy_used(fake), ssh.LogAndAcceptHostKeyPolicy) + + def test_strict_host_key_checking_rejects_unknown_hosts(self): + """strict_host_key_checking opts into refusing unknown hosts.""" + fake, _ = self._connect_with(dict(self.SERVER, strict_host_key_checking=True)) + self.assertIs(self._policy_used(fake), ssh.RejectUnknownHostKeyPolicy) + + def test_configured_key_is_offered_as_the_identity(self): + """The key must reach connect() as key_filename, not only host keys.""" + fake, _ = self._connect_with(dict(self.SERVER)) + self.assertEqual(fake.connect.call_args.kwargs['key_filename'], '/dev/null') + + def test_transport_error_is_retried_once(self): + """A banner/reset SSHException retries, matching the documented flake.""" + fake, raised = self._connect_with( + dict(self.SERVER), + connect_side_effect=[paramiko.SSHException('Error reading SSH protocol banner'), None]) + self.assertIsNone(raised) + self.assertEqual(fake.connect.call_count, 2) + + def test_keyboard_interrupt_is_not_swallowed(self): + """The retry must not catch KeyboardInterrupt, as a bare except did.""" + fake, raised = self._connect_with(dict(self.SERVER), + connect_side_effect=KeyboardInterrupt()) + self.assertIsInstance(raised, KeyboardInterrupt) + self.assertEqual(fake.connect.call_count, 1, 'must not retry after an interrupt') + + def test_second_transport_failure_propagates(self): + """If the retry also fails, the error surfaces rather than being hidden.""" + fake, raised = self._connect_with( + dict(self.SERVER), + connect_side_effect=[paramiko.SSHException('first'), paramiko.SSHException('second')]) + self.assertIsInstance(raised, paramiko.SSHException) + self.assertEqual(fake.connect.call_count, 2) + + +class TestSSHOptionalKey(unittest.TestCase): + """``servers[...]['key']`` is optional, for ssh-agent and default-key-path authentication.""" + + SERVER = {'address': 'host.example.edu', 'un': 'user', 'cluster_soft': 'PBS'} + + def _client(self, server_cfg): + """Instantiate an SSHClient against ``server_cfg`` without touching the network.""" + with patch.object(ssh, 'servers', {'srv': server_cfg}): + return ssh.SSHClient('srv') + + def _key_filename_used(self, server_cfg): + """Return the ``key_filename`` that _connect() hands to paramiko.""" + fake = MagicMock() + with patch.object(ssh, 'servers', {'srv': server_cfg}), \ + patch.object(ssh.paramiko, 'SSHClient', return_value=fake): + ssh.SSHClient('srv')._connect() + return fake.connect.call_args.kwargs['key_filename'] + + def test_configured_key_is_stored(self): + """A configured key is still read off the server settings.""" + self.assertEqual(self._client(dict(self.SERVER, key='/dev/null')).key, '/dev/null') + + def test_configured_key_is_forwarded_as_key_filename(self): + """A configured key is offered to paramiko as the connection identity.""" + self.assertEqual(self._key_filename_used(dict(self.SERVER, key='/dev/null')), '/dev/null') + + def test_missing_key_does_not_raise(self): + """A server entry without a key must not raise a KeyError on instantiation.""" + self.assertIsNone(self._client(dict(self.SERVER)).key) + + def test_missing_key_is_not_offered_as_an_identity(self): + """Without a key, paramiko must be free to fall back to the agent and default keys.""" + self.assertIsNone(self._key_filename_used(dict(self.SERVER))) + + def test_empty_key_is_treated_as_unset(self): + """An empty key path is not a usable identity, and must not be handed to paramiko.""" + self.assertIsNone(self._client(dict(self.SERVER, key='')).key) + self.assertIsNone(self._key_filename_used(dict(self.SERVER, key=''))) + + +class TestMissingKeyFileIsPermanent(unittest.TestCase): + """A ``key`` path that does not exist cannot be resolved by retrying it for 24 hours.""" + + SERVER = {'address': 'host.example.edu', 'un': 'user', 'key': '/no/such/key', + 'cluster_soft': 'PBS'} + + def _client(self, **overrides): + with patch.object(ssh, 'servers', {'srv': dict(self.SERVER, **overrides)}): + return ssh.SSHClient('srv', connection_attempts=5) + + def _connect_with(self, error, **overrides): + """Call connect() with _connect() failing on ``error``, returning (raised, calls, sleeps).""" + client = self._client(**overrides) + inner = MagicMock(side_effect=error) + raised = None + with patch.object(ssh, 'servers', {'srv': dict(self.SERVER, **overrides)}), \ + patch.object(client, '_connect', inner), \ + patch.object(ssh.time, 'sleep') as sleep: + try: + client.connect() + except ServerError as exc: + raised = exc + return raised, inner.call_count, sleep.call_count + + def test_a_missing_key_file_is_not_retried(self): + """paramiko reads the identity outside the SSHException it guards that read with.""" + error = FileNotFoundError(2, 'No such file or directory', '/no/such/key') + raised, calls, sleeps = self._connect_with(error) + self.assertIsInstance(raised, ServerError) + self.assertEqual(calls, 1) + self.assertEqual(sleeps, 0) + + def test_the_error_names_the_key_and_the_way_out(self): + """A 24 hour stall used to be the only report that the path was wrong.""" + error = FileNotFoundError(2, 'No such file or directory', '/no/such/key') + message = str(self._connect_with(error)[0]) + self.assertIn('/no/such/key', message) + self.assertIn('ssh-agent', message) + + def test_a_network_failure_is_still_retried(self): + """A refused or reset connection is an OSError too, and is exactly what the retry is for.""" + raised, calls, sleeps = self._connect_with(ConnectionRefusedError(111, 'Connection refused')) + self.assertIsInstance(raised, ServerError) + self.assertEqual(calls, 5) + self.assertEqual(sleeps, 4) + + def test_a_missing_file_that_is_not_the_key_is_still_retried(self): + """Only the configured key path is permanent; another absent file is not classified here.""" + error = FileNotFoundError(2, 'No such file or directory', '/some/other/file') + raised, calls, sleeps = self._connect_with(error) + self.assertIsInstance(raised, ServerError) + self.assertEqual(calls, 5) + self.assertEqual(sleeps, 4) + + def test_a_missing_file_without_a_configured_key_is_still_retried(self): + """With no key configured there is no key file to be missing.""" + error = FileNotFoundError(2, 'No such file or directory', '/no/such/key') + raised, calls, sleeps = self._connect_with(error, key=None) + self.assertIsInstance(raised, ServerError) + self.assertEqual(calls, 5) + self.assertEqual(sleeps, 4) + + def test_the_inner_retry_does_not_reattempt_a_missing_key(self): + """_connect() retries a transport failure once, which a missing key file is not.""" + fake = MagicMock() + fake.connect.side_effect = FileNotFoundError(2, 'No such file or directory', '/no/such/key') + with patch.object(ssh, 'servers', {'srv': dict(self.SERVER)}), \ + patch.object(ssh.paramiko, 'SSHClient', return_value=fake): + self.assertRaises(FileNotFoundError, ssh.SSHClient('srv')._connect) + self.assertEqual(fake.connect.call_count, 1) + + +class TestKeepaliveSurvivesReconnects(unittest.TestCase): + """A keepalive lives on a paramiko Transport, and a reconnect replaces the transport.""" + + SERVER = {'address': 'host.example.edu', 'un': 'user', 'key': '/dev/null', + 'cluster_soft': 'PBS'} + + def _client(self): + with patch.object(ssh, 'servers', {'srv': dict(self.SERVER)}): + return ssh.SSHClient('srv') + + def _connect_onto(self, client, transport): + """Connect ``client`` so that its paramiko client reports ``transport``.""" + paramiko_client = MagicMock() + paramiko_client.get_transport.return_value = transport + with patch.object(ssh, 'servers', {'srv': dict(self.SERVER)}), \ + patch.object(client, '_connect', MagicMock(return_value=(MagicMock(), paramiko_client))): + client.connect() + + def test_a_client_never_asked_for_a_keepalive_does_not_set_one(self): + """A one-shot client is not held open long enough to be dropped while idle.""" + transport = MagicMock() + client = self._client() + self._connect_onto(client, transport) + transport.set_keepalive.assert_not_called() + + def test_the_keepalive_is_reapplied_to_a_new_transport(self): + """check_connections reconnects a pooled client in place, replacing its transport.""" + client = self._client() + client._keepalive_interval = 30 + second = MagicMock() + self._connect_onto(client, second) + second.set_keepalive.assert_called_once_with(30) + + def test_a_reconnect_without_a_transport_does_not_raise(self): + """A client whose paramiko client has no transport has nothing to keep alive.""" + client = self._client() + client._keepalive_interval = 30 + self._connect_onto(client, None) + self.assertFalse(client._apply_keepalive()) + + +class TestKnownHostsCheck(unittest.TestCase): + """The startup report of servers that have no host key on this machine.""" + + @classmethod + def setUpClass(cls): + """Generate one host key, reused by every test as the key of a 'known' host.""" + key = paramiko.ECDSAKey.generate() + cls.key_type, cls.key_b64 = key.get_name(), key.get_base64() + + def setUp(self): + self.tmp_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp_dir, True) + self.known_hosts = os.path.join(self.tmp_dir, 'known_hosts') + + def _write_known_hosts(self, *host_patterns): + """Write a known_hosts file listing ``host_patterns``, and return its path.""" + with open(self.known_hosts, 'w') as f: + for host_pattern in host_patterns: + f.write(f'{host_pattern} {self.key_type} {self.key_b64}\n') + return self.known_hosts + + def test_a_known_host_is_not_reported(self): + """A server whose address is in known_hosts must not be reported.""" + self._write_known_hosts('login.cluster.edu') + missing = ssh.get_servers_missing_host_keys( + server_dict={'srv': {'address': 'login.cluster.edu', 'un': 'me', 'cluster_soft': 'PBS'}}, + known_hosts_path=self.known_hosts) + self.assertEqual(missing, {}) + + def test_an_unknown_host_is_reported_with_its_address(self): + """A server absent from known_hosts is reported, keyed by its server name.""" + self._write_known_hosts('other.cluster.edu') + missing = ssh.get_servers_missing_host_keys( + server_dict={'srv': {'address': 'login.cluster.edu', 'un': 'me', 'cluster_soft': 'PBS'}}, + known_hosts_path=self.known_hosts) + self.assertEqual(missing, {'srv': 'login.cluster.edu'}) + + def test_a_hashed_entry_is_recognized(self): + """``ssh-keyscan -H`` writes hashed host names; paramiko's lookup must resolve them.""" + self._write_known_hosts(paramiko.HostKeys.hash_host('login.cluster.edu')) + missing = ssh.get_servers_missing_host_keys( + server_dict={'srv': {'address': 'login.cluster.edu', 'un': 'me', 'cluster_soft': 'PBS'}}, + known_hosts_path=self.known_hosts) + self.assertEqual(missing, {}) + + def test_an_absent_known_hosts_file_reports_every_server(self): + """No known_hosts file at all must report the servers, not raise.""" + missing = ssh.get_servers_missing_host_keys( + server_dict={'srv': {'address': 'login.cluster.edu', 'un': 'me', 'cluster_soft': 'PBS'}}, + known_hosts_path=os.path.join(self.tmp_dir, 'does_not_exist')) + self.assertEqual(missing, {'srv': 'login.cluster.edu'}) + + def test_local_and_addressless_servers_are_skipped(self): + """A 'local' server and a server without an address are not reachable over SSH.""" + missing = ssh.get_servers_missing_host_keys( + server_dict={'local': {'cluster_soft': 'PBS', 'un': 'me'}, + 'no_address': {'cluster_soft': 'PBS', 'un': 'me'}}, + known_hosts_path=self.known_hosts) + self.assertEqual(missing, {}) + + def test_placeholder_servers_are_skipped(self): + """The repository's shipped placeholders must not warn on every default install.""" + missing = ssh.get_servers_missing_host_keys( + server_dict={'server1': {'address': 'server1.host.edu', 'un': '', + 'cluster_soft': 'OGE'}, + 'named_user': {'address': 'real.cluster.edu', 'un': '', + 'cluster_soft': 'PBS'}}, + known_hosts_path=self.known_hosts) + self.assertEqual(missing, {}) + + def test_check_warns_naming_the_server_and_the_fix(self): + """The warning must name the server, the address and the ssh-keyscan command.""" + server_dict = {'srv': {'address': 'login.cluster.edu', 'un': 'me', 'cluster_soft': 'PBS'}} + with self.assertLogs(ssh.logger, level='WARNING') as captured: + missing = ssh.check_servers_known_hosts(server_dict=server_dict, + known_hosts_path=self.known_hosts) + self.assertEqual(missing, {'srv': 'login.cluster.edu'}) + logged = '\n'.join(captured.output) + self.assertIn('srv', logged) + self.assertIn('login.cluster.edu', logged) + self.assertIn(f'ssh-keyscan -H login.cluster.edu >> {self.known_hosts}', logged) + + def test_check_reports_refusal_for_a_strict_server(self): + """With strict_host_key_checking the consequence is a refusal, not a warning.""" + server_dict = {'srv': {'address': 'login.cluster.edu', 'un': 'me', 'cluster_soft': 'PBS', + 'strict_host_key_checking': True}} + with self.assertLogs(ssh.logger, level='WARNING') as captured: + ssh.check_servers_known_hosts(server_dict=server_dict, + known_hosts_path=self.known_hosts) + self.assertIn('will be refused', '\n'.join(captured.output)) + + def test_check_is_silent_when_every_host_is_known(self): + """No warning may be emitted when nothing is missing.""" + self._write_known_hosts('login.cluster.edu') + server_dict = {'srv': {'address': 'login.cluster.edu', 'un': 'me', 'cluster_soft': 'PBS'}} + with patch.object(ssh.logger, 'warning') as warning: + missing = ssh.check_servers_known_hosts(server_dict=server_dict, + known_hosts_path=self.known_hosts) + self.assertEqual(missing, {}) + warning.assert_not_called() + + +class TestTransfersCheckTheConnection(unittest.TestCase): + """A pooled client is long-lived, so a transfer must re-establish a dropped connection.""" + + SERVER = {'address': 'host.example.edu', 'un': 'user', 'key': '/dev/null', + 'cluster_soft': 'PBS'} + + def setUp(self): + """Build an unconnected client whose connect() only records that it was called.""" + self.tmp_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp_dir, ignore_errors=True) + with patch.object(ssh, 'servers', {'srv': dict(self.SERVER)}): + self.client = ssh.SSHClient('srv') + + def _connect(): + self.client._sftp = MagicMock() + self.client._ssh = MagicMock() + self.connect = MagicMock(side_effect=_connect) + patcher = patch.object(self.client, 'connect', self.connect) + patcher.start() + self.addCleanup(patcher.stop) + + def test_upload_file_connects_an_unconnected_client(self): + """Without this, the SFTP handle is None and the upload raises AttributeError.""" + with patch.object(self.client, '_check_dir_exists', return_value=True): + self.client.upload_file(remote_file_path='/remote/in.gjf', file_string='#p opt\n') + self.connect.assert_called_once() + self.client._sftp.open.assert_called_once() + + def test_download_file_connects_an_unconnected_client(self): + with patch.object(self.client, '_check_file_exists', return_value=True): + self.client.download_file(remote_file_path='/remote/out.txt', + local_file_path=os.path.join(self.tmp_dir, 'out.txt')) + self.connect.assert_called_once() + self.client._sftp.get.assert_called_once() + + def test_upload_file_reconnects_a_dead_connection(self): + """The case the pool makes routine: the transport died between two jobs.""" + self.client._sftp, self.client._ssh = MagicMock(), MagicMock() + self.client._ssh.exec_command.side_effect = OSError('Socket is closed') + with patch.object(self.client, '_check_dir_exists', return_value=True): + self.client.upload_file(remote_file_path='/remote/in.gjf', file_string='#p opt\n') + self.connect.assert_called_once() + + def test_download_file_reconnects_a_dead_connection(self): + self.client._sftp, self.client._ssh = MagicMock(), MagicMock() + self.client._ssh.exec_command.side_effect = OSError('Socket is closed') + with patch.object(self.client, '_check_file_exists', return_value=True): + self.client.download_file(remote_file_path='/remote/out.txt', + local_file_path=os.path.join(self.tmp_dir, 'out.txt')) + self.connect.assert_called_once() + + def test_an_unconnected_client_does_not_unpack_the_connect_result(self): + """connect() assigns the handles and returns None, so unpacking it raised TypeError.""" + self.client.read_remote_file(remote_file_path='/remote/out.txt') + self.connect.assert_called_once() + self.client._sftp.open.assert_called_once() + + +class TestConflictingHostKeys(unittest.TestCase): + """known_hosts entries that contradict each other are reported without connecting.""" + + @classmethod + def setUpClass(cls): + """Generate two distinct keys of one type, which is what a contradiction is made of.""" + cls.key_1 = paramiko.ECDSAKey.generate() + cls.key_2 = paramiko.ECDSAKey.generate() + cls.rsa_key = paramiko.RSAKey.generate(2048) + + def setUp(self): + self.tmp_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp_dir, True) + self.known_hosts = os.path.join(self.tmp_dir, 'known_hosts') + + SERVER = {'address': 'login.cluster.edu', 'un': 'me', 'cluster_soft': 'PBS'} + + def _write(self, *entries): + """Write ``(host pattern, key)`` pairs to known_hosts, and return its path.""" + with open(self.known_hosts, 'w') as f: + for host_pattern, key in entries: + f.write(f'{host_pattern} {key.get_name()} {key.get_base64()}\n') + return self.known_hosts + + def _conflicting(self, server_dict=None): + """Return the conflict report for ``server_dict``, defaulting to one ordinary server.""" + return ssh.get_servers_with_conflicting_host_keys( + server_dict=server_dict if server_dict is not None else {'srv': dict(self.SERVER)}, + known_hosts_path=self.known_hosts) + + def test_two_keys_of_one_type_for_one_host_are_reported(self): + """This is the shape of a stale entry, and equally of one placed there to shadow.""" + self._write(('login.cluster.edu', self.key_1), ('login.cluster.edu', self.key_2)) + self.assertEqual(self._conflicting(), {'srv': [self.key_1.get_name()]}) + + def test_one_key_per_type_is_not_a_conflict(self): + """A host legitimately has one key of each type, which must not be reported.""" + self._write(('login.cluster.edu', self.key_1), ('login.cluster.edu', self.rsa_key)) + self.assertEqual(self._conflicting(), {}) + + def test_a_single_entry_is_not_a_conflict(self): + self._write(('login.cluster.edu', self.key_1)) + self.assertEqual(self._conflicting(), {}) + + def test_a_hashed_entry_contradicting_a_plain_one_is_reported(self): + """ssh-keyscan -H writes hashed names, so a stale pair may not look like a pair.""" + self._write((paramiko.HostKeys.hash_host('login.cluster.edu'), self.key_1), + ('login.cluster.edu', self.key_2)) + self.assertEqual(self._conflicting(), {'srv': [self.key_1.get_name()]}) + + def test_an_unknown_host_is_not_reported_as_conflicting(self): + """Absence is the sibling check's business, not this one's.""" + self._write(('other.cluster.edu', self.key_1)) + self.assertEqual(self._conflicting(), {}) + + def test_an_absent_known_hosts_file_reports_nothing(self): + self.assertEqual( + ssh.get_servers_with_conflicting_host_keys( + server_dict={'srv': dict(self.SERVER)}, + known_hosts_path=os.path.join(self.tmp_dir, 'does_not_exist')), + {}) + + def test_local_and_placeholder_servers_are_skipped(self): + """The same servers the absence check skips, for the same reasons.""" + self._write(('server1.host.edu', self.key_1), ('server1.host.edu', self.key_2)) + self.assertEqual( + self._conflicting({'local': {'cluster_soft': 'PBS', 'un': 'me'}, + 'server1': {'address': 'server1.host.edu', 'un': '', + 'cluster_soft': 'OGE'}}), + {}) + + def test_the_check_reports_a_conflict_at_the_error_level(self): + """A file that disagrees with itself about a server's identity is not a warning.""" + self._write(('login.cluster.edu', self.key_1), ('login.cluster.edu', self.key_2)) + with self.assertLogs(ssh.logger, level='ERROR') as captured: + ssh.check_servers_known_hosts(server_dict={'srv': dict(self.SERVER)}, + known_hosts_path=self.known_hosts) + logged = '\n'.join(captured.output) + self.assertIn('srv', logged) + self.assertIn('login.cluster.edu', logged) + self.assertIn('ssh-keygen -R login.cluster.edu', logged) + + def test_the_check_is_silent_when_the_single_key_is_known(self): + """One key, no absence and no contradiction, must produce no report at all.""" + self._write(('login.cluster.edu', self.key_1)) + with patch.object(ssh.logger, 'warning') as warning, \ + patch.object(ssh.logger, 'error') as error: + ssh.check_servers_known_hosts(server_dict={'srv': dict(self.SERVER)}, + known_hosts_path=self.known_hosts) + warning.assert_not_called() + error.assert_not_called() + + +class TestHostKeyMismatch(unittest.TestCase): + """A stored key contradicted by the server is the security-relevant case.""" + + SERVER = {'address': 'host.example.edu', 'un': 'user', 'key': '/dev/null', + 'cluster_soft': 'PBS'} + + def setUp(self): + with patch.object(ssh, 'servers', {'srv': dict(self.SERVER)}): + self.client = ssh.SSHClient('srv') + self.stored = FakeHostKey(b'the-key-known_hosts-has') + self.presented = FakeHostKey(b'the-key-the-server-sent') + self.error = paramiko.BadHostKeyException(self.SERVER['address'], + self.presented, self.stored) + + def _connect(self): + """Call connect() with _connect() failing on the mismatch, and return what was raised.""" + with patch.object(ssh, 'servers', {'srv': dict(self.SERVER)}), \ + patch.object(self.client, '_connect', MagicMock(side_effect=self.error)), \ + patch.object(ssh.time, 'sleep'): + try: + self.client.connect() + except ServerError as exc: + return exc + return None + + def test_a_mismatch_raises_its_own_error_type(self): + """Told apart from an absent key and from a wrong password, which mean other things.""" + raised = self._connect() + self.assertIsInstance(raised, ssh.HostKeyMismatchError) + self.assertIsInstance(raised, ServerError) + self.assertIs(raised.__cause__, self.error) + + def test_the_error_names_both_fingerprints(self): + """Which of the two the reader recognises is what decides re-keyed from intercepted.""" + message = str(self._connect()) + self.assertIn(ssh.get_host_key_fingerprint(self.stored), message) + self.assertIn(ssh.get_host_key_fingerprint(self.presented), message) + + def test_the_error_gives_the_command_that_replaces_the_stale_key(self): + message = str(self._connect()) + self.assertIn(f'ssh-keygen -R {self.SERVER["address"]}', message) + + def test_the_mismatch_is_reported_at_the_error_level(self): + """A mismatch buried at the warning level reads as one more failed connection.""" + with self.assertLogs(ssh.logger, level='ERROR') as captured: + self._connect() + self.assertIn('does not match', '\n'.join(captured.output)) + + def test_a_mismatch_is_not_retried(self): + """Retrying an intercepted or re-keyed server cannot resolve it.""" + inner = MagicMock(side_effect=self.error) + with patch.object(ssh, 'servers', {'srv': dict(self.SERVER)}), \ + patch.object(self.client, '_connect', inner), \ + patch.object(ssh.time, 'sleep') as sleep: + self.assertRaises(ssh.HostKeyMismatchError, self.client.connect) + self.assertEqual(inner.call_count, 1) + self.assertEqual(sleep.call_count, 0) + + +class TestRemoteCommandQuoting(unittest.TestCase): + """Caller-derived values must reach the remote shell as single arguments.""" + + SERVER = {'address': 'host.example.edu', 'un': 'user', 'key': '/dev/null', + 'cluster_soft': 'PBS'} + + INJECTING = '/home/u/runs/a"; touch /tmp/pwned; "b' + SPACED = '/home/u/my runs/proj 1' + DASHED = '-rf' + PLAIN = '/home/u/runs/ARC/proj/calcs' + + HOSTILE = (INJECTING, SPACED, DASHED) + + def _client(self): + """Build an SSHClient without connecting to anything.""" + with patch.object(ssh, 'servers', {'srv': dict(self.SERVER)}): + return ssh.SSHClient('srv') + + def _command_from(self, method_name, *args, **kwargs): + """Return the command string the named method hands to the transport.""" + client = self._client() + sender = MagicMock(return_value=([], [])) + with patch.object(ssh, 'servers', {'srv': dict(self.SERVER)}), \ + patch.object(client, '_send_command_to_server', sender): + getattr(client, method_name)(*args, **kwargs) + return sender.call_args[0][0] + + def _tokens(self, command): + """Split a single-command string the way a POSIX shell would.""" + return shlex.split(command) + + def test_remove_dir_passes_the_path_as_one_argument(self): + """A path with a quote, a space or a leading dash must not become shell syntax.""" + for path in self.HOSTILE: + with self.subTest(path=path): + tokens = self._tokens(self._command_from('remove_dir', path)) + self.assertEqual(tokens, ['rm', '-rf', '--', path]) + + def test_remove_dir_does_not_leak_an_injected_command(self): + """The injected payload must survive as data inside the path operand.""" + command = self._command_from('remove_dir', self.INJECTING) + self.assertNotIn('; touch /tmp/pwned; ', command.replace(shlex.quote(self.INJECTING), '')) + + def test_create_dir_passes_the_path_as_one_argument(self): + """mkdir must receive the path as a single operand after an end-of-options marker.""" + for path in self.HOSTILE: + with self.subTest(path=path): + tokens = self._tokens(self._command_from('_create_dir', path)) + self.assertEqual(tokens, ['mkdir', '-p', '--', path]) + + def test_check_file_exists_keeps_a_valid_test_expression(self): + """The path must be one word inside [ -f ... ] and the && echo must be intact.""" + for path in self.HOSTILE: + with self.subTest(path=path): + tokens = self._tokens(self._command_from('_check_file_exists', path)) + self.assertEqual(tokens, ['[', '-f', path, ']', '&&', 'echo', 'File exists']) + + def test_check_dir_exists_keeps_a_valid_test_expression(self): + """The path must be one word inside [ -d ... ] and the && echo must be intact.""" + for path in self.HOSTILE: + with self.subTest(path=path): + tokens = self._tokens(self._command_from('_check_dir_exists', path)) + self.assertEqual(tokens, ['[', '-d', path, ']', '&&', 'echo', 'Dir exists']) + + def test_change_mode_quotes_only_the_file_name(self): + """The mode stays literal shell while the file name becomes one operand.""" + for name in self.HOSTILE: + with self.subTest(name=name): + tokens = self._tokens( + self._command_from('change_mode', '+x', name, remote_path='')) + self.assertEqual(tokens, ['chmod', '--', '+x', name]) + + def test_change_mode_keeps_the_recursive_flag_before_the_marker(self): + """Recursion is an ARC-controlled option and must precede the end-of-options marker.""" + tokens = self._tokens( + self._command_from('change_mode', '+x', self.SPACED, recursive=True, remote_path='')) + self.assertEqual(tokens, ['chmod', '-R', '--', '+x', self.SPACED]) + + def test_find_package_passes_the_name_as_one_argument(self): + """A package name reaches which() as a single argument.""" + tokens = self._tokens(self._command_from('find_package', 'g16')) + self.assertEqual(tokens, ['.', '~/.bashrc;', 'which', 'g16']) + + def test_find_package_does_not_let_a_name_become_shell_syntax(self): + """A hostile package name must not close the command and start another.""" + command = self._command_from('find_package', 'g16; touch /tmp/pwned') + self.assertTrue(command.endswith(shlex.quote('g16; touch /tmp/pwned'))) + + def test_ordinary_paths_are_unchanged(self): + """A shell-safe path must be interpolated verbatim, with no quoting added.""" + self.assertEqual(self._command_from('remove_dir', self.PLAIN), + f'rm -rf -- {self.PLAIN}') + self.assertEqual(self._command_from('_create_dir', self.PLAIN), + f'mkdir -p -- {self.PLAIN}') + self.assertEqual(self._command_from('_check_file_exists', self.PLAIN), + f'[ -f {self.PLAIN} ] && echo "File exists"') + + +class TestRemotePathCdQuoting(unittest.TestCase): + """The remote_path a command is executed in is quoted, the command itself is not.""" + + SERVER = {'address': 'host.example.edu', 'un': 'user', 'key': '/dev/null', + 'cluster_soft': 'PBS'} + + def _command_sent(self, inner_command, remote_path): + """Return the string handed to exec_command for a command run inside remote_path.""" + fake_ssh = MagicMock() + fake_ssh.exec_command.return_value = (MagicMock(), MagicMock(), MagicMock()) + with patch.object(ssh, 'servers', {'srv': dict(self.SERVER)}): + client = ssh.SSHClient('srv') + client._ssh = fake_ssh + with patch.object(client, '_check_dir_exists', return_value=True): + client._send_command_to_server(inner_command, remote_path) + return fake_ssh.exec_command.call_args_list[-1][0][0] + + def test_the_directory_is_quoted_and_the_command_is_not(self): + """Only the path is a value; the assembled command stays shell.""" + command = self._command_sent('ls -alF', '/home/u/a"; touch /tmp/pwned; "b') + self.assertEqual( + command, + "cd -- '/home/u/a\"; touch /tmp/pwned; \"b'; ls -alF; cd ") + + def test_a_path_with_a_space_stays_one_argument(self): + """A spaced directory must not split into two cd operands.""" + command = self._command_sent('ls -alF', '/home/u/my runs') + self.assertTrue(command.startswith("cd -- '/home/u/my runs'; ")) + + def test_a_path_with_a_leading_dash_is_not_read_as_an_option(self): + """The end-of-options marker keeps a dashed path an operand.""" + command = self._command_sent('ls -alF', '-P') + self.assertTrue(command.startswith('cd -- -P; ')) + + def test_an_ordinary_path_is_interpolated_verbatim(self): + """A shell-safe path gains no quoting.""" + command = self._command_sent('ls -alF', '/home/u/runs/proj') + self.assertEqual(command, 'cd -- /home/u/runs/proj; ls -alF; cd ') + + +class TestHostKeyPolicies(unittest.TestCase): + """The unknown-host-key policies must reach the user and be classifiable.""" + + HOST = 'host.example.edu' + + def setUp(self): + """Build a host key and its expected fingerprint, derived independently of ssh.py.""" + self.key = FakeHostKey() + digest = hashlib.sha256(self.key.asbytes()).digest() + self.fingerprint = 'SHA256:' + base64.b64encode(digest).decode().rstrip('=') + + def _accept(self): + """Run the default policy against the fake key, returning the logged records.""" + with self.assertLogs('arc', level='WARNING') as logged: + result = ssh.LogAndAcceptHostKeyPolicy().missing_host_key( + client=None, hostname=self.HOST, key=self.key) + return result, logged.output + + def test_the_fingerprint_is_the_openssh_sha256_form(self): + """The fingerprint must be comparable to what ssh-keygen -lf prints.""" + self.assertEqual(ssh.get_host_key_fingerprint(self.key), self.fingerprint) + self.assertNotIn('=', ssh.get_host_key_fingerprint(self.key)) + + def test_the_default_policy_logs_the_host_and_the_fingerprint(self): + """Both are needed to verify the key against a trusted source.""" + _, output = self._accept() + self.assertEqual(len(output), 1) + self.assertIn(self.HOST, output[0]) + self.assertIn(self.fingerprint, output[0]) + + def test_the_default_policy_still_accepts_the_key(self): + """Logging must not change the connect-anyway behavior.""" + result, _ = self._accept() + self.assertIsNone(result) + + def test_the_log_survives_the_paramiko_warnings_filter(self): + """initialize_log() ignores paramiko warnings; ARC's own report must not be ignored.""" + with warnings.catch_warnings(): + warnings.filterwarnings(action='ignore', module='.*paramiko.*') + with self.assertLogs('arc', level='WARNING') as logged: + ssh.LogAndAcceptHostKeyPolicy().missing_host_key( + client=None, hostname=self.HOST, key=self.key) + self.assertEqual(len(logged.output), 1) + + def test_paramikos_own_policy_is_silenced_by_that_filter(self): + """The reason ARC cannot rely on paramiko.WarningPolicy: one warning becomes none.""" + with warnings.catch_warnings(record=True) as unfiltered: + warnings.simplefilter('always') + paramiko.WarningPolicy().missing_host_key(None, self.HOST, self.key) + with warnings.catch_warnings(record=True) as filtered: + warnings.simplefilter('always') + warnings.filterwarnings(action='ignore', module='.*paramiko.*') + paramiko.WarningPolicy().missing_host_key(None, self.HOST, self.key) + self.assertEqual(len(unfiltered), 1) + self.assertEqual(len(filtered), 0) + + def test_the_strict_policy_raises_a_distinct_error(self): + """A refused host key must be identifiable by type, not by message text.""" + with self.assertRaises(ssh.UnknownHostKeyError) as raised: + ssh.RejectUnknownHostKeyPolicy().missing_host_key( + client=None, hostname=self.HOST, key=self.key) + self.assertIsInstance(raised.exception, ServerError) + self.assertIsInstance(raised.exception, paramiko.SSHException) + self.assertIn(self.HOST, str(raised.exception)) + self.assertIn(self.fingerprint, str(raised.exception)) + + +class TestConnectRetryClassification(unittest.TestCase): + """connect() retries for 24 hours; only failures that retrying can resolve may enter it.""" + + SERVER = {'address': 'host.example.edu', 'un': 'user', 'key': '/dev/null', + 'cluster_soft': 'PBS'} + + def _connect(self, side_effect): + """Call connect() with _connect() and the retry interval faked out. + + Returns: tuple + The raised exception (or None), the number of _connect() calls, + and the number of sleeps between retries. + """ + with patch.object(ssh, 'servers', {'srv': dict(self.SERVER)}): + client = ssh.SSHClient('srv') + inner = MagicMock(side_effect=side_effect) + raised = None + with patch.object(ssh, 'servers', {'srv': dict(self.SERVER)}), \ + patch.object(client, '_connect', inner), \ + patch.object(ssh.time, 'sleep') as sleep: + try: + client.connect() + except ServerError as exc: + raised = exc + return raised, inner.call_count, sleep.call_count + + def _assert_not_retried(self, error): + """A permanent failure surfaces as a ServerError on the first attempt.""" + raised, calls, sleeps = self._connect(error) + self.assertIsInstance(raised, ServerError) + self.assertIs(raised.__cause__, error) + self.assertEqual(calls, 1) + self.assertEqual(sleeps, 0) + + def test_a_rejected_authentication_is_not_retried(self): + """A wrong key or username is not going to be accepted an hour later.""" + self._assert_not_retried(paramiko.AuthenticationException('Authentication failed.')) + + def test_a_required_password_is_not_retried(self): + """PasswordRequiredException is an authentication failure, and equally permanent.""" + self._assert_not_retried(paramiko.PasswordRequiredException('Private key file is encrypted')) + + def test_a_changed_host_key_is_not_retried(self): + """A key that contradicts known_hosts needs a human, not another attempt.""" + key = FakeHostKey() + self._assert_not_retried(paramiko.BadHostKeyException(self.SERVER['address'], key, key)) + + def test_a_refused_host_key_is_not_retried(self): + """strict_host_key_checking refuses statelessly, so every retry refuses too.""" + self._assert_not_retried(ssh.UnknownHostKeyError('not in known_hosts')) + + def test_a_transport_failure_is_still_retried(self): + """The banner/reset flake the retry was written for must keep being retried.""" + raised, calls, sleeps = self._connect( + [paramiko.SSHException('Error reading SSH protocol banner'), ('sftp', 'ssh')]) + self.assertIsNone(raised) + self.assertEqual(calls, 2) + self.assertEqual(sleeps, 1) + + def test_an_unreachable_server_is_still_retried(self): + """A refused socket may well be a server that is rebooting.""" + raised, calls, sleeps = self._connect([ConnectionRefusedError(111, 'Connection refused'), + ('sftp', 'ssh')]) + self.assertIsNone(raised) + self.assertEqual(calls, 2) + self.assertEqual(sleeps, 1) + + +class TestDownloadFileWithoutARemoteFile(unittest.TestCase): + """A local file must never survive a missing remote file as if it were this job's output.""" + + SERVER = {'address': 'host.example.edu', 'un': 'user', 'key': '/dev/null', + 'cluster_soft': 'PBS'} + + def setUp(self): + """Create a connected client whose remote files are all absent, and a temporary directory.""" + self.tmp_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp_dir, ignore_errors=True) + with patch.object(ssh, 'servers', {'srv': dict(self.SERVER)}): + self.client = ssh.SSHClient('srv') + self.client._sftp = MagicMock() + self.client._ssh = MagicMock() + self.local_path = os.path.join(self.tmp_dir, 'out.txt') + + def _download(self, exists=False): + """Download a remote file whose existence check answers ``exists``. + + Returns: tuple + The existence check and the sleep between attempts, as mocks. + """ + with patch.object(self.client, '_check_file_exists', + side_effect=exists if isinstance(exists, list) else None, + return_value=None if isinstance(exists, list) else exists) as checked, \ + patch.object(ssh.time, 'sleep') as slept: + self.client.download_file(remote_file_path='/remote/out.txt', + local_file_path=self.local_path) + return checked, slept + + def test_a_stale_local_file_is_emptied(self): + """Otherwise a previous job's out.txt is read back as this job's server output.""" + with open(self.local_path, 'w') as f: + f.write('slurmstepd: *** JOB 1 CANCELLED AT 2019-03-27 DUE TO TIME LIMIT ***\n') + self._download() + self.assertTrue(os.path.isfile(self.local_path)) + self.assertEqual(os.path.getsize(self.local_path), 0) + + def test_an_absent_local_file_is_created_empty(self): + """The base behavior, which ESS troubleshooting reads: an empty file, not no file.""" + self._download() + self.assertTrue(os.path.isfile(self.local_path)) + self.assertEqual(os.path.getsize(self.local_path), 0) + + def test_no_download_is_attempted(self): + """Emptying the local file must not cost a pointless SFTP round trip.""" + self._download() + self.client._sftp.get.assert_not_called() + + def test_the_miss_is_reported_at_the_warning_level(self): + """A job that produced no stdout at all leaves this log line as its only trace.""" + with self.assertLogs('arc', level='WARNING') as logged: + self._download() + self.assertEqual(len(logged.output), 1) + self.assertIn('/remote/out.txt', logged.output[0]) + self.assertIn('srv', logged.output[0]) + self.assertIn(self.local_path, logged.output[0]) + + def test_the_existence_check_is_retried_three_times(self): + """Scheduler epilogues can flush stdout a second or two after the job leaves the queue.""" + checked, slept = self._download() + self.assertEqual(checked.call_count, 3) + self.assertEqual(slept.call_count, 2) + self.assertEqual([call.args[0] for call in slept.call_args_list], [1.0, 1.0]) + + def test_a_file_that_appears_on_the_second_attempt_is_downloaded(self): + """The retry exists to download that file, not merely to delay the warning.""" + with open(self.local_path, 'w') as f: + f.write('an earlier download\n') + checked, slept = self._download(exists=[False, True]) + self.assertEqual(checked.call_count, 2) + self.assertEqual(slept.call_count, 1) + self.client._sftp.get.assert_called_once_with(remotepath='/remote/out.txt', + localpath=self.local_path) + self.assertEqual(os.path.getsize(self.local_path), len('an earlier download\n')) + + def test_an_unwritable_local_path_does_not_raise(self): + """Downloads are best-effort; a directory that is gone must not abort the job.""" + self.client._empty_local_file(os.path.join(self.tmp_dir, 'no_such_dir', 'out.txt')) + + class TestSSHClientConnect(unittest.TestCase): - """ - Contains unit tests for connecting to a server. - """ + """Connection trial counting and interval scoping in SSHClient.connect().""" + + SERVER = {'address': 'host.example.edu', 'un': 'user', 'key': '/dev/null', 'cluster_soft': 'PBS'} def setUp(self): """ @@ -62,10 +976,11 @@ def setUp(self): Count the connection trials instead of attempting to reach a server. """ self.trials, self.intervals = list(), list() - for patch in [mock.patch.object(SSHClient, '_connect', lambda ssh_client: self.fail_to_connect()), - mock.patch.object(ssh.time, 'sleep', lambda interval: self.intervals.append(interval))]: - patch.start() - self.addCleanup(patch.stop) + for patcher in [patch.object(ssh, 'servers', {'srv': self.SERVER}), + patch.object(ssh.SSHClient, '_connect', lambda ssh_client: self.fail_to_connect()), + patch.object(ssh.time, 'sleep', lambda interval: self.intervals.append(interval))]: + patcher.start() + self.addCleanup(patcher.stop) def fail_to_connect(self): """ @@ -79,7 +994,7 @@ def fail_to_connect(self): def test_connect_gives_up_after_a_single_requested_trial(self): """Test that a single connection trial is not followed by an interval, so teardown cannot block""" - ssh_client = SSHClient('server2', connection_attempts=1) + ssh_client = ssh.SSHClient('srv', connection_attempts=1) with self.assertRaises(ServerError): ssh_client.connect() self.assertEqual(len(self.trials), 1) @@ -87,7 +1002,7 @@ def test_connect_gives_up_after_a_single_requested_trial(self): def test_connect_does_not_wait_after_its_last_trial(self): """Test that connecting waits between trials, but not after the last one""" - ssh_client = SSHClient('server2', connection_attempts=3) + ssh_client = ssh.SSHClient('srv', connection_attempts=3) with self.assertRaises(ServerError): ssh_client.connect() self.assertEqual(len(self.trials), 3) @@ -95,13 +1010,23 @@ def test_connect_does_not_wait_after_its_last_trial(self): def test_connect_defaults_to_the_long_haul(self): """Test that the default number of connection trials, used while jobs are running, is unchanged""" - self.assertEqual(SSHClient('server2').connection_attempts, 1440) + self.assertEqual(ssh.SSHClient('srv').connection_attempts, 1440) + + def test_a_permanent_failure_raises_on_the_first_trial_however_many_are_allowed(self): + """Test that the retry budget does not resurrect retrying of a failure that retrying cannot fix""" + with patch.object(ssh.SSHClient, '_connect', + side_effect=paramiko.AuthenticationException('nope')) as connect: + ssh_client = ssh.SSHClient('srv', connection_attempts=1440) + with self.assertRaises(ServerError): + ssh_client.connect() + self.assertEqual(connect.call_count, 1) + self.assertEqual(self.intervals, list()) class TestDeleteCheckFilesOnServers(unittest.TestCase): - """ - Contains unit tests for deleting ESS checkfiles on the servers a project ran on. - """ + """Deleting ESS checkfiles on the servers a project ran on.""" + + SERVER = {'address': 'host.example.edu', 'un': 'user', 'key': '/dev/null', 'cluster_soft': 'PBS'} def setUp(self): """ @@ -110,16 +1035,18 @@ def setUp(self): to a server are actually executed, so that the real cleanup code path is exercised. """ self.remote_root = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.remote_root, ignore_errors=True) self.commands = list() - self.server = 'server2' + self.server = 'srv' self.project_path = os.path.join('runs', 'ARC_Projects', 'a_project') self.other_project_path = os.path.join('runs', 'ARC_Projects', 'an_unrelated_project') - for patch in [mock.patch.object(SSHClient, 'connect', lambda ssh_client: None), - mock.patch.object(SSHClient, '_send_command_to_server', - lambda ssh_client, command, remote_path='': - self.execute_on_fake_server(command, remote_path))]: - patch.start() - self.addCleanup(patch.stop) + for patcher in [patch.object(ssh, 'servers', {self.server: self.SERVER, 'local': self.SERVER}), + patch.object(ssh.SSHClient, 'connect', lambda ssh_client: None), + patch.object(ssh.SSHClient, '_send_command_to_server', + lambda ssh_client, command, remote_path='': + self.execute_on_fake_server(command, remote_path))]: + patcher.start() + self.addCleanup(patcher.stop) self.check_file = self.write_remote_file(self.project_path, 'spc1', 'opt_a1', 'check.chk') self.output_file = self.write_remote_file(self.project_path, 'spc1', 'opt_a1', 'input.log') self.other_check_file = self.write_remote_file(self.other_project_path, 'spc2', 'opt_a1', 'check.chk') @@ -159,19 +1086,21 @@ def write_remote_file(self, *args) -> str: def test_only_check_files_of_the_given_project_are_deleted(self): """Test that check files are deleted, and that nothing else on the server is""" - delete_check_files_on_servers({self.server: self.project_path}) + ssh.delete_check_files_on_servers({self.server: self.project_path}) self.assertFalse(os.path.isfile(self.check_file)) self.assertTrue(os.path.isfile(self.output_file)) self.assertTrue(os.path.isfile(self.other_check_file)) def test_a_local_server_is_skipped(self): """Test that a server named 'local' is skipped, its check files are deleted by the local cleanup""" - delete_check_files_on_servers({'local': self.project_path}) + ssh.delete_check_files_on_servers({'local': self.project_path}) + self.assertEqual(self.commands, list()) self.assertTrue(os.path.isfile(self.check_file)) def test_a_missing_remote_directory_is_a_no_op(self): """Test that a project directory which does not exist on the server is silently skipped""" - delete_check_files_on_servers({self.server: os.path.join('runs', 'ARC_Projects', 'no_such_project')}) + ssh.delete_check_files_on_servers({self.server: os.path.join('runs', 'ARC_Projects', 'no_such_project')}) + self.assertEqual([command for command in self.commands if command.startswith('find')], list()) self.assertTrue(os.path.isfile(self.check_file)) self.assertTrue(os.path.isfile(self.other_check_file)) @@ -179,9 +1108,10 @@ def test_a_path_with_shell_characters_is_not_interpreted(self): """Test that the remote path is quoted, so that its content cannot become part of the command""" wild_path = os.path.join('runs', 'ARC_Projects', 'a project $(touch injected.txt)') wild_check_file = self.write_remote_file(wild_path, 'spc1', 'opt_a1', 'check.chk') - delete_check_files_on_servers({self.server: wild_path}) + with patch.object(ssh.SSHClient, '_check_dir_exists', lambda ssh_client, remote_dir_path: True): + ssh.delete_check_files_on_servers({self.server: wild_path}) self.assertEqual(len(self.commands), 1) - self.assertEqual([token for token in shlex.split(self.commands[0]) if token == wild_path], [wild_path] * 2) + self.assertEqual(shlex.split(self.commands[0])[1], wild_path) self.assertFalse(os.path.isfile(wild_check_file)) self.assertFalse(os.path.isfile(os.path.join(self.remote_root, 'injected.txt'))) @@ -190,15 +1120,17 @@ def test_an_unreachable_server_is_only_logged(self): def raise_server_error(ssh_client): raise ServerError(f'Could not connect to server {ssh_client.server}.') - with mock.patch.object(SSHClient, 'connect', raise_server_error): - delete_check_files_on_servers({self.server: self.project_path}) # Must not raise. + with patch.object(ssh.SSHClient, 'connect', raise_server_error): + ssh.delete_check_files_on_servers({self.server: self.project_path}) self.assertTrue(os.path.isfile(self.check_file)) - def tearDown(self): - """ - A method that is run after each unit test in this class. - """ - shutil.rmtree(self.remote_root, ignore_errors=True) + def test_the_cleanup_connects_with_a_single_attempt(self): + """Test that the cleanup does not inherit the 24 hour retry, which would stall ARC's teardown""" + attempts = list() + with patch.object(ssh.SSHClient, 'connect', + lambda ssh_client: attempts.append(ssh_client.connection_attempts)): + ssh.delete_check_files_on_servers({self.server: self.project_path}) + self.assertEqual(attempts, [1]) if __name__ == '__main__': diff --git a/arc/main.py b/arc/main.py index 328054ef10..e79cb85532 100644 --- a/arc/main.py +++ b/arc/main.py @@ -31,7 +31,8 @@ from arc.imports import settings from arc.level import Level, assign_frequency_scale_factor from arc.job.factory import _registered_job_adapters -from arc.job.ssh import SSHClient, delete_check_files_on_servers +from arc.job.ssh import check_servers_known_hosts, delete_check_files_on_servers +from arc.job.ssh_pool import borrow_ssh_client, reset_default_pool from arc.output import write_output_yml from arc.processor import process_arc_project, resolve_neb_level from arc.reaction import ARCReaction @@ -318,6 +319,7 @@ def __init__(self, self.adaptive_levels = process_adaptive_levels(adaptive_levels) initialize_log(log_file=os.path.join(self.project_directory, 'arc.log'), project=self.project, project_directory=self.project_directory, verbose=self.verbose) + check_servers_known_hosts() self.dont_gen_confs = dont_gen_confs or list() self.t0 = time.time() # init time self.execution_time = None @@ -395,7 +397,8 @@ def __init__(self, if self.adaptive_levels is not None: logger.info(f'Using the following adaptive levels of theory:\n{self.adaptive_levels}') - self.ess_settings = check_ess_settings(ess_settings or global_ess_settings) + self.ess_settings = check_ess_settings(ess_settings or global_ess_settings, + ts_adapters=self.ts_adapters) if not self.ess_settings: # Use the "radar" feature if ess_settings are still unavailable. self.determine_ess_settings() @@ -558,7 +561,24 @@ def write_input_file(self, path=None): def execute(self) -> dict: """ - Execute ARC. + Execute ARC, releasing the SSH connections the run opened once it ends. + + The pooled connections (:mod:`arc.job.ssh_pool`) are held open for the lifetime of the + run and closed here, in a ``finally``, so they are also released when the run raises or + is interrupted, and so a consumer that drives ARC in-process rather than through + ``ARC.py`` -- a library caller, a test, a pipe worker -- releases them too. + + Returns: dict + Status dictionary indicating which species converged successfully. + """ + try: + return self._execute() + finally: + reset_default_pool() + + def _execute(self) -> dict: + """ + Run the project: schedule every job, process the results and write the output. Returns: dict Status dictionary indicating which species converged successfully. @@ -781,7 +801,7 @@ def determine_ess_settings(self, diagnostics=False): if `diagnostics` is True, this method will not raise errors, and will print its findings. """ if self.ess_settings and not diagnostics: - self.ess_settings = check_ess_settings(self.ess_settings) + self.ess_settings = check_ess_settings(self.ess_settings, ts_adapters=self.ts_adapters) return if diagnostics: @@ -832,7 +852,7 @@ def determine_ess_settings(self, diagnostics=False): continue if diagnostics: logger.info('\nTrying {0}'.format(server)) - with SSHClient(server) as ssh: + with borrow_ssh_client(server) as ssh: g03 = ssh.find_package('g03') g09 = ssh.find_package('g09') diff --git a/arc/main_test.py b/arc/main_test.py index 5ca22d3e85..bf373d514d 100644 --- a/arc/main_test.py +++ b/arc/main_test.py @@ -11,7 +11,7 @@ import subprocess import tempfile import unittest -from unittest import mock +from unittest.mock import patch from arc.common import ARC_PATH, get_logger from arc.exceptions import InputError @@ -566,6 +566,82 @@ def tearDownClass(cls): shutil.rmtree(project_directory, ignore_errors=True) +class TestExecuteReleasesPooledConnections(unittest.TestCase): + """The SSH connections a run holds open must be released by the run, not by interpreter exit.""" + + @staticmethod + def _arc(): + """An ARC object without the project setup __init__ does, which this does not need.""" + return ARC.__new__(ARC) + + def test_the_pool_is_released_when_the_run_finishes(self): + """A consumer that never goes through ARC.py must still release its connections.""" + with patch.object(ARC, '_execute', return_value={'spc': 'converged'}), \ + patch('arc.main.reset_default_pool') as released: + status = self._arc().execute() + self.assertEqual(status, {'spc': 'converged'}) + released.assert_called_once() + + def test_the_pool_is_released_when_the_run_raises(self): + """An interrupted or failed run is exactly when connections would otherwise be left open.""" + with patch.object(ARC, '_execute', side_effect=ValueError('the run went wrong')), \ + patch('arc.main.reset_default_pool') as released: + self.assertRaises(ValueError, self._arc().execute) + released.assert_called_once() + + def test_the_pool_is_released_on_a_keyboard_interrupt(self): + """Ctrl-C is how a long run usually ends, and it is not an Exception.""" + with patch.object(ARC, '_execute', side_effect=KeyboardInterrupt), \ + patch('arc.main.reset_default_pool') as released: + self.assertRaises(KeyboardInterrupt, self._arc().execute) + released.assert_called_once() + + +class TestServerMappingBorrowsItsConnection(unittest.TestCase): + """The connection the ESS survey opens is the one the run's jobs then need.""" + + REMOTE = {'zeus': {'cluster_soft': 'PBS', 'address': 'z.example.edu', 'un': 'u'}} + + def _map_servers(self, found): + """Survey the remote servers with every find_package() answering ``found``.""" + arc_object = ARC.__new__(ARC) + arc_object.ess_settings = dict() + with patch('arc.main.servers', self.REMOTE), \ + patch('arc.main.borrow_ssh_client') as borrow: + borrow.return_value.__enter__.return_value.find_package.return_value = found + arc_object.determine_ess_settings() + return arc_object, borrow + + def test_one_connection_is_borrowed_per_server(self): + """The survey asks after five packages, and used to open one connection for all of them.""" + _, borrow = self._map_servers(found=[]) + borrow.assert_called_once_with('zeus') + + def test_the_borrowed_connection_is_released(self): + _, borrow = self._map_servers(found=[]) + borrow.return_value.__exit__.assert_called_once() + + def test_what_the_survey_finds_is_unchanged(self): + """Borrowing instead of opening must not change the answer the survey gives.""" + arc_object, _ = self._map_servers(found=['/usr/bin/g16']) + self.assertEqual(arc_object.ess_settings['gaussian'], ['zeus']) + self.assertEqual(arc_object.ess_settings['orca'], ['zeus']) + + +class ReachedTheCleanup(Exception): + """Raised to stop a run right after its check file cleanup, so the rest of the run is not needed.""" + + +class SchedulerStub(object): + """Stands in for a Scheduler that has finished running a project's jobs on a server.""" + + def __init__(self, remote_project_paths: dict): + self.remote_project_paths = remote_project_paths + self.output = dict() + self.species_dict = dict() + self.rxn_list = list() + + class TestCheckFileCleanup(unittest.TestCase): """ Contains unit tests for deleting ESS checkfiles when ARC terminates, both locally and on the servers. @@ -591,19 +667,19 @@ def setUp(self): A method that is run before each unit test in this class. Set up a fake remote server: a temporary directory in which the commands ARC would have sent to a server are actually executed, so that the real cleanup code path is exercised. + The server definition is pinned so that neither a user settings file nor a missing 'server2' + entry can change the remote path this test builds and cleans. """ self.remote_root = tempfile.mkdtemp() self.project_directory = os.path.join(tempfile.mkdtemp(), self.project) - # Pin the server definition so that neither a user settings file nor a missing 'server2' - # entry can change the remote path this test builds and cleans. - for patch in [mock.patch.dict('arc.job.adapter.servers', {self.server: self.server_settings}), - mock.patch.dict('arc.job.ssh.servers', {self.server: self.server_settings}), - mock.patch.object(SSHClient, 'connect', lambda ssh_client: None), - mock.patch.object(SSHClient, '_send_command_to_server', - lambda ssh_client, command, remote_path='': - self.send_command_to_fake_server(command, remote_path))]: - patch.start() - self.addCleanup(patch.stop) + for patcher in [patch.dict('arc.job.adapter.servers', {self.server: self.server_settings}), + patch.dict('arc.job.ssh.servers', {self.server: self.server_settings}), + patch.object(SSHClient, 'connect', lambda ssh_client: None), + patch.object(SSHClient, '_send_command_to_server', + lambda ssh_client, command, remote_path='': + self.send_command_to_fake_server(command, remote_path))]: + patcher.start() + self.addCleanup(patcher.stop) def send_command_to_fake_server(self, command: str, remote_path: str = '') -> tuple: """ @@ -678,12 +754,12 @@ def set_up_arc_and_check_files(self, keep_checks: bool) -> ARC: return arc0 def test_check_files_are_deleted_locally_and_remotely(self): - """Test that check files are deleted on the server as well as locally when keep_checks is False""" + """Test that check files are deleted on the server as well as locally when keep_checks is False, + and that only check files, and only those under this project's own remote directory, are deleted""" arc0 = self.set_up_arc_and_check_files(keep_checks=False) arc0.clean_check_files(remote_project_paths=self.remote_project_paths) self.assertFalse(os.path.isfile(self.local_check_path)) self.assertFalse(os.path.isfile(self.remote_check_path)) - # Only check files, and only under this project's own remote directory, are deleted: self.assertTrue(os.path.isfile(self.remote_output_path)) self.assertTrue(os.path.isfile(self.other_project_check_path)) @@ -703,6 +779,19 @@ def test_check_files_are_deleted_locally_when_no_server_was_used(self): self.assertFalse(os.path.isfile(self.local_check_path)) self.assertTrue(os.path.isfile(self.remote_check_path)) + def test_a_run_reaches_the_remote_cleanup_with_the_scheduler_remote_paths(self): + """Test that executing a project actually deletes the server's check files, the cleanup is wired""" + arc0 = self.set_up_arc_and_check_files(keep_checks=False) + scheduler = SchedulerStub(remote_project_paths=self.remote_project_paths) + with patch('arc.main.Scheduler', return_value=scheduler), \ + patch.object(ARC, 'delete_leftovers', side_effect=ReachedTheCleanup): + with self.assertRaises(ReachedTheCleanup): + arc0.execute() + self.assertFalse(os.path.isfile(self.local_check_path)) + self.assertFalse(os.path.isfile(self.remote_check_path)) + self.assertTrue(os.path.isfile(self.remote_output_path)) + self.assertTrue(os.path.isfile(self.other_project_check_path)) + def tearDown(self): """ A method that is run after each unit test in this class. diff --git a/arc/settings/settings.py b/arc/settings/settings.py index 83328319c2..198b5b7580 100644 --- a/arc/settings/settings.py +++ b/arc/settings/settings.py @@ -21,6 +21,12 @@ # Users should update the following server dictionary. # Instructions for RSA key generation can be found here: # https://www.digitalocean.com/community/tutorials/how-to-set-up-ssh-keys--2 +# 'key' is the path to the SSH *private* key to authenticate with, and is optional: +# omit it to authenticate via a running ssh-agent (including a forwarded one) or via the +# default key paths '~/.ssh/id_rsa', '~/.ssh/id_ecdsa' and '~/.ssh/id_ed25519'. +# ARC does not read '~/.ssh/config', so IdentityFile/ProxyJump/ProxyCommand have no effect. +# Set 'strict_host_key_checking': True on a server to refuse hosts absent from known_hosts +# instead of only warning about them. # If ARC is being executed on a server, and ESS are available on that server, define a server named 'local', # for which only the cluster software and username are required. # servers = { @@ -28,7 +34,6 @@ # 'cluster_soft': 'OGE', # Oracle Grid Engine (Sun Grin Engine) # 'address': 'pharos.mit.edu', # 'un': '', -# 'key': '/home//.ssh/known_hosts', # }, # 'rmg': { # 'cluster_soft': 'Slurm', # Simple Linux Utility for Resource Management @@ -46,6 +51,7 @@ 'server1': { 'cluster_soft': 'OGE', 'address': 'server1.host.edu', + 'path': '/home', # an absolute path on the server holding the user directories; ARC runs under //runs/ARC_Projects/ 'un': '', 'key': 'path_to_rsa_key', 'max_simultaneous_jobs': 10, # optional, "check_status_command" must be set to only return jobs for your user @@ -53,6 +59,7 @@ 'server2': { 'cluster_soft': 'Slurm', 'address': 'server2.host.edu', + 'path': '/home', # an absolute path on the server holding the user directories; ARC runs under //runs/ARC_Projects/ 'un': '', 'key': 'path_to_rsa_key', 'cpus': 24, # number of cpu's per node, optional (default: 8) @@ -61,6 +68,7 @@ 'server3': { 'cluster_soft': 'PBS', 'address': 'server3.host.edu', + 'path': '/home', # an absolute path on the server holding the user directories; ARC runs under //runs/ARC_Projects/ 'un': '', 'key': 'path_to_rsa_key', }, From 09b36728034c8f11a888893322f353a904fd3a4f Mon Sep 17 00:00:00 2001 From: Calvin Pieters Date: Sat, 22 Aug 2026 13:08:58 +0300 Subject: [PATCH 3/9] Route the remaining SSH callers through the connection pool The pool was built and tested but the highest-frequency caller never used it. Scheduler.get_server_job_ids() opens a connection per server per poll cycle, for every cycle of every job's lifetime -- on a run of any length that is the dominant source of connections by an order of magnitude, and it is exactly the traffic shape a per-user connection limit is there to stop. It borrows now, so a run's polling costs one connection per server rather than one per poll. The same for the three sites in trsh_job_on_server() and for CFour's execute_queue(). CFour overrides execute_queue() rather than calling JobAdapter.legacy_queue_execution(), so it did not inherit the sharing the other adapters got; it goes through _open_or_borrow_ssh(), which means its submission also reuses the client its upload just used. One of those trsh sites leaked. `ssh = SSHClient(server)` with no `with` and no close() left a connection open for the rest of the process every time a job was troubleshooted by changing node. It never actually reached the server, since check_connections() raised TypeError on an unconnected client (fixed with the rest of the SSH work), but the leak is real for any caller that got past it. Not routed: delete_all_arc_jobs() in arc/job/ssh.py. Its only caller is arc/utils/delete.py, a standalone command-line utility that deletes jobs and exits, outside any ARC run; it opens no connection ARC would otherwise reuse and its `with` already closes what it opens, so pooling would swap a closed connection for one left open until the interpreter exits. ssh.py is also the module ssh_pool.py imports, so pooling there would have to be a function-local import to avoid a cycle -- a cost with nothing bought. Absorbed from PR #1000 by @alongd, brought in here rather than merged so the two pull requests do not conflict over the same lines: the scheduler records the remote project path of each server it spawns a job on, which is what the end of a run hands to the check file cleanup. Fix logger.denug in the unknown-cluster-software branch of trsh_job_on_server, which raised AttributeError instead of declining to troubleshoot. --- arc/job/adapters/cfour.py | 3 +- arc/job/adapters/cfour_test.py | 51 ++++++++++++++++++ arc/job/trsh.py | 16 +++--- arc/job/trsh_test.py | 95 ++++++++++++++++++++++++++++++++++ arc/scheduler.py | 4 +- arc/scheduler_test.py | 87 +++++++++++++++++++++++++++++++ 6 files changed, 244 insertions(+), 12 deletions(-) diff --git a/arc/job/adapters/cfour.py b/arc/job/adapters/cfour.py index a704f23501..62d3a07cc5 100644 --- a/arc/job/adapters/cfour.py +++ b/arc/job/adapters/cfour.py @@ -23,7 +23,6 @@ ) from arc.job.factory import register_job_adapter from arc.job.local import execute_command, submit_job -from arc.job.ssh import SSHClient from arc.level import Level from arc.species.converter import zmat_from_xyz, zmat_to_str @@ -302,7 +301,7 @@ def execute_queue(self): """ self._log_job_execution() if self.server != 'local': - with SSHClient(self.server) as ssh: + with self._open_or_borrow_ssh() as ssh: self.job_status[0], self.job_id = ssh.submit_job(remote_path=self.remote_path) else: self.job_status[0], self.job_id = submit_job(path=self.local_path) diff --git a/arc/job/adapters/cfour_test.py b/arc/job/adapters/cfour_test.py index d2bcde5b0a..c552960b57 100644 --- a/arc/job/adapters/cfour_test.py +++ b/arc/job/adapters/cfour_test.py @@ -9,6 +9,8 @@ import os import shutil import unittest +from types import SimpleNamespace +from unittest.mock import MagicMock, patch from arc.common import ARC_TESTING_PATH from arc.job.adapters.cfour import CFourAdapter @@ -147,5 +149,54 @@ def tearDownClass(cls): shutil.rmtree(os.path.join(ARC_TESTING_PATH, 'test_CFourAdapter'), ignore_errors=True) +class TestCFourQueueSubmissionSharesAConnection(unittest.TestCase): + """CFour overrides execute_queue(), so it needs the shared borrow path of its own accord.""" + + @staticmethod + def _adapter(server): + """ + A stand-in carrying only what ``execute_queue()`` reads, with the borrow path mocked. + + A real adapter is not built here because constructing one runs the whole adapter + initialization for a test about which connection the submission uses. Naming the + attributes explicitly also states what ``execute_queue()`` is allowed to touch, so a + method that starts reading something else fails here rather than passing on a mock + that answers to anything. + """ + return SimpleNamespace(server=server, + remote_path='/remote/job', + local_path='/local/job', + job_status=['initializing', {'status': 'initializing'}], + job_id=0, + _log_job_execution=lambda: None, + _open_or_borrow_ssh=MagicMock()) + + def test_a_remote_submission_borrows_its_connection(self): + """Submitting used to open a second connection right after the upload used one.""" + adapter = self._adapter('zeus') + ssh = MagicMock() + ssh.submit_job.return_value = ('running', 4242) + adapter._open_or_borrow_ssh.return_value.__enter__.return_value = ssh + CFourAdapter.execute_queue(adapter) + adapter._open_or_borrow_ssh.assert_called_once() + ssh.submit_job.assert_called_once_with(remote_path='/remote/job') + self.assertEqual((adapter.job_status[0], adapter.job_id), ('running', 4242)) + + def test_the_borrowed_connection_is_released(self): + adapter = self._adapter('zeus') + adapter._open_or_borrow_ssh.return_value.__enter__.return_value.submit_job.return_value = \ + ('running', 1) + CFourAdapter.execute_queue(adapter) + adapter._open_or_borrow_ssh.return_value.__exit__.assert_called_once() + + def test_a_local_submission_opens_no_connection(self): + """A local queue job is submitted with a local command and must not touch SSH.""" + adapter = self._adapter('local') + with patch('arc.job.adapters.cfour.submit_job', return_value=('running', 9)) as submitted: + CFourAdapter.execute_queue(adapter) + adapter._open_or_borrow_ssh.assert_not_called() + submitted.assert_called_once_with(path='/local/job') + + if __name__ == '__main__': unittest.main(testRunner=unittest.TextTestRunner(verbosity=2)) diff --git a/arc/job/trsh.py b/arc/job/trsh.py index db0a153f0f..777f6a5b1b 100644 --- a/arc/job/trsh.py +++ b/arc/job/trsh.py @@ -24,7 +24,7 @@ from arc.imports import settings from arc.level import Level from arc.job.local import execute_command -from arc.job.ssh import SSHClient +from arc.job.ssh_pool import borrow_ssh_client from arc.species import ARCSpecies from arc.species.conformers import determine_smallest_atom_index_in_scan from arc.species.converter import (displace_xyz, ics_to_scan_constraints) @@ -1481,13 +1481,13 @@ def trsh_job_on_server(server: str, execute_command(cmd) return None, True else: - with SSHClient(server) as ssh: + with borrow_ssh_client(server) as ssh: ssh.delete_job(job_id) # find available node logger.error('Troubleshooting by changing node.') - ssh = SSHClient(server) - nodes = ssh.list_available_nodes() + with borrow_ssh_client(server) as ssh: + nodes = ssh.list_available_nodes() for node in nodes: if node not in server_nodes: server_nodes.append(node) @@ -1500,7 +1500,7 @@ def trsh_job_on_server(server: str, # modify the submit file remote_submit_file = os.path.join(remote_path, submit_filenames[cluster_soft]) - with SSHClient(server) as ssh: + with borrow_ssh_client(server) as ssh: content = ssh.read_remote_file(remote_file_path=remote_submit_file) if cluster_soft.lower() == 'oge': node_assign = '#$ -l h=' @@ -1510,19 +1510,19 @@ def trsh_job_on_server(server: str, insert_line_num = 5 else: # Other software? - logger.denug(f'Unknown cluster software {cluster_soft} is encountered when ' + logger.debug(f'Unknown cluster software {cluster_soft} is encountered when ' f'troubleshooting by changing node.') return None, False for i, line in enumerate(content): if node_assign in line: content[i] = node_assign + node - break + break else: content.insert(insert_line_num, node_assign + node) content = ''.join(content) # convert list into a single string, not to upset paramiko # resubmit - with SSHClient(server) as ssh: + with borrow_ssh_client(server) as ssh: ssh.upload_file(remote_file_path=os.path.join(remote_path, submit_filenames[cluster_soft]), file_string=content) return node, True diff --git a/arc/job/trsh_test.py b/arc/job/trsh_test.py index 3f3a5fb7af..120a5b9ba1 100644 --- a/arc/job/trsh_test.py +++ b/arc/job/trsh_test.py @@ -1192,5 +1192,100 @@ def test_determine_ess_status_of_a_yaml_output(self): shutil.rmtree(tmp_dir, ignore_errors=True) +class TestTrshJobOnServerConnections(unittest.TestCase): + """Troubleshooting a server must not leave an SSH connection open behind it.""" + + def _trsh(self, nodes=None): + """Troubleshoot a job on a remote server, and return the borrow mock and the result.""" + client = patch.object(trsh, 'borrow_ssh_client') + borrow = client.start() + self.addCleanup(client.stop) + ssh = borrow.return_value.__enter__.return_value + ssh.list_available_nodes.return_value = nodes if nodes is not None else ['node01'] + ssh.read_remote_file.return_value = ['#!/bin/bash\n'] * 10 + result = trsh.trsh_job_on_server(server='server1', + job_name='opt_a103', + job_id=123, + job_server_status='errored', + remote_path='/home/u/runs/job', + server_nodes=list()) + return borrow, ssh, result + + def test_every_connection_is_borrowed_and_released(self): + """The node lookup used to open a client with no context manager, and never close it.""" + borrow, _, _ = self._trsh() + self.assertEqual(borrow.call_count, 4) + self.assertEqual(borrow.return_value.__exit__.call_count, borrow.call_count) + + def test_every_connection_is_for_the_troubleshooted_server(self): + borrow, _, _ = self._trsh() + self.assertEqual({call.args[0] for call in borrow.call_args_list}, {'server1'}) + + def test_the_node_lookup_still_picks_a_node(self): + """The leak fix must not change what troubleshooting decides.""" + _, ssh, result = self._trsh(nodes=['node01', 'node02']) + ssh.list_available_nodes.assert_called_once() + self.assertEqual(result, ('node01', True)) + + def test_an_unknown_cluster_software_is_reported_rather_than_raising(self): + """ + The report of an unknown cluster software was spelled ``logger.denug``, so reaching this + branch raised AttributeError instead of declining to troubleshoot. + """ + client = patch.object(trsh, 'borrow_ssh_client') + borrow = client.start() + self.addCleanup(client.stop) + ssh = borrow.return_value.__enter__.return_value + ssh.list_available_nodes.return_value = ['node01'] + ssh.read_remote_file.return_value = ['#!/bin/bash\n'] * 10 + server = dict(trsh.servers['server1'], cluster_soft='Cobalt') + with patch.dict(trsh.servers, {'server1': server}), \ + patch.dict(trsh.submit_filenames, {'Cobalt': 'submit.sh'}), \ + self.assertLogs(trsh.logger, level='DEBUG') as captured: + result = trsh.trsh_job_on_server(server='server1', + job_name='opt_a103', + job_id=123, + job_server_status='errored', + remote_path='/home/u/runs/job', + server_nodes=list()) + self.assertEqual(result, (None, False)) + self.assertIn('Unknown cluster software Cobalt', '\n'.join(captured.output)) + + def test_the_node_directive_is_inserted_when_the_submit_file_has_none(self): + """ + The loop that looks for an existing node directive broke on its first iteration + regardless of what that line held, so a submit file without one was uploaded unchanged. + """ + borrow, ssh, result = self._trsh() + uploaded = ssh.upload_file.call_args.kwargs['file_string'] + self.assertIn('#$ -l h=node01', uploaded) + self.assertEqual(result, ('node01', True)) + + def test_an_existing_node_directive_is_replaced_rather_than_added(self): + """A submit file that already names a node must come back naming the new one, once.""" + client = patch.object(trsh, 'borrow_ssh_client') + borrow = client.start() + self.addCleanup(client.stop) + ssh = borrow.return_value.__enter__.return_value + ssh.list_available_nodes.return_value = ['node02'] + ssh.read_remote_file.return_value = ['#!/bin/bash\n'] * 4 + ['#$ -l h=node01\n'] + ['#!/bin/bash\n'] * 5 + trsh.trsh_job_on_server(server='server1', + job_name='opt_a103', + job_id=123, + job_server_status='errored', + remote_path='/home/u/runs/job', + server_nodes=list()) + uploaded = ssh.upload_file.call_args.kwargs['file_string'] + self.assertEqual(uploaded.count('#$ -l h='), 1) + self.assertIn('#$ -l h=node02', uploaded) + + def test_no_node_available_gives_up_without_uploading(self): + """With nothing to switch to there is nothing to resubmit, and no third connection.""" + borrow, ssh, result = self._trsh(nodes=list()) + self.assertEqual(result, (None, False)) + ssh.upload_file.assert_not_called() + self.assertEqual(borrow.return_value.__exit__.call_count, borrow.call_count) + + if __name__ == "__main__": unittest.main(testRunner=unittest.TextTestRunner(verbosity=2)) diff --git a/arc/scheduler.py b/arc/scheduler.py index ecc17c53b9..f606976396 100644 --- a/arc/scheduler.py +++ b/arc/scheduler.py @@ -44,7 +44,7 @@ from arc.job.local import check_running_jobs_ids from arc.job.pipe.pipe_coordinator import PipeCoordinator from arc.job.pipe.pipe_planner import PipePlanner -from arc.job.ssh import SSHClient +from arc.job.ssh_pool import borrow_ssh_client from arc.job.trsh import (scan_quality_check, trsh_conformer_isomorphism, trsh_ess_job, @@ -3559,7 +3559,7 @@ def get_server_job_ids(self, specific_server: str | None = None): for server in self.servers: if specific_server is None or server == specific_server: if server != 'local': - with SSHClient(server) as ssh: + with borrow_ssh_client(server) as ssh: self.server_job_ids.extend(ssh.check_running_jobs_ids()) else: self.server_job_ids.extend(check_running_jobs_ids()) diff --git a/arc/scheduler_test.py b/arc/scheduler_test.py index dc18fad095..7803e010c2 100644 --- a/arc/scheduler_test.py +++ b/arc/scheduler_test.py @@ -410,6 +410,7 @@ def test_determine_adaptive_level(self): def test_initialize_output_dict(self): """Test Scheduler.initialize_output_dict""" + self.sched1.output['C2H6']['info'] = 'some text' self.assertTrue(self.sched1._does_output_dict_contain_info()) self.sched1.output = dict() self.assertEqual(self.sched1.output, dict()) @@ -2060,6 +2061,39 @@ def test_run_job_does_not_alias_level_args(self, mock_job_factory): args['keyword']['dft_grid'] = 'defgrid2' self.assertEqual(level.args, {'keyword': {'opt': 'opt=(verytight)'}, 'block': dict()}) + @patch('arc.scheduler.Scheduler.check_max_simultaneous_jobs_limit') + @patch('arc.scheduler.job_factory') + def test_run_job_records_the_remote_project_path_of_each_server(self, mock_job_factory, mock_limit): + """Test that run_job() records where the project lives on every server it spawns a job on.""" + project_directory = os.path.join(ARC_PATH, 'Projects', 'arc_project_run_job_remote_paths') + self.addCleanup(shutil.rmtree, project_directory, ignore_errors=True) + sched = Scheduler(project='test_run_job_remote_paths', ess_settings=self.ess_settings, + species_list=[ARCSpecies(label='C2H6', smiles='CC')], + opt_level=Level(repr=default_levels_of_theory['opt']), + freq_level=Level(repr=default_levels_of_theory['freq']), + sp_level=Level(repr=default_levels_of_theory['sp']), + ts_guess_level=Level(repr=default_levels_of_theory['ts_guesses']), + project_directory=project_directory, + testing=True, + job_types=self.job_types1, + ) + self.assertEqual(sched.remote_project_paths, dict()) + + for job_name, server, remote_project_path in [('opt_a0000', 'server1', 'runs/ARC_Projects/a_project'), + ('opt_a0001', 'server1', 'a_later_job_does_not_overwrite'), + ('opt_a0002', 'server2', 'runs/ARC_Projects/a_project'), + ('opt_a0003', 'local', None), + ('opt_a0004', None, 'no_server_is_not_recorded')]: + job_mock = MagicMock() + job_mock.job_name, job_mock.server = job_name, server + job_mock.remote_project_path = remote_project_path + mock_job_factory.return_value = job_mock + sched.run_job(label='C2H6', job_type='opt', + level_of_theory=Level(repr=default_levels_of_theory['opt']), job_adapter='gaussian') + + self.assertEqual(sched.remote_project_paths, {'server1': 'runs/ARC_Projects/a_project', + 'server2': 'runs/ARC_Projects/a_project'}) + @classmethod def tearDownClass(cls): """ @@ -2310,5 +2344,58 @@ def test_apply_adaptive_reaction_levels_label_collision(self): self.build_scheduler(rxn, r + p + [collider], 'adaptive_collision') +class TestGetServerJobIds(unittest.TestCase): + """The status poll runs every cycle for every job, so it is the hottest SSH caller there is.""" + + @staticmethod + def _sched(servers): + """A stand-in carrying only what get_server_job_ids() reads.""" + return SimpleNamespace(servers=servers, server_job_ids=None) + + def test_a_remote_server_is_polled_through_a_pooled_client(self): + """Opening a connection per poll is what the pool exists to stop.""" + sched = self._sched(['zeus']) + client = MagicMock() + client.check_running_jobs_ids.return_value = ['101', '102'] + with patch('arc.scheduler.borrow_ssh_client') as borrow: + borrow.return_value.__enter__.return_value = client + Scheduler.get_server_job_ids(sched) + borrow.assert_called_once_with('zeus') + self.assertEqual(sched.server_job_ids, ['101', '102']) + + def test_the_borrowed_client_is_released(self): + """A borrow that is not exited would hold the pool's client for the rest of the run.""" + sched = self._sched(['zeus']) + with patch('arc.scheduler.borrow_ssh_client') as borrow: + borrow.return_value.__enter__.return_value = MagicMock() + Scheduler.get_server_job_ids(sched) + borrow.return_value.__exit__.assert_called_once() + + def test_every_poll_of_one_server_goes_through_one_borrow(self): + """Each cycle borrows once per server, which is one pooled client for the whole run.""" + sched = self._sched(['zeus']) + with patch('arc.scheduler.borrow_ssh_client') as borrow: + borrow.return_value.__enter__.return_value = MagicMock() + for _ in range(50): + Scheduler.get_server_job_ids(sched) + self.assertEqual(borrow.call_count, 50) + + def test_a_local_server_is_not_polled_over_ssh(self): + """The local queue is read with a local command, and must not touch the pool.""" + sched = self._sched(['local']) + with patch('arc.scheduler.borrow_ssh_client') as borrow, \ + patch('arc.scheduler.check_running_jobs_ids', return_value=['7']): + Scheduler.get_server_job_ids(sched) + borrow.assert_not_called() + self.assertEqual(sched.server_job_ids, ['7']) + + def test_a_specific_server_limits_the_poll_to_it(self): + sched = self._sched(['zeus', 'atlas']) + with patch('arc.scheduler.borrow_ssh_client') as borrow: + borrow.return_value.__enter__.return_value = MagicMock() + Scheduler.get_server_job_ids(sched, specific_server='atlas') + borrow.assert_called_once_with('atlas') + + if __name__ == '__main__': unittest.main(testRunner=unittest.TextTestRunner(verbosity=2)) From cdfcd1789646de8d9b4375a01534bf1df20a4103 Mon Sep 17 00:00:00 2001 From: Calvin Pieters Date: Sat, 22 Aug 2026 13:08:58 +0300 Subject: [PATCH 4/9] Report a ~/.arc overlay that could not be imported arc/imports.py caught ImportError from the local settings.py, submit.py and inputs.py overlays and passed. An overlay that fails to load therefore leaves ARC running on the repository defaults with nothing said, and the defaults are a working configuration, so there is no other symptom: the cluster templates in submit.py or the server definitions in settings.py are simply not the ones the user wrote, and a run goes to the wrong place, or to ARC's dummy servers, for hours. The usual cause is an overlay that imports something not installed in the environment ARC is running in, which is easy to produce and invisible once produced. Report it instead. Control flow is unchanged -- the defaults still stand, the run still starts -- and the message names the file and the error, and is queued so it survives the log being initialized later in the run. Loudness follows what actually failed, since the two cases mean opposite things. A file that loaded but does not define the name is a partial overlay, which is the ordinary way to override one setting and leave the rest alone: a submit.py that defines submit_scripts and neither incore_commands nor pipe_submit is correct, and warning about it would put two lines in every run's log of every user who has one. That is a debug line. A file that did not load at all loses every setting in it, and is a warning, reported once per file rather than once per name imported from it. The two are told apart by whether the module is in sys.modules after the failure. A syntax error in an overlay is not covered, and cannot be: it is a SyntaxError, not an ImportError, and it propagates out of arc/imports.py and stops ARC from starting -- loudly, if confusingly, but never silently. --- arc/imports.py | 63 ++++++++++++++++++++++++++++++++++++++------- arc/imports_test.py | 59 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 111 insertions(+), 11 deletions(-) diff --git a/arc/imports.py b/arc/imports.py index 70022765ad..860ff6020b 100644 --- a/arc/imports.py +++ b/arc/imports.py @@ -56,6 +56,49 @@ def resolve_overridden_dependents(settings: dict, local_settings_dict: dict) -> queue_deferred_warning(msg) +_UNUSABLE_OVERLAYS_REPORTED = set() + + +def _report_unusable_overlay(path: str, module: str, error: ImportError, what: str = '') -> None: + """ + Report that a local ~/.arc overlay could not supply a setting, so ARC's default is used. + + An overlay that fails to load leaves ARC running on the repository defaults, which is a + working configuration and therefore produces no other symptom: the submit script or the + server definitions the user wrote are simply not the ones in use, and a run goes to the + wrong cluster or with the wrong resources. Naming the file and the error is what makes that + visible instead of silent. + + Whether the overlay file itself loaded decides how loud the report is, since the two cases + mean opposite things. A file that loaded but does not define the name is a partial overlay, + which is the ordinary way to override one setting and leave the rest alone, and is reported + at the debug level. A file that did not load at all, most often because something it imports + is not installed, loses every setting in it; that is reported once at the warning level, and + queued so it survives the log being initialized later in the run. + + Note that a syntax error in an overlay is not reported here and never reaches this function: + it is a ``SyntaxError`` rather than an ``ImportError``, and it propagates out of this module + and stops ARC from starting at all. + + Args: + path (str): The overlay file that could not be used. + module (str): The module name the overlay is imported under. + error (ImportError): The import failure. + what (str, optional): The name that could not be imported, when the file itself loaded. + """ + if module in sys.modules: + logger.debug(f'{path} does not define "{what}", so ARC\'s default is used. ' + f'Got {type(error).__name__}: {error}') + return + if path in _UNUSABLE_OVERLAYS_REPORTED: + return + _UNUSABLE_OVERLAYS_REPORTED.add(path) + msg = f'Could not import {path}, so none of the settings in it are used and ARC\'s defaults ' \ + f'are used instead. Got {type(error).__name__}: {error}' + logger.warning(msg) + queue_deferred_warning(msg) + + # Common imports where the user can optionally put a modified copy of settings.py or submit.py file under ~/.arc home = os.getenv("HOME") or os.path.expanduser("~") local_arc_path = os.path.join(home, '.arc') @@ -68,8 +111,8 @@ def resolve_overridden_dependents(settings: dict, local_settings_dict: dict) -> sys.path.insert(1, local_arc_path) try: import settings as local_settings - except ImportError: - pass + except ImportError as e: + _report_unusable_overlay(local_arc_settings_path, 'settings', e) if local_settings: local_settings_dict = {key: val for key, val in vars(local_settings).items() if '__' not in key} settings.update(local_settings_dict) @@ -85,16 +128,16 @@ def resolve_overridden_dependents(settings: dict, local_settings_dict: dict) -> sys.path.insert(1, local_arc_path) try: from submit import incore_commands as local_incore_commands - except ImportError: - pass + except ImportError as e: + _report_unusable_overlay(local_arc_submit_path, 'submit', e, 'incore_commands') try: from submit import pipe_submit as local_pipe_submit - except ImportError: - pass + except ImportError as e: + _report_unusable_overlay(local_arc_submit_path, 'submit', e, 'pipe_submit') try: from submit import submit_scripts as local_submit_scripts - except ImportError: - pass + except ImportError as e: + _report_unusable_overlay(local_arc_submit_path, 'submit', e, 'submit_scripts') if local_incore_commands: incore_commands.update(local_incore_commands) if local_pipe_submit: @@ -109,7 +152,7 @@ def resolve_overridden_dependents(settings: dict, local_settings_dict: dict) -> sys.path.insert(1, local_arc_path) try: from inputs import input_files as local_input_files - except ImportError: - pass + except ImportError as e: + _report_unusable_overlay(local_arc_inputs_path, 'inputs', e, 'input_files') if local_input_files: input_files.update(local_input_files) diff --git a/arc/imports_test.py b/arc/imports_test.py index 19b5597429..85e44a1e27 100644 --- a/arc/imports_test.py +++ b/arc/imports_test.py @@ -2,12 +2,13 @@ This module contains unit tests for the arc.imports module. """ +import logging import os import tempfile import unittest from unittest.mock import patch -from arc.imports import resolve_overridden_dependents +from arc.imports import _report_unusable_overlay, resolve_overridden_dependents from arc.settings import external_paths # The finders consult these before ``repo_path``, and CI exports two of them @@ -133,5 +134,61 @@ def test_env_var_override_outranks_the_re_derived_repo_path(self): self.assertEqual(settings['RITS_CKPT_PATH'], os.path.abspath(env_ckpt)) +class TestReportUnusableOverlay(unittest.TestCase): + """A ~/.arc overlay that cannot be used must say so, not leave ARC quietly on its defaults.""" + + OVERLAY = '/home/user/.arc/submit.py' + + def setUp(self): + """Report every overlay afresh, since the warning is emitted once per file per run.""" + reported = patch('arc.imports._UNUSABLE_OVERLAYS_REPORTED', set()) + reported.start() + self.addCleanup(reported.stop) + queued = patch('arc.imports.queue_deferred_warning') + self.queued = queued.start() + self.addCleanup(queued.stop) + + def _report(self, module, what='submit_scripts'): + """Report a failed overlay import, and return the records it logged. + + The levels are read as ``levelno`` rather than ``levelname`` because + ``arc.common.initialize_log`` renames the level names process-wide, so + ``levelname`` is 'WARNING' or 'Warning: ' depending on whether anything has + initialized ARC's log yet. + """ + error = ImportError(f'No module named {module!r}') + with self.assertLogs('arc', level='DEBUG') as captured: + _report_unusable_overlay(self.OVERLAY, module, error, what) + return captured.records + + def test_a_file_that_did_not_load_is_a_warning(self): + """Every setting in the file is lost, and nothing else in the run says so.""" + records = self._report('a_module_that_is_not_loaded') + self.assertEqual([record.levelno for record in records], [logging.WARNING]) + self.assertIn(self.OVERLAY, records[0].getMessage()) + self.assertIn('ImportError', records[0].getMessage()) + + def test_a_file_that_did_not_load_is_queued_for_the_log_file(self): + """The overlay is read before the log exists, so the warning has to survive until it does.""" + self._report('a_module_that_is_not_loaded') + self.queued.assert_called_once() + self.assertIn(self.OVERLAY, self.queued.call_args[0][0]) + + def test_a_file_that_did_not_load_is_reported_once(self): + """submit.py is imported from three times, and one broken file is one problem.""" + self._report('a_module_that_is_not_loaded') + with patch('arc.imports.logger.warning') as warning: + _report_unusable_overlay(self.OVERLAY, 'a_module_that_is_not_loaded', + ImportError('boom'), 'pipe_submit') + warning.assert_not_called() + + def test_a_loaded_file_missing_one_name_is_only_a_debug_line(self): + """Overriding one setting and not the rest is the ordinary way to use an overlay.""" + records = self._report('unittest') + self.assertEqual([record.levelno for record in records], [logging.DEBUG]) + self.assertIn('submit_scripts', records[0].getMessage()) + self.queued.assert_not_called() + + if __name__ == '__main__': unittest.main(testRunner=unittest.TextTestRunner(verbosity=2)) From 27e25e667b0ee836c40956cc7e218c7d37970cd5 Mon Sep 17 00:00:00 2001 From: Calvin Pieters Date: Sat, 22 Aug 2026 13:08:58 +0300 Subject: [PATCH 5/9] pipe: refuse remote servers instead of deadlocking PipeRun.submit_to_scheduler() invokes qsub/sbatch on the machine running ARC, and the worker (python -m arc.scripts.pipe_worker) reads pipe_root from its local filesystem. When the engine's resolved server is remote, that submission errors silently and the run deadlocks waiting for results that can never arrive. Make should_use_pipe() refuse a non-local server so the planner falls back to per-job queue submission over SSH, and say in the log which engine and server triggered the refusal and what is being used instead -- that fallback is slower than a pipe run, so without the message the only symptom is an unexplained slowdown. Supporting pipe on a remote server needs it rebuilt around batch jobs staged on the remote side, which is out of scope here. The guard resolved its server with `next((s for s in server_list if s in servers_dict), None)`, which fails open in three ways: it skips an entry that names an unconfigured server and silently judges the next one instead, it compares server names case-sensitively when a server name is a settings key whose casing the user chose, and it permits the pipe when nothing resolves at all. That last one matters most, because derive_cluster_software() applies the same "skip what is not configured" rule and then falls back to guessing slurm, so an unresolvable server produced a pipe submitted with a guessed template. Resolve the first entry unconditionally, compare case-insensitively, and refuse unless the result is a configured server that is this machine. Refusing costs the run only the bundling -- the planner submits the tasks as individual queue jobs, which works for a local and a remote server alike -- so failing closed here is cheap and failing open is not. "Cannot be resolved" is not the same as "has no server", and conflating the two would have disabled TSG pipe mode outright. A TS-guess batch carries engine=, and gcn, kinbot, xtb_gsm and the rest are not ESS: they are absent from ess_settings by design and run in this process, which is why _initialize_adapter resolves a server only for an engine ess_settings names and leaves every other one with server=None, and why set_file_paths gives such a job no remote path at all. In process is this machine, so those tasks pipe. The refusal is for an engine that ess_settings does name and that still does not resolve to a configured local server -- an ESS declared and available nowhere, or named on a server that is not configured. The resolution itself is not a second implementation. _initialize_adapter() already decided which server a job goes to, inline: a trsh override first, then the first entry of the ESS settings for the adapter, with a bare string read as a single server. That is now resolve_job_server() in arc/job/adapters/common.py, the module that owns the concept, called by both, so the pipe's answer is the answer the job would have got rather than a lookalike. Extracting it also fixes an IndexError on an empty server list, and drops a redundant re-check of a condition the enclosing `if` had already established. Absorbed from PR #1000 by @alongd, brought in here rather than merged so the two pull requests do not conflict over the same lines: _initialize_adapter() initializes the new remote_project_path attribute. --- arc/job/adapters/common.py | 44 ++++++++++++++--- arc/job/adapters/common_test.py | 70 ++++++++++++++++++++------- arc/job/pipe/pipe_coordinator.py | 66 ++++++++++++++++++++++++- arc/job/pipe/pipe_coordinator_test.py | 59 +++++++++++++++++++++- arc/job/pipe/pipe_planner_test.py | 4 +- arc/scheduler_pipe_test.py | 4 +- 6 files changed, 219 insertions(+), 28 deletions(-) diff --git a/arc/job/adapters/common.py b/arc/job/adapters/common.py index a50954a732..43d567cddc 100644 --- a/arc/job/adapters/common.py +++ b/arc/job/adapters/common.py @@ -258,13 +258,9 @@ def _initialize_adapter(obj: JobAdapter, obj.args = set_job_args(args=obj.args, level=obj.level, job_name=obj.job_name) if obj.execution_type != 'incore' and obj.job_adapter in obj.ess_settings.keys() and obj.server is None: - if 'server' in obj.args['trsh']: - obj.server = obj.args['trsh']['server'] - elif obj.job_adapter in obj.ess_settings.keys(): - if isinstance(obj.ess_settings[obj.job_adapter], list): - obj.server = obj.ess_settings[obj.job_adapter][0] - else: - obj.server = obj.ess_settings[obj.job_adapter] + obj.server = resolve_job_server(ess_settings=obj.ess_settings, + job_adapter=obj.job_adapter, + args=obj.args) obj.set_file_paths() obj.set_cpu_and_mem() @@ -281,6 +277,40 @@ def _initialize_adapter(obj: JobAdapter, check_argument_consistency(obj) +def resolve_job_server(ess_settings: dict, + job_adapter: str, + args: dict | None = None, + ) -> str | None: + """ + Return the server that a ``job_adapter`` job will be submitted to. + + A troubleshooting override in ``args['trsh']['server']`` wins, since it is set precisely to + move a job off the server that failed it, and it is honoured even when it is empty so that + the caller sees the same server the job itself would be given. Otherwise the server is the + first one the ESS settings name for the adapter, which is the entry ARC submits to; a bare + string there is read as a single server. + + Args: + ess_settings (dict): The ESS settings, mapping an adapter to the server or the list of + servers it is available on. + job_adapter (str): The job adapter to resolve a server for. + args (dict, optional): The job's arguments, whose ``'trsh'`` entry may carry a + ``'server'`` override. + + Returns: str | None + The server name, or ``None`` when neither an override nor the ESS settings name one. + """ + trsh_args = (args or dict()).get('trsh') or dict() + if isinstance(trsh_args, dict) and 'server' in trsh_args: + return trsh_args['server'] + servers_for_adapter = (ess_settings or dict()).get(job_adapter) + if isinstance(servers_for_adapter, str): + return servers_for_adapter or None + if isinstance(servers_for_adapter, (list, tuple)) and len(servers_for_adapter): + return servers_for_adapter[0] + return None + + def is_restricted(obj: JobAdapter) -> bool | list[bool]: """ Check whether a Job Adapter should be executed as restricted or unrestricted. diff --git a/arc/job/adapters/common_test.py b/arc/job/adapters/common_test.py index abab9432db..56176b16bd 100644 --- a/arc/job/adapters/common_test.py +++ b/arc/job/adapters/common_test.py @@ -5,12 +5,11 @@ This module contains unit tests of the arc.job.adapters.common module """ -import os import shutil +import tempfile import unittest import arc.job.adapters.common as common -from arc.common import ARC_TESTING_PATH from arc.job.adapters.gaussian import GaussianAdapter from arc.job.adapters.molpro import MolproAdapter from arc.level import Level @@ -27,11 +26,13 @@ def setUpClass(cls): A method that is run before all unit tests in this class. """ cls.maxDiff = None + cls.project_directory = tempfile.mkdtemp(prefix='test_JobAdaptersCommon_') + cls.addClassCleanup(shutil.rmtree, cls.project_directory, ignore_errors=True) cls.job_1 = GaussianAdapter(execution_type='incore', job_type='composite', level=Level(method='cbs-qb3-paraskevas'), project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[ARCSpecies(label='spc1', xyz=['O 0 0 1'], multiplicity=1)], testing=True, args={'keyword': {'general': 'IOp(1/12=5,3/44=0)'}}, @@ -41,7 +42,7 @@ def setUpClass(cls): torsions=[[1, 2, 3, 4]], level=Level(method='wb97xd', basis='def2tzvp'), project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[ARCSpecies(label='spc1', xyz=['O 0 0 1'], multiplicity=3)], testing=True, args={'keyword': {'general': 'IOp(1/12=5,3/44=0)'}}, @@ -51,7 +52,7 @@ def setUpClass(cls): torsions=[[1, 2, 3, 4]], level=Level(method='wb97xd', basis='def2tzvp'), project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[ARCSpecies(label='spc1', xyz=['O 0 0 1'], multiplicity=1, number_of_radicals=2)], testing=True, args={'keyword': {'general': 'IOp(1/12=5,3/44=0)'}}, @@ -61,7 +62,7 @@ def setUpClass(cls): torsions=[[1, 2, 3, 4]], level=Level(method='wb97xd', basis='def2tzvp'), project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[ARCSpecies(label='spc1', xyz=['O 0 0 1'], multiplicity=1, number_of_radicals=2, multi_species='mltspc1'), ARCSpecies(label='spc1', xyz=['O 0 0 1'], multiplicity=1, number_of_radicals=1, multi_species='mltspc1')], testing=True, @@ -86,7 +87,7 @@ def test_check_argument_consistency(self): job_type='irc', level=Level(method='ccsd(t)', basis='cc-pvtz'), project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_MolproAdapter'), + project_directory=self.project_directory, species=[ARCSpecies(label='spc1', xyz=['O 0 0 1'], multiplicity=1)], testing=True, ) @@ -95,7 +96,7 @@ def test_check_argument_consistency(self): job_type='irc', level=Level(method='b3lyp', basis='def2svp'), project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=self.project_directory, species=[ARCSpecies(label='spc1', xyz=['O 0 0 1'], multiplicity=1)], testing=True, args={'keyword': {'general': 'IOp(1/12=5,3/44=0)'}}, @@ -109,7 +110,7 @@ def test_check_argument_consistency(self): torsions=[[1, 2, 3, 4]], level=Level(method='ccsd(t)', basis='cc-pvtz'), project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_MolproAdapter'), + project_directory=self.project_directory, species=[spc], testing=True, ) @@ -335,14 +336,49 @@ def test_input_dict_strip(self): stripped_dict = common.input_dict_strip(input_dict) self.assertEqual(stripped_dict, expected_stripped_dict) - @classmethod - def tearDownClass(cls): - """ - A function that is run ONCE after all unit tests in this class. - Delete all project directories created during these unit tests - """ - shutil.rmtree(os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), ignore_errors=True) - shutil.rmtree(os.path.join(ARC_TESTING_PATH, 'test_MolproAdapter'), ignore_errors=True) + +class TestResolveJobServer(unittest.TestCase): + """The one answer to "which server will this job go to", shared by the adapters and the pipe.""" + + def test_the_first_server_listed_is_the_one_used(self): + """ARC submits to the first entry, so anything else would describe a different job.""" + self.assertEqual(common.resolve_job_server({'gaussian': ['zeus', 'atlas']}, 'gaussian'), + 'zeus') + + def test_a_bare_string_is_read_as_a_single_server(self): + self.assertEqual(common.resolve_job_server({'gaussian': 'zeus'}, 'gaussian'), 'zeus') + + def test_an_unlisted_adapter_resolves_to_nothing(self): + self.assertIsNone(common.resolve_job_server({'gaussian': ['zeus']}, 'orca')) + + def test_an_empty_server_list_resolves_to_nothing(self): + """The open-coded version indexed [0] here and raised IndexError.""" + self.assertIsNone(common.resolve_job_server({'gaussian': []}, 'gaussian')) + + def test_empty_ess_settings_resolve_to_nothing(self): + self.assertIsNone(common.resolve_job_server(dict(), 'gaussian')) + self.assertIsNone(common.resolve_job_server(None, 'gaussian')) + + def test_a_troubleshooting_override_wins(self): + """The override exists to move a job off the server that failed it.""" + self.assertEqual(common.resolve_job_server({'gaussian': ['zeus']}, 'gaussian', + args={'trsh': {'server': 'atlas'}}), + 'atlas') + + def test_an_override_is_honoured_even_when_it_is_empty(self): + """The caller must see the same server the job itself would be given.""" + self.assertIsNone(common.resolve_job_server({'gaussian': ['zeus']}, 'gaussian', + args={'trsh': {'server': None}})) + + def test_trsh_args_without_a_server_fall_through(self): + self.assertEqual(common.resolve_job_server({'gaussian': ['zeus']}, 'gaussian', + args={'trsh': {'scan_res': 8}}), + 'zeus') + + def test_args_without_a_trsh_entry_fall_through(self): + self.assertEqual(common.resolve_job_server({'gaussian': ['zeus']}, 'gaussian', + args={'keyword': {}}), + 'zeus') if __name__ == '__main__': diff --git a/arc/job/pipe/pipe_coordinator.py b/arc/job/pipe/pipe_coordinator.py index c3fb4691ed..364e5d771a 100644 --- a/arc/job/pipe/pipe_coordinator.py +++ b/arc/job/pipe/pipe_coordinator.py @@ -15,6 +15,7 @@ import arc.parser.parser as parser from arc.common import get_logger from arc.imports import settings +from arc.job.adapters.common import resolve_job_server from arc.level import Level from arc.job.pipe.pipe_run import ( @@ -87,7 +88,9 @@ def should_use_pipe(self, tasks: list[TaskSpec]) -> bool: Returns ``True`` only if: 1. Pipe mode is enabled. 2. There are at least ``min_tasks`` tasks. - 3. All tasks are homogeneous in engine, task_family, owner_type, + 3. The tasks' server resolves to this machine + (:meth:`_pipe_server_is_this_machine`). + 4. All tasks are homogeneous in engine, task_family, owner_type, level, required_cores, and required_memory_mb. """ if not pipe_settings.get('enabled', True): @@ -97,6 +100,8 @@ def should_use_pipe(self, tasks: list[TaskSpec]) -> bool: min_tasks = pipe_settings.get('min_tasks', 10) if len(tasks) < min_tasks: return False + if not self._pipe_server_is_this_machine(tasks[0]): + return False ref = tasks[0] return all(t.engine == ref.engine and t.task_family == ref.task_family @@ -106,6 +111,65 @@ def should_use_pipe(self, tasks: list[TaskSpec]) -> bool: and t.required_memory_mb == ref.required_memory_mb for t in tasks[1:]) + def _pipe_server_is_this_machine(self, task: TaskSpec) -> bool: + """ + Determine whether a task's server is the machine ARC is running on. + + ``PipeRun.submit_to_scheduler`` invokes qsub/sbatch on the orchestrator machine, and the + worker (``python -m arc.scripts.pipe_worker``) reads ``pipe_root`` from the local + filesystem. A pipe therefore only works when the tasks would have gone to the local + server; sent anywhere else the submission errors and the run deadlocks waiting for + results that are never produced. + + An engine the ESS settings do not name at all is not sent to a server. It is a TS-guess + method or another adapter that runs in this process, which is why + :func:`arc.job.adapters.common._initialize_adapter` resolves a server only for an engine + the ESS settings do name and leaves every other one with none. In process is this + machine, so those tasks may be piped. + + An engine the ESS settings do name is resolved the same way the job adapter resolves its + own (:func:`arc.job.adapters.common.resolve_job_server`), so the answer is about the + server the tasks would actually be sent to. It is compared case-insensitively, because a + server name is a settings key the user chose the casing of, while ``'local'`` is spelled + in lower case throughout ARC. + + Such an engine that resolves to nothing, or to a server that is not configured, is not + known to run on this machine and so refuses the pipe. Refusing costs the run only the + bundling: the planner falls back to submitting the tasks as individual queue jobs, which + works for a local and for a remote server alike. + + Args: + task (TaskSpec): The task whose server decides the whole bundle's, the bundle being + homogeneous in engine. + + Returns: bool + Whether the pipe may be used for this task's server. + """ + ess_settings = getattr(self.sched, 'ess_settings', None) or dict() + if task.engine not in ess_settings: + return True + server = resolve_job_server(ess_settings=ess_settings, + job_adapter=task.engine, + args=getattr(task, 'args', None)) + fallback = 'Falling back to per-job queue submission.' + if not server: + logger.info(f'Not using the pipe for {task.engine} jobs: the ESS settings name ' + f'{task.engine} but no server for it, so the tasks cannot be shown to ' + f'run on this machine, which is the only place a pipe worker can read ' + f'its payload from. {fallback}') + return False + if server.casefold() not in {name.casefold() for name in settings['servers']}: + logger.info(f'Not using the pipe for {task.engine} jobs: the resolved server ' + f'{server!r} is not among the configured servers, so it cannot be shown ' + f'to be this machine. {fallback}') + return False + if server.casefold() != 'local': + logger.info(f'Not using the pipe for {task.engine} jobs: server {server!r} is ' + f'remote, and the pipe worker reads its job payload from the ' + f'orchestrator\'s local filesystem. {fallback}') + return False + return True + def _compute_pipe_root(self, run_id: str, tasks: list[TaskSpec]) -> str: """ Compute the pipe_root path under ``calcs/``, following ARC's directory convention. diff --git a/arc/job/pipe/pipe_coordinator_test.py b/arc/job/pipe/pipe_coordinator_test.py index 505ad71014..8998c44676 100644 --- a/arc/job/pipe/pipe_coordinator_test.py +++ b/arc/job/pipe/pipe_coordinator_test.py @@ -67,11 +67,12 @@ def _make_spec(task_id, task_family='conf_opt', engine='mockter', level=None, ) -def _make_mock_sched(project_directory): +def _make_mock_sched(project_directory, ess_settings=None): """Create a mock Scheduler with the attributes PipeCoordinator needs.""" sched = MagicMock() sched.project_directory = project_directory sched.server_job_ids = list() + sched.ess_settings = ess_settings if ess_settings is not None else {'mockter': ['local']} spc = ARCSpecies(label='H2O', smiles='O') spc.conformers = [None] * 5 spc.conformer_energies = [None] * 5 @@ -131,6 +132,62 @@ def test_false_when_disabled(self): tasks = [_make_spec(f't_{i}') for i in range(15)] self.assertFalse(self.coord.should_use_pipe(tasks)) + @patch('arc.job.pipe.pipe_coordinator.settings', + {'servers': {'zeus': {'cluster_soft': 'PBS', 'address': 'z.example.edu', 'un': 'u'}}}) + def test_false_when_engine_resolves_to_remote_server(self): + coord = PipeCoordinator(_make_mock_sched(self.tmpdir, ess_settings={'mockter': ['zeus']})) + tasks = [_make_spec(f't_{i}') for i in range(15)] + self.assertFalse(coord.should_use_pipe(tasks)) + + def _should_use_pipe(self, ess_settings, tasks=None): + """Return should_use_pipe() for a scheduler whose ESS settings are ``ess_settings``.""" + coord = PipeCoordinator(_make_mock_sched(self.tmpdir, ess_settings=ess_settings)) + return coord.should_use_pipe(tasks if tasks is not None + else [_make_spec(f't_{i}') for i in range(15)]) + + def test_true_when_the_engine_is_not_an_ess(self): + """TS-guess methods run in this process and are given no server, which is this machine.""" + self.assertTrue(self._should_use_pipe({'gaussian': ['local']})) + + def test_false_when_the_engine_names_an_empty_server_list(self): + """An ESS available nowhere resolves to nothing, and used to raise IndexError.""" + self.assertFalse(self._should_use_pipe({'mockter': []})) + + @patch('arc.job.pipe.pipe_coordinator.settings', + {'servers': {'zeus': {'cluster_soft': 'PBS', 'address': 'z.example.edu', 'un': 'u'}}}) + def test_false_when_the_resolved_server_is_not_configured(self): + """An unconfigured server is not known to be this machine, so the pipe is refused.""" + self.assertFalse(self._should_use_pipe({'mockter': ['not_a_configured_server']})) + + @patch('arc.job.pipe.pipe_coordinator.settings', + {'servers': {'zeus': {'cluster_soft': 'PBS', 'address': 'z.example.edu', 'un': 'u'}, + 'local': {'cluster_soft': 'PBS', 'un': 'u'}}}) + def test_the_first_server_decides_even_when_a_later_one_is_local(self): + """ARC submits to the first server named, so that is the one the pipe must be judged on.""" + self.assertFalse(self._should_use_pipe({'mockter': ['zeus', 'local']})) + + @patch('arc.job.pipe.pipe_coordinator.settings', + {'servers': {'local': {'cluster_soft': 'PBS', 'un': 'u'}}}) + def test_true_when_the_server_is_local_in_another_case(self): + """A server name is a settings key whose casing the user chose.""" + self.assertTrue(self._should_use_pipe({'mockter': ['LOCAL']})) + + @patch('arc.job.pipe.pipe_coordinator.settings', + {'servers': {'local': {'cluster_soft': 'PBS', 'un': 'u'}}}) + def test_true_when_the_server_is_named_as_a_bare_string(self): + """The ESS settings allow one server as a string rather than a one-item list.""" + self.assertTrue(self._should_use_pipe({'mockter': 'local'})) + + @patch('arc.job.pipe.pipe_coordinator.settings', + {'servers': {'zeus': {'cluster_soft': 'PBS', 'address': 'z.example.edu', 'un': 'u'}, + 'local': {'cluster_soft': 'PBS', 'un': 'u'}}}) + def test_a_troubleshooting_server_override_is_honoured(self): + """A job moved to another server by troubleshooting does not go through the pipe.""" + tasks = [_make_spec(f't_{i}') for i in range(15)] + for task in tasks: + task.args = {'trsh': {'server': 'zeus'}} + self.assertFalse(self._should_use_pipe({'mockter': ['local']}, tasks=tasks)) + class TestSubmitPipeRun(unittest.TestCase): """Tests for PipeCoordinator.submit_pipe_run().""" diff --git a/arc/job/pipe/pipe_planner_test.py b/arc/job/pipe/pipe_planner_test.py index bc1b38b113..74bc0185a2 100644 --- a/arc/job/pipe/pipe_planner_test.py +++ b/arc/job/pipe/pipe_planner_test.py @@ -50,7 +50,9 @@ def _make_mock_sched(project_directory): sched.freq_level = Level(method='wb97xd', basis='def2-tzvp') sched.scan_level = Level(method='wb97xd', basis='def2-tzvp') sched.irc_level = Level(method='wb97xd', basis='def2-tzvp') - sched.ess_settings = {'gaussian': ['server1']} + # PipeCoordinator refuses pipe for non-'local' servers (remote pipe + # support lives on a separate branch — see pipe_coordinator.py:70-76). + sched.ess_settings = {'gaussian': ['local']} sched.job_types = {'conf_opt': True, 'conf_sp': True, 'opt': True, 'freq': True, 'sp': True, 'rotors': True} spc = ARCSpecies(label='H2O', smiles='O') diff --git a/arc/scheduler_pipe_test.py b/arc/scheduler_pipe_test.py index 35c5e40604..4bbd973a87 100644 --- a/arc/scheduler_pipe_test.py +++ b/arc/scheduler_pipe_test.py @@ -51,7 +51,9 @@ def _make_task_spec(task_id, engine='mockter', task_family='conf_opt', def _make_scheduler(project_directory): """Create a minimal Scheduler for testing pipe methods.""" - ess_settings = {'gaussian': ['server1'], 'molpro': ['server2', 'server1'], 'qchem': ['server1']} + # PipeCoordinator refuses pipe for non-'local' servers (remote pipe + # support lives on a separate branch — see pipe_coordinator.py:70-76). + ess_settings = {'gaussian': ['local'], 'molpro': ['local'], 'qchem': ['local']} spc = ARCSpecies(label='H2O', smiles='O') spc.conformers = [None] * 5 spc.conformer_energies = [None] * 5 From 9115ebfde853180a8a1b3d6fd178b44286f12505 Mon Sep 17 00:00:00 2001 From: Calvin Pieters Date: Sat, 22 Aug 2026 13:08:58 +0300 Subject: [PATCH 6/9] Orca NEB: resolve reactant/product paths on the executing machine The NEB input template embeds absolute paths to reactant.xyz and product.xyz, built from self.local_path. For a remote server that names a directory on the machine running ARC, which does not exist on the cluster, so Orca cannot open the geometries. set_files() already uploads both files, and they land in remote_path. Choose the path accordingly -- remote_path for a remote server, local_path otherwise -- mirroring the choice JobAdapter already makes for the pipe payload's "pwd". This needs no change to any submit script, since the files are staged where the input now points. remote_path is only absolute when the server carries a 'path' in the settings: JobAdapter builds it as os.path.join(servers[server].get('path', '').lower(), 'runs', 'ARC_Projects', ...), which is relative when 'path' is unset, and the shipped Orca submit scripts cd into a scratch directory before running. Refuse to write the deck in that case, naming the server and the setting, rather than emitting one Orca will fail on for a reason that is not visible in the input. The relative path is pre-existing behaviour of arc/job/adapter.py shared by every adapter. The commit adding the connection pool fixes what can be fixed there -- the lowercasing of a configured path, and a report naming any server without one -- but a server with no 'path' configured has no absolute remote path that ARC can know before it connects, so refusing here is the other half of that fix. The adapter's tests only ever ran with server='local', where reverting the path choice to local_path still passed. They now cover a remote server too: the deck must point into the absolute remote_path, and a relative remote_path must be refused. Raise SettingsError rather than ValueError when the remote path is not absolute, matching the error arc.common.check_remote_paths_of_path_naming_adapters raises for the same condition at startup, so the two read as one settings problem. Validation now reports this before any job is spawned for every server the adapter is configured to run on; the refusal here remains the backstop for the paths validation does not see, such as a job moved to another server by troubleshooting. --- arc/job/adapters/ts/orca_neb.py | 38 ++++++++++- arc/job/adapters/ts/orca_neb_test.py | 95 +++++++++++++++++++++++++--- 2 files changed, 123 insertions(+), 10 deletions(-) diff --git a/arc/job/adapters/ts/orca_neb.py b/arc/job/adapters/ts/orca_neb.py index 916f882140..cabc84b037 100644 --- a/arc/job/adapters/ts/orca_neb.py +++ b/arc/job/adapters/ts/orca_neb.py @@ -12,6 +12,7 @@ from mako.template import Template from arc.common import get_logger +from arc.exceptions import SettingsError from arc.imports import incore_commands, settings from arc.job.adapters.common import is_restricted, which from arc.job.adapters.orca import OrcaAdapter, _format_orca_method, _format_orca_basis @@ -212,6 +213,15 @@ def __init__(self, def write_input_file(self) -> None: """ Write the input file to execute the job on the server. + + The NEB endpoint files are named in the deck by an absolute path, since Orca reads them + after the submit script has changed into a scratch directory. The path is + ``self.remote_path`` for a remote server, where the files are uploaded to, and + ``self.local_path`` for the local server. + + Raises: + ValueError: If the reaction has no atom map. + SettingsError: If the remote path of a remote server is not absolute. """ input_dict = dict() @@ -222,7 +232,7 @@ def write_input_file(self) -> None: input_dict['cpus'] = self.cpu_cores input_dict['charge'] = self.charge input_dict['multiplicity'] = self.multiplicity - input_dict['abs_path'] = self.local_path + input_dict['abs_path'] = self._get_abs_path() # NEB specific parameters neb_settings = orca_neb_settings.get('keyword', {}) @@ -247,6 +257,32 @@ def write_input_file(self) -> None: with open(os.path.join(self.local_path, input_filenames[self.job_adapter]), 'w') as f: f.write(Template(input_template).render(**input_dict)) + def _get_abs_path(self) -> str: + """ + Determine the directory Orca will read the NEB endpoint files from. + + A server that a run reaches here without an absolute path was not caught by + :func:`arc.common.check_remote_paths_of_path_naming_adapters`, which reports the same + condition at startup for every server this adapter is configured to run on. This is the + backstop for the paths that validation does not see, such as a job moved to another + server by troubleshooting, so it refuses rather than writing a deck Orca cannot follow. + + Returns: str + ``self.remote_path`` for a remote server, ``self.local_path`` otherwise. + + Raises: + SettingsError: If the server is remote and its remote path is not absolute. + """ + if self.server is None or self.server.lower() == 'local': + return self.local_path + if not self.remote_path or not os.path.isabs(self.remote_path): + raise SettingsError(f'Cannot write an Orca NEB input file for server "{self.server}": ' + f'the remote path is {self.remote_path!r}, which is not absolute, ' + f'and Orca reads the NEB endpoint files after the submit script ' + f'changed into a scratch directory. Set an absolute "path" for this ' + f'server in the settings.') + return self.remote_path + def set_files(self) -> None: """ Set files to be uploaded and downloaded. Writes the files if needed. diff --git a/arc/job/adapters/ts/orca_neb_test.py b/arc/job/adapters/ts/orca_neb_test.py index 1930c58dc6..315fe862e3 100644 --- a/arc/job/adapters/ts/orca_neb_test.py +++ b/arc/job/adapters/ts/orca_neb_test.py @@ -7,12 +7,13 @@ import os import shutil +import tempfile import datetime import unittest import unittest.mock import pytest -from arc.common import ARC_TESTING_PATH +from arc.exceptions import SettingsError from arc.job.adapters.ts.orca_neb import OrcaNEBAdapter from arc.level import Level from arc.reaction import ARCReaction @@ -31,21 +32,24 @@ def setUpClass(cls): """ cls.maxDiff = None - cls.project_directory = os.path.join(ARC_TESTING_PATH, 'test_OrcaNEBAdapter') - if os.path.exists(cls.project_directory): - shutil.rmtree(cls.project_directory) + cls.project_directory = tempfile.mkdtemp(prefix='test_OrcaNEBAdapter_') cls.addClassCleanup(shutil.rmtree, cls.project_directory, ignore_errors=True) - os.makedirs(cls.project_directory) # Mock objects for both orca_neb and orca/adapter modules mock_input_filenames = {'orca_neb': 'input.in', 'orca': 'input.in'} mock_output_filenames = {'orca_neb': 'input.log', 'orca': 'input.log'} - mock_servers = {'local': {'cluster_soft': 'local', 'un': 'user', 'queues': {}}} - mock_submit_filenames = {'local': 'submit.sub'} + mock_servers = {'local': {'cluster_soft': 'local', 'un': 'user', 'queues': {}}, + 'remote_server': {'cluster_soft': 'PBS', 'un': 'user', 'path': '/home/user', + 'address': 'remote.host.edu', 'queues': {'q': '24:00:00'}}, + 'server_without_a_path': {'cluster_soft': 'PBS', 'un': 'user', + 'address': 'remote.host.edu', 'queues': {'q': '24:00:00'}}} + mock_submit_filenames = {'local': 'submit.sub', 'PBS': 'submit.sub'} mock_orca_neb_settings = {'keyword': {'interpolation': 'IDPP', 'nnodes': 15, 'preopt': 'true'}} mock_default_job_settings = {'job_total_memory_gb': 14, 'job_cpu_cores': 8} - mock_t_max_format = {'local': 'hours'} - mock_submit_scripts = {'local': {'orca': 'mock submit script content'}} + mock_t_max_format = {'local': 'hours', 'PBS': 'hours'} + mock_submit_scripts = {'local': {'orca': 'mock submit script content'}, + 'remote_server': {'orca': 'mock submit script content'}, + 'server_without_a_path': {'orca': 'mock submit script content'}} # 1. Mock settings in orca_neb module cls.settings_patcher = unittest.mock.patch('arc.job.adapters.ts.orca_neb.settings', { @@ -89,6 +93,13 @@ def setUpClass(cls): cls.orca_neb_output_filenames_patcher = unittest.mock.patch('arc.job.adapters.ts.orca_neb.output_filenames', mock_output_filenames) cls.mock_orca_neb_output_filenames = cls.orca_neb_output_filenames_patcher.start() + cls.orca_neb_servers_patcher = unittest.mock.patch('arc.job.adapters.ts.orca_neb.servers', mock_servers) + cls.mock_orca_neb_servers = cls.orca_neb_servers_patcher.start() + + cls.orca_neb_submit_filenames_patcher = unittest.mock.patch('arc.job.adapters.ts.orca_neb.submit_filenames', + mock_submit_filenames) + cls.mock_orca_neb_submit_filenames = cls.orca_neb_submit_filenames_patcher.start() + # 4. Setup species and reaction cls.r_species = ARCSpecies(label='i-C3H7', smiles='C[CH]C') cls.p_species = ARCSpecies(label='n-C3H7', smiles='CC[CH2]') @@ -138,6 +149,70 @@ def test_task_1_preparation(self): self.assertIn('reactant.xyz', content) self.assertIn('product.xyz', content) + def test_task_1b_a_remote_server_resolves_the_endpoint_files_on_the_server(self): + """Orca reads reactant.xyz and product.xyz on the machine the job runs on.""" + job = OrcaNEBAdapter(project='test_orca_neb', + job_type='tsg', + project_directory=self.project_directory, + reactions=[self.reaction], + level=self.level, + server='remote_server') + abs_path = job._get_abs_path() + self.assertEqual(abs_path, job.remote_path) + self.assertNotEqual(abs_path, job.local_path) + self.assertTrue(os.path.isabs(abs_path)) + job.write_input_file() + with open(os.path.join(job.local_path, 'input.in'), 'r') as f: + content = f.read() + self.assertIn(f'{job.remote_path}/reactant.xyz', content) + self.assertIn(f'{job.remote_path}/product.xyz', content) + self.assertNotIn(job.local_path, content) + + def test_task_1c_a_relative_remote_path_is_refused(self): + """A server without a configured path yields a relative remote path Orca cannot follow.""" + job = OrcaNEBAdapter(project='test_orca_neb', + job_type='tsg', + project_directory=self.project_directory, + reactions=[self.reaction], + level=self.level, + server='remote_server') + job.remote_path = os.path.join('runs', 'ARC_Projects', 'test_orca_neb') + with self.assertRaises(SettingsError): + job.write_input_file() + + def test_task_1e_a_server_without_a_path_is_refused_at_construction(self): + """ + The backstop for a server that startup validation did not see, such as one a job was + moved to by troubleshooting. It reports the same condition as the startup check and with + the same error type, so the two read as one settings problem rather than two faults. + """ + with self.assertRaises(SettingsError): + OrcaNEBAdapter(project='test_orca_neb', + job_type='tsg', + project_directory=self.project_directory, + reactions=[self.reaction], + level=self.level, + server='server_without_a_path') + + def test_task_1f_the_refusal_names_the_server_and_the_setting_to_fix(self): + """A refusal the reader cannot act on costs a support round.""" + job = OrcaNEBAdapter(project='test_orca_neb', + job_type='tsg', + project_directory=self.project_directory, + reactions=[self.reaction], + level=self.level, + server='remote_server') + job.remote_path = os.path.join('runs', 'ARC_Projects', 'test_orca_neb') + with self.assertRaises(SettingsError) as raised: + job.write_input_file() + self.assertIn('remote_server', str(raised.exception)) + self.assertIn('path', str(raised.exception)) + + def test_task_1d_the_local_server_still_resolves_locally(self): + """A local job reads the endpoint files from the directory ARC wrote them to.""" + self.assertTrue(os.path.isabs(self.job.remote_path)) + self.assertEqual(self.job._get_abs_path(), self.job.local_path) + def test_task_2_post_processing(self): """ Task 2: Post processing parts. @@ -181,6 +256,8 @@ def tearDownClass(cls): cls.adapter_t_max_format_patcher.stop() cls.adapter_output_filenames_patcher.stop() cls.orca_neb_output_filenames_patcher.stop() + cls.orca_neb_servers_patcher.stop() + cls.orca_neb_submit_filenames_patcher.stop() if __name__ == '__main__': From 70457034d4a25ca180263ab56db733b719584fd7 Mon Sep 17 00:00:00 2001 From: Calvin Pieters Date: Sat, 22 Aug 2026 13:08:58 +0300 Subject: [PATCH 7/9] Docker: support driving a remote cluster over SSH from the container ARC inside the image can now reach a remote HPC cluster either through a forwarded SSH agent socket (preferred -- keys never enter the container, and passphrase-protected keys keep working) or through a read-only bind mount of the user's key material, with the ~/.arc settings overlay mounted alongside it. entrywrapper.sh: - pass SSH_AUTH_SOCK explicitly across the `runuser -u mambauser` privilege drop, and explicitly `env -u` it when the socket turned out to be unusable, so the SSH client falls back to key files instead of failing against a dead socket; - never chown/chmod a bind-mounted /home/mambauser/.ssh (detected by comparing the device of the path with its parent), so read-only mounts neither fail nor leak ownership changes back to the host. paramiko does not enforce 0600 on key files, so a read-only mount is fine; - emit actionable PUID/PGID diagnostics when a mounted socket or .ssh directory is not accessible to the container user. PUID/PGID remap: the base image carried a vestigial 'ubuntu' account at 1000:1000 and the entrypoint refused to remap onto any occupied ID, so `-e PUID=$(id -u) -e PGID=$(id -g)` -- the very flags needed for bind-mount ownership -- aborted with exit 1 for the majority of Linux desktop users, and docker-compose.yml defaults both to 1000, so the compose path was broken by default. The Dockerfile deletes that unused account in the final stage, and the entrypoint no longer depends on the image being fixed: a collision with an ordinary, idle account is resolved by sharing the ID (usermod -o), since permissions are numeric and nothing then has to be deleted or renamed. "Unused" is established from /proc rather than assumed from the account's name. A collision with the superuser, with a system account (<= 999), or with an account owning running processes is still fatal, exit code unchanged, but now names the exact flag to drop and states what ownership the mounts would fall back to. Agent socket: prepare_ssh_agent_socket() ran `chmod o+rw` on the forwarded socket whenever the container user could not open it. A bind mount shares the inode, so that mutated the user's live agent socket on the host -- 600 -> 606, verified -- leaving it readable and writable by every local user for as long as the agent runs, never restored, and contradicting the rule this same file applies to every other bind-mounted path. Restoring it is impossible here in any case, since the entrypoint hands off with exec and no EXIT trap can fire. The remap above is the mechanism instead: a container user carrying the host UID opens a 0600 socket with nothing changed. The widening survives only behind an explicit ARC_WIDEN_AGENT_SOCKET=1, and when used it reports what it changed on the host and how to undo it; without the opt-in, an inaccessible socket is reported with the PUID/PGID fix and agent forwarding is skipped. Dockerfile: add openssh-client to the final stage (ssh-keyscan and friends for debugging; ARC itself uses paramiko) and pre-create /home/mambauser/.ssh so a bind mount lands with sane ownership. Final stage only. docker-compose.yml: replace the stale definition (foreign image, non-existent /home/rmguser/KMClass path, a CONTAINER_MODE variable the entrypoint never read) with one matching the real entrypoint contract: /work bind mount, ~/.arc, agent socket forwarding, PUID/PGID. arc_preflight.py: arc/imports.py catches ImportError from the local settings.py and passes, so a settings.py that cannot be imported leaves ARC running against its *dummy* servers with no output at all -- verified: a settings.py whose first line imports a missing module yields settings['servers']['server1']['address'] == 'server1.host.edu', silently. In a container a mistyped mount path lands in exactly that state, and the only symptom is a run that spends hours failing to reach a host that never existed. The pre-flight imports the overlay exactly as ARC does -- as a top-level module on sys.path, without importing ARC itself -- and exits 78 (EX_CONFIG) if it is present but unimportable; a directory with no settings.py is reported as absent and exits 0. Everything else is a warning, deliberately: a missing or unreadable `key` file, a server with no `key` and neither an agent nor a default key, and strict_host_key_checking with no known_hosts. A server that is configured but unused in a given run must not be able to abort it. ARC_SKIP_PREFLIGHT=1 bypasses the whole check. The check runs for `arc` only, never for `rmg`, and the overlay directory is passed as argv so the entrypoint and the tests cannot drift apart on the path. The overlay is mounted read-only, with PYTHONDONTWRITEBYTECODE=1 so Python does not pointlessly attempt __pycache__ writes into it; runuser sets HOME=/home/mambauser, hence the mount target. test_docker_smoke.py: SSH-oriented smoke checks that need no remote server (paramiko importable and constructible, arc.job.ssh imports, openssh-client present, entrypoint agent-socket handling, forwarded socket usable when present, the PUID remap and the socket mode -- each of the last three confirmed to fail against the pre-fix entrypoint), plus an opt-in live-cluster test skipped unless ARC_SMOKE_SSH_HOST is set. The overlay is exercised through a subprocess, never by importing arc in the pytest process, since some ARC branches disable the ~/.arc overlay whenever pytest is loaded, which would silently hollow out an in-process test. Not changed: usermod -u recursively chowns the home directory, which was raised as a possible startup cost now that ~/.julia is 2.9 GB. Measured on the built image -- 28,795 files, 0.375 s without the remap against 0.925 s with it. That does not warrant working around usermod, so it is left alone. Julia is pinned to 1.10.11 rather than tracking the 1.10 channel. juliacall segfaults on import under 1.10.12 and the channel floats to the newest patch, so an unchanged Dockerfile silently changed what it installed; RMG-Py pinned the same version for the same reason in 62eb728c0. docker-compose.yml also mounts known_hosts read-only at /home/mambauser/.ssh/known_hosts, paramiko's only host-key location, so the container gets the same treatment as the key material and the ~/.arc overlay and so ARC's startup host-key check means something there. The source is ${ARC_KNOWN_HOSTS:-/dev/null} rather than ${HOME}/.ssh/known_hosts directly: Docker materialises a missing bind-mount source as a directory, so naming that path unconditionally would leave a root-owned directory at $HOME/.ssh/known_hosts on any host that has never written the file, which then breaks ssh on the host itself. /dev/null always exists and reads as an empty key list. --- Dockerfile | 26 +- docker-compose.yml | 88 ++++- dockerfiles/arc_preflight.py | 181 ++++++++++ dockerfiles/docker_tests/test_docker_smoke.py | 339 ++++++++++++++++++ dockerfiles/entrywrapper.sh | 244 ++++++++++++- 5 files changed, 848 insertions(+), 30 deletions(-) create mode 100644 dockerfiles/arc_preflight.py diff --git a/Dockerfile b/Dockerfile index 659cfefd75..61a97a0a4a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -34,11 +34,13 @@ ENV MAMBA_USER=mambauser # which then dies with NoChannelsConfiguredError (non-fatally) and silently drops that pin. RUN printf 'channels:\n - conda-forge\n' > /home/mambauser/.condarc -# Set JuliaUp PATH and install Julia 1.10 as req. by RMG (mirrors RMG-Py's own Dockerfile) +# Set JuliaUp PATH and install Julia as req. by RMG (mirrors RMG-Py's own Dockerfile). +# Pinned to a patch version, not the 1.10 channel: juliacall segfaults on import under 1.10.12, +# and the channel floats to the newest patch. RMG-Py pinned the same version in 62eb728c0. ENV PATH="/home/mambauser/.juliaup/bin:$PATH" -RUN wget -qO- https://install.julialang.org | sh -s -- --yes --default-channel 1.10 && \ - juliaup add 1.10 && \ - juliaup default 1.10 && \ +RUN wget -qO- https://install.julialang.org | sh -s -- --yes --default-channel 1.10.11 && \ + juliaup add 1.10.11 && \ + juliaup default 1.10.11 && \ juliaup list && \ rm -rf /home/mambauser/.juliaup/downloads /home/mambauser/.juliaup/tmp @@ -153,7 +155,7 @@ ENV PYTHON_JULIAPKG_EXE=/home/mambauser/.juliaup/bin/julia # any --entrypoint override) lands on root with HOME=/root rather than going through # entrywrapper.sh's `runuser -u mambauser`. juliaup then finds no config, falls back to the # `release` channel, and downloads a newer Julia over the network - silently bypassing the pinned -# 1.10 and every pkgimage baked above, or hard-failing when the host is offline. Pinning the depot +# 1.10.11 and every pkgimage baked above, or hard-failing when the host is offline. Pinning the depot # explicitly makes resolution HOME-independent. Note this is JULIAUP_DEPOT_PATH, not # JULIA_DEPOT_PATH; juliaup does not read the latter for channel lookup. ENV JULIAUP_DEPOT_PATH=/home/mambauser/.julia @@ -166,7 +168,14 @@ RUN apt-get update && \ ca-certificates \ nano \ make \ + openssh-client \ && apt-get clean && rm -rf /var/lib/apt/lists/* +# Drop the vestigial 'ubuntu' account the base image ships at 1000:1000. Nothing here uses it +# (micromamba runs as mambauser, uid 57439), but it occupies exactly the IDs that the documented +# `-e PUID=$(id -u) -e PGID=$(id -g)` asks for on a typical Linux desktop, where the entrypoint +# would then refuse to remap mambauser and abort the container. +RUN if getent passwd ubuntu >/dev/null; then userdel -r ubuntu 2>/dev/null || userdel ubuntu; fi && \ + if getent group ubuntu >/dev/null; then groupdel ubuntu; fi USER mambauser COPY --from=builder --chown=mambauser:mambauser /opt/conda /opt/conda @@ -187,6 +196,13 @@ COPY --chmod=755 dockerfiles/entrywrapper.sh /usr/local/bin/entrywrapper.sh COPY --chmod=644 dockerfiles/aliases.sh /etc/profile.d/aliases.sh COPY --chmod=755 dockerfiles/job_helpers.sh /usr/local/bin/arc_job_helpers.sh COPY --chmod=755 dockerfiles/aliases_print.sh /usr/local/bin/aliases +COPY --chmod=755 dockerfiles/arc_preflight.py /usr/local/bin/arc_preflight.py +# Mount points for the user's SSH material (agent socket or read-only key/known_hosts mounts) +# and for the personal ARC settings overlay, so both bind mounts land on an existing path +# owned by mambauser. +RUN mkdir -p /home/mambauser/.ssh && chmod 700 /home/mambauser/.ssh && \ + mkdir -p /home/mambauser/.arc && chown mambauser:mambauser /home/mambauser/.arc + RUN touch /home/mambauser/.bashrc && \ grep -qxF 'source /etc/profile.d/aliases.sh' /home/mambauser/.bashrc || \ echo 'source /etc/profile.d/aliases.sh' >> /home/mambauser/.bashrc diff --git a/docker-compose.yml b/docker-compose.yml index f3744bae37..0a72cf0f63 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,16 +1,78 @@ -version: '3.8' +# ARC / RMG container. +# +# The image entrypoint is dockerfiles/entrywrapper.sh, which accepts: +# arc run ARC in arc_env +# rmg run RMG in rmg_env +# pass-through +# (no arguments) interactive login shell +# +# Note that this service defines a `command`, and `docker compose run` falls back to it +# whenever no command is given on the command line. Running the service with no arguments +# therefore runs ARC, not the entrypoint's interactive shell; ask for a shell explicitly. +# +# Usage: +# ARC_INPUT=my_case/input.yml docker compose run --rm arc # run ARC on that input +# docker compose run --rm arc # same, on ${ARC_INPUT:-input.yml} +# docker compose run --rm arc rmg my_case/input.py # run RMG instead +# docker compose run --rm arc bash # interactive shell services: - rmg_container: - image: laxzal/arc:latest + arc: + image: ${ARC_IMAGE:-laxzal/arc:latest} platform: linux/amd64 - container_name: rmg_container - volumes: - # Mount a local directory to the container. Replace with your actual local directory path. - # Ensure to use absolute paths. Examples of directory formats for different operating systems: - # Windows: C:/Users/YourUsername/Documents/MyFolder - # macOS: /Users/YourUsername/Documents/MyFolder - # Linux: /home/YourUsername/Documents/MyFolder - - :/home/rmguser/KMClass + # Uncomment to build the image from this repository instead of pulling it. + # build: + # context: . + # dockerfile: Dockerfile + working_dir: /work + stdin_open: true + tty: true environment: - # Set the mode of the container. Replace with either "interactive" or "non-interactive". - - CONTAINER_MODE=interactive + # Remap the in-container mambauser account to your host UID/GID so files written to + # the bind mounts below are owned by you, and so mounts owned by you stay readable. + - PUID=${PUID:-1000} + - PGID=${PGID:-1000} + # Path of the forwarded SSH agent socket *inside* the container. The entrypoint keeps + # this variable across its privilege drop; if nothing is forwarded it warns and + # continues without an agent. + - SSH_AUTH_SOCK=/ssh-agent + # The ~/.arc overlay below is mounted read-only, so keep Python from trying to drop + # __pycache__ next to it. Python ignores the failure, but the attempt is pointless. + - PYTHONDONTWRITEBYTECODE=1 + volumes: + # Working directory: inputs are read from here and all ARC/RMG output lands here. + - ${ARC_WORKDIR:-.}:/work + # ARC personal settings, read-only. ARC reads settings.py, submit.py and inputs.py from + # here; submit.py in particular holds the cluster's PBS/Slurm submit templates, so a + # remote run needs this mount as much as it needs the SSH material below. + # Each file replaces the repository default for the top-level names it defines - the + # overlay is a name-level replacement, not a deep merge - and note that ARC forces + # global_ess_settings to None whenever a local settings.py exists unless that file + # defines a truthy value of its own. + # The entrypoint refuses to start ARC if this settings.py cannot be imported, because + # ARC itself would ignore it silently and fall back to its dummy servers. + - ${HOME}/.arc:/home/mambauser/.arc:ro + # SSH agent forwarding for remote job submission - keys never enter the container. + # Falls back to /dev/null when no agent is running, which the entrypoint detects and + # reports instead of failing. On macOS use /run/host-services/ssh-auth.sock instead. + # Beware a *stale* SSH_AUTH_SOCK, pointing at a socket of an agent that has since died: + # Docker creates any missing bind-mount source, so it will silently make a root-owned + # directory at that path on the host, and the entrypoint then reports a non-socket. + # The same applies to the ${HOME}/.arc source above if that directory does not exist. + - ${SSH_AUTH_SOCK:-/dev/null}:/ssh-agent + # Fallback for headless runs with no agent: mount your key material read-only, and point + # the server's 'key' at the in-container path. Read-only is fine - ARC uses paramiko, + # which does not enforce 0600 on key files. + # - ${HOME}/.ssh:/home/mambauser/.ssh:ro + # Host keys, read-only. paramiko reads them from its default location, so the target path + # must be exactly this one. Required for servers with 'strict_host_key_checking': True, + # which refuse any host that is not listed here; without it, unknown hosts are only warned + # about - once by ARC's startup check and again on every connection. + # The source defaults to /dev/null rather than to ${HOME}/.ssh/known_hosts because Docker + # creates any missing bind-mount source: on a host that has never written a known_hosts + # file, naming that path directly would leave a root-owned *directory* at + # ${HOME}/.ssh/known_hosts, which then breaks ssh on the host itself. /dev/null always + # exists and reads as an empty key list, so the container simply starts with no host keys. + # Point ARC_KNOWN_HOSTS at your file to share it: + # ARC_KNOWN_HOSTS=$HOME/.ssh/known_hosts docker compose run --rm arc + - ${ARC_KNOWN_HOSTS:-/dev/null}:/home/mambauser/.ssh/known_hosts:ro + command: ["arc", "${ARC_INPUT:-input.yml}"] diff --git a/dockerfiles/arc_preflight.py b/dockerfiles/arc_preflight.py new file mode 100644 index 0000000000..39c4f9fd83 --- /dev/null +++ b/dockerfiles/arc_preflight.py @@ -0,0 +1,181 @@ +""" +A pre-flight check for the ARC container's ``~/.arc`` settings overlay. + +ARC loads user overrides from ``$HOME/.arc/{settings,submit,inputs}.py`` (see ``arc/imports.py``). +That loader is deliberately forgiving: a ``settings.py`` which raises ``ImportError`` is skipped +without a word, and ARC then continues with the repository's dummy servers +(``server1.host.edu``, ````). Inside a container the usual cause is a mis-typed bind +mount, and the only symptom is a run that spends hours failing to reach a host that never +existed. This script turns that into an immediate, actionable message, and additionally reports +SSH identities that cannot work inside the container. + +It deliberately imports ``settings`` the same way ``arc.imports`` does -- as a top-level module +found on ``sys.path`` -- rather than importing ARC itself, which is slow and would obscure the +very failure being checked for. + +Usage: + arc_preflight.py [overlay directory] + + The directory defaults to ``DEFAULT_ARC_DIR``. The entrypoint passes its own constant so + that the two cannot drift apart, and the tests pass a temporary directory. + +Exit codes: + 0: the overlay is usable, or there is no ``settings.py`` to check. Warnings may still + have been printed. + 1: ``settings.py`` is present but raised while being imported. The caller should abort. +""" + +import os +import sys +import traceback + + +DEFAULT_ARC_DIR = '/home/mambauser/.arc' +DUMMY_ADDRESS_SUFFIX = '.host.edu' +DUMMY_USERNAME = '' +DEFAULT_KEY_NAMES = ('id_rsa', 'id_ecdsa', 'id_ed25519') + + +def warn(message: str) -> None: + """ + Print an actionable warning to stderr. + + Args: + message (str): The warning text, without a prefix. + """ + print(f'preflight: warning: {message}', file=sys.stderr) + + +def load_local_settings(arc_dir: str): + """ + Import ``settings.py`` from ``arc_dir`` exactly as ``arc.imports`` does. + + Args: + arc_dir (str): The directory holding the personal ARC settings. + + Returns: + module: The imported local settings module. + """ + if arc_dir not in sys.path: + sys.path.insert(0, arc_dir) + import settings + return settings + + +def ssh_dir() -> str: + """ + Returns: + str: The path of the container user's ``.ssh`` directory. + """ + return os.path.join(os.path.expanduser('~'), '.ssh') + + +def check_key(name: str, cfg: dict) -> None: + """ + Warn when a server's SSH identity cannot be used from inside the container. + + ``key`` is the path of a private key on the machine running ARC, and is optional: without it + paramiko falls back to a running ssh-agent and then to the default key paths. Both routes are + legitimate, so nothing here is fatal -- a server that is configured but never used must not + abort the run. + + Args: + name (str): The server name. + cfg (dict): The server settings. + """ + key = cfg.get('key') + if key: + if not os.path.isfile(key): + warn(f"server '{name}' sets key '{key}', which does not exist inside the container. " + f"Mount it read-only, e.g. -v \"$HOME/.ssh/id_ed25519:{key}:ro\" , or remove " + f"'key' from the server entry and forward your ssh-agent instead.") + elif not os.access(key, os.R_OK): + warn(f"server '{name}' sets key '{key}', which exists but is not readable by uid " + f"{os.getuid()}. Re-run with -e PUID=$(id -u) -e PGID=$(id -g) so the container " + f"user matches the owner of the mount.") + return + if os.environ.get('SSH_AUTH_SOCK'): + return + if any(os.path.isfile(os.path.join(ssh_dir(), key_name)) for key_name in DEFAULT_KEY_NAMES): + return + warn(f"server '{name}' sets no 'key', and this container has neither a forwarded ssh-agent " + f"(SSH_AUTH_SOCK is empty) nor a default key under {ssh_dir()}. Forward your agent with " + f"-v \"$SSH_AUTH_SOCK:/ssh-agent\" -e SSH_AUTH_SOCK=/ssh-agent , or mount a private key " + f"and point 'key' at it.") + + +def check_host_keys(name: str, cfg: dict, known_hosts: str) -> None: + """ + Warn when strict host key checking cannot succeed for lack of a seeded ``known_hosts``. + + Args: + name (str): The server name. + cfg (dict): The server settings. + known_hosts (str): The path paramiko reads known host keys from. + """ + if not cfg.get('strict_host_key_checking'): + return + if os.path.isfile(known_hosts): + return + warn(f"server '{name}' sets strict_host_key_checking, but {known_hosts} does not exist inside " + f"the container, so every connection will be refused. Mount your host keys at that exact " + f"path, or seed them with ssh-keyscan {cfg.get('address', 'HOST')} >> {known_hosts} .") + + +def main(arc_dir: str) -> int: + """ + Run the pre-flight checks. + + Args: + arc_dir (str): The directory holding the personal ARC settings. + + Returns: + int: 0 when the overlay is usable, 1 when it is present but unimportable. + """ + settings_path = os.path.join(arc_dir, 'settings.py') + if not os.path.isfile(settings_path): + warn(f'{settings_path} does not exist, so ARC will use its dummy server settings. ' + f'Nothing to check.') + return 0 + try: + local_settings = load_local_settings(arc_dir) + except ImportError: + traceback.print_exc() + print(f'preflight: error: {settings_path} could not be imported (traceback above).', + file=sys.stderr) + print('preflight: ARC catches exactly this error and carries on with its dummy server ' + 'settings, without printing anything, so the run is being stopped here instead.', + file=sys.stderr) + return 1 + except Exception: + traceback.print_exc() + print(f'preflight: error: {settings_path} raised while being imported (traceback above).', + file=sys.stderr) + print('preflight: ARC tolerates only an ImportError here, so it would abort on this ' + 'too, a moment later and with nothing but the traceback. Fix the file and re-run.', + file=sys.stderr) + return 1 + servers = getattr(local_settings, 'servers', None) + if not isinstance(servers, dict) or not servers: + warn(f'{arc_dir}/settings.py defines no non-empty "servers" dict, so ARC will use its ' + f'dummy server settings. This is expected only for runs that submit nothing.') + return 0 + known_hosts = os.path.join(ssh_dir(), 'known_hosts') + for name, cfg in servers.items(): + if not isinstance(cfg, dict): + warn(f"server '{name}' is not a settings dictionary; ARC will not be able to use it.") + continue + address = cfg.get('address') + if not address: + continue + if address.endswith(DUMMY_ADDRESS_SUFFIX) or cfg.get('un') == DUMMY_USERNAME: + warn(f"server '{name}' still carries ARC's placeholder address or username " + f"('{address}', '{cfg.get('un')}'); it cannot be reached.") + continue + check_key(name, cfg) + check_host_keys(name, cfg, known_hosts) + return 0 + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1] if len(sys.argv) > 1 else DEFAULT_ARC_DIR)) diff --git a/dockerfiles/docker_tests/test_docker_smoke.py b/dockerfiles/docker_tests/test_docker_smoke.py index fddb032879..dd7a2b5ce6 100644 --- a/dockerfiles/docker_tests/test_docker_smoke.py +++ b/dockerfiles/docker_tests/test_docker_smoke.py @@ -1,9 +1,47 @@ import pytest +import os +import shutil +import stat import subprocess import tempfile import pathlib import textwrap +ENTRYWRAPPER = os.environ.get("ARC_ENTRYWRAPPER_PATH", "/usr/local/bin/entrywrapper.sh") +PREFLIGHT = os.environ.get("ARC_PREFLIGHT_PATH", "/usr/local/bin/arc_preflight.py") +ARC_SETTINGS_DIR = os.environ.get("ARC_SETTINGS_DIR", "/home/mambauser/.arc") + +REMOTE_SETTINGS = """\ +servers = { + 'smoke_cluster': { + 'cluster_soft': 'Slurm', + 'address': 'login.smoke-cluster.invalid', + 'un': 'smoke_user', + }, +} +""" + + +def run_in_arc_env(code, env=None): + """Run a snippet in arc_env through a login shell, never in this pytest process. + + ARC's ~/.arc overlay is loaded at import time and some ARC branches disable it outright + while pytest is loaded, so any test of the overlay has to cross a process boundary to + keep testing what it claims to test. + """ + full_env = dict(os.environ) + full_env.update(env or {}) + cmd = ["bash", "-lc", f"micromamba run -n arc_env python - <<'PY'\n{code}\nPY"] + return subprocess.run(cmd, capture_output=True, text=True, env=full_env) + + +def run_preflight(arc_dir, env=None): + """Run the entrypoint's settings pre-flight check against ``arc_dir``.""" + full_env = dict(os.environ) + full_env.update(env or {}) + cmd = ["bash", "-lc", f"micromamba run -n arc_env python {PREFLIGHT} {arc_dir}"] + return subprocess.run(cmd, capture_output=True, text=True, env=full_env) + @pytest.mark.smoke def test_import_arc(): @@ -73,3 +111,304 @@ def test_rmg_cli_help_runs(): p = subprocess.run(cmd, capture_output=True, text=True) # Just ensure it executes and prints usage/help assert "rmg" in (p.stdout + p.stderr).lower() + + +@pytest.mark.smoke +def test_paramiko_available(): + """Test that paramiko is importable and an SSH client can be constructed.""" + code = r""" +import paramiko +client = paramiko.SSHClient() +try: + print('paramiko OK', paramiko.__version__) +finally: + client.close() +""" + cmd = ["bash", "-lc", f"micromamba run -n arc_env python - <<'PY'\n{code}\nPY"] + p = subprocess.run(cmd, capture_output=True, text=True) + assert p.returncode == 0, p.stderr + assert "paramiko OK" in p.stdout + + +@pytest.mark.smoke +def test_arc_ssh_module_imports(): + """Test that ARC's SSH layer imports in the docker image.""" + code = r""" +from arc.job.ssh import SSHClient +print('arc.job.ssh OK', SSHClient.__name__) +""" + cmd = ["bash", "-lc", f"micromamba run -n arc_env python - <<'PY'\n{code}\nPY"] + p = subprocess.run(cmd, capture_output=True, text=True) + assert p.returncode == 0, p.stderr + assert "arc.job.ssh OK" in p.stdout + + +@pytest.mark.smoke +def test_openssh_client_installed(): + """Test that the OpenSSH client tools are available for debugging remote connections.""" + for tool in ("ssh", "ssh-keyscan", "ssh-keygen"): + assert shutil.which(tool) is not None, f"{tool} is missing from the image" + + +@pytest.mark.smoke +def test_entrywrapper_is_valid_bash(): + """Test that the entrypoint script is present, executable, and syntactically valid.""" + path = pathlib.Path(ENTRYWRAPPER) + assert path.is_file(), f"{ENTRYWRAPPER} is missing" + assert path.stat().st_mode & stat.S_IXUSR, f"{ENTRYWRAPPER} is not executable" + p = subprocess.run(["bash", "-n", ENTRYWRAPPER], capture_output=True, text=True) + assert p.returncode == 0, p.stderr + + +@pytest.mark.smoke +def test_entrywrapper_forwards_ssh_auth_sock_across_privilege_drop(): + """Test that the entrypoint hands SSH_AUTH_SOCK to the unprivileged user explicitly.""" + text = pathlib.Path(ENTRYWRAPPER).read_text() + assert "SSH_AUTH_SOCK=$SSH_AUTH_SOCK" in text, \ + "the entrypoint must pass SSH_AUTH_SOCK through the runuser privilege drop" + assert "-u SSH_AUTH_SOCK" in text, \ + "the entrypoint must unset SSH_AUTH_SOCK when the forwarded socket is unusable" + + +@pytest.mark.smoke +def test_entrypoint_does_not_widen_the_agent_socket_unasked(): + """Test that relaxing the agent socket's mode stays behind an explicit opt-in. + + A bind-mounted socket shares its inode with the host, so a chmod here mutates the user's + real agent socket and is never restored, the entrypoint having handed off with exec. + """ + text = pathlib.Path(ENTRYWRAPPER).read_text() + assert "ARC_WIDEN_AGENT_SOCKET" in text, \ + "the opt-in guard for relaxing the agent socket's mode has gone missing" + for line in text.splitlines(): + stripped = line.strip() + if stripped.startswith("chmod") and "sock" in stripped: + pytest.fail(f"unguarded chmod of the agent socket: {stripped}") + + +@pytest.mark.smoke +def test_forwarded_agent_socket_mode_is_left_alone(): + """Test that a forwarded agent socket was not made accessible to other users. + + Skipped unless an agent was forwarded, and when the widening was explicitly requested. + """ + sock = os.environ.get("SSH_AUTH_SOCK") + if not sock: + pytest.skip("no SSH agent forwarded into the container") + if os.environ.get("ARC_WIDEN_AGENT_SOCKET") == "1": + pytest.skip("the socket mode was relaxed on explicit request") + mode = os.stat(sock).st_mode + assert not mode & stat.S_IWOTH, \ + f"{sock} is world-writable; the entrypoint must not relax a bind-mounted agent socket" + assert not mode & stat.S_IROTH, \ + f"{sock} is world-readable; the entrypoint must not relax a bind-mounted agent socket" + + +@pytest.mark.smoke +def test_container_user_matches_requested_puid(): + """Test that PUID/PGID actually remapped the container user. + + The base image ships an unused 'ubuntu' account at 1000:1000, which is what PUID/PGID ask + for on a typical Linux desktop, and a collision there used to abort the container. + Skipped when the container was started without the remap. + """ + puid = os.environ.get("PUID") + pgid = os.environ.get("PGID") + if not puid and not pgid: + pytest.skip("container started without PUID/PGID") + if puid: + assert os.getuid() == int(puid), \ + f"asked for PUID={puid} but running as uid {os.getuid()}" + if pgid: + assert os.getgid() == int(pgid), \ + f"asked for PGID={pgid} but running as gid {os.getgid()}" + + +@pytest.mark.smoke +def test_ssh_home_directory_is_usable(): + """Test that the SSH material directory exists and is usable by the current user.""" + ssh_dir = pathlib.Path.home() / ".ssh" + if not ssh_dir.exists(): + pytest.skip(f"{ssh_dir} is not present in this container") + assert os.access(ssh_dir, os.R_OK | os.X_OK), \ + f"{ssh_dir} is not readable by the container user; re-run with -e PUID=$(id -u) -e PGID=$(id -g)" + + +@pytest.mark.smoke +def test_forwarded_ssh_agent_socket_is_usable(): + """Test that a forwarded SSH agent socket survived the privilege drop and is usable. + + Skipped unless the container was started with an agent socket forwarded. + """ + sock = os.environ.get("SSH_AUTH_SOCK") + if not sock: + pytest.skip("no SSH agent forwarded into the container") + assert stat.S_ISSOCK(os.stat(sock).st_mode), f"SSH_AUTH_SOCK={sock} is not a socket" + assert os.access(sock, os.R_OK | os.W_OK), \ + f"{sock} is not accessible to the container user; re-run with -e PUID=$(id -u) -e PGID=$(id -g)" + + +@pytest.mark.smoke +def test_arc_settings_overlay_is_loaded(): + """Test that a ~/.arc/settings.py replaces ARC's dummy servers.""" + code = r""" +from arc.imports import settings +print('SERVERS', sorted(settings['servers'])) +""" + with tempfile.TemporaryDirectory() as td: + arc_dir = pathlib.Path(td, ".arc") + arc_dir.mkdir() + (arc_dir / "settings.py").write_text(REMOTE_SETTINGS) + p = run_in_arc_env(code, env={"HOME": td}) + assert p.returncode == 0, p.stderr + assert "smoke_cluster" in p.stdout, p.stdout + assert "server1" not in p.stdout, "the repository's dummy servers were not replaced" + + +@pytest.mark.smoke +def test_arc_submit_overlay_is_loaded(): + """Test that a ~/.arc/submit.py replaces the repository's submit templates. + + submit.py carries the cluster's PBS/Slurm templates, so a remote run depends on this + mount just as much as on settings.py. + """ + code = r""" +from arc.imports import submit_scripts +print('TEMPLATE', submit_scripts.get('smoke_cluster', {}).get('gaussian')) +""" + with tempfile.TemporaryDirectory() as td: + arc_dir = pathlib.Path(td, ".arc") + arc_dir.mkdir() + (arc_dir / "submit.py").write_text( + "submit_scripts = {'smoke_cluster': {'gaussian': 'SMOKE-TEMPLATE'}}\n") + p = run_in_arc_env(code, env={"HOME": td}) + assert p.returncode == 0, p.stderr + assert "TEMPLATE SMOKE-TEMPLATE" in p.stdout, p.stdout + + +@pytest.mark.smoke +def test_arc_ignores_an_unimportable_settings_overlay(): + """Test the failure mode the entrypoint pre-flight exists to catch. + + ARC swallows an ImportError from ~/.arc/settings.py and falls back to its dummy servers + without a word. If this ever stops being true the pre-flight can be relaxed, so assert it + rather than assume it. + """ + code = r""" +from arc.imports import settings +print('ADDRESS', settings['servers'].get('server1', {}).get('address')) +""" + with tempfile.TemporaryDirectory() as td: + arc_dir = pathlib.Path(td, ".arc") + arc_dir.mkdir() + (arc_dir / "settings.py").write_text( + "import a_module_that_is_not_installed\n" + REMOTE_SETTINGS) + p = run_in_arc_env(code, env={"HOME": td}) + assert p.returncode == 0, p.stderr + assert "ADDRESS server1.host.edu" in p.stdout, \ + f"expected a silent fallback to the dummy servers, got: {p.stdout}" + + +@pytest.mark.smoke +def test_preflight_accepts_a_usable_settings_overlay(): + """Test that the pre-flight passes a settings overlay that imports cleanly.""" + with tempfile.TemporaryDirectory() as td: + pathlib.Path(td, "settings.py").write_text(REMOTE_SETTINGS) + p = run_preflight(td, env={"SSH_AUTH_SOCK": "/ssh-agent"}) + assert p.returncode == 0, p.stderr + + +@pytest.mark.smoke +def test_preflight_rejects_an_unimportable_settings_overlay(): + """Test that a settings overlay ARC would silently drop is reported as fatal.""" + with tempfile.TemporaryDirectory() as td: + pathlib.Path(td, "settings.py").write_text( + "import a_module_that_is_not_installed\n" + REMOTE_SETTINGS) + p = run_preflight(td) + assert p.returncode == 1, f"expected a fatal pre-flight, got {p.returncode}: {p.stdout}" + assert "could not be imported" in p.stderr, p.stderr + assert "dummy server settings" in p.stderr, p.stderr + + +@pytest.mark.smoke +def test_preflight_warns_about_a_key_that_is_missing_in_the_container(): + """Test that a 'key' pointing outside the container is reported, but is not fatal. + + A server that is configured but unused in this run must not abort it. + """ + with tempfile.TemporaryDirectory() as td: + pathlib.Path(td, "settings.py").write_text(REMOTE_SETTINGS.replace( + "'un': 'smoke_user',", "'un': 'smoke_user',\n 'key': '/no/such/key',")) + p = run_preflight(td) + assert p.returncode == 0, p.stderr + assert "/no/such/key" in p.stderr, p.stderr + + +@pytest.mark.smoke +def test_preflight_accepts_a_server_without_a_key(): + """Test that omitting 'key' is accepted when an agent is forwarded. + + This is the preferred remote-submission setup: no key file exists in the container at all, + and paramiko authenticates through the forwarded agent. + """ + with tempfile.TemporaryDirectory() as td: + pathlib.Path(td, "settings.py").write_text(REMOTE_SETTINGS) + p = run_preflight(td, env={"SSH_AUTH_SOCK": "/ssh-agent"}) + assert p.returncode == 0, p.stderr + assert "sets no 'key'" not in p.stderr, p.stderr + + +@pytest.mark.smoke +def test_preflight_warns_when_strict_host_key_checking_has_no_known_hosts(): + """Test that strict host key checking without a seeded known_hosts is reported.""" + with tempfile.TemporaryDirectory() as td: + pathlib.Path(td, "settings.py").write_text(REMOTE_SETTINGS.replace( + "'un': 'smoke_user',", "'un': 'smoke_user',\n 'strict_host_key_checking': True,")) + p = run_preflight(td, env={"HOME": td, "SSH_AUTH_SOCK": "/ssh-agent"}) + assert p.returncode == 0, p.stderr + assert "known_hosts" in p.stderr, p.stderr + + +@pytest.mark.smoke +def test_mounted_arc_settings_are_readable(): + """Test that a mounted ~/.arc is readable by the container user. + + Skipped when nothing is mounted there, which is the case for local-only runs. + """ + settings_file = pathlib.Path(ARC_SETTINGS_DIR, "settings.py") + if not settings_file.is_file(): + pytest.skip(f"no settings overlay mounted at {ARC_SETTINGS_DIR}") + assert os.access(settings_file, os.R_OK), \ + f"{settings_file} is not readable by the container user; " \ + f"re-run with -e PUID=$(id -u) -e PGID=$(id -g)" + p = run_preflight(ARC_SETTINGS_DIR) + assert p.returncode == 0, f"the mounted settings overlay is unusable:\n{p.stderr}" + + +@pytest.mark.smoke +@pytest.mark.skipif(not os.environ.get("ARC_SMOKE_SSH_HOST"), + reason="live remote cluster test; set ARC_SMOKE_SSH_HOST and ARC_SMOKE_SSH_USER to run") +def test_live_remote_ssh_connection(): + """Test an actual SSH connection to a remote cluster. Requires a live server, opt-in only. + + No missing-host-key policy is set, so paramiko's default RejectPolicy applies and the host + must already be in /home/mambauser/.ssh/known_hosts. Seed it first, e.g. by mounting the + host's file or running ssh-keyscan, or the test fails on the host key rather than on the + connection it means to exercise. + """ + code = r""" +import os +import paramiko +client = paramiko.SSHClient() +client.load_system_host_keys() +client.connect(hostname=os.environ['ARC_SMOKE_SSH_HOST'], + username=os.environ.get('ARC_SMOKE_SSH_USER'), + banner_timeout=200) +_, stdout, _ = client.exec_command('echo remote OK') +print(stdout.read().decode().strip()) +client.close() +""" + cmd = ["bash", "-lc", f"micromamba run -n arc_env python - <<'PY'\n{code}\nPY"] + p = subprocess.run(cmd, capture_output=True, text=True) + assert p.returncode == 0, p.stderr + assert "remote OK" in p.stdout diff --git a/dockerfiles/entrywrapper.sh b/dockerfiles/entrywrapper.sh index 43cdde4c8e..c0da6078a2 100644 --- a/dockerfiles/entrywrapper.sh +++ b/dockerfiles/entrywrapper.sh @@ -1,27 +1,223 @@ #!/usr/bin/env bash set -euo pipefail +SSH_DIR="/home/mambauser/.ssh" +ARC_SETTINGS_DIR="/home/mambauser/.arc" +ARC_PREFLIGHT="/usr/local/bin/arc_preflight.py" + +# Key file names paramiko looks for under $HOME/.ssh when no explicit key is supplied. +# (paramiko >= 4 dropped DSA support, hence no id_dsa.) +SSH_DEFAULT_KEYS=(id_rsa id_ecdsa id_ed25519) + +warn() { echo "entrywrapper: warning: $*" >&2; } + +# True when "$1" sits on a different device than its parent directory, i.e. it is a bind +# mount from the host. Such paths must never be chown'ed or chmod'ed: they may be mounted +# read-only (the chown would simply fail), and when they are writable the change would +# leak out to the user's real files on the host. +is_bind_mount() { + local path="$1" parent dev_path dev_parent + parent="$(dirname "$path")" + dev_path="$(stat -c %d "$path" 2>/dev/null || echo 'no-such-path')" + dev_parent="$(stat -c %d "$parent" 2>/dev/null || echo 'no-such-parent')" + [[ "$dev_path" != "$dev_parent" ]] +} + +# The single quotes are deliberate: $1 must be expanded by the inner shell, not by this one. +# shellcheck disable=SC2016 +readable_by_mambauser() { + runuser -u mambauser -- bash -c '[[ -r "$1" ]]' _ "$1" +} + +# shellcheck disable=SC2016 +rw_by_mambauser() { + runuser -u mambauser -- bash -c '[[ -r "$1" && -w "$1" ]]' _ "$1" +} + +# Make a forwarded SSH agent socket usable after the privilege drop, or disable it. +# Sets SSH_AUTH_SOCK to "" when the socket cannot be used, so the SSH client falls back to +# key files instead of failing against a socket it cannot open. +# +# The supported way to gain access is the PUID/PGID remap above: an agent socket is mode 0600 +# and owned by the host user, so a container user with that same uid can open it and nothing +# needs changing. Widening the socket's mode is NOT done by default. The bind mount shares the +# inode with the host, so `chmod o+rw` mutates the user's real, live agent socket, making it +# readable and writable by every other user on the host for as long as their agent runs. The +# entrypoint cannot undo it either: it hands off with `exec`, so no EXIT trap would ever fire. +# That contradicts the rule applied to bind-mounted paths everywhere else in this file, so it +# is available only on explicit request via ARC_WIDEN_AGENT_SOCKET=1, and says what it did. +prepare_ssh_agent_socket() { + local sock="${SSH_AUTH_SOCK:-}" + if [[ -z "$sock" ]]; then + return 0 + fi + if [[ ! -S "$sock" ]]; then + warn "SSH_AUTH_SOCK is set to '$sock', which is not a socket inside the container; ignoring it." + warn "forward the host agent with: -v \"\$SSH_AUTH_SOCK:/ssh-agent\" -e SSH_AUTH_SOCK=/ssh-agent" + SSH_AUTH_SOCK="" + return 0 + fi + if rw_by_mambauser "$sock"; then + return 0 + fi + if [[ "${ARC_WIDEN_AGENT_SOCKET:-0}" == "1" ]]; then + if chmod o+rw "$sock" && rw_by_mambauser "$sock"; then + warn "ARC_WIDEN_AGENT_SOCKET=1: relaxed the mode of the agent socket '$sock' to $(stat -c %a "$sock")." + warn "this changed the socket ON THE HOST, where it is now readable and writable by any" + warn "local user, and it is NOT restored when this container exits." + warn "restore it yourself afterwards with: chmod 600 \"\$SSH_AUTH_SOCK\"" + return 0 + fi + warn "ARC_WIDEN_AGENT_SOCKET=1 was set, but relaxing the mode did not make '$sock' usable." + fi + warn "the forwarded SSH agent socket '$sock' is not accessible to mambauser (uid $(id -u mambauser))." + warn "re-run with -e PUID=\$(id -u) -e PGID=\$(id -g) so the container user matches the socket's owner;" + warn "that is the supported fix and it leaves the socket on the host untouched." + warn "continuing without agent forwarding." + SSH_AUTH_SOCK="" +} + +# Prepare /home/mambauser/.ssh. A bind-mounted .ssh is left completely untouched, including +# when it is read-only: paramiko, unlike the OpenSSH CLI, does not require 0600 on key files, +# so a read-only mount owned by the host user works as long as it is readable. +prepare_ssh_dir() { + if [[ ! -e "$SSH_DIR" ]]; then + return 0 + fi + if is_bind_mount "$SSH_DIR"; then + if ! readable_by_mambauser "$SSH_DIR"; then + warn "the mounted $SSH_DIR is not readable by mambauser (uid $(id -u mambauser))." + warn "re-run with -e PUID=\$(id -u) -e PGID=\$(id -g) so the container user matches the mount owner." + fi + return 0 + fi + if ! chown mambauser:mambauser "$SSH_DIR" 2>/dev/null; then + warn "failed to change ownership of $SSH_DIR to mambauser:mambauser." + fi + local key + for key in "${SSH_DEFAULT_KEYS[@]}"; do + if [[ -e "$SSH_DIR/$key" ]] && ! readable_by_mambauser "$SSH_DIR/$key"; then + warn "$SSH_DIR/$key exists but is not readable by mambauser." + fi + done +} + +# Fail fast on a mis-mounted ARC settings overlay. +# +# ARC reads $HOME/.arc/{settings,submit,inputs}.py, and arc/imports.py swallows an ImportError +# from settings.py without a word: ARC then runs against the repository's *dummy* servers +# ('server1.host.edu', ''). In a container a mis-typed mount path or a settings.py +# whose own imports are unavailable produces exactly that, and the only symptom is a run that +# fails hours later against a host that never existed. arc_preflight.py imports the overlay the +# same way ARC does and reports what it finds; an overlay that is present but unimportable is a +# configuration error, not something to continue past. +# +# Set ARC_SKIP_PREFLIGHT=1 to bypass this entirely. +preflight_arc_settings() { + if [[ "${ARC_SKIP_PREFLIGHT:-0}" == "1" ]]; then + return 0 + fi + if [[ ! -f "$ARC_SETTINGS_DIR/settings.py" ]]; then + warn "no settings.py under $ARC_SETTINGS_DIR, so ARC will use its dummy server settings." + warn "to drive a remote cluster, mount your settings with -v \"\$HOME/.arc:$ARC_SETTINGS_DIR:ro\"" + warn "if you did mount it, the source side must be the directory *containing* settings.py." + return 0 + fi + if [[ ! -f "$ARC_PREFLIGHT" ]]; then + warn "$ARC_PREFLIGHT is missing from the image; skipping the settings pre-flight check." + return 0 + fi + local output status=0 + output="$(micromamba run -n arc_env python "$ARC_PREFLIGHT" "$ARC_SETTINGS_DIR" 2>&1)" || status=$? + if [[ -n "$output" ]]; then + printf '%s\n' "$output" >&2 + fi + if [[ "$status" -ne 0 ]]; then + echo "Error: the ARC settings overlay mounted at $ARC_SETTINGS_DIR is unusable (see above)." >&2 + exit 78 # EX_CONFIG + fi +} + +# Highest ID reserved for system accounts on Debian/Ubuntu. Sharing one of those with +# mambauser would hand it that account's file access, so such a collision stays fatal. +SYSTEM_ID_MAX=999 + +# True when a running process belongs to the given real UID (or GID, with field "Gid:"). +# Read from /proc rather than via ps, which the image does not install. +id_has_running_process() { + local field="$1" id="$2" + [[ -r /proc/1/status ]] || return 1 + awk -v field="$field:" -v want="$id" \ + '$1 == field && $2 == want { found = 1 } END { exit !found }' /proc/[0-9]*/status 2>/dev/null +} + +# Explain an ID collision that must stay fatal, naming the flag to drop. Always returns 1. +refuse_remap() { + local kind="$1" id="$2" holder="$3" reason="$4" var + var="P${kind^^}" + echo "Error: cannot remap mambauser to $kind $id: it is already held by '$holder', $reason." >&2 + echo "Drop -e $var=$id to keep mambauser's built-in IDs, or pass an unused ID instead." >&2 + echo "Bind-mounted files would then be owned by uid $(id -u mambauser)/gid $(id -g mambauser)." >&2 + return 1 +} + +# Decide whether a requested ID that another account already holds may be shared with +# mambauser. Sharing is safe for an ordinary, idle account: file permissions are numeric, so +# mambauser gets exactly the access the bind mounts need, and nothing is deleted or renamed. +may_share_id() { + local kind="$1" id="$2" holder="$3" field="$4" + if [[ "$id" -eq 0 ]]; then + refuse_remap "$kind" "$id" "$holder" "the superuser" + return 1 + fi + if [[ "$id" -le "$SYSTEM_ID_MAX" ]]; then + refuse_remap "$kind" "$id" "$holder" "a system account reserved by the distribution" + return 1 + fi + if id_has_running_process "$field" "$id"; then + refuse_remap "$kind" "$id" "$holder" "an account with running processes" + return 1 + fi + warn "$kind $id is also held by the idle account '$holder'; sharing it with mambauser." + return 0 +} + # 0) If root, optionally remap mambauser UID/GID for bind mounts, then drop privileges. +# +# A collision here used to be fatal outright. The base image ships an unused 'ubuntu' account at +# 1000:1000, which is exactly what `-e PUID=$(id -u) -e PGID=$(id -g)` asks for on a typical +# Linux desktop, so that rule aborted the container for most users. The Dockerfile now removes +# that account, but the collision can return from a rebased base image or a derived one, so it +# is also handled here instead of trusting the image alone. if [[ "$(id -u)" -eq 0 ]]; then if [[ -n "${PGID:-}" ]]; then - existing_group="" - if getent group "$PGID" >/dev/null; then - existing_group="$(getent group "$PGID" | cut -d: -f1)" - fi - if [[ -n "$existing_group" && "$existing_group" != "mambauser" ]]; then - echo "Error: requested PGID '$PGID' is already in use by group '$existing_group', cannot remap mambauser group." >&2 - exit 1 + current_gid="$(id -g mambauser)" + if [[ "$PGID" != "$current_gid" ]]; then + existing_group="" + if getent group "$PGID" >/dev/null; then + existing_group="$(getent group "$PGID" | cut -d: -f1)" + fi + if [[ -n "$existing_group" && "$existing_group" != "mambauser" ]]; then + may_share_id gid "$PGID" "$existing_group" Gid || exit 1 + groupmod -o -g "$PGID" mambauser + else + groupmod -g "$PGID" mambauser + fi fi - groupmod -g "$PGID" mambauser fi if [[ -n "${PUID:-}" ]]; then current_uid="$(id -u mambauser)" if [[ "$PUID" != "$current_uid" ]]; then + existing_user="" if getent passwd "$PUID" >/dev/null; then - echo "Error: Cannot remap mambauser to UID '$PUID' because it is already in use by another user." >&2 - exit 1 + existing_user="$(getent passwd "$PUID" | cut -d: -f1)" + fi + if [[ -n "$existing_user" && "$existing_user" != "mambauser" ]]; then + may_share_id uid "$PUID" "$existing_user" Uid || exit 1 + usermod -o -u "$PUID" mambauser + else + usermod -u "$PUID" mambauser fi - usermod -u "$PUID" mambauser fi fi @@ -36,8 +232,23 @@ if [[ "$(id -u)" -eq 0 ]]; then fi fi + # SSH setup runs after the UID/GID remap so the checks reflect the final mambauser IDs. + prepare_ssh_agent_socket + prepare_ssh_dir + if [[ "${ENTRYWRAPPER_AS_USER:-0}" != "1" ]]; then - exec runuser -u mambauser -- env ENTRYWRAPPER_AS_USER=1 /usr/local/bin/entrywrapper.sh "$@" + # runuser resets HOME/SHELL/USER/LOGNAME/PATH. SSH_AUTH_SOCK is passed on explicitly so + # that it survives the privilege drop, and is unset explicitly when the socket was found + # to be unusable above. + drop_env=() + if [[ -z "${SSH_AUTH_SOCK:-}" ]]; then + drop_env+=(-u SSH_AUTH_SOCK) + fi + drop_env+=(ENTRYWRAPPER_AS_USER=1) + if [[ -n "${SSH_AUTH_SOCK:-}" ]]; then + drop_env+=("SSH_AUTH_SOCK=$SSH_AUTH_SOCK") + fi + exec runuser -u mambauser -- env "${drop_env[@]}" /usr/local/bin/entrywrapper.sh "$@" fi fi @@ -69,6 +280,14 @@ Run with a bind mount so the container can read your input file, e.g. Notes: - / must be a non-flag argument - if you pass flags (e.g. -n 8), put them before the file: rmg -n 8 input.py +- to let ARC submit ESS jobs to a remote cluster over SSH, either forward your host SSH + agent with -v "$SSH_AUTH_SOCK:/ssh-agent" -e SSH_AUTH_SOCK=/ssh-agent (preferred, keys + never enter the container), or mount your keys read-only with + -v "$HOME/.ssh:/home/mambauser/.ssh:ro"; pass -e PUID=$(id -u) -e PGID=$(id -g) so the + container user matches the owner of those mounts +- your server definitions live in your personal ARC settings, which must be mounted with + -v "$HOME/.arc:/home/mambauser/.arc:ro"; without it ARC runs against its dummy servers. + `arc` checks this before starting and refuses to run on an unimportable settings.py USAGE } @@ -118,6 +337,7 @@ case "$cmd" in # no defaults: user must provide their file path if [[ "$cmd" == "arc" ]]; then + preflight_arc_settings exec micromamba run -n arc_env python /home/mambauser/Code/ARC/ARC.py "$@" else exec micromamba run -n rmg_env python /home/mambauser/Code/RMG-Py/rmg.py "$@" From 187767ad6494bfa09bb3b06cfbf18eae546eca83 Mon Sep 17 00:00:00 2001 From: Calvin Pieters Date: Sat, 22 Aug 2026 13:08:59 +0300 Subject: [PATCH 8/9] Document remote submission over SSH and from the Docker image The container and SSH halves of remote submission were in place; what was missing was the user-facing configuration around them. These docs are written against the `key` and host-key semantics this branch introduces, not against main's. docs/source/remote_submission.rst (new, in the toctree): authentication via a forwarded agent (preferred -- keys never enter the container and passphrase-protected keys keep working) or via a mounted key file; host key verification and the new per-server strict_host_key_checking, including why an unseeded known_hosts matters in a fresh container now that WarningPolicy and RejectPolicy have replaced the silent AutoAdd; the ~/.arc overlay mount, which a remote run needs as much as the SSH material since submit.py carries the cluster's PBS/Slurm templates; both `docker run` invocations and the compose equivalent; and the entrypoint's exit codes. Two limitations are documented rather than worked around: - ARC never builds a paramiko.SSHConfig, so ~/.ssh/config is not read at all and ProxyJump/bastion hosts are unsupported. This is true on bare metal too, and is called out so nobody blames the container for it. - a default-bridge container reaches an ordinary login node with no extra flags, since paramiko speaks SSH itself; the exceptions are a host VPN whose routing excludes docker0, and internal names served only by a VPN-pushed resolver. Each claim is checked against the code rather than assumed: paramiko's load_system_host_keys() reads ~/.ssh/known_hosts and nothing else, so /etc/ssh/ssh_known_hosts is not mentioned as an alternative (seeding it under strict_host_key_checking would have refused every connection); a rejected host key raises into the same 24-hour retry loop and so presents as a hang rather than a fast failure; the retry reason reaches the logger only on every tenth attempt, the others going to stdout; and Docker materialises a missing bind-mount source as a root-owned directory, which is what a stale SSH_AUTH_SOCK or an absent ~/.arc produces on the host. installation.rst and running.rst described `key` as a private key path, which was wrong on main and is right as of this branch; they now say so, present the agent route as the default, and mention strict_host_key_checking. docker.rst gains the remote-submission pointer, index.rst the toctree entry, and the stray `key` in the advanced.rst node-limits example is dropped, since that example is about cpus and memory. remote_submission.rst now also states why warning rather than rejecting is the default host-key policy -- a refused key does not fail a long-running scheduler once but starves it while the driver stays alive, and it presents as a hang rather than an error -- and how to opt into refusal per server. The Startup Checks section leads with the check ARC itself performs on every run, in a container or not, and describes the compose file's known_hosts mount and why ARC_KNOWN_HOSTS defaults to /dev/null. running.rst gains a pointer to the same startup report. Absorbed from PR #1000 by @alongd, brought in here rather than merged so the two pull requests do not conflict over the same lines: keep_checks is documented as covering the servers a project ran on, not only the local project directory. Document the server 'path' key and give every remote server example one. No example defined it, so the documented remote configuration was the one in which an input file that must name a path on the server, such as Orca NEB's, cannot be written. --- docs/source/advanced.rst | 2 +- docs/source/docker.rst | 4 +- docs/source/index.rst | 1 + docs/source/installation.rst | 39 ++- docs/source/remote_submission.rst | 476 ++++++++++++++++++++++++++++++ docs/source/running.rst | 23 +- 6 files changed, 536 insertions(+), 9 deletions(-) create mode 100644 docs/source/remote_submission.rst diff --git a/docs/source/advanced.rst b/docs/source/advanced.rst index 10f7560a68..c8c1325c5d 100644 --- a/docs/source/advanced.rst +++ b/docs/source/advanced.rst @@ -288,8 +288,8 @@ Server entries can also define node limits: 'my_slurm': { 'cluster_soft': 'Slurm', 'address': 'login.cluster.edu', + 'path': '/home', 'un': 'my_user', - 'key': '/home/my_user/.ssh/id_rsa', 'cpus': 32, 'memory': 128, }, diff --git a/docs/source/docker.rst b/docs/source/docker.rst index bf61b220b2..ec06718ca2 100644 --- a/docs/source/docker.rst +++ b/docs/source/docker.rst @@ -59,7 +59,9 @@ Open an interactive shell:: laxzal/arc:latest For job submission, the scheduler client tools must be available in the container -or accessed via SSH on a remote host. +or accessed via SSH on a remote host. To submit to a remote cluster you also need +to mount your personal ARC settings and your SSH material into the container; see +:ref:`remote_submission` for the required mounts and the two authentication routes. Aliases in interactive shells ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/docs/source/index.rst b/docs/source/index.rst index d2370a07ab..bc3b0055e3 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -58,6 +58,7 @@ Reference installation docker running + remote_submission how_it_works input_reference examples diff --git a/docs/source/installation.rst b/docs/source/installation.rst index dc2aa1aecf..81e5be44c3 100644 --- a/docs/source/installation.rst +++ b/docs/source/installation.rst @@ -175,14 +175,18 @@ defaults. Missing values fall back to ARC's repository defaults. Remote Servers and SSH ---------------------- -Use SSH keys for remote servers. A remote server entry needs: +Remote servers are reached over SSH. A remote server entry needs: * ``cluster_soft`` - one of the cluster software names configured in ARC; * ``address`` - SSH hostname; * ``un`` - username; -* ``key`` - path to the private SSH key on the machine running ARC. +* ``path`` - the absolute path on the server holding the user directories, under which + ARC runs. See :doc:`remote_submission`. -Example: +Authentication is by SSH key. The recommended setup is to load the key into an +ssh-agent and leave the server entry without a ``key``, in which case ARC falls back +to the agent and then to the default key paths (``~/.ssh/id_rsa``, +``~/.ssh/id_ecdsa``, ``~/.ssh/id_ed25519``): .. code-block:: python @@ -190,13 +194,40 @@ Example: 'my_slurm': { 'cluster_soft': 'Slurm', 'address': 'login.cluster.edu', + 'path': '/home', 'un': 'my_user', - 'key': '/home/my_user/.ssh/id_rsa', 'cpus': 32, 'memory': 128, }, } +Two optional keys control the connection: + +* ``key`` - the path, on the machine running ARC, of the SSH **private** key to + authenticate with. Set it only when you are not using an agent. The file must + exist and be readable, or the connection fails. +* ``strict_host_key_checking`` - ``False`` by default, meaning a host that is + absent from ``known_hosts`` is only warned about. Set it to ``True`` to refuse + such hosts outright, having first seeded ``~/.ssh/known_hosts``. + +.. code-block:: python + + servers = { + 'my_slurm': { + 'cluster_soft': 'Slurm', + 'address': 'login.cluster.edu', + 'path': '/home', + 'un': 'my_user', + 'key': '/home/my_user/.ssh/id_ed25519', + 'strict_host_key_checking': True, + 'cpus': 32, + 'memory': 128, + }, + } + +See :ref:`remote_submission` for the full details, including how to do this from +the Docker image. + Local and HPC Execution ----------------------- diff --git a/docs/source/remote_submission.rst b/docs/source/remote_submission.rst new file mode 100644 index 0000000000..52dd5d827d --- /dev/null +++ b/docs/source/remote_submission.rst @@ -0,0 +1,476 @@ +.. _remote_submission: + +Remote Job Submission over SSH +============================== + +ARC submits electronic structure jobs to a remote cluster by opening an SSH session +with paramiko. This page describes how to authenticate that session, how host keys +are verified, and how to do both from inside the Docker image. + +If ARC and the electronic structure software run on the same machine, you do not +need any of this - define a server named ``local`` instead, as described in +:ref:`running`. + +Server Settings +--------------- + +Remote servers are declared in the ``servers`` dictionary of your personal +``~/.arc/settings.py``: + +.. code-block:: python + + servers = { + 'cluster_a': { + 'cluster_soft': 'Slurm', + 'address': 'login.cluster.edu', + 'path': '/home', + 'un': 'my_user', + }, + } + +The keys that affect the SSH connection itself are: + +* ``address`` - the hostname ARC connects to; +* ``un`` - the username on the remote machine; +* ``path`` - the **absolute** path on the server holding the user directories, under + which ARC runs (``//runs/ARC_Projects/``). Without it the remote job + directories are relative to the SSH login directory, which an input file that must + name a path on the server, such as Orca NEB's, cannot follow; ARC reports such a + server at startup; +* ``key`` - **optional**. The path, on the machine running ARC, of the SSH + **private key** to authenticate with; +* ``strict_host_key_checking`` - optional, ``False`` by default. See + `Host Key Verification`_. + +Authentication +-------------- + +ARC hands ``key`` to paramiko as the identity to authenticate with. There are two +supported ways to authenticate, and the choice is made simply by whether you set +``key``. + +Using an ssh-agent (preferred) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Omit ``key`` entirely. paramiko then looks for a running ssh-agent, and after that +for the default key paths ``~/.ssh/id_rsa``, ``~/.ssh/id_ecdsa`` and +``~/.ssh/id_ed25519``. There is no ``id_dsa`` in that list; paramiko 4 dropped DSA. + +.. code-block:: python + + servers = { + 'cluster_a': { + 'cluster_soft': 'Slurm', + 'address': 'login.cluster.edu', + 'path': '/home', + 'un': 'my_user', + }, + } + +Add the key to your agent once per login session: + +.. code-block:: bash + + ssh-add ~/.ssh/id_ed25519 + +This is the preferred route. Passphrase-protected keys keep working, because the +agent holds the decrypted key and ARC never has to prompt for the passphrase. In a +container it also means the key material itself never has to enter the container. + +Using a key file +^^^^^^^^^^^^^^^^ + +Set ``key`` to the path of the private key: + +.. code-block:: python + + servers = { + 'cluster_a': { + 'cluster_soft': 'Slurm', + 'address': 'login.cluster.edu', + 'path': '/home', + 'un': 'my_user', + 'key': '/home/my_user/.ssh/id_ed25519', + }, + } + +The path must exist and be readable **on the machine running ARC**, and it must +name the private key, not the public ``.pub`` half and not ``known_hosts``. +paramiko raises if it cannot read the file, and ARC then retries the connection for +up to 24 hours, so a wrong path shows up as a run that appears to hang rather than +as an immediate error. + +``~/.ssh/config`` is not read +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +ARC connects with paramiko, and paramiko only applies ``~/.ssh/config`` when the +application explicitly builds a ``paramiko.SSHConfig``. ARC does not, so **no** +directive in that file has any effect: not ``IdentityFile``, not ``User``, not +``HostName``, not ``Port``, and not ``ProxyJump`` or ``ProxyCommand``. + +The practical consequences are: + +* every connection detail must be spelled out in the server entry - ``address``, + ``un``, and ``key`` if you use one - even when your ``ssh`` command line works + without them; +* **bastion and jump hosts are not supported**. A cluster that can only be reached + through a jump host cannot be driven by ARC directly. Run ARC on a machine that + has direct access to the login node instead, or establish the tunnel outside ARC + and point ``address`` at the local end of it. + +This is a property of ARC on any machine, not something introduced by running it +in a container. + +Host Key Verification +--------------------- + +ARC loads the system host keys from paramiko's default location, which is +``~/.ssh/known_hosts`` on the machine running ARC, and nothing else. In particular +``/etc/ssh/ssh_known_hosts`` is **not** read, even though the OpenSSH command line +client reads it; paramiko's ``load_system_host_keys()`` consults the user file only, +and silently ignores it if it cannot be read. The path is not configurable from +``settings.py``. + +What happens when a host is *not* in ``known_hosts`` depends on the per-server +``strict_host_key_checking`` flag: + +.. list-table:: + :header-rows: 1 + :widths: 22 78 + + * - Value + - Behavior for an unknown host + * - ``False`` (default) + - The connection proceeds and a warning is logged. Convenient, but a + machine-in-the-middle is indistinguishable from a first-ever connection, + so the warning is the only signal you get. + * - ``True`` + - The connection is refused. The host must be present in ``known_hosts`` + before ARC can reach it. The refusal is immediate -- it raises its own + exception type, which ARC does not retry -- so it is reported as an error + naming the host rather than as a silent wait. Seed ``known_hosts`` first. + +.. code-block:: python + + servers = { + 'cluster_a': { + 'cluster_soft': 'Slurm', + 'address': 'login.cluster.edu', + 'path': '/home', + 'un': 'my_user', + 'strict_host_key_checking': True, + }, + } + +Seed ``known_hosts`` before enabling the flag: + +.. code-block:: bash + + ssh-keyscan -H login.cluster.edu >> ~/.ssh/known_hosts + +Verify the fingerprints against a trusted source before trusting the result - +``ssh-keyscan`` trusts whatever answers on the network. + +Why warning is the default +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Refusing an unknown host is the safer policy in the abstract, and it is a poor +default for ARC specifically, because ARC is a scheduler that runs unattended for +days rather than a client that makes one connection. + +A refused host key does not fail the run once and stop. Every submission, status +poll and download for that server fails while the ARC driver stays alive, so the run +continues, submits nothing, and looks like a stall. Each individual refusal is +reported at once rather than retried, but nothing takes the driver down, so whoever +is running it typically finds out hours later, and recovery means stopping ARC, +running ``ssh-keyscan``, and starting again. +Weighed against a first-ever connection to a login node reached over a network the +user already trusts enough to submit jobs to, that failure mode costs more than the +risk it removes. + +What makes the default defensible is that the risk is *reported*, not hidden. ARC +checks ``known_hosts`` at startup (see `Startup Checks`_), so an unseeded host is +named before any calculation is submitted, and +``'strict_host_key_checking': True`` remains available per server for anyone who +wants the connection refused instead. + +Running in the Docker Image +--------------------------- + +Everything above applies unchanged inside the container; what changes is that +``~/.arc``, the SSH material, and possibly the agent socket have to be bind-mounted +in. See :ref:`docker` for the image's general usage. + +Mounting your ARC settings +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The container user is ``mambauser`` with ``HOME=/home/mambauser``, so your personal +settings must be mounted at ``/home/mambauser/.arc``: + +.. code-block:: bash + + -v "$HOME/.arc:/home/mambauser/.arc:ro" + +ARC reads ``settings.py``, ``submit.py`` and ``inputs.py`` from that directory. All +three matter for remote submission: ``submit.py`` holds the cluster's PBS/Slurm +submit script templates, and without it ARC will submit jobs with the repository's +generic templates. + +Two properties of that overlay are worth knowing: + +* It is a **replacement of top-level names**, not a deep merge. A ``settings.py`` + that defines only ``servers`` replaces the whole ``servers`` dictionary and + leaves every other setting at its repository default - which is what you + usually want - but a ``settings.py`` that defines, say, ``levels_ess`` replaces + that whole dictionary too. +* Whenever a local ``settings.py`` exists, ARC forces ``global_ess_settings`` to + ``None`` unless that file defines a truthy value of its own. This is deliberate: + the repository defaults are dummies. If you route software to servers through + ``global_ess_settings``, define it in your own ``settings.py``. + +Mount the directory read-only, and set ``PYTHONDONTWRITEBYTECODE=1`` so Python does +not try to write ``__pycache__`` into a read-only mount: + +.. code-block:: bash + + -e PYTHONDONTWRITEBYTECODE=1 + +With agent forwarding (preferred) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Forward the host's ssh-agent socket and omit ``key`` from the server entry. No key +material enters the container: + +.. code-block:: bash + + docker run --rm \ + -v "$PWD:/work" -w /work \ + -e PUID=$(id -u) -e PGID=$(id -g) \ + -e PYTHONDONTWRITEBYTECODE=1 \ + -v "$HOME/.arc:/home/mambauser/.arc:ro" \ + -v "$HOME/.ssh/known_hosts:/home/mambauser/.ssh/known_hosts:ro" \ + -v "$SSH_AUTH_SOCK:/ssh-agent" -e SSH_AUTH_SOCK=/ssh-agent \ + laxzal/arc:latest arc my_case/input.yml + +On macOS, Docker Desktop exposes the host agent at a fixed path, so use +``-v /run/host-services/ssh-auth.sock:/ssh-agent`` instead of ``$SSH_AUTH_SOCK``. + +``PUID``/``PGID`` remap the ``mambauser`` account to your host UID/GID. This is not +optional for agent forwarding, it is the mechanism: an agent socket is mode ``0600`` +and owned by you, so only a container user carrying your UID can open it. + +The entrypoint deliberately does **not** relax the socket's permissions to work +around a missing remap. A bind mount shares the inode with the host, so doing that +would make your real, live agent socket readable and writable by every other user on +the machine, for as long as the agent runs - and because the entrypoint hands off +with ``exec``, nothing would ever restore it. If you genuinely need that behaviour, +set ``ARC_WIDEN_AGENT_SOCKET=1``; the entrypoint will then relax the mode, say +exactly what it changed, and remind you to run ``chmod 600 "$SSH_AUTH_SOCK"`` on the +host afterwards. Passing ``PUID``/``PGID`` is almost always the better answer. + +A note on stale sockets: if ``$SSH_AUTH_SOCK`` points at an agent that has since +exited, Docker creates the missing bind-mount source rather than failing, leaving a +root-owned directory at that path on your host. The entrypoint reports it as "not a +socket" and continues without agent forwarding. The same applies to ``$HOME/.arc`` +if that directory does not exist. + +With a mounted key (fallback) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +For headless runs where no agent is available, mount the key read-only and point +``key`` at the path **inside** the container: + +.. code-block:: python + + servers = { + 'cluster_a': { + 'cluster_soft': 'Slurm', + 'address': 'login.cluster.edu', + 'path': '/home', + 'un': 'my_user', + 'key': '/home/mambauser/.ssh/id_ed25519', + }, + } + +.. code-block:: bash + + docker run --rm \ + -v "$PWD:/work" -w /work \ + -e PUID=$(id -u) -e PGID=$(id -g) \ + -e PYTHONDONTWRITEBYTECODE=1 \ + -v "$HOME/.arc:/home/mambauser/.arc:ro" \ + -v "$HOME/.ssh/id_ed25519:/home/mambauser/.ssh/id_ed25519:ro" \ + -v "$HOME/.ssh/known_hosts:/home/mambauser/.ssh/known_hosts:ro" \ + laxzal/arc:latest arc my_case/input.yml + +A read-only mount is fine: ARC uses paramiko, which unlike the OpenSSH command line +client does not refuse key files with permissive modes. The key must be +passphrase-free, since there is nothing to prompt for it in a batch run. + +To mount the whole directory instead of individual files, use +``-v "$HOME/.ssh:/home/mambauser/.ssh:ro"``. The entrypoint detects that +``/home/mambauser/.ssh`` is a bind mount and leaves its ownership and modes alone, +so nothing leaks back to your host files. + +Docker Compose +^^^^^^^^^^^^^^ + +``docker-compose.yml`` in the repository root wires all of this up already: + +.. code-block:: bash + + ARC_WORKDIR=$PWD ARC_INPUT=my_case/input.yml \ + PUID=$(id -u) PGID=$(id -g) \ + docker compose run --rm arc + +It mounts ``$HOME/.arc`` read-only, forwards ``$SSH_AUTH_SOCK`` to ``/ssh-agent``, +mounts ``$ARC_KNOWN_HOSTS`` read-only at ``/home/mambauser/.ssh/known_hosts``, and +sets ``PYTHONDONTWRITEBYTECODE=1``. The key-file mount is present but commented out; +uncomment it if you are not using an agent. + +Point ``ARC_KNOWN_HOSTS`` at your host keys to share them with the container: + +.. code-block:: bash + + ARC_WORKDIR=$PWD ARC_INPUT=my_case/input.yml \ + ARC_KNOWN_HOSTS=$HOME/.ssh/known_hosts \ + PUID=$(id -u) PGID=$(id -g) \ + docker compose run --rm arc + +The variable defaults to ``/dev/null``, which reads as an empty set of host keys, +rather than to ``$HOME/.ssh/known_hosts`` directly. Docker creates any bind-mount +source that does not exist, so naming that path unconditionally would leave a +root-owned *directory* at ``$HOME/.ssh/known_hosts`` on a host that has never +written the file - which then breaks ``ssh`` on the host itself. + +known_hosts in a fresh container +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +A fresh container starts with an empty ``/home/mambauser/.ssh``, so *every* host is +an unknown host. With the default ``strict_host_key_checking: False`` that only +produces a warning per connection, but with ``strict_host_key_checking: True`` it +refuses every connection until the file is seeded. paramiko reads it from its +default location, so the mount target has to be exactly +``/home/mambauser/.ssh/known_hosts`` - mounting it anywhere else has no effect. + +Either mount the host's file, as in the examples above, or seed one inside the +container. The image ships ``openssh-client`` for exactly this: + +.. code-block:: bash + + docker run --rm -it \ + -v "$HOME/.ssh:/home/mambauser/.ssh" \ + -e PUID=$(id -u) -e PGID=$(id -g) \ + laxzal/arc:latest \ + ssh-keyscan login.cluster.edu + +Append the verified output to the ``known_hosts`` file you then mount read-only. + +Network reachability +^^^^^^^^^^^^^^^^^^^^ + +A container on Docker's default bridge network reaches an ordinary cluster login +node without any extra flags: the host acts as a NAT router, DNS resolves, and +paramiko speaks SSH itself rather than shelling out to ``ssh``, so nothing but +outbound TCP on port 22 is required. ``--network host`` is not needed in the normal +case. + +Two situations do differ from running on the host: + +* **The cluster is reachable only over a VPN on the host.** Split-tunnel routing + and firewall rules frequently exclude the ``docker0`` bridge, so the container's + traffic never enters the tunnel. ``--network host`` is the escape hatch: the + container then shares the host's network namespace and its VPN routes. +* **Internal hostnames resolved by a VPN-pushed resolver.** The container uses + Docker's resolver, not the host's, so a name that only the VPN's DNS server knows + will not resolve. Use the IP address in ``address``, or pass + ``--dns ``. + +To tell the two apart, check reachability from inside the container before blaming +ARC: + +.. code-block:: bash + + docker run --rm laxzal/arc:latest \ + bash -lc 'getent hosts login.cluster.edu && ssh-keyscan -T 5 login.cluster.edu' + +A resolved address followed by a host key means the network path is fine and the +problem is authentication. + +Startup Checks +-------------- + +Every ARC run, in a container or not, reports the configured servers that have no +host key in ``~/.ssh/known_hosts`` before it submits anything. The message names the +server, its address, and the ``ssh-keyscan`` command that fixes it. It is a warning, +never fatal, and the check is entirely local: the ``known_hosts`` file is read, and +no name is resolved and no connection opened. + +Three kinds of server entry are skipped, because reporting them would be noise +rather than information: the ``local`` server, entries with no ``address``, and +entries still carrying the repository's placeholder ``*.host.edu`` address or +```` user name. + +Inside the container the entrypoint checks more, and earlier - before ARC itself +starts - because the failure modes there are otherwise invisible until much later in +the run. Most notably, ARC ignores a ``~/.arc/settings.py`` that fails to import +and silently continues with the repository's dummy servers +(``server1.host.edu``, ````) - in a container, a mis-typed mount path +produces exactly that. + +The entrypoint therefore: + +* refuses to start ARC, with exit code 78, if ``/home/mambauser/.arc/settings.py`` + exists but cannot be imported, printing the import traceback; +* warns if no ``settings.py`` is mounted at all; +* warns about a server whose ``key`` does not exist or is not readable inside the + container; +* warns about a server with no ``key`` when the container has neither a forwarded + agent nor a default key under ``/home/mambauser/.ssh``; +* warns about a server with ``strict_host_key_checking`` when no ``known_hosts`` + file is present. + +Only the unimportable ``settings.py`` is fatal. The rest are warnings, so a server +that is configured but not used in this particular run cannot abort it. Set +``ARC_SKIP_PREFLIGHT=1`` to skip the checks entirely. + +Exit codes from the entrypoint follow ``sysexits.h``: 64 for a usage error, 66 for +a missing input file, and 78 for a configuration error. + +Troubleshooting +--------------- + +**ARC connects to** ``server1.host.edu`` **, or reports unknown server names.** +Your ``~/.arc/settings.py`` was not picked up. Outside a container, check that the +file is at ``$HOME/.arc/settings.py`` and imports cleanly with +``python -c "import sys; sys.path.insert(0, '$HOME/.arc'); import settings"``. +Inside a container, check the mount target is ``/home/mambauser/.arc``. + +**The run appears to hang while connecting.** ARC retries a failed connection every +60 seconds, up to 1440 times, so a full 24 hours -- but only a transport-level one, +such as a login node that is down or a ``key`` pointing at a missing file. The +reason is reported on every attempt, but only every tenth attempt goes through the +logger, at info level; the others are written straight to standard output, so look +there as well as in the log if the run seems stuck. + +Failures that retrying cannot resolve are not retried at all: a rejected identity +(a wrong username, or a key the server does not accept), a host key that contradicts +``known_hosts``, and a rejected host key under ``strict_host_key_checking`` raise a +``ServerError`` naming the cause on the first attempt. + +**Permission denied, or the forwarded agent is not usable.** Pass +``-e PUID=$(id -u) -e PGID=$(id -g)``. Without it the container user's UID does not +match the owner of your mounts or of the agent socket, and the entrypoint will say +so. + +**"cannot remap mambauser to uid/gid N".** The requested ID belongs to an account +the container cannot safely displace - the superuser, or a system account reserved +by the distribution. Ordinary, idle accounts are shared automatically, so this only +appears for IDs below 1000. Drop the flag named in the message and mount ownership +will fall back to the image's own ``mambauser`` IDs. + +**Host key refused.** The server sets ``strict_host_key_checking: True`` and the +host is not in ``known_hosts``. Seed it with ``ssh-keyscan``, verify the +fingerprint, and mount the file at ``/home/mambauser/.ssh/known_hosts``. + +.. include:: links.txt diff --git a/docs/source/running.rst b/docs/source/running.rst index b6ddb4bd7c..f448dedd2f 100644 --- a/docs/source/running.rst +++ b/docs/source/running.rst @@ -174,8 +174,8 @@ Run over SSH ------------ Use SSH mode when ARC runs on your workstation but submits jobs on one or more -remote servers. Configure each remote server with ``address``, ``un``, and -``key``, then route ESS names to those servers: +remote servers. Configure each remote server with ``address`` and ``un``, then +route ESS names to those servers: .. code-block:: python @@ -183,8 +183,8 @@ remote servers. Configure each remote server with ``address``, ``un``, and 'cluster_a': { 'cluster_soft': 'Slurm', 'address': 'login.cluster.edu', + 'path': '/home', 'un': 'my_user', - 'key': '/home/my_user/.ssh/id_rsa', }, } @@ -193,6 +193,23 @@ remote servers. Configure each remote server with ``address``, ``un``, and 'molpro': 'cluster_a', } +With no ``key`` in the entry, ARC authenticates through a running ssh-agent and +then through the default key paths, so ``ssh-add ~/.ssh/id_ed25519`` is all that is +needed. To point at a specific private key instead, add +``'key': '/home/my_user/.ssh/id_ed25519'``; the path is read on the machine running +ARC and must name the private key. + +A host that is absent from ``~/.ssh/known_hosts`` is warned about but still +connected to. Add ``'strict_host_key_checking': True`` to a server entry to refuse +unknown hosts instead. ARC names every such server at startup, before it submits +anything, together with the ``ssh-keyscan`` command that seeds the missing key. + +ARC does not read ``~/.ssh/config``, so every connection detail has to be given in +the server entry, and jump hosts (``ProxyJump``/``ProxyCommand``) are not supported. + +See :ref:`remote_submission` for authentication, host key verification, and remote +submission from the Docker image. + Run on HPC ---------- From dd1f0af9d69770066f0eea970c2e8dce505d1377 Mon Sep 17 00:00:00 2001 From: Calvin Pieters Date: Sat, 22 Aug 2026 13:08:59 +0300 Subject: [PATCH 9/9] gaussian_test: give the adapter fixtures a private project directory arc/job/adapters/gaussian_test.py and arc/job/adapters/common_test.py both built their adapter fixtures under arc/testing/test_GaussianAdapter and both deleted that directory in tearDownClass. Under pytest-xdist the two modules run on different workers, so whichever class finished first removed the tree the other was still writing input files into, and the three tests that render an input file and read it back failed with FileNotFoundError on the input.gjf they had just written. The same collision is possible within this module alone, since the worksteal scheduler may split a class across workers and each worker runs its own setUpClass and tearDownClass. Create the project directory with tempfile.mkdtemp() in setUpClass and remove it through addClassCleanup, so every class setup owns a directory no other class or worker can name, and each removes only the directory it created. No test asserts on the directory's path; they all derive it from the adapter's local_path. Reproduced by running this module together with arc/job/adapters/common_test.py under -n 4 --dist worksteal: 8 of 8 runs failed before, 7 of 7 pass after. --- arc/job/adapters/gaussian_test.py | 70 ++++++++++++++----------------- 1 file changed, 32 insertions(+), 38 deletions(-) diff --git a/arc/job/adapters/gaussian_test.py b/arc/job/adapters/gaussian_test.py index 96f65af276..a0b8af26eb 100644 --- a/arc/job/adapters/gaussian_test.py +++ b/arc/job/adapters/gaussian_test.py @@ -8,9 +8,9 @@ import math import os import shutil +import tempfile import unittest -from arc.common import ARC_TESTING_PATH from arc.job.adapters.gaussian import GaussianAdapter, get_memory_headroom_fraction from arc.level import Level from arc.settings.settings import input_filenames, output_filenames, servers, submit_filenames @@ -28,11 +28,13 @@ def setUpClass(cls): A method that is run before all unit tests in this class. """ cls.maxDiff = None + cls.project_directory = tempfile.mkdtemp(prefix='test_GaussianAdapter_') + cls.addClassCleanup(shutil.rmtree, cls.project_directory, ignore_errors=True) cls.job_1 = GaussianAdapter(execution_type='incore', job_type='composite', level=Level(method='cbs-qb3-paraskevas'), project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[ARCSpecies(label='spc1', xyz=['O 0 0 1'], multiplicity=3)], testing=True, args={'keyword': {'general': 'IOp(1/12=5,3/44=0)'}}, @@ -44,7 +46,7 @@ def setUpClass(cls): solvation_method='SMD', solvent='Water'), project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[ARCSpecies(label='spc1', xyz=['O 0 0 1'], multiplicity=3), ARCSpecies(label='spc2', xyz=['O 0 0 2'], multiplicity=3)], testing=True, @@ -56,7 +58,7 @@ def setUpClass(cls): solvation_method='SMD', solvent='Water'), project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[ARCSpecies(label='spc1', xyz=['O 0 0 1'], multiplicity=3)], testing=True, ) @@ -76,7 +78,7 @@ def setUpClass(cls): level=Level(method='wb97xd', basis='def2-TZVP'), project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[spc_4], rotor_index=0, testing=True, @@ -87,7 +89,7 @@ def setUpClass(cls): level=Level(method='wb97xd', basis='def2-TZVP'), project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[ARCSpecies(label='birad singlet', xyz=['O 0 0 1'], multiplicity=1, @@ -99,7 +101,7 @@ def setUpClass(cls): level=Level(method='wb97xd', basis='def2-TZVP'), project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[ARCSpecies(label='anion', xyz=['O 0 0 1'], charge=-1, is_ts=False)], testing=True, ) @@ -108,7 +110,7 @@ def setUpClass(cls): level=Level(method='wb97xd', basis='def2-TZVP'), project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[ARCSpecies(label='IRC', xyz=['O 0 0 1'], is_ts=True, multiplicity=3)], irc_direction='reverse', testing=True, @@ -117,7 +119,7 @@ def setUpClass(cls): job_type='composite', level=Level(method='cbs-qb3-paraskevas'), project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[ARCSpecies(label='spc1', xyz=['O 0 0 1'], multiplicity=3)], testing=True, args={'keyword': {'general': 'IOp(1/12=5,3/44=0)'}}, @@ -127,7 +129,7 @@ def setUpClass(cls): level=Level(method='wb97xd', basis='def2-TZVP'), project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[ARCSpecies(label='anion', xyz=['O 0 0 1'], charge=-1, is_ts=False)], testing=True, ) @@ -136,7 +138,7 @@ def setUpClass(cls): level=Level(method='wb97xd'), fine=True, project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[ARCSpecies(label='anion', xyz=['O 0 0 1'], charge=-1, is_ts=False)], testing=True, args={'trsh': {'trsh': ['int=(Acc2E=14)']}}, @@ -145,7 +147,7 @@ def setUpClass(cls): job_type='opt', level=Level(method='uff'), project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[ARCSpecies(label='spc1', xyz=['O 0 0 1'], multiplicity=3)], testing=True, ) @@ -156,7 +158,7 @@ def setUpClass(cls): solvation_method='SMD', solvent='Water'), project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[ARCSpecies(label='spc1', xyz=['O 0 0 1'], multi_species='mltspc1', multiplicity=3), ARCSpecies(label='spc2', xyz=['O 0 0 2'], multi_species='mltspc1', multiplicity=3), ARCSpecies(label='ethanol', xyz=["""C 1.1658210 -0.4043550 0.0000000 @@ -212,7 +214,7 @@ def setUpClass(cls): fine=True, ess_trsh_methods=ess_trsh_methods, project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[spc_11], testing=True, args=args @@ -234,7 +236,7 @@ def setUpClass(cls): fine=True, ess_trsh_methods=ess_trsh_methods, project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[spc_11], testing=True, args=args @@ -256,7 +258,7 @@ def setUpClass(cls): fine=True, ess_trsh_methods=ess_trsh_methods, project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[spc_11], testing=True, args=args @@ -278,7 +280,7 @@ def setUpClass(cls): fine=True, ess_trsh_methods=ess_trsh_methods, project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[spc_11], testing=True, args=args @@ -301,7 +303,7 @@ def setUpClass(cls): fine=True, ess_trsh_methods=ess_trsh_methods, project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[spc_11], testing=True, args=args @@ -323,7 +325,7 @@ def setUpClass(cls): fine=True, ess_trsh_methods=ess_trsh_methods, project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[spc_11], testing=True, args=args @@ -346,7 +348,7 @@ def setUpClass(cls): fine=True, ess_trsh_methods=ess_trsh_methods, project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[spc_11], testing=True, args=args @@ -368,7 +370,7 @@ def setUpClass(cls): fine=True, ess_trsh_methods=ess_trsh_methods, project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[spc_11], testing=True, args=args @@ -390,7 +392,7 @@ def setUpClass(cls): fine=True, ess_trsh_methods=ess_trsh_methods, project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[spc_11], testing=True, args=args @@ -412,7 +414,7 @@ def setUpClass(cls): fine=True, ess_trsh_methods=ess_trsh_methods, project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[spc_11], testing=True, args=args @@ -434,7 +436,7 @@ def setUpClass(cls): fine=True, ess_trsh_methods=ess_trsh_methods, project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[spc_11], testing=True, args=args @@ -457,7 +459,7 @@ def setUpClass(cls): fine=True, ess_trsh_methods=ess_trsh_methods, project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[spc_11], testing=True, args=args @@ -480,7 +482,7 @@ def setUpClass(cls): fine=True, ess_trsh_methods=ess_trsh_methods, project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[spc_11], testing=True, args=args @@ -504,7 +506,7 @@ def setUpClass(cls): fine=True, ess_trsh_methods=ess_trsh_methods, project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=cls.project_directory, species=[spc_11], testing=True, args=args @@ -537,7 +539,7 @@ def test_set_input_file_memory_with_headroom_marker(self): job_type='opt', level=Level(method='wb97xd', basis='def2tzvp'), project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=self.project_directory, species=[ARCSpecies(label='spc_headroom', xyz=['O 0 0 1'], multiplicity=3)], testing=True, ess_trsh_methods=ess_trsh_methods, @@ -553,7 +555,7 @@ def test_memory_headroom_marker_not_in_trsh_keyword(self): job_type='opt', level=Level(method='wb97xd', basis='def2tzvp'), project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=self.project_directory, species=[ARCSpecies(label='spc_headroom_marker', xyz=['O 0 0 1'], multiplicity=3)], testing=True, ess_trsh_methods=['memory_headroom_0.6'], @@ -1214,7 +1216,7 @@ def test_user_keyword_args_survive_a_level_round_trip(self): job_type='opt', level=rebuilt_level, project='test', - project_directory=os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), + project_directory=self.project_directory, species=[ARCSpecies(label='spc1', xyz=['O 0 0 1'], multiplicity=3)], testing=True, ) @@ -1225,14 +1227,6 @@ def test_user_keyword_args_survive_a_level_round_trip(self): self.assertEqual(len(route_section), 1) self.assertIn('verytight', route_section[0]) - @classmethod - def tearDownClass(cls): - """ - A function that is run ONCE after all unit tests in this class. - Delete all project directories created during these unit tests. - """ - shutil.rmtree(os.path.join(ARC_TESTING_PATH, 'test_GaussianAdapter'), ignore_errors=True) - class TestGetMemoryHeadroomFraction(unittest.TestCase): """