Tests: stop xdist workers from deleting each other's ARC project directories

arc/checks/nmd_test.py

@@ -12,7 +12,7 @@
 import numpy as np
 
 import arc.checks.nmd as nmd
-from arc.common import ARC_PATH, ARC_TESTING_PATH, almost_equal_coords
+from arc.common import ARC_PATH, ARC_TESTING_PATH, almost_equal_coords, get_test_project_directory
 from arc.job.factory import job_factory
 from arc.level import Level
 from arc.molecule import Molecule
@@ -32,12 +32,13 @@ def setUpClass(cls):
         A method that is run before all unit tests in this class.
         """
         cls.maxDiff = None
+        cls.project_directory = get_test_project_directory('tmp_nmd_project')
         cls.generic_job = job_factory(job_adapter='gaussian',
                                       species=[ARCSpecies(label='SPC', smiles='C')],
                                       job_type='composite',
                                       level=Level(method='CBS-QB3'),
                                       project='test_project',
-                                      project_directory=os.path.join(ARC_PATH, 'Projects', 'tmp_nmd_project'),
+                                      project_directory=cls.project_directory,
                                       )
         cls.xyz_1 = {'symbols': ('C', 'N', 'H', 'H', 'H', 'H'),
                      'isotopes': (13, 14, 1, 1, 1, 1),
@@ -941,10 +942,7 @@ def tearDownClass(cls):
         A function that is run ONCE after all unit tests in this class.
         Delete all project directories created during these unit tests
         """
-        projects = ['tmp_nmd_project']
-        for project in projects:
-            project_directory = os.path.join(ARC_PATH, 'Projects', project)
-            shutil.rmtree(project_directory, ignore_errors=True)
+        shutil.rmtree(cls.project_directory, ignore_errors=True)
         file_paths = [os.path.join(ARC_PATH, 'arc', 'checks', 'nul'), os.path.join(ARC_PATH, 'arc', 'checks', 'run.out')]
         for file_path in file_paths:
             if os.path.isfile(file_path):

arc/checks/ts_test.py

@@ -12,7 +12,7 @@
 import numpy as np
 
 import arc.checks.ts as ts
-from arc.common import ARC_PATH, ARC_TESTING_PATH, almost_equal_lists
+from arc.common import ARC_PATH, ARC_TESTING_PATH, almost_equal_lists, get_test_project_directory
 from arc.job.factory import job_factory
 from arc.level import Level
 from arc.parser.parser import parse_normal_mode_displacement, parse_geometry
@@ -103,9 +103,7 @@ def setUpClass(cls):
                                job_type='composite',
                                level=Level(method='CBS-QB3'),
                                project='test_project',
-                               project_directory=os.path.join(ARC_PATH,
-                                                              'Projects',
-                                                              'arc_project_for_testing_delete_after_usage4'),
+                               project_directory=get_test_project_directory('arc_project_for_testing_delete_after_usage4'),
                                )
 
         cls.rxn_3 = ARCReaction(r_species=[ARCSpecies(label='NH3', smiles='N'), ARCSpecies(label='H', smiles='[H]')],
@@ -213,7 +211,7 @@ def setUpClass(cls):
                                    (-1.1265684046717404, -0.2344009055503307, -1.0127644068816903))}
 
         cls.species_dict_8 = {spc.label: spc for spc in cls.rxn_8.r_species + cls.rxn_8.p_species + [cls.rxn_8.ts_species]}
-        cls.project_directory_8 = os.path.join(ts.ARC_PATH, 'Projects', 'arc_project_for_testing_delete_after_usage5')
+        cls.project_directory_5 = get_test_project_directory('arc_project_for_testing_delete_after_usage5')
         cls.output_dict_8 = {'iC3H7': {'paths': {'freq': os.path.join(ARC_TESTING_PATH, 'freq', 'iC3H7.out'),
                                                  'sp': os.path.join(ARC_TESTING_PATH, 'opt', 'iC3H7.out'),
                                                  'opt': os.path.join(ARC_TESTING_PATH, 'opt', 'iC3H7.out'),
@@ -331,9 +329,9 @@ def test_compute_rxn_e0(self):
         """Test the compute_rxn_e0() function."""
         for spc_label in self.rxn_8.reactants + self.rxn_8.products + [self.rxn_8.ts_label]:
             folder = 'rxns' if self.species_dict_8[spc_label].is_ts else 'Species'
-            base_path = os.path.join(self.project_directory_8, 'output', folder, spc_label, 'geometry')
+            base_path = os.path.join(self.project_directory_5, 'output', folder, spc_label, 'geometry')
             os.makedirs(base_path, exist_ok=True)
-            freq_path = os.path.join(self.project_directory_8, 'output', folder, spc_label, 'geometry', 'freq.out')
+            freq_path = os.path.join(self.project_directory_5, 'output', folder, spc_label, 'geometry', 'freq.out')
             shutil.copy(src=self.output_dict_8[spc_label]['paths']['freq'], dst=freq_path)
 
         self.assertIsNone(self.rxn_8.r_species[0].e0)
@@ -342,7 +340,7 @@ def test_compute_rxn_e0(self):
 
         rxn_copy = ts.compute_rxn_e0(reaction=self.rxn_8,
                                      species_dict=self.species_dict_8,
-                                     project_directory=self.project_directory_8,
+                                     project_directory=self.project_directory_5,
                                      kinetics_adapter='arkane',
                                      output=self.output_dict_8,
                                      sp_level=Level(repr='cbs-qb3'),
@@ -356,13 +354,13 @@ def test_check_rxn_e0(self):
         """Test the check_rxn_e0() function."""
         for spc_label in self.rxn_8.reactants + self.rxn_8.products + [self.rxn_8.ts_label]:
             folder = 'rxns' if self.species_dict_8[spc_label].is_ts else 'Species'
-            base_path = os.path.join(self.project_directory_8, 'output', folder, spc_label, 'geometry')
+            base_path = os.path.join(self.project_directory_5, 'output', folder, spc_label, 'geometry')
             os.makedirs(base_path, exist_ok=True)
-            freq_path = os.path.join(self.project_directory_8, 'output', folder, spc_label, 'geometry', 'freq.out')
+            freq_path = os.path.join(self.project_directory_5, 'output', folder, spc_label, 'geometry', 'freq.out')
             shutil.copy(src=self.output_dict_8[spc_label]['paths']['freq'], dst=freq_path)
         rxn_copy = ts.compute_rxn_e0(reaction=self.rxn_8,
                                      species_dict=self.species_dict_8,
-                                     project_directory=self.project_directory_8,
+                                     project_directory=self.project_directory_5,
                                      kinetics_adapter='arkane',
                                      output=self.output_dict_8,
                                      sp_level=Level(repr='CBS-QB3'),
@@ -858,9 +856,8 @@ def tearDownClass(cls):
         A function that is run ONCE after all unit tests in this class.
         Delete all project directories created during these unit tests
         """
-        projects = ['arc_project_for_testing_delete_after_usage4', 'arc_project_for_testing_delete_after_usage5']
-        for project in projects:
-            project_directory = os.path.join(ARC_PATH, 'Projects', project)
+        for project_directory in [get_test_project_directory('arc_project_for_testing_delete_after_usage4'),
+                                  cls.project_directory_5]:
             shutil.rmtree(project_directory, ignore_errors=True)
         file_paths = [os.path.join(ARC_PATH, 'arc', 'checks', 'nul'), os.path.join(ARC_PATH, 'arc', 'checks', 'run.out')]
         for file_path in file_paths:

arc/common.py

@@ -508,6 +508,35 @@ def globalize_path(string: str,
     return string
 
 
+def get_test_project_name(base_name: str) -> str:
+    """
+    Get a project name for a unit test that is unique per pytest-xdist worker.
+
+    Args:
+        base_name (str): The base project name.
+
+    Returns:
+        str: ``base_name`` suffixed by the pytest-xdist worker ID when running under pytest-xdist,
+             otherwise ``base_name`` unchanged.
+    """
+    worker_id = os.environ.get('PYTEST_XDIST_WORKER')
+    return f'{base_name}_{worker_id}' if worker_id else base_name
+
+
+def get_test_project_directory(base_name: str) -> str:
+    """
+    Get a path to a project directory for a unit test that is unique per pytest-xdist worker.
+
+    Args:
+        base_name (str): The base project name.
+
+    Returns:
+        str: The path under ARC's ``Projects`` folder to a directory named ``base_name`` suffixed by the
+             pytest-xdist worker ID when running under pytest-xdist, and named ``base_name`` otherwise.
+    """
+    return os.path.join(ARC_PATH, 'Projects', get_test_project_name(base_name))
+
+
 def delete_check_files(project_directory: str):
     """
     Delete local ESS checkfiles. They usually take up lots of space and are not needed after ARC terminates.

arc/common_test.py

@@ -1023,6 +1023,34 @@ def test_globalize_path(self):
         globalized_string = common.globalize_path(string=string, project_directory='~/Code/runs/run_1/')
         self.assertEqual(globalized_string, '    project_directory: ~/Code/runs/run_1/')
 
+    def test_get_test_project_name(self):
+        """Test getting a pytest-xdist worker scoped project name"""
+        original = os.environ.get('PYTEST_XDIST_WORKER')
+        self.addCleanup(self._restore_xdist_worker_env, original)
+        os.environ.pop('PYTEST_XDIST_WORKER', None)
+        self.assertEqual(common.get_test_project_name('project_1'), 'project_1')
+        os.environ['PYTEST_XDIST_WORKER'] = 'gw3'
+        self.assertEqual(common.get_test_project_name('project_1'), 'project_1_gw3')
+
+    def test_get_test_project_directory(self):
+        """Test getting a pytest-xdist worker scoped project directory"""
+        original = os.environ.get('PYTEST_XDIST_WORKER')
+        self.addCleanup(self._restore_xdist_worker_env, original)
+        os.environ['PYTEST_XDIST_WORKER'] = 'gw5'
+        self.assertEqual(common.get_test_project_directory('project_2'),
+                         os.path.join(common.ARC_PATH, 'Projects', 'project_2_gw5'))
+        os.environ.pop('PYTEST_XDIST_WORKER', None)
+        self.assertEqual(common.get_test_project_directory('project_2'),
+                         os.path.join(common.ARC_PATH, 'Projects', 'project_2'))
+
+    @staticmethod
+    def _restore_xdist_worker_env(original: str | None):
+        """Restore the PYTEST_XDIST_WORKER environment variable to its original value"""
+        if original is None:
+            os.environ.pop('PYTEST_XDIST_WORKER', None)
+        else:
+            os.environ['PYTEST_XDIST_WORKER'] = original
+
     def test_estimate_orca_mem_cpu_requirement(self):
         """Test estimating memory and cpu requirements for an Orca job."""
         num_heavy_atoms_0 = 0

arc/job/adapter_test.py

@@ -15,7 +15,7 @@
 import unittest
 from unittest.mock import patch
 
-from arc.common import ARC_TESTING_PATH
+from arc.common import ARC_TESTING_PATH, get_test_project_name
 from arc.imports import settings
 from arc.job.adapter import JobAdapter, JobEnum, JobTypeEnum, JobExecutionTypeEnum
 from arc.job.adapters.gaussian import GaussianAdapter
@@ -24,6 +24,10 @@
 
 servers, submit_filenames = settings['servers'], settings['submit_filenames']
 
+JOB_ADAPTER_DIRS = tuple(os.path.join(ARC_TESTING_PATH, get_test_project_name(f'test_JobAdapter{suffix}'))
+                         for suffix in ('', '_scan', '_ServerTimeLimit'))
+JOB_ADAPTER_DIR, JOB_ADAPTER_SCAN_DIR, JOB_ADAPTER_STL_DIR = JOB_ADAPTER_DIRS
+
 
 class TestEnumerationClasses(unittest.TestCase):
     """
@@ -89,13 +93,13 @@ def setUpClass(cls):
         A method that is run before all unit tests in this class.
         """
         cls.maxDiff = None
-        for dir_name in ('test_JobAdapter', 'test_JobAdapter_scan', 'test_JobAdapter_ServerTimeLimit'):
-            cls.addClassCleanup(shutil.rmtree, os.path.join(ARC_TESTING_PATH, dir_name), ignore_errors=True)
+        for dir_path in JOB_ADAPTER_DIRS:
+            cls.addClassCleanup(shutil.rmtree, dir_path, ignore_errors=True)
         cls.job_1 = GaussianAdapter(execution_type='queue',
                                     job_type='conf_opt',
                                     level=Level(method='cbs-qb3'),
                                     project='test',
-                                    project_directory=os.path.join(ARC_TESTING_PATH, 'test_JobAdapter'),
+                                    project_directory=JOB_ADAPTER_DIR,
                                     species=[ARCSpecies(label='spc1',
                                                         xyz=['O 0 0 1',
                                                              'O 0 0 2',
@@ -124,7 +128,7 @@ def setUpClass(cls):
                                     job_type='opt',
                                     level=Level(method='cbs-qb3'),
                                     project='test',
-                                    project_directory=os.path.join(ARC_TESTING_PATH, 'test_JobAdapter'),
+                                    project_directory=JOB_ADAPTER_DIR,
                                     species=[ARCSpecies(label='spc1', xyz=['O 0 0 1'])],
                                     testing=True,
                                     )
@@ -151,20 +155,20 @@ def setUpClass(cls):
                                     torsions=[[1, 2, 3, 4]],
                                     level=Level(method='wb97xd', basis='def2-tzvp'),
                                     project='test_scans',
-                                    project_directory=os.path.join(ARC_TESTING_PATH, 'test_JobAdapter_scan'),
+                                    project_directory=JOB_ADAPTER_SCAN_DIR,
                                     species=[cls.spc_3a, cls.spc_3b, cls.spc_3c, cls.spc_3d, cls.spc_3e, cls.spc_3f],
                                     testing=True,
                                     )
         cls.job_4 = GaussianAdapter(execution_type='queue',
                                     job_type='opt',
                                     level=Level(method='cbs-qb3'),
                                     project='test',
-                                    project_directory=os.path.join(ARC_TESTING_PATH, 'test_JobAdapter'),
+                                    project_directory=JOB_ADAPTER_DIR,
                                     species=[ARCSpecies(label='spc1', xyz=['O 0 0 1'])],
                                     testing=True,
                                     )
         # Copy the PBS time limit fixture into the directory structure the adapter expects.
-        stl_dir = os.path.join(ARC_TESTING_PATH, 'test_JobAdapter_ServerTimeLimit')
+        stl_dir = JOB_ADAPTER_STL_DIR
         err_dest = os.path.join(stl_dir, 'calcs', 'Species', 'spc1', 'opt_101')
         os.makedirs(err_dest, exist_ok=True)
         shutil.copy(os.path.join(ARC_TESTING_PATH, 'server', 'pbs', 'timelimit', 'err.txt'),
@@ -288,7 +292,7 @@ def test_add_to_args(self):
                                         job_type='opt',
                                         level=Level(method='cbs-qb3'),
                                         project='test',
-                                        project_directory=os.path.join(ARC_TESTING_PATH, 'test_JobAdapter'),
+                                        project_directory=JOB_ADAPTER_DIR,
                                         species=[ARCSpecies(label='spc1', xyz=['O 0 0 1'])],
                                         testing=True,
                                         args={'keyword': {'general': 'val_tst_1 val_tst_2     val_tst_3'},