Give each test its own scratch directory and each TestScheduler test its own fixtures

arc/checks/nmd_test.py

@@ -9,6 +9,7 @@
 import math
 import os
 import shutil
+import tempfile
 from unittest.mock import patch
 
 import numpy as np
@@ -37,12 +38,14 @@ def setUpClass(cls):
         A method that is run before all unit tests in this class.
         """
         cls.maxDiff = None
+        cls.scratch_dir = tempfile.mkdtemp(prefix='arc_test_nmd_')
+        cls.addClassCleanup(shutil.rmtree, cls.scratch_dir, ignore_errors=True)
         cls.generic_job = job_factory(job_adapter='gaussian',
                                       species=[ARCSpecies(label='SPC', smiles='C')],
                                       job_type='composite',
                                       level=Level(method='CBS-QB3'),
                                       project='test_project',
-                                      project_directory=os.path.join(ARC_PATH, 'Projects', 'tmp_nmd_project'),
+                                      project_directory=os.path.join(cls.scratch_dir, 'tmp_nmd_project'),
                                       )
         cls.xyz_1 = {'symbols': ('C', 'N', 'H', 'H', 'H', 'H'),
                      'isotopes': (13, 14, 1, 1, 1, 1),
@@ -812,7 +815,7 @@ def test_the_scheduler_does_not_switch_a_ts_for_an_unsupported_ess(self):
         rxn.ts_label = 'TS_unsupported_ess'
         rxn.ts_species = ARCSpecies(label='TS_unsupported_ess', is_ts=True, xyz=self.ts_1_xyz)
         rxn.ts_species.rxn_index = 0
-        project_directory = os.path.join(ARC_PATH, 'Projects', 'tmp_nmd_unsupported_ess_project')
+        project_directory = tempfile.mkdtemp(prefix='arc_test_nmd_unsupported_ess_')
         self.addCleanup(shutil.rmtree, project_directory, ignore_errors=True)
         sched = Scheduler(project='tmp_nmd_unsupported_ess_project',
                           ess_settings={'gaussian': ['local']},
@@ -1487,12 +1490,8 @@ def test_analyze_ts_normal_mode_displacement_without_any_reactive_bonds(self):
     def tearDownClass(cls):
         """
         A function that is run ONCE after all unit tests in this class.
-        Delete all project directories created during these unit tests
+        Delete files created during these unit tests
         """
-        projects = ['tmp_nmd_project']
-        for project in projects:
-            project_directory = os.path.join(ARC_PATH, 'Projects', project)
-            shutil.rmtree(project_directory, ignore_errors=True)
         file_paths = [os.path.join(ARC_PATH, 'arc', 'checks', 'nul'), os.path.join(ARC_PATH, 'arc', 'checks', 'run.out')]
         for file_path in file_paths:
             if os.path.isfile(file_path):

arc/checks/ts_test.py

@@ -8,6 +8,7 @@
 import unittest
 import os
 import shutil
+import tempfile
 from unittest.mock import patch
 
 import numpy as np
@@ -33,6 +34,8 @@ def setUpClass(cls):
         A method that is run before all unit tests in this class.
         """
         cls.maxDiff = None
+        cls.scratch_dir = tempfile.mkdtemp(prefix='arc_test_ts_checks_')
+        cls.addClassCleanup(shutil.rmtree, cls.scratch_dir, ignore_errors=True)
 
         cls.rms_list_1 = [0.01414213562373095, 0.05, 0.04, 0.5632938842203065, 0.7993122043357026, 0.08944271909999159,
                           0.10677078252031312, 0.09000000000000001, 0.05, 0.09433981132056604]
@@ -105,9 +108,7 @@ def setUpClass(cls):
                                job_type='composite',
                                level=Level(method='CBS-QB3'),
                                project='test_project',
-                               project_directory=os.path.join(ARC_PATH,
-                                                              'Projects',
-                                                              'arc_project_for_testing_delete_after_usage4'),
+                               project_directory=os.path.join(cls.scratch_dir, 'job1_project'),
                                )
 
         cls.rxn_3 = ARCReaction(r_species=[ARCSpecies(label='NH3', smiles='N'), ARCSpecies(label='H', smiles='[H]')],
@@ -215,7 +216,7 @@ def setUpClass(cls):
                                    (-1.1265684046717404, -0.2344009055503307, -1.0127644068816903))}
 
         cls.species_dict_8 = {spc.label: spc for spc in cls.rxn_8.r_species + cls.rxn_8.p_species + [cls.rxn_8.ts_species]}
-        cls.project_directory_8 = os.path.join(ts.ARC_PATH, 'Projects', 'arc_project_for_testing_delete_after_usage5')
+        cls.project_directory_8 = os.path.join(cls.scratch_dir, 'rxn_8_project')
         cls.output_dict_8 = {'iC3H7': {'paths': {'freq': os.path.join(ARC_TESTING_PATH, 'freq', 'iC3H7.out'),
                                                  'sp': os.path.join(ARC_TESTING_PATH, 'opt', 'iC3H7.out'),
                                                  'opt': os.path.join(ARC_TESTING_PATH, 'opt', 'iC3H7.out'),
@@ -1034,12 +1035,8 @@ def test_check_irc_isomorphism_mismatch_alone_is_not_a_failure(self):
     def tearDownClass(cls):
         """
         A function that is run ONCE after all unit tests in this class.
-        Delete all project directories created during these unit tests
+        Delete files created during these unit tests
         """
-        projects = ['arc_project_for_testing_delete_after_usage4', 'arc_project_for_testing_delete_after_usage5']
-        for project in projects:
-            project_directory = os.path.join(ARC_PATH, 'Projects', project)
-            shutil.rmtree(project_directory, ignore_errors=True)
         file_paths = [os.path.join(ARC_PATH, 'arc', 'checks', 'nul'), os.path.join(ARC_PATH, 'arc', 'checks', 'run.out')]
         for file_path in file_paths:
             if os.path.isfile(file_path):

arc/common_test.py

@@ -8,6 +8,7 @@
 import copy
 import datetime
 import os
+import shutil
 import tempfile
 import time
 import unittest
@@ -32,33 +33,11 @@ class TestCommon(unittest.TestCase):
     """
     Contains unit tests for ARC's common module
     """
-    @classmethod
-    def _clean_globalized_restart_artifact(cls):
-        """Remove the globalized restart-paths artifact written by
-        :meth:`test_globalize_paths`.
-
-        Called from BOTH ``setUpClass`` (defensive: wipes a stale
-        artifact left behind by a previously interrupted run) and
-        ``tearDownClass`` (the normal cleanup path).  This makes the
-        cleanup self-healing: a Ctrl+C, ``kill``, or hard error during
-        a previous run cannot leave the next run inheriting the prior
-        ``restart_paths_globalized.yml``.
-        """
-        globalized_restart_path = os.path.join(
-            common.ARC_TESTING_PATH, 'restart', '4_globalized_paths',
-            'restart_paths_globalized.yml')
-        if os.path.isfile(globalized_restart_path):
-            try:
-                os.remove(path=globalized_restart_path)
-            except OSError as e:
-                print(f'Could not remove stale globalized restart artifact {globalized_restart_path}: {e}')
-
     @classmethod
     def setUpClass(cls):
         """
         A method that is run before all unit tests in this class.
         """
-        cls._clean_globalized_restart_artifact()
         cls.maxDiff = None
         cls.default_job_types = {'conf_opt': True,
                                  'opt': True,
@@ -991,19 +970,21 @@ def test_get_bonds_from_dmat(self):
 
     def test_globalize_paths(self):
         """Test modifying a file's contents to correct absolute file paths"""
-        project_directory = os.path.join(common.ARC_TESTING_PATH, 'restart', '4_globalized_paths')
+        project_directory = os.path.join(tempfile.mkdtemp(prefix='arc_test_globalize_'), '4_globalized_paths')
+        self.addCleanup(shutil.rmtree, os.path.dirname(project_directory), ignore_errors=True)
+        shutil.copytree(os.path.join(common.ARC_TESTING_PATH, 'restart', '4_globalized_paths'), project_directory)
         restart_path = os.path.join(project_directory, 'restart_paths.yml')
         common.globalize_paths(file_path=restart_path, project_directory=project_directory)
         globalized_restart_path = os.path.join(project_directory, 'restart_paths_globalized.yml')
         content = common.read_yaml_file(globalized_restart_path)
         self.assertEqual(content['output']['restart'], 'Restarted ARC at 2020-02-28 12:51:14.446086; ')
-        self.assertIn('arc/testing/restart/4_globalized_paths/calcs/Species/HCN/freq_a38229/output.out',
+        self.assertIn(os.path.join(project_directory, 'calcs', 'Species', 'HCN', 'freq_a38229', 'output.out'),
                       content['output']['spc']['paths']['freq'])
         self.assertNotIn('gpfs/workspace/users/user', content['output']['spc']['paths']['freq'])
 
         path = '/home/user/runs/ARC/ARC_Project/calcs/Species/H/sp_a4339/output.out'
         new_path = common.globalize_path(path, project_directory)
-        self.assertIn('arc/testing/restart/4_globalized_paths/calcs/Species/H/sp_a4339/output.out', new_path)
+        self.assertIn(os.path.join(project_directory, 'calcs', 'Species', 'H', 'sp_a4339', 'output.out'), new_path)
 
     def test_globalize_path(self):
         """Test rebasing a single path to the current ARC project"""
@@ -1346,13 +1327,14 @@ def test_calc_rmsd(self):
 
     def test_safe_copy_file(self):
         """tests the safe_copy_file() function."""
+        scratch_dir = tempfile.mkdtemp(prefix='arc_test_safe_copy_')
+        self.addCleanup(shutil.rmtree, scratch_dir, ignore_errors=True)
         source_path = os.path.join(common.ARC_TESTING_PATH, 'freq', 'CO2_xtb.out')
-        destination_path = os.path.join(common.ARC_TESTING_PATH, 'freq', 'CO2_xtb_copy.out')
+        destination_path = os.path.join(scratch_dir, 'CO2_xtb_copy.out')
         common.safe_copy_file(source=source_path, destination=destination_path)
         self.assertTrue(os.path.isfile(destination_path))
         # Check that no error is being raised if we attempt to copy to the same destination.
         common.safe_copy_file(source=source_path, destination=destination_path)
-        os.remove(destination_path)
 
     def test_sort_atoms_in_descending_label_order(self):
         """tests the sort_atoms_in_descending_label_order function"""
@@ -1520,13 +1502,6 @@ def test_calculate_arrhenius_rate_coefficient(self):
             with self.assertRaises(ValueError):
                 common.calculate_arrhenius_rate_coefficient(A=1e12, n=0.5, Ea=10, T=bad_T, Ea_units='kJ/mol')
 
-    @classmethod
-    def tearDownClass(cls):
-        """
-        A function that is run ONCE after all unit tests in this class.
-        """
-        cls._clean_globalized_restart_artifact()
-
 
 class TestInitializeLogDeferredWarnings(unittest.TestCase):
     """initialize_log() must flush any deferred import-time warnings

arc/job/adapter_test.py

@@ -89,13 +89,13 @@ def setUpClass(cls):
         A method that is run before all unit tests in this class.
         """
         cls.maxDiff = None
-        for dir_name in ('test_JobAdapter', 'test_JobAdapter_scan', 'test_JobAdapter_ServerTimeLimit'):
-            cls.addClassCleanup(shutil.rmtree, os.path.join(ARC_TESTING_PATH, dir_name), ignore_errors=True)
+        cls.scratch_dir = tempfile.mkdtemp(prefix='arc_test_job_adapter_')
+        cls.addClassCleanup(shutil.rmtree, cls.scratch_dir, ignore_errors=True)
         cls.job_1 = GaussianAdapter(execution_type='queue',
                                     job_type='conf_opt',
                                     level=Level(method='cbs-qb3'),
                                     project='test',
-                                    project_directory=os.path.join(ARC_TESTING_PATH, 'test_JobAdapter'),
+                                    project_directory=os.path.join(cls.scratch_dir, 'test_JobAdapter'),
                                     species=[ARCSpecies(label='spc1',
                                                         xyz=['O 0 0 1',
                                                              'O 0 0 2',
@@ -124,7 +124,7 @@ def setUpClass(cls):
                                     job_type='opt',
                                     level=Level(method='cbs-qb3'),
                                     project='test',
-                                    project_directory=os.path.join(ARC_TESTING_PATH, 'test_JobAdapter'),
+                                    project_directory=os.path.join(cls.scratch_dir, 'test_JobAdapter'),
                                     species=[ARCSpecies(label='spc1', xyz=['O 0 0 1'])],
                                     testing=True,
                                     )
@@ -151,20 +151,20 @@ def setUpClass(cls):
                                     torsions=[[1, 2, 3, 4]],
                                     level=Level(method='wb97xd', basis='def2-tzvp'),
                                     project='test_scans',
-                                    project_directory=os.path.join(ARC_TESTING_PATH, 'test_JobAdapter_scan'),
+                                    project_directory=os.path.join(cls.scratch_dir, 'test_JobAdapter_scan'),
                                     species=[cls.spc_3a, cls.spc_3b, cls.spc_3c, cls.spc_3d, cls.spc_3e, cls.spc_3f],
                                     testing=True,
                                     )
         cls.job_4 = GaussianAdapter(execution_type='queue',
                                     job_type='opt',
                                     level=Level(method='cbs-qb3'),
                                     project='test',
-                                    project_directory=os.path.join(ARC_TESTING_PATH, 'test_JobAdapter'),
+                                    project_directory=os.path.join(cls.scratch_dir, 'test_JobAdapter'),
                                     species=[ARCSpecies(label='spc1', xyz=['O 0 0 1'])],
                                     testing=True,
                                     )
         # Copy the PBS time limit fixture into the directory structure the adapter expects.
-        stl_dir = os.path.join(ARC_TESTING_PATH, 'test_JobAdapter_ServerTimeLimit')
+        stl_dir = os.path.join(cls.scratch_dir, 'test_JobAdapter_ServerTimeLimit')
         err_dest = os.path.join(stl_dir, 'calcs', 'Species', 'spc1', 'opt_101')
         os.makedirs(err_dest, exist_ok=True)
         shutil.copy(os.path.join(ARC_TESTING_PATH, 'server', 'pbs', 'timelimit', 'err.txt'),
@@ -288,7 +288,7 @@ def test_add_to_args(self):
                                         job_type='opt',
                                         level=Level(method='cbs-qb3'),
                                         project='test',
-                                        project_directory=os.path.join(ARC_TESTING_PATH, 'test_JobAdapter'),
+                                        project_directory=os.path.join(self.scratch_dir, 'test_JobAdapter'),
                                         species=[ARCSpecies(label='spc1', xyz=['O 0 0 1'])],
                                         testing=True,
                                         args={'keyword': {'general': 'val_tst_1 val_tst_2     val_tst_3'},

arc/job/adapters/ase_test.py

@@ -8,14 +8,15 @@
 
 import os
 import shutil
+import tempfile
 import unittest
 from unittest.mock import patch
 import numpy as np
 
 from ase import Atoms
 from ase.calculators.emt import EMT
 
-from arc.common import ARC_TESTING_PATH, read_yaml_file, save_yaml_file
+from arc.common import read_yaml_file, save_yaml_file
 from arc.job.adapters.ase_adapter import ASEAdapter
 from arc.parser.parser import parse_1d_scan_coords, parse_1d_scan_energies
 from arc.species.species import ARCSpecies
@@ -51,8 +52,8 @@ def setUpClass(cls):
         A method that is run before all unit tests in this class.
         """
         cls.maxDiff = None
-        cls.project_directory = os.path.join(ARC_TESTING_PATH, 'test_ASEAdapter')
-        os.makedirs(cls.project_directory, exist_ok=True)  # parallel workers race here
+        cls.project_directory = tempfile.mkdtemp(prefix='arc_test_ase_')
+        cls.addClassCleanup(shutil.rmtree, cls.project_directory, ignore_errors=True)
 
         xyz = {'symbols': ('O', 'H', 'H'),
                'isotopes': (16, 1, 1),
@@ -403,14 +404,6 @@ def test_set_scan_torsions_from_rotors_dict(self):
         self.assertEqual(data['torsions'], expected)
         self.assertIsNotNone(data['torsions'][0])
 
-    @classmethod
-    def tearDownClass(cls):
-        """
-        A function that is run ONCE after all unit tests in this class.
-        Delete all project directories created during these unit tests
-        """
-        shutil.rmtree(cls.project_directory, ignore_errors=True)
-
 
 if __name__ == '__main__':
     unittest.main(testRunner=unittest.TextTestRunner(verbosity=2))

arc/job/adapters/cfour_test.py

@@ -8,9 +8,9 @@
 import math
 import os
 import shutil
+import tempfile
 import unittest
 
-from arc.common import ARC_TESTING_PATH
 from arc.job.adapters.cfour import CFourAdapter
 from arc.level import Level
 from arc.settings.settings import input_filenames, output_filenames
@@ -27,6 +27,8 @@ def setUpClass(cls):
         A method that is run before all unit tests in this class.
         """
         cls.maxDiff = None
+        cls.scratch_dir = tempfile.mkdtemp(prefix='arc_test_cfour_')
+        cls.addClassCleanup(shutil.rmtree, cls.scratch_dir, ignore_errors=True)
         xyz = {'symbols': ('O', 'C', 'C', 'C', 'H', 'H', 'H', 'H', 'H', 'H', 'H', 'H'),
                'isotopes': (16, 12, 12, 12, 1, 1, 1, 1, 1, 1, 1, 1),
                'coords': ((2.094965350070438, -0.6820312883655302, 0.41738812543556636),
@@ -45,7 +47,7 @@ def setUpClass(cls):
                                  job_type='sp',
                                  level=Level(method='CCSD(T)', basis='cc-pVTZ'),
                                  project='test',
-                                 project_directory=os.path.join(ARC_TESTING_PATH, 'test_CFourAdapter'),
+                                 project_directory=os.path.join(cls.scratch_dir, 'test_CFourAdapter'),
                                  species=[ARCSpecies(label='spc1', xyz=xyz)],
                                  testing=True,
                                  )
@@ -138,14 +140,6 @@ def test_set_files(self):
         self.assertEqual(self.job_1.files_to_upload, job_1_files_to_upload)
         self.assertEqual(self.job_1.files_to_download, job_1_files_to_download)
 
-    @classmethod
-    def tearDownClass(cls):
-        """
-        A function that is run ONCE after all unit tests in this class.
-        Delete all project directories created during these unit tests
-        """
-        shutil.rmtree(os.path.join(ARC_TESTING_PATH, 'test_CFourAdapter'), ignore_errors=True)
-
 
 if __name__ == '__main__':
     unittest.main(testRunner=unittest.TextTestRunner(verbosity=2))