From 2a98a10d1ea20e6f963727b4e25b3518cb3bc35d Mon Sep 17 00:00:00 2001 From: matthewholman Date: Sat, 15 Aug 2026 13:52:55 -0400 Subject: [PATCH] Document LAYUP_NUM_WORKERS, and correct the --num-workers help (#463) The environment override added in #462 was documented only in a docstring, so there was no way for a user to discover it. Adds a short README section and a docs page in the toctree. Also fixes something #462 left behind: all four CLI verbs still described -1 as "uses all available CPUs", which stopped being true. It now resolves to $LAYUP_NUM_WORKERS if set, else 1 when layup is already inside another worker process, else the CPUs actually available (the affinity mask on Linux, so taskset and cgroup limits are respected). Wrong help text is worse than absent documentation, so that is the part of this that is a defect rather than an addition. Closes #463. Co-Authored-By: Claude Opus 5 (1M context) --- README.md | 18 ++++++++++++++++++ docs/index.rst | 1 + docs/parallelism.rst | 24 ++++++++++++++++++++++++ src/layup_cmdline/comet.py | 4 +++- src/layup_cmdline/convert.py | 4 +++- src/layup_cmdline/orbitfit.py | 4 +++- src/layup_cmdline/predict.py | 4 +++- 7 files changed, 55 insertions(+), 4 deletions(-) create mode 100644 docs/parallelism.rst diff --git a/README.md b/README.md index b44da23e..ea3653ab 100644 --- a/README.md +++ b/README.md @@ -89,6 +89,24 @@ method, non-gravitational parameters, parallel workers, …): layup orbitfit --help ``` +### Control how many CPUs layup uses + +`--num-workers` (CLI) and `num_workers=` (API) default to `-1`, meaning decide +automatically: `$LAYUP_NUM_WORKERS` if set, otherwise 1 when layup is already +running inside another worker process, otherwise the CPUs available to this +process. + +Set `LAYUP_NUM_WORKERS` when layup does not own the whole machine — running it +from your own process pool, or as one of several jobs on a shared node: + +``` +export LAYUP_NUM_WORKERS=4 +``` + +Otherwise each copy would size its pool to the whole machine and oversubscribe +it. This is separate from `OMP_NUM_THREADS` and friends, which control threads +within a worker rather than the number of workers. + ### Use the Python API The same load → fit → convert → predict workflow is available directly from diff --git a/docs/index.rst b/docs/index.rst index 594738fb..b8542330 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -57,5 +57,6 @@ Notes: :hidden: Home page + Controlling parallelism API Reference Notebooks diff --git a/docs/parallelism.rst b/docs/parallelism.rst new file mode 100644 index 00000000..83a080cd --- /dev/null +++ b/docs/parallelism.rst @@ -0,0 +1,24 @@ +Controlling parallelism +======================================================================================== + +``--num-workers`` (command line) and ``num_workers=`` (Python API) default to ``-1``, +meaning decide automatically: + +1. ``$LAYUP_NUM_WORKERS``, if set. +2. Otherwise ``1``, when layup is already running inside another worker process. +3. Otherwise the CPUs available to this process (the affinity mask on Linux, so + ``taskset`` and cgroup limits are respected). + +Set ``LAYUP_NUM_WORKERS`` when layup does not own the whole machine — for example when +running it from your own process pool, or as one of several jobs on a shared node: + +.. code-block:: console + + >> export LAYUP_NUM_WORKERS=4 + +Without it, each copy sizes its pool to the whole machine and oversubscribes it. + +.. note:: + + This is separate from ``OMP_NUM_THREADS`` and the other threadpool variables, which + control the number of threads *within* a worker rather than the number of workers. diff --git a/src/layup_cmdline/comet.py b/src/layup_cmdline/comet.py index f8c04721..a622457a 100644 --- a/src/layup_cmdline/comet.py +++ b/src/layup_cmdline/comet.py @@ -77,7 +77,9 @@ def main(): optional.add_argument( "-n", "--num-workers", - help="Number of CPU workers to use for parallel processing each chunk. -1 uses all available CPUs.", + help="Number of CPU workers for parallel processing. -1 (default) decides " + "automatically: $LAYUP_NUM_WORKERS if set, else 1 when layup is already " + "running inside another worker process, else the CPUs available to this process.", dest="n", type=int, default=-1, diff --git a/src/layup_cmdline/convert.py b/src/layup_cmdline/convert.py index 22446e4e..21e269dd 100644 --- a/src/layup_cmdline/convert.py +++ b/src/layup_cmdline/convert.py @@ -85,7 +85,9 @@ def main(): optional.add_argument( "-n", "--num-workers", - help="Number of CPU workers to use for parallel processing each chunk. -1 uses all available CPUs.", + help="Number of CPU workers for parallel processing. -1 (default) decides " + "automatically: $LAYUP_NUM_WORKERS if set, else 1 when layup is already " + "running inside another worker process, else the CPUs available to this process.", dest="n", type=int, default=-1, diff --git a/src/layup_cmdline/orbitfit.py b/src/layup_cmdline/orbitfit.py index 5a90ffd8..72ec0915 100644 --- a/src/layup_cmdline/orbitfit.py +++ b/src/layup_cmdline/orbitfit.py @@ -114,7 +114,9 @@ def main(): optional.add_argument( "-n", "--num-workers", - help="Number of CPU workers to use for parallel processing each chunk. -1 uses all available CPUs.", + help="Number of CPU workers for parallel processing. -1 (default) decides " + "automatically: $LAYUP_NUM_WORKERS if set, else 1 when layup is already " + "running inside another worker process, else the CPUs available to this process.", dest="n", type=int, default=-1, diff --git a/src/layup_cmdline/predict.py b/src/layup_cmdline/predict.py index 455ee859..8750b035 100644 --- a/src/layup_cmdline/predict.py +++ b/src/layup_cmdline/predict.py @@ -120,7 +120,9 @@ def main(): optional.add_argument( "-n", "--num-workers", - help="Number of CPU workers to use for parallel processing each chunk. -1 uses all available CPUs.", + help="Number of CPU workers for parallel processing. -1 (default) decides " + "automatically: $LAYUP_NUM_WORKERS if set, else 1 when layup is already " + "running inside another worker process, else the CPUs available to this process.", dest="n", type=int, default=-1,