From 674e8511c90a2fa6801ab23d23d5d494e7f0c81f Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Tue, 8 Jun 2021 16:50:32 +0100
Subject: [PATCH 01/44] Update plotman.yaml

Moved tmp_overrides into scheduling: to better reflect that they change the scheduling behaviour.
Added more override options to allow different tmpdir characteristics to be catered for.
---
 src/plotman/resources/plotman.yaml | 33 ++++++++++++++++++------------
 1 file changed, 20 insertions(+), 13 deletions(-)

diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index 09ee52ad..21004605 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -44,19 +44,6 @@ directories:
                 - /mnt/tmp/02
                 - /mnt/tmp/03
 
-        # Optional: Allows overriding some characteristics of certain tmp
-        # directories. This contains a map of tmp directory names to
-        # attributes. If a tmp directory and attribute is not listed here,
-        # it uses the default attribute setting from the main configuration.
-        #
-        # Currently support override parameters:
-        #     - tmpdir_max_jobs
-        tmp_overrides:
-                # In this example, /mnt/tmp/00 is larger than the other tmp
-                # dirs and it can hold more plots than the default.
-                "/mnt/tmp/00":
-                        tmpdir_max_jobs: 5
-
         # Optional: tmp2 directory.  If specified, will be passed to
         # chia plots create as -2.  Only one tmp2 directory is supported.
         # tmp2: /mnt/tmp/a
@@ -131,6 +118,26 @@ scheduling:
         # How often the daemon wakes to consider starting a new plot job, in seconds.
         polling_time_s: 20
 
+        # Optional: Allows overriding some scheduling characteristics of certain tmp
+        # directories. This contains a map of tmp directory names to
+        # attributes. If a tmp directory and attribute is not listed here,
+        # it uses the default attribute setting from the main configuration.
+        #
+        # Currently support override parameters:
+        #     - tmpdir_stagger_phase_major
+        #     - tmpdir_stagger_phase_minor
+        #     - tmpdir_stagger_phase_limit
+        #     - tmpdir_max_jobs
+        tmp_overrides:
+                # In this example, /mnt/tmp/00 is larger than the other tmp
+                # dirs and it can hold more plots than the default, allowing
+                # more simultaneous plots and they are being started earlier
+                # than the global setting above.
+                "/mnt/tmp/00":
+                        tmpdir_stagger_phase_major: 1
+                        tmpdir_stagger_phase_minr: 5
+                        tmpdir_stagger_phase_limit: 1
+                        tmpdir_max_jobs: 5
 
 # Plotting parameters.  These are pass-through parameters to chia plots create.
 # See documentation at

From 15663757531f5154c4c484fbb3bfb72c00267ce3 Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Tue, 8 Jun 2021 16:58:07 +0100
Subject: [PATCH 02/44] Update configuration.py

Changes for reading the tmpdir overrides
---
 src/plotman/configuration.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 220dc538..06b82a78 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -186,6 +186,9 @@ def maybe_create_scripts(self, temp):
 
 @attr.frozen
 class TmpOverrides:
+    tmpdir_stagger_phase_major: Optional[int] = None
+    tmpdir_stagger_phase_minor: Optional[int] = None
+    tmpdir_stagger_phase_limit: Optional[int] = None
     tmpdir_max_jobs: Optional[int] = None
 
 @attr.frozen
@@ -222,7 +225,6 @@ class Directories:
     tmp: List[str]
     dst: Optional[List[str]] = None
     tmp2: Optional[str] = None
-    tmp_overrides: Optional[Dict[str, TmpOverrides]] = None
 
     def dst_is_tmp(self):
         return self.dst is None and self.tmp2 is None
@@ -250,6 +252,7 @@ class Scheduling:
     tmpdir_stagger_phase_major: int
     tmpdir_stagger_phase_minor: int
     tmpdir_stagger_phase_limit: int = 1  # If not explicit, "tmpdir_stagger_phase_limit" will default to 1
+    tmp_overrides: Optional[Dict[str, TmpOverrides]] = None
 
 @attr.frozen
 class Plotting:

From f253d060d6531c754a4796e9b625ee94b733f331 Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Tue, 8 Jun 2021 17:12:47 +0100
Subject: [PATCH 03/44] Update manager.py

---
 src/plotman/manager.py | 33 +++++++++++++++++++++++----------
 1 file changed, 23 insertions(+), 10 deletions(-)

diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index 7c5b116d..4982f2a9 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -53,21 +53,34 @@ def phases_permit_new_job(phases, d, sched_cfg, dir_cfg):
     if len(phases) == 0:
         return True
 
-    milestone = job.Phase(
-        major=sched_cfg.tmpdir_stagger_phase_major,
-        minor=sched_cfg.tmpdir_stagger_phase_minor,
-    )
+    # Check if any overrides exist for the current job
+    if sched_cfg.tmp_overrides is not None and d in sched_cfg.tmp_overrides:
+        curr_overrides = sched_cfg.tmp_overrides[d]
+    
+    # Check and apply overrides for major, minor and phase limit
+    if curr_overrides.tmpdir_stagger_phase_major is not None:
+        major = curr_overrides.tmpdir_stagger_phase_major
+    else:
+        major = sched_cfg.tmpdir_stagger_phase_major
+    if curr_overrides.tmpdir_stagger_phase_minor is not None:
+        minor = curr_overrides.tmpdir_stagger_phase_minor
+    else:
+        minor = sched_cfg.tmpdir_stagger_phase_minor
+    milestone = job.Phase(major,minor,)
+    if curr_overrides.tmpdir_stagger_phase_limit is not None:
+        stagger_phase_limit = curr_overrides.tmpdir_stagger_phase_limit
+    else:
+        stagger_phase_limit = sched_cfg.tmpdir_stagger_phase_limit
     # tmpdir_stagger_phase_limit default is 1, as declared in configuration.py
-    if len([p for p in phases if p < milestone]) >= sched_cfg.tmpdir_stagger_phase_limit:
+    if len([p for p in phases if p < milestone]) >= stagger_phase_limit:
         return False
 
     # Limit the total number of jobs per tmp dir. Default to the overall max
     # jobs configuration, but restrict to any configured overrides.
-    max_plots = sched_cfg.tmpdir_max_jobs
-    if dir_cfg.tmp_overrides is not None and d in dir_cfg.tmp_overrides:
-        curr_overrides = dir_cfg.tmp_overrides[d]
-        if curr_overrides.tmpdir_max_jobs is not None:
-            max_plots = curr_overrides.tmpdir_max_jobs
+    if curr_overrides.tmpdir_max_jobs is not None:
+        max_plots = curr_overrides.tmpdir_max_jobs
+    else:
+        max_plots = sched_cfg.tmpdir_max_jobs
     if len(phases) >= max_plots:
         return False
 

From 082c1e4deebb94daf2d6580bd30d1db5b9792e66 Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Tue, 8 Jun 2021 17:59:18 +0100
Subject: [PATCH 04/44] Update plotman.yaml

---
 src/plotman/resources/plotman.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index 21004605..9835d16e 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -135,7 +135,7 @@ scheduling:
                 # than the global setting above.
                 "/mnt/tmp/00":
                         tmpdir_stagger_phase_major: 1
-                        tmpdir_stagger_phase_minr: 5
+                        tmpdir_stagger_phase_minor: 5
                         tmpdir_stagger_phase_limit: 1
                         tmpdir_max_jobs: 5
 

From c73c63d813e9cf6c9e36bc541ba6d3e5cf023189 Mon Sep 17 00:00:00 2001
From: Graeme Seaton <graemes@graemes.com>
Date: Sun, 13 Jun 2021 15:39:41 +0000
Subject: [PATCH 05/44] Add docker build

---
 .gitignore | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.gitignore b/.gitignore
index ca379a55..609e8dfd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -2,4 +2,5 @@ __pycache__
 venv
 .DS_Store
 .vscode
+*.code-workspace
 src/plotman.egg-info

From 834f2fe3514b195eccdb345b9e4c8793e67d1520 Mon Sep 17 00:00:00 2001
From: Graeme Seaton <graemes@graemes.com>
Date: Sun, 13 Jun 2021 15:39:59 +0000
Subject: [PATCH 06/44] Add docker build

---
 .dockerfile             |  4 +++
 Dockerfile              | 60 +++++++++++++++++++++++++++++++++++++++++
 build-docker-plotman.sh | 20 ++++++++++++++
 3 files changed, 84 insertions(+)
 create mode 100644 .dockerfile
 create mode 100644 Dockerfile
 create mode 100755 build-docker-plotman.sh

diff --git a/.dockerfile b/.dockerfile
new file mode 100644
index 00000000..756b8274
--- /dev/null
+++ b/.dockerfile
@@ -0,0 +1,4 @@
+.git
+.github
+.coveragerc
+.gitignore
diff --git a/Dockerfile b/Dockerfile
new file mode 100644
index 00000000..4435ec98
--- /dev/null
+++ b/Dockerfile
@@ -0,0 +1,60 @@
+# Build deployable artifacts
+FROM ubuntu:latest as plotman-builder
+
+ARG CHIA_BRANCH
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq python3 ansible tar bash ca-certificates git openssl unzip wget python3-pip sudo acl build-essential python3-dev python3.8-venv python3.8-distutils apt nfs-common python-is-python3 cmake libsodium-dev g++
+
+RUN echo "cloning ${CHIA_BRANCH}"
+RUN git clone --branch ${CHIA_BRANCH} https://github.com/Chia-Network/chia-blockchain.git \
+&& cd chia-blockchain \
+&& git submodule update --init mozilla-ca \
+&& chmod +x install.sh 
+
+WORKDIR /chia-blockchain
+# Placeholder for patches
+RUN /usr/bin/sh ./install.sh
+
+COPY . /plotman
+
+RUN . ./activate \
+&& pip install -e /plotman \
+&& deactivate 
+
+# Build deployment container
+FROM ubuntu:latest as plotman
+
+ARG UID=10001
+ARG GID=10001
+
+RUN DEBIAN_FRONTEND=noninteractive apt-get update \
+&& DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq python3 python3.8-venv python3.8-distutils ca-certificates tzdata vim ssh less rsync git tmux libsodium23 \
+&& apt-get clean all \
+&& rm -rf /var/lib/apt/lists
+
+COPY --from=plotman-builder /chia-blockchain /chia-blockchain
+COPY --from=plotman-builder /plotman /plotman
+
+RUN groupadd -g ${GID} chia
+RUN useradd -m -u ${UID} -g ${GID} chia
+
+RUN mkdir -p /data/chia/tmp \
+&& mkdir -p /data/chia/plots \
+&& mkdir -p /data/chia/logs
+
+VOLUME ["/data/chia/tmp","/data/chia/plots","/data/chia/logs"]
+
+RUN chown -R chia:chia /chia-blockchain \
+&& chown -R chia:chia /plotman \
+&& chown -R chia:chia /data/chia
+
+WORKDIR /chia-blockchain
+USER chia
+
+ENV VIRTUAL_ENV="/chia-blockchain/venv"
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+# Kick off plots (assumes the environemnt is good to go)
+CMD ["/bin/bash", "-c", "plotman plot" ]
+# Alternative command to simply provide shell environment
+#CMD ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait" ]
diff --git a/build-docker-plotman.sh b/build-docker-plotman.sh
new file mode 100755
index 00000000..7e9a3be7
--- /dev/null
+++ b/build-docker-plotman.sh
@@ -0,0 +1,20 @@
+#!/bin/sh
+
+DOCKER_REGISTRY="graemes"
+LOCAL_REGISTRY="registry.graemes.com/graemes"
+PROJECT="chia-plotman"
+TAG="plotter"
+CHIA_BRANCH="1.1.7"
+
+docker rmi ${LOCAL_REGISTRY}/${PROJECT}:${TAG}
+
+docker build . \
+	--squash \
+	--build-arg CHIA_BRANCH=${CHIA_BRANCH} \
+	-f Dockerfile \
+	-t ${LOCAL_REGISTRY}/${PROJECT}:${TAG}
+
+#     -t ${DOCKER_REGISTRY}/${PROJECT}:${TAG} \
+
+docker push ${LOCAL_REGISTRY}/${PROJECT}:${TAG}
+#docker push ${DOCKER_REGISTRY}/${PROJECT}:${TAG}

From de6ea5e95b9f12f518bc98ed44e9c84f0e36301f Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Tue, 15 Jun 2021 23:43:03 +0100
Subject: [PATCH 07/44] Moved tmp_overrides from directories into scheduling,
 added extra configuration options with description and examples

---
 src/plotman/resources/plotman.yaml | 29 ++++++++++++++++++-----------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index 9835d16e..f50016d2 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -118,26 +118,33 @@ scheduling:
         # How often the daemon wakes to consider starting a new plot job, in seconds.
         polling_time_s: 20
 
-        # Optional: Allows overriding some scheduling characteristics of certain tmp
-        # directories. This contains a map of tmp directory names to
-        # attributes. If a tmp directory and attribute is not listed here,
-        # it uses the default attribute setting from the main configuration.
+        # Optional: Allows the overriding of some scheduling characteristics of the
+        # tmp directories specified here.
+        # This contains a map of tmp directory names to attributes. If a tmp directory 
+        # and attribute is not listed here, the default attribute setting from the main 
+        # configuration will be used
         #
         # Currently support override parameters:
-        #     - tmpdir_stagger_phase_major
-        #     - tmpdir_stagger_phase_minor
+        #     - tmpdir_stagger_phase_major (requires tmpdir_stagger_phase_minor)
+        #     - tmpdir_stagger_phase_minor (requires tmpdir_stagger_phase_major)
         #     - tmpdir_stagger_phase_limit
         #     - tmpdir_max_jobs
         tmp_overrides:
-                # In this example, /mnt/tmp/00 is larger than the other tmp
-                # dirs and it can hold more plots than the default, allowing
-                # more simultaneous plots and they are being started earlier
-                # than the global setting above.
+                # In this example, /mnt/tmp/00 is larger and faster than the
+                # other tmp dirs and it can hold more plots than the default,
+                # allowing more simultaneous plots, so they are being started
+                # earlier than the global setting above.
                 "/mnt/tmp/00":
                         tmpdir_stagger_phase_major: 1
                         tmpdir_stagger_phase_minor: 5
-                        tmpdir_stagger_phase_limit: 1
                         tmpdir_max_jobs: 5
+                # Here, /mnt/tmp/03 is smaller, so a different config might be
+                # to space the phase stagger further apart and only allow 2 jobs
+                # to run concurrently in it
+                @/mnt/tmp/03::
+                        tmpdir_stagger_phase_major: 3
+                        tmpdir_stagger_phase_minor: 1
+                        tmpdir_max_jobs: 2
 
 # Plotting parameters.  These are pass-through parameters to chia plots create.
 # See documentation at

From 38d7a743f0f43f66e396321254d06b2ac67c0900 Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Wed, 16 Jun 2021 00:24:30 +0100
Subject: [PATCH 08/44] additional overrides in phases_permit_new_job and
 different wait_reason messages

---
 src/plotman/manager.py | 56 +++++++++++++++++++++---------------------
 1 file changed, 28 insertions(+), 28 deletions(-)

diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index 425f9aa1..ccdf8d37 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -53,38 +53,38 @@ def phases_permit_new_job(phases: typing.List[job.Phase], d: str, sched_cfg: plo
     if len(phases) == 0:
         return True
 
+    # Assign variables
+    major = sched_cfg.tmpdir_stagger_phase_major
+    minor = sched_cfg.tmpdir_stagger_phase_minor
+    # tmpdir_stagger_phase_limit default is 1, as declared in configuration.py
+    stagger_phase_limit = sched_cfg.tmpdir_stagger_phase_limit
+    
+    # Limit the total number of jobs per tmp dir. Default to overall max
+    # jobs configuration, but restrict to any configured overrides.
+    max_plots = sched_cfg.tmpdir_max_jobs
+    
     # Check if any overrides exist for the current job
     if sched_cfg.tmp_overrides is not None and d in sched_cfg.tmp_overrides:
         curr_overrides = sched_cfg.tmp_overrides[d]
     
-    # Check and apply overrides for major, minor and phase limit
-    if curr_overrides.tmpdir_stagger_phase_major is not None:
-        major = curr_overrides.tmpdir_stagger_phase_major
-    else:
-        major = sched_cfg.tmpdir_stagger_phase_major
-    if curr_overrides.tmpdir_stagger_phase_minor is not None:
-        minor = curr_overrides.tmpdir_stagger_phase_minor
-    else:
-        minor = sched_cfg.tmpdir_stagger_phase_minor
+        # Check for and assign major & minor phase overrides
+        if curr_overrides.tmpdir_stagger_phase_major is not None:
+            major = curr_overrides.tmpdir_stagger_phase_major
+            minor = curr_overrides.tmpdir_stagger_phase_minor
+        # Check for and assign stagger phase limit override
+        if curr_overrides.tmpdir_stagger_phase_limit is not None:
+            stagger_phase_limit = curr_overrides.tmpdir_stagger_phase_limit
+        # Check for and assign stagger phase limit override
+        if curr_overrides.tmpdir_max_jobs is not None:
+            max_plots = curr_overrides.tmpdir_max_jobs
+        
     milestone = job.Phase(major,minor,)
-    if curr_overrides.tmpdir_stagger_phase_limit is not None:
-        stagger_phase_limit = curr_overrides.tmpdir_stagger_phase_limit
-    else:
-        stagger_phase_limit = sched_cfg.tmpdir_stagger_phase_limit
-    # tmpdir_stagger_phase_limit default is 1, as declared in configuration.py
-    if len([p for p in phases if p < milestone]) >= stagger_phase_limit:
+    
+    # Check if phases pass the criteria
+    if len([p for p in phases if p < milestone]) >= stagger_phase_limit or len(phases) >= max_plots:
         return False
-
-    # Limit the total number of jobs per tmp dir. Default to the overall max
-    # jobs configuration, but restrict to any configured overrides.
-    if curr_overrides.tmpdir_max_jobs is not None:
-        max_plots = curr_overrides.tmpdir_max_jobs
     else:
-        max_plots = sched_cfg.tmpdir_max_jobs
-    if len(phases) >= max_plots:
-        return False
-
-    return True
+        return True
 
 def maybe_start_new_plot(dir_cfg: plotman.configuration.Directories, sched_cfg: plotman.configuration.Scheduling, plotting_cfg: plotman.configuration.Plotting, log_cfg: plotman.configuration.Logging) -> typing.Tuple[bool, str]:
     jobs = job.Job.get_running_jobs(log_cfg.plots)
@@ -94,9 +94,9 @@ def maybe_start_new_plot(dir_cfg: plotman.configuration.Directories, sched_cfg:
     youngest_job_age = min(jobs, key=job.Job.get_time_wall).get_time_wall() if jobs else MAX_AGE
     global_stagger = int(sched_cfg.global_stagger_m * MIN)
     if (youngest_job_age < global_stagger):
-        wait_reason = 'stagger (%ds/%ds)' % (youngest_job_age, global_stagger)
+        wait_reason = 'global stagger (%ds/%ds)' % (youngest_job_age, global_stagger)
     elif len(jobs) >= sched_cfg.global_max_jobs:
-        wait_reason = 'max jobs (%d) - (%ds/%ds)' % (sched_cfg.global_max_jobs, youngest_job_age, global_stagger)
+        wait_reason = 'global max jobs reached (%d)' % (sched_cfg.global_max_jobs)
     else:
         tmp_to_all_phases = [(d, job.job_phases_for_tmpdir(d, jobs)) for d in dir_cfg.tmp]
         eligible = [ (d, phases) for (d, phases) in tmp_to_all_phases
@@ -105,7 +105,7 @@ def maybe_start_new_plot(dir_cfg: plotman.configuration.Directories, sched_cfg:
                 for (d, phases) in eligible ]
 
         if not eligible:
-            wait_reason = 'no eligible tempdirs (%ds/%ds)' % (youngest_job_age, global_stagger)
+            wait_reason = 'waiting for phase match'
         else:
             # Plot to oldest tmpdir.
             tmpdir = max(rankable, key=operator.itemgetter(1))[0]

From cc02768bb10720414f010ffc5771a2488299b6d1 Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Wed, 16 Jun 2021 02:17:56 +0100
Subject: [PATCH 09/44] Moved overrides to scheduling and added extras

---
 src/plotman/configuration.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 96c3365b..493f38a0 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -209,9 +209,11 @@ def maybe_create_scripts(self, temp: str) -> None:
 @attr.frozen
 class TmpOverrides:
     tmpdir_stagger_phase_major: Optional[int] = None
-    tmpdir_stagger_phase_minor: Optional[int] = None
+    tmpdir_stagger_phase_minor: Optional[int] = None        
     tmpdir_stagger_phase_limit: Optional[int] = None
     tmpdir_max_jobs: Optional[int] = None
+    
+    raise Exception(tmpdir_stagger_phase_major)
 
 @attr.frozen
 class Logging:

From 5720fc0f1fd3ada84d1514278068f1cd9c9c42da Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Wed, 16 Jun 2021 03:28:08 +0100
Subject: [PATCH 10/44] Update configuration.py

removed raise Exception as no longer testing
---
 src/plotman/configuration.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 493f38a0..3608871a 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -212,8 +212,6 @@ class TmpOverrides:
     tmpdir_stagger_phase_minor: Optional[int] = None        
     tmpdir_stagger_phase_limit: Optional[int] = None
     tmpdir_max_jobs: Optional[int] = None
-    
-    raise Exception(tmpdir_stagger_phase_major)
 
 @attr.frozen
 class Logging:

From 1ec475c0c000bb4e6c11cd58420e0ef830a9c815 Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Wed, 16 Jun 2021 03:37:23 +0100
Subject: [PATCH 11/44] Updated for #758

---
 CHANGELOG.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0b291813..744c5845 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [unreleased]
 ### Added
+- optional tmpdir_overrides added for each specified tmpdir
+- ([#758](https://github.com/ericaltendorf/plotman/pull/758))
 - `plotman export` command to output summaries from plot logs in `.csv` format.
   ([#557](https://github.com/ericaltendorf/plotman/pull/557))
 - `--json` option for `plotman status`.

From 5b000b188a7a5d0de93dcea473beb713c0f402dd Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Wed, 16 Jun 2021 04:06:44 +0100
Subject: [PATCH 12/44] Move tmpdir_overrides from Directories to Scheduling

---
 src/plotman/_tests/manager_test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/plotman/_tests/manager_test.py b/src/plotman/_tests/manager_test.py
index 0c0b752e..a2ecd381 100755
--- a/src/plotman/_tests/manager_test.py
+++ b/src/plotman/_tests/manager_test.py
@@ -15,15 +15,15 @@ def sched_cfg() -> configuration.Scheduling:
         polling_time_s=2,
         tmpdir_stagger_phase_major=3,
         tmpdir_stagger_phase_minor=0,
-        tmpdir_max_jobs=3
+        tmpdir_max_jobs=3,
+        tmp_overrides={"/mnt/tmp/04": configuration.TmpOverrides(tmpdir_max_jobs=4)
     )
 
 @pytest.fixture
 def dir_cfg() -> configuration.Directories:
     return configuration.Directories(
         tmp=["/var/tmp", "/tmp"],
-        dst=["/mnt/dst/00", "/mnt/dst/01", "/mnt/dst/03"],
-        tmp_overrides={"/mnt/tmp/04": configuration.TmpOverrides(tmpdir_max_jobs=4)}
+        dst=["/mnt/dst/00", "/mnt/dst/01", "/mnt/dst/03"]}
     )
 
 def test_permit_new_job_post_milestone(sched_cfg: configuration.Scheduling, dir_cfg: configuration.Directories) -> None:

From b691b0415c8b16553629ca81d7a0253320fa7652 Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Wed, 16 Jun 2021 04:12:22 +0100
Subject: [PATCH 13/44] Update manager_test.py

Ad missing }
---
 src/plotman/_tests/manager_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/_tests/manager_test.py b/src/plotman/_tests/manager_test.py
index a2ecd381..006b6539 100755
--- a/src/plotman/_tests/manager_test.py
+++ b/src/plotman/_tests/manager_test.py
@@ -16,7 +16,7 @@ def sched_cfg() -> configuration.Scheduling:
         tmpdir_stagger_phase_major=3,
         tmpdir_stagger_phase_minor=0,
         tmpdir_max_jobs=3,
-        tmp_overrides={"/mnt/tmp/04": configuration.TmpOverrides(tmpdir_max_jobs=4)
+        tmp_overrides={"/mnt/tmp/04": configuration.TmpOverrides(tmpdir_max_jobs=4)}
     )
 
 @pytest.fixture

From dcd419f6698af7fe2ba485bbb6028441c9841f37 Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Wed, 16 Jun 2021 04:18:47 +0100
Subject: [PATCH 14/44] Update manager_test.py

Correcting a stupid mistake
---
 src/plotman/_tests/manager_test.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/_tests/manager_test.py b/src/plotman/_tests/manager_test.py
index 006b6539..d9fbae85 100755
--- a/src/plotman/_tests/manager_test.py
+++ b/src/plotman/_tests/manager_test.py
@@ -23,7 +23,7 @@ def sched_cfg() -> configuration.Scheduling:
 def dir_cfg() -> configuration.Directories:
     return configuration.Directories(
         tmp=["/var/tmp", "/tmp"],
-        dst=["/mnt/dst/00", "/mnt/dst/01", "/mnt/dst/03"]}
+        dst=["/mnt/dst/00", "/mnt/dst/01", "/mnt/dst/03"]
     )
 
 def test_permit_new_job_post_milestone(sched_cfg: configuration.Scheduling, dir_cfg: configuration.Directories) -> None:

From 44fd610caacba7615280762ab105e8e49ff27e50 Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Wed, 16 Jun 2021 04:23:14 +0100
Subject: [PATCH 15/44] Correcting typos

---
 src/plotman/resources/plotman.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index f50016d2..a88dc549 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -141,7 +141,7 @@ scheduling:
                 # Here, /mnt/tmp/03 is smaller, so a different config might be
                 # to space the phase stagger further apart and only allow 2 jobs
                 # to run concurrently in it
-                @/mnt/tmp/03::
+                "/mnt/tmp/03":
                         tmpdir_stagger_phase_major: 3
                         tmpdir_stagger_phase_minor: 1
                         tmpdir_max_jobs: 2

From a8599fab0324d969f42c25bd94828a49910f75e2 Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Wed, 16 Jun 2021 04:32:57 +0100
Subject: [PATCH 16/44] remove unexpected whitespaces

---
 src/plotman/configuration.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 3608871a..96c3365b 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -209,7 +209,7 @@ def maybe_create_scripts(self, temp: str) -> None:
 @attr.frozen
 class TmpOverrides:
     tmpdir_stagger_phase_major: Optional[int] = None
-    tmpdir_stagger_phase_minor: Optional[int] = None        
+    tmpdir_stagger_phase_minor: Optional[int] = None
     tmpdir_stagger_phase_limit: Optional[int] = None
     tmpdir_max_jobs: Optional[int] = None
 

From 9299f28cdd905161f1e11dadacf960000d89d075 Mon Sep 17 00:00:00 2001
From: racergoodwin <racer.goodwin@googlemail.com>
Date: Wed, 16 Jun 2021 11:21:42 +0100
Subject: [PATCH 17/44] added global_ to variables defined in scheduling

---
 src/plotman/configuration.py       | 8 ++++----
 src/plotman/manager.py             | 8 ++++----
 src/plotman/resources/plotman.yaml | 8 ++++----
 3 files changed, 12 insertions(+), 12 deletions(-)

diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 96c3365b..09e8ecc0 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -270,10 +270,10 @@ class Scheduling:
     global_max_jobs: int
     global_stagger_m: int
     polling_time_s: int
-    tmpdir_max_jobs: int
-    tmpdir_stagger_phase_major: int
-    tmpdir_stagger_phase_minor: int
-    tmpdir_stagger_phase_limit: int = 1  # If not explicit, "tmpdir_stagger_phase_limit" will default to 1
+    global_tmpdir_max_jobs: int
+    global_tmpdir_stagger_phase_major: int
+    global_tmpdir_stagger_phase_minor: int
+    global_tmpdir_stagger_phase_limit: int = 1  # If not explicit, "tmpdir_stagger_phase_limit" will default to 1
     tmp_overrides: Optional[Dict[str, TmpOverrides]] = None
 
 @attr.frozen
diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index ccdf8d37..af1456c4 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -54,14 +54,14 @@ def phases_permit_new_job(phases: typing.List[job.Phase], d: str, sched_cfg: plo
         return True
 
     # Assign variables
-    major = sched_cfg.tmpdir_stagger_phase_major
-    minor = sched_cfg.tmpdir_stagger_phase_minor
+    major = sched_cfg.global_tmpdir_stagger_phase_major
+    minor = sched_cfg.global_tmpdir_stagger_phase_minor
     # tmpdir_stagger_phase_limit default is 1, as declared in configuration.py
-    stagger_phase_limit = sched_cfg.tmpdir_stagger_phase_limit
+    stagger_phase_limit = sched_cfg.global_tmpdir_stagger_phase_limit
     
     # Limit the total number of jobs per tmp dir. Default to overall max
     # jobs configuration, but restrict to any configured overrides.
-    max_plots = sched_cfg.tmpdir_max_jobs
+    max_plots = sched_cfg.global_tmpdir_max_jobs
     
     # Check if any overrides exist for the current job
     if sched_cfg.tmp_overrides is not None and d in sched_cfg.tmp_overrides:
diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index a88dc549..6212ecb2 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -101,13 +101,13 @@ scheduling:
         # e.g, with default settings, a new plot will start only when your plot
         # reaches phase [2 : 1] on your temp drive. This setting takes precidence
         # over global_stagger_m
-        tmpdir_stagger_phase_major: 2
-        tmpdir_stagger_phase_minor: 1
+        global_tmpdir_stagger_phase_major: 2
+        global_tmpdir_stagger_phase_minor: 1
         # Optional: default is 1
-        tmpdir_stagger_phase_limit: 1
+        global_tmpdir_stagger_phase_limit: 1
 
         # Don't run more than this many jobs at a time on a single temp dir.
-        tmpdir_max_jobs: 3
+        global_tmpdir_max_jobs: 3
 
         # Don't run more than this many jobs at a time in total.
         global_max_jobs: 12

From 9b448e8a3ff21740243db170d9e6921fab4861e5 Mon Sep 17 00:00:00 2001
From: racergoodwin <racer.goodwin@googlemail.com>
Date: Wed, 16 Jun 2021 11:35:23 +0100
Subject: [PATCH 18/44] update _test/manager_test.py

---
 src/plotman/_tests/manager_test.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/plotman/_tests/manager_test.py b/src/plotman/_tests/manager_test.py
index d9fbae85..98710087 100755
--- a/src/plotman/_tests/manager_test.py
+++ b/src/plotman/_tests/manager_test.py
@@ -13,9 +13,9 @@ def sched_cfg() -> configuration.Scheduling:
         global_max_jobs=1,
         global_stagger_m=2,
         polling_time_s=2,
-        tmpdir_stagger_phase_major=3,
-        tmpdir_stagger_phase_minor=0,
-        tmpdir_max_jobs=3,
+        global_tmpdir_stagger_phase_major=3,
+        global_tmpdir_stagger_phase_minor=0,
+        global_tmpdir_max_jobs=3,
         tmp_overrides={"/mnt/tmp/04": configuration.TmpOverrides(tmpdir_max_jobs=4)}
     )
 

From 85d95d6893dacf625c65a21b4717c8a7a786c614 Mon Sep 17 00:00:00 2001
From: racergoodwin <racer.goodwin@googlemail.com>
Date: Wed, 16 Jun 2021 11:50:24 +0100
Subject: [PATCH 19/44] added seperate check for tmpdir_stagger_phase_minor and
 removed global_ from tmpdir variables

---
 src/plotman/_tests/manager_test.py | 6 +++---
 src/plotman/configuration.py       | 8 ++++----
 src/plotman/manager.py             | 9 +++++----
 src/plotman/resources/plotman.yaml | 8 ++++----
 4 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/src/plotman/_tests/manager_test.py b/src/plotman/_tests/manager_test.py
index 98710087..d9fbae85 100755
--- a/src/plotman/_tests/manager_test.py
+++ b/src/plotman/_tests/manager_test.py
@@ -13,9 +13,9 @@ def sched_cfg() -> configuration.Scheduling:
         global_max_jobs=1,
         global_stagger_m=2,
         polling_time_s=2,
-        global_tmpdir_stagger_phase_major=3,
-        global_tmpdir_stagger_phase_minor=0,
-        global_tmpdir_max_jobs=3,
+        tmpdir_stagger_phase_major=3,
+        tmpdir_stagger_phase_minor=0,
+        tmpdir_max_jobs=3,
         tmp_overrides={"/mnt/tmp/04": configuration.TmpOverrides(tmpdir_max_jobs=4)}
     )
 
diff --git a/src/plotman/configuration.py b/src/plotman/configuration.py
index 09e8ecc0..96c3365b 100644
--- a/src/plotman/configuration.py
+++ b/src/plotman/configuration.py
@@ -270,10 +270,10 @@ class Scheduling:
     global_max_jobs: int
     global_stagger_m: int
     polling_time_s: int
-    global_tmpdir_max_jobs: int
-    global_tmpdir_stagger_phase_major: int
-    global_tmpdir_stagger_phase_minor: int
-    global_tmpdir_stagger_phase_limit: int = 1  # If not explicit, "tmpdir_stagger_phase_limit" will default to 1
+    tmpdir_max_jobs: int
+    tmpdir_stagger_phase_major: int
+    tmpdir_stagger_phase_minor: int
+    tmpdir_stagger_phase_limit: int = 1  # If not explicit, "tmpdir_stagger_phase_limit" will default to 1
     tmp_overrides: Optional[Dict[str, TmpOverrides]] = None
 
 @attr.frozen
diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index af1456c4..c109d577 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -54,14 +54,14 @@ def phases_permit_new_job(phases: typing.List[job.Phase], d: str, sched_cfg: plo
         return True
 
     # Assign variables
-    major = sched_cfg.global_tmpdir_stagger_phase_major
-    minor = sched_cfg.global_tmpdir_stagger_phase_minor
+    major = sched_cfg.tmpdir_stagger_phase_major
+    minor = sched_cfg.tmpdir_stagger_phase_minor
     # tmpdir_stagger_phase_limit default is 1, as declared in configuration.py
-    stagger_phase_limit = sched_cfg.global_tmpdir_stagger_phase_limit
+    stagger_phase_limit = sched_cfg.tmpdir_stagger_phase_limit
     
     # Limit the total number of jobs per tmp dir. Default to overall max
     # jobs configuration, but restrict to any configured overrides.
-    max_plots = sched_cfg.global_tmpdir_max_jobs
+    max_plots = sched_cfg.tmpdir_max_jobs
     
     # Check if any overrides exist for the current job
     if sched_cfg.tmp_overrides is not None and d in sched_cfg.tmp_overrides:
@@ -70,6 +70,7 @@ def phases_permit_new_job(phases: typing.List[job.Phase], d: str, sched_cfg: plo
         # Check for and assign major & minor phase overrides
         if curr_overrides.tmpdir_stagger_phase_major is not None:
             major = curr_overrides.tmpdir_stagger_phase_major
+        if curr_overrides.tmpdir_stagger_phase_minor is not None:
             minor = curr_overrides.tmpdir_stagger_phase_minor
         # Check for and assign stagger phase limit override
         if curr_overrides.tmpdir_stagger_phase_limit is not None:
diff --git a/src/plotman/resources/plotman.yaml b/src/plotman/resources/plotman.yaml
index 6212ecb2..a88dc549 100644
--- a/src/plotman/resources/plotman.yaml
+++ b/src/plotman/resources/plotman.yaml
@@ -101,13 +101,13 @@ scheduling:
         # e.g, with default settings, a new plot will start only when your plot
         # reaches phase [2 : 1] on your temp drive. This setting takes precidence
         # over global_stagger_m
-        global_tmpdir_stagger_phase_major: 2
-        global_tmpdir_stagger_phase_minor: 1
+        tmpdir_stagger_phase_major: 2
+        tmpdir_stagger_phase_minor: 1
         # Optional: default is 1
-        global_tmpdir_stagger_phase_limit: 1
+        tmpdir_stagger_phase_limit: 1
 
         # Don't run more than this many jobs at a time on a single temp dir.
-        global_tmpdir_max_jobs: 3
+        tmpdir_max_jobs: 3
 
         # Don't run more than this many jobs at a time in total.
         global_max_jobs: 12

From 8163dac106d0930dfc6b835b96db6daf724d395f Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Fri, 18 Jun 2021 17:54:55 +0100
Subject: [PATCH 20/44] Remove unnecessary comma

Co-authored-by: Kyle Altendorf <sda@fstab.net>
---
 src/plotman/manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index c109d577..a63bf7a2 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -79,7 +79,7 @@ def phases_permit_new_job(phases: typing.List[job.Phase], d: str, sched_cfg: plo
         if curr_overrides.tmpdir_max_jobs is not None:
             max_plots = curr_overrides.tmpdir_max_jobs
         
-    milestone = job.Phase(major,minor,)
+    milestone = job.Phase(major,minor)
     
     # Check if phases pass the criteria
     if len([p for p in phases if p < milestone]) >= stagger_phase_limit or len(phases) >= max_plots:

From 64a8eb75a82e4534973366aac641e24c195a5993 Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Fri, 18 Jun 2021 17:55:16 +0100
Subject: [PATCH 21/44] Update CHANGELOG.md

Co-authored-by: Kyle Altendorf <sda@fstab.net>
---
 CHANGELOG.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 744c5845..f0127041 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,7 +8,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [unreleased]
 ### Added
 - optional tmpdir_overrides added for each specified tmpdir
-- ([#758](https://github.com/ericaltendorf/plotman/pull/758))
+  ([#758](https://github.com/ericaltendorf/plotman/pull/758))
 - `plotman export` command to output summaries from plot logs in `.csv` format.
   ([#557](https://github.com/ericaltendorf/plotman/pull/557))
 - `--json` option for `plotman status`.

From f06400e5fe9122e56c41d112c841b29d60fcc8ab Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Fri, 18 Jun 2021 17:57:06 +0100
Subject: [PATCH 22/44] Undo change

Co-authored-by: Kyle Altendorf <sda@fstab.net>
---
 src/plotman/manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index a63bf7a2..fa544b12 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -97,7 +97,7 @@ def maybe_start_new_plot(dir_cfg: plotman.configuration.Directories, sched_cfg:
     if (youngest_job_age < global_stagger):
         wait_reason = 'global stagger (%ds/%ds)' % (youngest_job_age, global_stagger)
     elif len(jobs) >= sched_cfg.global_max_jobs:
-        wait_reason = 'global max jobs reached (%d)' % (sched_cfg.global_max_jobs)
+        wait_reason = 'global max jobs reached (%d) - (%ds/%ds)' % (sched_cfg.global_max_jobs, youngest_job_age, global_stagger)
     else:
         tmp_to_all_phases = [(d, job.job_phases_for_tmpdir(d, jobs)) for d in dir_cfg.tmp]
         eligible = [ (d, phases) for (d, phases) in tmp_to_all_phases

From 69fed66332db1e5b0bc184f85678bc697c9f73c0 Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Fri, 18 Jun 2021 18:01:18 +0100
Subject: [PATCH 23/44] Update src/plotman/manager.py

Co-authored-by: Kyle Altendorf <sda@fstab.net>
---
 src/plotman/manager.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index fa544b12..24f2b965 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -82,7 +82,10 @@ def phases_permit_new_job(phases: typing.List[job.Phase], d: str, sched_cfg: plo
     milestone = job.Phase(major,minor)
     
     # Check if phases pass the criteria
-    if len([p for p in phases if p < milestone]) >= stagger_phase_limit or len(phases) >= max_plots:
+    if len([p for p in phases if p < milestone]) >= stagger_phase_limit
+        return False
+
+    if len(phases) >= max_plots:
         return False
     else:
         return True

From ad0b9e07bd0cd35350e126c88c3dea01981fd018 Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Fri, 18 Jun 2021 18:01:25 +0100
Subject: [PATCH 24/44] Update src/plotman/manager.py

Co-authored-by: Kyle Altendorf <sda@fstab.net>
---
 src/plotman/manager.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index 24f2b965..b7f3a2a6 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -87,8 +87,8 @@ def phases_permit_new_job(phases: typing.List[job.Phase], d: str, sched_cfg: plo
 
     if len(phases) >= max_plots:
         return False
-    else:
-        return True
+
+    return True
 
 def maybe_start_new_plot(dir_cfg: plotman.configuration.Directories, sched_cfg: plotman.configuration.Scheduling, plotting_cfg: plotman.configuration.Plotting, log_cfg: plotman.configuration.Logging) -> typing.Tuple[bool, str]:
     jobs = job.Job.get_running_jobs(log_cfg.plots)

From 3a92068aa54decf19b9afe4046c4f507f52f13b4 Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Sat, 19 Jun 2021 16:02:49 +0100
Subject: [PATCH 25/44] Add missing colon

---
 src/plotman/manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index b7f3a2a6..c204d00f 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -82,7 +82,7 @@ def phases_permit_new_job(phases: typing.List[job.Phase], d: str, sched_cfg: plo
     milestone = job.Phase(major,minor)
     
     # Check if phases pass the criteria
-    if len([p for p in phases if p < milestone]) >= stagger_phase_limit
+    if len([p for p in phases if p < milestone]) >= stagger_phase_limit:
         return False
 
     if len(phases) >= max_plots:

From b74e2ad938dcd996b78d4cc6ca5e5dead450433c Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Sun, 20 Jun 2021 18:26:21 +0100
Subject: [PATCH 26/44] Return wait_reason to orig

---
 src/plotman/manager.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index c204d00f..f7d5e8b1 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -98,9 +98,9 @@ def maybe_start_new_plot(dir_cfg: plotman.configuration.Directories, sched_cfg:
     youngest_job_age = min(jobs, key=job.Job.get_time_wall).get_time_wall() if jobs else MAX_AGE
     global_stagger = int(sched_cfg.global_stagger_m * MIN)
     if (youngest_job_age < global_stagger):
-        wait_reason = 'global stagger (%ds/%ds)' % (youngest_job_age, global_stagger)
+        wait_reason = 'stagger (%ds/%ds)' % (youngest_job_age, global_stagger)
     elif len(jobs) >= sched_cfg.global_max_jobs:
-        wait_reason = 'global max jobs reached (%d) - (%ds/%ds)' % (sched_cfg.global_max_jobs, youngest_job_age, global_stagger)
+        wait_reason = 'max jobs (%d) - (%ds/%ds)' % (sched_cfg.global_max_jobs, youngest_job_age, global_stagger)
     else:
         tmp_to_all_phases = [(d, job.job_phases_for_tmpdir(d, jobs)) for d in dir_cfg.tmp]
         eligible = [ (d, phases) for (d, phases) in tmp_to_all_phases
@@ -109,7 +109,7 @@ def maybe_start_new_plot(dir_cfg: plotman.configuration.Directories, sched_cfg:
                 for (d, phases) in eligible ]
 
         if not eligible:
-            wait_reason = 'waiting for phase match'
+            wait_reason = 'no eligible tempdirs (%ds/%ds)' % (youngest_job_age, global_stagger)'
         else:
             # Plot to oldest tmpdir.
             tmpdir = max(rankable, key=operator.itemgetter(1))[0]

From 3c6d8fa25c16b4487f9190cbf70b5965e949bb96 Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Sun, 20 Jun 2021 18:32:52 +0100
Subject: [PATCH 27/44] abolish aberrant apostrophe!

---
 src/plotman/manager.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/src/plotman/manager.py b/src/plotman/manager.py
index f7d5e8b1..88d18127 100644
--- a/src/plotman/manager.py
+++ b/src/plotman/manager.py
@@ -109,7 +109,7 @@ def maybe_start_new_plot(dir_cfg: plotman.configuration.Directories, sched_cfg:
                 for (d, phases) in eligible ]
 
         if not eligible:
-            wait_reason = 'no eligible tempdirs (%ds/%ds)' % (youngest_job_age, global_stagger)'
+            wait_reason = 'no eligible tempdirs (%ds/%ds)' % (youngest_job_age, global_stagger)
         else:
             # Plot to oldest tmpdir.
             tmpdir = max(rankable, key=operator.itemgetter(1))[0]

From e364d757edb7b1042bcc5c8b5d775cd97b548641 Mon Sep 17 00:00:00 2001
From: Graeme Seaton <graemes@graemes.com>
Date: Tue, 22 Jun 2021 20:00:28 +0100
Subject: [PATCH 28/44] Incorporate review comments

---
 Dockerfile              | 28 ++++++++++++----------------
 build-docker-plotman.sh | 15 ++++++++-------
 2 files changed, 20 insertions(+), 23 deletions(-)

diff --git a/Dockerfile b/Dockerfile
index 4435ec98..58908986 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,39 +1,36 @@
 # Build deployable artifacts
-FROM ubuntu:latest as plotman-builder
+ARG BASE_CONTAINER=ubuntu:20.04
+FROM ${BASE_CONTAINER} as plotman-builder
 
-ARG CHIA_BRANCH
+ARG CHIA_GIT_REFERENCE
 
-RUN DEBIAN_FRONTEND=noninteractive apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq python3 ansible tar bash ca-certificates git openssl unzip wget python3-pip sudo acl build-essential python3-dev python3.8-venv python3.8-distutils apt nfs-common python-is-python3 cmake libsodium-dev g++
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq python3 ansible tar bash ca-certificates git openssl unzip wget python3-pip sudo acl build-essential python3-dev python3.8-venv python3.8-distutils apt nfs-common python-is-python3
 
-RUN echo "cloning ${CHIA_BRANCH}"
-RUN git clone --branch ${CHIA_BRANCH} https://github.com/Chia-Network/chia-blockchain.git \
+RUN echo "cloning ${CHIA_GIT_REFERENCE}"
+RUN git clone --branch "${CHIA_GIT_REFERENCE}" https://github.com/Chia-Network/chia-blockchain.git \
 && cd chia-blockchain \
-&& git submodule update --init mozilla-ca \
-&& chmod +x install.sh 
+&& git submodule update --init mozilla-ca
 
 WORKDIR /chia-blockchain
 # Placeholder for patches
-RUN /usr/bin/sh ./install.sh
+RUN /bin/bash ./install.sh
 
 COPY . /plotman
 
-RUN . ./activate \
-&& pip install -e /plotman \
-&& deactivate 
+RUN ["/bin/bash", "-c", "source ./activate && pip install /plotman && deactivate"] 
 
 # Build deployment container
-FROM ubuntu:latest as plotman
+FROM ${BASE_CONTAINER} as plotman
 
 ARG UID=10001
 ARG GID=10001
 
 RUN DEBIAN_FRONTEND=noninteractive apt-get update \
-&& DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq python3 python3.8-venv python3.8-distutils ca-certificates tzdata vim ssh less rsync git tmux libsodium23 \
+&& DEBIAN_FRONTEND=noninteractive apt-get install -y curl jq python3 python3.8-venv ca-certificates tzdata ssh rsync \
 && apt-get clean all \
 && rm -rf /var/lib/apt/lists
 
 COPY --from=plotman-builder /chia-blockchain /chia-blockchain
-COPY --from=plotman-builder /plotman /plotman
 
 RUN groupadd -g ${GID} chia
 RUN useradd -m -u ${UID} -g ${GID} chia
@@ -45,7 +42,6 @@ RUN mkdir -p /data/chia/tmp \
 VOLUME ["/data/chia/tmp","/data/chia/plots","/data/chia/logs"]
 
 RUN chown -R chia:chia /chia-blockchain \
-&& chown -R chia:chia /plotman \
 && chown -R chia:chia /data/chia
 
 WORKDIR /chia-blockchain
@@ -57,4 +53,4 @@ ENV PATH="$VIRTUAL_ENV/bin:$PATH"
 # Kick off plots (assumes the environemnt is good to go)
 CMD ["/bin/bash", "-c", "plotman plot" ]
 # Alternative command to simply provide shell environment
-#CMD ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait" ]
+# CMD ["/bin/bash", "-c", "trap : TERM INT; sleep infinity & wait" ]
diff --git a/build-docker-plotman.sh b/build-docker-plotman.sh
index 7e9a3be7..61809a11 100755
--- a/build-docker-plotman.sh
+++ b/build-docker-plotman.sh
@@ -1,20 +1,21 @@
-#!/bin/sh
+#!/bin/bash
 
-DOCKER_REGISTRY="graemes"
-LOCAL_REGISTRY="registry.graemes.com/graemes"
+LOCAL_REGISTRY="<local-registry>"
+#DOCKER_REGISTRY="<docker-registry>"
 PROJECT="chia-plotman"
 TAG="plotter"
-CHIA_BRANCH="1.1.7"
+BASE_CONTAINER="ubuntu:20.04"
+CHIA_GIT_REFERENCE="1.1.7"
 
 docker rmi ${LOCAL_REGISTRY}/${PROJECT}:${TAG}
 
 docker build . \
 	--squash \
-	--build-arg CHIA_BRANCH=${CHIA_BRANCH} \
+	--build-arg BASE_CONTAINER=${BASE_CONTAINER} \
+	--build-arg CHIA_GIT_REFERENCE=${CHIA_GIT_REFERENCE} \
 	-f Dockerfile \
 	-t ${LOCAL_REGISTRY}/${PROJECT}:${TAG}
-
-#     -t ${DOCKER_REGISTRY}/${PROJECT}:${TAG} \
+#     -t ${DOCKER_REGISTRY}/${PROJECT}:${TAG}
 
 docker push ${LOCAL_REGISTRY}/${PROJECT}:${TAG}
 #docker push ${DOCKER_REGISTRY}/${PROJECT}:${TAG}

From a8f24b5242c1b4dd68205214b00a38f084db1c82 Mon Sep 17 00:00:00 2001
From: Rafael Steil <rafael.steil@shopify.com>
Date: Mon, 21 Jun 2021 11:53:15 -0400
Subject: [PATCH 29/44] Look from job temp files in the filesystem instead of
 open files

---
 src/plotman/job.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/src/plotman/job.py b/src/plotman/job.py
index 3009df6b..c99ae257 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -7,6 +7,7 @@
 import random
 import re
 import sys
+import glob
 import time
 from datetime import datetime
 from enum import Enum, auto
@@ -555,13 +556,11 @@ def resume(self) -> None:
     def get_temp_files(self) -> typing.Set[str]:
         # Prevent duplicate file paths by using set.
         temp_files = set([])
-        for f in self.proc.open_files():
-            if any(
-                dir in f.path
-                for dir in [self.tmpdir, self.tmp2dir, self.dstdir]
-                if dir is not None
-            ):
-                temp_files.add(f.path)
+
+        for dir in [self.tmpdir, self.tmp2dir, self.dstdir]:
+            if dir is not None:
+                temp_files.update(glob.glob(os.path.join(dir, "plot-*-{0}.*".format(self.plot_id))))
+
         return temp_files
 
     def cancel(self) -> None:

From 834e8240830416a8c84b645a5f1e8089dfae554c Mon Sep 17 00:00:00 2001
From: Rafael Steil <rafael.steil@shopify.com>
Date: Mon, 21 Jun 2021 11:34:15 -0400
Subject: [PATCH 30/44] Add -f argument to plotman kill to bypass confirmation

---
 src/plotman/job.py     |  2 +-
 src/plotman/plotman.py | 17 +++++++++++++----
 2 files changed, 14 insertions(+), 5 deletions(-)

diff --git a/src/plotman/job.py b/src/plotman/job.py
index c99ae257..96dd72fb 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -559,7 +559,7 @@ def get_temp_files(self) -> typing.Set[str]:
 
         for dir in [self.tmpdir, self.tmp2dir, self.dstdir]:
             if dir is not None:
-                temp_files.update(glob.glob(os.path.join(dir, "plot-*-{0}.*".format(self.plot_id))))
+                temp_files.update(glob.glob(os.path.join(dir, f"plot-*-{self.plot_id}.*")))
 
         return temp_files
 
diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py
index 7aa9536c..3b67ef6a 100755
--- a/src/plotman/plotman.py
+++ b/src/plotman/plotman.py
@@ -73,6 +73,7 @@ def parse_args(self) -> typing.Any:
         self.add_idprefix_arg(p_files)
 
         p_kill = sp.add_parser('kill', help='kill job (and cleanup temp files)')
+        p_kill.add_argument('-f', '--force', action='store_true', default=False, help="Don't ask for confirmation before killing the plot job")
         self.add_idprefix_arg(p_kill)
 
         p_suspend = sp.add_parser('suspend', help='suspend job')
@@ -309,15 +310,23 @@ def main() -> None:
                         job.suspend()
 
                         temp_files = job.get_temp_files()
-                        print('Will kill pid %d, plot id %s' % (job.proc.pid, job.plot_id))
-                        print('Will delete %d temp files' % len(temp_files))
-                        conf = input('Are you sure? ("y" to confirm): ')
+                        
+                        if args.force:
+                            conf = 'y'
+                        else:
+                            conf = input('Are you sure? ("y" to confirm): ')
+
                         if (conf != 'y'):
-                            print('canceled.  If you wish to resume the job, do so manually.')
+                            print('Canceled.  If you wish to resume the job, do so manually.')
                         else:
+                            print('Will kill pid %d, plot id %s' % (job.proc.pid, job.plot_id))
+                            print('Will delete %d temp files' % len(temp_files))
                             print('killing...')
+
                             job.cancel()
+
                             print('cleaning up temp files...')
+
                             for f in temp_files:
                                 os.remove(f)
 

From 49f97f0d93ea61d398f9143e4d82a09f5b0c5262 Mon Sep 17 00:00:00 2001
From: Rafael Steil <rafael.steil@shopify.com>
Date: Tue, 22 Jun 2021 23:46:04 -0400
Subject: [PATCH 31/44] Update CHANGELOG

---
 CHANGELOG.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4e717de8..6db63086 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -20,6 +20,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Support the [madMAx plotter](https://github.com/madMAx43v3r/chia-plotter).
   See the [configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Configuration#2-v05) for help setting it up.
   ([#797](https://github.com/ericaltendorf/plotman/pull/797))
+- Added argument `-f / --force` to `plotman kill` to skip confirmation before killing the job
+
+### Fixed
+- `plotman kill` doesn't leave any temporary files behind anymore
 
 ## [0.4.1] - 2021-06-11
 ### Fixed

From d80aa87dcd52e541fb652257dae47b441140be4b Mon Sep 17 00:00:00 2001
From: Graeme Seaton <graemes@graemes.com>
Date: Wed, 23 Jun 2021 12:33:24 +0100
Subject: [PATCH 32/44] Add uid/gid args

---
 build-docker-plotman.sh | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/build-docker-plotman.sh b/build-docker-plotman.sh
index 61809a11..1f3e0bc4 100755
--- a/build-docker-plotman.sh
+++ b/build-docker-plotman.sh
@@ -6,6 +6,8 @@ PROJECT="chia-plotman"
 TAG="plotter"
 BASE_CONTAINER="ubuntu:20.04"
 CHIA_GIT_REFERENCE="1.1.7"
+UID=10001
+GID=10001
 
 docker rmi ${LOCAL_REGISTRY}/${PROJECT}:${TAG}
 
@@ -13,6 +15,8 @@ docker build . \
 	--squash \
 	--build-arg BASE_CONTAINER=${BASE_CONTAINER} \
 	--build-arg CHIA_GIT_REFERENCE=${CHIA_GIT_REFERENCE} \
+	--build-arg UID=${UID} \
+	--build-arg GID=${GID} \
 	-f Dockerfile \
 	-t ${LOCAL_REGISTRY}/${PROJECT}:${TAG}
 #     -t ${DOCKER_REGISTRY}/${PROJECT}:${TAG}

From 336d73c9acf67fcd35d47ecea28810e76968d1e7 Mon Sep 17 00:00:00 2001
From: Graeme Seaton <graemes@graemes.com>
Date: Wed, 23 Jun 2021 13:21:51 +0100
Subject: [PATCH 33/44] Move docker artifacts to own subdirectory

---
 .dockerfile => .dockerignore    | 1 +
 build-docker-plotman.sh         | 6 ++++--
 Dockerfile => docker/Dockerfile | 0
 3 files changed, 5 insertions(+), 2 deletions(-)
 rename .dockerfile => .dockerignore (83%)
 rename Dockerfile => docker/Dockerfile (100%)

diff --git a/.dockerfile b/.dockerignore
similarity index 83%
rename from .dockerfile
rename to .dockerignore
index 756b8274..aba6e3ad 100644
--- a/.dockerfile
+++ b/.dockerignore
@@ -2,3 +2,4 @@
 .github
 .coveragerc
 .gitignore
+docker
diff --git a/build-docker-plotman.sh b/build-docker-plotman.sh
index 1f3e0bc4..24a23698 100755
--- a/build-docker-plotman.sh
+++ b/build-docker-plotman.sh
@@ -6,6 +6,8 @@ PROJECT="chia-plotman"
 TAG="plotter"
 BASE_CONTAINER="ubuntu:20.04"
 CHIA_GIT_REFERENCE="1.1.7"
+
+# The UID/GID should match the 'chia' owner of the directories on the host system
 UID=10001
 GID=10001
 
@@ -17,9 +19,9 @@ docker build . \
 	--build-arg CHIA_GIT_REFERENCE=${CHIA_GIT_REFERENCE} \
 	--build-arg UID=${UID} \
 	--build-arg GID=${GID} \
-	-f Dockerfile \
+	-f docker/Dockerfile \
 	-t ${LOCAL_REGISTRY}/${PROJECT}:${TAG}
-#     -t ${DOCKER_REGISTRY}/${PROJECT}:${TAG}
+#        -t ${DOCKER_REGISTRY}/${PROJECT}:${TAG}
 
 docker push ${LOCAL_REGISTRY}/${PROJECT}:${TAG}
 #docker push ${DOCKER_REGISTRY}/${PROJECT}:${TAG}
diff --git a/Dockerfile b/docker/Dockerfile
similarity index 100%
rename from Dockerfile
rename to docker/Dockerfile

From 70f281f0a655e62c7d1c4c76c51a5b423f982738 Mon Sep 17 00:00:00 2001
From: Graeme Seaton <graemes@graemes.com>
Date: Wed, 23 Jun 2021 13:46:32 +0100
Subject: [PATCH 34/44] Tidy up registry reference

---
 build-docker-plotman.sh | 9 +++------
 1 file changed, 3 insertions(+), 6 deletions(-)

diff --git a/build-docker-plotman.sh b/build-docker-plotman.sh
index 24a23698..1ae200b0 100755
--- a/build-docker-plotman.sh
+++ b/build-docker-plotman.sh
@@ -1,7 +1,6 @@
 #!/bin/bash
 
-LOCAL_REGISTRY="<local-registry>"
-#DOCKER_REGISTRY="<docker-registry>"
+DOCKER_REGISTRY="<docker-registry>"
 PROJECT="chia-plotman"
 TAG="plotter"
 BASE_CONTAINER="ubuntu:20.04"
@@ -20,8 +19,6 @@ docker build . \
 	--build-arg UID=${UID} \
 	--build-arg GID=${GID} \
 	-f docker/Dockerfile \
-	-t ${LOCAL_REGISTRY}/${PROJECT}:${TAG}
-#        -t ${DOCKER_REGISTRY}/${PROJECT}:${TAG}
+        -t ${DOCKER_REGISTRY}/${PROJECT}:${TAG}
 
-docker push ${LOCAL_REGISTRY}/${PROJECT}:${TAG}
-#docker push ${DOCKER_REGISTRY}/${PROJECT}:${TAG}
+docker push ${DOCKER_REGISTRY}/${PROJECT}:${TAG}

From 46cb570d2881ee545062c649bcea9daeae02a0da Mon Sep 17 00:00:00 2001
From: Graeme Seaton <graemes@graemes.com>
Date: Wed, 23 Jun 2021 19:13:43 +0100
Subject: [PATCH 35/44] Add samples and update CHANGELOG

---
 CHANGELOG.md                     |  4 ++++
 docker/sample.docker-compose.yml | 19 +++++++++++++++++++
 docker/sample.env                |  7 +++++++
 3 files changed, 30 insertions(+)
 create mode 100644 docker/sample.docker-compose.yml
 create mode 100644 docker/sample.env

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 4e717de8..26481fda 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -20,6 +20,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Support the [madMAx plotter](https://github.com/madMAx43v3r/chia-plotter).
   See the [configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Configuration#2-v05) for help setting it up.
   ([#797](https://github.com/ericaltendorf/plotman/pull/797))
+- Docker container support.
+  See the [docker configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Docker-Configuration) for help setting it up.
+  ([#783](https://github.com/ericaltendorf/plotman/pull/783))
+
 
 ## [0.4.1] - 2021-06-11
 ### Fixed
diff --git a/docker/sample.docker-compose.yml b/docker/sample.docker-compose.yml
new file mode 100644
index 00000000..af6ae14f
--- /dev/null
+++ b/docker/sample.docker-compose.yml
@@ -0,0 +1,19 @@
+version: "3"
+
+services:
+  chia_plotman:
+    restart: always
+    container_name: chia-plotman
+    image: ${DOCKER_IMAGE}
+    volumes:
+      - ${HOME}/.ssh:/home/chia/.ssh
+      - ${HOME}/.chia:/home/chia/.chia
+      - ${HOME}/.config:/home/chia/.config
+      - ${LOGS_DIR}:/data/chia/logs
+      - ${PLOTS_DIR}:/data/chia/plots
+      - ${PLOTS_TMP_DIR}:/data/chia/tmp
+      - /tmp:/tmp
+    logging:
+      options:
+        max-size: ${DOCKER_LOG_MAX_SIZE}
+        max-file: ${DOCKER_LOG_MAX_FILE}
diff --git a/docker/sample.env b/docker/sample.env
new file mode 100644
index 00000000..725fd840
--- /dev/null
+++ b/docker/sample.env
@@ -0,0 +1,7 @@
+DOCKER_IMAGE=<docker-registry>/chia-plotman:plotter
+LOGS_DIR=/data/chia/logs
+PLOTS_DIR=/data/chia/plots
+PLOTS_TMP_DIR=/data/chia/tmp
+DOCKER_LOG_MAX_SIZE=4m
+DOCKER_LOG_MAX_FILE=10
+

From 1429ef8b52397a8000f7d8ed123486800723c76f Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 24 Jun 2021 19:37:23 -0700
Subject: [PATCH 36/44] Apply suggestions from code review

---
 CHANGELOG.md       | 2 +-
 src/plotman/job.py | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6db63086..f6330aea 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -20,7 +20,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Support the [madMAx plotter](https://github.com/madMAx43v3r/chia-plotter).
   See the [configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Configuration#2-v05) for help setting it up.
   ([#797](https://github.com/ericaltendorf/plotman/pull/797))
-- Added argument `-f / --force` to `plotman kill` to skip confirmation before killing the job
+- Added argument `-f`/`--force` to `plotman kill` to skip confirmation before killing the job.
 
 ### Fixed
 - `plotman kill` doesn't leave any temporary files behind anymore
diff --git a/src/plotman/job.py b/src/plotman/job.py
index 96dd72fb..ffa0dc71 100644
--- a/src/plotman/job.py
+++ b/src/plotman/job.py
@@ -559,7 +559,7 @@ def get_temp_files(self) -> typing.Set[str]:
 
         for dir in [self.tmpdir, self.tmp2dir, self.dstdir]:
             if dir is not None:
-                temp_files.update(glob.glob(os.path.join(dir, f"plot-*-{self.plot_id}.*")))
+                temp_files.update(glob.glob(os.path.join(dir, f"plot-*-{self.plot_id}.tmp")))
 
         return temp_files
 

From 5adea47069790356bd3c0631cf88a6c8903b4af6 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 24 Jun 2021 22:39:39 -0400
Subject: [PATCH 37/44] Update CHANGELOG.md

---
 CHANGELOG.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index f6330aea..48102953 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -6,6 +6,9 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ## [unreleased]
+### Fixed
+- `plotman kill` doesn't leave any temporary files behind anymore.
+  ([#801](https://github.com/ericaltendorf/plotman/pull/801))
 ### Added
 - `plotman export` command to output summaries from plot logs in `.csv` format.
   ([#557](https://github.com/ericaltendorf/plotman/pull/557))
@@ -21,9 +24,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   See the [configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Configuration#2-v05) for help setting it up.
   ([#797](https://github.com/ericaltendorf/plotman/pull/797))
 - Added argument `-f`/`--force` to `plotman kill` to skip confirmation before killing the job.
-
-### Fixed
-- `plotman kill` doesn't leave any temporary files behind anymore
+  ([#801](https://github.com/ericaltendorf/plotman/pull/801))
 
 ## [0.4.1] - 2021-06-11
 ### Fixed

From 078f353ba5b3b0b45e74c15b640a31dfba951308 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Thu, 24 Jun 2021 22:41:07 -0400
Subject: [PATCH 38/44] Update plotman.py

---
 src/plotman/plotman.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/src/plotman/plotman.py b/src/plotman/plotman.py
index 3b67ef6a..5312b78c 100755
--- a/src/plotman/plotman.py
+++ b/src/plotman/plotman.py
@@ -311,6 +311,9 @@ def main() -> None:
 
                         temp_files = job.get_temp_files()
                         
+                        print('Will kill pid %d, plot id %s' % (job.proc.pid, job.plot_id))
+                        print('Will delete %d temp files' % len(temp_files))
+
                         if args.force:
                             conf = 'y'
                         else:
@@ -319,8 +322,6 @@ def main() -> None:
                         if (conf != 'y'):
                             print('Canceled.  If you wish to resume the job, do so manually.')
                         else:
-                            print('Will kill pid %d, plot id %s' % (job.proc.pid, job.plot_id))
-                            print('Will delete %d temp files' % len(temp_files))
                             print('killing...')
 
                             job.cancel()

From 1e20fce67130f5c9f785944acd584223ed3437a9 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Sun, 27 Jun 2021 16:59:01 -0400
Subject: [PATCH 39/44] update MANIFEST.in for Docker files

---
 MANIFEST.in | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/MANIFEST.in b/MANIFEST.in
index 08d64bbe..8e5ab768 100644
--- a/MANIFEST.in
+++ b/MANIFEST.in
@@ -9,3 +9,6 @@ include .coveragerc
 recursive-include src *.py
 recursive-include src/plotman/_tests/resources *
 recursive-include src/plotman/resources *
+recursive-exclude docker *
+exclude .dockerignore
+exclude build-docker-plotman.sh

From eb82eaa9631e78f50781f842fbf3df008641894c Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Mon, 28 Jun 2021 22:42:56 -0400
Subject: [PATCH 40/44] some references are still k32

---
 src/plotman/_tests/plot_util_test.py | 4 ++--
 src/plotman/archive.py               | 4 ++--
 src/plotman/plot_util.py             | 2 +-
 src/plotman/reporting.py             | 2 +-
 4 files changed, 6 insertions(+), 6 deletions(-)

diff --git a/src/plotman/_tests/plot_util_test.py b/src/plotman/_tests/plot_util_test.py
index 5a0f9879..ab912e55 100755
--- a/src/plotman/_tests/plot_util_test.py
+++ b/src/plotman/_tests/plot_util_test.py
@@ -45,7 +45,7 @@ def test_columns() -> None:
               [ 1 ],
               [ 2 ] ] )
 
-def test_list_k_plots(fs: pyfakefs.fake_filesystem.FakeFilesystem) -> None:
+def test_list_plots(fs: pyfakefs.fake_filesystem.FakeFilesystem) -> None:
     fs.create_file('/t/plot-k32-0.plot', st_size=108 * GB)
     fs.create_file('/t/plot-k32-1.plot', st_size=108 * GB)
     fs.create_file('/t/.plot-k32-2.plot', st_size=108 * GB)
@@ -53,7 +53,7 @@ def test_list_k_plots(fs: pyfakefs.fake_filesystem.FakeFilesystem) -> None:
     fs.create_file('/t/plot-k32-4.plot', st_size=100 * GB)
     fs.create_file('/t/plot-k32-5.plot', st_size=108 * GB)
 
-    assert (plot_util.list_k_plots('/t/') ==
+    assert (plot_util.list_plots('/t/') ==
             [ '/t/plot-k32-0.plot',
               '/t/plot-k32-1.plot',
               '/t/plot-k32-5.plot' ] )
diff --git a/src/plotman/archive.py b/src/plotman/archive.py
index 0c78ee07..59a9a9e6 100644
--- a/src/plotman/archive.py
+++ b/src/plotman/archive.py
@@ -96,7 +96,7 @@ def spawn_archive_process(dir_cfg: configuration.Directories, arch_cfg: configur
 
 def compute_priority(phase: job.Phase, gb_free: float, n_plots: int) -> int:
     # All these values are designed around dst buffer dirs of about
-    # ~2TB size and containing k plots.  TODO: Generalize, and
+    # ~2TB size and containing k32 plots.  TODO: Generalize, and
     # rewrite as a sort function.
 
     priority = 50
@@ -210,7 +210,7 @@ def archive(dir_cfg: configuration.Directories, arch_cfg: configuration.Archivin
     dst_dir = dir_cfg.get_dst_directories()
     for d in dst_dir:
         ph = dir2ph.get(d, job.Phase(0, 0))
-        dir_plots = plot_util.list_k_plots(d)
+        dir_plots = plot_util.list_plots(d)
         gb_free = plot_util.df_b(d) / plot_util.GB
         n_plots = len(dir_plots)
         priority = compute_priority(ph, gb_free, n_plots)
diff --git a/src/plotman/plot_util.py b/src/plotman/plot_util.py
index 5bad17bd..38d5f78c 100644
--- a/src/plotman/plot_util.py
+++ b/src/plotman/plot_util.py
@@ -51,7 +51,7 @@ def split_path_prefix(items: typing.List[str]) -> typing.Tuple[str, typing.List[
         remainders = [ os.path.relpath(i, prefix) for i in items ]
         return (prefix, remainders)
 
-def list_k_plots(d: str) -> typing.List[str]:
+def list_plots(d: str) -> typing.List[str]:
     'List completed plots in a directory (not recursive)'
     plots = []
     for plot in os.listdir(d):
diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py
index 83fe5230..75ad5a8c 100644
--- a/src/plotman/reporting.py
+++ b/src/plotman/reporting.py
@@ -216,7 +216,7 @@ def dst_dir_report(jobs: typing.List[job.Job], dstdirs: typing.List[str], width:
         eldest_ph = dir2oldphase.get(d, job.Phase(0, 0))
         phases = job.job_phases_for_dstdir(d, jobs)
 
-        dir_plots = plot_util.list_k_plots(d)
+        dir_plots = plot_util.list_plots(d)
         gb_free = int(plot_util.df_b(d) / plot_util.GB)
         n_plots = len(dir_plots)
         priority = archive.compute_priority(eldest_ph, gb_free, n_plots)

From 20284713d6475100519db4656e6d042d66b53128 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Mon, 28 Jun 2021 22:43:04 -0400
Subject: [PATCH 41/44] add changelog entry for #803

---
 CHANGELOG.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index eaf67106..adb3cdbd 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -28,6 +28,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Docker container support.
   See the [docker configuration wiki page](https://github.com/ericaltendorf/plotman/wiki/Docker-Configuration) for help setting it up.
   ([#783](https://github.com/ericaltendorf/plotman/pull/783))
+- Plot sizes other than k32 are handled.
+  ([#803](https://github.com/ericaltendorf/plotman/pull/803))
 
 ## [0.4.1] - 2021-06-11
 ### Fixed

From deace18001c0a471c3cdfefb5108f2b87c2c85cb Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Mon, 28 Jun 2021 22:45:43 -0400
Subject: [PATCH 42/44] add a small k33 test case

---
 src/plotman/_tests/plot_util_test.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/src/plotman/_tests/plot_util_test.py b/src/plotman/_tests/plot_util_test.py
index ab912e55..6718cb44 100755
--- a/src/plotman/_tests/plot_util_test.py
+++ b/src/plotman/_tests/plot_util_test.py
@@ -53,10 +53,14 @@ def test_list_plots(fs: pyfakefs.fake_filesystem.FakeFilesystem) -> None:
     fs.create_file('/t/plot-k32-4.plot', st_size=100 * GB)
     fs.create_file('/t/plot-k32-5.plot', st_size=108 * GB)
 
+    fs.create_file('/t/plot-k33-6.plot', st_size=108 * GB)
+    fs.create_file('/t/plot-k33-7.plot', st_size=216 * GB)
+
     assert (plot_util.list_plots('/t/') ==
             [ '/t/plot-k32-0.plot',
               '/t/plot-k32-1.plot',
-              '/t/plot-k32-5.plot' ] )
+              '/t/plot-k32-5.plot',
+              '/t/plot-k33-7.plot' ] )
 
 
 def test_get_plotsize() -> None:

From b4f8b905281672243ecace2e7f409caf6f4786b4 Mon Sep 17 00:00:00 2001
From: racergoodwin <84841970+racergoodwin@users.noreply.github.com>
Date: Tue, 29 Jun 2021 21:56:06 +0100
Subject: [PATCH 43/44] Update CHANGELOG.md

Agree to commit suggestion

Co-authored-by: Kyle Altendorf <sda@fstab.net>
---
 CHANGELOG.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 9e920bae..40dac73f 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -10,7 +10,9 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - `plotman kill` doesn't leave any temporary files behind anymore.
   ([#801](https://github.com/ericaltendorf/plotman/pull/801))
 ### Added
-- optional tmpdir_overrides added for each specified tmpdir
+- tmp directory overrides moved to `scheduling:` `tmp_overrides:`.
+  ([#758](https://github.com/ericaltendorf/plotman/pull/758))
+- Per tmp directory phase limit control added to `scheduling:` `tmp_overrides:`.
   ([#758](https://github.com/ericaltendorf/plotman/pull/758))
 - `plotman export` command to output summaries from plot logs in `.csv` format.
   ([#557](https://github.com/ericaltendorf/plotman/pull/557))

From 1a4a055ea8d3fbb848c11bd1044c052c22a23502 Mon Sep 17 00:00:00 2001
From: Kyle Altendorf <sda@fstab.net>
Date: Tue, 29 Jun 2021 20:09:51 -0400
Subject: [PATCH 44/44] no walrus in 3.7

---
 src/plotman/plot_util.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/src/plotman/plot_util.py b/src/plotman/plot_util.py
index 38d5f78c..75d85eca 100644
--- a/src/plotman/plot_util.py
+++ b/src/plotman/plot_util.py
@@ -55,7 +55,8 @@ def list_plots(d: str) -> typing.List[str]:
     'List completed plots in a directory (not recursive)'
     plots = []
     for plot in os.listdir(d):
-        if matches := re.search(r"^plot-k(\d*)-.*plot$", plot):
+        matches = re.search(r"^plot-k(\d*)-.*plot$", plot)
+        if matches is not None:
             grps = matches.groups()
             plot_k = int(grps[0])
             plot = os.path.join(d, plot)