From 261d91c5077f393bb9c8c1babd6304abe3b22acd Mon Sep 17 00:00:00 2001 From: Pablo Garcia de los Salmones Valencia Date: Wed, 23 Jun 2021 01:58:50 +0200 Subject: [PATCH 1/2] Find generalized plots from k32 to any k value --- src/plotman/plot_util.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/plotman/plot_util.py b/src/plotman/plot_util.py index 37215527..5238f5e4 100644 --- a/src/plotman/plot_util.py +++ b/src/plotman/plot_util.py @@ -55,17 +55,18 @@ def split_path_prefix(items: typing.List[str]) -> typing.Tuple[str, typing.List[ return (prefix, remainders) def list_k32_plots(d: str) -> typing.List[str]: - 'List completed k32 plots in a directory (not recursive)' + 'List completed k32 (and others) plots in a directory (not recursive)' plots = [] for plot in os.listdir(d): - if re.match(r'^plot-k32-.*plot$', plot): + if matches := re.search(r"^plot-k(\d*)-.*plot$", plot): + grps = matches.groups() + plot_k = int(grps[0]) plot = os.path.join(d, plot) try: - if os.stat(plot).st_size > (0.95 * get_k32_plotsize()): + if os.stat(plot).st_size > (0.95 * get_plotsize(plot_k)): plots.append(plot) except FileNotFoundError: continue - return plots def column_wrap( From fc3996ac3b6a70fd993aa1acf81c5961c08deae7 Mon Sep 17 00:00:00 2001 From: Pablo Garcia de los Salmones Valencia Date: Wed, 23 Jun 2021 02:02:42 +0200 Subject: [PATCH 2/2] Generalized code form k32 to any size of k --- src/plotman/_tests/plot_util_test.py | 4 ++-- src/plotman/archive.py | 4 ++-- src/plotman/plot_util.py | 7 ++----- src/plotman/reporting.py | 2 +- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/plotman/_tests/plot_util_test.py b/src/plotman/_tests/plot_util_test.py index 4a9a8c0b..5a0f9879 100755 --- a/src/plotman/_tests/plot_util_test.py +++ b/src/plotman/_tests/plot_util_test.py @@ -45,7 +45,7 @@ def test_columns() -> None: [ 1 ], [ 2 ] ] ) -def test_list_k32_plots(fs: pyfakefs.fake_filesystem.FakeFilesystem) -> None: +def test_list_k_plots(fs: pyfakefs.fake_filesystem.FakeFilesystem) -> None: fs.create_file('/t/plot-k32-0.plot', st_size=108 * GB) fs.create_file('/t/plot-k32-1.plot', st_size=108 * GB) fs.create_file('/t/.plot-k32-2.plot', st_size=108 * GB) @@ -53,7 +53,7 @@ def test_list_k32_plots(fs: pyfakefs.fake_filesystem.FakeFilesystem) -> None: fs.create_file('/t/plot-k32-4.plot', st_size=100 * GB) fs.create_file('/t/plot-k32-5.plot', st_size=108 * GB) - assert (plot_util.list_k32_plots('/t/') == + assert (plot_util.list_k_plots('/t/') == [ '/t/plot-k32-0.plot', '/t/plot-k32-1.plot', '/t/plot-k32-5.plot' ] ) diff --git a/src/plotman/archive.py b/src/plotman/archive.py index 13c36ddd..0c78ee07 100644 --- a/src/plotman/archive.py +++ b/src/plotman/archive.py @@ -96,7 +96,7 @@ def spawn_archive_process(dir_cfg: configuration.Directories, arch_cfg: configur def compute_priority(phase: job.Phase, gb_free: float, n_plots: int) -> int: # All these values are designed around dst buffer dirs of about - # ~2TB size and containing k32 plots. TODO: Generalize, and + # ~2TB size and containing k plots. TODO: Generalize, and # rewrite as a sort function. priority = 50 @@ -210,7 +210,7 @@ def archive(dir_cfg: configuration.Directories, arch_cfg: configuration.Archivin dst_dir = dir_cfg.get_dst_directories() for d in dst_dir: ph = dir2ph.get(d, job.Phase(0, 0)) - dir_plots = plot_util.list_k32_plots(d) + dir_plots = plot_util.list_k_plots(d) gb_free = plot_util.df_b(d) / plot_util.GB n_plots = len(dir_plots) priority = compute_priority(ph, gb_free, n_plots) diff --git a/src/plotman/plot_util.py b/src/plotman/plot_util.py index 5238f5e4..5bad17bd 100644 --- a/src/plotman/plot_util.py +++ b/src/plotman/plot_util.py @@ -14,9 +14,6 @@ def df_b(d: str) -> int: usage = shutil.disk_usage(d) return usage.free -def get_k32_plotsize() -> int: - return get_plotsize(32) - def get_plotsize(k: int) -> int: return (int)(_get_plotsize_scaler(k) * k * pow(2, k)) @@ -54,8 +51,8 @@ def split_path_prefix(items: typing.List[str]) -> typing.Tuple[str, typing.List[ remainders = [ os.path.relpath(i, prefix) for i in items ] return (prefix, remainders) -def list_k32_plots(d: str) -> typing.List[str]: - 'List completed k32 (and others) plots in a directory (not recursive)' +def list_k_plots(d: str) -> typing.List[str]: + 'List completed plots in a directory (not recursive)' plots = [] for plot in os.listdir(d): if matches := re.search(r"^plot-k(\d*)-.*plot$", plot): diff --git a/src/plotman/reporting.py b/src/plotman/reporting.py index 0a74d84d..be2fca9f 100644 --- a/src/plotman/reporting.py +++ b/src/plotman/reporting.py @@ -215,7 +215,7 @@ def dst_dir_report(jobs: typing.List[job.Job], dstdirs: typing.List[str], width: eldest_ph = dir2oldphase.get(d, job.Phase(0, 0)) phases = job.job_phases_for_dstdir(d, jobs) - dir_plots = plot_util.list_k32_plots(d) + dir_plots = plot_util.list_k_plots(d) gb_free = int(plot_util.df_b(d) / plot_util.GB) n_plots = len(dir_plots) priority = archive.compute_priority(eldest_ph, gb_free, n_plots)