trying out pre commit hook [TEST]

.pre-commit-config.yaml

@@ -0,0 +1,12 @@
+repos:
+-   repo: https://github.com/ambv/black
+    rev: stable
+    hooks:
+    - id: black
+      args: [--safe]
+      language_version: python3.6
+-   repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v1.2.3
+    hooks:
+    -   id: flake8
+        args: [--max-line-length=88]

circle.yml

@@ -1,7 +1,8 @@
 dependencies:
 
     pre:
-        - pip install tox tox-pyenv
+        - pip install tox tox-pyenv flake8 pre-commit
 
     override:
+        - "if $(python -V 2>&1 | grep -q 'Python 3.6'); then pre-commit run --all-files; fi"
         - pyenv local 2.7.10 3.6.1

nodata/alphamask.py

@@ -19,7 +19,9 @@ def value_grab(a, b):
         out[b] = measures[a[0] - 1][m_key]
         return None
 
-    sci_meas.labeled_comprehension(labeled_img, labeled_img, u_labels, value_grab, float, 0, pass_positions=True)
+    sci_meas.labeled_comprehension(
+        labeled_img, labeled_img, u_labels, value_grab, float, 0, pass_positions=True
+    )
 
     return out.reshape(labeled_img.shape)
 
@@ -49,7 +51,7 @@ def all_valid_edges(data, ndv, threshold=0):
 
 
 def simple_mask(data, ndv):
-    '''Exact nodata masking'''
+    """Exact nodata masking"""
     depth, rows, cols = data.shape
     nd = np.iinfo(data.dtype).max
     alpha = np.invert(np.all(np.dstack(data) == ndv, axis=2)).astype(data.dtype) * nd
@@ -69,12 +71,19 @@ def slic_mask(arr, nodata, n_clusters=50, threshold=5, debug=False):
     clusters = slic(near_nodata, n_clusters)
     labeled = measure.label(clusters) + 1
     measures = measure.regionprops(labeled, intensity_image=near_nodata, cache=True)
-    mean_intensity = _hacky_make_image(labeled, np.unique(labeled), measures, 'mean_intensity')
+    mean_intensity = _hacky_make_image(
+        labeled, np.unique(labeled), measures, "mean_intensity"
+    )
     mask = binary_fill_holes(np.invert(binary_fill_holes(mean_intensity >= threshold)))
     nd = np.iinfo(arr.dtype).max
     mask = np.invert(mask) * nd
     if debug:
         d = dict([(m.label, m.mean_intensity) for m in measures])
-        return mask.astype('uint8'), labeled.astype('uint32'), mean_intensity.astype('uint32'), d
+        return (
+            mask.astype("uint8"),
+            labeled.astype("uint32"),
+            mean_intensity.astype("uint32"),
+            d,
+        )
     else:
-        return mask.astype('uint8')
+        return mask.astype("uint8")

nodata/blob.py

@@ -17,16 +17,13 @@ def pad_window(wnd, pad):
         # rasterio < 1.0, already a tuple
         pass
 
-    return (
-        (wnd[0][0] - pad, wnd[0][1] + pad),
-        (wnd[1][0] - pad, wnd[1][1] + pad)
-    )
+    return ((wnd[0][0] - pad, wnd[0][1] + pad), (wnd[1][0] - pad, wnd[1][1] + pad))
 
 
 def test_rgb(count, nodata, alphafy, outCount):
     if count == 3 and alphafy:
         if not isinstance(nodata, Number):
-            raise ValueError('3 band imagery must have a defined nodata value')
+            raise ValueError("3 band imagery must have a defined nodata value")
 
         return None, nodata, outCount
     elif alphafy:
@@ -56,60 +53,63 @@ def handle_RGB(img, mask):
 
 
 def blob_worker(srcs, window, ij, globalArgs):
-    pad = globalArgs['max_search_distance'] + 1
+    pad = globalArgs["max_search_distance"] + 1
     padded = pad_window(window, pad)
     padWindow = Window.from_slices(*padded, boundless=True)
     img = srcs[0].read(boundless=True, window=padWindow)
 
-    if isinstance(globalArgs['selectNodata'], Number):
+    if isinstance(globalArgs["selectNodata"], Number):
         mask = srcs[0].read_masks(boundless=True, window=padWindow)[0]
         img = handle_RGB(img, mask)
         alphamask = False
     else:
         mask = img[-1]
         alphamask = True
 
-    if globalArgs['maskThreshold'] is not None and alphamask:
-        img[-1] = (np.invert(img[-1] < globalArgs['maskThreshold']).astype(img.dtype)
-                   * np.iinfo(img.dtype).max)
+    if globalArgs["maskThreshold"] is not None and alphamask:
+        img[-1] = (
+            np.invert(img[-1] < globalArgs["maskThreshold"]).astype(img.dtype)
+            * np.iinfo(img.dtype).max
+        )
         mask = img[-1]
 
     if runNodataFiller(mask, pad):
         img = fill_nodata(
-            img, mask, globalArgs['bands'],
-            globalArgs['max_search_distance'])[:, pad: -pad, pad: -pad]
+            img, mask, globalArgs["bands"], globalArgs["max_search_distance"]
+        )[:, pad:-pad, pad:-pad]
 
-        if globalArgs['nibblemask'] \
-                and alphamask is False \
-                and 'nodata' in srcs[0].meta:
+        if globalArgs["nibblemask"] and alphamask is False and "nodata" in srcs[0].meta:
 
             img = nibble_filled_mask(
-                img,
-                srcs[0].meta['nodata'],
-                globalArgs['max_search_distance'])
+                img, srcs[0].meta["nodata"], globalArgs["max_search_distance"]
+            )
 
-        elif globalArgs['nibblemask'] and alphamask:
+        elif globalArgs["nibblemask"] and alphamask:
             img[-1] = nibble_filled_mask(
-                img[-1],
-                None,
-                globalArgs['max_search_distance'],
-                True)
+                img[-1], None, globalArgs["max_search_distance"], True
+            )
 
     else:
-        img = img[:, pad: -pad, pad: -pad]
+        img = img[:, pad:-pad, pad:-pad]
 
     return img
 
 
 def blob_nodata(
-        src_path, dst_path, bidx, max_search_distance, nibblemask,
-        creation_options, maskThreshold, workers, alphafy):
+    src_path,
+    dst_path,
+    bidx,
+    max_search_distance,
+    nibblemask,
+    creation_options,
+    maskThreshold,
+    workers,
+    alphafy,
+):
     """
     """
     with rio.open(src_path) as src:
-        windows = [
-            [window, ij] for ij, window in src.block_windows()
-        ]
+        windows = [[window, ij] for ij, window in src.block_windows()]
 
         options = src.meta.copy()
         kwds = src.profile.copy()
@@ -128,29 +128,33 @@ def blob_nodata(
 
             if bidx and (len(bidx) == 0 or len(bidx) > src.count):
                 raise ValueError(
-                    "Bands %s differ from source count of %s" %
-                    (', '.join([str(b) for b in bidx]), src.count))
+                    "Bands %s differ from source count of %s"
+                    % (", ".join([str(b) for b in bidx]), src.count)
+                )
         elif alphafy and src.count == 3:
             bidx = list(src.indexes)
             bidx.append(src.indexes[-1] + 1)
         else:
             bidx = list(src.indexes)
 
         if maskThreshold is not None:
-            maskThreshold = np.iinfo(options['dtype']).max - maskThreshold
+            maskThreshold = np.iinfo(options["dtype"]).max - maskThreshold
 
     with riomucho.RioMucho(
-            [src_path], dst_path, blob_worker,
-            windows=windows,
-            global_args={
-                'max_search_distance': max_search_distance,
-                'nibblemask': nibblemask,
-                'bands': bidx,
-                'maskThreshold': maskThreshold,
-                'selectNodata': selectNodata
-            },
-            options=options,
-            mode='manual_read') as rm:
+        [src_path],
+        dst_path,
+        blob_worker,
+        windows=windows,
+        global_args={
+            "max_search_distance": max_search_distance,
+            "nibblemask": nibblemask,
+            "bands": bidx,
+            "maskThreshold": maskThreshold,
+            "selectNodata": selectNodata,
+        },
+        options=options,
+        mode="manual_read",
+    ) as rm:
 
         rm.run(workers)
 
@@ -160,27 +164,31 @@ def nibble_filled_mask(filled, nodataval, max_search_distance, is_mask=False):
     if is_mask:
         filled = minimum_filter(filled, size=(max_search_distance * 2 + 1))
     else:
-        nmsk = (filled[0] == nodataval) & (filled[1] == nodataval) & (filled[2] == nodataval)
+        nmsk = (
+            (filled[0] == nodataval)
+            & (filled[1] == nodataval)
+            & (filled[2] == nodataval)
+        )
         nmsk = maximum_filter(nmsk, size=(max_search_distance * 2 + 1))
-        mskInds = np.where(nmsk == True)
+        mskInds = np.where(nmsk)
         filled[:, mskInds[0], mskInds[1]] = nodataval
 
     return filled
 
 
 def make_nibbled(src_path, dst_path, nibble):
-    with rio.open(src_path, 'r') as src:
+    with rio.open(src_path, "r") as src:
         img = src.read(masked=False)
         kwargs = src.meta
-        if 'nodata' in src.meta:
-            nodataval = src.meta['nodata']
+        if "nodata" in src.meta:
+            nodataval = src.meta["nodata"]
         else:
             nodataval = 0.0
 
-    kwargs.update(compress='lzw')
+    kwargs.update(compress="lzw")
 
     nibbled = nibble_filled_mask(img, nodataval, nibble)
 
-    with rio.open(dst_path, 'w', **kwargs) as dst:
+    with rio.open(dst_path, "w", **kwargs) as dst:
         for i, arr in enumerate(nibbled, 1):
             dst.write(arr, i)

nodata/scripts/alpha.py

@@ -1,7 +1,7 @@
 from itertools import repeat
 from multiprocessing import cpu_count, Pool
-import sys
 import zlib
+
 try:
     from itertools import izip
 except ImportError:
@@ -35,6 +35,7 @@ def finalize_worker():
 
 # The following function is executed by worker processes.
 
+
 def compute_window_mask(args):
     """Execute the given function with keyword arguments to compute a
     valid data mask.
@@ -44,7 +45,7 @@ def compute_window_mask(args):
     window, nodata, extra_args = args
     global mask_function, src_dataset
 
-    padding = int(extra_args.get('padding', 0))
+    padding = int(extra_args.get("padding", 0))
 
     if padding:
         start, stop = zip(*window)
@@ -76,8 +77,7 @@ class NodataPoolMan:
     windows of a dataset.
     """
 
-    def __init__(self, input_path, func, nodata, num_workers=None,
-            max_tasks=100):
+    def __init__(self, input_path, func, nodata, num_workers=None, max_tasks=100):
         """Create a pool of workers to process window masks"""
         self.input_path = input_path
         self.func = func
@@ -89,8 +89,8 @@ def __init__(self, input_path, func, nodata, num_workers=None,
             self.dtype = src.dtypes[0]
 
         self.pool = Pool(
-            num_workers or cpu_count()-1, init_worker, (input_path, func),
-            max_tasks)
+            num_workers or cpu_count() - 1, init_worker, (input_path, func), max_tasks
+        )
 
     def mask(self, windows, **kwargs):
         """Iterate over windows and compute mask arrays.
@@ -101,11 +101,10 @@ def mask(self, windows, **kwargs):
         Yields window, ndarray pairs.
         """
         iterargs = izip(windows, repeat(self.nodata), repeat(kwargs))
-        for out_window, data in self.pool.imap_unordered(
-                compute_window_mask, iterargs):
+        for out_window, data in self.pool.imap_unordered(compute_window_mask, iterargs):
 
-            out_data = numpy.fromstring(
-                        zlib.decompress(data), self.dtype).reshape(
-                            [int(x) for x in rasterio.windows.shape(out_window)])
+            out_data = numpy.fromstring(zlib.decompress(data), self.dtype).reshape(
+                [int(x) for x in rasterio.windows.shape(out_window)]
+            )
 
             yield out_window, out_data