From b71458cbac47d83f18cb5a13308bff609b9c516f Mon Sep 17 00:00:00 2001
From: Hamdah Shafqat Abbasi <hamdahshafqat@hotmail.com>
Date: Wed, 27 Nov 2024 17:17:22 -0500
Subject: [PATCH 01/12] adding new tabular statistic plugin

---
 .../polus-csv-statistics-plugin/Dockerfile    |  17 -
 features/polus-csv-statistics-plugin/VERSION  |   1 -
 .../build-docker.sh                           |   4 -
 features/polus-csv-statistics-plugin/ict.yaml |  60 ---
 .../polus-csv-statistics-plugin/src/main.py   | 309 --------------
 .../src/requirements.txt                      |   1 -
 .../tabular-statistics-tool/.bumpversion.cfg  |  31 ++
 features/tabular-statistics-tool/Dockerfile   |  24 ++
 .../README.md                                 |  17 +-
 .../TabularStatistics.cwl                     |  36 ++
 features/tabular-statistics-tool/VERSION      |   1 +
 .../tabular-statistics-tool/build-docker.sh   |  23 +
 features/tabular-statistics-tool/gitignore    |  23 +
 features/tabular-statistics-tool/ict.yaml     |  62 +++
 .../plugin.json                               |  43 +-
 .../tabular-statistics-tool/pyproject.toml    |  31 ++
 .../features/tabular_statistics/__init__.py   |   4 +
 .../features/tabular_statistics/__main__.py   | 117 +++++
 .../tabular_statistics/tabular_statistics.py  | 399 ++++++++++++++++++
 19 files changed, 790 insertions(+), 413 deletions(-)
 delete mode 100644 features/polus-csv-statistics-plugin/Dockerfile
 delete mode 100644 features/polus-csv-statistics-plugin/VERSION
 delete mode 100755 features/polus-csv-statistics-plugin/build-docker.sh
 delete mode 100644 features/polus-csv-statistics-plugin/ict.yaml
 delete mode 100644 features/polus-csv-statistics-plugin/src/main.py
 delete mode 100644 features/polus-csv-statistics-plugin/src/requirements.txt
 create mode 100644 features/tabular-statistics-tool/.bumpversion.cfg
 create mode 100644 features/tabular-statistics-tool/Dockerfile
 rename features/{polus-csv-statistics-plugin => tabular-statistics-tool}/README.md (67%)
 create mode 100644 features/tabular-statistics-tool/TabularStatistics.cwl
 create mode 100644 features/tabular-statistics-tool/VERSION
 create mode 100755 features/tabular-statistics-tool/build-docker.sh
 create mode 100644 features/tabular-statistics-tool/gitignore
 create mode 100644 features/tabular-statistics-tool/ict.yaml
 rename features/{polus-csv-statistics-plugin => tabular-statistics-tool}/plugin.json (71%)
 create mode 100644 features/tabular-statistics-tool/pyproject.toml
 create mode 100644 features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/__init__.py
 create mode 100644 features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/__main__.py
 create mode 100644 features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py

diff --git a/features/polus-csv-statistics-plugin/Dockerfile b/features/polus-csv-statistics-plugin/Dockerfile
deleted file mode 100644
index d6b8f9f..0000000
--- a/features/polus-csv-statistics-plugin/Dockerfile
+++ /dev/null
@@ -1,17 +0,0 @@
-FROM polusai/bfio:2.1.9
-
-COPY VERSION /
-		
-ARG EXEC_DIR="/opt/executables"
-ARG DATA_DIR="/data"
-
-RUN mkdir -p ${EXEC_DIR} \
-    && mkdir -p ${DATA_DIR}/inputs \
-    && mkdir ${DATA_DIR}/outputs
-
-COPY src ${EXEC_DIR}/
-WORKDIR ${EXEC_DIR}
-
-RUN pip3 install -r ${EXEC_DIR}/requirements.txt
-
-ENTRYPOINT ["python3", "/opt/executables/main.py"]
\ No newline at end of file
diff --git a/features/polus-csv-statistics-plugin/VERSION b/features/polus-csv-statistics-plugin/VERSION
deleted file mode 100644
index 7dff5b8..0000000
--- a/features/polus-csv-statistics-plugin/VERSION
+++ /dev/null
@@ -1 +0,0 @@
-0.2.1
\ No newline at end of file
diff --git a/features/polus-csv-statistics-plugin/build-docker.sh b/features/polus-csv-statistics-plugin/build-docker.sh
deleted file mode 100755
index ff8f13c..0000000
--- a/features/polus-csv-statistics-plugin/build-docker.sh
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/bash
-
-version=$(<VERSION)
-docker build . -t polusai/csv-statistics-plugin:${version}
\ No newline at end of file
diff --git a/features/polus-csv-statistics-plugin/ict.yaml b/features/polus-csv-statistics-plugin/ict.yaml
deleted file mode 100644
index 21e2a19..0000000
--- a/features/polus-csv-statistics-plugin/ict.yaml
+++ /dev/null
@@ -1,60 +0,0 @@
-author:
-- Nick Schaub
-contact: nick.schaub@nih.gov
-container: polusai/csv-statistics-plugin:0.2.1
-description: Calculate simple statistics to groups of data in a csv file.
-entrypoint: '[python3, main.py]'
-inputs:
-- description: The csv statistics to generate
-  format:
-  - statistics
-  name: statistics
-  required: true
-  type: array
-- description: Input csv collection to be processed by this plugin
-  format:
-  - inpDir
-  name: inpDir
-  required: true
-  type: path
-- description: The filepattern of the images represented in the csv files
-  format:
-  - filePattern
-  name: filePattern
-  required: false
-  type: string
-- description: The variable(s) to group the images by
-  format:
-  - groupBy
-  name: groupBy
-  required: false
-  type: string
-name: polusai/CSVStatistics
-outputs:
-- description: Output collection
-  format:
-  - outDir
-  name: outDir
-  required: true
-  type: path
-repository: https://github.com/labshare/polus-plugins
-specVersion: 1.0.0
-title: CSV Statistics
-ui:
-- description: Types of statistics to calculate
-  key: inputs.statistics
-  title: Statistics
-  type: text
-- description: Input csv collection to be processed by this plugin
-  key: inputs.inpDir
-  title: Input collection
-  type: path
-- description: The filepattern of the images represented in the csv files
-  key: inputs.filePattern
-  title: Filepattern
-  type: text
-- description: The variable(s) to group the images by
-  key: inputs.groupBy
-  title: Groupby Variables(s)
-  type: text
-version: 0.2.1
diff --git a/features/polus-csv-statistics-plugin/src/main.py b/features/polus-csv-statistics-plugin/src/main.py
deleted file mode 100644
index 7a01596..0000000
--- a/features/polus-csv-statistics-plugin/src/main.py
+++ /dev/null
@@ -1,309 +0,0 @@
-import argparse, logging, math
-from pathlib import Path
-from filepattern.functions import get_regex, parse_filename, output_name
-
-
-def get_vars(file, groupBy, regex, var):
-    if isinstance(file, list):
-        r = parse_filename(file[0], regex, var)
-    else:
-        r = parse_filename(file, regex, var)
-    vars = {var:value for var,value in r.items() if var in groupBy}
-    return vars
-
-def get_number(s):
-    """ Check that s is number
-    
-    If s is a number, returns it as a float. If not, returns s without modification.
-
-    Inputs:
-        s - An input string or number
-    Outputs:
-        value - Either float(s) or False if s cannot be cast to float
-    """
-    try:
-        return float(s)
-    except ValueError:
-        return s
-    
-def count(data_list,data_dict):
-    if 'count' in data_dict.keys():
-        return
-    data_dict['count'] = len(data_list)
-    return data_dict
-    
-def minval(data_list,data_dict):
-    if 'min' in data_dict.keys():
-        return
-    data_dict['min'] = min(data_list)
-    return data_dict
-
-def maxval(data_list,data_dict):
-    if 'max' in data_dict.keys():
-        return
-    data_dict['max'] = max(data_list)
-    return data_dict
-    
-def mean(data_list,data_dict):
-    if 'mean' in data_dict.keys():
-        return
-    count(data_list,data_dict)
-    data_dict['mean'] = sum(data_list)/data_dict['count']
-    return data_dict
-    
-def var(data_list,data_dict):
-    if 'var' in data_dict.keys():
-        return
-    mean(data_list,data_dict)
-    data_dict['var'] = sum([x**2 for x in data_list])/data_dict['count'] - data_dict['mean']**2
-    return data_dict
-
-def median(data_list,data_dict):
-    if 'median' in data_dict.keys():
-        return
-    count(data_list,data_dict)
-    data_list.sort()
-    data_dict['median'] = (data_list[(data_dict['count']-1)//2] + data_list[data_dict['count']//2]) / 2
-    return data_dict
-    
-def std(data_list,data_dict):
-    if 'std' in data_dict.keys():
-        return
-    var(data_list,data_dict)
-    try:
-        data_dict['std'] = math.sqrt(data_dict['var'])
-    except ValueError as err:
-        print(data_list)
-        print(data_dict['mean'])
-        print(data_dict['var'])
-        ValueError(err)
-    return data_dict
-    
-def skewness(data_list,data_dict):
-    if 'skew' in data_dict.keys():
-        return
-    std(data_list,data_dict)
-    try:
-        data_dict['skew'] = sum([(x-data_dict['mean'])**3 for x in data_list])/(data_dict['count']*data_dict['std']**3)
-    except ZeroDivisionError:
-        data_dict['skew'] = 'NaN'
-    return data_dict
-    
-def kurtosis(data_list,data_dict):
-    if 'kurt' in data_dict.keys():
-        return
-    std(data_list,data_dict)
-    try:
-        data_dict['kurt'] = sum([(x-data_dict['mean'])**4 for x in data_list])/(data_dict['count']*data_dict['std']**4) - 3
-    except ZeroDivisionError:
-        data_dict['kurt'] = 'NaN'
-    return data_dict
-
-def iqr(data_list,data_dict):
-    if 'iqr' in data_dict.keys():
-        return
-    count(data_list, data_dict)
-    data_list.sort()
-    cnt = (int(data_dict['count']))//2
-    l_half = data_list[:cnt]
-    u_half = data_list[-cnt:]
-    q1 = (l_half[(len(l_half)-1)//2] + l_half[len(l_half)//2]) / 2
-    q3 = (u_half[(len(u_half)-1)//2] + u_half[len(u_half)//2]) / 2
-    data_dict['iqr'] = q3-q1
-    return data_dict
-
-# Dictionary of input statistics
-STATS = {'mean': mean,
-         'median': median,
-         'std': std,
-         'var': var,
-         'skew': skewness,
-         'kurt': kurtosis,
-         'count': count,
-         'max': maxval,
-         'min': minval,
-         'iqr': iqr}
-
-if __name__=="__main__":
-    # Initialize the logger
-    logging.basicConfig(format='%(asctime)s - %(name)-8s - %(levelname)-8s - %(message)s',
-                        datefmt='%d-%b-%y %H:%M:%S')
-    logger = logging.getLogger("main")
-    logger.setLevel(logging.INFO)
-
-    # Setup the argument parsing
-    logger.info("Parsing arguments...")
-    parser = argparse.ArgumentParser(prog='main', description='Calculate simple statistics to groups of data in a csv file.')
-    parser.add_argument('--statistics', dest='statistics', type=str,
-                        help='Types of statistics to calculate', required=True)
-    parser.add_argument('--inpDir', dest='inpDir', type=str,
-                        help='Input csv collection to be processed by this plugin', required=True)
-    parser.add_argument('--filePattern', dest='filePattern', type=str,
-                        help='The filepattern of the files in the file column', required=False)
-    parser.add_argument('--groupBy', dest='groupBy', type=str,
-                        help='The varaibles in the filepattern to groupby', required=False)
-    parser.add_argument('--outDir', dest='outDir', type=str,
-                        help='Output collection', required=True)
-
-    # Parse the arguments
-    args = parser.parse_args()
-    if args.statistics == 'all':
-        statistics = [i for i,v in STATS.items() ]
-    else:
-        statistics = args.statistics.split(',')
-    logger.info('statistics = {}'.format(statistics))
-    inpDir = args.inpDir
-    logger.info('inpDir = {}'.format(inpDir))
-
-    filePattern = args.filePattern
-    logger.info('filePattern = {}'.format(filePattern))
-    groupBy = args.groupBy
-    logger.info('groupBy = {}'.format(groupBy))
-    outDir = args.outDir
-    logger.info('outDir = {}'.format(outDir))
-
-    # Get a list of all input files
-    csv_files = [f for f in Path(inpDir).iterdir() if f.name.endswith('csv')]
-    
-    if filePattern:
-        assert groupBy, 'groupBy variables must be specified when grouping with a filePattern'
-        # Get the regex and variables of the filePattern
-        regex, variables = get_regex(filePattern)
-        ind = ''.join([v for v in variables if v not in groupBy])
-
-    else:
-        logger.info('Grouping by filenames')
-        
-    if groupBy:
-        assert filePattern, 'filePattern must be specified when specifying groupBy variables'
-
-    file_column = 'intensity_image'
-
-    # Open each csv files
-    for feat_file in csv_files:
-        fpath = str(feat_file.absolute())
-        out = str(Path(outDir).joinpath(feat_file.name).absolute())
-        with open(fpath,'r') as fr:
-            with open(str(out),'w') as fw:
-                # Read the first line, which should contain headers
-                first_line = fr.readline()
-                headers = first_line.replace(file_column, 'file').rstrip('\n').split(',')
-                var_ind = {key:val for key,val in enumerate(headers)} # map headers to line positions
-                # If no column is labeled file, throw an error
-                if 'file' not in headers:
-                    raise ValueError('At least one column must have a header title file.')
-
-                # Generate the output dictionary template and format string
-                line_dict = {'file': 'NaN'}
-                for key in headers:
-                    if key == 'file':
-                        continue
-                    for stat in statistics:
-                        line_dict[key + '_' + stat] = 'NaN'
-
-                        # Generate the line template
-                line_template = ','.join([('{' + h + '}') for h in line_dict.keys()]) + '\n'
-
-                # Write headers to the new file
-                fw.write(','.join(line_dict.keys()) + '\n')
-
-                # Get the first line of data
-                line = fr.readline()
-
-                # Read each line in the stitching vector
-                fnum = 0
-                fcheck = 0
-                while line:
-                    # Parse the current line as a dictionary
-                    p_line = {var_ind[ind]:val for ind,val in enumerate(line.rstrip('\n').split(','))}
-                    for key,val in p_line.items():
-                        v = get_number(val)
-                        p_line[key] = [v]
-
-                    line = fr.readline()
-                    np_line = {var_ind[ind]:val for ind,val in enumerate(line.rstrip('\n').split(','))}
-                    match = False
-                    
-                    if filePattern:
-                        # The groupBy variable values for current file
-                        group_list = [
-                            {k:v for k,v in parse_filename(p_line['file'][0], regex, variables).items() if k in groupBy}
-                            ]
-                        
-                        p_line_vars = get_vars(p_line['file'], ind, regex, variables)
-                        np_line_vars = get_vars(np_line['file'], ind, regex, variables)
-                        
-                        if p_line_vars == np_line_vars:
-                            match = True
-                    
-                    else:
-                        if p_line['file'][0] == np_line['file']:
-                            match = True
-                    
-                    # Loop through rows until the filename changes
-                    while line and match:
-                        # Store the values in a feature list
-                        for key,val in np_line.items():
-                            if isinstance(val,str):
-                                # p_line[key].append(get_number(val[0]))
-                                p_line[key].append(get_number(val))
-                        
-                        if filePattern:
-                            # Add the file's groupBy variables to the output row label list
-                            group_list.append(
-                                {k:v for k,v in parse_filename(np_line['file'], regex, variables).items() 
-                                 if k in groupBy}
-                                )
-
-                        # Get the next line
-                        line = fr.readline()
-                        np_line = {var_ind[ind]:val for ind,val in enumerate(line.rstrip('\n').split(','))}
-                        
-                        match = False
-                        
-                        if filePattern and line:
-                            np_line_vars = get_vars(np_line['file'], ind, regex, variables)
-                            if p_line_vars == np_line_vars:
-                                match = True
-                                
-                    
-                        elif line:
-                            if p_line['file'][0] == np_line['file']:
-                                match = True
-
-                    # Get the mean of the feature list, save in the file dictionary
-                    for key,val in p_line.items():
-                        
-                        # Set the file name
-                        if key=='file':
-                            if filePattern:
-                                ind_list = {
-                                    k:v for k,v in parse_filename(val[0], regex, variables).items() if k not in groupBy
-                                    }
-                                line_dict['file'] = output_name(filePattern, group_list, ind_list)
-                            
-                            else:
-                                line_dict['file'] = val[0]
-                                
-                            continue
-
-                        # Grab only float values
-                        inp_data = [d for d in val if isinstance(d,float)]
-
-                        # If inp_data contains no floats, skip it
-                        if len(inp_data) == 0:
-                            continue
-
-                        # Calculate the statistics for the feature
-                        data_dict = {}
-                        for stat in statistics:
-                            STATS[stat](inp_data,data_dict)
-                            line_dict[key + '_' + stat] = data_dict[stat]
-                    fw.write(line_template.format(**line_dict))
-                    line_dict = {key:'NaN' for key in line_dict.keys()}
-
-                    # Checkpoint
-                    fnum += 1
-                    if fnum > fcheck:
-                        fcheck += 1
-                        logger.info('Unique Files parsed: {}'.format(fnum))
\ No newline at end of file
diff --git a/features/polus-csv-statistics-plugin/src/requirements.txt b/features/polus-csv-statistics-plugin/src/requirements.txt
deleted file mode 100644
index 6dd96c6..0000000
--- a/features/polus-csv-statistics-plugin/src/requirements.txt
+++ /dev/null
@@ -1 +0,0 @@
-filepattern==1.4.7
\ No newline at end of file
diff --git a/features/tabular-statistics-tool/.bumpversion.cfg b/features/tabular-statistics-tool/.bumpversion.cfg
new file mode 100644
index 0000000..026c094
--- /dev/null
+++ b/features/tabular-statistics-tool/.bumpversion.cfg
@@ -0,0 +1,31 @@
+[bumpversion]
+current_version = 0.1.0-dev0
+commit = True
+tag = False
+parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-(?P<release>[a-z]+)(?P<dev>\d+))?
+serialize = 
+	{major}.{minor}.{patch}-{release}{dev}
+	{major}.{minor}.{patch}
+
+[bumpversion:part:release]
+optional_value = _
+first_value = dev
+values = 
+	dev
+	_
+
+[bumpversion:part:dev]
+
+[bumpversion:file:pyproject.toml]
+search = version = "{current_version}"
+replace = version = "{new_version}"
+
+[bumpversion:file:plugin.json]
+
+[bumpversion:file:VERSION]
+
+[bumpversion:file:README.md]
+[bumpversion:file:ict.yaml]
+[bumpversion:file:TabularStatistics.cwl]
+
+[bumpversion:file:src/polus/tabular/features/tabular_statistics/__init__.py]
\ No newline at end of file
diff --git a/features/tabular-statistics-tool/Dockerfile b/features/tabular-statistics-tool/Dockerfile
new file mode 100644
index 0000000..5d23a5b
--- /dev/null
+++ b/features/tabular-statistics-tool/Dockerfile
@@ -0,0 +1,24 @@
+FROM polusai/bfio:2.4.5
+
+# environment variables defined in polusai/bfio
+ENV EXEC_DIR="/opt/executables"
+ENV POLUS_TAB_EXT=".csv"
+ENV POLUS_LOG="INFO"
+
+# Work directory defined in the base container
+WORKDIR ${EXEC_DIR}
+
+# TODO: Change the tool_dir to the tool directory
+ENV TOOL_DIR="features/tabular-statistics-tool"
+
+# Copy the repository into the container
+RUN mkdir tabular-tools
+COPY . ${EXEC_DIR}/tabular-tools
+
+# Install the tool
+RUN pip3 install "${EXEC_DIR}/tabular-tools/${TOOL_DIR}" --no-cache-dir
+
+# Set the entrypoint
+# TODO: Change the entrypoint to the tool entrypoint
+ENTRYPOINT ["python3", "-m", "polus.tabular.features.tabular_statistics"]
+CMD ["--help"]
\ No newline at end of file
diff --git a/features/polus-csv-statistics-plugin/README.md b/features/tabular-statistics-tool/README.md
similarity index 67%
rename from features/polus-csv-statistics-plugin/README.md
rename to features/tabular-statistics-tool/README.md
index 51ac1c4..60ac1f7 100644
--- a/features/polus-csv-statistics-plugin/README.md
+++ b/features/tabular-statistics-tool/README.md
@@ -1,8 +1,8 @@
-# CSV Statistics
+# Tabular Statistics
 
-This WIPP plugin performs statistics on values in each column of a csv file if the data is numeric. Rows of data are grouped together by rows that have a matching value in a column with header named `file`. If no columns have the `file` header, then this plugin throws and error.
+This plugin computes statistical measures on numeric and floating point data columns in tabular files. The supported input file formats are `CSV`, `Feather`, `Arrow`, and `Parquet`, leveraging the  [PyArrow](https://arrow.apache.org/) library for efficient processing. If no columns have the `file` header, then this plugin throws and error.
 
-Available statistics are:
+## Available Statistics:
 
 1. [mean (arithmetic mean)](https://en.wikipedia.org/wiki/Mean#Arithmetic_mean_(AM))
 2. [median](https://en.wikipedia.org/wiki/Median#The_sample_median)
@@ -12,6 +12,13 @@ Available statistics are:
 6. [kurt (excess kurtosis)](https://www.itl.nist.gov/div898/handbook/eda/section3/eda35b.htm)
 7. count (number of rows sampled)
 8. [iqr (Interquartile_range)](https://en.wikipedia.org/wiki/Interquartile_range)
+9. Counts the numbers greater than 0 and divides this count by the total number of elements in the sequence
+
+
+## Usage:
+- A directory containing one or more tabular files in the supported formats
+- Each file must include numeric and floating point data columns
+- If a `groupBy` column is specified in the input arguments, it must be present in the data
 
 For more information on WIPP, visit the [official WIPP page](https://isg.nist.gov/deepzoomweb/software/wipp).
 
@@ -31,7 +38,7 @@ This plugin takes two input argument and one output argument:
 | Name            | Description                                         | I/O    | Type          |
 | --------------- | --------------------------------------------------- | ------ | ------------- |
 | `--statistics`  | Types of statistics to calculate                    | Input  | array         |
-| `--inpDir`      | Input csv collection to be processed by this plugin | Input  | csvCollection |
+| `--inpDir`      | Input csv collection to be processed by this plugin | Input  | genericData   |
 | `--filePattern` | The filePattern of the images in represented in csv | Input  | string        |
 | `--groupBy`     | The variable(s) of how the images should be grouped | Input  | string        |
-| `--outDir`      | Output collection                                   | Output | csvCollection |
+| `--outDir`      | Output collection                                   | Output | genericData   | 
diff --git a/features/tabular-statistics-tool/TabularStatistics.cwl b/features/tabular-statistics-tool/TabularStatistics.cwl
new file mode 100644
index 0000000..aa061c2
--- /dev/null
+++ b/features/tabular-statistics-tool/TabularStatistics.cwl
@@ -0,0 +1,36 @@
+class: CommandLineTool
+cwlVersion: v1.2
+inputs:
+  filePattern:
+    inputBinding:
+      prefix: --filePattern
+    type: string?
+  groupBy:
+    inputBinding:
+      prefix: --groupBy
+    type: string?
+  inpDir:
+    inputBinding:
+      prefix: --inpDir
+    type: Directory
+  outDir:
+    inputBinding:
+      prefix: --outDir
+    type: Directory
+  statistics:
+    inputBinding:
+      prefix: --statistics
+    type: string
+outputs:
+  outDir:
+    outputBinding:
+      glob: $(inputs.outDir.basename)
+    type: Directory
+requirements:
+  DockerRequirement:
+    dockerPull: polusai/tabular-statistics-tool:0.1.0-dev0
+  InitialWorkDirRequirement:
+    listing:
+    - entry: $(inputs.outDir)
+      writable: true
+  InlineJavascriptRequirement: {}
diff --git a/features/tabular-statistics-tool/VERSION b/features/tabular-statistics-tool/VERSION
new file mode 100644
index 0000000..15a06be
--- /dev/null
+++ b/features/tabular-statistics-tool/VERSION
@@ -0,0 +1 @@
+0.1.0-dev0
\ No newline at end of file
diff --git a/features/tabular-statistics-tool/build-docker.sh b/features/tabular-statistics-tool/build-docker.sh
new file mode 100755
index 0000000..d734881
--- /dev/null
+++ b/features/tabular-statistics-tool/build-docker.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+# TODO: Change the name of the tool here
+tool_dir="features"
+tool_name="tabular-statistics-tool"
+
+# The version is read from the VERSION file
+version=$(<VERSION)
+tag="polusai/${tool_name}:${version}"
+echo "Building docker image with tag: ${tag}"
+
+# The current directory and the repository root are saved in variables
+cur_dir=$(pwd)
+repo_root=$(git rev-parse --show-toplevel)
+
+# The Dockerfile and .dockerignore files are copied to the repository root before building the image
+cd ${repo_root}
+cp ./${tool_dir}/${tool_name}/Dockerfile .
+cp .gitignore .dockerignore
+docker build . -t ${tag}
+rm Dockerfile .dockerignore
+cd ${cur_dir}
+version=$(<VERSION)
\ No newline at end of file
diff --git a/features/tabular-statistics-tool/gitignore b/features/tabular-statistics-tool/gitignore
new file mode 100644
index 0000000..57b14d3
--- /dev/null
+++ b/features/tabular-statistics-tool/gitignore
@@ -0,0 +1,23 @@
+# Jupyter Notebook
+.ipynb_checkpoints
+poetry.lock
+../../poetry.lock
+# Environments
+.env
+.myenv
+.venv
+env/
+venv/
+# test data directory
+data
+# yaml file
+.pre-commit-config.yaml
+# hidden files
+.DS_Store
+.ds_store
+# flake8
+.flake8
+../../.flake8
+__pycache__
+.mypy_cache
+requirements.txt
\ No newline at end of file
diff --git a/features/tabular-statistics-tool/ict.yaml b/features/tabular-statistics-tool/ict.yaml
new file mode 100644
index 0000000..5bd0256
--- /dev/null
+++ b/features/tabular-statistics-tool/ict.yaml
@@ -0,0 +1,62 @@
+author:
+- Hamdah Shafqat
+- Nick Schaub
+contact: hamdahshafqat.abbasi@nih.gov
+container: polusai/tabular-statistics-tool:0.1.0-dev0
+description: Calculating Statistics for Numeric and Floating-Point Columns in Tabular
+  Files
+entrypoint: python3 -m polus.tabular.features.tabular_statistics
+inputs:
+- description: The csv statistics to generate
+  format:
+  - array
+  name: statistics
+  required: true
+  type: array
+- description: Input tabular collection to be processed by this plugin
+  format:
+  - genericData
+  name: inpDir
+  required: true
+  type: path
+- description: Pattern to select files for grouping
+  format:
+  - string
+  name: filePattern
+  required: false
+  type: string
+- description: The variable(s) to group rows
+  format:
+  - string
+  name: groupBy
+  required: false
+  type: string
+name: polusai/TabularStatistics
+outputs:
+- description: Output collection
+  format:
+  - genericData
+  name: outDir
+  required: true
+  type: path
+repository: https://github.com/PolusAI/tabular-tools
+specVersion: 1.0.0
+title: Tabular Statistics
+ui:
+- description: Types of statistics to calculate
+  key: inputs.statistics
+  title: Statistics
+  type: text
+- description: Input tabular collection to be processed by this plugin
+  key: inputs.inpDir
+  title: Input collection
+  type: path
+- description: Pattern to select files for grouping
+  key: inputs.filePattern
+  title: Filepattern
+  type: text
+- description: The variable(s) to group rows
+  key: inputs.groupBy
+  title: Groupby Variables(s)
+  type: text
+version: 0.1.0-dev0
diff --git a/features/polus-csv-statistics-plugin/plugin.json b/features/tabular-statistics-tool/plugin.json
similarity index 71%
rename from features/polus-csv-statistics-plugin/plugin.json
rename to features/tabular-statistics-tool/plugin.json
index a0d8e41..880e36b 100644
--- a/features/polus-csv-statistics-plugin/plugin.json
+++ b/features/tabular-statistics-tool/plugin.json
@@ -1,14 +1,19 @@
 {
-  "name": "CSV Statistics",
-  "version": "0.2.1",
-  "title": "CSV Statistics",
-  "description": "Calculate simple statistics to groups of data in a csv file.",
-  "author": "Nick Schaub (nick.schaub@nih.gov)",
+  "name": "Tabular Statistics",
+  "version": "0.1.0-dev0",
+  "title": "Tabular Statistics",
+  "description": "Calculating Statistics for Numeric and Floating-Point Columns in Tabular Files",
+  "author": "Hamdah Shafqat abbasi (hamdahshafqat.abbasi@nih.gov), Nick Schaub (nick.schaub@nih.gov)",
   "institution": "National Center for Advancing Translational Sciences, National Institutes of Health",
-  "repository": "https://github.com/labshare/polus-plugins",
+  "repository": "https://github.com/PolusAI/tabular-tools",
   "website": "https://ncats.nih.gov/preclinical/core/informatics",
   "citation": "",
-  "containerId": "polusai/csv-statistics-plugin:0.2.1",
+  "containerId": "polusai/tabular-statistics-tool:0.1.0-dev0",
+  "baseCommand": [
+    "python3",
+    "-m",
+    "polus.tabular.features.tabular_statistics"
+  ],
   "inputs": [
     {
       "name": "statistics",
@@ -18,7 +23,7 @@
         "items": {
           "type": "string",
           "title": "Select statistic",
-          "description": "Csv Statistics",
+          "description": "Tabular Statistics",
           "required": true,
           "oneOf": [
             {
@@ -81,6 +86,12 @@
                 "iqr"
               ]
             },
+            {
+              "description": "Proportion",
+              "enum": [
+                "prop"
+              ]
+            },
             {
               "description": "All parameters",
               "enum": [
@@ -97,27 +108,27 @@
     },
     {
       "name": "inpDir",
-      "type": "csvCollection",
-      "description": "Input csv collection to be processed by this plugin",
+      "type": "genericData",
+      "description": "Input tabular collection to be processed by this plugin",
       "required": true
     },
     {
       "name": "filePattern",
       "type": "string",
-      "description": "The filepattern of the images represented in the csv files",
+      "description": "Pattern to select files for grouping",
       "required": false
     },
     {
       "name": "groupBy",
       "type": "string",
-      "description": "The variable(s) to group the images by",
+      "description": "The variable(s) to group rows",
       "required": false
     }
   ],
   "outputs": [
     {
       "name": "outDir",
-      "type": "csvCollection",
+      "type": "genericData",
       "description": "Output collection"
     }
   ],
@@ -130,17 +141,17 @@
     {
       "key": "inputs.inpDir",
       "title": "Input collection",
-      "description": "Input csv collection to be processed by this plugin"
+      "description": "Input tabular collection to be processed by this plugin"
     },
     {
       "key": "inputs.filePattern",
       "title": "Filepattern",
-      "description": "The filepattern of the images represented in the csv files"
+      "description": "Pattern to select files for grouping"
     },
     {
       "key": "inputs.groupBy",
       "title": "Groupby Variables(s)",
-      "description": "The variable(s) to group the images by"
+      "description": "The variable(s) to group rows"
     }
   ]
 }
\ No newline at end of file
diff --git a/features/tabular-statistics-tool/pyproject.toml b/features/tabular-statistics-tool/pyproject.toml
new file mode 100644
index 0000000..ee9244b
--- /dev/null
+++ b/features/tabular-statistics-tool/pyproject.toml
@@ -0,0 +1,31 @@
+[tool.poetry]
+name = "polus-tabular-features-tabular-statistics"
+version = "0.1.0-dev0"
+description = "Calculating Statistics for Numeric and Floating-Point Columns in Tabular Files"
+authors = [
+"Hamdah Shafqat abbasi <hamdahshafqat.abbasi@nih.gov>",
+"Nick Schaub <nick.schaub@nih.gov>",
+]
+readme = "README.md"
+packages = [{include = "polus", from = "src"}]
+
+[tool.poetry.dependencies]
+python = ">=3.9"
+typer = "^0.7.0"
+tqdm = "^4.65.0"
+filepattern = "^2.0.5"
+pyarrow = "^18.0.0"
+pandas = "^2.2.2"                     
+
+
+[tool.poetry.group.dev.dependencies]
+bump2version = "^1.0.1"
+pre-commit = "^3.1.0"
+black = "^23.1.0"
+flake8 = "^6.0.0"
+mypy = "^1.0.1"
+pytest = "^7.2.1"
+
+[build-system]
+requires = ["poetry-core"]
+build-backend = "poetry.core.masonry.api"
\ No newline at end of file
diff --git a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/__init__.py b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/__init__.py
new file mode 100644
index 0000000..037f583
--- /dev/null
+++ b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/__init__.py
@@ -0,0 +1,4 @@
+"""Tabular Statistics."""
+__version__ = "0.1.0-dev0"
+
+from . import tabular_statistics
\ No newline at end of file
diff --git a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/__main__.py b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/__main__.py
new file mode 100644
index 0000000..6af6ebc
--- /dev/null
+++ b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/__main__.py
@@ -0,0 +1,117 @@
+"""Tabular Statistics."""
+import json
+import logging
+import os
+import pathlib
+import time
+import pyarrow as pa
+import pyarrow.ipc as ipc
+import pyarrow.csv as csv
+
+from typing import Any
+from typing import Optional
+import filepattern as fp
+import typer
+# import tabular_statistics as ts
+
+from polus.tabular.features.tabular_statistics import tabular_statistics as ts
+
+app = typer.Typer()
+
+# Initialize the logger
+logging.basicConfig(
+    format="%(asctime)s - %(name)-8s - %(levelname)-8s - %(message)s",
+    datefmt="%d-%b-%y %H:%M:%S",
+)
+logger = logging.getLogger("polus.tabular.features.tabular_statistics")
+logger.setLevel(os.environ.get("POLUS_LOG", logging.INFO))
+
+
+@app.command()
+def main(  # noqa: PLR0913
+    inp_dir: pathlib.Path = typer.Option(
+        ...,
+        "--inpDir",
+        help="Input generic data collection to be processed by this plugin",
+    ),
+    file_pattern: str = typer.Option(".+", "--filePattern", help="file_pattern"),
+   
+    statistics: str = typer.Option(
+        ...,
+        "--statistics",
+        help="Only merge files with the same number of rows?",
+    ),
+    group_by: str = typer.Option(
+        None,
+        "--groupBy",
+        help="Merge files with common header",
+    ),
+    out_dir: pathlib.Path = typer.Option(..., "--outDir", help="Output collection"),
+    preview: Optional[bool] = typer.Option(
+        False,
+        "--preview",
+        help="Output a JSON preview of files",
+    ),
+) -> None:
+    """CLI for the tool."""
+    logger.info(f"inpDir = {inp_dir}")
+    logger.info(f"outDir = {out_dir}")
+    logger.info(f"filePattern = {file_pattern}")
+    logger.info(f"statistics = {statistics}")
+    logger.info(f"groupBy= {group_by}")
+
+    start_time = time.time()
+
+    inp_dir = pathlib.Path(inp_dir).resolve()
+    out_dir = pathlib.Path(out_dir).resolve()
+
+    assert inp_dir.exists(), f"{inp_dir} doesnot exists!! Please check input path again"
+    assert (
+        out_dir.exists()
+    ), f"{out_dir} doesnot exists!! Please check output path again"
+
+    # By default it ingests all input files if not file_pattern is defined
+
+    EXTS = [".arrow", ".feather", ".csv", ".parquet"]
+
+    file_pattern = ".*" + file_pattern
+
+    fps = fp.FilePattern(inp_dir, file_pattern)
+
+    if preview:
+        ts.preview(out_dir, file_pattern)
+
+    else:
+        flist = [f for f in pathlib.Path(inp_dir).iterdir() if f.suffix in  EXTS]
+
+        table = ts.load_files(flist)
+
+        columns = table.column_names
+        file_column = "intensity_image"
+
+        # Validate required column
+        if file_column not in columns:
+            raise ValueError(f"Column '{file_column}' not found")
+        
+        logger.info(f"Merging files into a single table")
+
+        if group_by:
+            logger.info(f"Applying statistics by grouping data on: {group_by}")
+            group_by = [col.strip() for col in group_by.split(",") if col.strip()] 
+            # Apply statistics to the grouped data
+            aggregated_df = ts.apply_statistics(table, statistics, group_by)
+        else:
+            logger.info(f"Applying statistics on data")
+            aggregated_df = ts.apply_statistics_to_group(table, statistics)
+
+
+        ts.save_outputs(aggregated_df, out_dir)
+        
+
+    exec_time = time.time() - start_time
+    logger.info(f"Execution time: {time.strftime('%H:%M:%S', time.gmtime(exec_time))}")
+    logger.info("Finished processing of files!")
+
+
+if __name__ == "__main__":
+    app()
\ No newline at end of file
diff --git a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py
new file mode 100644
index 0000000..86a52a3
--- /dev/null
+++ b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py
@@ -0,0 +1,399 @@
+"""Tabular Statistics."""
+import logging
+import os
+from scipy.stats import kurtosis
+from typing import Optional
+import numpy as np
+import pandas as pd
+import math
+import pyarrow as pa
+import pyarrow.csv as csv
+import pyarrow.ipc as ipc
+import pyarrow.feather as feather
+import pyarrow.parquet as pq
+from pathlib import Path
+from typing import List
+from typing import Any
+from tqdm import tqdm
+import json
+import filepattern as fp
+
+logger = logging.getLogger(__name__)
+logger.setLevel(os.environ.get("POLUS_LOG", logging.INFO))
+POLUS_TAB_EXT = os.environ.get("POLUS_TAB_EXT", ".csv")
+
+
+
+def mean(values: list[float]) -> float:
+    """
+    Calculate the mean of a list of numbers.
+
+    Args:
+        values: A list of numerical values.
+
+    Returns:
+        float: The mean or average of the input values.
+
+    Raises:
+        ValueError: If the input list is empty.
+    """
+    if values.empty:
+        raise ValueError("The input list is empty. Cannot calculate the mean.")
+    return np.mean(values)
+
+
+def median(values: list[float]) -> float:
+    """
+    Calculate the median of a list of numbers.
+
+    Args:
+        values: A list of numerical values.
+
+    Returns:
+        float: The median value of the input values.
+
+    Raises:
+        ValueError: If the input list is empty.
+    """
+    if values.empty:
+        raise ValueError("The input list is empty. Cannot calculate the median.")
+    return np.median(values)
+
+def std(values: list[float]) -> float:
+    """
+    Calculate the standard deviation of a list of numbers.
+
+    Args:
+        values: A list of numerical values.
+
+    Returns:
+        float: The standard deviation of the input values.
+
+    Raises:
+        ValueError: If the input list is empty.
+    """
+    if values.empty:
+        raise ValueError("The input list is empty. Cannot calculate the standard deviation.")
+    return np.std(values)
+
+def var(values: list[float]) -> float:
+    """
+    Calculate the variance of a list of numbers.
+
+    Args:
+        values: A list of numerical values.
+
+    Returns:
+        float: The variance of the input values.
+
+    Raises:
+        ValueError: If the input list is empty.
+    """
+    if values.empty:
+        raise ValueError("The input list is empty. Cannot calculate the variance.")
+    return np.var(values)
+
+
+def skew(values: list[float]) -> float:
+    """
+    Calculate the skewness of a list of numbers.
+
+    Args:
+        values: A list of numerical values.
+
+    Returns:
+        float: The skewness of the input values.
+
+    Raises:
+        ValueError: If the input list is empty.
+    """
+    if values.empty:
+        raise ValueError("The input list is empty. Cannot calculate the skewness.")
+    
+    n = len(values)
+    mean = sum(values) / n
+    std_dev = math.sqrt(sum((x - mean) ** 2 for x in values) / n)
+    
+    if std_dev == 0:
+        return float('nan')  # If standard deviation is 0, skewness is undefined
+    
+    skew_value = sum((x - mean) ** 3 for x in values) / (n * std_dev ** 3)
+    return skew_value
+
+def kurt(values: list[float]) -> float:
+    """
+    Calculate the kurtosis of a list of numbers.
+
+    Args:
+        values: A list of numerical values.
+
+    Returns:
+        float: The kurtosis of the input values.
+
+    Raises:
+        ValueError: If the input list is empty.
+    """
+    if values.empty:
+        raise ValueError("The input list is empty. Cannot calculate the kurtosis.")
+    return kurtosis(values)
+
+def count(values: list[float]) -> int:
+    """
+    Count the number of elements in a list.
+
+    Args:
+        values: A list of numerical values.
+
+    Returns:
+        int: The number of elements in the input list.
+
+    Raises:
+        ValueError: If the input list is empty.
+    """
+    if values.empty:
+        raise ValueError("The input list is empty. Cannot calculate the count.")
+    return len(values)
+
+
+
+def max(values: list[float]) -> float:
+    """
+    Calculate the maximum value from a list of numbers.
+
+    Args:
+        values: A list of numerical values.
+
+    Returns:
+        float: The maximum value in the input list.
+
+    Raises:
+        ValueError: If the input list is empty.
+    """
+    if values.empty:
+        raise ValueError("The input list is empty. Cannot calculate the maximum.")
+    return np.max(values)
+
+def min(values: list[float]) -> float:
+    """
+    Calculate the minimum value from a list of numbers.
+
+    Args:
+        values: A list of numerical values.
+
+    Returns:
+        float: The minimum value in the input list.
+
+    Raises:
+        ValueError: If the input list is empty.
+    """
+    if values.empty:
+        raise ValueError("The input list is empty. Cannot calculate the minimum.")
+    return np.min(values)
+
+def iqr(values: list[float]) -> float:
+    """
+    Calculate the interquartile range (IQR) of a list of numbers.
+
+    Args:
+        values: A list of numerical values.
+
+    Returns:
+        float: The IQR of the input values.
+
+    Raises:
+        ValueError: If the input list is empty.
+    """
+    if values.empty:
+        raise ValueError("The input list is empty. Cannot calculate the IQR.")
+    return np.percentile(values, 75) - np.percentile(values, 25)
+
+def proportion(values: list[float]) -> float:
+    """
+    Calculate the proportion of positive values in a list.
+
+    Args:
+        values: A list of numerical values.
+
+    Returns:
+        float: The proportion of positive values in the input list.
+
+    Raises:
+        ValueError: If the input list is empty.
+    """
+    if values.empty:
+        raise ValueError("The input list is empty. Cannot calculate the proportion.")
+    return sum(1 for x in values if x > 0) / len(values)
+
+
+# STATS dictionary with statistical functions
+STATS = {
+    "mean": mean,
+    "median": median,
+    "std": std,
+    "var": var,
+    "skew": skew,
+    "kurt": kurt,
+    "count": count,
+    "max": max,
+    "min": min,
+    "iqr": iqr,
+    "prop": proportion
+}
+
+
+ # Function to apply statistics to each group
+def apply_statistics(table: pa.Table,  statistics: str, group_by_columns:Optional[str]=None): 
+        """
+        Apply statistical functions to each group in a PyArrow Table.
+
+        Args:
+            table: Input PyArrow Table.
+            group_by_columns: Columns to group by.
+            statistics: Comma-separated list of statistics or 'all' to apply all available statistics.
+        
+        Returns:
+            pyarrow.Table: Aggregated statistics for each numeric column.
+        """
+        # Step 1: Convert PyArrow Table to Pandas DataFrame for easy manipulation
+        if statistics == "all":
+            statistics = STATS
+        else:
+            statistics = statistics.split(',')
+            statistics = {k: v for k, v in STATS.items() if k in statistics}
+
+
+        df = table.to_pandas()
+
+        numeric_columns = df.select_dtypes(include='number').columns.tolist()
+
+        # # Step 5: Apply the statistics to each group
+        aggregated_results = {}
+
+        for stat_name, stat_func in statistics.items():
+            for col in  numeric_columns:
+                col_stat_name = f"{col}_{stat_name}"
+                if group_by_columns:
+                    # Aggregate without grouping
+                    aggregated_results[col_stat_name] = df.groupby(group_by_columns)[col].apply(stat_func)
+                else:
+                    # Aggregate without grouping
+                    aggregated_results[col_stat_name] = [stat_func(df[col])]
+
+        if group_by_columns:
+            group_table = pd.DataFrame(aggregated_results)
+            # Perform an left join on the "group_by_columns" column
+            arrow_table = pd.merge(df, group_table, on=group_by_columns, how='left')
+
+            arrow_table.reset_index(inplace=True)
+            arrow_table = pa.Table.from_pandas(arrow_table)
+
+
+        else:
+            arrow_table = pa.Table.from_pydict(aggregated_results)
+
+        return arrow_table
+
+
+def save_outputs(data:pa.Table, out_dir:Path) -> None:
+    """
+    Save data to a file in CSV, Feather, Arrow, or Parquet format.
+
+    Args:
+        data: The data to save.
+        out_dir: Directory to save the file.
+        file: Base filename for the saved file.
+    """
+
+    # Determine the output file path and extension based on the desired format
+    output_file = out_dir.joinpath(f"tabular_statistics_output{POLUS_TAB_EXT}")
+
+    if POLUS_TAB_EXT == ".csv":
+        logger.info(f"Saved outputs:{output_file}")
+        csv.write_csv(data, output_file)
+
+    elif POLUS_TAB_EXT == ".feather":
+        logger.info(f"Saved outputs: {output_file}")
+        feather.write_feather(data, output_file)
+
+    elif POLUS_TAB_EXT == ".arrow":
+        logger.info(f"Saved outputs:{output_file}")
+        with pa.OSFile(str(output_file), 'wb') as arrow_file:
+            with ipc.new_file(data,data.schema) as writer:
+                writer.write(data)
+
+    elif POLUS_TAB_EXT == ".parquet":
+        logger.info(f"Saved outputs:{output_file}")
+        pq.write_table(data, output_file)
+
+    else:
+        raise ValueError(f"Unsupported extension: {POLUS_TAB_EXT}. Supported: .csv, .feather, .arrow, .parquet.")
+
+    
+    logger.info(f"File saved successfully:{output_file}")
+
+
+
+def load_files(flist: List[Path]) -> pa.Table:
+    """
+    Load multiple files (CSV, Feather, Arrow, Parquet) and return a concatenated PyArrow Table.
+
+    Args:
+        flist: List of file paths to load.
+
+    Returns:
+        pa.Table: Concatenated table of all loaded data.
+    """
+    tables = []
+    
+    for file in tqdm(flist, desc="Processing files"):
+        try:
+            if file.suffix == ".csv":
+                logger.info(f"Loading CSV file: {file.name}")
+                # Read CSV with pandas and convert to PyArrow Table
+                df = pd.read_csv(file)
+                table = pa.Table.from_pandas(df)
+            elif file.suffix == ".feather":
+                logger.info(f"Loading Feather file: {file.name}")
+                # Read Feather file directly using PyArrow
+                table = feather.read_table(str(file))
+            elif file.suffix == ".arrow":
+                logger.info(f"Loading Arrow file: {file.name}")
+                # Read Arrow file using PyArrow IPC
+                with pa.memory_map(str(file), "r") as source:
+                    table = ipc.open_file(source).read_all()
+            elif file.suffix == ".parquet":
+                logger.info(f"Loading Parquet file: {file.name}")
+                # Read Parquet file using PyArrow
+                table = pq.read_table(str(file))
+            else:
+                logger.error(f"Unsupported file format: {file.suffix}")
+                continue  # Skip unsupported files
+
+            tables.append(table)
+        except Exception as e:
+            logger.error(f"Error processing file {file.name}: {e}")
+
+    # Concatenate all loaded tables into a single table
+    combined_table = pa.concat_tables(tables)
+
+    return combined_table
+
+
+def preview(out_dir:Path, file_pattern:str) -> None:
+    """
+    Create a preview JSON file with output directory and file pattern details.
+
+    Args:
+        out_dir: Directory for the outputs.
+        file_pattern: The file pattern used for processing.
+    """
+
+    out_name= out_dir.joinpath(f"tabular_statistics_output{POLUS_TAB_EXT}")
+
+    out_json: dict[str, Any] = {
+        "filepattern": file_pattern,
+        "outDir": str(out_name),
+    }
+    with Path(out_dir, "preview.json").open("w") as jfile:
+        json.dump(out_json, jfile, indent=2)
+

From c320408e7813494cdc1f97f9894fbfe1d85ad04f Mon Sep 17 00:00:00 2001
From: Hamdah Shafqat Abbasi <hamdahshafqat@hotmail.com>
Date: Wed, 27 Nov 2024 18:41:59 -0500
Subject: [PATCH 02/12] added tests for csv statistics

---
 .../tabular-statistics-tool/pyproject.toml    |   1 +
 .../tabular-statistics-tool/tests/__init__.py |   1 +
 .../tests/test_main.py                        | 132 ++++++++++++++++++
 3 files changed, 134 insertions(+)
 create mode 100644 features/tabular-statistics-tool/tests/__init__.py
 create mode 100644 features/tabular-statistics-tool/tests/test_main.py

diff --git a/features/tabular-statistics-tool/pyproject.toml b/features/tabular-statistics-tool/pyproject.toml
index ee9244b..c6b0032 100644
--- a/features/tabular-statistics-tool/pyproject.toml
+++ b/features/tabular-statistics-tool/pyproject.toml
@@ -15,6 +15,7 @@ typer = "^0.7.0"
 tqdm = "^4.65.0"
 filepattern = "^2.0.5"
 pyarrow = "^18.0.0"
+scipy = "1.11.4"
 pandas = "^2.2.2"                     
 
 
diff --git a/features/tabular-statistics-tool/tests/__init__.py b/features/tabular-statistics-tool/tests/__init__.py
new file mode 100644
index 0000000..49a8fb4
--- /dev/null
+++ b/features/tabular-statistics-tool/tests/__init__.py
@@ -0,0 +1 @@
+"""Test for Tabular Statistics."""
\ No newline at end of file
diff --git a/features/tabular-statistics-tool/tests/test_main.py b/features/tabular-statistics-tool/tests/test_main.py
new file mode 100644
index 0000000..85f0a9a
--- /dev/null
+++ b/features/tabular-statistics-tool/tests/test_main.py
@@ -0,0 +1,132 @@
+import pathlib
+import string
+import numpy as np
+import pandas as pd
+import pyarrow as pa
+from polus.tabular.features.tabular_statistics import tabular_statistics as ts
+
+
+class Generatedata:
+    """Generate tabular data with several different file formats."""
+
+    def __init__(
+        self,
+        file_pattern: str,
+        out_name: str
+    ) -> None:
+        """Define instance attributes."""
+        self.data_dir = pathlib.Path.cwd().parent.joinpath("data")
+
+        self.inp_dir = pathlib.Path(self.data_dir, "input")
+        if not self.inp_dir.exists():
+            self.inp_dir.mkdir(exist_ok=True, parents=True)
+
+        self.out_dir = pathlib.Path(self.data_dir, "output")
+        if not self.out_dir.exists():
+            self.out_dir.mkdir(exist_ok=True, parents=True)
+
+        self.file_pattern = file_pattern
+        self.out_name = out_name
+        self.df = self.create_dataframe()
+
+    def get_inp_dir(self) -> pathlib.Path:
+        """Get input directory."""
+        return self.inp_dir
+
+    def get_out_dir(self) -> pathlib.Path:
+        """Get output directory."""
+        return self.out_dir
+
+    def create_dataframe(self) -> pd.DataFrame:
+        """Create Pandas dataframe."""
+        df_size = 100
+        rng = np.random.default_rng()
+        letters = list(string.ascii_lowercase)
+
+        diction_1 = {
+            "A": [rng.choice(letters) for i in range(df_size)],
+            "B": rng.integers(low=1, high=100, size=df_size),
+            "C": rng.normal(0.0, 1.0, size=df_size),
+        }
+
+        return pd.DataFrame(diction_1)
+
+    def arrow_func(self) -> None:
+        """Convert pandas dataframe to Arrow file format."""
+        self.df.to_feather(pathlib.Path(self.inp_dir, self.out_name))
+
+    def csv_func(self) -> None:
+        """Convert pandas dataframe to csv file format."""
+        self.df.to_csv(pathlib.Path(self.inp_dir, self.out_name), index=False)
+
+    def parquet_func(self) -> None:
+        """Convert pandas dataframe to parquet file format."""
+        self.df.to_parquet(
+            pathlib.Path(self.inp_dir, self.out_name),
+            engine="auto",
+            compression=None,
+        )
+
+    def __call__(self) -> None:
+        """To make a class callable."""
+        data_ext = {
+            ".csv": self.csv_func,
+            ".parquet": self.parquet_func,
+            ".arrow": self.arrow_func,
+        }
+        return data_ext[self.file_pattern]()
+
+
+
+def test_apply_statistics() -> None:
+    """Test applying statistics on PyArrow table."""
+
+    for i in [".parquet", ".csv",  ".arrow"]:
+        d1 = Generatedata(file_pattern=i, out_name=f"data_1{i}")
+        d1()
+        table = pa.table(d1.df)
+        statistics_list = list(ts.STATS.keys())
+
+        # Test applying each statistic in STATS to the table
+        for statistic in statistics_list:
+
+            result_table = ts.apply_statistics(table, statistics=statistic)
+            
+            assert isinstance(result_table, pa.Table)
+            
+            for col_name in table.column_names:
+                col = table[col_name] 
+                if not pa.types.is_string(col.type): 
+                    # Check if the new column with the statistic is present in the result table
+                    expected_col_name = f"{col_name}_{statistic}"
+                    assert expected_col_name in result_table.column_names, \
+                        f"Column {expected_col_name} not found in result table"
+                    
+
+def test_all_statistics() -> None:
+    """Test applying all statistics in STATS to the table."""
+
+    for i in [".parquet", ".csv",  ".arrow"]:
+        d1 = Generatedata(file_pattern=i, out_name=f"data_1{i}")
+        d1()
+        table = pa.table(d1.df)
+        statistics = 'all'
+        result_table = ts.apply_statistics(table, statistics=statistics)
+
+        # Check that the result is a PyArrow Table
+        assert isinstance(result_table, pa.Table)
+
+            # For each numeric column in the table, check if all statistics are applied
+        for col_name in table.column_names:
+            col = table[col_name]  # Get the actual column data
+            if not pa.types.is_string(col.type):  # Skip string columns
+                for stat_name in ts.STATS.keys():
+                    # Check if the new column with each statistic is present in the result table
+                    expected_col_name = f"{col_name}_{stat_name}"
+                    assert expected_col_name in result_table.column_names, \
+                        f"Column {expected_col_name} not found in result table"
+                    
+
+
+
+

From c18d6b04447a41389965446294d22da0e2286dc9 Mon Sep 17 00:00:00 2001
From: Hamdah Shafqat Abbasi <hamdahshafqat@hotmail.com>
Date: Tue, 3 Dec 2024 13:14:17 -0500
Subject: [PATCH 03/12] fix filepattern groupby column

---
 .../tabular-statistics-tool/pyproject.toml    |  2 +-
 .../features/tabular_statistics/__main__.py   | 32 ++++++++++++-------
 .../tabular_statistics/tabular_statistics.py  |  8 ++++-
 .../tests/test_main.py                        | 10 ++++--
 4 files changed, 35 insertions(+), 17 deletions(-)

diff --git a/features/tabular-statistics-tool/pyproject.toml b/features/tabular-statistics-tool/pyproject.toml
index c6b0032..3f0eab6 100644
--- a/features/tabular-statistics-tool/pyproject.toml
+++ b/features/tabular-statistics-tool/pyproject.toml
@@ -13,7 +13,7 @@ packages = [{include = "polus", from = "src"}]
 python = ">=3.9"
 typer = "^0.7.0"
 tqdm = "^4.65.0"
-filepattern = "^2.0.5"
+filepattern = "^2.1.0"
 pyarrow = "^18.0.0"
 scipy = "1.11.4"
 pandas = "^2.2.2"                     
diff --git a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/__main__.py b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/__main__.py
index 6af6ebc..f720a1f 100644
--- a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/__main__.py
+++ b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/__main__.py
@@ -1,18 +1,13 @@
 """Tabular Statistics."""
-import json
 import logging
 import os
 import pathlib
 import time
-import pyarrow as pa
-import pyarrow.ipc as ipc
-import pyarrow.csv as csv
-
-from typing import Any
 from typing import Optional
 import filepattern as fp
 import typer
-# import tabular_statistics as ts
+import numpy as np
+import pyarrow as pa
 
 from polus.tabular.features.tabular_statistics import tabular_statistics as ts
 
@@ -74,15 +69,12 @@ def main(  # noqa: PLR0913
 
     EXTS = [".arrow", ".feather", ".csv", ".parquet"]
 
-    file_pattern = ".*" + file_pattern
-
-    fps = fp.FilePattern(inp_dir, file_pattern)
 
     if preview:
         ts.preview(out_dir, file_pattern)
 
     else:
-        flist = [f for f in pathlib.Path(inp_dir).iterdir() if f.suffix in  EXTS]
+        flist = [f for f in pathlib.Path(inp_dir).iterdir() if f.suffix in EXTS]
 
         table = ts.load_files(flist)
 
@@ -98,8 +90,24 @@ def main(  # noqa: PLR0913
         if group_by:
             logger.info(f"Applying statistics by grouping data on: {group_by}")
             group_by = [col.strip() for col in group_by.split(",") if col.strip()] 
+            
+            images = list(np.array(table[file_column]))
+
+            # Generate the 'groupby_columnlist' in the same order as 'images' as images are sorted in filepattern object
+            groupby_columnlist = []
+            for idx in range(len(images)):
+                file = images[idx]
+                fps = fp.FilePattern([file], file_pattern)
+                file_dict, _ = list(fps())[0]
+                values = [file_dict[key] for key in group_by if key in file_dict]
+                joined_string = ''.join(map(str, values))
+                groupby_columnlist.append(joined_string)
+            # Convert the list to a PyArrow Array
+            groupby_column = pa.array(groupby_columnlist)
+            table = table.append_column("group_by", groupby_column)
+            
             # Apply statistics to the grouped data
-            aggregated_df = ts.apply_statistics(table, statistics, group_by)
+            aggregated_df = ts.apply_statistics(table, statistics, group_by_columns="group_by")
         else:
             logger.info(f"Applying statistics on data")
             aggregated_df = ts.apply_statistics_to_group(table, statistics)
diff --git a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py
index 86a52a3..8271f57 100644
--- a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py
+++ b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py
@@ -16,7 +16,6 @@
 from typing import Any
 from tqdm import tqdm
 import json
-import filepattern as fp
 
 logger = logging.getLogger(__name__)
 logger.setLevel(os.environ.get("POLUS_LOG", logging.INFO))
@@ -259,6 +258,11 @@ def apply_statistics(table: pa.Table,  statistics: str, group_by_columns:Optiona
             statistics = STATS
         else:
             statistics = statistics.split(',')
+            #Check if all keys in statistics are in the STATS dictionary
+            missing_keys = [key for key in statistics if key not in STATS]
+            if missing_keys:
+                raise KeyError(f"Invalid statistics: {', '.join(missing_keys)}")
+            
             statistics = {k: v for k, v in STATS.items() if k in statistics}
 
 
@@ -351,6 +355,8 @@ def load_files(flist: List[Path]) -> pa.Table:
                 logger.info(f"Loading CSV file: {file.name}")
                 # Read CSV with pandas and convert to PyArrow Table
                 df = pd.read_csv(file)
+                df[df.select_dtypes(include=['object']).columns] = \
+                    df.select_dtypes(include=['object']).map(str)
                 table = pa.Table.from_pandas(df)
             elif file.suffix == ".feather":
                 logger.info(f"Loading Feather file: {file.name}")
diff --git a/features/tabular-statistics-tool/tests/test_main.py b/features/tabular-statistics-tool/tests/test_main.py
index 85f0a9a..885db1b 100644
--- a/features/tabular-statistics-tool/tests/test_main.py
+++ b/features/tabular-statistics-tool/tests/test_main.py
@@ -3,9 +3,14 @@
 import numpy as np
 import pandas as pd
 import pyarrow as pa
+from typer.testing import CliRunner
+from polus.tabular.features.tabular_statistics.__main__ import app
 from polus.tabular.features.tabular_statistics import tabular_statistics as ts
 
 
+
+runner = CliRunner()
+
 class Generatedata:
     """Generate tabular data with several different file formats."""
 
@@ -28,6 +33,8 @@ def __init__(
         self.file_pattern = file_pattern
         self.out_name = out_name
         self.df = self.create_dataframe()
+        self.inpdir=self.get_inp_dir()
+        self.outdir=self.get_out_dir()
 
     def get_inp_dir(self) -> pathlib.Path:
         """Get input directory."""
@@ -127,6 +134,3 @@ def test_all_statistics() -> None:
                         f"Column {expected_col_name} not found in result table"
                     
 
-
-
-

From 0f64ddc81f57e1c5de3a5398e1ff7c379f831de4 Mon Sep 17 00:00:00 2001
From: Hamdah Shafqat Abbasi <hamdahshafqat@hotmail.com>
Date: Tue, 3 Dec 2024 15:05:03 -0500
Subject: [PATCH 04/12] fixed pre-commmit checks

---
 features/tabular-statistics-tool/Dockerfile   |   2 +-
 features/tabular-statistics-tool/README.md    |   2 +-
 features/tabular-statistics-tool/VERSION      |   2 +-
 .../tabular-statistics-tool/build-docker.sh   |   2 +-
 features/tabular-statistics-tool/gitignore    |   2 +-
 features/tabular-statistics-tool/plugin.json  |   2 +-
 .../tabular-statistics-tool/pyproject.toml    |   4 +-
 .../features/tabular_statistics/__init__.py   |   2 +-
 .../features/tabular_statistics/__main__.py   |  43 +--
 .../tabular_statistics/tabular_statistics.py  | 289 +++++++++---------
 .../tabular-statistics-tool/tests/__init__.py |   2 +-
 .../tests/test_main.py                        |  44 ++-
 12 files changed, 199 insertions(+), 197 deletions(-)

diff --git a/features/tabular-statistics-tool/Dockerfile b/features/tabular-statistics-tool/Dockerfile
index 5d23a5b..ad94cc8 100644
--- a/features/tabular-statistics-tool/Dockerfile
+++ b/features/tabular-statistics-tool/Dockerfile
@@ -21,4 +21,4 @@ RUN pip3 install "${EXEC_DIR}/tabular-tools/${TOOL_DIR}" --no-cache-dir
 # Set the entrypoint
 # TODO: Change the entrypoint to the tool entrypoint
 ENTRYPOINT ["python3", "-m", "polus.tabular.features.tabular_statistics"]
-CMD ["--help"]
\ No newline at end of file
+CMD ["--help"]
diff --git a/features/tabular-statistics-tool/README.md b/features/tabular-statistics-tool/README.md
index 60ac1f7..3c85868 100644
--- a/features/tabular-statistics-tool/README.md
+++ b/features/tabular-statistics-tool/README.md
@@ -41,4 +41,4 @@ This plugin takes two input argument and one output argument:
 | `--inpDir`      | Input csv collection to be processed by this plugin | Input  | genericData   |
 | `--filePattern` | The filePattern of the images in represented in csv | Input  | string        |
 | `--groupBy`     | The variable(s) of how the images should be grouped | Input  | string        |
-| `--outDir`      | Output collection                                   | Output | genericData   | 
+| `--outDir`      | Output collection                                   | Output | genericData   |
diff --git a/features/tabular-statistics-tool/VERSION b/features/tabular-statistics-tool/VERSION
index 15a06be..206c085 100644
--- a/features/tabular-statistics-tool/VERSION
+++ b/features/tabular-statistics-tool/VERSION
@@ -1 +1 @@
-0.1.0-dev0
\ No newline at end of file
+0.1.0-dev0
diff --git a/features/tabular-statistics-tool/build-docker.sh b/features/tabular-statistics-tool/build-docker.sh
index d734881..aaafec9 100755
--- a/features/tabular-statistics-tool/build-docker.sh
+++ b/features/tabular-statistics-tool/build-docker.sh
@@ -20,4 +20,4 @@ cp .gitignore .dockerignore
 docker build . -t ${tag}
 rm Dockerfile .dockerignore
 cd ${cur_dir}
-version=$(<VERSION)
\ No newline at end of file
+version=$(<VERSION)
diff --git a/features/tabular-statistics-tool/gitignore b/features/tabular-statistics-tool/gitignore
index 57b14d3..9ed1c37 100644
--- a/features/tabular-statistics-tool/gitignore
+++ b/features/tabular-statistics-tool/gitignore
@@ -20,4 +20,4 @@ data
 ../../.flake8
 __pycache__
 .mypy_cache
-requirements.txt
\ No newline at end of file
+requirements.txt
diff --git a/features/tabular-statistics-tool/plugin.json b/features/tabular-statistics-tool/plugin.json
index 880e36b..a448110 100644
--- a/features/tabular-statistics-tool/plugin.json
+++ b/features/tabular-statistics-tool/plugin.json
@@ -154,4 +154,4 @@
       "description": "The variable(s) to group rows"
     }
   ]
-}
\ No newline at end of file
+}
diff --git a/features/tabular-statistics-tool/pyproject.toml b/features/tabular-statistics-tool/pyproject.toml
index 3f0eab6..b4d5e43 100644
--- a/features/tabular-statistics-tool/pyproject.toml
+++ b/features/tabular-statistics-tool/pyproject.toml
@@ -16,7 +16,7 @@ tqdm = "^4.65.0"
 filepattern = "^2.1.0"
 pyarrow = "^18.0.0"
 scipy = "1.11.4"
-pandas = "^2.2.2"                     
+pandas = "^2.2.2"
 
 
 [tool.poetry.group.dev.dependencies]
@@ -29,4 +29,4 @@ pytest = "^7.2.1"
 
 [build-system]
 requires = ["poetry-core"]
-build-backend = "poetry.core.masonry.api"
\ No newline at end of file
+build-backend = "poetry.core.masonry.api"
diff --git a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/__init__.py b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/__init__.py
index 037f583..fe578a6 100644
--- a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/__init__.py
+++ b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/__init__.py
@@ -1,4 +1,4 @@
 """Tabular Statistics."""
 __version__ = "0.1.0-dev0"
 
-from . import tabular_statistics
\ No newline at end of file
+from . import tabular_statistics
diff --git a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/__main__.py b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/__main__.py
index f720a1f..9d2db0f 100644
--- a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/__main__.py
+++ b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/__main__.py
@@ -4,11 +4,11 @@
 import pathlib
 import time
 from typing import Optional
+
 import filepattern as fp
-import typer
 import numpy as np
 import pyarrow as pa
-
+import typer
 from polus.tabular.features.tabular_statistics import tabular_statistics as ts
 
 app = typer.Typer()
@@ -29,8 +29,7 @@ def main(  # noqa: PLR0913
         "--inpDir",
         help="Input generic data collection to be processed by this plugin",
     ),
-    file_pattern: str = typer.Option(".+", "--filePattern", help="file_pattern"),
-   
+    file_pattern: str = typer.Option(..., "--filePattern", help="file_pattern"),
     statistics: str = typer.Option(
         ...,
         "--statistics",
@@ -67,14 +66,13 @@ def main(  # noqa: PLR0913
 
     # By default it ingests all input files if not file_pattern is defined
 
-    EXTS = [".arrow", ".feather", ".csv", ".parquet"]
-
+    outformats = [".arrow", ".feather", ".csv", ".parquet"]
 
     if preview:
         ts.preview(out_dir, file_pattern)
 
     else:
-        flist = [f for f in pathlib.Path(inp_dir).iterdir() if f.suffix in EXTS]
+        flist = [f for f in pathlib.Path(inp_dir).iterdir() if f.suffix in outformats]
 
         table = ts.load_files(flist)
 
@@ -83,38 +81,41 @@ def main(  # noqa: PLR0913
 
         # Validate required column
         if file_column not in columns:
-            raise ValueError(f"Column '{file_column}' not found")
-        
-        logger.info(f"Merging files into a single table")
+            msg = f"Column '{file_column}' not found"
+            raise ValueError(msg)
+
+        logger.info("Merging files into a single table")
 
         if group_by:
             logger.info(f"Applying statistics by grouping data on: {group_by}")
-            group_by = [col.strip() for col in group_by.split(",") if col.strip()] 
-            
+            groupby = [col.strip() for col in group_by.split(",") if col.strip()]
+
             images = list(np.array(table[file_column]))
 
-            # Generate the 'groupby_columnlist' in the same order as 'images' as images are sorted in filepattern object
+            # Generate the 'groupby_columnlist' in the same order as 'images'
             groupby_columnlist = []
             for idx in range(len(images)):
                 file = images[idx]
                 fps = fp.FilePattern([file], file_pattern)
                 file_dict, _ = list(fps())[0]
-                values = [file_dict[key] for key in group_by if key in file_dict]
-                joined_string = ''.join(map(str, values))
+                values = [file_dict[key] for key in groupby if key in file_dict]
+                joined_string = "".join(map(str, values))
                 groupby_columnlist.append(joined_string)
             # Convert the list to a PyArrow Array
             groupby_column = pa.array(groupby_columnlist)
             table = table.append_column("group_by", groupby_column)
-            
+
             # Apply statistics to the grouped data
-            aggregated_df = ts.apply_statistics(table, statistics, group_by_columns="group_by")
+            aggregated_df = ts.apply_statistics(
+                table,
+                statistics,
+                group_by_columns="group_by",
+            )
         else:
-            logger.info(f"Applying statistics on data")
+            logger.info("Applying statistics on data")
             aggregated_df = ts.apply_statistics_to_group(table, statistics)
 
-
         ts.save_outputs(aggregated_df, out_dir)
-        
 
     exec_time = time.time() - start_time
     logger.info(f"Execution time: {time.strftime('%H:%M:%S', time.gmtime(exec_time))}")
@@ -122,4 +123,4 @@ def main(  # noqa: PLR0913
 
 
 if __name__ == "__main__":
-    app()
\ No newline at end of file
+    app()
diff --git a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py
index 8271f57..4c2f341 100644
--- a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py
+++ b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py
@@ -1,31 +1,29 @@
 """Tabular Statistics."""
+import json
 import logging
+import math
 import os
-from scipy.stats import kurtosis
+from pathlib import Path
+from typing import Any
 from typing import Optional
+
 import numpy as np
 import pandas as pd
-import math
 import pyarrow as pa
-import pyarrow.csv as csv
-import pyarrow.ipc as ipc
-import pyarrow.feather as feather
 import pyarrow.parquet as pq
-from pathlib import Path
-from typing import List
-from typing import Any
+from pyarrow import csv
+from pyarrow import feather
+from pyarrow import ipc
+from scipy.stats import kurtosis
 from tqdm import tqdm
-import json
 
 logger = logging.getLogger(__name__)
 logger.setLevel(os.environ.get("POLUS_LOG", logging.INFO))
-POLUS_TAB_EXT = os.environ.get("POLUS_TAB_EXT", ".csv")
-
+POLUS_TAB_EXT = os.environ.get("POLUS_TAB_EXT", ".feather")
 
 
 def mean(values: list[float]) -> float:
-    """
-    Calculate the mean of a list of numbers.
+    """Calculate the mean of a list of numbers.
 
     Args:
         values: A list of numerical values.
@@ -36,14 +34,14 @@ def mean(values: list[float]) -> float:
     Raises:
         ValueError: If the input list is empty.
     """
-    if values.empty:
-        raise ValueError("The input list is empty. Cannot calculate the mean.")
+    if len(values) == 0:
+        msg = "The input list is empty. Cannot calculate the mean."
+        raise ValueError(msg)
     return np.mean(values)
 
 
 def median(values: list[float]) -> float:
-    """
-    Calculate the median of a list of numbers.
+    """Calculate the median of a list of numbers.
 
     Args:
         values: A list of numerical values.
@@ -54,13 +52,14 @@ def median(values: list[float]) -> float:
     Raises:
         ValueError: If the input list is empty.
     """
-    if values.empty:
-        raise ValueError("The input list is empty. Cannot calculate the median.")
+    if len(values) == 0:
+        msg = "The input list is empty. Cannot calculate the median."
+        raise ValueError(msg)
     return np.median(values)
 
+
 def std(values: list[float]) -> float:
-    """
-    Calculate the standard deviation of a list of numbers.
+    """Calculate the standard deviation of a list of numbers.
 
     Args:
         values: A list of numerical values.
@@ -71,13 +70,16 @@ def std(values: list[float]) -> float:
     Raises:
         ValueError: If the input list is empty.
     """
-    if values.empty:
-        raise ValueError("The input list is empty. Cannot calculate the standard deviation.")
+    if len(values) == 0:
+        msg = "The input list is empty. Cannot calculate the standard deviation."
+        raise ValueError(
+            msg,
+        )
     return np.std(values)
 
+
 def var(values: list[float]) -> float:
-    """
-    Calculate the variance of a list of numbers.
+    """Calculate the variance of a list of numbers.
 
     Args:
         values: A list of numerical values.
@@ -88,14 +90,14 @@ def var(values: list[float]) -> float:
     Raises:
         ValueError: If the input list is empty.
     """
-    if values.empty:
-        raise ValueError("The input list is empty. Cannot calculate the variance.")
+    if len(values) == 0:
+        msg = "The input list is empty. Cannot calculate the variance."
+        raise ValueError(msg)
     return np.var(values)
 
 
 def skew(values: list[float]) -> float:
-    """
-    Calculate the skewness of a list of numbers.
+    """Calculate the skewness of a list of numbers.
 
     Args:
         values: A list of numerical values.
@@ -106,22 +108,22 @@ def skew(values: list[float]) -> float:
     Raises:
         ValueError: If the input list is empty.
     """
-    if values.empty:
-        raise ValueError("The input list is empty. Cannot calculate the skewness.")
-    
+    if len(values) == 0:
+        msg = "The input list is empty. Cannot calculate the skewness."
+        raise ValueError(msg)
+
     n = len(values)
     mean = sum(values) / n
     std_dev = math.sqrt(sum((x - mean) ** 2 for x in values) / n)
-    
+
     if std_dev == 0:
-        return float('nan')  # If standard deviation is 0, skewness is undefined
-    
-    skew_value = sum((x - mean) ** 3 for x in values) / (n * std_dev ** 3)
-    return skew_value
+        return float("nan")  # If standard deviation is 0, skewness is undefined
+
+    return sum((x - mean) ** 3 for x in values) / (n * std_dev**3)
+
 
 def kurt(values: list[float]) -> float:
-    """
-    Calculate the kurtosis of a list of numbers.
+    """Calculate the kurtosis of a list of numbers.
 
     Args:
         values: A list of numerical values.
@@ -132,13 +134,14 @@ def kurt(values: list[float]) -> float:
     Raises:
         ValueError: If the input list is empty.
     """
-    if values.empty:
-        raise ValueError("The input list is empty. Cannot calculate the kurtosis.")
+    if len(values) == 0:
+        msg = "The input list is empty. Cannot calculate the kurtosis."
+        raise ValueError(msg)
     return kurtosis(values)
 
+
 def count(values: list[float]) -> int:
-    """
-    Count the number of elements in a list.
+    """Count the number of elements in a list.
 
     Args:
         values: A list of numerical values.
@@ -149,15 +152,14 @@ def count(values: list[float]) -> int:
     Raises:
         ValueError: If the input list is empty.
     """
-    if values.empty:
-        raise ValueError("The input list is empty. Cannot calculate the count.")
+    if len(values) == 0:
+        msg = "The input list is empty. Cannot calculate the count."
+        raise ValueError(msg)
     return len(values)
 
 
-
-def max(values: list[float]) -> float:
-    """
-    Calculate the maximum value from a list of numbers.
+def max(values: list[float]) -> float:  # noqa: A001
+    """Calculate the maximum value from a list of numbers.
 
     Args:
         values: A list of numerical values.
@@ -168,13 +170,14 @@ def max(values: list[float]) -> float:
     Raises:
         ValueError: If the input list is empty.
     """
-    if values.empty:
-        raise ValueError("The input list is empty. Cannot calculate the maximum.")
+    if len(values) == 0:
+        msg = "The input list is empty. Cannot calculate the maximum."
+        raise ValueError(msg)
     return np.max(values)
 
-def min(values: list[float]) -> float:
-    """
-    Calculate the minimum value from a list of numbers.
+
+def min(values: list[float]) -> float:  # noqa: A001
+    """Calculate the minimum value from a list of numbers.
 
     Args:
         values: A list of numerical values.
@@ -185,13 +188,14 @@ def min(values: list[float]) -> float:
     Raises:
         ValueError: If the input list is empty.
     """
-    if values.empty:
-        raise ValueError("The input list is empty. Cannot calculate the minimum.")
+    if len(values) == 0:
+        msg = "The input list is empty. Cannot calculate the minimum."
+        raise ValueError(msg)
     return np.min(values)
 
+
 def iqr(values: list[float]) -> float:
-    """
-    Calculate the interquartile range (IQR) of a list of numbers.
+    """Calculate the interquartile range (IQR) of a list of numbers.
 
     Args:
         values: A list of numerical values.
@@ -202,13 +206,14 @@ def iqr(values: list[float]) -> float:
     Raises:
         ValueError: If the input list is empty.
     """
-    if values.empty:
-        raise ValueError("The input list is empty. Cannot calculate the IQR.")
+    if len(values) == 0:
+        msg = "The input list is empty. Cannot calculate the IQR."
+        raise ValueError(msg)
     return np.percentile(values, 75) - np.percentile(values, 25)
 
+
 def proportion(values: list[float]) -> float:
-    """
-    Calculate the proportion of positive values in a list.
+    """Calculate the proportion of positive values in a list.
 
     Args:
         values: A list of numerical values.
@@ -219,8 +224,9 @@ def proportion(values: list[float]) -> float:
     Raises:
         ValueError: If the input list is empty.
     """
-    if values.empty:
-        raise ValueError("The input list is empty. Cannot calculate the proportion.")
+    if len(values) == 0:
+        msg = "The input list is empty. Cannot calculate the proportion."
+        raise ValueError(msg)
     return sum(1 for x in values if x > 0) / len(values)
 
 
@@ -236,78 +242,80 @@ def proportion(values: list[float]) -> float:
     "max": max,
     "min": min,
     "iqr": iqr,
-    "prop": proportion
+    "prop": proportion,
 }
 
 
- # Function to apply statistics to each group
-def apply_statistics(table: pa.Table,  statistics: str, group_by_columns:Optional[str]=None): 
-        """
-        Apply statistical functions to each group in a PyArrow Table.
+# Function to apply statistics to each group
+def apply_statistics(
+    table: pa.Table,
+    statistics: str,
+    group_by_columns: Optional[str] = None,
+) -> pa.Table:
+    """Apply statistical functions to each group in a PyArrow Table.
 
-        Args:
-            table: Input PyArrow Table.
-            group_by_columns: Columns to group by.
-            statistics: Comma-separated list of statistics or 'all' to apply all available statistics.
-        
-        Returns:
-            pyarrow.Table: Aggregated statistics for each numeric column.
-        """
-        # Step 1: Convert PyArrow Table to Pandas DataFrame for easy manipulation
-        if statistics == "all":
-            statistics = STATS
-        else:
-            statistics = statistics.split(',')
-            #Check if all keys in statistics are in the STATS dictionary
-            missing_keys = [key for key in statistics if key not in STATS]
-            if missing_keys:
-                raise KeyError(f"Invalid statistics: {', '.join(missing_keys)}")
-            
-            statistics = {k: v for k, v in STATS.items() if k in statistics}
-
-
-        df = table.to_pandas()
-
-        numeric_columns = df.select_dtypes(include='number').columns.tolist()
-
-        # # Step 5: Apply the statistics to each group
-        aggregated_results = {}
-
-        for stat_name, stat_func in statistics.items():
-            for col in  numeric_columns:
-                col_stat_name = f"{col}_{stat_name}"
-                if group_by_columns:
-                    # Aggregate without grouping
-                    aggregated_results[col_stat_name] = df.groupby(group_by_columns)[col].apply(stat_func)
-                else:
-                    # Aggregate without grouping
-                    aggregated_results[col_stat_name] = [stat_func(df[col])]
+    Args:
+        table: Input PyArrow Table.
+        group_by_columns: Columns to group by.
+        statistics: Comma-separated list or 'all' for all statistics.
 
-        if group_by_columns:
-            group_table = pd.DataFrame(aggregated_results)
-            # Perform an left join on the "group_by_columns" column
-            arrow_table = pd.merge(df, group_table, on=group_by_columns, how='left')
+    Returns:
+        pyarrow.Table: Aggregated statistics for each numeric column.
+    """
+    # Step 1: Convert PyArrow Table to Pandas DataFrame for easy manipulation
+    if statistics == "all":
+        metrics = STATS
+    else:
+        metrics = statistics.split(",")  # type: ignore
+        # Check if all keys in statistics are in the STATS dictionary
+        missing_keys = [key for key in metrics if key not in STATS]
+        if missing_keys:
+            msg = f"Invalid statistics: {', '.join(missing_keys)}"
+            raise KeyError(msg)
+
+        metrics = {k: v for k, v in STATS.items() if k in metrics}
+
+    df = table.to_pandas()
+
+    numeric_columns = df.select_dtypes(include="number").columns.tolist()
+
+    # # Step 5: Apply the statistics to each group
+    aggregated_results = {}
+
+    for stat_name, stat_func in metrics.items():
+        for col in numeric_columns:
+            col_stat_name = f"{col}_{stat_name}"
+            if group_by_columns:
+                # Aggregate without grouping
+                aggregated_results[col_stat_name] = df.groupby(group_by_columns)[
+                    col
+                ].apply(stat_func)
+            else:
+                # Aggregate without grouping
+                aggregated_results[col_stat_name] = [stat_func(df[col])]
 
-            arrow_table.reset_index(inplace=True)
-            arrow_table = pa.Table.from_pandas(arrow_table)
+    if group_by_columns:
+        group_table = pd.DataFrame(aggregated_results)
+        # Perform an left join on the "group_by_columns" column
+        arrow_table = pd.merge(df, group_table, on=group_by_columns, how="left")
 
+        arrow_table.reset_index(inplace=True)
+        arrow_table = pa.Table.from_pandas(arrow_table)
 
-        else:
-            arrow_table = pa.Table.from_pydict(aggregated_results)
+    else:
+        arrow_table = pa.Table.from_pydict(aggregated_results)
 
-        return arrow_table
+    return arrow_table
 
 
-def save_outputs(data:pa.Table, out_dir:Path) -> None:
-    """
-    Save data to a file in CSV, Feather, Arrow, or Parquet format.
+def save_outputs(data: pa.Table, out_dir: Path) -> None:
+    """Save data to a file in CSV, Feather, Arrow, or Parquet format.
 
     Args:
         data: The data to save.
         out_dir: Directory to save the file.
         file: Base filename for the saved file.
     """
-
     # Determine the output file path and extension based on the desired format
     output_file = out_dir.joinpath(f"tabular_statistics_output{POLUS_TAB_EXT}")
 
@@ -321,25 +329,28 @@ def save_outputs(data:pa.Table, out_dir:Path) -> None:
 
     elif POLUS_TAB_EXT == ".arrow":
         logger.info(f"Saved outputs:{output_file}")
-        with pa.OSFile(str(output_file), 'wb') as arrow_file:
-            with ipc.new_file(data,data.schema) as writer:
-                writer.write(data)
+        record_batch = data.to_batches()
+        # Write the data using ipc.new_file
+        with pa.OSFile(str(output_file), "wb") as sink:  # noqa:SIM117
+            with ipc.new_file(sink, data.schema) as writer:
+                for batch in record_batch:
+                    writer.write(batch)
 
     elif POLUS_TAB_EXT == ".parquet":
         logger.info(f"Saved outputs:{output_file}")
         pq.write_table(data, output_file)
 
     else:
-        raise ValueError(f"Unsupported extension: {POLUS_TAB_EXT}. Supported: .csv, .feather, .arrow, .parquet.")
+        msg = f"Unsupported extension: {POLUS_TAB_EXT}"
+        raise ValueError(
+            msg,
+        )
 
-    
     logger.info(f"File saved successfully:{output_file}")
 
 
-
-def load_files(flist: List[Path]) -> pa.Table:
-    """
-    Load multiple files (CSV, Feather, Arrow, Parquet) and return a concatenated PyArrow Table.
+def load_files(flist: list[Path]) -> pa.Table:
+    """Load and concatenate multiple files into a PyArrow Table.
 
     Args:
         flist: List of file paths to load.
@@ -348,15 +359,16 @@ def load_files(flist: List[Path]) -> pa.Table:
         pa.Table: Concatenated table of all loaded data.
     """
     tables = []
-    
+
     for file in tqdm(flist, desc="Processing files"):
         try:
             if file.suffix == ".csv":
                 logger.info(f"Loading CSV file: {file.name}")
                 # Read CSV with pandas and convert to PyArrow Table
                 df = pd.read_csv(file)
-                df[df.select_dtypes(include=['object']).columns] = \
-                    df.select_dtypes(include=['object']).map(str)
+                df[df.select_dtypes(include=["object"]).columns] = df.select_dtypes(
+                    include=["object"],
+                ).map(str)
                 table = pa.Table.from_pandas(df)
             elif file.suffix == ".feather":
                 logger.info(f"Loading Feather file: {file.name}")
@@ -376,25 +388,21 @@ def load_files(flist: List[Path]) -> pa.Table:
                 continue  # Skip unsupported files
 
             tables.append(table)
-        except Exception as e:
+        except (OSError, ValueError) as e:
             logger.error(f"Error processing file {file.name}: {e}")
 
     # Concatenate all loaded tables into a single table
-    combined_table = pa.concat_tables(tables)
-
-    return combined_table
+    return pa.concat_tables(tables)
 
 
-def preview(out_dir:Path, file_pattern:str) -> None:
-    """
-    Create a preview JSON file with output directory and file pattern details.
+def preview(out_dir: Path, file_pattern: str) -> None:
+    """Create a preview JSON file with output directory and file pattern details.
 
     Args:
         out_dir: Directory for the outputs.
         file_pattern: The file pattern used for processing.
     """
-
-    out_name= out_dir.joinpath(f"tabular_statistics_output{POLUS_TAB_EXT}")
+    out_name = out_dir.joinpath(f"tabular_statistics_output{POLUS_TAB_EXT}")
 
     out_json: dict[str, Any] = {
         "filepattern": file_pattern,
@@ -402,4 +410,3 @@ def preview(out_dir:Path, file_pattern:str) -> None:
     }
     with Path(out_dir, "preview.json").open("w") as jfile:
         json.dump(out_json, jfile, indent=2)
-
diff --git a/features/tabular-statistics-tool/tests/__init__.py b/features/tabular-statistics-tool/tests/__init__.py
index 49a8fb4..d1e4fc5 100644
--- a/features/tabular-statistics-tool/tests/__init__.py
+++ b/features/tabular-statistics-tool/tests/__init__.py
@@ -1 +1 @@
-"""Test for Tabular Statistics."""
\ No newline at end of file
+"""Test for Tabular Statistics."""
diff --git a/features/tabular-statistics-tool/tests/test_main.py b/features/tabular-statistics-tool/tests/test_main.py
index 885db1b..cb2c906 100644
--- a/features/tabular-statistics-tool/tests/test_main.py
+++ b/features/tabular-statistics-tool/tests/test_main.py
@@ -8,17 +8,13 @@
 from polus.tabular.features.tabular_statistics import tabular_statistics as ts
 
 
-
 runner = CliRunner()
 
+
 class Generatedata:
     """Generate tabular data with several different file formats."""
 
-    def __init__(
-        self,
-        file_pattern: str,
-        out_name: str
-    ) -> None:
+    def __init__(self, file_pattern: str, out_name: str) -> None:
         """Define instance attributes."""
         self.data_dir = pathlib.Path.cwd().parent.joinpath("data")
 
@@ -33,8 +29,8 @@ def __init__(
         self.file_pattern = file_pattern
         self.out_name = out_name
         self.df = self.create_dataframe()
-        self.inpdir=self.get_inp_dir()
-        self.outdir=self.get_out_dir()
+        self.inpdir = self.get_inp_dir()
+        self.outdir = self.get_out_dir()
 
     def get_inp_dir(self) -> pathlib.Path:
         """Get input directory."""
@@ -84,11 +80,10 @@ def __call__(self) -> None:
         return data_ext[self.file_pattern]()
 
 
-
 def test_apply_statistics() -> None:
     """Test applying statistics on PyArrow table."""
 
-    for i in [".parquet", ".csv",  ".arrow"]:
+    for i in [".parquet", ".csv", ".arrow"]:
         d1 = Generatedata(file_pattern=i, out_name=f"data_1{i}")
         d1()
         table = pa.table(d1.df)
@@ -96,41 +91,40 @@ def test_apply_statistics() -> None:
 
         # Test applying each statistic in STATS to the table
         for statistic in statistics_list:
-
             result_table = ts.apply_statistics(table, statistics=statistic)
-            
+
             assert isinstance(result_table, pa.Table)
-            
+
             for col_name in table.column_names:
-                col = table[col_name] 
-                if not pa.types.is_string(col.type): 
+                col = table[col_name]
+                if not pa.types.is_string(col.type):
                     # Check if the new column with the statistic is present in the result table
                     expected_col_name = f"{col_name}_{statistic}"
-                    assert expected_col_name in result_table.column_names, \
-                        f"Column {expected_col_name} not found in result table"
-                    
+                    assert (
+                        expected_col_name in result_table.column_names
+                    ), f"Column {expected_col_name} not found in result table"
+
 
 def test_all_statistics() -> None:
     """Test applying all statistics in STATS to the table."""
 
-    for i in [".parquet", ".csv",  ".arrow"]:
+    for i in [".parquet", ".csv", ".arrow"]:
         d1 = Generatedata(file_pattern=i, out_name=f"data_1{i}")
         d1()
         table = pa.table(d1.df)
-        statistics = 'all'
+        statistics = "all"
         result_table = ts.apply_statistics(table, statistics=statistics)
 
         # Check that the result is a PyArrow Table
         assert isinstance(result_table, pa.Table)
 
-            # For each numeric column in the table, check if all statistics are applied
+        # For each numeric column in the table, check if all statistics are applied
         for col_name in table.column_names:
             col = table[col_name]  # Get the actual column data
             if not pa.types.is_string(col.type):  # Skip string columns
                 for stat_name in ts.STATS.keys():
                     # Check if the new column with each statistic is present in the result table
                     expected_col_name = f"{col_name}_{stat_name}"
-                    assert expected_col_name in result_table.column_names, \
-                        f"Column {expected_col_name} not found in result table"
-                    
-
+                    assert (
+                        expected_col_name in result_table.column_names
+                    ), f"Column {expected_col_name} not found in result table"

From 549350fa41bb5882b7dd175450ef0fd85fbda03d Mon Sep 17 00:00:00 2001
From: Hamdah Shafqat Abbasi <hamdahshafqat@hotmail.com>
Date: Tue, 3 Dec 2024 15:24:01 -0500
Subject: [PATCH 05/12] add bash script for running plugin

---
 .../tabular-statistics-tool/run-plugin.sh     | 24 +++++++++++++++++++
 .../tabular_statistics/tabular_statistics.py  |  2 +-
 2 files changed, 25 insertions(+), 1 deletion(-)
 create mode 100644 features/tabular-statistics-tool/run-plugin.sh

diff --git a/features/tabular-statistics-tool/run-plugin.sh b/features/tabular-statistics-tool/run-plugin.sh
new file mode 100644
index 0000000..7978941
--- /dev/null
+++ b/features/tabular-statistics-tool/run-plugin.sh
@@ -0,0 +1,24 @@
+#!/bin/bash
+
+version=$(<VERSION)
+datapath=$(readlink --canonicalize data)
+
+# Inputs
+inp_dir=/data/input
+file_pattern="{r:c+}_{c:d+}_c{p:d}.ome.tif"
+outformat=".arrow"
+group_by=r,c
+statistics=mean,median
+
+# Output paths
+out_dir=/data/output
+
+# Run the plugin
+docker run -e POLUS_TAB_EXT=${outformat} --mount type=bind,source=${datapath},target=/data/ \
+            polusai/tabular-statistics-tool:${version} \
+            --inpDir ${inp_dir} \
+            --outDir ${out_dir} \
+            --filePattern ${file_pattern} \
+            --groupBy ${group_by} \
+            --statistics ${statistics} \
+         
\ No newline at end of file
diff --git a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py
index 4c2f341..2a03b55 100644
--- a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py
+++ b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py
@@ -19,7 +19,7 @@
 
 logger = logging.getLogger(__name__)
 logger.setLevel(os.environ.get("POLUS_LOG", logging.INFO))
-POLUS_TAB_EXT = os.environ.get("POLUS_TAB_EXT", ".feather")
+POLUS_TAB_EXT = os.environ.get("POLUS_TAB_EXT", ".csv")
 
 
 def mean(values: list[float]) -> float:

From c7e92874c3683269cf724b0e1adeda866353f813 Mon Sep 17 00:00:00 2001
From: Hamdah Shafqat Abbasi <hamdahshafqat@hotmail.com>
Date: Tue, 3 Dec 2024 15:32:53 -0500
Subject: [PATCH 06/12] fix README.md

---
 features/tabular-statistics-tool/README.md | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/features/tabular-statistics-tool/README.md b/features/tabular-statistics-tool/README.md
index 3c85868..40d7ea8 100644
--- a/features/tabular-statistics-tool/README.md
+++ b/features/tabular-statistics-tool/README.md
@@ -12,7 +12,9 @@ This plugin computes statistical measures on numeric and floating point data col
 6. [kurt (excess kurtosis)](https://www.itl.nist.gov/div898/handbook/eda/section3/eda35b.htm)
 7. count (number of rows sampled)
 8. [iqr (Interquartile_range)](https://en.wikipedia.org/wiki/Interquartile_range)
-9. Counts the numbers greater than 0 and divides this count by the total number of elements in the sequence
+9. prop: counts the numbers greater than 0 and divides this count by the total number of elements in the sequence
+10. min: Minimum value in the dataset.
+11. max: Maximum value in the dataset.
 
 
 ## Usage:

From 22cc9a3cfd0837236cb4955d25c5d3abd25a24ae Mon Sep 17 00:00:00 2001
From: Hamdah Shafqat Abbasi <hamdahshafqat@hotmail.com>
Date: Tue, 3 Dec 2024 15:37:20 -0500
Subject: [PATCH 07/12] updated groupBy in docs

---
 features/tabular-statistics-tool/README.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/features/tabular-statistics-tool/README.md b/features/tabular-statistics-tool/README.md
index 40d7ea8..d8d9fbb 100644
--- a/features/tabular-statistics-tool/README.md
+++ b/features/tabular-statistics-tool/README.md
@@ -20,7 +20,8 @@ This plugin computes statistical measures on numeric and floating point data col
 ## Usage:
 - A directory containing one or more tabular files in the supported formats
 - Each file must include numeric and floating point data columns
-- If a `groupBy` column is specified in the input arguments, it must be present in the data
+- Ensure that the `groupBy` variables are included in the file pattern
+
 
 For more information on WIPP, visit the [official WIPP page](https://isg.nist.gov/deepzoomweb/software/wipp).
 

From 7cf19d730a61f18de5845bd0541c7caf8c9af116 Mon Sep 17 00:00:00 2001
From: Hamdah Shafqat Abbasi <hamdahshafqat@hotmail.com>
Date: Wed, 11 Dec 2024 12:38:14 -0500
Subject: [PATCH 08/12] Only selecting uppercase features for calculations

---
 .../features/tabular_statistics/tabular_statistics.py       | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py
index 2a03b55..b1a2439 100644
--- a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py
+++ b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py
@@ -277,8 +277,12 @@ def apply_statistics(
 
     df = table.to_pandas()
 
-    numeric_columns = df.select_dtypes(include="number").columns.tolist()
+    numeric_columns = [col for col in df.columns if col.isupper()]
 
+    if len(numeric_columns) == 0:
+        msg = "No features found!! Features should start with a capital letter."
+        raise ValueError(msg)
+    
     # # Step 5: Apply the statistics to each group
     aggregated_results = {}
 

From 2b03c24aee6b376029680d3349f2257337fa26d8 Mon Sep 17 00:00:00 2001
From: Hamdah Shafqat Abbasi <hamdahshafqat@hotmail.com>
Date: Wed, 11 Dec 2024 12:41:47 -0500
Subject: [PATCH 09/12] updated docs

---
 features/tabular-statistics-tool/README.md | 1 +
 1 file changed, 1 insertion(+)

diff --git a/features/tabular-statistics-tool/README.md b/features/tabular-statistics-tool/README.md
index d8d9fbb..e027307 100644
--- a/features/tabular-statistics-tool/README.md
+++ b/features/tabular-statistics-tool/README.md
@@ -21,6 +21,7 @@ This plugin computes statistical measures on numeric and floating point data col
 - A directory containing one or more tabular files in the supported formats
 - Each file must include numeric and floating point data columns
 - Ensure that the `groupBy` variables are included in the file pattern
+- Please note that all features on which statistics are computed are capitalized.
 
 
 For more information on WIPP, visit the [official WIPP page](https://isg.nist.gov/deepzoomweb/software/wipp).

From 7bda75b08aa889ecc5231945f4a68a88df7d5d5f Mon Sep 17 00:00:00 2001
From: Hamdah Shafqat Abbasi <hamdahshafqat@hotmail.com>
Date: Wed, 11 Dec 2024 17:52:11 -0500
Subject: [PATCH 10/12] updated env var

---
 features/tabular-statistics-tool/Dockerfile                     | 2 +-
 .../tabular/features/tabular_statistics/tabular_statistics.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/features/tabular-statistics-tool/Dockerfile b/features/tabular-statistics-tool/Dockerfile
index ad94cc8..c2d71a0 100644
--- a/features/tabular-statistics-tool/Dockerfile
+++ b/features/tabular-statistics-tool/Dockerfile
@@ -2,7 +2,7 @@ FROM polusai/bfio:2.4.5
 
 # environment variables defined in polusai/bfio
 ENV EXEC_DIR="/opt/executables"
-ENV POLUS_TAB_EXT=".csv"
+ENV POLUS_TAB_EXT=".arrow"
 ENV POLUS_LOG="INFO"
 
 # Work directory defined in the base container
diff --git a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py
index b1a2439..dfe2f19 100644
--- a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py
+++ b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py
@@ -19,7 +19,7 @@
 
 logger = logging.getLogger(__name__)
 logger.setLevel(os.environ.get("POLUS_LOG", logging.INFO))
-POLUS_TAB_EXT = os.environ.get("POLUS_TAB_EXT", ".csv")
+POLUS_TAB_EXT = os.environ.get("POLUS_TAB_EXT", ".arrow")
 
 
 def mean(values: list[float]) -> float:

From e67830ec48ffcd425c8925c9d5fbedb85405d62f Mon Sep 17 00:00:00 2001
From: Hamdah Shafqat Abbasi <hamdahshafqat@hotmail.com>
Date: Fri, 10 Jan 2025 14:54:32 -0500
Subject: [PATCH 11/12] fix string column issue

---
 features/tabular-statistics-tool/tests/test_main.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/features/tabular-statistics-tool/tests/test_main.py b/features/tabular-statistics-tool/tests/test_main.py
index cb2c906..024643b 100644
--- a/features/tabular-statistics-tool/tests/test_main.py
+++ b/features/tabular-statistics-tool/tests/test_main.py
@@ -87,11 +87,12 @@ def test_apply_statistics() -> None:
         d1 = Generatedata(file_pattern=i, out_name=f"data_1{i}")
         d1()
         table = pa.table(d1.df)
+        numeric_table = table.drop([col for col in table.column_names if pa.types.is_string(table[col].type)])
         statistics_list = list(ts.STATS.keys())
 
         # Test applying each statistic in STATS to the table
         for statistic in statistics_list:
-            result_table = ts.apply_statistics(table, statistics=statistic)
+            result_table = ts.apply_statistics(numeric_table, statistics=statistic)
 
             assert isinstance(result_table, pa.Table)
 
@@ -112,8 +113,9 @@ def test_all_statistics() -> None:
         d1 = Generatedata(file_pattern=i, out_name=f"data_1{i}")
         d1()
         table = pa.table(d1.df)
+        numeric_table = table.drop([col for col in table.column_names if pa.types.is_string(table[col].type)])
         statistics = "all"
-        result_table = ts.apply_statistics(table, statistics=statistics)
+        result_table = ts.apply_statistics(numeric_table, statistics=statistics)
 
         # Check that the result is a PyArrow Table
         assert isinstance(result_table, pa.Table)

From c70d45de71036e29a20057eeda3017c45027dcfd Mon Sep 17 00:00:00 2001
From: Hamdah Shafqat Abbasi <hamdahshafqat@hotmail.com>
Date: Fri, 10 Jan 2025 15:08:46 -0500
Subject: [PATCH 12/12] fix black formatting

---
 .../features/tabular_statistics/tabular_statistics.py     | 1 +
 features/tabular-statistics-tool/tests/test_main.py       | 8 ++++++--
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py
index dfe2f19..49261f8 100644
--- a/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py
+++ b/features/tabular-statistics-tool/src/polus/tabular/features/tabular_statistics/tabular_statistics.py
@@ -411,6 +411,7 @@ def preview(out_dir: Path, file_pattern: str) -> None:
     out_json: dict[str, Any] = {
         "filepattern": file_pattern,
         "outDir": str(out_name),
+        
     }
     with Path(out_dir, "preview.json").open("w") as jfile:
         json.dump(out_json, jfile, indent=2)
diff --git a/features/tabular-statistics-tool/tests/test_main.py b/features/tabular-statistics-tool/tests/test_main.py
index 024643b..26a6b4e 100644
--- a/features/tabular-statistics-tool/tests/test_main.py
+++ b/features/tabular-statistics-tool/tests/test_main.py
@@ -87,7 +87,9 @@ def test_apply_statistics() -> None:
         d1 = Generatedata(file_pattern=i, out_name=f"data_1{i}")
         d1()
         table = pa.table(d1.df)
-        numeric_table = table.drop([col for col in table.column_names if pa.types.is_string(table[col].type)])
+        numeric_table = table.drop(
+            [col for col in table.column_names if pa.types.is_string(table[col].type)]
+        )
         statistics_list = list(ts.STATS.keys())
 
         # Test applying each statistic in STATS to the table
@@ -113,7 +115,9 @@ def test_all_statistics() -> None:
         d1 = Generatedata(file_pattern=i, out_name=f"data_1{i}")
         d1()
         table = pa.table(d1.df)
-        numeric_table = table.drop([col for col in table.column_names if pa.types.is_string(table[col].type)])
+        numeric_table = table.drop(
+            [col for col in table.column_names if pa.types.is_string(table[col].type)]
+        )
         statistics = "all"
         result_table = ts.apply_statistics(numeric_table, statistics=statistics)