From b1e760632c850469a61d5392a91ce215036aa41e Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Tue, 9 Dec 2025 11:22:23 +0000 Subject: [PATCH 1/5] Update submodules to latest release tags (or main) --- services/cogstack-jupyter-hub | 2 +- services/cogstack-nlp | 2 +- services/ocr-service | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/services/cogstack-jupyter-hub b/services/cogstack-jupyter-hub index 0d87fc6ef..269f6926f 160000 --- a/services/cogstack-jupyter-hub +++ b/services/cogstack-jupyter-hub @@ -1 +1 @@ -Subproject commit 0d87fc6ef8cada74131014e4277c03004e888344 +Subproject commit 269f6926f5cd47fa902383b714cbf4045f652d6d diff --git a/services/cogstack-nlp b/services/cogstack-nlp index 623be373f..bfc5f6ff0 160000 --- a/services/cogstack-nlp +++ b/services/cogstack-nlp @@ -1 +1 @@ -Subproject commit 623be373fb15621fb8906105663085c65320c197 +Subproject commit bfc5f6ff09643fc6fc1bda4459956a62ef9241ec diff --git a/services/ocr-service b/services/ocr-service index 452efc47b..f102f2d29 160000 --- a/services/ocr-service +++ b/services/ocr-service @@ -1 +1 @@ -Subproject commit 452efc47bd8a3a4db83faeb6de42918be7ea6f64 +Subproject commit f102f2d29dac8d19337e7eeb72c4456dd8e916db From 6bb0487be738d6c6f6d720f8d02bf5d9c4bfa574 Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Wed, 10 Dec 2025 14:41:22 +0000 Subject: [PATCH 2/5] Updated nifiapi stubs. --- nifi/requirements.txt | 2 +- pyproject.toml | 2 +- typings/nifiapi/__init__.pyi | 23 +++++++++++ typings/nifiapi/flowfiletransform.py | 0 typings/nifiapi/flowfiletransform.pyi | 18 +++++---- typings/nifiapi/properties.py | 0 typings/nifiapi/properties.pyi | 57 +++++++++++++++++++++++++++ typings/nifiapi/relationship.py | 0 typings/nifiapi/relationship.pyi | 8 ++++ typings/py4j/__init__.pyi | 0 typings/py4j/java_gateway.pyi | 2 +- 11 files changed, 101 insertions(+), 11 deletions(-) create mode 100644 typings/nifiapi/__init__.pyi create mode 100644 typings/nifiapi/flowfiletransform.py create mode 100644 typings/nifiapi/properties.py create mode 100644 typings/nifiapi/properties.pyi create mode 100644 typings/nifiapi/relationship.py create mode 100644 typings/nifiapi/relationship.pyi create mode 100644 typings/py4j/__init__.pyi diff --git a/nifi/requirements.txt b/nifi/requirements.txt index a424dbc9d..9a23aadcf 100644 --- a/nifi/requirements.txt +++ b/nifi/requirements.txt @@ -10,6 +10,7 @@ pandas==1.5.3 dill>=0.3.6,<1.0.0 bokeh==3.8.0 psycopg[c,binary]==3.2.9 +override==7.7.0 # used in NiFi scripts: geolocation, avro conversion etc. py4j==0.10.9.9 @@ -34,7 +35,6 @@ xnat==0.7.2 opensearch-py==3.0.0 elasticsearch9==9.1.0 neo4j==5.28.2 -eland==9.0.1 # git utils dvc==3.62.0 diff --git a/pyproject.toml b/pyproject.toml index 2f17afdc7..6fae60f28 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.ruff] line-length = 120 -exclude = ["nifi/user-scripts/legacy_scripts"] +exclude = ["nifi/user-scripts/legacy_scripts", "services"] target-version = "py311" indent-width = 4 diff --git a/typings/nifiapi/__init__.pyi b/typings/nifiapi/__init__.pyi new file mode 100644 index 000000000..8fe092830 --- /dev/null +++ b/typings/nifiapi/__init__.pyi @@ -0,0 +1,23 @@ +__all__ = [ + "FlowFileTransform", + "FlowFileTransformResult", + "ProcessContext", + "PropertyDescriptor", + "StandardValidators", + "Relationship", +] + +import nifiapi.flowfiletransform # noqa: F401 +import nifiapi.properties # noqa: F401 +import nifiapi.relationship # noqa: F401 + +from .flowfiletransform import ( + FlowFileTransform, + FlowFileTransformResult, + ProcessContext, +) +from .properties import ( + PropertyDescriptor, + StandardValidators, +) +from .relationship import Relationship diff --git a/typings/nifiapi/flowfiletransform.py b/typings/nifiapi/flowfiletransform.py new file mode 100644 index 000000000..e69de29bb diff --git a/typings/nifiapi/flowfiletransform.pyi b/typings/nifiapi/flowfiletransform.pyi index 32345142d..65306797a 100644 --- a/typings/nifiapi/flowfiletransform.pyi +++ b/typings/nifiapi/flowfiletransform.pyi @@ -1,17 +1,19 @@ from typing import Any, Protocol -from nifiapi.properties import ProcessContext from py4j.java_gateway import JavaObject +from .properties import ProcessContext + +# --------------------- +# FlowFileTransform + Result +# --------------------- +class FlowFileTransform(Protocol): + def transform(self, context: ProcessContext, flowFile: JavaObject) -> Any: ... + class FlowFileTransformResult: def __init__( self, relationship: str, attributes: dict[str, str], - contents: bytes | str | None = None - ) -> None : ... - - - -class FlowFileTransform(Protocol): - def transform(self, context: ProcessContext, flowFile: JavaObject) -> Any: ... + contents: bytes | None = None, + ) -> None: ... diff --git a/typings/nifiapi/properties.py b/typings/nifiapi/properties.py new file mode 100644 index 000000000..e69de29bb diff --git a/typings/nifiapi/properties.pyi b/typings/nifiapi/properties.pyi new file mode 100644 index 000000000..526ebd935 --- /dev/null +++ b/typings/nifiapi/properties.pyi @@ -0,0 +1,57 @@ +from enum import Enum +from typing import Any, Protocol + +class ExpressionLanguageScope(Enum): + NONE = 1 + ENVIRONMENT = 2 + FLOWFILE_ATTRIBUTES = 3 + + +class ProcessContext(Protocol): + def getProperties(self) -> dict[str, Any]: ... + + +class PropertyDescriptor: + def __init__(self, name, description, required=False, sensitive=False, + display_name=None, default_value=None, allowable_values=None, + dependencies=None, expression_language_scope=ExpressionLanguageScope.NONE, + dynamic=False, validators=None, + resource_definition=None, controller_service_definition=None): + if validators is None: + validators = [StandardValidators.ALWAYS_VALID] + + self.name = name + self.description = description + self.required = required + self.sensitive = sensitive + self.displayName = display_name + self.defaultValue = None if default_value is None else str(default_value) + self.allowableValues = allowable_values + self.dependencies = dependencies + self.expressionLanguageScope = expression_language_scope + self.dynamic = dynamic + self.validators = validators + self.resourceDefinition = resource_definition + self.controllerServiceDefinition = controller_service_definition + + +class StandardValidators: + ALWAYS_VALID: object + NON_EMPTY_VALIDATOR: object + INTEGER_VALIDATOR: object + POSITIVE_INTEGER_VALIDATOR: object + POSITIVE_LONG_VALIDATOR: object + NON_NEGATIVE_INTEGER_VALIDATOR: object + NUMBER_VALIDATOR: object + LONG_VALIDATOR: object + PORT_VALIDATOR: object + NON_EMPTY_EL_VALIDATOR: object + HOSTNAME_PORT_LIST_VALIDATOR: object + BOOLEAN_VALIDATOR: object + URL_VALIDATOR: object + URI_VALIDATOR: object + REGULAR_EXPRESSION_VALIDATOR: object + REGULAR_EXPRESSION_WITH_EL_VALIDATOR: object + TIME_PERIOD_VALIDATOR: object + DATA_SIZE_VALIDATOR: object + FILE_EXISTS_VALIDATOR: object diff --git a/typings/nifiapi/relationship.py b/typings/nifiapi/relationship.py new file mode 100644 index 000000000..e69de29bb diff --git a/typings/nifiapi/relationship.pyi b/typings/nifiapi/relationship.pyi new file mode 100644 index 000000000..90acc47e2 --- /dev/null +++ b/typings/nifiapi/relationship.pyi @@ -0,0 +1,8 @@ + +class Relationship: + def __init__(self, name: str, description: str = "") -> None: ... + + name: str + description: str + + raise NotImplementedError \ No newline at end of file diff --git a/typings/py4j/__init__.pyi b/typings/py4j/__init__.pyi new file mode 100644 index 000000000..e69de29bb diff --git a/typings/py4j/java_gateway.pyi b/typings/py4j/java_gateway.pyi index 80bce30e5..f618f3df3 100644 --- a/typings/py4j/java_gateway.pyi +++ b/typings/py4j/java_gateway.pyi @@ -7,4 +7,4 @@ class JavaObject(Protocol): class JVMView(Protocol): """Stub JVM view""" - pass \ No newline at end of file + pass From 92a486b7a8ee3f6fcbc98268964d87ce9bedb644 Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Wed, 10 Dec 2025 14:52:01 +0000 Subject: [PATCH 3/5] Requirements update. --- nifi/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi/requirements.txt b/nifi/requirements.txt index 9a23aadcf..99794aef2 100644 --- a/nifi/requirements.txt +++ b/nifi/requirements.txt @@ -10,7 +10,7 @@ pandas==1.5.3 dill>=0.3.6,<1.0.0 bokeh==3.8.0 psycopg[c,binary]==3.2.9 -override==7.7.0 +override==7.5.0 # used in NiFi scripts: geolocation, avro conversion etc. py4j==0.10.9.9 From dc54c294658d065cb5bd892640ba2e7786fbdd39 Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Wed, 10 Dec 2025 14:56:21 +0000 Subject: [PATCH 4/5] Requirements fix II. --- nifi/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nifi/requirements.txt b/nifi/requirements.txt index 99794aef2..45303fa15 100644 --- a/nifi/requirements.txt +++ b/nifi/requirements.txt @@ -10,7 +10,7 @@ pandas==1.5.3 dill>=0.3.6,<1.0.0 bokeh==3.8.0 psycopg[c,binary]==3.2.9 -override==7.5.0 +override>=7.3.0 # used in NiFi scripts: geolocation, avro conversion etc. py4j==0.10.9.9 From b74887d0e29680ce49c15d5d170e9ab3db64cdb0 Mon Sep 17 00:00:00 2001 From: vladd-bit Date: Wed, 10 Dec 2025 15:09:45 +0000 Subject: [PATCH 5/5] Dockerfile + req update. --- nifi/Dockerfile | 3 --- nifi/requirements.txt | 6 +++--- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/nifi/Dockerfile b/nifi/Dockerfile index 66ad8fc89..35aa3f2d7 100644 --- a/nifi/Dockerfile +++ b/nifi/Dockerfile @@ -83,9 +83,6 @@ RUN uv pip install --no-cache-dir --break-system-packages --system --upgrade pip COPY ./requirements.txt ./requirements.txt RUN uv pip install --no-cache-dir --break-system-packages --target=${NIFI_PYTHON_FRAMEWORK_SOURCE_DIRECTORY} -r "./requirements.txt" -# install MedCAT from the working with cogstack repo (as it is assumed to be stable and the latest version) -RUN uv pip install --no-cache-dir --break-system-packages --target=${NIFI_PYTHON_FRAMEWORK_SOURCE_DIRECTORY} -r https://raw.githubusercontent.com/CogStack/working_with_cogstack/refs/heads/main/requirements.txt --extra-index-url https://download.pytorch.org/whl/cpu/ - ####################################################################################################### # solve groovy grape proxy issues, grape ignores the current environment's proxy settings diff --git a/nifi/requirements.txt b/nifi/requirements.txt index 45303fa15..1632088d4 100644 --- a/nifi/requirements.txt +++ b/nifi/requirements.txt @@ -3,21 +3,21 @@ seaborn==0.13.2 matplotlib==3.10.6 graphviz==0.21 plotly==6.3.0 -keras==3.12.0 +keras==3.12.0 nltk==3.9.1 numpy>=1.26.0,<2.0.0 pandas==1.5.3 dill>=0.3.6,<1.0.0 bokeh==3.8.0 psycopg[c,binary]==3.2.9 -override>=7.3.0 +overrides==7.0.0 # used in NiFi scripts: geolocation, avro conversion etc. py4j==0.10.9.9 rancoord==0.0.6 geocoder==1.38.1 avro==1.12.0 -nipyapi==1.0.0 +nipyapi==1.1.0 py7zr==1.0.0 ipyparallel==9.0.1 cython==3.1.3