Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .copier-answers.yml
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# Changes here will be overwritten by Copier
_commit: v7.0.0
_commit: v7.1.0
_src_path: gh:eccenca/cmem-plugin-template
author_mail: cmempy-developer@eccenca.com
author_name: eccenca GmbH
Expand Down
2 changes: 1 addition & 1 deletion .gitlab-ci.yml
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
---
default:
image: docker-registry.eccenca.com/eccenca-python:v3.11.4
image: docker-registry.eccenca.com/eccenca-python:v3.11.9-2
# all jobs can be interrupted in case a new commit is pushed
interruptible: true
before_script:
Expand Down
4 changes: 1 addition & 3 deletions Taskfile.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -157,9 +157,7 @@ tasks:
<<: *preparation
cmds:
# ignore 51358 safety - dev dependency only
# ignore 67599 pip - dev dependency only
# ignore 70612 jinja2 - dev dependency only
- poetry run safety check -i 51358 -i 67599 -i 70612
- poetry run safety check -i 51358

check:ruff:
desc: Complain about everything else
Expand Down
6 changes: 4 additions & 2 deletions cmem_plugin_splitfile/plugin_splitfile.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
"""A task splitting a text file into multiple parts with a specified size"""

from collections import OrderedDict
from collections.abc import Sequence
from io import BytesIO
from pathlib import Path
from shutil import move
Expand All @@ -16,6 +17,7 @@
)
from cmem_plugin_base.dataintegration.context import ExecutionContext, ExecutionReport
from cmem_plugin_base.dataintegration.description import Icon, Plugin, PluginParameter
from cmem_plugin_base.dataintegration.entity import Entities
from cmem_plugin_base.dataintegration.parameter.choice import ChoiceParameterType
from cmem_plugin_base.dataintegration.plugins import WorkflowPlugin
from cmem_plugin_base.dataintegration.ports import FixedNumberOfInputs
Expand Down Expand Up @@ -134,7 +136,7 @@ def __init__( # noqa: C901 PLR0913
errors += "Minimum chunk size is 1024 bytes. "

if use_directory:
test_path = projects_path[1:] if projects_path.startswith("/") else projects_path
test_path = projects_path.removeprefix("/")
if not is_valid_filepath(test_path):
errors += 'Invalid path for parameter "Internal projects directory". '
elif not Path(projects_path).is_dir():
Expand Down Expand Up @@ -246,7 +248,7 @@ def execute_filesystem(self) -> bool:
(resources_path / self.input_filename).unlink()
return True

def execute(self, inputs: None, context: ExecutionContext) -> None: # noqa: ARG002
def execute(self, inputs: Sequence[Entities], context: ExecutionContext) -> None: # noqa: ARG002
"""Execute plugin with temporary directory"""
self.context = context
context.report.update(ExecutionReport(entity_count=0, operation_desc="files generated"))
Expand Down
56 changes: 28 additions & 28 deletions poetry.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

15 changes: 7 additions & 8 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@ keywords = [

[tool.poetry.dependencies]# if you need to change python version here, change it also in .python-version
python = "^3.11"
pathvalidate = "^3.2.1"
pathvalidate = "^3.2.3"
filesplit = "^4.1.0"
types-requests = "^2.32.0.20241016"
urllib3 = "^2.2.3"
Expand All @@ -25,18 +25,18 @@ version = "^4.5.0"
allow-prereleases = false

[tool.poetry.group.dev.dependencies.cmem-cmemc]
version = "^24.2.0"
version = "^24.3.0"

[tool.poetry.group.dev.dependencies]
genbadge = {extras = ["coverage"], version = "^1.1.1"}
mypy = "^1.11.1"
pip = "^24"
pytest = "^8.3.2"
pytest-cov = "^5.0.0"
mypy = "^1.14.1"
pip = "^25.0"
pytest = "^8.3.4"
pytest-cov = "^6.0.0"
pytest-dotenv = "^0.5.2"
pytest-html = "^4.1.1"
pytest-memray = { version = "^1.7.0", markers = "platform_system != 'Windows'" }
ruff = "^0.6.1"
ruff = "^0.9.4"
safety = "^1.10.3"

[build-system]
Expand Down Expand Up @@ -78,7 +78,6 @@ line-ending = "lf" # Use `\n` line endings for all files
[tool.ruff.lint]
select = ["ALL"]
ignore = [
"ANN101", # Missing type annotation for self in method
"ANN204", # Missing return type annotation for special method `__init__`
"COM812", # missing-trailing-comma
"D107", # Missing docstring in __init__
Expand Down
51 changes: 23 additions & 28 deletions tests/test_splitnt.py → tests/test_splitfile.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@
import pytest
from cmem.cmempy.workspace.projects.project import delete_project, make_new_project
from cmem.cmempy.workspace.projects.resources.resource import create_resource, get_resource
from requests.exceptions import HTTPError
from requests import HTTPError

from cmem_plugin_splitfile.plugin_splitfile import SplitFilePlugin
from tests.utils import TestExecutionContext, needs_cmem
Expand Down Expand Up @@ -57,12 +57,12 @@ def test_filesystem_size() -> None:
size_unit="KB",
projects_path=__path__[0],
use_directory=True,
).execute(None, context=TestExecutionContext(PROJECT_ID))
).execute(inputs=[], context=TestExecutionContext(PROJECT_ID))

for n in range(3):
assert cmp(
Path(__path__[0]) / PROJECT_ID / "resources" / f"{UUID4}_00000000{n+1}.nt",
Path(__path__[0]) / "test_files" / f"{UUID4}_size_00000000{n+1}.nt",
Path(__path__[0]) / PROJECT_ID / "resources" / f"{UUID4}_00000000{n + 1}.nt",
Path(__path__[0]) / "test_files" / f"{UUID4}_size_00000000{n + 1}.nt",
)

if not (Path(__path__[0]) / PROJECT_ID / "resources" / TEST_FILENAME).is_file():
Expand All @@ -80,12 +80,12 @@ def test_filesystem_size_header() -> None:
include_header=True,
projects_path=__path__[0],
use_directory=True,
).execute(None, context=TestExecutionContext(PROJECT_ID))
).execute(inputs=[], context=TestExecutionContext(PROJECT_ID))

for n in range(3):
assert cmp(
Path(__path__[0]) / PROJECT_ID / "resources" / f"{UUID4}_00000000{n+1}.nt",
Path(__path__[0]) / "test_files" / f"{UUID4}_size_header_00000000{n+1}.nt",
Path(__path__[0]) / PROJECT_ID / "resources" / f"{UUID4}_00000000{n + 1}.nt",
Path(__path__[0]) / "test_files" / f"{UUID4}_size_header_00000000{n + 1}.nt",
)

if not (Path(__path__[0]) / PROJECT_ID / "resources" / TEST_FILENAME).is_file():
Expand All @@ -101,13 +101,13 @@ def test_api_size() -> None:
chunk_size=6,
size_unit="KB",
projects_path=__path__[0],
).execute(None, context=TestExecutionContext(PROJECT_ID))
).execute(inputs=[], context=TestExecutionContext(PROJECT_ID))

for n in range(3):
f = get_resource(project_name=PROJECT_ID, resource_name=f"{UUID4}_00000000{n+1}.nt")
f = get_resource(project_name=PROJECT_ID, resource_name=f"{UUID4}_00000000{n + 1}.nt")
assert (
f
== (Path(__path__[0]) / "test_files" / f"{UUID4}_size_00000000{n+1}.nt")
== (Path(__path__[0]) / "test_files" / f"{UUID4}_size_00000000{n + 1}.nt")
.open("rb")
.read()
)
Expand All @@ -126,12 +126,12 @@ def test_filesystem_size_delete() -> None:
projects_path=__path__[0],
use_directory=True,
delete_file=True,
).execute(None, context=TestExecutionContext(PROJECT_ID))
).execute(inputs=[], context=TestExecutionContext(PROJECT_ID))

for n in range(3):
assert cmp(
Path(__path__[0]) / PROJECT_ID / "resources" / f"{UUID4}_00000000{n+1}.nt",
Path(__path__[0]) / "test_files" / f"{UUID4}_size_00000000{n+1}.nt",
Path(__path__[0]) / PROJECT_ID / "resources" / f"{UUID4}_00000000{n + 1}.nt",
Path(__path__[0]) / "test_files" / f"{UUID4}_size_00000000{n + 1}.nt",
)

if (Path(__path__[0]) / PROJECT_ID / "resources" / TEST_FILENAME).is_file():
Expand All @@ -148,24 +148,19 @@ def test_api_size_delete() -> None:
size_unit="KB",
projects_path=__path__[0],
delete_file=True,
).execute(None, context=TestExecutionContext(PROJECT_ID))
).execute(inputs=[], context=TestExecutionContext(PROJECT_ID))

for n in range(3):
f = get_resource(project_name=PROJECT_ID, resource_name=f"{UUID4}_00000000{n+1}.nt")
f = get_resource(project_name=PROJECT_ID, resource_name=f"{UUID4}_00000000{n + 1}.nt")
assert (
f
== (Path(__path__[0]) / "test_files" / f"{UUID4}_size_00000000{n+1}.nt")
== (Path(__path__[0]) / "test_files" / f"{UUID4}_size_00000000{n + 1}.nt")
.open("rb")
.read()
)

try:
with pytest.raises(HTTPError, match="404 Client Error: Not Found for url:"):
get_resource(project_name=PROJECT_ID, resource_name=TEST_FILENAME)
except Exception as exc:
if type(exc) is HTTPError and exc.status_code == 404: # noqa: PLR2004
pass
else:
raise


@needs_cmem
Expand All @@ -178,12 +173,12 @@ def test_filesystem_lines() -> None:
size_unit="Lines",
projects_path=__path__[0],
use_directory=True,
).execute(None, context=TestExecutionContext(PROJECT_ID))
).execute(inputs=[], context=TestExecutionContext(PROJECT_ID))

for n in range(3):
assert cmp(
Path(__path__[0]) / PROJECT_ID / "resources" / f"{UUID4}_00000000{n+1}.nt",
Path(__path__[0]) / "test_files" / f"{UUID4}_lines_00000000{n+1}.nt",
Path(__path__[0]) / PROJECT_ID / "resources" / f"{UUID4}_00000000{n + 1}.nt",
Path(__path__[0]) / "test_files" / f"{UUID4}_lines_00000000{n + 1}.nt",
)


Expand All @@ -198,10 +193,10 @@ def test_filesystem_lines_header() -> None:
include_header=True,
projects_path=__path__[0],
use_directory=True,
).execute(None, context=TestExecutionContext(PROJECT_ID))
).execute(inputs=[], context=TestExecutionContext(PROJECT_ID))

for n in range(3):
assert cmp(
Path(__path__[0]) / PROJECT_ID / "resources" / f"{UUID4}_00000000{n+1}.nt",
Path(__path__[0]) / "test_files" / f"{UUID4}_lines_header_00000000{n+1}.nt",
Path(__path__[0]) / PROJECT_ID / "resources" / f"{UUID4}_00000000{n + 1}.nt",
Path(__path__[0]) / "test_files" / f"{UUID4}_lines_header_00000000{n + 1}.nt",
)