Skip to content

Commit acd1fe9

Browse files
JamesParrottJamesParrott
authored
Fix tests. Merge pull request #320 from JamesParrott/run_network_tests_on_localhost
Use localhost instead of remote server for network marked tests.
2 parents 89a4125 + 19a9238 commit acd1fe9

5 files changed

Lines changed: 234 additions & 36 deletions

File tree

‎.github/actions/test/action.yml‎

Lines changed: 86 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,119 @@
11
name:
2-
Test
2+
Run Doctests and Pytest
33

44
description:
55
Run pytest, and run the doctest runner (shapefile.py as a script).
66

77
inputs:
88
extra_args:
9-
type: string
9+
description: Extra command line args for Pytest and python shapefile.py
1010
default: '-m "not network"'
11+
required: false
12+
replace_remote_urls_with_localhost:
13+
description: yes or no. Test loading shapefiles from a url, without overloading an external server from 30 parallel workflows.
14+
default: 'no'
15+
required: false
16+
pyshp_repo_directory:
17+
description: Path to where the PyShp repo was checked out to (to keep separate from Shapefiles & artefacts repo).
18+
required: false
19+
default: '.'
20+
python-version:
21+
description: Set to "2.7" to use caddy instead of python -m SimpleHTTPServer
22+
required: true
23+
24+
1125

1226
runs:
1327
using: "composite"
1428
steps:
15-
# The Repo is required to already be checked out, e.g. by the calling workflow
29+
# The PyShp repo is required to already be checked out into pyshp_repo_directory,
30+
# e.g. by the calling workflow using:
31+
# steps:
32+
# - uses: actions/checkout@v4
33+
# with:
34+
# path: ./Pyshp
35+
# and then calling this Action with:
36+
# - name: Run tests
37+
# uses: ./Pyshp/.github/actions/test
38+
# with:
39+
# extra_args: ""
40+
# replace_remote_urls_with_localhost: 'yes'
41+
# pyshp_repo_directory: ./Pyshp
1642

1743
# The Python to be tested with is required to already be setup, with "python" and "pip" on the system Path
1844

45+
- name: Checkout shapefiles and zip file artefacts repo
46+
if: ${{ inputs.replace_remote_urls_with_localhost == 'yes' }}
47+
uses: actions/checkout@v4
48+
with:
49+
repository: JamesParrott/PyShp_test_shapefile
50+
path: ./PyShp_test_shapefile
51+
52+
- name: Serve shapefiles and zip file artefacts on localhost
53+
if: ${{ inputs.replace_remote_urls_with_localhost == 'yes' && inputs.python-version != '2.7'}}
54+
shell: bash
55+
working-directory: ./PyShp_test_shapefile
56+
run: |
57+
python -m http.server 8000 &
58+
echo "HTTP_SERVER_PID=$!" >> $GITHUB_ENV
59+
sleep 4 # give server time to start
60+
61+
- name: Download and unzip Caddy binary
62+
if: ${{ inputs.replace_remote_urls_with_localhost == 'yes' && inputs.python-version == '2.7'}}
63+
working-directory: .
64+
shell: bash
65+
run: |
66+
curl -L https://github.com/caddyserver/caddy/releases/download/v2.10.0/caddy_2.10.0_linux_amd64.tar.gz --output caddy.tar.gz
67+
tar -xzf caddy.tar.gz
68+
69+
- name: Serve shapefiles and zip file artefacts on localhost using Caddy
70+
if: ${{ inputs.replace_remote_urls_with_localhost == 'yes' && inputs.python-version == '2.7'}}
71+
shell: bash
72+
working-directory: .
73+
run: |
74+
./caddy file-server --root ./PyShp_test_shapefile --listen :8000 &
75+
echo "HTTP_SERVER_PID=$!" >> $GITHUB_ENV
76+
sleep 2 # give server time to start
77+
1978
- name: Doctests
2079
shell: bash
80+
working-directory: ${{ inputs.pyshp_repo_directory }}
81+
env:
82+
REPLACE_REMOTE_URLS_WITH_LOCALHOST: ${{ inputs.replace_remote_urls_with_localhost }}
2183
run: python shapefile.py ${{ inputs.extra_args }}
2284

2385
- name: Install test dependencies.
2486
shell: bash
87+
working-directory: ${{ inputs.pyshp_repo_directory }}
2588
run: |
2689
python -m pip install --upgrade pip
2790
pip install -r requirements.test.txt
2891
2992
- name: Pytest
3093
shell: bash
94+
working-directory: ${{ inputs.pyshp_repo_directory }}
95+
env:
96+
REPLACE_REMOTE_URLS_WITH_LOCALHOST: ${{ inputs.replace_remote_urls_with_localhost }}
3197
run: |
32-
pytest ${{ inputs.extra_args }}
98+
pytest -rA --tb=short ${{ inputs.extra_args }}
3399
34100
- name: Show versions for logs.
35101
shell: bash
36102
run: |
37103
python --version
38-
python -m pytest --version
104+
python -m pytest --version
105+
106+
107+
# - name: Test http server
108+
# # (needs a full Github Actions runner or a Python non-slim Docker image,
109+
# # as the slim Debian images don't have curl or wget).
110+
# if: ${{ inputs.replace_remote_urls_with_localhost == 'yes' }}
111+
# shell: bash
112+
# run: curl http://localhost:8000/ne_110m_admin_0_tiny_countries.shp
113+
114+
- name: Stop http server
115+
if: ${{ inputs.replace_remote_urls_with_localhost == 'yes' }}
116+
shell: bash
117+
run: |
118+
echo Killing http server process ID: ${{ env.HTTP_SERVER_PID }}
119+
kill ${{ env.HTTP_SERVER_PID }}

‎.github/workflows/run_tests_hooks_and_tools.yml‎

Lines changed: 34 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ name: Run pre-commit hooks and tests
55
on:
66
push:
77
pull_request:
8-
branches: [ master ]
8+
branches: [ master, ]
99
workflow_call:
1010
workflow_dispatch:
1111

@@ -30,7 +30,7 @@ jobs:
3030
run: |
3131
pylint --disable=R,C test_shapefile.py
3232
33-
test_on_old_Pythons:
33+
test_on_EOL_Pythons:
3434
strategy:
3535
fail-fast: false
3636
matrix:
@@ -44,16 +44,28 @@ jobs:
4444

4545
runs-on: ubuntu-latest
4646
container:
47-
image: python:${{ matrix.python-version }}-slim
47+
image: python:${{ matrix.python-version }}
4848

4949
steps:
5050
- uses: actions/checkout@v4
51+
with:
52+
path: ./Pyshp
5153

52-
- name: Run tests
53-
uses: ./.github/actions/test
54+
- name: Non-network tests
55+
uses: ./Pyshp/.github/actions/test
56+
with:
57+
pyshp_repo_directory: ./Pyshp
58+
python-version: ${{ matrix.python-version }}
5459

60+
- name: Network tests
61+
uses: ./Pyshp/.github/actions/test
62+
with:
63+
extra_args: '-m network'
64+
replace_remote_urls_with_localhost: 'yes'
65+
pyshp_repo_directory: ./Pyshp
66+
python-version: ${{ matrix.python-version }}
5567

56-
run_tests:
68+
test_on_supported_Pythons:
5769
strategy:
5870
fail-fast: false
5971
matrix:
@@ -74,11 +86,24 @@ jobs:
7486

7587
runs-on: ${{ matrix.os }}
7688
steps:
89+
- uses: actions/setup-python@v5
90+
with:
91+
python-version: ${{ matrix.python-version }}
92+
7793
- uses: actions/checkout@v4
94+
with:
95+
path: ./Pyshp
7896

79-
- uses: actions/setup-python@v5
97+
- name: Non-network tests
98+
uses: ./Pyshp/.github/actions/test
8099
with:
100+
pyshp_repo_directory: ./Pyshp
81101
python-version: ${{ matrix.python-version }}
82102

83-
- name: Run tests
84-
uses: ./.github/actions/test
103+
- name: Network tests
104+
uses: ./Pyshp/.github/actions/test
105+
with:
106+
extra_args: '-m network'
107+
replace_remote_urls_with_localhost: 'yes'
108+
pyshp_repo_directory: ./Pyshp
109+
python-version: ${{ matrix.python-version }}

‎.pre-commit-config.yaml‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,15 @@
11
repos:
2-
- repo: https://github.com/pre-commit/pre-commit-hooks
3-
rev: v2.3.0
2+
- repo: https://github.com/astral-sh/ruff-pre-commit
3+
rev: v0.6.4
44
hooks:
5-
- id: check-yaml
6-
- id: trailing-whitespace
5+
- id: ruff-format
76
- repo: https://github.com/pycqa/isort
87
rev: 5.13.2
98
hooks:
109
- id: isort
1110
name: isort (python)
12-
- repo: https://github.com/astral-sh/ruff-pre-commit
13-
rev: v0.6.4
11+
- repo: https://github.com/pre-commit/pre-commit-hooks
12+
rev: v2.3.0
1413
hooks:
15-
- id: ruff-format
14+
- id: check-yaml
15+
- id: trailing-whitespace

‎shapefile.py‎

Lines changed: 88 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,11 @@
2525
# Module settings
2626
VERBOSE = True
2727

28+
# Test config (for the Doctest runner and test_shapefile.py)
29+
REPLACE_REMOTE_URLS_WITH_LOCALHOST = (
30+
os.getenv("REPLACE_REMOTE_URLS_WITH_LOCALHOST", "").lower() == "yes"
31+
)
32+
2833
# Constants for shape types
2934
NULL = 0
3035
POINT = 1
@@ -2794,41 +2799,113 @@ def _get_doctests():
27942799
return tests
27952800

27962801

2797-
def _get_no_network_doctests(examples):
2802+
def _filter_network_doctests(examples, include_network=False, include_non_network=True):
27982803
globals_from_network_doctests = set()
2799-
for example in examples:
2804+
2805+
if not (include_network or include_non_network):
2806+
return
2807+
2808+
examples_it = iter(examples)
2809+
2810+
yield next(examples_it)
2811+
2812+
for example in examples_it:
2813+
# Track variables in doctest shell sessions defined from commands
2814+
# that poll remote URLs, to skip subsequent commands until all
2815+
# such dependent variables are reassigned.
2816+
28002817
if 'sf = shapefile.Reader("https://' in example.source:
28012818
globals_from_network_doctests.add("sf")
2819+
if include_network:
2820+
yield example
28022821
continue
2822+
28032823
lhs = example.source.partition("=")[0]
28042824

28052825
for target in lhs.split(","):
28062826
target = target.strip()
28072827
if target in globals_from_network_doctests:
28082828
globals_from_network_doctests.remove(target)
28092829

2830+
# Non-network tests dependent on the network tests.
28102831
if globals_from_network_doctests:
2832+
if include_network:
2833+
yield example
2834+
continue
2835+
2836+
if not include_non_network:
28112837
continue
28122838

28132839
yield example
28142840

28152841

2816-
def _test(verbosity=0):
2842+
def _replace_remote_url(
2843+
old_url,
2844+
# Default port of Python http.server and Python 2's SimpleHttpServer
2845+
port=8000,
2846+
scheme="http",
2847+
netloc="localhost",
2848+
path=None,
2849+
params="",
2850+
query="",
2851+
fragment="",
2852+
):
2853+
old_parsed = urlparse(old_url)
2854+
2855+
# Strip subpaths, so an artefacts
2856+
# repo or file tree can be simpler and flat
2857+
if path is None:
2858+
path = old_parsed.path.rpartition("/")[2]
2859+
2860+
if port not in (None, ""):
2861+
netloc = "%s:%s" % (netloc, port)
2862+
2863+
new_parsed = old_parsed._replace(
2864+
scheme=scheme,
2865+
netloc=netloc,
2866+
path=path,
2867+
params=params,
2868+
query=query,
2869+
fragment=fragment,
2870+
)
2871+
2872+
new_url = urlunparse(new_parsed) if PYTHON3 else urlunparse(list(new_parsed))
2873+
return new_url
2874+
2875+
2876+
def _test(args=sys.argv[1:], verbosity=0):
28172877
if verbosity == 0:
28182878
print("Getting doctests...")
2819-
tests = _get_doctests()
2820-
2821-
if len(sys.argv) >= 3 and sys.argv[1:3] == ["-m", "not network"]:
2822-
if verbosity == 0:
2823-
print("Removing doctests requiring internet access...")
2824-
tests.examples = list(_get_no_network_doctests(tests.examples))
28252879

28262880
import doctest
2881+
import re
28272882

28282883
doctest.NORMALIZE_WHITESPACE = 1
28292884

2830-
# ignore py2-3 unicode differences
2831-
import re
2885+
tests = _get_doctests()
2886+
2887+
if len(args) >= 2 and args[0] == "-m":
2888+
if verbosity == 0:
2889+
print("Filtering doctests...")
2890+
tests.examples = list(
2891+
_filter_network_doctests(
2892+
tests.examples,
2893+
include_network=args[1] == "network",
2894+
include_non_network=args[1] == "not network",
2895+
)
2896+
)
2897+
2898+
if REPLACE_REMOTE_URLS_WITH_LOCALHOST:
2899+
if verbosity == 0:
2900+
print("Replacing remote urls with http://localhost in doctests...")
2901+
2902+
for example in tests.examples:
2903+
match_url_str_literal = re.search(r'"(https://.*)"', example.source)
2904+
if not match_url_str_literal:
2905+
continue
2906+
old_url = match_url_str_literal.group(1)
2907+
new_url = _replace_remote_url(old_url)
2908+
example.source = example.source.replace(old_url, new_url)
28322909

28332910
class Py23DocChecker(doctest.OutputChecker):
28342911
def check_output(self, want, got, optionflags):

0 commit comments

Comments
 (0)