diff --git a/apps/api/plane/api/serializers/__init__.py b/apps/api/plane/api/serializers/__init__.py index d0278eb1415..29e60c60a9f 100644 --- a/apps/api/plane/api/serializers/__init__.py +++ b/apps/api/plane/api/serializers/__init__.py @@ -61,6 +61,7 @@ GenericAssetUpdateSerializer, FileAssetSerializer, ) +from .page import PageSearchSerializer from .invite import WorkspaceInviteSerializer from .member import ( ProjectMemberSerializer, diff --git a/apps/api/plane/api/serializers/page.py b/apps/api/plane/api/serializers/page.py new file mode 100644 index 00000000000..4e5eb433932 --- /dev/null +++ b/apps/api/plane/api/serializers/page.py @@ -0,0 +1,35 @@ +# Copyright (c) 2023-present Plane Software, Inc. and contributors +# SPDX-License-Identifier: AGPL-3.0-only +# See the LICENSE file for details. + +# Third party imports +from rest_framework import serializers + +# Module imports +from plane.utils.page_search import build_page_snippet + + +class PageSearchSerializer(serializers.Serializer): + """ + Serializer for page search result data formatting. + + Provides a lightweight, read-only projection of a page for search + responses: identity, project and parent context, last-modified time, and a + short text snippet extracted around the query match. + """ + + id = serializers.UUIDField(read_only=True, help_text="Page ID") + name = serializers.CharField(read_only=True, help_text="Page name") + project_id = serializers.UUIDField( + source="matched_project_id", + read_only=True, + allow_null=True, + help_text="ID of an accessible project the page belongs to", + ) + parent_id = serializers.UUIDField(read_only=True, allow_null=True, help_text="Parent page ID") + updated_at = serializers.DateTimeField(read_only=True, help_text="Last modified timestamp") + snippet = serializers.SerializerMethodField(help_text="Short text excerpt around the search match") + + def get_snippet(self, obj) -> str: + query = self.context.get("query") or "" + return build_page_snippet(obj.description_stripped, query) diff --git a/apps/api/plane/api/urls/__init__.py b/apps/api/plane/api/urls/__init__.py index 4a202431bc7..676e1e2c161 100644 --- a/apps/api/plane/api/urls/__init__.py +++ b/apps/api/plane/api/urls/__init__.py @@ -8,6 +8,7 @@ from .label import urlpatterns as label_patterns from .member import urlpatterns as member_patterns from .module import urlpatterns as module_patterns +from .page import urlpatterns as page_patterns from .project import urlpatterns as project_patterns from .state import urlpatterns as state_patterns from .user import urlpatterns as user_patterns @@ -22,6 +23,7 @@ *label_patterns, *member_patterns, *module_patterns, + *page_patterns, *project_patterns, *state_patterns, *user_patterns, diff --git a/apps/api/plane/api/urls/page.py b/apps/api/plane/api/urls/page.py new file mode 100644 index 00000000000..3720e2d9c59 --- /dev/null +++ b/apps/api/plane/api/urls/page.py @@ -0,0 +1,15 @@ +# Copyright (c) 2023-present Plane Software, Inc. and contributors +# SPDX-License-Identifier: AGPL-3.0-only +# See the LICENSE file for details. + +from django.urls import path + +from plane.api.views import PageSearchEndpoint + +urlpatterns = [ + path( + "workspaces//pages/search/", + PageSearchEndpoint.as_view(http_method_names=["get"]), + name="page-search", + ), +] diff --git a/apps/api/plane/api/views/__init__.py b/apps/api/plane/api/views/__init__.py index 5e4660a7b2b..a9caddacbb3 100644 --- a/apps/api/plane/api/views/__init__.py +++ b/apps/api/plane/api/views/__init__.py @@ -65,6 +65,8 @@ IntakeIssueDetailAPIEndpoint, ) +from .page import PageSearchEndpoint + from .asset import UserAssetEndpoint, UserServerAssetEndpoint, GenericAssetEndpoint from .user import UserEndpoint diff --git a/apps/api/plane/api/views/page.py b/apps/api/plane/api/views/page.py new file mode 100644 index 00000000000..82109207e26 --- /dev/null +++ b/apps/api/plane/api/views/page.py @@ -0,0 +1,270 @@ +# Copyright (c) 2023-present Plane Software, Inc. and contributors +# SPDX-License-Identifier: AGPL-3.0-only +# See the LICENSE file for details. + +# Python imports +import uuid +from functools import reduce +from operator import and_ + +# Django imports +from django.db.models import Exists, OuterRef, Q, Subquery + +# Third party imports +from drf_spectacular.types import OpenApiTypes +from drf_spectacular.utils import ( + OpenApiExample, + OpenApiParameter, + extend_schema, +) +from rest_framework import status +from rest_framework.exceptions import ParseError +from rest_framework.response import Response + +# Module imports +from plane.api.serializers import PageSearchSerializer +from plane.app.permissions import ROLE +from plane.db.models import Page, ProjectPage, Workspace +from plane.utils.openapi import ( + BAD_SEARCH_REQUEST_RESPONSE, + CURSOR_PARAMETER, + FORBIDDEN_RESPONSE, + PER_PAGE_PARAMETER, + UNAUTHORIZED_RESPONSE, + WORKSPACE_SLUG_PARAMETER, + create_paginated_response, +) + +from .base import BaseAPIView + +# The only columns the search response reads. Pages can hold very large bodies, +# so loading them for every hit would pull megabytes out of the database just to +# render a short snippet. +PAGE_SEARCH_FIELDS = ("id", "name", "parent_id", "updated_at", "description_stripped") + +# Page size for search results. Each hit carries a text snippet, so results are +# heavier than a plain id/name list; a smaller default keeps responses light and +# matches the advertised PER_PAGE_PARAMETER contract (default 20, max 100). +PAGE_SEARCH_DEFAULT_PER_PAGE = 20 +PAGE_SEARCH_MAX_PER_PAGE = 100 + +# Ceiling on distinct keywords in one query. Each keyword adds two ILIKE +# predicates with a leading wildcard, which no index can serve, so the cost of a +# search is linear in a number the caller picks for free. Measured on a 200-page +# workspace with ~3 KB bodies where every keyword matches (no AND short-circuit, +# the worst case): 4 keywords ~170 ms, 16 ~530 ms, 100 ~2.9 s — roughly 29 ms per +# keyword, unbounded. 16 sits far above any genuine keyword search (typical +# queries are three to five words) while holding the WHERE clause to 32 +# predicates. Duplicates are collapsed before this limit applies, since AND is +# idempotent and repeating one keyword must not multiply the work. +PAGE_SEARCH_MAX_TOKENS = 16 + +# Query parameters specific to page search. +PAGE_SEARCH_QUERY_PARAMETER = OpenApiParameter( + name="query", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + description=( + "Search query, split on whitespace into keywords. Every keyword must appear " + "(case-insensitively) in the page name or the page text content; they need not " + "appear together or in order. Repeated keywords are collapsed, and a query with " + "more than 16 distinct keywords is rejected." + ), + required=True, + examples=[ + OpenApiExample( + name="Keyword search", + value="latency spike rollback", + description="Find pages containing all of these keywords, in the name or the body", + ) + ], +) + +PAGE_SEARCH_PROJECTS_PARAMETER = OpenApiParameter( + name="projects", + type=OpenApiTypes.STR, + location=OpenApiParameter.QUERY, + description="Optional comma-separated list of project IDs to restrict the search to", + required=False, + examples=[ + OpenApiExample( + name="Two projects", + value="550e8400-e29b-41d4-a716-446655440010,550e8400-e29b-41d4-a716-446655440011", + ) + ], +) + +PAGE_SEARCH_ARCHIVED_PARAMETER = OpenApiParameter( + name="archived", + type=OpenApiTypes.BOOL, + location=OpenApiParameter.QUERY, + description="Include archived pages in the results. Archived pages are excluded unless this is 'true'.", + required=False, + examples=[OpenApiExample(name="Include archived", value=True)], +) + + +class PageSearchEndpoint(BaseAPIView): + """Endpoint to search project pages by name and text content.""" + + use_read_replica = True + + @extend_schema( + operation_id="search_pages", + tags=["Pages"], + description=( + "Search pages across a workspace by name and text content. The query is split on " + "whitespace into keywords and a page matches only when every keyword appears in its " + "name or its text content, so the keywords may be scattered across the document. " + "Only pages in projects the requesting user is a member of are returned; private " + "pages are visible only to their owner and archived pages are excluded unless " + "``archived=true``." + ), + parameters=[ + WORKSPACE_SLUG_PARAMETER, + PAGE_SEARCH_QUERY_PARAMETER, + PAGE_SEARCH_PROJECTS_PARAMETER, + PAGE_SEARCH_ARCHIVED_PARAMETER, + CURSOR_PARAMETER, + PER_PAGE_PARAMETER, + ], + responses={ + 200: create_paginated_response( + item_schema=PageSearchSerializer, + schema_name="PaginatedPageSearchResponse", + description="Paginated page search results", + example_name="Page Search Response", + ), + 400: BAD_SEARCH_REQUEST_RESPONSE, + 401: UNAUTHORIZED_RESPONSE, + 403: FORBIDDEN_RESPONSE, + }, + ) + def get(self, request, slug): + """Search pages + + Perform a case-insensitive search across page names and page text + content, scoped to the pages the requesting user is allowed to see. + Results are cursor paginated. + """ + # An unknown slug would otherwise match no pages and look like a genuine + # empty result. Report it like the other token-API workspace endpoints do. + if not Workspace.objects.filter(slug=slug).exists(): + return Response( + {"error": "Provided workspace does not exist"}, + status=status.HTTP_400_BAD_REQUEST, + ) + + query = request.query_params.get("query", "").strip() + # Collapse repeats before counting: ANDing a keyword with itself changes + # nothing but would double the scan, so "latency latency" must cost the + # same as "latency". First occurrence wins so query order is preserved, + # which is what the snippet anchors on. + seen_tokens = set() + tokens = [] + for token in query.split(): + folded = token.casefold() + if folded not in seen_tokens: + seen_tokens.add(folded) + tokens.append(token) + + if not tokens: + return Response( + {"error": "The 'query' parameter is required to search pages."}, + status=status.HTTP_400_BAD_REQUEST, + ) + + if len(tokens) > PAGE_SEARCH_MAX_TOKENS: + return Response( + {"error": f"The 'query' parameter accepts at most {PAGE_SEARCH_MAX_TOKENS} keywords."}, + status=status.HTTP_400_BAD_REQUEST, + ) + + # Parse the optional project filter into a list of validated UUIDs. + raw_projects = request.query_params.get("projects", "") + try: + project_ids = [uuid.UUID(pid.strip()) for pid in raw_projects.split(",") if pid.strip()] + except (ValueError, TypeError): + return Response( + {"error": "The 'projects' parameter must be a comma-separated list of valid project IDs."}, + status=status.HTTP_400_BAD_REQUEST, + ) + + include_archived = request.query_params.get("archived", "false").lower() == "true" + + # Callers send keyword queries, not phrases, so the query is tokenised on + # whitespace: every token must appear (AND), and each token may appear in + # either the page name or the maintained stripped text content. Matching + # the whole query as one literal substring would only find pages carrying + # the exact phrase; OR-ing the tokens instead would flood the results with + # pages that happen to contain just one common word. A single-token query + # reduces to the same condition as before. + match_query = reduce( + and_, + (Q(name__icontains=token) | Q(description_stripped__icontains=token) for token in tokens), + ) + + # Scoping (security critical) — mirror the internal GlobalSearchEndpoint / + # PageViewSet rules using an Exists() subquery so the project membership + # join does not fan out (and duplicate) rows: + # * only pages that belong to at least one project where the requesting + # user is an active member and the project is not archived, + # * in a project where the user is only a guest, and that project has not + # opted guests into seeing everything, only their own pages — the same + # rule PageViewSet.list/retrieve enforce, + # * optionally narrowed to the requested projects. + # + # Every membership predicate stays in this single filter() call so they all + # bind to the SAME ProjectMember row; splitting them across filter() calls + # would let the role check match a different membership than the user's. + # + # The subquery is pinned to the same workspace as the outer query so a + # ProjectPage row pointing at a project in another workspace could never + # let membership over there grant access to a page in this one. + accessible_project_pages = ProjectPage.objects.filter( + Q(project__project_projectmember__role__gt=ROLE.GUEST.value) + | Q(project__guest_view_all_features=True) + | Q(page__owned_by=request.user), + page_id=OuterRef("pk"), + project__workspace__slug=slug, + project__project_projectmember__member=request.user, + project__project_projectmember__is_active=True, + project__archived_at__isnull=True, + ) + if project_ids: + accessible_project_pages = accessible_project_pages.filter(project_id__in=project_ids) + + # A representative accessible project id to report for the page. + representative_project = accessible_project_pages.order_by("created_at").values("project_id")[:1] + + pages = ( + Page.objects.filter(workspace__slug=slug) + .filter(match_query) + # Private pages are visible only to their owner. + .filter(Q(access=Page.PUBLIC_ACCESS) | Q(owned_by=request.user)) + .annotate( + matched_project_id=Subquery(representative_project), + has_access=Exists(accessible_project_pages), + ) + .filter(has_access=True) + .only(*PAGE_SEARCH_FIELDS) + ) + + # Archived pages are excluded unless explicitly requested. + if not include_archived: + pages = pages.filter(archived_at__isnull=True) + + try: + return self.paginate( + request=request, + queryset=pages, + on_results=lambda results: PageSearchSerializer(results, many=True, context={"query": query}).data, + order_by="-updated_at", + default_per_page=PAGE_SEARCH_DEFAULT_PER_PAGE, + max_per_page=PAGE_SEARCH_MAX_PER_PAGE, + ) + except ParseError as exc: + # The paginator reports bad cursor/per_page values as DRF's + # {"detail": ...}; restate them in the {"error": ...} envelope this + # endpoint uses for its own validation failures. + return Response({"error": str(exc.detail)}, status=status.HTTP_400_BAD_REQUEST) diff --git a/apps/api/plane/tests/contract/api/test_page_search.py b/apps/api/plane/tests/contract/api/test_page_search.py new file mode 100644 index 00000000000..460cc21a456 --- /dev/null +++ b/apps/api/plane/tests/contract/api/test_page_search.py @@ -0,0 +1,558 @@ +# Copyright (c) 2023-present Plane Software, Inc. and contributors +# SPDX-License-Identifier: AGPL-3.0-only +# See the LICENSE file for details. + +"""Contract tests for the token-API page search endpoint. + + GET /api/v1/workspaces/{slug}/pages/search/ + +Covers matching (name and text content), the security-critical scoping rules +(project membership, private-page ownership, archived exclusion), the optional +project filter, and cursor pagination. +""" + +import pytest +from django.db import connection +from django.test.utils import CaptureQueriesContext +from rest_framework import status + +from plane.api.rate_limit import ApiKeyRateThrottle +from plane.api.views.page import PAGE_SEARCH_MAX_TOKENS +from plane.db.models import Page, Project, ProjectMember, ProjectPage, User, Workspace, WorkspaceMember + + +def _url(slug): + return f"/api/v1/workspaces/{slug}/pages/search/" + + +def _make_project(workspace, creator, identifier, member=None, is_active=True, archived_at=None): + """Create a project; optionally add ``member`` (active or not) as an admin member.""" + project = Project.objects.create( + name=f"Project {identifier}", + identifier=identifier, + workspace=workspace, + created_by=creator, + archived_at=archived_at, + ) + if member is not None: + ProjectMember.objects.create(project=project, member=member, role=20, is_active=is_active) + return project + + +def _make_page(workspace, project, owner, name="", content="", access=0, archived_at=None): + """Create a page in ``project``. ``content`` is stored as HTML so the model's + ``save()`` populates ``description_stripped`` exactly like production.""" + page = Page.objects.create( + name=name, + workspace=workspace, + owned_by=owner, + description_html=f"

{content}

" if content else "

", + access=access, + archived_at=archived_at, + ) + ProjectPage.objects.create(page=page, project=project, workspace=workspace) + return page + + +@pytest.fixture(autouse=True) +def _reset_api_key_throttle(api_token): + """Keep these tests isolated from the API-key rate-limit counter, which is + keyed on the (shared) test token and otherwise accumulates across the suite + in the backing cache — producing spurious HTTP 429s here. + + Only this token's throttle key is dropped; flushing the whole cache would + reach into unrelated tests sharing the backend.""" + from django.core.cache import cache + + throttle_key = f"{ApiKeyRateThrottle.scope}:{api_token.token}" + cache.delete(throttle_key) + yield + cache.delete(throttle_key) + + +@pytest.fixture +def other_user(db): + """A second user who is never the API caller.""" + user = User.objects.create( + email="other@plane.so", + username="other-user", + first_name="Other", + last_name="User", + ) + user.set_password("other-password") + user.save() + return user + + +@pytest.fixture +def project(db, workspace, create_user): + """A project the requesting user (``create_user``) is an active member of.""" + return _make_project(workspace, create_user, "TP", member=create_user) + + +@pytest.mark.contract +@pytest.mark.django_db +class TestPageSearch: + def test_missing_query_returns_400(self, api_key_client, workspace, project): + response = api_key_client.get(_url(workspace.slug)) + assert response.status_code == status.HTTP_400_BAD_REQUEST, response.data + + def test_name_match(self, api_key_client, workspace, project, create_user): + match = _make_page(workspace, project, create_user, name="Quarterly Roadmap") + _make_page(workspace, project, create_user, name="Team Lunch Notes") + + response = api_key_client.get(_url(workspace.slug), {"query": "roadmap"}) + + assert response.status_code == status.HTTP_200_OK, response.data + ids = {r["id"] for r in response.data["results"]} + assert ids == {str(match.id)} + + def test_content_match(self, api_key_client, workspace, project, create_user): + # Name does NOT contain the term; the body does. + match = _make_page( + workspace, + project, + create_user, + name="Untitled", + content="Remember to renew the SSL certificate before it expires.", + ) + _make_page(workspace, project, create_user, name="Untitled", content="Nothing relevant here.") + + response = api_key_client.get(_url(workspace.slug), {"query": "ssl certificate"}) + + assert response.status_code == status.HTTP_200_OK, response.data + ids = {r["id"] for r in response.data["results"]} + assert ids == {str(match.id)} + + def test_case_insensitive(self, api_key_client, workspace, project, create_user): + match = _make_page(workspace, project, create_user, name="ONBOARDING Guide") + + response = api_key_client.get(_url(workspace.slug), {"query": "onboarding"}) + + assert response.status_code == status.HTTP_200_OK, response.data + assert {r["id"] for r in response.data["results"]} == {str(match.id)} + + def test_multi_keyword_query_matches_tokens_in_different_sentences( + self, api_key_client, workspace, project, create_user + ): + """Callers send keyword queries. The tokens need not appear together, or + even in the same sentence — matching the query as one literal phrase + would find none of these.""" + scattered = _make_page( + workspace, + project, + create_user, + name="Untitled", + content=( + "We saw a latency regression on Tuesday. A spike in error rates followed. " + "The rollback was clean and the incident is closed." + ), + ) + # Tokens split across the name and the body. + across_name_and_body = _make_page( + workspace, + project, + create_user, + name="Rollback runbook", + content="Mitigation for a latency spike during an incident.", + ) + + response = api_key_client.get(_url(workspace.slug), {"query": "latency spike rollback incident"}) + + assert response.status_code == status.HTTP_200_OK, response.data + assert {r["id"] for r in response.data["results"]} == { + str(scattered.id), + str(across_name_and_body.id), + } + + def test_page_with_only_some_tokens_does_not_match(self, api_key_client, workspace, project, create_user): + """Tokens are ANDed: a page missing any one of them is not a result. + OR-ing would flood the response with single-common-word matches.""" + _make_page( + workspace, + project, + create_user, + name="Untitled", + content="A latency spike happened, but this page never mentions the other terms.", + ) + + response = api_key_client.get(_url(workspace.slug), {"query": "latency spike rollback incident"}) + + assert response.status_code == status.HTTP_200_OK, response.data + assert response.data["results"] == [] + + def test_snippet_anchors_on_first_matching_token(self, api_key_client, workspace, project, create_user): + """With the phrase absent, the excerpt is anchored on the first query + token so the reader sees something they searched for.""" + filler = "z" * 400 + page = _make_page( + workspace, + project, + create_user, + name="Untitled", + content=f"{filler} a latency regression appeared. {filler} and later a spike followed.", + ) + + response = api_key_client.get(_url(workspace.slug), {"query": "latency spike"}) + + assert response.status_code == status.HTTP_200_OK, response.data + snippet = next(r for r in response.data["results"] if r["id"] == str(page.id))["snippet"] + assert "latency" in snippet.lower() + # 'spike' sits ~400 characters later, outside the 200-character budget. + assert "spike" not in snippet.lower() + + def test_snippet_anchor_follows_query_order(self, api_key_client, workspace, project, create_user): + """Reversing the keywords moves the anchor, proving the excerpt follows + query order rather than whichever keyword appears first in the page.""" + filler = "z" * 400 + page = _make_page( + workspace, + project, + create_user, + name="Untitled", + content=f"a latency regression appeared. {filler} and later a spike followed.", + ) + + def snippet_for(query): + response = api_key_client.get(_url(workspace.slug), {"query": query}) + assert response.status_code == status.HTTP_200_OK, response.data + return next(r for r in response.data["results"] if r["id"] == str(page.id))["snippet"].lower() + + forward = snippet_for("latency spike") + assert "latency regression" in forward and "spike" not in forward + + reverse = snippet_for("spike latency") + assert "spike followed" in reverse and "latency regression" not in reverse + + def test_too_many_keywords_returns_400(self, api_key_client, workspace, project, create_user): + """Each keyword adds two unindexable ILIKE predicates, so the keyword + count — which the caller picks for free — has to be bounded.""" + _make_page(workspace, project, create_user, name="Roadmap") + + at_limit = " ".join(f"term{i}" for i in range(PAGE_SEARCH_MAX_TOKENS)) + response = api_key_client.get(_url(workspace.slug), {"query": at_limit}) + assert response.status_code == status.HTTP_200_OK, response.data + + over_limit = " ".join(f"term{i}" for i in range(PAGE_SEARCH_MAX_TOKENS + 1)) + response = api_key_client.get(_url(workspace.slug), {"query": over_limit}) + assert response.status_code == status.HTTP_400_BAD_REQUEST, response.data + assert "error" in response.data + + def test_repeated_keywords_are_collapsed(self, api_key_client, workspace, project, create_user): + """AND is idempotent, so repeats must not count towards the limit or + multiply the scan — a single keyword repeated past the cap still works.""" + page = _make_page(workspace, project, create_user, name="Roadmap") + + repeated = " ".join(["roadmap"] * (PAGE_SEARCH_MAX_TOKENS * 3)) + response = api_key_client.get(_url(workspace.slug), {"query": repeated}) + + assert response.status_code == status.HTTP_200_OK, response.data + assert {r["id"] for r in response.data["results"]} == {str(page.id)} + + # Case-insensitive repeats collapse too. + mixed_case = api_key_client.get(_url(workspace.slug), {"query": "roadmap ROADMAP RoAdMaP"}) + assert mixed_case.status_code == status.HTTP_200_OK, mixed_case.data + assert {r["id"] for r in mixed_case.data["results"]} == {str(page.id)} + + def test_whitespace_only_query_returns_400(self, api_key_client, workspace, project): + """A query that is empty once tokenised is rejected, as before.""" + for value in (" ", "\t", "\n "): + response = api_key_client.get(_url(workspace.slug), {"query": value}) + assert response.status_code == status.HTTP_400_BAD_REQUEST, (value, response.data) + + def test_multi_keyword_results_ordered_by_updated_at_desc(self, api_key_client, workspace, project, create_user): + """Ordering is unchanged by tokenisation: most recently updated first.""" + content = "latency spike rollback incident notes" + first = _make_page(workspace, project, create_user, name="First", content=content) + second = _make_page(workspace, project, create_user, name="Second", content=content) + third = _make_page(workspace, project, create_user, name="Third", content=content) + + # Touch them in a known order; updated_at is auto_now. + for page in (first, third, second): + page.save() + + response = api_key_client.get(_url(workspace.slug), {"query": "latency spike rollback incident"}) + + assert response.status_code == status.HTTP_200_OK, response.data + assert [r["id"] for r in response.data["results"]] == [ + str(second.id), + str(third.id), + str(first.id), + ] + + def test_membership_scoping_non_member_sees_nothing(self, api_key_client, workspace, create_user, other_user): + # A project the caller is NOT a member of, holding a matching page. + foreign_project = _make_project(workspace, other_user, "FP", member=other_user) + _make_page(workspace, foreign_project, other_user, name="Secret Roadmap") + + response = api_key_client.get(_url(workspace.slug), {"query": "roadmap"}) + + assert response.status_code == status.HTTP_200_OK, response.data + assert response.data["results"] == [] + + def test_private_page_excluded_for_non_owner_but_visible_to_owner( + self, api_key_client, workspace, project, create_user, other_user + ): + # Private page owned by someone else, in a project the caller can access. + _make_page(workspace, project, other_user, name="Private Roadmap", access=Page.PRIVATE_ACCESS) + # Private page owned by the caller. + own_private = _make_page(workspace, project, create_user, name="My Private Roadmap", access=Page.PRIVATE_ACCESS) + # Public page, visible to any member. + public = _make_page(workspace, project, other_user, name="Public Roadmap", access=Page.PUBLIC_ACCESS) + + response = api_key_client.get(_url(workspace.slug), {"query": "roadmap"}) + + assert response.status_code == status.HTTP_200_OK, response.data + ids = {r["id"] for r in response.data["results"]} + assert ids == {str(own_private.id), str(public.id)} + + def test_project_filter(self, api_key_client, workspace, create_user): + project_a = _make_project(workspace, create_user, "PA", member=create_user) + project_b = _make_project(workspace, create_user, "PB", member=create_user) + page_a = _make_page(workspace, project_a, create_user, name="Roadmap A") + _make_page(workspace, project_b, create_user, name="Roadmap B") + + response = api_key_client.get(_url(workspace.slug), {"query": "roadmap", "projects": str(project_a.id)}) + + assert response.status_code == status.HTTP_200_OK, response.data + ids = {r["id"] for r in response.data["results"]} + assert ids == {str(page_a.id)} + # The reported project id is the accessible project the page belongs to. + assert response.data["results"][0]["project_id"] == str(project_a.id) + + def test_invalid_projects_filter_returns_400(self, api_key_client, workspace, project): + response = api_key_client.get(_url(workspace.slug), {"query": "roadmap", "projects": "not-a-uuid"}) + assert response.status_code == status.HTTP_400_BAD_REQUEST, response.data + + def test_archived_excluded_by_default_and_included_with_flag(self, api_key_client, workspace, project, create_user): + from django.utils import timezone + + active = _make_page(workspace, project, create_user, name="Active Roadmap") + archived = _make_page( + workspace, project, create_user, name="Archived Roadmap", archived_at=timezone.now().date() + ) + + # Default: archived excluded. + response = api_key_client.get(_url(workspace.slug), {"query": "roadmap"}) + assert response.status_code == status.HTTP_200_OK, response.data + assert {r["id"] for r in response.data["results"]} == {str(active.id)} + + # With archived=true both are returned. + response = api_key_client.get(_url(workspace.slug), {"query": "roadmap", "archived": "true"}) + assert response.status_code == status.HTTP_200_OK, response.data + assert {r["id"] for r in response.data["results"]} == {str(active.id), str(archived.id)} + + def test_result_shape_and_snippet(self, api_key_client, workspace, project, create_user): + parent = _make_page(workspace, project, create_user, name="Parent Page") + page = _make_page( + workspace, + project, + create_user, + name="Untitled", + content="The quarterly budget review covers spend across every team this period.", + ) + page.parent = parent + page.save() + + response = api_key_client.get(_url(workspace.slug), {"query": "budget review"}) + + assert response.status_code == status.HTTP_200_OK, response.data + result = next(r for r in response.data["results"] if r["id"] == str(page.id)) + assert set(result.keys()) == {"id", "name", "project_id", "parent_id", "updated_at", "snippet"} + assert result["project_id"] == str(project.id) + assert result["parent_id"] == str(parent.id) + assert "budget review" in result["snippet"].lower() + + def test_pagination(self, api_key_client, workspace, project, create_user): + created = {str(_make_page(workspace, project, create_user, name=f"Roadmap {i}").id) for i in range(3)} + + # First page of 2. + response = api_key_client.get(_url(workspace.slug), {"query": "roadmap", "per_page": 2}) + assert response.status_code == status.HTTP_200_OK, response.data + assert response.data["count"] == 2 + assert response.data["total_count"] == 3 + assert response.data["next_page_results"] is True + seen = {r["id"] for r in response.data["results"]} + + # Follow the cursor for the remainder. + response = api_key_client.get( + _url(workspace.slug), + {"query": "roadmap", "per_page": 2, "cursor": response.data["next_cursor"]}, + ) + assert response.status_code == status.HTTP_200_OK, response.data + assert response.data["count"] == 1 + assert response.data["next_page_results"] is False + seen |= {r["id"] for r in response.data["results"]} + + assert seen == created + + def test_inactive_membership_sees_nothing(self, api_key_client, workspace, create_user): + # The caller once belonged to the project but the membership is deactivated. + project = _make_project(workspace, create_user, "IA", member=create_user, is_active=False) + _make_page(workspace, project, create_user, name="Roadmap") + + response = api_key_client.get(_url(workspace.slug), {"query": "roadmap"}) + + assert response.status_code == status.HTTP_200_OK, response.data + assert response.data["results"] == [] + + def test_archived_project_pages_excluded(self, api_key_client, workspace, create_user): + from django.utils import timezone + + # Page lives in an archived project the caller is an active member of. + archived_project = _make_project(workspace, create_user, "AP", member=create_user, archived_at=timezone.now()) + _make_page(workspace, archived_project, create_user, name="Roadmap") + + # Excluded by default and even when archived pages are requested — the + # ?archived flag controls page archival, not project archival. + for params in ({"query": "roadmap"}, {"query": "roadmap", "archived": "true"}): + response = api_key_client.get(_url(workspace.slug), params) + assert response.status_code == status.HTTP_200_OK, response.data + assert response.data["results"] == [], params + + def test_cross_workspace_isolation(self, api_key_client, workspace, create_user): + # A second workspace the caller is fully a member of, with a matching page. + other_workspace = Workspace.objects.create(name="Other Workspace", owner=create_user, slug="other-workspace") + WorkspaceMember.objects.create(workspace=other_workspace, member=create_user, role=20) + other_project = _make_project(other_workspace, create_user, "OW", member=create_user) + _make_page(other_workspace, other_project, create_user, name="Roadmap") + + # Searching the first workspace must not surface the other workspace's page. + response = api_key_client.get(_url(workspace.slug), {"query": "roadmap"}) + assert response.status_code == status.HTTP_200_OK, response.data + assert response.data["results"] == [] + + def test_guest_without_view_all_sees_only_own_pages(self, api_key_client, workspace, create_user, other_user): + """A guest in a project that has not opted guests into full visibility + may only see the pages they own — the rule PageViewSet enforces.""" + project = _make_project(workspace, other_user, "GP", member=other_user) + project.guest_view_all_features = False + project.save() + ProjectMember.objects.create(project=project, member=create_user, role=5, is_active=True) + + _make_page(workspace, project, other_user, name="Roadmap Owned By Other") + own = _make_page(workspace, project, create_user, name="Roadmap Owned By Me") + + response = api_key_client.get(_url(workspace.slug), {"query": "roadmap"}) + + assert response.status_code == status.HTTP_200_OK, response.data + assert {r["id"] for r in response.data["results"]} == {str(own.id)} + + def test_guest_with_view_all_sees_project_pages(self, api_key_client, workspace, create_user, other_user): + """When the project opts guests into full visibility, a guest sees the + project's public pages like any other member.""" + project = _make_project(workspace, other_user, "GV", member=other_user) + project.guest_view_all_features = True + project.save() + ProjectMember.objects.create(project=project, member=create_user, role=5, is_active=True) + + others = _make_page(workspace, project, other_user, name="Roadmap Owned By Other") + own = _make_page(workspace, project, create_user, name="Roadmap Owned By Me") + + response = api_key_client.get(_url(workspace.slug), {"query": "roadmap"}) + + assert response.status_code == status.HTTP_200_OK, response.data + assert {r["id"] for r in response.data["results"]} == {str(others.id), str(own.id)} + + def test_unknown_workspace_returns_400(self, api_key_client, workspace, project, create_user): + """An unrecognised slug is reported, not silently returned as an empty + result set — matching the other token-API workspace endpoints.""" + _make_page(workspace, project, create_user, name="Roadmap") + + response = api_key_client.get(_url("no-such-workspace"), {"query": "roadmap"}) + + assert response.status_code == status.HTTP_400_BAD_REQUEST, response.data + assert "error" in response.data + + def test_error_responses_use_a_consistent_envelope(self, api_key_client, workspace, project, create_user): + """Every 400 from this endpoint uses {"error": ...}, including the ones + the paginator raises (which are DRF's {"detail": ...} by default).""" + _make_page(workspace, project, create_user, name="Roadmap") + + cases = [ + {}, # missing query + {"query": "roadmap", "projects": "not-a-uuid"}, + {"query": "roadmap", "per_page": 0}, + {"query": "roadmap", "per_page": 101}, # above this endpoint's max + {"query": "roadmap", "cursor": "not-a-cursor"}, + ] + for params in cases: + response = api_key_client.get(_url(workspace.slug), params) + assert response.status_code == status.HTTP_400_BAD_REQUEST, (params, response.data) + assert "error" in response.data, (params, response.data) + assert "detail" not in response.data, (params, response.data) + + def test_heavy_page_columns_are_not_loaded(self, api_key_client, workspace, project, create_user): + """The response only needs identity fields plus the stripped text, so the + large description columns must never reach the SELECT. + + Asserted against the SQL the request actually runs — checking a queryset + built here instead would only prove that Django's .only() works, and + would keep passing if the endpoint stopped deferring anything.""" + page = _make_page(workspace, project, create_user, name="Roadmap", content="Some body text") + page.description_json = {"big": "payload"} + page.save() + + with CaptureQueriesContext(connection) as captured: + response = api_key_client.get(_url(workspace.slug), {"query": "roadmap"}) + + assert response.status_code == status.HTTP_200_OK, response.data + assert {r["id"] for r in response.data["results"]} == {str(page.id)} + + page_selects = [ + entry["sql"] + for entry in captured.captured_queries + if entry["sql"].lstrip().upper().startswith("SELECT") and '"pages"."description_stripped"' in entry["sql"] + ] + assert page_selects, "no page SELECT captured — the assertions below would be vacuous" + + for sql in captured.captured_queries: + for heavy_column in ("description_html", "description_binary", "description_json"): + assert heavy_column not in sql["sql"], f"{heavy_column} was selected: {sql['sql']}" + + def test_page_linked_to_project_in_another_workspace_is_not_exposed( + self, api_key_client, workspace, create_user, other_user + ): + """Access is decided strictly within the searched workspace: a stray + ProjectPage row pointing at a project elsewhere must not grant access.""" + # Caller is NOT a member of the project holding the page in this workspace. + foreign_project = _make_project(workspace, other_user, "FW", member=other_user) + page = _make_page(workspace, foreign_project, other_user, name="Roadmap") + + # ...but is an admin of a project in a different workspace, which is then + # linked to the same page (the corrupted/cross-workspace row). + other_workspace = Workspace.objects.create(name="Other WS", owner=create_user, slug="other-ws") + WorkspaceMember.objects.create(workspace=other_workspace, member=create_user, role=20) + elsewhere = _make_project(other_workspace, create_user, "EW", member=create_user) + ProjectPage.objects.create(page=page, project=elsewhere, workspace=other_workspace) + + response = api_key_client.get(_url(workspace.slug), {"query": "roadmap"}) + + assert response.status_code == status.HTTP_200_OK, response.data + assert response.data["results"] == [] + + def test_per_page_zero_returns_400(self, api_key_client, workspace, project, create_user): + _make_page(workspace, project, create_user, name="Roadmap") + response = api_key_client.get(_url(workspace.slug), {"query": "roadmap", "per_page": 0}) + assert response.status_code == status.HTTP_400_BAD_REQUEST, response.data + + def test_per_page_over_max_returns_400(self, api_key_client, workspace, project, create_user): + _make_page(workspace, project, create_user, name="Roadmap") + response = api_key_client.get(_url(workspace.slug), {"query": "roadmap", "per_page": 1000}) + assert response.status_code == status.HTTP_400_BAD_REQUEST, response.data + + def test_snippet_alignment_with_unicode(self, api_key_client, workspace, project, create_user): + # A leading character that expands when lowercased ("İ".lower() has length 2) + # must not shift the snippet window off the match. + page = _make_page( + workspace, + project, + create_user, + name="Untitled", + content="İ office note. The budget review happens on Friday afternoon here.", + ) + + response = api_key_client.get(_url(workspace.slug), {"query": "budget review"}) + assert response.status_code == status.HTTP_200_OK, response.data + result = next(r for r in response.data["results"] if r["id"] == str(page.id)) + assert "budget review" in result["snippet"].lower() diff --git a/apps/api/plane/tests/unit/utils/test_page_search.py b/apps/api/plane/tests/unit/utils/test_page_search.py new file mode 100644 index 00000000000..eb0287c5543 --- /dev/null +++ b/apps/api/plane/tests/unit/utils/test_page_search.py @@ -0,0 +1,292 @@ +# Copyright (c) 2023-present Plane Software, Inc. and contributors +# SPDX-License-Identifier: AGPL-3.0-only +# See the LICENSE file for details. + +"""Unit tests for the page search snippet helper.""" + +import re + +import pytest + +from plane.utils import page_search +from plane.utils.page_search import ( + SNIPPET_LEAD_CHARS, + SNIPPET_MAX_LENGTH, + build_page_snippet, +) + +_ELLIPSIS = "…" + + +def _reference_snippet(stripped_text, query, max_length=SNIPPET_MAX_LENGTH, lead=SNIPPET_LEAD_CHARS): + """Straightforward implementation used as an oracle: normalize the whole + document, then excerpt around the match. The shipped version must agree with + this while only normalizing a bounded window.""" + if not stripped_text or max_length <= 0: + return "" + text = re.sub(r"\s+", " ", stripped_text).strip() + if not text: + return "" + + tokens = [re.escape(token) for token in query.split()] + match = re.search(r"\s+".join(tokens), text, re.IGNORECASE) if tokens else None + if match is None: + # Phrase absent: anchor on the first query token present in the text. + for token in tokens: + match = re.search(token, text, re.IGNORECASE) + if match is not None: + break + + if match is None: + if len(text) <= max_length: + return text + return text[: max_length - len(_ELLIPSIS)] + _ELLIPSIS + + start = max(0, match.start() - lead) + prefix = _ELLIPSIS if start > 0 else "" + budget = max_length - len(prefix) + if start + budget < len(text): + budget -= len(_ELLIPSIS) + if budget <= 0: + return _ELLIPSIS[:max_length] + end = min(len(text), start + budget) + suffix = _ELLIPSIS if end < len(text) else "" + return prefix + text[start:end] + suffix + + +@pytest.mark.unit +class TestBuildPageSnippet: + def test_empty_or_none_text_returns_empty(self): + assert build_page_snippet(None, "anything") == "" + assert build_page_snippet("", "anything") == "" + assert build_page_snippet(" \n\t ", "anything") == "" + + def test_match_at_start_has_no_leading_ellipsis(self): + text = "Budget review notes for the quarter." + snippet = build_page_snippet(text, "budget") + assert not snippet.startswith("…") + assert "budget" in snippet.lower() + + def test_match_in_middle_is_surrounded_by_context(self): + text = "x" * 300 + " needle " + "y" * 300 + snippet = build_page_snippet(text, "needle") + assert "needle" in snippet + assert snippet.startswith("…") + assert snippet.endswith("…") + # Leading context is bounded by SNIPPET_LEAD_CHARS (plus the ellipsis char). + assert snippet.index("needle") <= SNIPPET_LEAD_CHARS + 1 + + def test_length_is_capped(self): + text = "word " * 500 + snippet = build_page_snippet(text, "word") + assert len(snippet) <= SNIPPET_MAX_LENGTH + + @pytest.mark.parametrize( + "text,query", + [ + ("word " * 500, "word"), # match at the start + ("x" * 300 + " needle " + "y" * 300, "needle"), # match in the middle + ("a" * 500, "absent-term"), # no content match + ("b" * 500 + " tail", "tail"), # match at the very end + ], + ) + def test_ellipses_are_paid_for_out_of_the_budget(self, text, query): + """The ellipsis markers must never push the snippet past max_length.""" + assert len(build_page_snippet(text, query)) <= SNIPPET_MAX_LENGTH + + @pytest.mark.parametrize("max_length", [-5, -1, 0]) + def test_non_positive_max_length_returns_empty(self, max_length): + """A non-positive budget must yield nothing. Previously the negative + slice bound (text[:max_length - 1]) returned nearly the whole document.""" + text = "The quarterly budget review covers spend across every team." + assert build_page_snippet(text, "budget", max_length=max_length) == "" + assert build_page_snippet(text, "absent-term", max_length=max_length) == "" + + def test_phrase_is_preferred_when_present(self): + """When the whole phrase occurs it stays the anchor, even though the + first token appears on its own earlier in the document — dropping the + phrase branch would anchor on that earlier lone occurrence instead.""" + text = "latency alone here. " + "z" * 400 + " latency spike together." + snippet = build_page_snippet(text, "latency spike") + assert "latency spike" in snippet.lower() + assert "latency alone here" not in snippet.lower() + + def test_token_fallback_follows_query_order_not_document_order(self): + """The fallback anchors on the first token of the QUERY, not on whichever + token happens to appear first in the document.""" + text = "latency arrived early. " + "z" * 400 + " a spike arrived late." + + forward = build_page_snippet(text, "latency spike").lower() + assert "latency arrived early" in forward + assert "spike" not in forward + + # Same document, reversed query: the later occurrence now wins. + reversed_query = build_page_snippet(text, "spike latency").lower() + assert "spike arrived late" in reversed_query + assert "latency arrived early" not in reversed_query + + def test_repeated_tokens_are_collapsed_into_one_fallback_scan(self): + """Each fallback token costs a pass over the document, so repeats must be + dropped. Asserted on the compiled pattern list — equal output alone would + hold with or without the de-duplication.""" + phrase, token_patterns = page_search._query_patterns("spike Spike SPIKE latency") + assert [p.pattern for p in token_patterns] == ["spike", "latency"] + # The phrase keeps every occurrence: it is the literal text sought. + assert phrase.pattern == r"spike\s+Spike\s+SPIKE\s+latency" + + text = "z" * 400 + " a spike happened." + assert build_page_snippet(text, "spike spike spike") == build_page_snippet(text, "spike") + + def test_anchors_on_first_token_when_phrase_absent(self): + """Tokenised search matches pages whose tokens sit in different + sentences, so the phrase is often absent; anchor on the first token.""" + text = "intro " + "z" * 400 + " a latency problem. " + "y" * 400 + " and then a spike." + snippet = build_page_snippet(text, "latency spike") + assert "latency" in snippet.lower() + # 'spike' is ~400 characters further on, well outside a 200-char excerpt. + assert "spike" not in snippet.lower() + + def test_anchor_falls_through_to_a_later_token(self): + """If the first token is absent the next matching one anchors it.""" + text = "z" * 400 + " the spike happened overnight." + snippet = build_page_snippet(text, "latency spike") + assert "spike" in snippet.lower() + + def test_no_token_present_excerpts_from_the_start(self): + """A page that matched on its name alone still gets a preview.""" + text = "Body text that shares nothing with the query at all." + assert build_page_snippet(text, "latency spike").startswith("Body text") + + def test_single_token_behaviour_unchanged(self): + text = "x" * 300 + " needle " + "y" * 300 + assert build_page_snippet(text, "needle") == _reference_snippet(text, "needle") + + def test_match_is_found_across_whitespace_runs(self): + """The document is only normalized in a window, so the search itself runs + against raw text where the query's words may straddle newlines.""" + text = "intro\n\nThe budget\n review happens Friday." + snippet = build_page_snippet(text, "budget review") + assert "budget review" in snippet.lower() + assert "\n" not in snippet + + def test_only_a_bounded_window_is_normalized(self, monkeypatch): + """A large document must not be rewritten in full for a single snippet. + + Checked by observing what the whitespace normalizer is handed rather than + by elapsed time, which would depend on CI hardware and load.""" + real_pattern = page_search._WHITESPACE_RE + normalized_sizes = [] + + class RecordingPattern: + def sub(self, repl, string): + normalized_sizes.append(len(string)) + return real_pattern.sub(repl, string) + + monkeypatch.setattr(page_search, "_WHITESPACE_RE", RecordingPattern()) + + text = "a" * 5_000_000 + " needle tail" + snippet = build_page_snippet(text, "needle") + + assert "needle" in snippet + assert len(snippet) <= SNIPPET_MAX_LENGTH + assert normalized_sizes, "the normalizer was never called" + # With the default budget the window is ~1.3 KB; the bound below is loose + # enough to survive tuning but still orders of magnitude under the 5 MB + # document, so collapsing everything would fail the test. + assert max(normalized_sizes) <= 10_000, ( + f"normalizer received {max(normalized_sizes)} characters — the whole document was likely collapsed" + ) + + @pytest.mark.parametrize( + "text,query", + [ + ("word " * 500, "word"), + ("x" * 300 + " needle " + "y" * 300, "needle"), + ("a" * 500, "absent-term"), + ("b" * 500 + " tail", "tail"), + (" \n\n leading whitespace then needle here", "needle"), + ("needle at the very start of the document", "needle"), + ("trailing match needle \n\n ", "needle"), + ("\n\n".join(["para " * 40] * 30) + " needle", "needle"), + ("tiny", "tiny"), + ("no match at all here", "absent"), + ("İ" * 100 + " needle", "needle"), + # Multi-token queries: phrase present, phrase absent (anchors on the + # first token), first token absent, and no token present at all. + ("z" * 300 + " latency spike " + "y" * 300, "latency spike"), + ("z" * 300 + " latency here " + "y" * 300 + " spike there", "latency spike"), + ("z" * 300 + " only a spike here " + "y" * 300, "latency spike"), + ("nothing relevant in this document", "latency spike"), + ("latency at the very start " + "y" * 400 + " spike", "latency spike"), + ("a\n\nlatency\n\n b \n\n spike", "latency spike"), + # Phrase present but the first token also occurs alone earlier. + ("latency alone here. " + "z" * 400 + " latency spike together.", "latency spike"), + # Query order and document order disagree. + ("latency arrived early. " + "z" * 400 + " a spike arrived late.", "spike latency"), + # Whitespace runs longer than the window at the document edges: the + # window starts/ends inside them, so stripping has to key on content + # rather than on position or a spurious ellipsis appears. + ("\xa0" * 473 + "A" * 39 + " latency spike notes " + "b" * 300, "latency spike"), + ("\xa0" * 474 + "A" * 38 + " latency spike notes " + "b" * 300, "latency spike"), + (" " * 600 + "latency spike here", "latency spike"), + ("latency spike here" + " " * 600, "latency spike"), + ("\n" * 500 + "x" * 100 + " latency spike " + "y" * 100 + "\t" * 500, "latency spike"), + ("\t" * 900 + "only trailing", "absent"), + ], + ) + def test_matches_naive_full_normalization(self, text, query): + """The windowed implementation must agree with the obvious one that + normalizes the entire document up front.""" + assert build_page_snippet(text, query) == _reference_snippet(text, query) + + @pytest.mark.parametrize("max_length", range(1, 12)) + def test_small_max_length_stays_within_budget(self, max_length): + """Tiny budgets degrade to a marker rather than overflowing, for a match + at the start, a match far into the text, and no match at all.""" + long_text = "z" * 400 + cases = [ + ("budget review is here, " + long_text, "budget"), # match at start + (long_text + " budget review", "budget"), # match far in (leading ellipsis) + (long_text, "absent-term"), # no match + ] + for text, query in cases: + snippet = build_page_snippet(text, query, max_length=max_length) + assert len(snippet) <= max_length, (max_length, query, repr(snippet)) + + def test_no_content_match_excerpts_from_start(self): + text = "The introduction paragraph explains the overall context here." + snippet = build_page_snippet(text, "term-not-present") + assert snippet.startswith("The introduction") + + def test_empty_query_excerpts_from_start(self): + text = "Some leading content that should be previewed." + snippet = build_page_snippet(text, "") + assert snippet.startswith("Some leading content") + + def test_whitespace_is_collapsed(self): + text = "alpha\n\n beta\t\tgamma" + snippet = build_page_snippet(text, "beta") + assert "\n" not in snippet + assert " " not in snippet + + def test_short_text_has_no_ellipsis(self): + text = "tiny doc" + snippet = build_page_snippet(text, "tiny") + assert snippet == "tiny doc" + + def test_case_insensitive_match(self): + text = "The Budget Review is scheduled." + snippet = build_page_snippet(text, "budget review") + assert "Budget Review" in snippet + + def test_unicode_expanding_char_keeps_match_in_window(self): + # "İ".lower() == "i̇" (length 2); indexing text.lower() would drift the + # window right and could push the match out. The match must survive. + text = "İ " * 60 + "needle tail" + snippet = build_page_snippet(text, "needle") + assert "needle" in snippet + + def test_regex_metacharacters_in_query_are_literal(self): + text = "Price is $5 (approx) for the item." + snippet = build_page_snippet(text, "$5 (approx)") + assert "$5 (approx)" in snippet diff --git a/apps/api/plane/utils/page_search.py b/apps/api/plane/utils/page_search.py new file mode 100644 index 00000000000..d99d9038842 --- /dev/null +++ b/apps/api/plane/utils/page_search.py @@ -0,0 +1,220 @@ +# Copyright (c) 2023-present Plane Software, Inc. and contributors +# SPDX-License-Identifier: AGPL-3.0-only +# See the LICENSE file for details. + +"""Helpers for the page search API. + +Pages carry a maintained ``description_stripped`` column (plain text derived +from ``description_html`` on save). Search results include a short excerpt of +that text centered on the query match so callers can preview *why* a page +matched without downloading the full document. +""" + +import re + +# Snippet sizing. +# +# ``SNIPPET_MAX_LENGTH`` keeps each result light — a search response can carry a +# full page of hits, and pages can hold very large documents, so the excerpt is +# capped rather than returning the whole stripped body. 200 characters is enough +# to show the match with surrounding context on a single line while keeping the +# payload small. +# +# ``SNIPPET_LEAD_CHARS`` is how much preceding context to include so the matched +# term is not flush against the left edge of the excerpt; the remaining budget +# after the lead shows the match and the text that follows it. +SNIPPET_MAX_LENGTH = 200 +SNIPPET_LEAD_CHARS = 40 + +_ELLIPSIS = "…" +_WHITESPACE_RE = re.compile(r"\s+") +_NON_WHITESPACE_RE = re.compile(r"\S") + +# Smallest slice of the raw document to collapse whitespace in. Collapsing only +# ever shrinks text, so a window a few times the display budget supplies all the +# context a snippet can use for any realistic document; _normalized_window grows +# it when a pathologically whitespace-heavy document needs more. +_MIN_WINDOW_CHARS = 512 + + +def _query_patterns(query: str) -> tuple[re.Pattern | None, list[re.Pattern]]: + """Compile ``query`` into a whole-phrase pattern and its per-token patterns. + + Both allow any run of whitespace where the query has whitespace: the document + is only whitespace-normalized in a window around the match, so the search + itself runs against the raw text, where the query's words may be separated by + newlines or runs of spaces. + + For a single-token query the phrase pattern and the sole token pattern are + equivalent, so behaviour is unchanged. + """ + tokens = query.split() + if not tokens: + return None, [] + + # The phrase keeps every token, repeats included, since that is the literal + # text being looked for. The fallback list is de-duplicated: scanning the + # document again for a keyword already searched cannot find a new anchor, and + # each rescan costs a pass over the whole document. + phrase = re.compile(r"\s+".join(re.escape(token) for token in tokens), re.IGNORECASE) + + seen = set() + distinct = [] + for token in tokens: + folded = token.casefold() + if folded not in seen: + seen.add(folded) + distinct.append(token) + return phrase, [re.compile(re.escape(token), re.IGNORECASE) for token in distinct] + + +def _locate_match(raw: str, query: str) -> re.Match | None: + """Find where to anchor the snippet. + + The whole phrase is preferred, since that is the most informative excerpt. + Search is tokenised (every token must appear somewhere, in the name or the + body), so a page can match without containing the phrase at all — and its + tokens may even live in different sentences. In that case anchor on the first + query token that appears in this text, so the excerpt still shows the reader + something they searched for rather than the top of the document. + """ + phrase, token_patterns = _query_patterns(query) + if phrase is None: + return None + + match = phrase.search(raw) + if match is not None: + return match + + for pattern in token_patterns: + match = pattern.search(raw) + if match is not None: + return match + return None + + +def _normalized_window(raw: str, anchor: int, need_left: int, need_right: int) -> tuple[str, int, bool, bool]: + """Collapse whitespace in a bounded window of ``raw`` around ``anchor``. + + Returns the normalized window, the anchor's offset within it, and whether + real content was left outside the window on either side. + + The window grows until it holds all the context the caller can use, so the + excerpt matches what normalizing the entire document would produce without + paying to rewrite the entire document for every search hit. ``anchor`` + always sits on a non-whitespace character (or at 0), so no whitespace run + straddles it and the two halves can be collapsed independently. + """ + left_span = max(need_left * 4, _MIN_WINDOW_CHARS) + right_span = max(need_right * 4, _MIN_WINDOW_CHARS) + + while True: + start = max(0, anchor - left_span) + end = min(len(raw), anchor + right_span) + + left = _WHITESPACE_RE.sub(" ", raw[start:anchor]) + right = _WHITESPACE_RE.sub(" ", raw[anchor:end]) + + # Whether real content — not merely characters — lies outside the window. + # Each search is bounded to the discarded region and stops at the first + # non-whitespace character, so this does not rescan the document. + more_before = start > 0 and _NON_WHITESPACE_RE.search(raw, 0, start) is not None + more_after = end < len(raw) and _NON_WHITESPACE_RE.search(raw, end) is not None + + # Reproduce the document-level strip(). It has to key on content rather + # than on window position: a window that begins inside a long run of + # leading whitespace has characters before it but no content, and the + # whole-document form would have stripped that run away. + if not more_before: + if left: + left = left.lstrip() + else: + right = right.lstrip() + if not more_after: + if right: + right = right.rstrip() + else: + left = left.rstrip() + + # Sufficiency is judged the same way: there is nothing more to gather on a + # side that holds no further content, however many characters remain. + enough_left = not more_before or len(left) >= need_left + enough_right = not more_after or len(right) >= need_right + + if enough_left and enough_right: + return left + right, len(left), more_before, more_after + + left_span *= 4 + right_span *= 4 + + +def build_page_snippet( + stripped_text: str | None, + query: str, + max_length: int = SNIPPET_MAX_LENGTH, + lead: int = SNIPPET_LEAD_CHARS, +) -> str: + """Return a short single-line excerpt of ``stripped_text``. + + The excerpt is anchored, in order of preference, on the first + (case-insensitive) occurrence of the whole query phrase, then on the first + query token that appears in the text — search matches pages whose tokens are + scattered across different sentences, so the phrase is often absent. Failing + both (the page matched on its name only, or no query was supplied) the + excerpt is taken from the start of the text. ``lead`` characters of preceding + context are included, and an ellipsis marks either side that was truncated. + + ``max_length`` bounds the WHOLE returned string: the ellipsis markers are + paid for out of that budget, never appended on top of it, so a caller sizing + its layout on ``max_length`` is never handed a longer string. A budget too + small to hold any text yields just a marker, and a non-positive budget + yields an empty string — never a longer fallback. + """ + if not stripped_text: + return "" + + # A non-positive budget has no room for anything. Guarding here keeps the + # slice arithmetic below from going negative, which would otherwise turn + # text[:max_length - 1] into a near-complete copy of the document. + if max_length <= 0: + return "" + + # Locate the match in the raw text: scanning is cheap, whereas collapsing + # whitespace across a large document allocates a full copy of it per result. + match = _locate_match(stripped_text, query) + anchor = match.start() if match else 0 + + text, match_pos, more_before, more_after = _normalized_window( + stripped_text, + anchor, + need_left=lead if match else 0, + need_right=max_length, + ) + if not text: + return "" + + if match is None: + # No content match: excerpt from the start of the document. + if not more_after and len(text) <= max_length: + return text + # Reserve room for the trailing ellipsis inside the budget. + return text[: max_length - len(_ELLIPSIS)] + _ELLIPSIS + + start = max(0, match_pos - lead) + prefix = _ELLIPSIS if start > 0 or more_before else "" + + # Whatever the leading ellipsis costs comes out of the budget, not on top of it. + budget = max_length - len(prefix) + if start + budget < len(text) or more_after: + # The excerpt will not reach the end of the text, so a trailing ellipsis + # is needed — reserve its room before slicing. + budget -= len(_ELLIPSIS) + + if budget <= 0: + # The markers alone exhaust the budget; show a single one rather than + # letting a negative slice bound run backwards through the text. + return _ELLIPSIS[:max_length] + + end = min(len(text), start + budget) + suffix = _ELLIPSIS if end < len(text) or more_after else "" + return prefix + text[start:end] + suffix diff --git a/apps/api/plane/utils/paginator.py b/apps/api/plane/utils/paginator.py index 2082041f1ac..600727df65f 100644 --- a/apps/api/plane/utils/paginator.py +++ b/apps/api/plane/utils/paginator.py @@ -646,6 +646,13 @@ def get_per_page(self, request, default_per_page=1000, max_per_page=1000): except ValueError: raise ParseError(detail="Invalid per_page parameter.") + # A non-positive page size would otherwise reach the paginator as the + # SQL limit, where limit=0 raises ZeroDivisionError (count / limit) and + # negative values produce an unsupported negative slice — both surfacing + # as an opaque HTTP 500. Reject it here as a 400 for every paginated view. + if per_page < 1: + raise ParseError(detail="Invalid per_page value. Must be a positive integer.") + max_per_page = max(max_per_page, default_per_page) if per_page > max_per_page: raise ParseError(detail=f"Invalid per_page value. Cannot exceed {max_per_page}.")