SPARQL conformance commands

pyproject.toml

@@ -1,5 +1,5 @@
 [build-system]
-requires = ["setuptools>=61.0", "wheel"]
+requires = ["setuptools>=64.0", "wheel"]
 build-backend = "setuptools.build_meta"
 
 [project]
@@ -27,6 +27,7 @@ Github = "https://github.com/ad-freiburg/qlever"
 
 [project.scripts]
 "qlever" = "qlever.qlever_main:main"
+"sparql_conformance" = "qlever.qlever_main:main"
 
 [tool.setuptools]
 license-files = ["LICENSE"]

src/qlever/__init__.py

@@ -16,7 +16,7 @@ def snake_to_camel(str):
 
 ENGINE_NAMES = {
     "qlever": "QLever",
-    "qmdb": "MillenniumDB",
+    "sparql_conformance": "SPARQL Conformance",
 }
 # Default engine_name = script_name without starting 'q' and capitalized
 engine_name = ENGINE_NAMES.get(script_name, script_name[1:].capitalize())

src/qlever/commands/index.py

@@ -178,7 +178,7 @@ def get_input_options_for_json(self, args) -> str:
         # Return the concatenated command-line options.
         return " ".join(input_options)
 
-    def execute(self, args) -> bool:
+    def execute(self, args, called_from_conformance_test = False) -> bool:
         # The mandatory part of the command line (specifying the input, the
         # basename of the index, and the settings file). There are two ways
         # to specify the input: via a single stream or via multiple streams.
@@ -278,15 +278,16 @@ def execute(self, args) -> bool:
                 return False
 
         # Check if all of the input files exist.
-        for pattern in shlex.split(args.input_files):
-            if len(glob.glob(pattern)) == 0:
-                log.error(f'No file matching "{pattern}" found')
-                log.info("")
-                log.info(
-                    "Did you call `qlever get-data`? If you did, check "
-                    "GET_DATA_CMD and INPUT_FILES in the QLeverfile"
-                )
-                return False
+        if not called_from_conformance_test:
+            for pattern in shlex.split(args.input_files):
+                if len(glob.glob(pattern)) == 0:
+                    log.error(f'No file matching "{pattern}" found')
+                    log.info("")
+                    log.info(
+                        "Did you call `qlever get-data`? If you did, check "
+                        "GET_DATA_CMD and INPUT_FILES in the QLeverfile"
+                    )
+                    return False
 
         # Check if index files (name.index.*) already exist.
         existing_index_files = get_existing_index_files(args.name)
@@ -325,7 +326,7 @@ def execute(self, args) -> bool:
 
         # Run the index command.
         try:
-            run_command(index_cmd, show_output=True)
+            run_command(index_cmd, show_output=not called_from_conformance_test)
         except Exception as e:
             log.error(f"Building the index failed: {e}")
             return False

src/qlever/commands/query.py

@@ -15,6 +15,7 @@ class QueryCommand(QleverCommand):
     """
 
     def __init__(self):
+        self.query_output = ""
         self.predefined_queries = {
             "all-predicates": (
                 "SELECT (?p AS ?predicate) (COUNT(?p) AS ?count) "
@@ -84,7 +85,7 @@ def additional_arguments(self, subparser) -> None:
             help="Do not print the (end-to-end) time taken",
         )
 
-    def execute(self, args) -> bool:
+    def execute(self, args, called_from_conformance_test = False) -> bool:
         # Use a predefined query if requested.
         if args.predefined_query:
             args.query = self.predefined_queries[args.predefined_query]
@@ -105,6 +106,11 @@ def execute(self, args) -> bool:
             )
         else:
             curl_cmd_additions = ""
+        query_type = "query="
+        if called_from_conformance_test:
+            curl_cmd_additions += f" -w '\\nHTTP_STATUS:%{{http_code}}'"
+            query_type = args.content_type
+            curl_cmd_additions += f" --data-urlencode access-token={shlex.quote(args.access_token)}"
 
         # Show what the command will do.
         sparql_endpoint = (
@@ -115,7 +121,7 @@ def execute(self, args) -> bool:
         curl_cmd = (
             f"curl -s {sparql_endpoint}"
             f' -H "Accept: {args.accept}"'
-            f" --data-urlencode query={shlex.quote(args.query)}"
+            f" --data-urlencode {query_type}{shlex.quote(args.query)}"
             f"{curl_cmd_additions}"
         )
         self.show(curl_cmd, only_show=args.show)
@@ -125,7 +131,10 @@ def execute(self, args) -> bool:
         # Launch query.
         try:
             start_time = time.time()
-            run_command(curl_cmd, show_output=True)
+            if called_from_conformance_test:
+                self.query_output = run_command(curl_cmd, return_output=True)
+            else:
+                run_command(curl_cmd, show_output=True)
             time_msecs = round(1000 * (time.time() - start_time))
             if not args.no_time and args.log_level != "NO_LOG":
                 log.info("")

src/qlever/commands/start.py

@@ -166,7 +166,7 @@ def additional_arguments(self, subparser) -> None:
             "(default: run in the background with `nohup`)",
         )
 
-    def execute(self, args) -> bool:
+    def execute(self, args, called_from_conformance_test = False) -> bool:
         # Kill existing server with the same name if so desired.
         #
         # TODO: This is currently disabled because I never used it once over
@@ -267,8 +267,9 @@ def execute(self, args) -> bool:
                 f" (Ctrl-C stops following the log, but NOT the server)"
             )
         log.info("")
-        tail_cmd = f"exec tail -f {args.name}.server-log.txt"
-        tail_proc = subprocess.Popen(tail_cmd, shell=True)
+        if not called_from_conformance_test:
+            tail_cmd = f"exec tail -f {args.name}.server-log.txt"
+            tail_proc = subprocess.Popen(tail_cmd, shell=True)
         while not is_qlever_server_alive(endpoint_url):
             time.sleep(1)
 
@@ -288,7 +289,7 @@ def execute(self, args) -> bool:
                 return False
 
         # Kill the tail process. NOTE: `tail_proc.kill()` does not work.
-        if not args.run_in_foreground:
+        if not args.run_in_foreground and not called_from_conformance_test:
             tail_proc.terminate()
 
         # Execute the warmup command.

src/qlever/commands/status.py

@@ -3,6 +3,7 @@
 import psutil
 
 from qlever.command import QleverCommand
+from qlever.log import log
 from qlever.util import show_process_info
 
 
@@ -46,5 +47,5 @@ def execute(self, args) -> bool:
             if process_shown:
                 num_processes_found += 1
         if num_processes_found == 0:
-            print("No processes found")
+            log.error("No processes found")
         return True

src/qlever/qleverfile.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import json
 import re
 import socket
 import subprocess
@@ -21,6 +22,79 @@ class Qleverfile:
     Qleverfile + functions for parsing.
     """
 
+    @staticmethod
+    def get_conformance_arguments(arg):
+        """
+        Define all possible parameters for conformance checks.
+        """
+        args = {}
+        args["name"] = arg(
+            "--name",
+            type=str,
+            required=True,
+            help="Name of the result file of the conformance check.",
+        )
+        args["port"] = arg(
+            "--port",
+            type=str,
+            required=True,
+            help="Port which will be used for the SPARQL sever.",
+        )
+        args["graph_store"] = arg(
+            "--graph-store",
+            type=str,
+            required=True,
+            help="Name of the graph store endpoint used for graph store protocol tests.",
+        )
+        args["testsuite_dir"] = arg(
+            "--testsuite-dir",
+            type=str,
+            default=None,
+            help="Path to the directory of the testsuite.",
+        )
+        args["type_alias"] = arg(
+            "--type-alias",
+            type=json.loads,
+            required=False,
+            help=("Type mismatches that will be considered intended."
+                  "ex. \"[['http://www.w3.org/2001/XMLSchema#integer', "
+                  "'http://www.w3.org/2001/XMLSchema#int']..."
+                  "['http://www.w3.org/2001/XMLSchema#float',"
+                  "'http://www.w3.org/2001/XMLSchema#double']]\""
+            ),
+        )
+        args["engine"] = arg(
+            "--engine",
+            type=str,
+            choices=["qlever", "qlever-binaries"],# "mdb", "oxigraph"],
+            default="docker",
+            help="Which system to use to run the tests in"
+        )
+        args["exclude"] = arg(
+            "--exclude",
+            type=lambda s: s.split(","),
+            default=[],
+            help=("Tests (names) or test groups to exclude from the run."
+                  "ex. service,entailment,POST - existing graph"
+            )
+        )
+        args["include"] = arg(
+            "--include",
+            type=lambda s: s.split(","),
+            default=None,
+            help=("Tests (names) or test groups to include in the run."
+                  "ex. service,entailment,POST - existing graph"
+            )
+        )
+        args["binaries_directory"] = arg(
+            "--binaries-directory",
+            type=str,
+            required=False,
+            help="Path to the directory of the IndexBuilderMain and ServerMain binaries.",
+            default=""
+        )
+        return args
+
     @staticmethod
     def all_arguments():
         """
@@ -41,6 +115,11 @@ def arg(*args, **kwargs):
         server_args = all_args["server"] = {}
         runtime_args = all_args["runtime"] = {}
         ui_args = all_args["ui"] = {}
+        all_args["conformance"] = Qleverfile.get_conformance_arguments(arg)
+        qlever_binaries_args = all_args["qlever_binaries"] = {}
+        qlever_args = all_args["qlever"] = {}
+        oxigraph_args = all_args["oxigraph"] = {}
+        conformance_ui_args = all_args["conformance_ui"] = {}
 
         data_args["name"] = arg(
             "--name", type=str, required=True, help="The name of the dataset"
@@ -365,6 +444,34 @@ def arg(*args, **kwargs):
             help="The name of the container used for `qlever ui`",
         )
 
+
+        qlever_args["qlever_image"] = arg(
+            "--qlever-image",
+            type=str,
+            default="docker.io/adfreiburg/qlever",
+            help="The name of the image when running in a container",
+        )
+
+        oxigraph_args["oxigraph_image"] = arg(
+            "--oxigraph-image",
+            type=str,
+            default="ghcr.io/oxigraph/oxigraph",
+            help="The name of the image when running in a container",
+        )
+
+        conformance_ui_args["port"] = arg(
+            '--port',
+            required=False,
+            help='Port of the webserver (default: 3000)',
+            default='3000'
+        )
+        conformance_ui_args["result_directory"] = arg(
+            '--result-directory',
+            required=False,
+            help='Directory containing the results of the SPARQL conformance tests (default: current directory)',
+            default='$(pwd)'
+        )
+
         return all_args
 
     @staticmethod

src/sparql_conformance/Dockerfile

@@ -0,0 +1,31 @@
+FROM node:18
+ARG UID
+ARG GID
+
+RUN set -eux; \
+    if getent group "${GID}" >/dev/null; then \
+      echo "Using existing group with GID ${GID}"; \
+    else \
+      groupadd -g "${GID}" appgroup; \
+    fi; \
+    if getent passwd "${UID}" >/dev/null; then \
+      echo "Using existing user with UID ${UID}"; \
+    else \
+      useradd -m -u "${UID}" -g "${GID}" appuser; \
+    fi
+
+WORKDIR /app
+
+RUN apt-get update && apt-get install -y git && rm -rf /var/lib/apt/lists/*
+
+RUN git clone https://github.com/SIRDNARch/qlever-conformance-website.git .
+
+RUN mkdir -p /public/results
+
+RUN npm install
+
+EXPOSE 3000
+
+USER ${UID}:${GID}
+
+CMD [ "node", "server.js" ]

src/sparql_conformance/Qleverfiles/Qleverfile.qlever

@@ -0,0 +1,16 @@
+# Qleverfile for SPARQL conformance tests using the qlever engine
+# To exclude certain tests or test groups add them like this:
+# EXCLUDE = service-description,service,entailment,POST - existing graph,PUT - mismatched payload,query specifying dataset in both query string and protocol; test for use of protocol-specified dataset
+[data]
+NAME = ConformanceTest
+
+[runtime]
+SYSTEM = docker
+
+[conformance]
+NAME = ConformanceTest
+PORT = 7036
+ENGINE = qlever
+TESTSUITE_DIR = ./testsuite-files/sparql/sparql11/
+GRAPH_STORE = /http-graph-store
+TYPE_ALIAS = [["http://www.w3.org/2001/XMLSchema#int", "http://www.w3.org/2001/XMLSchema#integer"], ["http://www.w3.org/2001/XMLSchema#float", "http://www.w3.org/2001/XMLSchema#double"], ["http://www.w3.org/2001/XMLSchema#decimal", "http://www.w3.org/2001/XMLSchema#double"], ["http://www.w3.org/2001/XMLSchema#decimal", "http://www.w3.org/2001/XMLSchema#float"], ["http://www.w3.org/2001/XMLSchema#string", null]]