diff --git a/.dotstop.dot b/.dotstop.dot index 596103586c..6008c0d3ad 100644 --- a/.dotstop.dot +++ b/.dotstop.dot @@ -31,6 +31,7 @@ digraph G { "AOU-28" [sha="76887b683a1e5a2f36a029bf80c7489226d1728e6f148f69f7b2167211f1f443"]; "AOU-29" [sha="cb3302e313dae237d67bf249c61419232a83aa6245af30a0f33c795ab1b3a8f5"]; "AOU-30" [sha="ee747671350441b0ca2438c651e0c30a90ac9b126540308415e1645eeea2707a"]; +"AOU-31" [sha="e6dfb726b06bf38ff27602695bdfafee64605da7ab6540b2ec0ae479ab73f3dc"]; "JLEX-01" [sha="1cd16efee6db392909248a77c773e0c5aa76b4c05001ab0b981201004f2162b0"]; "JLEX-02" [sha=f40046dae47252d62f28c935db90460772263ced0fa9ed1a5e1383436b89daa9]; "JLS-01" [sha=dce41089598ceff4a3ce3df7a8d3d16183346b785d791d60c8292f51c055946d]; @@ -82,6 +83,9 @@ digraph G { "JLS-51" [sha="190e17d59795c9ed3b25a0a8bf57497de1e0d06ab90b3f6ba47b543c95edea43"]; "JLS-52" [sha="8539f924c31974a2722615d2410a25336a5d6a9f399f16dc485be83f7f87a5ff"]; "JLS-53" [sha="d9f7e732e34b0ec79305dde4c5b3d60906559ef1d90bc3ce2906e28a90293844"]; +"JLS-54" [sha="f1ff5d3aff697980bcf072e27e00f9916052aceb4796fe1417486ddc80d5e241"]; +"JLS-55" [sha="321c775e2acf0267ac1753eb772fedf1bcca18029714071c6e55d999f6dd008a"]; +"JLS-56" [sha="6fdd3ab1172c1df5658f140d104d51559080aa32389a5a842b93cfbd394593b3"]; "JLS-57" [sha="6261b6cf44be2e742af1e1d687f1233161ab7cdaf6f1c0a6e31e671a7451adc5"]; "JLS-58" [sha="dbdb83427fd82fd3be5e90ab761945a0346b33740b9ea80fd37122dfa6baaa60"]; "JLS-61" [sha="151f1cda2384ae4935d29d300c3424bca710378fa3689bbcff69b06dc86bb692"]; @@ -91,6 +95,11 @@ digraph G { "JLS-65" [sha="e413de6c831c1c019c67c3e3477b9dc9302cc79433ec894beaee0c95e053b545"]; "JLS-66" [sha="cf57eaf55654ef52589b1879c7294de13ddf1258ecdff4f6371178c6e8e6975b"]; "JLS-74" [sha="c161214f0f206f3c0826750978fcc4c99e2765a0c3333592e1293b323434ca34"]; +"JLS-70" [sha="5958335832f06baaa624a093c06ce9abc2ee88c8034819ace3be2c5fff0e0a74"]; +"JLS-71" [sha="83bb8658b1b8914e0f37cdd0333d5f9351a75763486565ce7804a240d9a418ac"]; +"JLS-72" [sha="0b061840170a819c7daf1005b2eee53d121e04e921413935f5eed4fec98a8726"]; +"JLS-73" [sha="42d8dd90dc4a321923567287216e216a30c55ede8836824e19fba046a2121a97"]; +"JLS-76" [sha="ebcfc023f88ef50a3c804cb72318428e7566dc44c3539c54742e25f261ff3249"]; "NJF-01" [sha="548dc86014e093974f68660942daa231271496a471885bbed092a375b3079bd8"]; "NJF-02" [sha="6ea015646d696e3f014390ff41612eab66ac940f20cf27ce933cbadf8482d526"]; "NJF-03" [sha="4bd1f8210b7bba9a248055a437f377d9da0b7576c5e3ed053606cf8b5b2febe3"]; @@ -407,6 +416,7 @@ digraph G { "TA-BEHAVIOURS" -> "JLEX-02" [sha=cb26451e31a56b1eb51a4d45283ba4a7c6e898efbd045b59cba10d3c6aa093de]; "TA-BEHAVIOURS" -> "JLS-03" [sha=cf9211c07452914cb2d0b455f859b26cb2724423eae5187e8cbfdff06d1b5ba3]; "TA-BEHAVIOURS" -> "JLS-27" [sha="880ec996ed026258b58299c356aab7d02652ae55cbf1f98494e2a7770fd96275"]; +"TA-BEHAVIOURS" -> "JLS-56" [sha="96132477a8b2090f75ca56db2af3ac142a26423915a2f76e328734ae3347a5dd"]; "TA-CONFIDENCE" -> "JLS-08" [sha="506164051180023c8533ea1f6dedf1bad894c3ee6020ff16b002e33b109c2791"]; "TA-CONFIDENCE" -> "JLS-09" [sha="80bbde95fc14f89acf3dad10b3831bc751943fe4a1d79d5cbf4702416c27530f"]; "TA-CONFIDENCE" -> "JLS-20" [sha="1bfd214ab8186a3c095262ae503451b8d71ada8db5b13ecc7b906739a05bc102"]; @@ -438,6 +448,11 @@ digraph G { "TA-CONSTRAINTS" -> "AOU-10" [sha="54e2cf65bcc7544eda66fd5aca9763008a094dd7389db2463b28c81e1c586375"]; "TA-CONSTRAINTS" -> "AOU-11" [sha="376c75481202bdc4c6d8f9073f5173ab307859b0495735b204de05e58ef2742c"]; "TA-CONSTRAINTS" -> "AOU-30" [sha="28a722163059e712168605d165311d28bbb6e0d88f34e4b8b317f18d47d02843"]; +"TA-CONSTRAINTS" -> "JLS-70" [sha="8672df0fe5ac8addb8adfb527e2735606966e66d45c112b87c23cf4e57f34909"]; +"TA-CONSTRAINTS" -> "JLS-71" [sha="06bc97101b9f4119cb3f5bdac28ec3f25f35bb51a593893fcfe1fc5d67cd17a3"]; +"TA-CONSTRAINTS" -> "JLS-72" [sha="ed3451acf602f0f25d33250a55bfe6311a47d6e5f7bd2b1b7d5beb08ca781094"]; +"TA-CONSTRAINTS" -> "JLS-73" [sha="4e3de24e57b1865345cebeb654b0b29d89558f3fddb0c76f8b26db0d4ff92993"]; +"TA-CONSTRAINTS" -> "AOU-31" [sha="858726a396aa2b29254e4f84a815ac07b0cddfd13d3f69b9e520cf8ebe5c0172"]; "TA-DATA" -> "JLS-18" [sha=af2567d28552ab32643e995a9c9c9fd73b2208ce3d5255fba333c8ddd28f72a6]; "TA-DATA" -> "JLS-45" [sha="9cc0a42711c2d7c03dab3a2d633abf41a7451c375f68bed9907429a86077d2d8"]; "TA-FIXES" -> "JLS-05" [sha="7264d31fb391f7ab77e3719a732b06a366579f7ee8b398d185daa4431aee4224"]; @@ -448,6 +463,9 @@ digraph G { "TA-FIXES" -> "JLS-28" [sha="0adb7ec9c6cc4338fa810442d22c8e4ca2a39cf33f0efaf859457ab32946991d"]; "TA-FIXES" -> "JLS-33" [sha="56b526e261afa2da0793ec172850d227e4ed1d8c8a06e616c680c3db29648d45"]; "TA-FIXES" -> "JLS-57" [sha="ada11447792ac70d2f87b81253379dd31d73eefa57554c9f0b1e067bf995812c"]; +"TA-INDICATORS" -> "JLS-54" [sha="b8d659e9fa897e4f13ceace28a2c71e38d41be22a52ca34162ca907c82e160fc"]; +"TA-INDICATORS" -> "JLS-55" [sha="0d048eab93d9d675bf346fb93b80a5af1b7752352b28e1b80b4b9e8b85e5fc3d"]; +"TA-INDICATORS" -> "JLS-56" [sha="a27def8655040fb7b0feb8c88ce79962442cd8f7fab0e64ec7a7ff576ff1b7e6"]; "TA-INPUTS" -> "JLS-04" [sha="262db6d430e99ef3a23645c93a1cc5bda1270ceba90b4d8cccb40b1eb85e9860"]; "TA-INPUTS" -> "JLS-47" [sha="b2da62290125ecc680f953dde166bb5f22e7f8c6e7e53a73136102e01dd013a7"]; "TA-INPUTS" -> "JLS-34" [sha="b39b1a808b02bfcd5450ffea835179c862c19f7759de0508bac9249c02db58c1"]; @@ -469,6 +487,8 @@ digraph G { "TA-MISBEHAVIOURS" -> "JLS-24" [sha=e8de01ff7c316debcd96afa4b3b6b62be73522e4531214c18b3ad7eec826275e]; "TA-MISBEHAVIOURS" -> "JLS-25" [sha="56ba396580f90e5a10fd5adfe33864921537d47e21b215a8faf531855af40ecd"]; "TA-MISBEHAVIOURS" -> "JLS-31" [sha="ff3352e20146a81904da6d8d94b003b4e0acbc2a8a63a73ea017ea0535e45e79"]; +"TA-MISBEHAVIOURS" -> "JLS-76" [sha="7528c4228bbe5a1d4ccfcd7efe4254b2f2443c9746b28de5903d2a925dba8436"]; +"TA-MISBEHAVIOURS" -> "JLS-11" [sha="d03c18218f8e022c1210c63a6dcea2729a0584bc3b31aba1370e5f5037b0ef94"]; "TA-RELEASES" -> "JLS-14" [sha="1202b9934353436fba927de6762cf62a8fc23ab0815a3c06f9d0a77b55830720"]; "TA-RELEASES" -> "JLS-38" [sha="ed0d250c8c191ac4fc03712a321bf399d26e932edabb13067fbcffae1640cb90"]; "TA-RELEASES" -> "JLS-10" [sha="33c316a9040c7d27c830ca453e39d3bb423acf42e14d1d561c952291ba66078b"]; diff --git a/.dotstop_extensions/references.py b/.dotstop_extensions/references.py index 999b6fdd83..85b98da8b6 100644 --- a/.dotstop_extensions/references.py +++ b/.dotstop_extensions/references.py @@ -36,16 +36,19 @@ class CPPTestReference(BaseReference): section declaration, and the closing brace `}` must have the same indentation as the opening brace. This is the case for the tests from nlohmann_json. """ - def __init__(self, name: str, path: str) -> None: + def __init__(self, name: str, path: str, description: str = "") -> None: """ Initialize CPPTestReference. Args: - name: Section name, use colon-separated for nested sections (e.g., "testcase1:section1:section2") + name: Section name, use colon-separated for nested sections + (e.g., "testcase1:section1:section2") path: Relative path from project root to the file + description: Optional human-readable description of the test section """ self._name = name self._path = Path(path) + self._description = description @classmethod def type(cls) -> str: @@ -167,13 +170,16 @@ def content(self) -> bytes: def as_markdown(self, filepath: None | str = None) -> str: content = self.content.decode('utf-8') content = self.remove_leading_whitespace_preserve_indentation(content) - return format_cpp_code_as_markdown(content) + md = format_cpp_code_as_markdown(content) + + if self._description: + return f"Description: {self._description}\n\n{md}" + return md def __str__(self) -> str: # this is used as a title in the trudag report return f"cpp-test: [{self._name}]\n({self._path})" - class JSONTestsuiteReference(CPPTestReference): """ Represents a reference to one or more JSON testsuite files, where the CPP test diff --git a/.github/workflows/coverage_gate.yml b/.github/workflows/coverage_gate.yml new file mode 100644 index 0000000000..1462ec2c4e --- /dev/null +++ b/.github/workflows/coverage_gate.yml @@ -0,0 +1,81 @@ +name: Coverage Gate + +on: + workflow_call: + inputs: + artifact_id: + description: "Artifact name for the coverage-gate result" + required: true + type: string + +jobs: + coverage_gate: + runs-on: ubuntu-latest + + steps: + - name: Checkout code + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + + - name: Download coverage HTML artifact + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: code-coverage-report + path: coverage_html + + - name: Debug list coverage files + run: | + echo "=== coverage_html contents ===" + ls -R coverage_html + + - name: Enforce coverage threshold + run: | + THRESHOLD=99.19 + + echo "=== Extracting line coverage for 'Lines:' from index.html ===" + + HEADER_BLOCK=$(grep -A1 'Lines:' coverage_html/index.html || true) + echo "$HEADER_BLOCK" + + LINE_COV=$(echo "$HEADER_BLOCK" | grep -oE "[0-9]+(\.[0-9]+)?" | head -n1 || true) + echo "Extracted Line coverage: '${LINE_COV}'" + + if [ -z "$LINE_COV" ]; then + echo "Could not extract line coverage for 'Lines:' from index.html" + exit 1 + fi + + COMPARE=$(awk -v cov="$LINE_COV" -v thr="$THRESHOLD" 'BEGIN { if (cov < thr) print "lt"; else print "ge"; }') + + if [ "$COMPARE" = "lt" ]; then + echo "Coverage below threshold, failing job." + exit 1 + fi + + echo "Coverage is above threshold." + + - name: Create coverage-gate result + if: always() + run: | + mkdir -p coverage_gate + + THRESHOLD=99.19 + + LINE_COV="" + if [ -f coverage_html/index.html ]; then + HEADER_BLOCK=$(grep -A1 'Lines:' coverage_html/index.html || true) + LINE_COV=$(echo "$HEADER_BLOCK" | grep -oE "[0-9]+(\.[0-9]+)?" | head -n1 || true) + fi + + { + echo "status=${{ job.status }}" + echo "line_coverage=${LINE_COV}" + echo "threshold=${THRESHOLD}" + } > coverage_gate/coverage_gate.txt + + + - name: Upload coverage-gate artifact + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ inputs.artifact_id }} + path: coverage_gate/coverage_gate.txt diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 1345f58914..11fe19d552 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -81,7 +81,7 @@ jobs: - name: Download trudag report artifact uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - name: trudag-report-${{ github.event.pull_request.head.sha || github.sha }} + name: trudag-report-${{ github.sha }} path: TSF/docs/generated/ - name: Build documentation diff --git a/.github/workflows/parent-workflow.yml b/.github/workflows/parent-workflow.yml index 775d88e6c0..a46c144083 100644 --- a/.github/workflows/parent-workflow.yml +++ b/.github/workflows/parent-workflow.yml @@ -10,6 +10,7 @@ on: permissions: contents: read + pull-requests: read jobs: labeler: @@ -63,6 +64,22 @@ jobs: with: artifact_id: "ubuntu-${{ github.sha }}" + coverage_gate: + name: Run Coverage Gate Workflow + needs: [ubuntu] + if: ${{ github.event_name == 'pull_request' && github.base_ref == 'main' + || github.event_name == 'push' && github.ref == 'refs/heads/main' + || github.event_name == 'schedule' }} + uses: ./.github/workflows/coverage_gate.yml + with: + artifact_id: "coverage_gate-${{ github.sha }}" + + pr_count_gate: + name: Run PR Count Gate Workflow + uses: ./.github/workflows/pr_count_gate.yml + with: + artifact_id: "pr_count_gate-${{ github.sha }}" + dependency_review: name: Run dependency_review Workflow if: ${{ github.event_name == 'pull_request' }} # only run dependency_review for PRs @@ -73,11 +90,11 @@ jobs: collect_artifacts_pr: name: "Collect Results & Deploy (PR)" if: github.event_name == 'pull_request' - needs: [labeler, SME_review_checker, check_amalgamation, test_trudag_extensions, dependency_review, codeql, ubuntu] + needs: [labeler, SME_review_checker, check_amalgamation, test_trudag_extensions, dependency_review, codeql, ubuntu, coverage_gate, pr_count_gate] runs-on: ubuntu-latest strategy: matrix: - target: [labeler, SME_review_checker, check_amalgamation, test_trudag_extensions, dependency_review, codeql, ubuntu] + target: [labeler, SME_review_checker, check_amalgamation, test_trudag_extensions, dependency_review, codeql, ubuntu, coverage_gate, pr_count_gate] steps: - name: Checkout code @@ -89,7 +106,7 @@ jobs: result="${{ needs[matrix.target].result }}" echo "${{ matrix.target }} workflow result: $result" - if [[ "$result" != "success" ]]; then + if [[ "$result" != "success" && "$result" != "skipped" ]]; then echo "❌ ${{ matrix.target }} workflow failed! Exiting..." exit 1 fi @@ -106,11 +123,11 @@ jobs: collect_artifacts_non_pr: name: "Collect Results & Deploy (Non-PR)" if: github.event_name != 'pull_request' - needs: [labeler, test_trudag_extensions, codeql, ubuntu] # no check_amalgamation, dependency_review or SME_review_checker if non PR + needs: [labeler, test_trudag_extensions, codeql, ubuntu, coverage_gate, pr_count_gate] # no check_amalgamation, dependency_review or SME_review_checker if non PR runs-on: ubuntu-latest strategy: matrix: - target: [labeler, test_trudag_extensions, codeql, ubuntu] + target: [labeler, test_trudag_extensions, codeql, ubuntu, coverage_gate, pr_count_gate] steps: - name: Checkout code diff --git a/.github/workflows/pr_count_gate.yml b/.github/workflows/pr_count_gate.yml new file mode 100644 index 0000000000..02f5fdbc3c --- /dev/null +++ b/.github/workflows/pr_count_gate.yml @@ -0,0 +1,78 @@ +name: PR Count Gate + +on: + workflow_call: + inputs: + artifact_id: + required: true + type: string + +jobs: + pr_count_gate: + runs-on: ubuntu-latest + + steps: + - name: Count open pull requests + id: pr-count + uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { owner, repo } = context.repo; + const per_page = 100; + let page = 1; + let total = 0; + + while (true) { + const { data } = await github.rest.pulls.list({ + owner, + repo, + state: 'open', + base: 'main', + per_page, + page, + }); + if (data.length === 0) break; + total += data.length; + if (data.length < per_page) break; + page++; + } + + core.info(`Open pull requests: ${total}`); + core.setOutput('open_prs', total.toString()); + + - name: fail if too many open PRs + id: gate + shell: bash + run: | + MAX=15 + OPEN="${{ steps.pr-count.outputs.open_prs }}" + + echo "Open PRs: ${OPEN}" + echo "Max allowed: ${MAX}" + + if [ "$OPEN" -gt "$MAX" ]; then + echo "Gate failed: too many open PRs (${OPEN} > ${MAX})" + exit 1 + fi + + echo "Gate passed: (${OPEN} <= ${MAX})" + + - name: Write PR count result + if: always() + run: | + mkdir -p pr_count_gate + echo "status=${{ job.status }}" > pr_count_gate/pr_count_gate.txt + echo "open_prs=${{ steps.pr-count.outputs.open_prs }}" >> pr_count_gate/pr_count_gate.txt + echo "max_open_prs=15" >> pr_count_gate/pr_count_gate.txt + echo "sha=${{ github.sha }}" >> pr_count_gate/pr_count_gate.txt + echo "run_id=${{ github.run_id }}" >> pr_count_gate/pr_count_gate.txt + + - name: Upload PR count gate artifact + if: always() + uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 + with: + name: ${{ inputs.artifact_id }} + path: pr_count_gate/pr_count_gate.txt + + diff --git a/.github/workflows/publish_documentation.yml b/.github/workflows/publish_documentation.yml index ec8e55757a..b0059dda28 100644 --- a/.github/workflows/publish_documentation.yml +++ b/.github/workflows/publish_documentation.yml @@ -189,6 +189,8 @@ jobs: TSF_SCORING_DB: ${{ steps.select_db.outputs.db_path }} run: | set -euo pipefail + git checkout "$branch_name" + git checkout save_historical_data -- "$TSF_SCORING_DB" TSF/misbehaviours.md echo "[verify] TSF_SCORING_DB='$TSF_SCORING_DB'" test -n "$TSF_SCORING_DB" test -f "$TSF_SCORING_DB" @@ -200,9 +202,13 @@ jobs: - name: Checkout data storage branch run: | - git stash push --include-untracked -- . ':(exclude)TSF/data_storage' ':(exclude)TSF/data_storage/**' ':(exclude)TSF/TrustableScoring.db' + set -euo pipefail + temp_db="$RUNNER_TEMP/$(basename "$TSF_SCORING_DB").working" + cp "$TSF_SCORING_DB" "$temp_db" + git stash push --include-untracked -- . git checkout save_historical_data git pull + cp "$temp_db" "$TSF_SCORING_DB" - name: Store persistent data run: | @@ -221,7 +227,8 @@ jobs: - name: Recover stash run: | - git checkout $branch_name + set -euo pipefail + git checkout "$branch_name" git stash apply - name: Upload trudag artifact diff --git a/TSF/trustable/assertions/TA-BEHAVIOURS-CHECKLIST.md b/TSF/trustable/assertions/TA-BEHAVIOURS-CHECKLIST.md index bfd9dfbb66..b73565646d 100644 --- a/TSF/trustable/assertions/TA-BEHAVIOURS-CHECKLIST.md +++ b/TSF/trustable/assertions/TA-BEHAVIOURS-CHECKLIST.md @@ -1,4 +1,4 @@ -#### Checklist for TA-BEHAVIOURS from [Codethink](https://codethinklabs.gitlab.io/trustable/trustable/print_page.html) +#### Checklist for TA-BEHAVIOURS from [Codethink](https://pages.eclipse.dev/eclipse/tsf/tsf/print_page.html) * How has the list of Expectations varied over time? diff --git a/TSF/trustable/assertions/TA-BEHAVIOURS_CONTEXT.md b/TSF/trustable/assertions/TA-BEHAVIOURS_CONTEXT.md index 2aea1e70c8..676790b6d1 100644 --- a/TSF/trustable/assertions/TA-BEHAVIOURS_CONTEXT.md +++ b/TSF/trustable/assertions/TA-BEHAVIOURS_CONTEXT.md @@ -3,12 +3,12 @@ level: 1.1 normative: false --- -(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://codethinklabs.gitlab.io/trustable/trustable/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) +(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://pages.eclipse.dev/eclipse/tsf/tsf/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) Although it is practically impossible to specify all of the necessary behaviours and required properties for complex software, we must clearly specify the most important of these (e.g. where harm could result if given criteria are not met), -and verify that these are correctly provided by nlohmann/json. +and verify that these are correctly provided by the nlohmann/json library. **Guidance** @@ -48,12 +48,11 @@ sufficient. **Evidence** - List of Expectations - - **Answer**: -- Argument of sufficiency for break-down of expected behaviour for all - Expectations - - **Answer**: + - **Answer**: The Expectations are provided via JLEX-01 and JLEX-02. +- Argument of sufficiency for break-down of expected behaviour for all Expectations + - **Answer**: See JLS-56. - Validation and verification of expected behaviour - - **Answer**: + - **Answer**: The validation and verification of expected behaviour is done via the evidence provided for all the statements below JLEX-01 and JLEX-02 in the trustable graph, in addition to JLS-03. **Confidence scoring** @@ -65,26 +64,26 @@ and that the resulting system and tests are validated by appropriate strategies. **Checklist** - How has the list of Expectations varied over time? - - **Answer**: + - **Answer**: The list of Expectations is taken from [here](https://eclipse-score.github.io/score/main/modules/baselibs/json/docs/requirements/index.html). The development can be retraced using git. - How confident can we be that this list is comprehensive? - - **Answer**: + - **Answer**: The list of Expectations has been collected amongst the stakeholders in S-CORE, so we are very confident that the list is comprehensive. - Could some participants have incentives to manipulate information? - - **Answer**: + - **Answer**: We consider intentional manipulation of information about nlohmann/json to be very unlikely because the library is open source, has no direct revenue or certification attached to this documentation, and all stakeholders share a strong interest in correctness and robustness. Any misrepresentation of expectations or verification would quickly become counterproductive by increasing integration risk, maintenance cost, and reputational damage for the participants. In addition, the requirements, code and history are publicly visible and version-controlled, so inconsistencies can be detected and challenged by other S-CORE stakeholders or the wider community. While unintentional errors are always possible, we see no realistic positive incentive, and several strong negative incentives, for deliberately manipulating this information. - Could there be whole categories of Expectations still undiscovered? - - **Answer**: + - **Answer**: Yes, it is always possible that whole categories of Expectations remain undiscovered, especially for a widely used and versatile library like nlohmann/json. However, this risk is mitigated by deriving Expectations from actual use cases, stakeholder input, and known integration contexts within S-CORE, and by revisiting them as new uses emerge. The requirements and their evolution are tracked in version control, so newly identified categories can be added transparently. We therefore acknowledge the possibility of missing categories, but consider the current set to be appropriate and proportionate for the identified scope and applications. - Can we identify Expectations that have been understood but not specified? - - **Answer**: + - **Answer**: There are currently no Expectations that have been understood but not specified. - Can we identify some new Expectations, right now? - - **Answer**: + - **Answer**: We currently do not see further Expectations to be identified because the existing set was derived systematically from the S-CORE stakeholders. - How confident can we be that this list covers all critical requirements? - - **Answer**: + - **Answer**: We are very confident that this list covers all critical requirements. - How comprehensive is the list of tests? - - **Answer**: + - **Answer**: Currently, branch coverage is 93.865% and line coverage is 99.186% (JLS-27). Deviation from 100% branch coverage is expected for this kind of template-heavy library due to known factors such as tooling artifacts, defensive/unreachable paths (e.g., LCOV-excluded branches), and configuration-dependent code paths. Coverage is continuously monitored in CI and reviewed in pull requests and maintainers have documented concrete coverage-artifact cases (e.g., https://github.com/nlohmann/json/pull/4595). In addition to raw percentages, we use statement-level traceability from Expectations JLEX-01/02 to concrete CI tests (JLS-74). Therefore, we deem the test set very comprehensive for the intended scope. - Is every Expectation covered by at least one implemented test? - - **Answer**: + - **Answer**: Yes, both Expectations (JLEX-01, JLEX-02) are covered by a broad set of implemented tests across their supporting statements (WFJ-, PJD-, NJF-, NPF-, TIJ-). Evidence is assessed through Trustable traceability and SME reviews (JLS-74), with direct CI-test links provided where available. In addition, completeness is assessed against the upstream basic_json API (JLS-72) with evidence of tests of a comprehensive set of arguments (JLS-31). - Are there any Expectations where we believe more coverage would help? - - **Answer**: + - **Answer**: No additional top-level Expectation is currently missing. Extra tests would mainly increase confidence for already identified boundary cases (especially around WFJ-06), rather than reveal a gap in the Expectation set. - How do dependencies affect Expectations, and are their properties verifiable? - - **Answer**: + - **Answer**: The nlohmann/json library does not have any external dependencies apart from the testing pipeline, so there are no dependencies that could possibly affect the Expectations. - Are input analysis findings from components, tools, and data considered in relation to Expectations? - - **Answer**: + - **Answer**: For components, there is no input analysis as the nlohmann/json library has no external components (see JLS-34). For Tools, a tool assessment is provided via JLS-50. In addition, the only data provided to the nlohmann/json library is the input data when using the libraries' functionality, as well as the test data taken from [here](https://github.com/nlohmann/json_test_data). For data, the json_test_data repository is an aggregation of widely used, independently curated JSON test suites (Big List of Naughty Strings, JSONTestSuite, JSON Patch tests, benchmark suites, etc.), each explicitly designed to cover malformed inputs, edge cases, and realistic JSON usage. We rely on the documented scope and intent of these suites, together with the high measured code coverage (JLS-27), to conclude that they exercise the behaviours captured in our Expectations (JLEX-01, JLEX-02), and we do not perform a separate, manual “input analysis” of every file in the corpus. In that sense, there are no additional local input‑analysis findings beyond the upstream characterisation of these corpora that would currently motivate changing or extending our Expectations. diff --git a/TSF/trustable/assertions/TA-CONSTRAINTS-CHECKLIST.md b/TSF/trustable/assertions/TA-CONSTRAINTS-CHECKLIST.md index 2c123d8ee7..dfecb0c259 100644 --- a/TSF/trustable/assertions/TA-CONSTRAINTS-CHECKLIST.md +++ b/TSF/trustable/assertions/TA-CONSTRAINTS-CHECKLIST.md @@ -1,4 +1,4 @@ -#### Checklist for TA-CONSTRAINTS from [Codethink](https://codethinklabs.gitlab.io/trustable/trustable/print_page.html) +#### Checklist for TA-CONSTRAINTS from [Codethink](https://pages.eclipse.dev/eclipse/tsf/tsf/print_page.html) * Are the constraints grounded in realistic expectations, backed by real-world examples? diff --git a/TSF/trustable/assertions/TA-CONSTRAINTS_CONTEXT.md b/TSF/trustable/assertions/TA-CONSTRAINTS_CONTEXT.md index c484995645..db19c997dc 100644 --- a/TSF/trustable/assertions/TA-CONSTRAINTS_CONTEXT.md +++ b/TSF/trustable/assertions/TA-CONSTRAINTS_CONTEXT.md @@ -3,7 +3,7 @@ level: 1.1 normative: false --- -(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://codethinklabs.gitlab.io/trustable/trustable/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) +(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://pages.eclipse.dev/eclipse/tsf/tsf/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) **Guidance** @@ -51,17 +51,15 @@ reporting bugs, issues, and requests. **Suggested evidence** - Installation manuals with worked examples - - **Answer**: + - **Answer**: See JLS-70. - Configuration manuals with worked examples - - **Answer**: + - **Answer**: See JLS-71. - Specification documentation with a clearly defined scope - - **Answer**: -- User guides detailing limitations in interfaces designed for expandability or - modularity - - **Answer**: -- Documented strategies used by external users to address constraints and - work with existing Statements - - **Answer**: + - **Answer**: See JLS-72. +- User guides detailing limitations in interfaces designed for expandability or modularity + - **Answer**: See JLS-73. +- Documented strategies used by external users to address constraints and work with existing Statements + - **Answer**: See AOU-10 and AOU-11. **Confidence scoring** @@ -72,21 +70,21 @@ contradictions and obvious pitfalls within the defined Statements. - Are the constraints grounded in realistic expectations, backed by real-world examples? - - **Answer**: + - **Answer**: The constraints are grounded in realistic expectations because they come from concrete AOUs covering practical integration duties (consistent dependencies and mirrored dependencies/tools for reproducible/offline builds (AOU-02/03/08/15), CI-tested toolchains (AOU-16), and release/update/security review processes (AOU-27/29)) and from nlohmann/json’s documented real-world pitfalls (exception handling/disablement (AOU-04/07), UTF-8-only input and invalid surrogates (AOU-05/21), brace-initialization ambiguity (AOU-06), and duplicate-key handling (AOU-20)). Upstream fuzz testing (JLS-02) further supports these constraints by exercising edge cases, increasing confidence without implying absolute certainty. - Do they effectively guide downstream consumers in expanding upon existing Statements? - - **Answer**: + - **Answer**: No downstream consumers exist yet to validate this. However, the AOUs are structured with the intent to guide downstream consumers in extending existing Statements. - Do they provide clear guidance for upstreams on reusing components with well-defined claims? - - **Answer**: + - **Answer**: Yes, to the extent that our constraints and AOUs explicitly state which behaviours and properties of nlohmann/json we rely on (e.g. UTF‑8 handling, exception behaviour, duplicate‑key handling, and integration/toolchain assumptions). This makes clear which claims must remain valid for our Statements to hold, and thus gives upstream maintainers concrete guidance on what changes would affect us and where they need to preserve compatibility or coordinate changes. - Are any Statements explicitly designated as not reusable or adaptable? - - **Answer**: + - **Answer**: No, all statements could theoretically be adapted or reused. - Are there worked examples from downstream or upstream users demonstrating these constraints in practice? - - **Answer**: + - **Answer**: As the nlohmann/json library is widely used, its constraints (like the installation manual) are regularly read and applied and therefore demonstrated. - Have there been any documented misunderstandings from users, and are these visibly resolved? - - **Answer**: + - **Answer**: Yes, some recurring misunderstandings are explicitly documented and addressed via upstream documentation and closed issues. For example, brace-initialization unexpectedly yielding arrays and differing across compilers is called out in the [FAQ](https://json.nlohmann.me/home/faq/) and referenced from issues [here](https://github.com/nlohmann/json/issues/4898), and duplicate-key behavior is clarified in the release notes as unspecified by RFC-8259 (see [release notes](https://json.nlohmann.me/home/releases) and issue [#2667](https://github.com/nlohmann/json/discussions/2667)). - Do external users actively keep up with updates, and are they properly notified of any changes? - - **Answer**: + - **Answer**: External users of the library are not necessarily automatically notified of an update, and are neither assumed nor required to keep up to date. If the external user forks the GitHub repository, however, then GitHub shows automatically whenever the upstream changes. diff --git a/TSF/trustable/assertions/TA-INDICATORS-CHECKLIST.md b/TSF/trustable/assertions/TA-INDICATORS-CHECKLIST.md index 0ca7405667..2b6929c89a 100644 --- a/TSF/trustable/assertions/TA-INDICATORS-CHECKLIST.md +++ b/TSF/trustable/assertions/TA-INDICATORS-CHECKLIST.md @@ -1,4 +1,4 @@ -#### Checklist for TA-INDICATORS from [Codethink](https://codethinklabs.gitlab.io/trustable/trustable/print_page.html) +#### Checklist for TA-INDICATORS from [Codethink](https://pages.eclipse.dev/eclipse/tsf/tsf/print_page.html) * How appropriate/thorough are the analyses that led to the indicators? diff --git a/TSF/trustable/assertions/TA-INDICATORS_CONTEXT.md b/TSF/trustable/assertions/TA-INDICATORS_CONTEXT.md index 6cf6af6b46..c294d60326 100644 --- a/TSF/trustable/assertions/TA-INDICATORS_CONTEXT.md +++ b/TSF/trustable/assertions/TA-INDICATORS_CONTEXT.md @@ -3,7 +3,7 @@ level: 1.1 normative: false --- -(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://codethinklabs.gitlab.io/trustable/trustable/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) +(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://pages.eclipse.dev/eclipse/tsf/tsf/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) Not all deviations from Expected Behaviour can be associated with a specific condition. Therefore, we must have a strategy for managing deviations that @@ -56,20 +56,20 @@ set of AWIs is incorrect, or the tolerance is too lax. **Evidence** -- Risk analyses - - **Answer**: +- Risk analysis + - **Answer**: Risk analysis has been performed (see docs/risk_analysis.md). - List of advance warning indicators - - **Answer**: + - **Answer**: The only two introduced AWIs are provided in JLS-54 and JLS-55. - List of Expectations for monitoring mechanisms - - **Answer**: + - **Answer**: There are no dedicated monitoring mechanisms defined. Any expectations for monitoring apply at system or integration level and are expected to be specified and implemented by the integrator. - List of implemented monitoring mechanisms - - **Answer**: + - **Answer**: There are no dedicated monitoring mechanisms for AWIs in the sense of continuous or in-field monitoring. The two AWIs (coverage and PR count) are evaluated only when CI workflows are executed and are used as quality gates at CI time, rather than as a separate, continuous monitoring system (see JLS-54 and JLS-55). - List of identified misbehaviours without advance warning indicators - - **Answer**: + - **Answer**: Provided by JLS-11. - List of advance warning indicators without implemented monitoring mechanisms - - **Answer**: + - **Answer**: All currently defined AWIs (JLS-54 and JLS-55) are evaluated via CI runs, but there is no additional, dedicated monitoring mechanism beyond these CI executions. - Advance warning signal data as time series (see TA-DATA) - - **Answer**: + - **Answer**: The only AWIs in JLS-54 and JLS-55 are implemented as part of the CI and therefore saved as time series (see JLS-18 and JLS-45). **Confidence scoring** @@ -80,27 +80,31 @@ monitoring mechanisms have been implemented to collect the required data. **Checklist** - How appropriate/thorough are the analyses that led to the indicators? - - **Answer**: + - **Answer**: For eclipse-score/inc_nlohmann_json, the library itself is a statically integrated, header-only component without stream processing loops. No runtime misbehaviours specific to this repository have been identified, and therefore no runtime AWIs are implemented for the library itself. The two AWIs that do exist (coverage threshold and PR-count limit) are based on the assumption that CI test results and review load correlate with potential misbehaviours in the library and its evolution, and are therefore focused on test and process quality rather than runtime behaviour (see JLS-54 and JLS-55). - How confident can we be that the list of indicators is comprehensive? - - **Answer**: + - **Answer**: For the scope of `eclipse-score/inc_nlohmann_json` as a static library, we are reasonably confident that CI-based indicators on test coverage and the count of open PRs are sufficient. - Could there be whole categories of warning indicators still missing? - - **Answer**: + - **Answer**: Yes, there could. In particular, runtime performance or stability indicators in systems that use the library are not covered here. Any missing warning indicators are expected to be implemented by the integrator (see AOU-09). - How has the list of advance warning indicators varied over time? - - **Answer**: + - **Answer**: The current AWIs (coverage threshold and pr count threshold on protected branches) were introduced as CI-based quality gates. No additional AWIs have been added or removed so far (see JLS-54 and JLS-55). - How confident are we that the indicators are leading/predictive? - - **Answer**: + - **Answer**: The indicators are leading in the sense that they prevent changes which reduce test coverage, or are made in an overloaded PR situation, from entering protected branches and being used as a basis for integration and release. - Are there misbehaviours that have no advance warning indicators? - - **Answer**: + - **Answer**: Potential runtime misbehaviours in consuming systems are not covered by AWIs in this repository. - Can we collect data for all indicators? - - **Answer**: + - **Answer**: Both indicators (JLS-54 and JLS-55) are derived from CI runs, and the required data (coverage and pr count) is collected automatically for each CI execution. - Are the monitoring mechanisms used included in our Trustable scope? - - **Answer**: + - **Answer**: There are no continuously running runtime monitoring mechanisms. The only related mechanisms are the CI workflows implementing JLS-54 and JLS-55, they run only when CI is executed. - Are there gaps or trends in the data? - - **Answer**: + - **Answer**: There is no trend analysis preformed on AWIs. However, there is trend analysis done as a proof of concept to the failure rate of CI test (see JLS-17). Potential gaps could arrise during the integration of the library (see AOU-09). - If there are gaps or trends, are they analysed and addressed? - - **Answer**: + - **Answer**: There are no trends identified (see the question above). Any gaps should be closed by the integrator. - Is the data actually predictive/useful? - - **Answer**: + - **Answer**: Yes, the CI data from the AWIs is useful to prevent regressions in the tested behaviour of the library and possible issues introduced due to a large number of open PRs from entering protected branches. - Are indicators from code, component, tool, or data inspections taken into consideration? - - **Answer**: + - **Answer**: All indicator types are considered in scope and in the analysis rationale, with different level of implementation in this repository. +For code inspections, we use CI-derived code evidence (JLS-54) and related CI analysis evidence. +For component inspections, no separate component-inspection AWI is implemented in this repository context because the library has no external components (JLS-34), and component-related runtime/system monitoring is therefore expected at integration level (AOU-19). +For tool inspections, tools are explicitly assessed (JLS-50), and a monitored CI/process indicator is implemented via the PR-count gate (JLS-55). +For data inspections, indicator data is collected and traceable for the implemented CI AWIs (JLS-54 and JLS-55), while continuous production/runtime monitoring is not implemented in this repository and is delegated to the integrator (AOU-09 and AOU-19). diff --git a/TSF/trustable/assertions/TA-METHODOLOGIES-CHECKLIST.md b/TSF/trustable/assertions/TA-METHODOLOGIES-CHECKLIST.md index 101fedf5b4..ad204c8b3d 100644 --- a/TSF/trustable/assertions/TA-METHODOLOGIES-CHECKLIST.md +++ b/TSF/trustable/assertions/TA-METHODOLOGIES-CHECKLIST.md @@ -1,4 +1,4 @@ -#### Checklist for TA-METHODOLOGIES from [Codethink](https://codethinklabs.gitlab.io/trustable/trustable/print_page.html) +#### Checklist for TA-METHODOLOGIES from [Codethink](https://pages.eclipse.dev/eclipse/tsf/tsfprint_page.html) This project follows purely the Methodologies of Eclipse S-CORE. diff --git a/TSF/trustable/assertions/TA-MISBEHAVIOURS-CHECKLIST.md b/TSF/trustable/assertions/TA-MISBEHAVIOURS-CHECKLIST.md index 773a481558..57a0eaddbb 100644 --- a/TSF/trustable/assertions/TA-MISBEHAVIOURS-CHECKLIST.md +++ b/TSF/trustable/assertions/TA-MISBEHAVIOURS-CHECKLIST.md @@ -1,4 +1,4 @@ -#### Checklist for TA-MISBEHAVIOURS from [Codethink](https://codethinklabs.gitlab.io/trustable/trustable/print_page.html) +#### Checklist for TA-MISBEHAVIOURS from [Codethink](https://pages.eclipse.dev/eclipse/tsf/tsf/print_page.html) * How has the list of misbehaviours varied over time? diff --git a/TSF/trustable/assertions/TA-MISBEHAVIOURS_CONTEXT.md b/TSF/trustable/assertions/TA-MISBEHAVIOURS_CONTEXT.md index 754b9c3e0c..17f06e3e4a 100644 --- a/TSF/trustable/assertions/TA-MISBEHAVIOURS_CONTEXT.md +++ b/TSF/trustable/assertions/TA-MISBEHAVIOURS_CONTEXT.md @@ -3,7 +3,7 @@ level: 1.1 normative: false --- -(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://codethinklabs.gitlab.io/trustable/trustable/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) +(Note: The guidance, evidence, confidence scoring and checklist sections below are copied from [CodeThink's documentation of TSF](https://pages.eclipse.dev/eclipse/tsf/tsf/trustable/TA.html). However, the answers to each point in the evidence list and checklist are specific to this project.) The goal of TA-MISBEHAVIOURS is to force engineers to think critically about their work. This means understanding and mitigating as many of the situations that cause the @@ -89,20 +89,20 @@ established and reusable solutions. **Suggested evidence** - List of identified Misbehaviours - - **Answer**: + - **Answer**: A list of Misbehaviours was identified through STPA risk analysis (risk_analysis.md). Upstream bug tracking (JLS-11 and nlohmann_misbehaviours_comments.md) is used as complementary empirical evidence to confirm coverage and update the list where needed. - List of Expectations for mitigations addressing identified Misbehaviours - - **Answer**: + - **Answer**: Mitigation expectations are expressed implicitly through (a) documented Quality assurance (https://json.nlohmann.me/community/quality_assurance) requirements and (b) concrete mitigation mechanisms captured by existing Statements: JLS-02 (fuzzing), JLS-31 (static analysis), JLS-25 (review/security policy), JLS-24 (defined failure mode via exceptions), and WFJ-06 (input validation via accept()). - Risk analysis - - **Answer**: + - **Answer**: Risk analysis has been performed (see docs/risk_analysis.md). - Test analysis, including: - False negative tests - - **Answer**: + - **Answer**: Fault induction via fuzzing (see JLS-02) provides a practical approach to triggering failures and edge cases that normal functional tests might miss. - Exception handling tests - - **Answer**: - - Stress tests - - **Answer**: + - **Answer**: The nlohmann/json library contains a set of exception handling tests, along with custom exception types, as described in JLS-24. + - Stress tests. + - **Answer**: Stress tests target system-level behavior under load, while nlohmann/json is a stateless JSON library. There is no long-lived state, connection pool, thread pool, queue or similar in nlohmann/json. Therefore, stress tests are generally not relevant for nlohmann/json. - Soak tests - - **Answer**: + - **Answer**: Soak tests are used to investigate long-running behaviour, not single operations like a JSON library. Soak tests are therefore not needed for the nlohmann/json library. **Confidence scoring** @@ -113,28 +113,26 @@ considered against the list of Expectations. **Checklist** - How has the list of misbehaviours varied over time? - - **Answer**: + - **Answer**: The list of misbehaviours for nlohmann/json (https://github.com/nlohmann/json/issues), as well as its history and development, is collected using Github. Statistics, e.g. about the number of open issues over time, are currently not tracked. - How confident can we be that this list is comprehensive? - - **Answer**: + - **Answer**: Due to the collaborative nature of the open source community, we deem it quite unlikely, but not impossible, that there are any known misbehaviours which are not reported to the nlohmann/json repository. - How well do the misbehaviours map to the expectations? - - **Answer**: + - **Answer**: The identified misbehaviours do not necessarily all have a direct impact on the defined expectations. A mapping of any misbehaviour to the expectations has to be done on a case-by-case basis. - Could some participants have incentives to manipulate information? - - **Answer**: + - **Answer**: We can not think of any incentive that any collaborateur could have to manipulate the information. - Could there be whole categories of misbehaviours still undiscovered? - - **Answer**: + - **Answer**: Due to the wide use and long-standing development of the library it is quite unlikely that any major misbehaviours, in particular regarding the parsing and validating of JSON data in the sense of RFC-8259, is undiscovered. - Can we identify misbehaviours that have been understood but not specified? - - **Answer**: + - **Answer**: We currently do not identify any misbehaviours that have been understood but not specified. - Can we identify some new misbehaviours, right now? - - **Answer**: + - **Answer**: No, currently no new misbehaviors can be identified. - Is every misbehaviour represented by at least one fault induction test? - - **Answer**: -- Are fault inductions used to demonstrate that tests which usually pass can - and do fail appropriately? - - **Answer**: + - **Answer**: The expected behaviour of nlohmann/json is described by JLS-24 and its substatements in the trustable graph. For every substatement at least one fault induction test is performed. Thus, every misbehaviour is represented by at least one fault induction test. +- Are fault inductions used to demonstrate that tests which usually pass can and do fail appropriately? + - **Answer**: The project uses several forms of fault induction on the library-behaviour level (malformed JSON, invalid API usage, simulated allocation failures, and fuzzing). Dedicated tests assert that these induced faults cause the library to fail in a well‑defined, expected way (e.g. by throwing specific exceptions). While the test framework itself is not tested by fault induction, there are some statements where positive and negative tests actively show expected failure behaviour of the library under induced faulty input, that pass with well-formed data (e.g., TIJ-04.1, evidence: nst's JSONTestSuite). CI then confirms that these "failure‑expecting" tests keep behaving as specified (JLS-76.). Due to the very comprehensive general fault induction testing employed, we see the instances of explicitly failing tests that usually pass rather as representative evidence than requiring a one-to-one matching to every graph leaf. - Are all the fault induction results actually collected? - - **Answer**: + - **Answer**: Partially. For Unit / regression tests: their results are only captured as normal test pass/fail status and CI logs; there is no separate, persistent database of all induced faults and outcomes in the repository. For Fuzz tests (OSS‑Fuzz): the fuzzing infrastructure stores crashing inputs, logs, and statistics on the OSS‑Fuzz side, not in the nlohmann/json repo itself. See JLS-76 for further information. - Are the results evaluated? - - **Answer**: -- Do input analysis findings on verifiable tool or component claims and features -identify additional misbehaviours or support existing mitigations? - - **Answer**: + - **Answer**: Yes. nlohmann/json has a multi‑layered QA process where all fault‑induction results (from unit tests, dynamic analysis, and fuzzing) are systematically evaluated. The CI pipeline runs 1000+ doctest-based tests across 50+ compiler/standard-library/platform configurations, with 100% code‑coverage targets; any failure (including sanitizer/Valgrind runs, static analysis warnings, or CI fuzz/CIFuzz/OSS‑Fuzz crashes) blocks merging, is inspected by maintainers, fixed, and typically covered by additional regression tests, in line with the documented quality assurance process. Together with the collection of fault‑induction results asserted in JLS‑76, this establishes a continuous loop from induced faults to analysed results to corrective actions. +- Do input analysis findings on verifiable tool or component claims and features identify additional misbehaviours or support existing mitigations? + - **Answer**: Currently, there is no analysis which identifies additional misbehaviours. The only such analysis is indirectly via the analysis of the fuzz testing, which currently does not identify additional misbehaviours. diff --git a/TSF/trustable/assumptions-of-use/AOU-31.md b/TSF/trustable/assumptions-of-use/AOU-31.md new file mode 100644 index 0000000000..5715129b59 --- /dev/null +++ b/TSF/trustable/assumptions-of-use/AOU-31.md @@ -0,0 +1,7 @@ +--- +level: '1.1' +normative: true +--- + +The integrator shall ensure that JSON parsing and validation using eclipse-score/nlohmann_json is performed under integration-defined resource and time budgets (e.g., maximum input size, maximum nesting/structure complexity, and maximum processing time), and that suitable enforcement mechanisms are in place at the integration boundary to prevent hangs or resource exhaustion when processing untrusted or extreme inputs. + diff --git a/TSF/trustable/statements/JLS-04.md b/TSF/trustable/statements/JLS-04.md index f0d793940c..3210582fbe 100644 --- a/TSF/trustable/statements/JLS-04.md +++ b/TSF/trustable/statements/JLS-04.md @@ -8,13 +8,15 @@ references: evidence: type: "check_artifact_exists" configuration: - check_amalgamation: exclude - codeql: exclude - dependency_review: include - labeler: exclude - publish_documentation: exclude - test_trudag_extensions: exclude - ubuntu: exclude + ubuntu: exclude + coverage_gate: exclude + codeql: exclude + labeler: exclude + test_trudag_extensions: exclude + dependency_review: include + check_amalgamation: exclude + publish_documentation: exclude + pr_count_gate: exclude score: Jonas-Kirchhoff: 1.0 Erikhu1: 1.0 diff --git a/TSF/trustable/statements/JLS-24.md b/TSF/trustable/statements/JLS-24.md index 8559289037..ee2b922e8d 100644 --- a/TSF/trustable/statements/JLS-24.md +++ b/TSF/trustable/statements/JLS-24.md @@ -1,6 +1,20 @@ --- level: 1.1 normative: true +references: + - type: project_website + url: "https://json.nlohmann.me/features/parsing/parse_exceptions/" + description: "Documents that invalid JSON input triggers a parse_error exception (with examples and recommended try/catch)." + - type: project_website + url: "https://json.nlohmann.me/home/exceptions/" + description: "Overview of exception types including parse errors thrown during deserialization." +evidence: + type: https_response_time + configuration: + target_seconds: 2 + urls: + - "https://json.nlohmann.me/features/parsing/parse_exceptions/" + - "https://json.nlohmann.me/home/exceptions/" --- -The nlohmann/json library recognizes malformed JSON and returns an exception. \ No newline at end of file +The nlohmann/json library recognizes malformed JSON and returns an exception or a discarded value. \ No newline at end of file diff --git a/TSF/trustable/statements/JLS-54.md b/TSF/trustable/statements/JLS-54.md new file mode 100644 index 0000000000..8d7f1f968a --- /dev/null +++ b/TSF/trustable/statements/JLS-54.md @@ -0,0 +1,22 @@ +--- +level: 1.1 +normative: true +references: + - type: verbose_file + path: "./.github/workflows/coverage_gate.yml" + description: "GitHub Actions workflow enforcing a minimum coverage threshold." +evidence: + type: "check_artifact_exists" + configuration: + ubuntu: exclude + coverage_gate: include + codeql: exclude + labeler: exclude + test_trudag_extensions: exclude + dependency_review: exclude + check_amalgamation: exclude + publish_documentation: exclude + pr_count_gate: exclude +--- + +In the eclipse-score/inc_nlohmann_json repository, code coverage is measured in CI and a minimum threshold is enforced for pull requests into main and pushes to main. If coverage falls below the threshold, the coverage_gate check fails and blocks merging into main until coverage is restored. \ No newline at end of file diff --git a/TSF/trustable/statements/JLS-55.md b/TSF/trustable/statements/JLS-55.md new file mode 100644 index 0000000000..a0eb631849 --- /dev/null +++ b/TSF/trustable/statements/JLS-55.md @@ -0,0 +1,22 @@ +--- +level: 1.1 +normative: true +references: + - type: verbose_file + path: "./.github/workflows/pr_count_gate.yml" + description: "GitHub Actions workflow enforcing a limit on open PRs." +evidence: + type: "check_artifact_exists" + configuration: + ubuntu: exclude + coverage_gate: exclude + codeql: exclude + labeler: exclude + test_trudag_extensions: exclude + dependency_review: exclude + check_amalgamation: exclude + publish_documentation: exclude + pr_count_gate: include +--- + +In eclipse-score/inc_nlohmann_json, a GitHub Actions workflow checks the number of open pull requests in the main branch. If the number exceeds a defined threshold, the workflow fails and blocks further merges until the number of open pull requests is reduced below that threshold. diff --git a/TSF/trustable/statements/JLS-56.md b/TSF/trustable/statements/JLS-56.md new file mode 100644 index 0000000000..6fbd228a59 --- /dev/null +++ b/TSF/trustable/statements/JLS-56.md @@ -0,0 +1,21 @@ +--- +level: 1.1 +normative: true +references: + - type: website + url: "https://eclipse-score.github.io/inc_nlohmann_json/main/generated/JLEX.html#jlex-01" + description: "Generated item page for JLEX-01, including its supporting subtree and references." + - type: website + url: "https://eclipse-score.github.io/inc_nlohmann_json/main/generated/JLEX.html#jlex-02" + description: "Generated item page for JLEX-02, including its supporting subtree and references." +evidence: + type: https_response_time + configuration: + target_seconds: 2 + urls: + - "https://eclipse-score.github.io/inc_nlohmann_json/main/generated/JLEX.html#jlex-01" + - "https://eclipse-score.github.io/inc_nlohmann_json/main/generated/JLEX.html#jlex-02" +--- + +The expected behaviour of the expectations JLEX-01 and JLEX-02 is sufficiently broken-down. + diff --git a/TSF/trustable/statements/JLS-70.md b/TSF/trustable/statements/JLS-70.md new file mode 100644 index 0000000000..73f47b9f48 --- /dev/null +++ b/TSF/trustable/statements/JLS-70.md @@ -0,0 +1,16 @@ +--- +level: 1.1 +normative: true +references: + - type: project_website + url: "https://github.com/nlohmann/json?tab=readme-ov-file#integration" + description: "README integration section describing how to integrate and install nlohmann/json." +evidence: + type: https_response_time + configuration: + target_seconds: 2 + urls: + - "https://github.com/nlohmann/json?tab=readme-ov-file#integration" +--- + +The nlohmann/json library provides installation manuals with worked examples. diff --git a/TSF/trustable/statements/JLS-71.md b/TSF/trustable/statements/JLS-71.md new file mode 100644 index 0000000000..54e555d200 --- /dev/null +++ b/TSF/trustable/statements/JLS-71.md @@ -0,0 +1,20 @@ +--- +level: 1.1 +normative: true +references: + - type: project_website + url: "https://json.nlohmann.me/features/macros/" + description: "Configuration documentation via supported preprocessor macros." + - type: project_website + url: "https://json.nlohmann.me/integration/cmake/" + description: "CMake integration documentation, including build options that map to configuration macros." +evidence: + type: https_response_time + configuration: + target_seconds: 2 + urls: + - "https://json.nlohmann.me/features/macros/" + - "https://json.nlohmann.me/integration/cmake/" +--- + +The nlohmann/json library provides configuration manuals with worked examples. diff --git a/TSF/trustable/statements/JLS-72.md b/TSF/trustable/statements/JLS-72.md new file mode 100644 index 0000000000..ed481e18a9 --- /dev/null +++ b/TSF/trustable/statements/JLS-72.md @@ -0,0 +1,20 @@ +--- +level: 1.1 +normative: true +references: + - type: project_website + url: "https://json.nlohmann.me/home/design_goals/" + description: "Design goals describing the intended scope and trade-offs of the library." + - type: project_website + url: "https://json.nlohmann.me/api/basic_json/" + description: "API reference describing the behavior and requirements of the core JSON type." +evidence: + type: https_response_time + configuration: + target_seconds: 2 + urls: + - "https://json.nlohmann.me/home/design_goals/" + - "https://json.nlohmann.me/api/basic_json/" +--- + +The nlohmann/json project documents the intended scope and design trade-offs of the library and specifies the documented interface and behavior of its core JSON type via its API reference. diff --git a/TSF/trustable/statements/JLS-73.md b/TSF/trustable/statements/JLS-73.md new file mode 100644 index 0000000000..ce1ac0195e --- /dev/null +++ b/TSF/trustable/statements/JLS-73.md @@ -0,0 +1,24 @@ +--- +level: 1.1 +normative: true +references: + - type: project_website + url: "https://json.nlohmann.me/features/arbitrary_types/" + description: "User guide for extending the library via serializers, including guidance for third-party types." + - type: project_website + url: "https://json.nlohmann.me/features/namespace/" + description: "Documentation of the inline-namespace mechanism for modular use of multiple versions/configurations, including its limitations." + - type: project_website + url: "https://json.nlohmann.me/integration/migration_guide/" + description: "Migration guidance on namespace/version/configuration handling to avoid integration pitfalls." +evidence: + type: https_response_time + configuration: + target_seconds: 2 + urls: + - "https://json.nlohmann.me/features/arbitrary_types/" + - "https://json.nlohmann.me/features/namespace/" + - "https://json.nlohmann.me/integration/migration_guide/" +--- + +The nlohmann/json documentation provides user-facing guidance for extensibility and modular integration, explicitly documenting limitations and constraints of extension interfaces. diff --git a/TSF/trustable/statements/JLS-76.md b/TSF/trustable/statements/JLS-76.md new file mode 100644 index 0000000000..3b0c63e435 --- /dev/null +++ b/TSF/trustable/statements/JLS-76.md @@ -0,0 +1,19 @@ +--- +level: 1.1 +normative: true +references: + - type: cpp_test + name: "parse;UTF-8;unexpected BOM" + path: "TSF/tests/unit-byte_order_mark.cpp" + description: "This test case is included purely as an example to illustrate the fault‑induction style tests used in nlohmann/json." + - type: cpp_test + name: "deserialization;contiguous containers;error cases;case 15" + path: "tests/src/unit-deserialization.cpp" + description: "This test case is included purely as an example to illustrate the fault‑induction style tests used in nlohmann/json." + - type: cpp_test + name: "parser class - core;parse;string;errors" + path: "TSF/tests/unit-class_parser_core.cpp" + description: "This test case is included purely as an example to illustrate the fault‑induction style tests used in nlohmann/json." +--- + +nlohmann/json does use fault‑induction–style techniques (invalid inputs, resource failures, fuzzing) to demonstrate that code paths which usually succeed in normal usage can and do fail in a controlled, specified way. \ No newline at end of file