✨ multi-scorecard experiment

cmd/multi-scorecard/LICENSE

@@ -0,0 +1,7 @@
+Copyright Jeff Mendoza <jlm@jlm.name>
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software and associated documentation files (the “Software”), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED “AS IS”, WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

cmd/multi-scorecard/README.md

@@ -0,0 +1,168 @@
+# `multi-scorecard`
+
+This program runs OpenSSF Scorecard over many repositories using a [GitHub App](https://docs.github.com/en/apps/creating-github-apps/about-creating-github-apps/about-creating-github-apps) credential.
+GitHub is queried to determine the orgs and repos the app is installed on to determine which repos to run Scorecard over.
+
+Results are printed to stdout in a JSON array.
+
+*`multi-scorecard` was originally featured as part of [Jeff Mendoza](https://github.com/jeffmendoza) and [Stephen Augustus](https://github.com/justaugustus)' SOSS Fusion talk, "Scorecard at Scale: Old and New Possibilities for Lifting Security on All Repositories".*
+
+- [Session page with slides](https://sched.co/1hcPq)
+- [Session recording](https://youtu.be/-XZqbO3hGcw?si=eGicz0sjgiIRhol4)
+- [Previous source repository](https://github.com/jeffmendoza/multi-scorecard)
+
+## Usage
+
+A [GitHub App](https://docs.github.com/en/apps/creating-github-apps/about-creating-github-apps/about-creating-github-apps) must be created and installed on the repositories you wish to scan.
+
+To install:
+
+```console
+go get github.com/ossf/scorecard/cmd/multi-scorecard@multi-scorecard
+```
+
+To run:
+
+```console
+multi-scorecard -appid 1234 -keyfile my-app.private-key.pem > results.json
+```
+
+Where `1234` is the App ID of the app, and `my-app.private-key.pem` is the private key file of the app.
+
+Once the program has finished running, move the `results.json` file to the `src/results.json` in the scorecard-visualizer repo.
+
+Example repo for local visualizer: http://localhost:3000/scorecard-visualizer/#/projects/github.com/uwu-tools/peribolos
+
+### Jeff notes
+
+OK, here is how to run the branches I have:
+
+scorecard-monitor:
+
+```console
+$ git clone
+$ cd scorecard-monitor/
+$ npm install
+Put a results.json at the top level from a multi-scorecard run
+you may need a scopes.json there as well, but it is ignored so can be empty
+$ node src/action.js
+reportfile.md is created as the current report
+databasefile.json is created, if it already exists then it will be used to show diffs between runs in the report.
+```
+
+scorecard-visualizer
+
+```console
+$ git clone
+$ cd scorecard-visualizer/
+$ npm install
+Put a results.json in the src/ subdirectory "src/results.json"
+$ npm start
+There will be some errors, but can be dismissed
+Scores from the results.json can be viewed at
+http://localhost:3000/scorecard-visualizer/#/projects/github.com/orgname/reponame
+```
+
+## TODO
+
+- Document required permissions for GitHub App
+  - Contents: read-only
+  - Metadata: read-only
+- Add logging
+- Resolve [workarounds](#workarounds)
+
+### Workarounds
+
+#### JSON format problems
+
+```log
+TS2345: Argument of type '(element: ScoreElement) => JSX.Element' is not assignable to parameter of type '(value: { details: null; score: number; reason: string; name: string; documentation: { url: string; short: string; }; }, index: number, array: { details: null; score: number; reason: string; name: string; documentation: { ...; }; }[]) => Element'.
+  Types of parameters 'element' and 'value' are incompatible.
+    Type '{ details: null; score: number; reason: string; name: string; documentation: { url: string; short: string; }; }' is not assignable to type 'ScoreElement'.
+      Types of property 'details' are incompatible.
+        Type 'null' is not assignable to type 'string[]'.
+```
+
+<details>
+
+```log
+    101 |
+    102 |       <hr />
+  > 103 |       {data.checks.map((element: ScoreElement) => (
+        |                        ^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+  > 104 |         <>
+        | ^^^^^^^^^^
+  > 105 |           <div key={element.name} className="card__wrapper">
+        | ^^^^^^^^^^
+  > 106 |             <div className="heading__wrapper" data-testid={element.name}>
+        | ^^^^^^^^^^
+  > 107 |               <h3>{element.name}</h3>
+        | ^^^^^^^^^^
+  > 108 |               {element.score !== -1 ? (
+        | ^^^^^^^^^^
+  > 109 |                 <span>{element.score}/10</span>
+        | ^^^^^^^^^^
+  > 110 |               ) : (
+        | ^^^^^^^^^^
+  > 111 |                 <NoAvailableDataMark />
+        | ^^^^^^^^^^
+  > 112 |               )}
+        | ^^^^^^^^^^
+  > 113 |             </div>
+        | ^^^^^^^^^^
+  > 114 |             <p>
+        | ^^^^^^^^^^
+  > 115 |               Description: {element.documentation.short.toLocaleLowerCase()}{" "}
+        | ^^^^^^^^^^
+  > 116 |               <a
+        | ^^^^^^^^^^
+  > 117 |                 href={`${element.documentation.url}`}
+        | ^^^^^^^^^^
+  > 118 |                 target="_blank"
+        | ^^^^^^^^^^
+  > 119 |                 rel="noreferrer"
+        | ^^^^^^^^^^
+  > 120 |               >
+        | ^^^^^^^^^^
+  > 121 |                 See documentation
+        | ^^^^^^^^^^
+  > 122 |               </a>
+        | ^^^^^^^^^^
+  > 123 |             </p>
+        | ^^^^^^^^^^
+  > 124 |             <p>Reasoning: {element?.reason.toLocaleLowerCase()}</p>
+        | ^^^^^^^^^^
+  > 125 |             {Array.isArray(element.details) && (
+        | ^^^^^^^^^^
+  > 126 |               <Collapsible details={element.details} />
+        | ^^^^^^^^^^
+  > 127 |             )}
+        | ^^^^^^^^^^
+  > 128 |           </div>
+        | ^^^^^^^^^^
+  > 129 |           <hr />
+        | ^^^^^^^^^^
+  > 130 |         </>
+        | ^^^^^^^^^^
+  > 131 |       ))}
+        | ^^^^^^^^
+    132 |     </>
+    133 |   );
+    134 | }
+```
+
+</details>
+
+Replace all instances of:
+
+```json
+"details": null
+```
+
+with:
+
+```json
+"details": ["details", "string", "array"]
+```
+
+This could be a permissions issue with the GitHub App i.e., it does not have the requisite scope to read details of the check.

cmd/multi-scorecard/main.go

@@ -0,0 +1,116 @@
+// Copyright (c) Jeff Mendoza <jlm@jlm.name>
+// SPDX-License-Identifier: MIT
+
+package main
+
+import (
+	"bytes"
+	"context"
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+
+	"github.com/bradleyfalzon/ghinstallation/v2"
+	"github.com/google/go-github/v53/github"
+
+	"github.com/ossf/scorecard/v5/clients/githubrepo"
+	"github.com/ossf/scorecard/v5/docs/checks"
+	"github.com/ossf/scorecard/v5/pkg/scorecard"
+)
+
+func main() {
+	var appID = flag.Int64("appid", 0, "")
+	var keyFile = flag.String("keyfile", "", "")
+	flag.Parse()
+
+	if err := do(*appID, *keyFile); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func do(appID int64, keyFile string) error {
+	dt := http.DefaultTransport
+	ctx := context.Background()
+
+	// Read in GitHub App private key
+	key, err := os.ReadFile(keyFile)
+	if err != nil {
+		return err
+	}
+
+	// Create authenticated transport for App
+	at, err := ghinstallation.NewAppsTransport(dt, appID, key)
+	if err != nil {
+		return err
+	}
+
+	// Query GitHub for list of installations for this App
+	ghc := github.NewClient(&http.Client{Transport: at})
+	insts, _, err := ghc.Apps.ListInstallations(ctx, &github.ListOptions{PerPage: 100})
+	if err != nil {
+		return err
+	}
+
+	checkDocs, err := checks.Read()
+	if err != nil {
+		return fmt.Errorf("cannot read yaml file: %w", err)
+	}
+
+	// Iterate through installations
+	var results [][]byte
+	for _, inst := range insts {
+		// Create a new authenticated transport for this App installation that will
+		// be used for Scorecard.
+		it := ghinstallation.NewFromAppsTransport(at, inst.GetID())
+
+		// Query GitHub for list of repos available to this installation
+		ic := github.NewClient(&http.Client{Transport: it})
+		repos, _, err := ic.Apps.ListRepos(ctx, &github.ListOptions{PerPage: 100})
+		if err != nil {
+			return err
+		}
+
+		// Create Scorecard RepoClient using authenticated transport
+		rc := githubrepo.CreateGithubRepoClientWithTransport(ctx, it)
+
+		// Iterate through installations
+		for _, repo := range repos.Repositories {
+			// Create Scorecard Repo object for repo name we want to scan
+			screpo, err := githubrepo.MakeGithubRepo(repo.GetFullName())
+			if err != nil {
+				return err
+			}
+
+			// Run scorecard with Repo and RepoClient
+			res, err := scorecard.Run(ctx, screpo,
+				scorecard.WithRepoClient(rc),
+			)
+			if err != nil {
+				return err
+			}
+
+			// Write results as json into buffer
+			var b bytes.Buffer
+			if err := res.AsJSON2(&b, checkDocs, nil); err != nil {
+				return err
+			}
+			results = append(results, b.Bytes())
+		}
+	}
+
+	// Output results as a JSON array
+	for i, r := range results {
+		if i == 0 {
+			fmt.Print("[")
+		}
+		fmt.Print(string(r))
+		if i == len(results)-1 {
+			fmt.Print("]")
+		} else {
+			fmt.Print(",")
+		}
+	}
+	return nil
+}