Merge remote-tracking branch 'origin/main' into cfg_cleanup

Author: nickrolfe

.codeqlmanifest.json

@@ -1,9 +1,10 @@
 {
     "provide": [
         "ql/src/qlpack.yml",
+        "ql/consistency-queries/qlpack.yml",
         "ql/test/qlpack.yml",
         "ql/examples/qlpack.yml",
         "upgrades/qlpack.yml",
         "extractor-pack/codeql-extractor.yml"
     ]
-}
+}

.devcontainer/Dockerfile

@@ -0,0 +1,15 @@
+# See here for image contents: https://github.com/microsoft/vscode-dev-containers/tree/v0.162.0/containers/rust/.devcontainer/base.Dockerfile
+
+FROM mcr.microsoft.com/vscode/devcontainers/rust:0-1
+
+RUN apt-key --keyring /usr/share/keyrings/githubcli-archive-keyring.gpg adv \
+    --keyserver keyserver.ubuntu.com --recv-key C99B11DEB97541F0 && \
+    echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/githubcli-archive-keyring.gpg] https://cli.github.com/packages $(lsb_release -cs) main" \
+    |  tee /etc/apt/sources.list.d/github-cli2.list > /dev/null
+
+
+RUN apt-get update && export DEBIAN_FRONTEND=noninteractive \
+    && apt-get -y install --no-install-recommends gh 
+
+COPY post_create.sh /bin/post_create.sh
+COPY post_attach.sh /bin/post_attach.sh

.devcontainer/devcontainer.json

@@ -0,0 +1,39 @@
+// For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
+// https://github.com/microsoft/vscode-dev-containers/tree/v0.162.0/containers/rust
+{
+	"name": "Rust",
+	"build": {
+		"dockerfile": "Dockerfile"
+	},
+	"runArgs": [
+		"--cap-add=SYS_PTRACE",
+		"--security-opt",
+		"seccomp=unconfined"
+	],
+	// Set *default* container specific settings.json values on container create.
+	"settings": {
+		"terminal.integrated.shell.linux": "/bin/bash",
+		"lldb.executable": "/usr/bin/lldb",
+		// VS Code don't watch files under ./target
+		"files.watcherExclude": {
+			"**/target/**": true
+		}
+	},
+	// Add the IDs of extensions you want installed when the container is created.
+	"extensions": [
+		"rust-lang.rust",
+		"bungcip.better-toml",
+		"vadimcn.vscode-lldb",
+		"mutantdino.resourcemonitor",
+		"ms-azuretools.vscode-docker",
+		"github.vscode-codeql"
+	],
+	// Use 'forwardPorts' to make a list of ports inside the container available locally.
+	// "forwardPorts": [],
+	// Use 'postCreateCommand' to run commands after the container is created.
+	// "postCreateCommand": "rustc --version",
+	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
+	"remoteUser": "vscode",
+	"postCreateCommand": [ "/bin/post_create.sh" ],
+	"postAttachCommand": [ "flock", "-E", "0", "-n", "/var/lock/post_attach.lock", "/bin/post_attach.sh" ]
+}

.devcontainer/post_attach.sh

@@ -0,0 +1,37 @@
+#! /bin/bash
+set -xe
+
+echo "Check installed CodeQL version"
+CURRENT_CODEQL_BIN=$(readlink -e /usr/local/bin/codeql || echo "")
+LATEST=$(gh release list --repo https://github.com/github/codeql-cli-binaries | cut -f 1 | sort --version-sort | tail -1)
+
+BASE_DIR=/home/vscode/codeql-binaries
+mkdir -p "${BASE_DIR}"
+LATEST_CODEQL_DIR="${BASE_DIR}/codeql-${LATEST}"
+LATEST_CODEQL_BIN="${LATEST_CODEQL_DIR}/codeql/codeql"
+
+if [ "${CURRENT_CODEQL_BIN}" != "${LATEST_CODEQL_BIN}" ]; then
+  echo "Installing CodeQL ${LATEST}"
+  TMPDIR=$(mktemp -d -p "$(dirname ${LATEST_CODEQL_DIR})")
+  gh release download --repo https://github.com/github/codeql-cli-binaries --pattern codeql-linux64.zip -D "${TMPDIR}" "$LATEST" 
+  unzip -oq "${TMPDIR}/codeql-linux64.zip" -d "${TMPDIR}"
+  rm -f "${TMPDIR}/codeql-linux64.zip"
+  mv "${TMPDIR}" "${LATEST_CODEQL_DIR}"
+  test -x "${LATEST_CODEQL_BIN}" && sudo ln -sf "${LATEST_CODEQL_BIN}" /usr/local/bin/codeql
+  if [[ "${CURRENT_CODEQL_BIN}" =~ .*/codeql/codeql ]]; then
+    rm -rf "$(dirname $(dirname ${CURRENT_CODEQL_BIN}))"
+  fi
+fi
+
+echo "Build the Ruby extractor"
+
+# clone the git dependencies using "git clone" because cargo's builtin git support is rather slow
+REPO_DIR="${CARGO_HOME:-/home/vscode/.cargo}/git/db" 
+REPO_DIR_ERB="${REPO_DIR}/tree-sitter-embedded-template-4c796e3340c233b6"
+REPO_DIR_RUBY="${REPO_DIR}/tree-sitter-ruby-666a40ce046f8e7a"
+
+mkdir -p "${REPO_DIR}"
+test -e "${REPO_DIR_ERB}" || git clone -q --bare https://github.com/tree-sitter/tree-sitter-embedded-template "${REPO_DIR_ERB}"
+test -e "${REPO_DIR_RUBY}" || git clone -q --bare https://github.com/tree-sitter/tree-sitter-ruby.git "${REPO_DIR_RUBY}"
+
+./create-extractor-pack.sh

.devcontainer/post_create.sh

@@ -0,0 +1,4 @@
+#! /bin/bash
+
+mkdir -p /home/vscode/.config/codeql
+echo '--search-path /workspaces/codeql-ruby' >> /home/vscode/.config/codeql/config

Cargo.lock

@@ -10,7 +10,7 @@ edition = "2018"
 flate2 = "1.0"
 node-types = { path = "../node-types" }
 tree-sitter = "0.17"
-tree-sitter-embedded-template = { git = "https://github.com/aibaars/tree-sitter-embedded-template", rev = "d4aac29c08aa7c596633d00b5ec2dd2d247eafe4" }
+tree-sitter-embedded-template = { git = "https://github.com/tree-sitter/tree-sitter-embedded-template", rev = "d4aac29c08aa7c596633d00b5ec2dd2d247eafe4" }
 tree-sitter-ruby = { git = "https://github.com/tree-sitter/tree-sitter-ruby.git", rev = "32cd5a04adb4accb0c121f037ab59df3c3488228" }
 clap = "2.33"
 tracing = "0.1"

codeql

@@ -305,22 +305,58 @@ struct Visitor<'a> {
 }
 
 impl Visitor<'_> {
+    fn record_parse_error(
+        &mut self,
+        error_message: String,
+        full_error_message: String,
+        loc: Label,
+    ) {
+        error!("{}", full_error_message);
+        let id = self.trap_writer.fresh_id();
+        self.trap_writer.add_tuple(
+            "diagnostics",
+            vec![
+                Arg::Label(id),
+                Arg::Int(40), // severity 40 = error
+                Arg::String("parse_error".to_string()),
+                Arg::String(error_message),
+                Arg::String(full_error_message),
+                Arg::Label(loc),
+            ],
+        );
+    }
+
+    fn record_parse_error_for_node(
+        &mut self,
+        error_message: String,
+        full_error_message: String,
+        node: Node,
+    ) {
+        let (start_line, start_column, end_line, end_column) = location_for(&self.source, node);
+        let loc = self.trap_writer.location(
+            self.file_label,
+            start_line,
+            start_column,
+            end_line,
+            end_column,
+        );
+        self.record_parse_error(error_message, full_error_message, loc);
+    }
+
     fn enter_node(&mut self, node: Node) -> bool {
-        if node.is_error() {
-            error!(
-                "{}:{}: parse error",
-                &self.path,
-                node.start_position().row + 1
-            );
-            return false;
-        }
-        if node.is_missing() {
-            error!(
-                "{}:{}: parse error: expecting '{}'",
+        if node.is_error() || node.is_missing() {
+            let error_message = if node.is_missing() {
+                format!("parse error: expecting '{}'", node.kind())
+            } else {
+                "parse error".to_string()
+            };
+            let full_error_message = format!(
+                "{}:{}: {}",
                 &self.path,
                 node.start_position().row + 1,
-                node.kind()
+                error_message
             );
+            self.record_parse_error_for_node(error_message, full_error_message, node);
             return false;
         }
 
@@ -405,12 +441,15 @@ impl Visitor<'_> {
                 }
             }
             _ => {
-                error!(
-                    "{}:{}: unknown table type: '{}'",
+                let error_message = format!("unknown table type: '{}'", node.kind());
+                let full_error_message = format!(
+                    "{}:{}: {}",
                     &self.path,
                     node.start_position().row + 1,
-                    node.kind()
+                    error_message
                 );
+                self.record_parse_error(error_message, full_error_message, loc);
+
                 valid = false;
             }
         }
@@ -456,26 +495,36 @@ impl Visitor<'_> {
                         values.push(Arg::Label(child_node.label));
                     }
                 } else if field.name.is_some() {
-                    error!(
-                        "{}:{}: type mismatch for field {}::{} with type {:?} != {:?}",
-                        &self.path,
-                        node.start_position().row + 1,
+                    let error_message = format!(
+                        "type mismatch for field {}::{} with type {:?} != {:?}",
                         node.kind(),
                         child_node.field_name.unwrap_or("child"),
                         child_node.type_name,
                         field.type_info
-                    )
+                    );
+                    let full_error_message = format!(
+                        "{}:{}: {}",
+                        &self.path,
+                        node.start_position().row + 1,
+                        error_message
+                    );
+                    self.record_parse_error_for_node(error_message, full_error_message, *node);
                 }
             } else {
                 if child_node.field_name.is_some() || child_node.type_name.named {
-                    error!(
-                        "{}:{}: value for unknown field: {}::{} and type {:?}",
-                        &self.path,
-                        node.start_position().row + 1,
+                    let error_message = format!(
+                        "value for unknown field: {}::{} and type {:?}",
                         node.kind(),
                         &child_node.field_name.unwrap_or("child"),
                         &child_node.type_name
                     );
+                    let full_error_message = format!(
+                        "{}:{}: {}",
+                        &self.path,
+                        node.start_position().row + 1,
+                        error_message
+                    );
+                    self.record_parse_error_for_node(error_message, full_error_message, *node);
                 }
             }
         }
@@ -489,18 +538,23 @@ impl Visitor<'_> {
                         args.push(child_values.first().unwrap().clone());
                     } else {
                         is_valid = false;
-                        error!(
-                            "{}:{}: {} for field: {}::{}",
-                            &self.path,
-                            node.start_position().row + 1,
+                        let error_message = format!(
+                            "{} for field: {}::{}",
                             if child_values.is_empty() {
                                 "missing value"
                             } else {
                                 "too many values"
                             },
                             node.kind(),
                             column_name
-                        )
+                        );
+                        let full_error_message = format!(
+                            "{}:{}: {}",
+                            &self.path,
+                            node.start_position().row + 1,
+                            error_message
+                        );
+                        self.record_parse_error_for_node(error_message, full_error_message, *node);
                     }
                 }
                 Storage::Table {

extractor/Cargo.toml

@@ -248,6 +248,11 @@ fn convert_nodes<'a>(nodes: &'a node_types::NodeTypeMap) -> Vec<dbscheme::Entry<
     entries.push(dbscheme::Entry::Table(token_table));
     entries.push(dbscheme::Entry::Case(token_case));
 
+    // Add the diagnostics table
+    let (diagnostics_case, diagnostics_table) = create_diagnostics();
+    entries.push(dbscheme::Entry::Table(diagnostics_table));
+    entries.push(dbscheme::Entry::Case(diagnostics_case));
+
     // Create a union of all database types.
     entries.push(dbscheme::Entry::Union(dbscheme::Union {
         name: "ast_node",
@@ -590,6 +595,69 @@ fn create_source_location_prefix_table<'a>() -> dbscheme::Entry<'a> {
     })
 }
 
+fn create_diagnostics<'a>() -> (dbscheme::Case<'a>, dbscheme::Table<'a>) {
+    let table = dbscheme::Table {
+        name: "diagnostics",
+        keysets: None,
+        columns: vec![
+            dbscheme::Column {
+                unique: true,
+                db_type: dbscheme::DbColumnType::Int,
+                name: "id",
+                ql_type: ql::Type::AtType("diagnostic"),
+                ql_type_is_ref: false,
+            },
+            dbscheme::Column {
+                unique: false,
+                db_type: dbscheme::DbColumnType::Int,
+                name: "severity",
+                ql_type: ql::Type::Int,
+                ql_type_is_ref: true,
+            },
+            dbscheme::Column {
+                unique: false,
+                db_type: dbscheme::DbColumnType::String,
+                name: "error_tag",
+                ql_type: ql::Type::String,
+                ql_type_is_ref: true,
+            },
+            dbscheme::Column {
+                unique: false,
+                db_type: dbscheme::DbColumnType::String,
+                name: "error_message",
+                ql_type: ql::Type::String,
+                ql_type_is_ref: true,
+            },
+            dbscheme::Column {
+                unique: false,
+                db_type: dbscheme::DbColumnType::String,
+                name: "full_error_message",
+                ql_type: ql::Type::String,
+                ql_type_is_ref: true,
+            },
+            dbscheme::Column {
+                unique: false,
+                db_type: dbscheme::DbColumnType::Int,
+                name: "location",
+                ql_type: ql::Type::AtType("location_default"),
+                ql_type_is_ref: true,
+            },
+        ],
+    };
+    let severities: Vec<(usize, &str)> = vec![
+        (10, "diagnostic_debug"),
+        (20, "diagnostic_info"),
+        (30, "diagnostic_warning"),
+        (40, "diagnostic_error"),
+    ];
+    let case = dbscheme::Case {
+        name: "diagnostic",
+        column: "severity",
+        branches: severities,
+    };
+    (case, table)
+}
+
 fn main() {
     tracing_subscriber::fmt()
         .with_target(false)