QL: Merge pull request #77 from github/erik-krogh/qlpacks

add pretty AST for YAML and a QLPack utility class

Author: MathiasVP

ql/src/codeql_ql/ast/Ast.qll

@@ -232,6 +232,45 @@ class Predicate extends TPredicate, AstNode, Declaration {
   override string getAPrimaryQlClass() { result = "Predicate" }
 }
 
+/**
+ * A relation in the database.
+ */
+class Relation extends TDBRelation, AstNode, Declaration {
+  Generated::DbTable table;
+
+  Relation() { this = TDBRelation(table) }
+
+  /**
+   * Gets the name of the relation.
+   */
+  override string getName() { result = table.getTableName().getChild().getValue() }
+
+  private Generated::DbColumn getColumn(int i) {
+    result =
+      rank[i + 1](Generated::DbColumn column, int child |
+        table.getChild(child) = column
+      |
+        column order by child
+      )
+  }
+
+  /** Gets the `i`th parameter name */
+  string getParameterName(int i) { result = getColumn(i).getColName().getValue() }
+
+  /** Gets the `i`th parameter type */
+  string getParameterType(int i) {
+    // TODO: This is just using the name of the type, not the actual type. Checkout Type.qll
+    result = getColumn(i).getColType().getChild().(Generated::Token).getValue()
+  }
+
+  /**
+   * Gets the number of parameters.
+   */
+  int getArity() { result = count(getColumn(_)) }
+
+  override string getAPrimaryQlClass() { result = "Relation" }
+}
+
 /**
  * An expression that refers to a predicate, e.g. `BasicBlock::succ/2`.
  */
@@ -2030,3 +2069,204 @@ class ModuleExpr extends TModuleExpr, ModuleRef {
     pred = directMember("getQualifier") and result = this.getQualifier()
   }
 }
+
+/**
+ * Classes modelling YAML AST nodes.
+ */
+module YAML {
+  /** A node in a YAML file */
+  class YAMLNode extends TYAMLNode, AstNode {
+    /** Holds if the predicate is a root node (has no parent) */
+    predicate isRoot() { not exists(getParent()) }
+  }
+
+  /** A YAML comment. */
+  class YAMLComment extends TYamlCommemt, YAMLNode {
+    Generated::YamlComment yamlcomment;
+
+    YAMLComment() { this = TYamlCommemt(yamlcomment) }
+
+    override string getAPrimaryQlClass() { result = "YAMLComment" }
+  }
+
+  /** A YAML entry. */
+  class YAMLEntry extends TYamlEntry, YAMLNode {
+    Generated::YamlEntry yamle;
+
+    YAMLEntry() { this = TYamlEntry(yamle) }
+
+    /** Gets the key of this YAML entry. */
+    YAMLKey getKey() {
+      exists(Generated::YamlKeyvaluepair pair |
+        pair.getParent() = yamle and
+        result = TYamlKey(pair.getKey())
+      )
+    }
+
+    /** Gets the value of this YAML entry. */
+    YAMLValue getValue() {
+      exists(Generated::YamlKeyvaluepair pair |
+        pair.getParent() = yamle and
+        result = TYamlValue(pair.getValue())
+      )
+    }
+
+    override string getAPrimaryQlClass() { result = "YAMLEntry" }
+  }
+
+  /** A YAML key. */
+  class YAMLKey extends TYamlKey, YAMLNode {
+    Generated::YamlKey yamlkey;
+
+    YAMLKey() { this = TYamlKey(yamlkey) }
+
+    /**
+     * Gets the value of this YAML key.
+     */
+    YAMLValue getValue() {
+      exists(Generated::YamlKeyvaluepair pair |
+        pair.getKey() = yamlkey and result = TYamlValue(pair.getValue())
+      )
+    }
+
+    override string getAPrimaryQlClass() { result = "YAMLKey" }
+
+    /** Gets the value of this YAML value. */
+    string getNamePart(int i) {
+      i = 0 and result = yamlkey.getChild(0).(Generated::SimpleId).getValue()
+      or
+      exists(YAMLKey child |
+        child = TYamlKey(yamlkey.getChild(1)) and
+        result = child.getNamePart(i - 1)
+      )
+    }
+
+    /**
+     * Gets all the name parts of this YAML key concatenated with `/`.
+     * Dashes are replaced with `/` (because we don't have that information in the generated AST).
+     */
+    string getQualifiedName() {
+      result = concat(string part, int i | part = getNamePart(i) | part, "/" order by i)
+    }
+  }
+
+  /** A YAML list item. */
+  class YAMLListItem extends TYamlListitem, YAMLNode {
+    Generated::YamlListitem yamllistitem;
+
+    YAMLListItem() { this = TYamlListitem(yamllistitem) }
+
+    /**
+     * Gets the value of this YAML list item.
+     */
+    YAMLValue getValue() { result = TYamlValue(yamllistitem.getChild()) }
+
+    override string getAPrimaryQlClass() { result = "YAMLListItem" }
+  }
+
+  /** A YAML value. */
+  class YAMLValue extends TYamlValue, YAMLNode {
+    Generated::YamlValue yamlvalue;
+
+    YAMLValue() { this = TYamlValue(yamlvalue) }
+
+    override string getAPrimaryQlClass() { result = "YAMLValue" }
+
+    /** Gets the value of this YAML value. */
+    string getValue() { result = yamlvalue.getValue() }
+  }
+
+  // to not expose the entire `File` API on `QlPack`.
+  private newtype TQLPack = MKQlPack(File file) { file.getBaseName() = "qlpack.yml" }
+
+  YAMLEntry test() { not result.isRoot() }
+
+  /**
+   * A `qlpack.yml` file.
+   */
+  class QLPack extends MKQlPack {
+    File file;
+
+    QLPack() { this = MKQlPack(file) }
+
+    private string getProperty(string name) {
+      exists(YAMLEntry entry |
+        entry.isRoot() and
+        entry.getKey().getQualifiedName() = name and
+        result = entry.getValue().getValue().trim() and
+        entry.getLocation().getFile() = file
+      )
+    }
+
+    /** Gets the name of this qlpack */
+    string getName() { result = getProperty("name") }
+
+    /** Gets the version of this qlpack */
+    string getVersion() { result = getProperty("version") }
+
+    /** Gets the extractor of this qlpack */
+    string getExtractor() { result = getProperty("extractor") }
+
+    string toString() { result = getName() }
+
+    /** Gets the file that this `QLPack` represents. */
+    File getFile() { result = file }
+
+    private predicate isADependency(YAMLEntry entry) {
+      exists(YAMLEntry deps |
+        deps.getLocation().getFile() = file and entry.getLocation().getFile() = file
+      |
+        deps.isRoot() and
+        deps.getKey().getQualifiedName() = "dependencies" and
+        entry.getLocation().getStartLine() = 1 + deps.getLocation().getStartLine() and
+        entry.getLocation().getStartColumn() > deps.getLocation().getStartColumn()
+      )
+      or
+      exists(YAMLEntry prev | isADependency(prev) |
+        prev.getLocation().getFile() = file and
+        entry.getLocation().getFile() = file and
+        entry.getLocation().getStartLine() = 1 + prev.getLocation().getStartLine() and
+        entry.getLocation().getStartColumn() = prev.getLocation().getStartColumn()
+      )
+    }
+
+    predicate hasDependency(string name, string version) {
+      exists(YAMLEntry entry | isADependency(entry) |
+        entry.getKey().getQualifiedName() = name and
+        entry.getValue().getValue() = version
+      )
+    }
+
+    /** Gets the database scheme of this qlpack */
+    File getDBScheme() {
+      result.getBaseName() = getProperty("dbscheme") and
+      result = file.getParentContainer().getFile(any(string s | s.matches("%.dbscheme")))
+    }
+
+    pragma[noinline]
+    Container getAFileInPack() {
+      result.getParentContainer() = file.getParentContainer()
+      or
+      result = getAFileInPack().(Folder).getAChildContainer()
+    }
+
+    /**
+     * Gets a QLPack that this QLPack depends on.
+     */
+    QLPack getADependency() {
+      exists(string name | hasDependency(name, _) | result.getName().replaceAll("-", "/") = name)
+    }
+
+    Location getLocation() {
+      // hacky, just pick the first node in the file.
+      result =
+        min(YAMLNode entry, Location l, File f |
+          entry.getLocation().getFile() = file and
+          f = file and
+          l = entry.getLocation()
+        |
+          entry order by l.getStartLine(), l.getStartColumn(), l.getEndColumn(), l.getEndLine()
+        ).getLocation()
+    }
+  }
+}

ql/src/codeql_ql/ast/internal/AstNodes.qll

@@ -10,6 +10,7 @@ newtype TAstNode =
   TClass(Generated::Dataclass dc) or
   TCharPred(Generated::Charpred pred) or
   TClassPredicate(Generated::MemberPredicate pred) or
+  TDBRelation(Generated::DbTable table) or
   TSelect(Generated::Select sel) or
   TModule(Generated::Module mod) or
   TNewType(Generated::Datatype dt) or
@@ -57,7 +58,12 @@ newtype TAstNode =
   TUnaryExpr(Generated::UnaryExpr unaryexpr) or
   TDontCare(Generated::Underscore dontcare) or
   TModuleExpr(Generated::ModuleExpr me) or
-  TPredicateExpr(Generated::PredicateExpr pe)
+  TPredicateExpr(Generated::PredicateExpr pe) or
+  TYamlCommemt(Generated::YamlComment yc) or
+  TYamlEntry(Generated::YamlEntry ye) or
+  TYamlKey(Generated::YamlKey yk) or
+  TYamlListitem(Generated::YamlListitem yli) or
+  TYamlValue(Generated::YamlValue yv)
 
 class TFormula =
   TDisjunction or TConjunction or TComparisonFormula or TQuantifier or TNegation or TIfFormula or
@@ -75,6 +81,8 @@ class TCall = TPredicateCall or TMemberCall or TNoneCall or TAnyCall;
 
 class TModuleRef = TImport or TModuleExpr;
 
+class TYAMLNode = TYamlCommemt or TYamlEntry or TYamlKey or TYamlListitem or TYamlValue;
+
 private Generated::AstNode toGeneratedFormula(AST::AstNode n) {
   n = TConjunction(result) or
   n = TDisjunction(result) or
@@ -105,6 +113,14 @@ private Generated::AstNode toGeneratedExpr(AST::AstNode n) {
   n = TDontCare(result)
 }
 
+private Generated::AstNode toGenerateYAML(AST::AstNode n) {
+  n = TYamlCommemt(result) or
+  n = TYamlEntry(result) or
+  n = TYamlKey(result) or
+  n = TYamlListitem(result) or
+  n = TYamlValue(result)
+}
+
 /**
  * Gets the underlying TreeSitter entity for a given AST node.
  */
@@ -113,6 +129,8 @@ Generated::AstNode toGenerated(AST::AstNode n) {
   or
   result = toGeneratedFormula(n)
   or
+  result = toGenerateYAML(n)
+  or
   result.(Generated::ParExpr).getChild() = toGenerated(n)
   or
   result =
@@ -135,6 +153,8 @@ Generated::AstNode toGenerated(AST::AstNode n) {
   or
   n = TClassPredicate(result)
   or
+  n = TDBRelation(result)
+  or
   n = TSelect(result)
   or
   n = TModule(result)
@@ -166,7 +186,7 @@ Generated::AstNode toGenerated(AST::AstNode n) {
   n = TSuper(result)
 }
 
-class TPredicate = TCharPred or TClasslessPredicate or TClassPredicate;
+class TPredicate = TCharPred or TClasslessPredicate or TClassPredicate or TDBRelation;
 
 class TModuleMember = TModuleDeclaration or TImport or TSelect or TQLDoc;
 

ql/src/codeql_ql/ast/internal/Module.qll

@@ -100,8 +100,10 @@ private predicate resolveQualifiedName(Import imp, ContainerOrModule m, int i) {
       exists(Container c, Container parent |
         // should ideally look at `qlpack.yml` files
         parent = imp.getLocation().getFile().getParentContainer+() and
-        exists(parent.getFile("qlpack.yml")) and
-        c.getParentContainer() = parent and
+        exists(YAML::QLPack pack |
+          pack.getFile().getParentContainer() = parent and
+          c.getParentContainer() = pack.getADependency*().getFile().getParentContainer()
+        ) and
         q = m.getName()
       |
         m = TFile(c)

ql/src/codeql_ql/ast/internal/Predicate.qll

@@ -88,11 +88,37 @@ private module Cached {
     )
   }
 
+  pragma[noinline]
+  private predicate candidate(Relation rel, PredicateCall pc) {
+    rel.getName() = pc.getPredicateName()
+  }
+
+  private predicate resolveDBRelation(PredicateCall pc, DefinedPredicate p) {
+    exists(Relation rel | p = TPred(rel) |
+      candidate(rel, pc) and
+      rel.getArity() = pc.getNumberOfArguments() and
+      (
+        exists(YAML::QLPack libPack, YAML::QLPack qlPack |
+          rel.getLocation().getFile() = libPack.getDBScheme() and
+          qlPack.getADependency*() = libPack and
+          qlPack.getAFileInPack() = pc.getLocation().getFile()
+        )
+        or
+        // upgrade scripts don't have a qlpack
+        rel.getLocation().getFile().getParentContainer() =
+          pc.getLocation().getFile().getParentContainer()
+      )
+    )
+  }
+
   cached
   predicate resolveCall(Call c, PredicateOrBuiltin p) {
     resolvePredicateCall(c, p)
     or
     resolveMemberCall(c, p)
+    or
+    not resolvePredicateCall(c, _) and
+    resolveDBRelation(c, p)
   }
 
   cached