Support query symbol information

docs/usage/commands/compile.md

@@ -12,9 +12,12 @@ jo compile [--sast <dir>] <file.jo>... [--lib <dir>]...
 jo compile --python|--ruby|--js [--sast <dir>] <file.jo>... \
            [--lib <dir>]... [--link-lib <dir>]... [--link <src>=<tgt>]... -o <output>
 
-# Generate documentation from source files (experimental)
+# Generate HTML documentation from source files (experimental)
 jo compile --doc [--out <dir>] [--title <name>] [--readme <file>] \
            [--include-private] [--include-source] <file.jo>...
+
+# Query API information as JSON (experimental)
+jo compile --query <selectors> [<file.jo>...] [--lib <dir>]...
 ```
 
 Without a backend flag, the compiler type-checks only. With a backend flag, it produces an executable or script. `--sast <dir>` is optional in both cases — if present, `.sast` files are written to `<dir>` alongside the primary output.
@@ -43,13 +46,36 @@ Experimental.
 
 | Flag | Description |
 |------|-------------|
-| `--doc` | Generate documentation instead of normal compile output |
+| `--doc` | Generate HTML documentation instead of normal compile output |
 | `--out <dir>` | Documentation output directory |
 | `--title <name>` | Documentation title |
 | `--readme <file>` | Markdown file to use as the generated documentation home page |
 | `--include-private` | Include private symbols |
 | `--include-source` | Embed source code in output |
 
+### Query
+
+Experimental.
+
+| Flag | Description |
+|------|-------------|
+| `--query <selectors>` | Query comma-separated symbols or source files and write a JSON array to stdout |
+| `--fields <fields>` | Select comma-separated query output fields (default: `name,signature,doc`) |
+
+Selectors may name symbols or source files with `file:<path>`.
+Symbol selectors are dot-separated names resolved from the root namespace, such
+as `jo.List.map`. Positional source files are optional. With no source files,
+the query searches the loaded SAST libraries, including stdlib, any `--lib`
+paths, and the runtime API selected by `--use-runtime-api`.
+
+A file selector may use the recorded source path, an absolute path, or a basename:
+
+```sh
+jo compile --query file:Byte.jo
+```
+
+Query output includes public symbols only.
+
 ### App compilation
 
 | Flag                      | Description |
@@ -104,6 +130,30 @@ jo compile --doc lib/Core.jo lib/List.jo \
   --title "Jo Standard Library"
 ```
 
+Query selected symbols from source:
+
+```sh
+jo compile --query 'MyAPI,jo.List' src/API.jo
+```
+
+Query stdlib API information from SAST files only:
+
+```sh
+jo compile --query jo.List.map
+```
+
+Select output fields:
+
+```sh
+jo compile --query jo.List.map --fields name,signature,loc,doc
+```
+
+Query a standard-library source file by basename:
+
+```sh
+jo compile --query file:Byte.jo
+```
+
 ## Notes
 
 `jo compile` is a low-level escape hatch for scripts, CI pipelines, or experiments that don't fit the standard project layout. For normal project workflows, prefer the higher-level commands such as `jo build`, `jo run <module>`, and [`jo doc`](doc.md).

docs/usage/reference/compiler-options.md

@@ -28,11 +28,13 @@ Backend selectors are handled before the shared compiler options are parsed.
 | `--ruby` | Compile a Ruby application. |
 | `--python` | Compile a Python application. |
 | `--js` | Compile a JavaScript application. Experimental. |
+| `--doc` | Generate HTML documentation from source files. Experimental. |
+| `--query <selectors>` | Query API information and write JSON to stdout. Experimental. |
 
 ## Common Options
 
 Common options are accepted by type-check-only compilation, app backend compilation,
-and experimental documentation generation.
+documentation generation, and API queries.
 
 | Option | Form | Description |
 |--------|------|-------------|
@@ -99,3 +101,28 @@ above plus these documentation-specific options.
 | `--readme <file>` | single value | Markdown file to use as the generated documentation home page. |
 | `--include-private` | flag | Include private symbols. |
 | `--include-source` | flag | Embed source code in the generated documentation. |
+
+Documentation output is always HTML.
+
+## Query Options
+
+`jo compile --query <selectors>` is an experimental API-query backend. It
+accepts common compiler options and optional positional source files, and writes
+a bare JSON array to stdout. Selectors are comma-separated:
+
+| Selector | Meaning |
+|----------|---------|
+| `<symbol>` | Dot-separated symbol name resolved from the root namespace, such as `jo.List.map`. Aggregate symbols are emitted structurally with their members. |
+| `file:<path>` | Public symbols whose recorded source file matches the normalized path, absolute path, or basename. |
+
+Positional source files are optional. With no source files, the query searches
+the loaded SAST libraries: stdlib by default, runtime
+API libraries selected by `--use-runtime-api`, and any `--lib` paths.
+
+Query output includes public symbols only.
+Each symbol includes `name`, `signature`, and `doc` by default. Use
+`--fields` to select any combination of `name`, `kind`, `signature`, `loc`,
+`visibility`, `flags`, `annotations`, and `doc`. Field order in the option does
+not affect JSON output order. Classes with views include `views` automatically.
+
+Kind-qualified selectors such as `def:<symbol>` are reserved for future use.

stack-lang/cli/Main.scala

@@ -120,6 +120,7 @@ object Main:
           case Some(backend) =>
             backend match
               case Backend.Doc            => doc.Compiler.main(flags.args)
+              case Backend.Query          => query.Compiler.main(flags.args)
               case Backend.Ruby           => ruby.Compiler.main(flags.args)
               case Backend.Python         => python.Compiler.main(flags.args)
               case Backend.JS             => js.Compiler.main(flags.args)
@@ -203,6 +204,7 @@ object Main:
 
   enum Backend:
     case Doc
+    case Query
     case Ruby
     case Python
     case JS
@@ -220,30 +222,45 @@ object Main:
     var remaining = List.empty[String]
     var i = 0
 
+    def selectBackend(next: Backend): Unit =
+      backend match
+        case None =>
+          backend = Some(next)
+        case Some(current) =>
+          System.err.println(
+            s"Error: compile backend already selected: ${current.toString.toLowerCase}; cannot select ${next.toString.toLowerCase}"
+          )
+          System.exit(1)
+
     while i < args.length do
       args(i) match
         case "--doc" =>
-          backend = Some(Backend.Doc)
+          selectBackend(Backend.Doc)
+          i += 1
+
+        case "--query" =>
+          selectBackend(Backend.Query)
+          remaining = remaining :+ args(i)
           i += 1
 
         case "--ruby" =>
-          backend = Some(Backend.Ruby)
+          selectBackend(Backend.Ruby)
           i += 1
 
         case "--python" =>
-          backend = Some(Backend.Python)
+          selectBackend(Backend.Python)
           i += 1
 
         case "--js" =>
-          backend = Some(Backend.JS)
+          selectBackend(Backend.JS)
           i += 1
 
         case "--stack" =>
-          backend = Some(Backend.LinuxX86Stack)
+          selectBackend(Backend.LinuxX86Stack)
           i += 1
 
         case "--reg" =>
-          backend = Some(Backend.LinuxX86Reg)
+          selectBackend(Backend.LinuxX86Reg)
           i += 1
 
         case other =>
@@ -274,7 +291,8 @@ object Main:
       |  jo versions use <version>              Switch the active compiler version
       |  jo versions remove <version>           Remove an installed compiler version
       |  jo compile [options] <source.jo>       Compile application or library
-      |  jo compile --doc [options] <files...>  Generate documentation from source files
+      |  jo compile --query <selectors> [files...] Query documentation from source files or SAST libraries
+      |  jo compile --doc [options] <files...>  Generate HTML documentation from source files
       |  jo doc [module]                        Generate module documentation
       |  jo help                                Show this help message
       |

stack-lang/doc/Compiler.scala

@@ -1,7 +1,7 @@
 package doc
 
 import sast.*
-
+import sast.Trees.FileUnit
 import typing.Typer
 import reporting.Reporter
 import reporting.Config
@@ -26,6 +26,8 @@ object Compiler:
 
     val (config, sources) = cli.OptionParser.parseConfig(args, docOptions)
 
+    given Config = config
+
     if sources.isEmpty then
       println("Usage: jo doc <sources...> [options]")
       println()
@@ -37,10 +39,7 @@ object Compiler:
       println()
       println("Examples:")
       println("  jo doc lib/Core.jo lib/List.jo --out site/api")
-      println("  jo doc src/main.jo --out docs --title MyProject")
-      return
-
-    given Config = config
+      System.exit(1)
 
     Reporter.monitor():
       compile(sources)
@@ -53,12 +52,12 @@ object Compiler:
     // Parse and type check
     val (units, _) = sources |> Typer.parseStep |> Typer.typeStep
 
-    if rp.hasErrors then
-      println("Errors occurred during type checking. Documentation not generated.")
-      return
+    if rp.hasErrors then return
 
-    given Definitions = lazyDefn.value
+    given defn: Definitions = lazyDefn.value
+    generateHtmlDoc(units)
 
+  def generateHtmlDoc(units: List[FileUnit])(using Config, Definitions): Unit =
     val outputPath = Paths.get(outputDir.value)
     val includePrivateVal = includePrivate.value
 

stack-lang/pickle/Decoder.scala

@@ -185,7 +185,7 @@ object Decoder:
 
       FileUnit(owner, imports, members, source)
 
-    LazyFileUnit(owner, state.accessed, delayed)
+    LazyFileUnit(owner, source.file, state.accessed, delayed)
 
   /** Decode all imports for a file unit */
   private def decodeImports

stack-lang/pickle/LazyFileUnit.scala

@@ -17,6 +17,7 @@ import scala.collection.mutable
  */
 class LazyFileUnit(
   val owner: Symbol,
+  val sourceFile: String,
   private val accessed: java.util.concurrent.atomic.AtomicBoolean,
   private val delayed: () => FileUnit
 ):

stack-lang/query/Compiler.scala

@@ -0,0 +1,70 @@
+package query
+
+import sast.*
+import sast.Trees.FileUnit
+
+import typing.Typer
+import reporting.{Config, Reporter}
+
+import java.io.{OutputStreamWriter, PrintWriter}
+import java.nio.charset.StandardCharsets
+
+object Compiler:
+  val selectors: Config.StringSetting =
+    Config.StringSetting("--query", "", "comma-separated documentation selectors")
+
+  val fields: Config.StringSetting =
+    Config.StringSetting("--fields", "name,signature,doc", "comma-separated query output fields")
+
+  val queryOptions: List[cli.OptionParser.Setting[?]] =
+    selectors :: fields :: Config.commonOptions
+
+  def main(args: Array[String]): Unit =
+    given Reporter = Reporter.createReporter()
+
+    val (config, sources) = cli.OptionParser.parseConfig(args, queryOptions)
+    given Config = config
+
+    val queryText = selectors.value.trim
+    if queryText.isEmpty then
+      println("Usage: jo compile --query <selectors> [files...] [options]")
+      println()
+      println("Options:")
+      println("  --query <selectors>    Select symbols or source files, e.g. jo.List.map,file:Byte.jo")
+      println("  --fields <fields>      Select output fields (default: name,signature,doc)")
+      System.exit(1)
+
+    Reporter.monitor():
+      val selectedFields = Query.parseFields(fields.value)
+      if !summon[Reporter].hasErrors then compile(queryText, selectedFields, sources)
+
+  def compile(queryText: String, fields: Set[String], sources: List[String])(using rp: Reporter, config: Config): Unit =
+    val rootNameTable = new NameTable
+    given lazyDefn: Definitions.Lazy = Definitions.Lazy(rootNameTable)
+
+    if sources.isEmpty && Config.libPaths.value.isEmpty then
+      Query.reportNoMatches(queryText)
+      return
+
+    val (units, delayedUnits) = sources |> Typer.parseStep |> Typer.typeStep
+    if rp.hasErrors then return
+
+    given defn: Definitions = lazyDefn.value
+    val filter = Query.parse(queryText, defn.rootNameTable)
+    if rp.hasErrors then return
+
+    delayedUnits.forceIf: unit =>
+      filter.selectsFile(unit.sourceFile) ||
+      filter.symbols.exists: querySymbol =>
+        unit.owner.containedIn(querySymbol) || querySymbol.containedIn(unit.owner)
+
+    val libraryUnits = delayedUnits.force()
+    val filteredUnits = Query.filterUnits(units, libraryUnits, filter)
+    if rp.hasErrors then return
+
+    writeJson(filteredUnits, filter, fields)
+
+  private def writeJson(units: List[FileUnit], filter: Query.Filter, fields: Set[String])(using Reporter, Definitions): Unit =
+    val out = new PrintWriter(new OutputStreamWriter(System.out, StandardCharsets.UTF_8))
+    Query.emitJson(units, filter, fields, false, out)
+    out.flush()

stack-lang/query/JsonUtil.scala

@@ -0,0 +1,23 @@
+package query
+
+import java.io.PrintWriter
+
+object JsonUtil:
+  def string(s: String): String =
+    val sb = new StringBuilder("\"")
+    for c <- s do
+      c match
+        case '"'  => sb.append("\\\"")
+        case '\\' => sb.append("\\\\")
+        case '\b' => sb.append("\\b")
+        case '\f' => sb.append("\\f")
+        case '\n' => sb.append("\\n")
+        case '\r' => sb.append("\\r")
+        case '\t' => sb.append("\\t")
+        case c if c < 32 => sb.append("\\u%04x".format(c.toInt))
+        case c => sb.append(c)
+    sb.append("\"")
+    sb.toString
+
+  def emitString(s: String, out: PrintWriter): Unit =
+    out.print(string(s))