🐛 Fix REPL error attribution by tracking source per span

compiler_tests/tests/compiler.rs

@@ -1,11 +1,11 @@
-use ndc_lexer::Lexer;
+use ndc_lexer::{Lexer, SourceId};
 use ndc_parser::Parser;
 use ndc_vm::chunk::OpCode;
 use ndc_vm::chunk::OpCode::*;
 use ndc_vm::compiler::Compiler;
 
 fn compile(input: &str) -> Vec<OpCode> {
-    let tokens = Lexer::new(input)
+    let tokens = Lexer::new(input, SourceId::SYNTHETIC)
         .collect::<Result<Vec<_>, _>>()
         .expect("lex failed");
     let expressions = Parser::from_tokens(tokens).parse().expect("parse failed");

ndc_bin/src/diagnostic.rs

@@ -1,70 +1,121 @@
 use codespan_reporting::diagnostic::{Diagnostic, Label};
-use codespan_reporting::files::SimpleFile;
+use codespan_reporting::files;
 use codespan_reporting::term;
 use codespan_reporting::term::termcolor::{ColorChoice, StandardStream};
 use ndc_interpreter::InterpreterError;
-use ndc_lexer::Span;
+use ndc_lexer::{SourceDb, SourceId};
+use std::ops::Range;
 
-fn span_to_range(span: Span) -> std::ops::Range<usize> {
-    span.offset()..span.end()
+struct DiagnosticFiles<'a>(&'a SourceDb);
+
+impl<'a> files::Files<'a> for DiagnosticFiles<'a> {
+    type FileId = SourceId;
+    type Name = &'a str;
+    type Source = &'a str;
+
+    fn name(&'a self, id: SourceId) -> Result<&'a str, files::Error> {
+        if id == SourceId::SYNTHETIC {
+            return Ok("<synthetic>");
+        }
+        Ok(self.0.name(id))
+    }
+
+    fn source(&'a self, id: SourceId) -> Result<&'a str, files::Error> {
+        if id == SourceId::SYNTHETIC {
+            return Ok("");
+        }
+        Ok(self.0.source(id))
+    }
+
+    fn line_index(&'a self, id: SourceId, byte_index: usize) -> Result<usize, files::Error> {
+        let source = self.source(id)?;
+        Ok(files::line_starts(source)
+            .take_while(|&start| start <= byte_index)
+            .count()
+            .saturating_sub(1))
+    }
+
+    fn line_range(&'a self, id: SourceId, line_index: usize) -> Result<Range<usize>, files::Error> {
+        let source = self.source(id)?;
+        let line_starts: Vec<usize> = files::line_starts(source).collect();
+        let start = *line_starts
+            .get(line_index)
+            .ok_or(files::Error::LineTooLarge {
+                given: line_index,
+                max: line_starts.len().saturating_sub(1),
+            })?;
+        let end = line_starts
+            .get(line_index + 1)
+            .copied()
+            .unwrap_or(source.len());
+        Ok(start..end)
+    }
 }
 
-fn into_diagnostic(err: InterpreterError) -> Diagnostic<()> {
+fn into_diagnostic(err: InterpreterError) -> Diagnostic<SourceId> {
     match err {
         InterpreterError::Lexer { cause } => {
+            let span = cause.span();
             let mut d = Diagnostic::error()
                 .with_code("lexer")
                 .with_message(cause.to_string())
                 .with_labels(vec![
-                    Label::primary((), span_to_range(cause.span())).with_message("here"),
+                    Label::primary(span.source_id(), span.range()).with_message("here"),
                 ]);
             if let Some(help) = cause.help_text() {
                 d = d.with_notes(vec![help.to_owned()]);
             }
             d
         }
         InterpreterError::Parser { cause } => {
+            let span = cause.span();
             let mut d = Diagnostic::error()
                 .with_code("parser")
                 .with_message(cause.to_string())
                 .with_labels(vec![
-                    Label::primary((), span_to_range(cause.span())).with_message("here"),
+                    Label::primary(span.source_id(), span.range()).with_message("here"),
                 ]);
             if let Some(help) = cause.help_text() {
                 d = d.with_notes(vec![help.to_owned()]);
             }
             d
         }
-        InterpreterError::Resolver { cause } => Diagnostic::error()
-            .with_code("resolver")
-            .with_message(cause.to_string())
-            .with_labels(vec![
-                Label::primary((), span_to_range(cause.span())).with_message("related to this"),
-            ]),
-        InterpreterError::Compiler { cause } => Diagnostic::error()
-            .with_code("compiler")
-            .with_message(cause.to_string())
-            .with_labels(vec![
-                Label::primary((), span_to_range(cause.span())).with_message("related to this"),
-            ]),
+        InterpreterError::Resolver { cause } => {
+            let span = cause.span();
+            Diagnostic::error()
+                .with_code("resolver")
+                .with_message(cause.to_string())
+                .with_labels(vec![
+                    Label::primary(span.source_id(), span.range()).with_message("related to this"),
+                ])
+        }
+        InterpreterError::Compiler { cause } => {
+            let span = cause.span();
+            Diagnostic::error()
+                .with_code("compiler")
+                .with_message(cause.to_string())
+                .with_labels(vec![
+                    Label::primary(span.source_id(), span.range()).with_message("related to this"),
+                ])
+        }
         InterpreterError::Vm(err) => {
             let mut d = Diagnostic::error()
                 .with_code("vm")
                 .with_message(&err.message);
             if let Some(span) = err.span {
                 d = d.with_labels(vec![
-                    Label::primary((), span_to_range(span)).with_message("related to this"),
+                    Label::primary(span.source_id(), span.range()).with_message("related to this"),
                 ]);
             }
             d
         }
     }
 }
 
-pub fn emit_error(filename: &str, source: &str, err: InterpreterError) {
+pub fn emit_error(source_db: &SourceDb, err: InterpreterError) {
     let diagnostic = into_diagnostic(err);
-    let file = SimpleFile::new(filename, source);
+    let files = DiagnosticFiles(source_db);
     let writer = StandardStream::stderr(ColorChoice::Auto);
     let config = term::Config::default();
-    let _ = term::emit(&mut writer.lock(), &config, &file, &diagnostic);
+    let _ = term::emit(&mut writer.lock(), &config, &files, &diagnostic);
 }

ndc_bin/src/highlighter.rs

@@ -1,6 +1,6 @@
 use ahash::AHashSet;
 use itertools::Itertools;
-use ndc_lexer::{Lexer, Token, TokenLocation};
+use ndc_lexer::{Lexer, SourceId, Token, TokenLocation};
 use ndc_parser::{Expression, ExpressionLocation, ForBody, ForIteration};
 use yansi::{Paint, Painted};
 
@@ -12,7 +12,7 @@ impl AndycppHighlighter {
     pub fn highlight_parsed(line: &str) -> Vec<Painted<&str>> {
         let mut function_spans = AHashSet::new();
 
-        let expressions = Lexer::new(line)
+        let expressions = Lexer::new(line, SourceId::SYNTHETIC)
             .collect::<Result<Vec<TokenLocation>, _>>()
             .ok()
             .and_then(|tokens| ndc_parser::Parser::from_tokens(tokens).parse().ok());
@@ -30,7 +30,8 @@ impl AndycppHighlighter {
         line: &'a str,
         function_spans: &AHashSet<usize>,
     ) -> Vec<Painted<&'a str>> {
-        let Ok(tokens) = Lexer::new(line).collect::<Result<Vec<_>, _>>() else {
+        let Ok(tokens) = Lexer::new(line, SourceId::SYNTHETIC).collect::<Result<Vec<_>, _>>()
+        else {
             return vec![line.red()];
         };
 

ndc_bin/src/main.rs

@@ -118,24 +118,20 @@ fn main() -> anyhow::Result<()> {
 
             let mut interpreter = Interpreter::new();
             interpreter.configure(ndc_stdlib::register);
-            if let Err(err) = interpreter.eval(&string) {
-                diagnostic::emit_error(&filename.expect("filename must exist"), &string, err);
+            let name = filename.as_deref().unwrap_or("<input>");
+            if let Err(err) = interpreter.eval_named(name, &string) {
+                diagnostic::emit_error(interpreter.source_db(), err);
                 process::exit(1);
             }
         }
         Action::DisassembleFile(path) => {
-            let filename = path
-                .file_name()
-                .and_then(|name| name.to_str())
-                .unwrap_or("<input>")
-                .to_string();
             let string = fs::read_to_string(path)?;
             let mut interpreter = Interpreter::new();
             interpreter.configure(ndc_stdlib::register);
             match interpreter.disassemble_str(&string) {
                 Ok(output) => print!("{output}"),
                 Err(e) => {
-                    diagnostic::emit_error(&filename, &string, e);
+                    diagnostic::emit_error(interpreter.source_db(), e);
                     process::exit(1);
                 }
             }

ndc_bin/src/repl.rs

@@ -34,22 +34,22 @@ pub fn run() -> anyhow::Result<()> {
 
     let mut interpreter = Interpreter::new();
     interpreter.configure(ndc_stdlib::register);
-    loop {
+    for command_nr in 1.. {
         match rl.readline("λ ") {
             Ok(line) => {
                 // If we can't append the history we just ignore this
                 let _ = rl.add_history_entry(line.as_str());
 
                 // Run the line we just read through the interpreter
-                match interpreter.eval(line.as_str()) {
+                match interpreter.eval_named(format!("<repl:{command_nr}>"), line.as_str()) {
                     Ok(value) => {
                         let output = value.to_string();
                         if !output.is_empty() {
                             println!("{output}")
                         }
                     }
                     Err(err) => {
-                        crate::diagnostic::emit_error("<repl>", &line, err);
+                        crate::diagnostic::emit_error(interpreter.source_db(), err);
                     }
                 }
             }

ndc_interpreter/src/lib.rs

@@ -1,6 +1,6 @@
 use ndc_analyser::{Analyser, ScopeTree};
 use ndc_core::FunctionRegistry;
-use ndc_lexer::{Lexer, TokenLocation};
+use ndc_lexer::{Lexer, SourceDb, SourceId, TokenLocation};
 use ndc_parser::ExpressionLocation;
 use ndc_vm::compiler::Compiler;
 use ndc_vm::value::CompiledFunction;
@@ -13,6 +13,7 @@ pub struct Interpreter {
     registry: FunctionRegistry<Rc<NativeFunction>>,
     capturing: bool,
     analyser: Analyser,
+    source_db: SourceDb,
     /// Persistent REPL VM and the compiler checkpoint from the last run.
     /// `None` until the first `eval` call; kept alive afterwards so that
     /// variables declared on one line are visible on subsequent lines.
@@ -38,6 +39,7 @@ impl Interpreter {
             registry: FunctionRegistry::default(),
             capturing,
             analyser: Analyser::from_scope_tree(ScopeTree::from_global_scope(vec![])),
+            source_db: SourceDb::new(),
             repl_state: None,
         }
     }
@@ -72,15 +74,21 @@ impl Interpreter {
         }
     }
 
+    pub fn source_db(&self) -> &SourceDb {
+        &self.source_db
+    }
+
     pub fn analyse_str(
         &mut self,
         input: &str,
     ) -> Result<Vec<ExpressionLocation>, InterpreterError> {
-        self.parse_and_analyse(input)
+        let source_id = self.source_db.add("<input>", input);
+        self.parse_and_analyse(input, source_id)
     }
 
     pub fn compile_str(&mut self, input: &str) -> Result<CompiledFunction, InterpreterError> {
-        let expressions = self.parse_and_analyse(input)?;
+        let source_id = self.source_db.add("<input>", input);
+        let expressions = self.parse_and_analyse(input, source_id)?;
         Ok(Compiler::compile(expressions.into_iter())?)
     }
 
@@ -95,15 +103,26 @@ impl Interpreter {
     ///
     /// Statements (semicolon-terminated) produce [`Value::unit()`].
     pub fn eval(&mut self, input: &str) -> Result<Value, InterpreterError> {
-        let expressions = self.parse_and_analyse(input)?;
+        self.eval_named("<input>", input)
+    }
+
+    /// Execute source code with a custom source name for diagnostics.
+    pub fn eval_named(
+        &mut self,
+        name: impl Into<String>,
+        input: &str,
+    ) -> Result<Value, InterpreterError> {
+        let source_id = self.source_db.add(name, input);
+        let expressions = self.parse_and_analyse(input, source_id)?;
         self.interpret_vm(input, expressions.into_iter())
     }
 
     fn parse_and_analyse(
         &mut self,
         input: &str,
+        source_id: SourceId,
     ) -> Result<Vec<ExpressionLocation>, InterpreterError> {
-        let tokens = Lexer::new(input).collect::<Result<Vec<TokenLocation>, _>>()?;
+        let tokens = Lexer::new(input, source_id).collect::<Result<Vec<TokenLocation>, _>>()?;
         let mut expressions = ndc_parser::Parser::from_tokens(tokens).parse()?;
 
         let checkpoint = self.analyser.checkpoint();

ndc_lexer/src/lib.rs

@@ -1,4 +1,5 @@
 mod number;
+mod source_db;
 mod span;
 mod string;
 mod token;
@@ -8,7 +9,8 @@ use std::collections::VecDeque;
 use std::str::Chars;
 use string::StringLexer;
 
-pub use span::Span;
+pub use source_db::SourceDb;
+pub use span::{SourceId, Span};
 pub use token::{Token, TokenLocation};
 
 pub struct Lexer<'a> {
@@ -41,12 +43,13 @@ impl<'a> Lexer<'a> {
     }
 
     #[must_use]
-    pub fn new(source: &'a str) -> Self {
+    pub fn new(source: &'a str, source_id: SourceId) -> Self {
         Self {
             source: SourceIterator {
                 inner: source.chars(),
                 buffer: VecDeque::default(),
                 offset: 0,
+                source_id,
             },
         }
     }
@@ -182,6 +185,7 @@ struct SourceIterator<'a> {
     inner: Chars<'a>,
     buffer: VecDeque<char>,
     offset: usize,
+    source_id: SourceId,
 }
 
 impl SourceIterator<'_> {
@@ -190,11 +194,11 @@ impl SourceIterator<'_> {
     }
 
     pub fn create_span(&self, start: usize) -> Span {
-        Span::new(start, (self.current_offset()) - start)
+        Span::new(self.source_id, start, self.current_offset() - start)
     }
 
     pub fn span(&self) -> Span {
-        Span::new(self.current_offset(), 1)
+        Span::new(self.source_id, self.current_offset(), 1)
     }
 
     pub fn consume(&mut self, count: usize) {